Continue after folders migration errors (#23)

* Update gitignore

* Bump go packages

* Allow migration to run if folders already exist

* Update bin version

* Revert "Update bin version"

This reverts commit e8bf720.

* Update bin version

.gitignore

@@ -5,3 +5,6 @@ teardown.sh
 # unit tests
 coverage.out
 coverage_badge.png
+
+# vscode
+.vscode

go.mod

@@ -13,12 +13,12 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
-	github.com/aws/aws-sdk-go v1.44.302
+	github.com/aws/aws-sdk-go v1.44.332
 	github.com/fatih/color v1.15.0
 	github.com/golang/mock v1.6.0
 	github.com/grafana/grafana-api-golang-client v0.23.0

go.sum

@@ -1,5 +1,5 @@
-github.com/aws/aws-sdk-go v1.44.302 h1:ST3ko6GrJKn3Xi+nAvxjG3uk/V1pW8KC52WLeIxqqNk=
-github.com/aws/aws-sdk-go v1.44.302/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.44.332 h1:Ze+98F41+LxoJUdsisAFThV+0yYYLYw17/Vt0++nFYM=
+github.com/aws/aws-sdk-go v1.44.332/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -67,8 +67,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

internal/pkg/app/app.go

@@ -39,15 +39,13 @@ func (a *App) Run(srcCustomGrafanaClient CustomGrafanaClient) error {
 	}
 	log.Success("Migrated ", migratedDs, " data sources")
 
-	fx, err := a.migrateFolders()
+	foldersResponse, err := a.migrateFolders()
 	if err != nil {
 		return err
 	}
-	log.Success("Migrated ", len(fx), " folders")
+	log.Success("Migrated ", len(foldersResponse.MigratedFolders), " folders")
 
-	// TODO: if there's an error on migrateFolders, query dashboard IDs in dst
-	// This will ensure that the migration is not partially completed
-	dashboards, err := a.migrateDashboards(&fx)
+	dashboards, err := a.migrateDashboards(&foldersResponse.SrcFolders)
 	if err != nil {
 		return err
 	}

internal/pkg/app/app_test.go

@@ -46,6 +46,18 @@ func mockFolder(_ *testing.T, m *mocks.Mockapi) {
 }
 
 func mockNewFolder(_ *testing.T, m *mocks.Mockapi) {
+	f := gapi.Folder{
+		ID:    1,
+		UID:   "uid",
+		Title: "test",
+		URL:   "http://test.com",
+	}
+	m.EXPECT().NewFolder(f.Title, f.UID).Return(
+		f, nil,
+	)
+}
+
+func mockNewFolderWithError(_ *testing.T, m *mocks.Mockapi) {
 	f := gapi.Folder{
 		ID:    1,
 		UID:   "uid",
@@ -56,6 +68,7 @@ func mockNewFolder(_ *testing.T, m *mocks.Mockapi) {
 		gapi.Folder{}, errors.New("some error while creating folder in dest"),
 	)
 }
+
 func mockDashboards(_ *testing.T, m *mocks.Mockapi) {
 	dsResponse := gapi.FolderDashboardSearchResponse{
 		FolderID:  1,

internal/pkg/app/folders.go

@@ -6,7 +6,17 @@ import (
 	gapi "github.com/grafana/grafana-api-golang-client"
 )
 
-func (a *App) migrateFolders() ([]gapi.Folder, error) {
+// FoldersResponse holds both folders in the source and destination Grafana
+type FoldersResponse struct {
+	SrcFolders      []gapi.Folder
+	MigratedFolders []gapi.Folder
+}
+
+// migrateFolders retrieve folders from source Grafana and use the api to
+// create them in the destination. We keep a copy of the source folders
+// in case the API fails to create one or more folders (because, it already
+// exists for example)
+func (a *App) migrateFolders() (*FoldersResponse, error) {
 	log.Info()
 	log.Info("Migrating folders:")
 
@@ -22,10 +32,11 @@ func (a *App) migrateFolders() ([]gapi.Folder, error) {
 		newF, err := a.Dst.NewFolder(f.Title, f.UID)
 		if err != nil {
 			log.Debugf("\terror: %s [%s]\n", f.Title, err)
+		} else {
+			newFx = append(newFx, newF)
 		}
-		newFx = append(newFx, newF)
 	}
-	return newFx, nil
+	return &FoldersResponse{fx, newFx}, nil
 }
 
 // From a list of folders, get a folder ID (used with destination folders)

internal/pkg/app/folders_test.go

@@ -33,25 +33,39 @@ func TestApp_migrateFolders(t *testing.T) {
 	tests := map[string]struct {
 		callMockSrc     func(m *mocks.Mockapi)
 		callMockDst     func(m *mocks.Mockapi)
-		expectedFolders int
+		migratedFolders int
+		sourceFolders   int
 		expectedError   error
 	}{
 		"error getting folders from src": {
 			callMockSrc: func(m *mocks.Mockapi) {
 				m.EXPECT().Folders().Return(nil, errors.New("some error")).AnyTimes()
 			},
 			callMockDst:     func(m *mocks.Mockapi) {},
-			expectedFolders: 0,
+			migratedFolders: 0,
+			sourceFolders:   0,
 			expectedError:   errors.New("some error"),
 		},
+		"syncing one folder with error": {
+			callMockSrc: func(m *mocks.Mockapi) {
+				mockFolder(t, m)
+			},
+			callMockDst: func(m *mocks.Mockapi) {
+				mockNewFolderWithError(t, m)
+			},
+			migratedFolders: 0,
+			sourceFolders:   1,
+			expectedError:   nil,
+		},
 		"syncing one folder": {
 			callMockSrc: func(m *mocks.Mockapi) {
 				mockFolder(t, m)
 			},
 			callMockDst: func(m *mocks.Mockapi) {
 				mockNewFolder(t, m)
 			},
-			expectedFolders: 1,
+			migratedFolders: 1,
+			sourceFolders:   1,
 			expectedError:   nil,
 		},
 	}
@@ -80,7 +94,8 @@ func TestApp_migrateFolders(t *testing.T) {
 				require.EqualError(t, err, tc.expectedError.Error())
 			} else {
 				require.NoError(t, err)
-				require.Len(t, fx, tc.expectedFolders)
+				require.Len(t, fx.SrcFolders, tc.sourceFolders)
+				require.Len(t, fx.MigratedFolders, tc.migratedFolders)
 			}
 		})
 	}

main.go

@@ -11,7 +11,7 @@ import (
 
 const (
 	shortDescription = "Amazon Managed Grafana migration utility"
-	version          = "0.1.9"
+	version          = "0.1.10"
 )
 
 var region string