From 7997352f981519ee52144398aa7e26e549881cf8 Mon Sep 17 00:00:00 2001 From: Dave Rolsky Date: Mon, 27 Jul 2026 16:59:17 -0500 Subject: [PATCH] TOOLS-4278 Convert mongoimport TestImportDocuments to testify At 827 lines this is the largest function in the file, and the only one so far where the GoConvey re-execution semantics mattered. Every leaf shared one collection, and GoConvey re-ran the outer block per leaf, so the Reset cleared that collection 51 times. Subtests do not work that way: a shared parent would run setup once and let each subtest inherit whatever its siblings left behind. The teardown is now a cleanImportCollection helper that every subtest calls, registering its own t.Cleanup, so the 51 leaf executions still get 51 clears. The cleanup deletes with context.Background() rather than t.Context(). t.Context() is cancelled just before Cleanup functions run, so the delete failed with "context canceled" until this was changed. Isolation was checked empirically rather than assumed: three consecutive runs, -count=5, three shuffled runs, all 52 subtests run individually, and zero residual documents afterwards. The three t.Fatalf checks in the old Reset body become require.NoError in the helper, which is the only reason the assertion count moves at all: 120 before, 123 after. No behavior change. --- mongoimport/mongoimport_test.go | 1448 ++++++++++++++++--------------- 1 file changed, 747 insertions(+), 701 deletions(-) diff --git a/mongoimport/mongoimport_test.go b/mongoimport/mongoimport_test.go index 51db3411d..a4ff18191 100644 --- a/mongoimport/mongoimport_test.go +++ b/mongoimport/mongoimport_test.go @@ -9,6 +9,7 @@ package mongoimport import ( "bufio" "bytes" + "context" "encoding/csv" "encoding/json" "fmt" @@ -658,104 +659,115 @@ func TestGetInputReader(t *testing.T) { func TestImportDocuments(t *testing.T) { testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) - Convey("With a mongoimport instance", t, func() { - Reset(func() { - sessionProvider, err := db.NewSessionProvider(*getBasicToolOptions()) - if err != nil { - t.Fatalf("error getting session provider session: %v", err) - } - session, err := sessionProvider.GetSession() - if err != nil { - t.Fatalf("error getting session: %v", err) - } - _, err = session.Database(testDb). - Collection(testCollection). - DeleteMany(t.Context(), bson.D{}) - if err != nil { - t.Fatalf("error dropping collection: %v", err) - } - }) - Convey("no error should be thrown for CSV import on test data and all "+ - "CSV data lines should be imported correctly", func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test.csv" - fields := "a,b,c" - imp.InputOptions.Fields = &fields - imp.IngestOptions.WriteConcern = "majority" - numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 3) - So(numFailed, ShouldEqual, 0) - }) - Convey("an error should be thrown for JSON import on test data that is "+ - "JSON array", func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.File = "testdata/test_array.json" - imp.IngestOptions.WriteConcern = "majority" - numProcessed, _, err := imp.ImportDocuments() - So(err, ShouldNotBeNil) - So(numProcessed, ShouldEqual, 0) - }) - Convey("TOOLS-247: no error should be thrown for JSON import on test "+ - "data and all documents should be imported correctly", func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.File = "testdata/test_plain2.json" - imp.IngestOptions.WriteConcern = "majority" - numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 10) - So(numFailed, ShouldEqual, 0) - }) - Convey("CSV import with --ignoreBlanks should import only non-blank fields", func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test_blanks.csv" - fields := "_id,b,c" - imp.InputOptions.Fields = &fields - imp.IngestOptions.IgnoreBlanks = true - numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 3) - So(numFailed, ShouldEqual, 0) - expectedDocuments := []bson.M{ - {"_id": int32(1), "b": int32(2)}, - {"_id": int32(5), "c": "6e"}, - {"_id": int32(7), "b": int32(8), "c": int32(6)}, - } - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - }) - Convey("CSV import without --ignoreBlanks should include blanks", func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test_blanks.csv" - fields := "_id,b,c" - imp.InputOptions.Fields = &fields - numProcessed, numFailed, err := imp.ImportDocuments() - So(numFailed, ShouldEqual, 0) - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 3) - expectedDocuments := []bson.M{ - {"_id": int32(1), "b": int32(2), "c": ""}, - {"_id": int32(5), "b": "", "c": "6e"}, - {"_id": int32(7), "b": int32(8), "c": int32(6)}, - } - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - }) - Convey("no error should be thrown for CSV import on test data with --upsertFields", func() { + t.Run("no error should be thrown for CSV import on test data and all "+ + "CSV data lines should be imported correctly", func(t *testing.T) { + cleanImportCollection(t) + + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test.csv" + fields := "a,b,c" + imp.InputOptions.Fields = &fields + imp.IngestOptions.WriteConcern = "majority" + numProcessed, numFailed, err := imp.ImportDocuments() + require.NoError(t, err, "should import CSV data without error") + assert.EqualValues(t, 3, numProcessed, "should process every CSV data line") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + }) + + t.Run("an error should be thrown for JSON import on test data that is "+ + "JSON array", func(t *testing.T) { + cleanImportCollection(t) + + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.File = "testdata/test_array.json" + imp.IngestOptions.WriteConcern = "majority" + numProcessed, _, err := imp.ImportDocuments() + require.Error(t, err, "should reject a JSON array without --jsonArray") + assert.EqualValues(t, 0, numProcessed, "should not process any document") + }) + + t.Run("TOOLS-247: no error should be thrown for JSON import on test "+ + "data and all documents should be imported correctly", func(t *testing.T) { + cleanImportCollection(t) + + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.File = "testdata/test_plain2.json" + imp.IngestOptions.WriteConcern = "majority" + numProcessed, numFailed, err := imp.ImportDocuments() + require.NoError(t, err, "should import JSON data without error") + assert.EqualValues(t, 10, numProcessed, "should process every document") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + }) + + t.Run("CSV import with --ignoreBlanks should import only non-blank fields", func(t *testing.T) { + cleanImportCollection(t) + + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test_blanks.csv" + fields := "_id,b,c" + imp.InputOptions.Fields = &fields + imp.IngestOptions.IgnoreBlanks = true + + numProcessed, numFailed, err := imp.ImportDocuments() + require.NoError(t, err, "should import CSV data without error") + assert.EqualValues(t, 3, numProcessed, "should process every CSV data line") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + expectedDocuments := []bson.M{ + {"_id": int32(1), "b": int32(2)}, + {"_id": int32(5), "c": "6e"}, + {"_id": int32(7), "b": int32(8), "c": int32(6)}, + } + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should import only the non-blank fields", + ) + }) + + t.Run("CSV import without --ignoreBlanks should include blanks", func(t *testing.T) { + cleanImportCollection(t) + + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test_blanks.csv" + fields := "_id,b,c" + imp.InputOptions.Fields = &fields + numProcessed, numFailed, err := imp.ImportDocuments() + assert.EqualValues(t, 0, numFailed, "should not fail any document") + require.NoError(t, err, "should import CSV data without error") + assert.EqualValues(t, 3, numProcessed, "should process every CSV data line") + expectedDocuments := []bson.M{ + {"_id": int32(1), "b": int32(2), "c": ""}, + {"_id": int32(5), "b": "", "c": "6e"}, + {"_id": int32(7), "b": int32(8), "c": int32(6)}, + } + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should import blank fields as empty strings", + ) + }) + + t.Run( + "no error should be thrown for CSV import on test data with --upsertFields", + func(t *testing.T) { + cleanImportCollection(t) + imp, err := NewMongoImport() - So(err, ShouldBeNil) + require.NoError(t, err, "should create a new mongoimport instance") imp.IngestOptions.Mode = modeInsert imp.InputOptions.Type = CSV imp.InputOptions.File = "testdata/test.csv" @@ -764,108 +776,142 @@ func TestImportDocuments(t *testing.T) { imp.IngestOptions.UpsertFields = "b,c" imp.IngestOptions.MaintainInsertionOrder = true numProcessed, numFailed, err := imp.ImportDocuments() - So(numFailed, ShouldEqual, 0) - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 3) - expectedDocuments := []bson.M{ - {"_id": int32(1), "b": int32(2), "c": int32(3)}, - {"_id": int32(3), "b": 5.4, "c": "string"}, - {"_id": int32(5), "b": int32(6), "c": int32(6)}, - } - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - }) - Convey("no error should be thrown for CSV import on test data with "+ - "--stopOnError. Only documents before error should be imported", func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test.csv" - fields := "_id,b,c" - imp.InputOptions.Fields = &fields - imp.IngestOptions.StopOnError = true - imp.IngestOptions.MaintainInsertionOrder = true - imp.IngestOptions.WriteConcern = "majority" - numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 3) - So(numFailed, ShouldEqual, 0) + assert.EqualValues(t, 0, numFailed, "should not fail any document") + require.NoError(t, err, "should import CSV data without error") + assert.EqualValues(t, 3, numProcessed, "should process every CSV data line") expectedDocuments := []bson.M{ {"_id": int32(1), "b": int32(2), "c": int32(3)}, {"_id": int32(3), "b": 5.4, "c": "string"}, {"_id": int32(5), "b": int32(6), "c": int32(6)}, } - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - }) - Convey( - "CSV import with duplicate _id's should not error if --stopOnError is not set", - func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test_duplicate.csv" - fields := "_id,b,c" - imp.InputOptions.Fields = &fields - imp.IngestOptions.StopOnError = false - numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 4) - So(numFailed, ShouldEqual, 1) - - expectedDocuments := []bson.M{ - {"_id": int32(1), "b": int32(2), "c": int32(3)}, - {"_id": int32(3), "b": 5.4, "c": "string"}, - {"_id": int32(5), "b": int32(6), "c": int32(6)}, - {"_id": int32(8), "b": int32(6), "c": int32(6)}, - } - // all docs except the one with duplicate _id - should be imported - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - }, + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should upsert on the given fields", + ) + }, + ) + + t.Run("no error should be thrown for CSV import on test data with "+ + "--stopOnError. Only documents before error should be imported", func(t *testing.T) { + cleanImportCollection(t) + + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test.csv" + fields := "_id,b,c" + imp.InputOptions.Fields = &fields + imp.IngestOptions.StopOnError = true + imp.IngestOptions.MaintainInsertionOrder = true + imp.IngestOptions.WriteConcern = "majority" + numProcessed, numFailed, err := imp.ImportDocuments() + require.NoError(t, err, "should import CSV data without error") + assert.EqualValues(t, 3, numProcessed, "should process every CSV data line") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + expectedDocuments := []bson.M{ + {"_id": int32(1), "b": int32(2), "c": int32(3)}, + {"_id": int32(3), "b": 5.4, "c": "string"}, + {"_id": int32(5), "b": int32(6), "c": int32(6)}, + } + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should import only documents before the error", ) - Convey("no error should be thrown for CSV import on test data with --drop", func() { + }) + + t.Run( + "CSV import with duplicate _id's should not error if --stopOnError is not set", + func(t *testing.T) { + cleanImportCollection(t) + imp, err := NewMongoImport() - So(err, ShouldBeNil) + require.NoError(t, err, "should create a new mongoimport instance") imp.IngestOptions.Mode = modeInsert imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test.csv" + imp.InputOptions.File = "testdata/test_duplicate.csv" fields := "_id,b,c" imp.InputOptions.Fields = &fields - imp.IngestOptions.Drop = true - imp.IngestOptions.MaintainInsertionOrder = true - imp.IngestOptions.WriteConcern = "majority" + imp.IngestOptions.StopOnError = false numProcessed, numFailed, err := imp.ImportDocuments() - So(numFailed, ShouldEqual, 0) - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 3) + require.NoError(t, err, "should not error on a duplicate _id without --stopOnError") + assert.EqualValues(t, 4, numProcessed, "should process every non-duplicate document") + assert.EqualValues(t, 1, numFailed, "should fail the duplicate document") + expectedDocuments := []bson.M{ {"_id": int32(1), "b": int32(2), "c": int32(3)}, {"_id": int32(3), "b": 5.4, "c": "string"}, {"_id": int32(5), "b": int32(6), "c": int32(6)}, + {"_id": int32(8), "b": int32(6), "c": int32(6)}, } - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - }) - Convey("CSV import on test data with --headerLine should succeed", func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test.csv" - fields := "_id,b,c" - imp.InputOptions.Fields = &fields - imp.InputOptions.HeaderLine = true - numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 2) - So(numFailed, ShouldEqual, 0) - }) - Convey("EOF should be thrown for CSV import with --headerLine if file is empty", func() { + // all docs except the one with duplicate _id - should be imported + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should import every document except the one with a duplicate _id", + ) + }, + ) + + t.Run("no error should be thrown for CSV import on test data with --drop", func(t *testing.T) { + cleanImportCollection(t) + + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test.csv" + fields := "_id,b,c" + imp.InputOptions.Fields = &fields + imp.IngestOptions.Drop = true + imp.IngestOptions.MaintainInsertionOrder = true + imp.IngestOptions.WriteConcern = "majority" + numProcessed, numFailed, err := imp.ImportDocuments() + assert.EqualValues(t, 0, numFailed, "should not fail any document") + require.NoError(t, err, "should import CSV data without error") + assert.EqualValues(t, 3, numProcessed, "should process every CSV data line") + expectedDocuments := []bson.M{ + {"_id": int32(1), "b": int32(2), "c": int32(3)}, + {"_id": int32(3), "b": 5.4, "c": "string"}, + {"_id": int32(5), "b": int32(6), "c": int32(6)}, + } + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should drop the collection before importing", + ) + }) + + t.Run("CSV import on test data with --headerLine should succeed", func(t *testing.T) { + cleanImportCollection(t) + + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test.csv" + fields := "_id,b,c" + imp.InputOptions.Fields = &fields + imp.InputOptions.HeaderLine = true + numProcessed, numFailed, err := imp.ImportDocuments() + require.NoError(t, err, "should import CSV data without error") + assert.EqualValues(t, 2, numProcessed, "should process every data line after the header") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + }) + + t.Run( + "EOF should be thrown for CSV import with --headerLine if file is empty", + func(t *testing.T) { + cleanImportCollection(t) + csvFile, err := os.CreateTemp("", "mongoimport_") - So(err, ShouldBeNil) + require.NoError(t, err, "should create a temporary CSV file") csvFile.Close() imp, err := NewMongoImport() - So(err, ShouldBeNil) + require.NoError(t, err, "should create a new mongoimport instance") imp.IngestOptions.Mode = modeInsert imp.InputOptions.Type = CSV imp.InputOptions.File = csvFile.Name() @@ -873,350 +919,395 @@ func TestImportDocuments(t *testing.T) { imp.InputOptions.Fields = &fields imp.InputOptions.HeaderLine = true numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldEqual, io.EOF) - So(numProcessed, ShouldEqual, 0) - So(numFailed, ShouldEqual, 0) - }) - Convey("CSV import with --mode=upsert and --upsertFields should succeed", func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test.csv" - fields := "_id,c,b" - imp.InputOptions.Fields = &fields - imp.IngestOptions.UpsertFields = "_id" - imp.IngestOptions.MaintainInsertionOrder = true - numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 3) - So(numFailed, ShouldEqual, 0) - expectedDocuments := []bson.M{ - {"_id": int32(1), "c": int32(2), "b": int32(3)}, - {"_id": int32(3), "c": 5.4, "b": "string"}, - {"_id": int32(5), "c": int32(6), "b": int32(6)}, - } - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - }) - Convey("CSV import with --mode=delete should succeed", func() { - // First import 3 documents - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test.csv" - fields := "_id,c,b" - imp.InputOptions.Fields = &fields - imp.IngestOptions.MaintainInsertionOrder = true - numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 3) - So(numFailed, ShouldEqual, 0) + require.Equal(t, io.EOF, err, "should report EOF for an empty file with --headerLine") + assert.EqualValues(t, 0, numProcessed, "should not process any document") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + }, + ) + + t.Run("CSV import with --mode=upsert and --upsertFields should succeed", func(t *testing.T) { + cleanImportCollection(t) + + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test.csv" + fields := "_id,c,b" + imp.InputOptions.Fields = &fields + imp.IngestOptions.UpsertFields = "_id" + imp.IngestOptions.MaintainInsertionOrder = true + numProcessed, numFailed, err := imp.ImportDocuments() + require.NoError(t, err, "should import CSV data without error") + assert.EqualValues(t, 3, numProcessed, "should process every CSV data line") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + expectedDocuments := []bson.M{ + {"_id": int32(1), "c": int32(2), "b": int32(3)}, + {"_id": int32(3), "c": 5.4, "b": "string"}, + {"_id": int32(5), "c": int32(6), "b": int32(6)}, + } + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should upsert on _id", + ) + }) + + t.Run("CSV import with --mode=delete should succeed", func(t *testing.T) { + cleanImportCollection(t) + + // First import 3 documents + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test.csv" + fields := "_id,c,b" + imp.InputOptions.Fields = &fields + imp.IngestOptions.MaintainInsertionOrder = true + numProcessed, numFailed, err := imp.ImportDocuments() + require.NoError(t, err, "should import CSV data without error") + assert.EqualValues(t, 3, numProcessed, "should process every CSV data line") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + + // Then delete two documents + imp, err = NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test_delete.csv" + fields = "_id,c,b" + imp.InputOptions.Fields = &fields + imp.IngestOptions.Mode = modeDelete + imp.IngestOptions.StopOnError = true + // Must specify upsert fields since option parsing is skipped in tests + imp.upsertFields = []string{"_id"} + numProcessed, numFailed, err = imp.ImportDocuments() + require.NoError(t, err, "should delete documents without error") + assert.EqualValues(t, 2, numProcessed, "should process every document to delete") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + + expectedDocuments := []bson.M{ + {"_id": int32(3), "c": 5.4, "b": "string"}, + } + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should delete the matching documents", + ) + }) + + t.Run("CSV import with --mode=delete and --upsertFields should succeed", func(t *testing.T) { + cleanImportCollection(t) + + // First import 3 documents + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test.csv" + fields := "_id,c,b" + imp.InputOptions.Fields = &fields + imp.IngestOptions.MaintainInsertionOrder = true + numProcessed, numFailed, err := imp.ImportDocuments() + require.NoError(t, err, "should import CSV data without error") + assert.EqualValues(t, 3, numProcessed, "should process every CSV data line") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + + // Then delete two documents + imp, err = NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test_delete.csv" + fields = "_id,c,b" + imp.InputOptions.Fields = &fields + imp.IngestOptions.Mode = modeDelete + imp.IngestOptions.StopOnError = true + // Must specify upsert fields since option parsing is skipped in tests + imp.upsertFields = []string{"b", "c"} + numProcessed, numFailed, err = imp.ImportDocuments() + require.NoError(t, err, "should delete documents without error") + assert.EqualValues(t, 1, numProcessed, "should process every document to delete") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + + expectedDocuments := []bson.M{ + {"_id": int32(3), "c": 5.4, "b": "string"}, + {"_id": int32(5), "c": int32(6), "b": int32(6)}, + } + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should delete only documents matching the upsert fields", + ) + }) + + t.Run("CSV import with --mode=delete and --ignoreBlanks should not take any action for "+ + "documents that have blank values for upsert fields", func(t *testing.T) { + cleanImportCollection(t) + + // First import 3 documents + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test.csv" + fields := "_id,c,b" + imp.InputOptions.Fields = &fields + imp.IngestOptions.MaintainInsertionOrder = true + numProcessed, numFailed, err := imp.ImportDocuments() + require.NoError(t, err, "should import CSV data without error") + assert.EqualValues(t, 3, numProcessed, "should process every CSV data line") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + + // Then delete two documents + imp, err = NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test_delete_with_blanks.csv" + fields = "_id,c,b" + imp.InputOptions.Fields = &fields + imp.IngestOptions.Mode = modeDelete + imp.IngestOptions.IgnoreBlanks = true + imp.IngestOptions.StopOnError = true + // Must specify upsert fields since option parsing is skipped in tests + imp.upsertFields = []string{"c"} + numProcessed, numFailed, err = imp.ImportDocuments() + require.NoError(t, err, "should delete documents without error") + assert.EqualValues( + t, + 1, + numProcessed, + "should process only the document with a non-blank upsert field", + ) + assert.EqualValues(t, 0, numFailed, "should not fail any document") + + expectedDocuments := []bson.M{ + {"_id": int32(3), "c": 5.4, "b": "string"}, + {"_id": int32(5), "c": int32(6), "b": int32(6)}, + } + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should not delete documents with a blank upsert field", + ) + }) + + t.Run("CSV import with --mode=upsert/--upsertFields with duplicate id should succeed "+ + "even if stopOnError is set", func(t *testing.T) { + cleanImportCollection(t) + + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test_duplicate.csv" + fields := "_id,b,c" + imp.InputOptions.Fields = &fields + imp.IngestOptions.Mode = modeUpsert + imp.IngestOptions.StopOnError = true + imp.upsertFields = []string{"_id"} + numProcessed, numFailed, err := imp.ImportDocuments() + require.NoError(t, err, "should upsert a duplicate _id without error") + assert.EqualValues(t, 5, numProcessed, "should process every document") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + expectedDocuments := []bson.M{ + {"_id": int32(1), "b": int32(2), "c": int32(3)}, + {"_id": int32(3), "b": 5.4, "c": "string"}, + {"_id": int32(5), "b": int32(6), "c": int32(9)}, + {"_id": int32(8), "b": int32(6), "c": int32(6)}, + } + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should upsert the duplicate document instead of failing", + ) + }) + + t.Run("an error should be thrown for CSV import on test data with "+ + "duplicate _id if --stopOnError is set", func(t *testing.T) { + cleanImportCollection(t) + + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test_duplicate.csv" + fields := "_id,b,c" + imp.InputOptions.Fields = &fields + imp.IngestOptions.StopOnError = true + imp.IngestOptions.WriteConcern = "1" + imp.IngestOptions.MaintainInsertionOrder = true + numInserted, numFailed, err := imp.ImportDocuments() + require.Error(t, err, "should stop on a duplicate _id when --stopOnError is set") + assert.EqualValues(t, 3, numInserted, "should process the documents before the error") + assert.EqualValues(t, 1, numFailed, "should fail the duplicate document") + expectedDocuments := []bson.M{ + {"_id": int32(1), "b": int32(2), "c": int32(3)}, + {"_id": int32(3), "b": 5.4, "c": "string"}, + {"_id": int32(5), "b": int32(6), "c": int32(6)}, + } + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should import only documents before the duplicate _id", + ) + }) + + t.Run("an error should be thrown for JSON import on test data that "+ + "is a JSON array without passing --jsonArray", func(t *testing.T) { + cleanImportCollection(t) + + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.File = "testdata/test_array.json" + imp.IngestOptions.WriteConcern = "1" + numInserted, _, err := imp.ImportDocuments() + require.Error(t, err, "should reject a JSON array without --jsonArray") + assert.EqualValues(t, 0, numInserted, "should not process any document") + }) - // Then delete two documents - imp, err = NewMongoImport() - So(err, ShouldBeNil) + t.Run("an error should be thrown if a plain JSON file is supplied", func(t *testing.T) { + cleanImportCollection(t) - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test_delete.csv" - fields = "_id,c,b" - imp.InputOptions.Fields = &fields - imp.IngestOptions.Mode = modeDelete - imp.IngestOptions.StopOnError = true - // Must specify upsert fields since option parsing is skipped in tests - imp.upsertFields = []string{"_id"} - numProcessed, numFailed, err = imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 2) - So(numFailed, ShouldEqual, 0) + fileHandle, err := os.Open("testdata/test_plain.json") + require.NoError(t, err, "should open the test fixture") + jsonInputReader := NewJSONInputReader(true, true, fileHandle, 1) + docChan := make(chan bson.D, 1) + require.Error( + t, + jsonInputReader.StreamDocument(t.Context(), true, docChan), + "should reject a plain JSON file", + ) + }) - expectedDocuments := []bson.M{ - {"_id": int32(3), "c": 5.4, "b": "string"}, - } - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - }) - Convey("CSV import with --mode=delete and --upsertFields should succeed", func() { - // First import 3 documents - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test.csv" - fields := "_id,c,b" - imp.InputOptions.Fields = &fields - imp.IngestOptions.MaintainInsertionOrder = true - numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 3) - So(numFailed, ShouldEqual, 0) + t.Run("an error should be thrown for invalid CSV import on test data", func(t *testing.T) { + cleanImportCollection(t) - // Then delete two documents - imp, err = NewMongoImport() - So(err, ShouldBeNil) + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") + imp.IngestOptions.Mode = modeInsert + imp.InputOptions.Type = CSV + imp.InputOptions.File = "testdata/test_bad.csv" + fields := "_id,b,c" + imp.InputOptions.Fields = &fields + imp.IngestOptions.StopOnError = true + imp.IngestOptions.WriteConcern = "1" + imp.IngestOptions.MaintainInsertionOrder = true + imp.IngestOptions.BulkBufferSize = 1 + _, _, err = imp.ImportDocuments() + require.Error(t, err, "should reject invalid CSV data") + }) - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test_delete.csv" - fields = "_id,c,b" - imp.InputOptions.Fields = &fields - imp.IngestOptions.Mode = modeDelete - imp.IngestOptions.StopOnError = true - // Must specify upsert fields since option parsing is skipped in tests - imp.upsertFields = []string{"b", "c"} - numProcessed, numFailed, err = imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 1) - So(numFailed, ShouldEqual, 0) + t.Run( + "CSV import with --mode=upsert/--upsertFields with a nested upsert field should succeed when repeated", + func(t *testing.T) { + cleanImportCollection(t) - expectedDocuments := []bson.M{ - {"_id": int32(3), "c": 5.4, "b": "string"}, - {"_id": int32(5), "c": int32(6), "b": int32(6)}, - } - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - }) - Convey("CSV import with --mode=delete and --ignoreBlanks should not take any action for "+ - "documents that have blank values for upsert fields", func() { - // First import 3 documents imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert + require.NoError(t, err, "should create a new mongoimport instance") imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test.csv" - fields := "_id,c,b" - imp.InputOptions.Fields = &fields - imp.IngestOptions.MaintainInsertionOrder = true + imp.InputOptions.File = "testdata/test_nested_upsert.csv" + imp.InputOptions.HeaderLine = true + imp.IngestOptions.Mode = modeUpsert + imp.upsertFields = []string{"level1.level2.key1"} numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 3) - So(numFailed, ShouldEqual, 0) - - // Then delete two documents + require.NoError(t, err, "should upsert a nested field without error") + assert.EqualValues(t, 1, numProcessed, "should process the document") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + n, err := countDocuments(t, imp.SessionProvider) + require.NoError(t, err, "should count the documents in the collection") + assert.Equal(t, 1, n, "should insert exactly one document") + + // Repeat must succeed imp, err = NewMongoImport() - So(err, ShouldBeNil) - + require.NoError(t, err, "should create a new mongoimport instance") imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test_delete_with_blanks.csv" - fields = "_id,c,b" - imp.InputOptions.Fields = &fields - imp.IngestOptions.Mode = modeDelete - imp.IngestOptions.IgnoreBlanks = true - imp.IngestOptions.StopOnError = true - // Must specify upsert fields since option parsing is skipped in tests - imp.upsertFields = []string{"c"} + imp.InputOptions.File = "testdata/test_nested_upsert.csv" + imp.InputOptions.HeaderLine = true + imp.IngestOptions.Mode = modeUpsert + imp.upsertFields = []string{"level1.level2.key1"} numProcessed, numFailed, err = imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 1) - So(numFailed, ShouldEqual, 0) + require.NoError(t, err, "should upsert the same nested field again without error") + assert.EqualValues(t, 1, numProcessed, "should process the document") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + n, err = countDocuments(t, imp.SessionProvider) + require.NoError(t, err, "should count the documents in the collection") + assert.Equal(t, 1, n, "should still hold exactly one document after the repeat upsert") + }, + ) - expectedDocuments := []bson.M{ - {"_id": int32(3), "c": 5.4, "b": "string"}, - {"_id": int32(5), "c": int32(6), "b": int32(6)}, - } - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - }) - Convey("CSV import with --mode=upsert/--upsertFields with duplicate id should succeed "+ - "even if stopOnError is set", func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test_duplicate.csv" - fields := "_id,b,c" - imp.InputOptions.Fields = &fields - imp.IngestOptions.Mode = modeUpsert - imp.IngestOptions.StopOnError = true - imp.upsertFields = []string{"_id"} - numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 5) - So(numFailed, ShouldEqual, 0) - expectedDocuments := []bson.M{ - {"_id": int32(1), "b": int32(2), "c": int32(3)}, - {"_id": int32(3), "b": 5.4, "c": "string"}, - {"_id": int32(5), "b": int32(6), "c": int32(9)}, - {"_id": int32(8), "b": int32(6), "c": int32(6)}, - } - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - }) - Convey("an error should be thrown for CSV import on test data with "+ - "duplicate _id if --stopOnError is set", func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test_duplicate.csv" - fields := "_id,b,c" - imp.InputOptions.Fields = &fields - imp.IngestOptions.StopOnError = true - imp.IngestOptions.WriteConcern = "1" - imp.IngestOptions.MaintainInsertionOrder = true - numInserted, numFailed, err := imp.ImportDocuments() - So(err, ShouldNotBeNil) - So(numInserted, ShouldEqual, 3) - So(numFailed, ShouldEqual, 1) - expectedDocuments := []bson.M{ - {"_id": int32(1), "b": int32(2), "c": int32(3)}, - {"_id": int32(3), "b": 5.4, "c": "string"}, - {"_id": int32(5), "b": int32(6), "c": int32(6)}, - } - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - }) - Convey("an error should be thrown for JSON import on test data that "+ - "is a JSON array without passing --jsonArray", func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.File = "testdata/test_array.json" - imp.IngestOptions.WriteConcern = "1" - numInserted, _, err := imp.ImportDocuments() - So(err, ShouldNotBeNil) - So(numInserted, ShouldEqual, 0) - }) - Convey("an error should be thrown if a plain JSON file is supplied", func() { - fileHandle, err := os.Open("testdata/test_plain.json") - So(err, ShouldBeNil) - jsonInputReader := NewJSONInputReader(true, true, fileHandle, 1) - docChan := make(chan bson.D, 1) - So(jsonInputReader.StreamDocument(t.Context(), true, docChan), ShouldNotBeNil) - }) - Convey("an error should be thrown for invalid CSV import on test data", func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.IngestOptions.Mode = modeInsert - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test_bad.csv" - fields := "_id,b,c" - imp.InputOptions.Fields = &fields - imp.IngestOptions.StopOnError = true - imp.IngestOptions.WriteConcern = "1" - imp.IngestOptions.MaintainInsertionOrder = true - imp.IngestOptions.BulkBufferSize = 1 - _, _, err = imp.ImportDocuments() - So(err, ShouldNotBeNil) - }) - Convey( - "CSV import with --mode=upsert/--upsertFields with a nested upsert field should succeed when repeated", - func() { - imp, err := NewMongoImport() - So(err, ShouldBeNil) - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test_nested_upsert.csv" - imp.InputOptions.HeaderLine = true - imp.IngestOptions.Mode = modeUpsert - imp.upsertFields = []string{"level1.level2.key1"} - numProcessed, numFailed, err := imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 1) - So(numFailed, ShouldEqual, 0) - n, err := countDocuments(t, imp.SessionProvider) - So(err, ShouldBeNil) - So(n, ShouldEqual, 1) - - // Repeat must succeed - imp, err = NewMongoImport() - So(err, ShouldBeNil) - imp.InputOptions.Type = CSV - imp.InputOptions.File = "testdata/test_nested_upsert.csv" - imp.InputOptions.HeaderLine = true - imp.IngestOptions.Mode = modeUpsert - imp.upsertFields = []string{"level1.level2.key1"} - numProcessed, numFailed, err = imp.ImportDocuments() - So(err, ShouldBeNil) - So(numProcessed, ShouldEqual, 1) - So(numFailed, ShouldEqual, 0) - n, err = countDocuments(t, imp.SessionProvider) - So(err, ShouldBeNil) - So(n, ShouldEqual, 1) - }, - ) - Convey("With --useArrayIndexFields: Top-level numerical fields should be document keys", - nestedFieldsTestHelper( - t, - "_id,0,1\n1,2,3", - []bson.M{ + t.Run("with --useArrayIndexFields", func(t *testing.T) { + cases := []struct { + name string + data string + expectedDocuments []bson.M + expectedErr error + }{ + { + name: "top-level numerical fields should be document keys", + data: "_id,0,1\n1,2,3", + expectedDocuments: []bson.M{ {"_id": int32(1), "0": int32(2), "1": int32(3)}, }, - nil, - ), - ) - Convey("With --useArrayIndexFields: Should insert nested document", - nestedFieldsTestHelper( - t, - "_id,a.a,a.b\n1,2,3", - []bson.M{ + }, + { + name: "should insert nested document", + data: "_id,a.a,a.b\n1,2,3", + expectedDocuments: []bson.M{ {"_id": int32(1), "a": bson.M{"a": int32(2), "b": int32(3)}}, }, - nil, - ), - ) - Convey("With --useArrayIndexFields: Should insert an array", - nestedFieldsTestHelper( - t, - "_id,a.0,a.1\n1,2,3", - []bson.M{ + }, + { + name: "should insert an array", + data: "_id,a.0,a.1\n1,2,3", + expectedDocuments: []bson.M{ {"_id": int32(1), "a": bson.A{int32(2), int32(3)}}, }, - nil, - ), - ) - Convey("With --useArrayIndexFields: Should insert an array of documents", - nestedFieldsTestHelper( - t, - "_id,a.0.a,a.0.b,a.1.a\n1,2,3,4", - []bson.M{ + }, + { + name: "should insert an array of documents", + data: "_id,a.0.a,a.0.b,a.1.a\n1,2,3,4", + expectedDocuments: []bson.M{ { "_id": int32(1), "a": bson.A{bson.M{"a": int32(2), "b": int32(3)}, bson.M{"a": int32(4)}}, }, }, - nil, - ), - ) - Convey("With --useArrayIndexFields: Should insert an array of arrays", - nestedFieldsTestHelper( - t, - "_id,a.0.0,a.0.1,a.1.0\n1,2,3,4", - []bson.M{ + }, + { + name: "should insert an array of arrays", + data: "_id,a.0.0,a.0.1,a.1.0\n1,2,3,4", + expectedDocuments: []bson.M{ {"_id": int32(1), "a": bson.A{bson.A{int32(2), int32(3)}, bson.A{int32(4)}}}, }, - nil, - ), - ) - Convey("With --useArrayIndexFields: Should insert an array when top-level key is \"0\"", - nestedFieldsTestHelper( - t, - "_id,0.0,0.1\n1,2,3", - []bson.M{ + }, + { + name: `should insert an array when top-level key is "0"`, + data: "_id,0.0,0.1\n1,2,3", + expectedDocuments: []bson.M{ {"_id": int32(1), "0": bson.A{int32(2), int32(3)}}, }, - nil, - ), - ) - Convey("With --useArrayIndexFields: Should insert an array in a document in an array", - nestedFieldsTestHelper( - t, - "_id,a.0.a.0,a.0.a.1\n1,2,3", - []bson.M{ + }, + { + name: "should insert an array in a document in an array", + data: "_id,a.0.a.0,a.0.a.1\n1,2,3", + expectedDocuments: []bson.M{ {"_id": int32(1), "a": bson.A{bson.M{"a": bson.A{int32(2), int32(3)}}}}, }, - nil, - ), - ) - Convey( - "With --useArrayIndexFields: If an array element is blank in the csv file, an empty string should be inserted", - nestedFieldsTestHelper( - t, - "_id,a.0,a.1,a.2\n1,2,,4", - []bson.M{ + }, + { + name: "if an array element is blank in the csv file, an empty string should be inserted", + data: "_id,a.0,a.1,a.2\n1,2,,4", + expectedDocuments: []bson.M{ {"_id": int32(1), "a": bson.A{int32(2), "", int32(4)}}, }, - nil, - ), - ) - Convey( - "With --useArrayIndexFields: If an array with more than 10 fields should be inserted", - nestedFieldsTestHelper( - t, - "_id,a.0,a.1,a.2,a.3,a.4,a.5,a.6,a.7,a.8,a.9,a.10\n0,1,2,3,4,5,6,7,8,9,10", - []bson.M{ + }, + { + name: "if an array with more than 10 fields should be inserted", + data: "_id,a.0,a.1,a.2,a.3,a.4,a.5,a.6,a.7,a.8,a.9,a.10\n0,1,2,3,4,5,6,7,8,9,10", + expectedDocuments: []bson.M{ { "_id": int32(0), "a": bson.A{ @@ -1233,252 +1324,207 @@ func TestImportDocuments(t *testing.T) { }, }, }, - nil, - ), - ) - Convey( - "With --useArrayIndexFields: An number with leading zeros should be interpreted as a document key, not an index", - nestedFieldsTestHelper( - t, - "_id,a.0001\n1,2", - []bson.M{ + }, + { + name: "a number with leading zeros should be interpreted as a document key, not an index", + data: "_id,a.0001\n1,2", + expectedDocuments: []bson.M{ {"_id": int32(1), "a": bson.M{"0001": int32(2)}}, }, - nil, - ), - ) - Convey( - "With --useArrayIndexFields: An number with leading plus should be interpreted as a document key, not an index", - nestedFieldsTestHelper( - t, - "_id,a.+15558675309\n1,2", - []bson.M{ + }, + { + name: "a number with leading plus should be interpreted as a document key, not an index", + data: "_id,a.+15558675309\n1,2", + expectedDocuments: []bson.M{ {"_id": int32(1), "a": bson.M{"+15558675309": int32(2)}}, }, - nil, - ), - ) - Convey( - "With --useArrayIndexFields: Should be able to make changes to document in an array once document has been created", - nestedFieldsTestHelper( - t, - "_id,a.0.a,a.1.a,a.0.b\n1,2,3,4", - []bson.M{ + }, + { + name: "should be able to make changes to document in an array once document has been created", + data: "_id,a.0.a,a.1.a,a.0.b\n1,2,3,4", + expectedDocuments: []bson.M{ { "_id": int32(1), "a": bson.A{bson.M{"a": int32(2), "b": int32(4)}, bson.M{"a": int32(3)}}, }, }, - nil, - ), - ) - Convey("With --useArrayIndexFields: Duplicate fields should throw an error", - nestedFieldsTestHelper( - t, - "_id,a.0,a.0\n1,2,3", - nil, - fmt.Errorf( + }, + { + name: "duplicate fields should throw an error", + data: "_id,a.0,a.0\n1,2,3", + expectedErr: fmt.Errorf( "array index error with field `a.0`: array indexes in fields must start from 0 and increase sequentially", ), - ), - ) - Convey("With --useArrayIndexFields: Array fields not starting at 0 should throw an error", - nestedFieldsTestHelper( - t, - "_id,a.1,a.0\n1,2,3", - nil, - fmt.Errorf( + }, + { + name: "array fields not starting at 0 should throw an error", + data: "_id,a.1,a.0\n1,2,3", + expectedErr: fmt.Errorf( "array index error with field `a.1`: array indexes in fields must start from 0 and increase sequentially", ), - ), - ) - Convey("With --useArrayIndexFields: Array fields skipping an index should throw an error", - nestedFieldsTestHelper( - t, - "_id,a.0,a.2\n1,2,3", - nil, - fmt.Errorf( + }, + { + name: "array fields skipping an index should throw an error", + data: "_id,a.0,a.2\n1,2,3", + expectedErr: fmt.Errorf( "array index error with field `a.2`: array indexes in fields must start from 0 and increase sequentially", ), - ), - ) - Convey( - "With --useArrayIndexFields: Array fields with sub documents skipping an index should throw an error", - nestedFieldsTestHelper( - t, - "_id,a.0.a,a.2.a\n1,2,3", - nil, - fmt.Errorf( + }, + { + name: "array fields with sub documents skipping an index should throw an error", + data: "_id,a.0.a,a.2.a\n1,2,3", + expectedErr: fmt.Errorf( "array index error with field `a.2.a`: array indexes in fields must start from 0 and increase sequentially", ), - ), - ) - Convey( - "With --useArrayIndexFields: Array field should throw an error if value has already been set as document", - nestedFieldsTestHelper( - t, - "_id,a.a,a.0\n1,2,3", - nil, - fmt.Errorf("fields `a.a` and `a.0` are incompatible"), - ), - ) - Convey( - "With --useArrayIndexFields: Array field should throw an error if value has already been set as document (deep object)", - nestedFieldsTestHelper( - t, - "_id,a.a.a.a,a.a.0.a\n1,2,3", - nil, - fmt.Errorf("fields `a.a.a.a` and `a.a.0.a` are incompatible"), - ), - ) - Convey( - "With --useArrayIndexFields: Document field should throw an error if value has already been set as array", - nestedFieldsTestHelper( - t, - "_id,a.0,a.a\n1,2,3", - nil, - fmt.Errorf("fields `a.0` and `a.a` are incompatible"), - ), - ) - Convey( - "With --useArrayIndexFields: Document field should throw an error if value has already been set as array (deep object)", - nestedFieldsTestHelper( - t, - "_id,a.a.a.0,a.a.a.a\n1,2,3", - nil, - fmt.Errorf("fields `a.a.a.0` and `a.a.a.a` are incompatible"), - ), - ) - Convey( - "With --useArrayIndexFields: Array field should throw an error if value has already been set as value", - nestedFieldsTestHelper( - t, - "_id,a,a.0\n1,2,3", - nil, - fmt.Errorf("fields `a` and `a.0` are incompatible"), - ), - ) - Convey( - "With --useArrayIndexFields: Array field should throw an error if value has already been set as value (deep object)", - nestedFieldsTestHelper( - t, - "_id,a.a.a,a.a.a.0\n1,2,3", - nil, - fmt.Errorf("fields `a.a.a` and `a.a.a.0` are incompatible"), - ), - ) - Convey( - "With --useArrayIndexFields: Array field should be incompatible with a document field starting with a symbol that is sorted before 0", - nestedFieldsTestHelper( - t, - "_id,a./,a.0\n1,2,3", - nil, - fmt.Errorf("fields `a./` and `a.0` are incompatible"), - ), - ) - Convey("With --useArrayIndexFields: Indexes in fields must start from 0", - nestedFieldsTestHelper( - t, - "_id,a,b.1\n1,2,3", - nil, - fmt.Errorf( + }, + { + name: "array field should throw an error if value has already been set as document", + data: "_id,a.a,a.0\n1,2,3", + expectedErr: fmt.Errorf("fields `a.a` and `a.0` are incompatible"), + }, + { + name: "array field should throw an error if value has already been set as document (deep object)", + data: "_id,a.a.a.a,a.a.0.a\n1,2,3", + expectedErr: fmt.Errorf("fields `a.a.a.a` and `a.a.0.a` are incompatible"), + }, + { + name: "document field should throw an error if value has already been set as array", + data: "_id,a.0,a.a\n1,2,3", + expectedErr: fmt.Errorf("fields `a.0` and `a.a` are incompatible"), + }, + { + name: "document field should throw an error if value has already been set as array (deep object)", + data: "_id,a.a.a.0,a.a.a.a\n1,2,3", + expectedErr: fmt.Errorf("fields `a.a.a.0` and `a.a.a.a` are incompatible"), + }, + { + name: "array field should throw an error if value has already been set as value", + data: "_id,a,a.0\n1,2,3", + expectedErr: fmt.Errorf("fields `a` and `a.0` are incompatible"), + }, + { + name: "array field should throw an error if value has already been set as value (deep object)", + data: "_id,a.a.a,a.a.a.0\n1,2,3", + expectedErr: fmt.Errorf("fields `a.a.a` and `a.a.a.0` are incompatible"), + }, + { + name: "array field should be incompatible with a document field starting with a symbol that is sorted before 0", + data: "_id,a./,a.0\n1,2,3", + expectedErr: fmt.Errorf("fields `a./` and `a.0` are incompatible"), + }, + { + name: "indexes in fields must start from 0", + data: "_id,a,b.1\n1,2,3", + expectedErr: fmt.Errorf( "array index error with field `b.1`: array indexes in fields must start from 0 and increase sequentially", ), - ), - ) - Convey( - "With --useArrayIndexFields: Indexes in fields must start from 0 (last field same length)", - nestedFieldsTestHelper( - t, - "_id,a.b,b.1\n1,2,3", - nil, - fmt.Errorf( + }, + { + name: "indexes in fields must start from 0 (last field same length)", + data: "_id,a.b,b.1\n1,2,3", + expectedErr: fmt.Errorf( "array index error with field `b.1`: array indexes in fields must start from 0 and increase sequentially", ), - ), - ) - Convey( - "With --useArrayIndexFields: Fields that are the same should throw an error (no arrays)", - nestedFieldsTestHelper( - t, - "_id,a.b,a.b\n1,2,3", - nil, - fmt.Errorf("fields cannot be identical: `a.b` and `a.b`"), - ), - ) - Convey("With --useArrayIndexFields: Repeated array index should throw error", - nestedFieldsTestHelper( - t, - "_id,a.0,a.1,a.2,a.0\n1,2,3,4,5", - nil, - fmt.Errorf( + }, + { + name: "fields that are the same should throw an error (no arrays)", + data: "_id,a.b,a.b\n1,2,3", + expectedErr: fmt.Errorf("fields cannot be identical: `a.b` and `a.b`"), + }, + { + name: "repeated array index should throw error", + data: "_id,a.0,a.1,a.2,a.0\n1,2,3,4,5", + expectedErr: fmt.Errorf( "array index error with field `a.0`: array indexes in fields must start from 0 and increase sequentially", ), - ), - ) - Convey("With --useArrayIndexFields: Array entries of different types should throw an error", - nestedFieldsTestHelper( - t, - "_id,a.a.0.a,a.a.0.1\n1,2,3", - nil, - fmt.Errorf("fields `a.a.0.a` and `a.a.0.1` are incompatible"), - ), - ) - Convey( - "With --useArrayIndexFields: Document field should throw an error if element has already been set to an array", - nestedFieldsTestHelper( - t, - "_id,a.0.0,a.0.a\n1,2,3", - nil, - fmt.Errorf("fields `a.0.0` and `a.0.a` are incompatible"), - ), - ) - Convey( - "With --useArrayIndexFields: Incompatible fields should throw error (one long, one short)", - nestedFieldsTestHelper( - t, - "_id,a.a.a.a,a.a\n1,2,3", - nil, - fmt.Errorf("fields `a.a.a.a` and `a.a` are incompatible"), - ), - ) + }, + { + name: "array entries of different types should throw an error", + data: "_id,a.a.0.a,a.a.0.1\n1,2,3", + expectedErr: fmt.Errorf("fields `a.a.0.a` and `a.a.0.1` are incompatible"), + }, + { + name: "document field should throw an error if element has already been set to an array", + data: "_id,a.0.0,a.0.a\n1,2,3", + expectedErr: fmt.Errorf("fields `a.0.0` and `a.0.a` are incompatible"), + }, + { + name: "incompatible fields should throw error (one long, one short)", + data: "_id,a.a.a.a,a.a\n1,2,3", + expectedErr: fmt.Errorf("fields `a.a.a.a` and `a.a` are incompatible"), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cleanImportCollection(t) + runNestedFieldsTestCase(t, tc.data, tc.expectedDocuments, tc.expectedErr) + }) + } + }) +} + +// Every subtest here writes to the same collection, so each one must clear it or it +// inherits documents from whichever sibling ran before it. +func cleanImportCollection(t *testing.T) { + t.Helper() + + // assert, not require: require calls FailNow, which is runtime.Goexit, and + // that unwinds out of the cleanup runner and skips every cleanup registered + // before this one. + t.Cleanup(func() { + sessionProvider, err := db.NewSessionProvider(*getBasicToolOptions()) + assert.NoError(t, err, "should get a session provider") + session, err := sessionProvider.GetSession() + assert.NoError(t, err, "should get a session") + _, err = session.Database(testDb). + Collection(testCollection). + DeleteMany(context.Background(), bson.D{}) + assert.NoError(t, err, "should drop the test collection") }) } -func nestedFieldsTestHelper( +func runNestedFieldsTestCase( t *testing.T, data string, expectedDocuments []bson.M, expectedErr error, -) func() { - return func() { - err := os.WriteFile(filepath.FromSlash("./temp_test_data.csv"), []byte(data), 0644) - So(err, ShouldBeNil) - defer func() { - err = os.Remove(filepath.FromSlash("./temp_test_data.csv")) - So(err, ShouldBeNil) - }() +) { + t.Helper() - imp, err := NewMongoImport() - So(err, ShouldBeNil) + err := os.WriteFile(filepath.FromSlash("./temp_test_data.csv"), []byte(data), 0644) + require.NoError(t, err, "should write the temporary CSV fixture") + t.Cleanup(func() { + require.NoError( + t, + os.Remove(filepath.FromSlash("./temp_test_data.csv")), + "should remove the temporary CSV fixture", + ) + }) - imp.InputOptions.Type = CSV - imp.InputOptions.File = "./temp_test_data.csv" - imp.InputOptions.HeaderLine = true - imp.InputOptions.UseArrayIndexFields = true - imp.IngestOptions.Mode = modeInsert + imp, err := NewMongoImport() + require.NoError(t, err, "should create a new mongoimport instance") - numImported, numFailed, err := imp.ImportDocuments() - if expectedDocuments == nil { - So(err, ShouldNotBeNil) - So(err, ShouldResemble, expectedErr) - } else { - So(err, ShouldBeNil) - So(numImported, ShouldEqual, len(expectedDocuments)) - So(numFailed, ShouldEqual, 0) + imp.InputOptions.Type = CSV + imp.InputOptions.File = "./temp_test_data.csv" + imp.InputOptions.HeaderLine = true + imp.InputOptions.UseArrayIndexFields = true + imp.IngestOptions.Mode = modeInsert - So(checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), ShouldBeNil) - } + numImported, numFailed, err := imp.ImportDocuments() + if expectedDocuments == nil { + require.Error(t, err, "should reject the incompatible or out-of-order fields") + require.Equal(t, expectedErr, err, "should report the specific field error") + } else { + require.NoError(t, err, "should import the array-indexed fields without error") + assert.EqualValues(t, len(expectedDocuments), numImported, "should process every document") + assert.EqualValues(t, 0, numFailed, "should not fail any document") + + require.NoError( + t, + checkOnlyHasDocuments(t, imp.SessionProvider, expectedDocuments), + "should insert the expected array-indexed document", + ) } }