Bug in copying collection

Anton shared this problem 3 years ago
Cannot Reproduce

I have 2 DB and 2 collections:


master cars

master_test cars_test


Auto generated script looks like this

const BATCH_SIZE = 10000;

let srcConnection = "localhost_admin";
let srcDb = "master";
let dstConnection = "localhost_admin";
let dstDb = "master_test";

//idPolicy: overwrite_with_same_id|always_insert_with_new_id|insert_with_new_id_if_id_exists|skip_documents_with_existing_id|abort_if_id_already_exists
let toCopyCollections = [
    { srcCollection: "cars", query: {}, projection: {}, dstCollection: "cars_test", idPolicy: "overwrite_with_same_id", dropDstCollection: true }
];

// if have more indexes than default index (_id), create index after copy data.
let shouldCreateIndex = true;

let totalCopyResult = {
    result: {},
    fails: [],
}

function copyCollection(params) {
    return
    let {srcCollection, dstCollection, query, projection, idPolicy, dropDstCollection} = params;
    let continueRead = true;

    console.log(`copy docs from ${srcConnection}:${srcDb}:${srcCollection} to ${dstConnection}:${dstDb}:${dstCollection} start...`);

    let isSameCollection = srcConnection === dstConnection && srcDb === dstDb && srcCollection === dstCollection;

    if (isSameCollection) {
        if (toCopyCollections.length === 1)
            shouldCreateIndex = false;
        else
            params.shouldCreateIndex = false;
    }

    if (dropDstCollection) {
        // srcCollection is same to dstCollection, can not drop dstCollection (equal to drop srcCollection)
        // drop dst collection and copy from same collection, means nothing to do.
        if (isSameCollection) return;

        mb.dropCollection({ connection: dstConnection, db: dstDb, collection: dstCollection });
    }

    totalCopyResult.result[dstCollection] = {
        nInserted: 0,
        nModified: 0,
        nSkipped: 0,
        failed: 0,
    };
    let collectionRst = totalCopyResult.result[dstCollection];

    let limitReadCount = Number.MAX_SAFE_INTEGER;
    if (isSameCollection)
        limitReadCount = mb.runScript({ connection: srcConnection, db: srcDb, script: `db.getCollection("${srcCollection}").find(${tojson(query)}).count()` })

    let skip = 0;
    while (continueRead) {
        let limit = limitReadCount > BATCH_SIZE ? BATCH_SIZE : limitReadCount;
        let docs = mb.readFromDb({ connection: srcConnection, db: srcDb, collection: srcCollection, query, projection, skip, limit });
        let readLength = docs.length;
        skip += readLength;
        limitReadCount -= readLength;

        if (readLength < BATCH_SIZE)
            continueRead = false;

        if (readLength) {
            let copyResult = mb.writeToDb({ connection: dstConnection, db: dstDb, collection: dstCollection, docs, idPolicy });
            let failed = copyResult.errors.length;
            let success = copyResult.nInserted + copyResult.nModified;

            collectionRst.nInserted += copyResult.nInserted;
            collectionRst.nModified += copyResult.nModified;
            collectionRst.nSkipped += copyResult.nSkipped;
            collectionRst.failed += failed;

            console.log(`${dstCollection}: ${collectionRst.nInserted + collectionRst.nModified} docs successfully imported, ${collectionRst.failed} docs failed.`);

            if (failed) {
                console.log("Failed objects", copyResult.errors);
            }

            totalCopyResult.fails = [...totalCopyResult.fails, ...copyResult.errors];
        }

        sleep(10)
    }

    console.log(`copy docs from ${srcConnection}:${srcDb}:${srcCollection} to ${dstConnection}:${dstDb}:${dstCollection} finished.`);
}

toCopyCollections.forEach(it => copyCollection(it));

if (shouldCreateIndex) {
    let indexCreationPrompted = false;
    function indexCreationPrompt() {
        if (indexCreationPrompted) return;

        let waitTime = 3;
        console.log(`Index creating will start in ${waitTime} seconds...`)
        sleep(1000 * waitTime);
        indexCreationPrompted = true;
    }

    let srcCollections = toCopyCollections.filter(it => it["shouldCreateIndex"] !== false).map(it => it.srcCollection)
    srcCollections.forEach(it => {
        let indexes = mb.runScript({ connection: srcConnection, db: srcDb, script: `db.getCollection("${it}").getIndexes();` });
        if (indexes.length > 1) {
            let toCopyCollection = _.find(toCopyCollections, { srcCollection: it });
            if (!toCopyCollection) return;

            let dstCollection = toCopyCollection.dstCollection;

            indexes.forEach(index => {
                if (index.name === "_id_") return;

                indexCreationPrompt();

                let newIndex = _.cloneDeep(index);

                // remove index version and engine info, these info may fail createIndexes operator.
                delete newIndex.v;
                delete newIndex.textIndexVersion;
                delete newIndex["2dsphereIndexVersion"];
                delete newIndex.storageEngine;

                newIndex.ns = `${dstDb}.${dstCollection}`;
                
                console.log(`create index ${newIndex.name} for ${dstDb}.${dstCollection}`);
                console.log(db.runCommand({
                    createIndexes: dstCollection,
                    indexes: [newIndex]
                }));
            })
        }
    });

    if (indexCreationPrompted)
        console.log("create index complete.")
}

if (totalCopyResult.result) {
    let successed = 0;
    let failed = 0;
    let collections = _.keys(totalCopyResult.result);
    collections.forEach((key) => {
        let obj = totalCopyResult.result[key];
        successed += obj.nInserted + obj.nModified;
        failed += obj.failed;
    });

    console.log(`${successed} document(s) of ${collections.length} collection(s) have been copied.`, totalCopyResult.result);

    if (failed) {
        console.log(`${failed} document(s) haven't been copied, please check failed list below.`);
    } else {
        console.log("All documents copied successfully.");
    }
}

if (totalCopyResult.fails.length) {
    console.log("All failed objects", totalCopyResult.fails);
}

However, it produces errors

#script:101:9  	+0.010s
Index creating will start in 3 seconds...
#script:136:9  	+3.012s
create index make_name_1 for master_test.cars_test
#script:137:9  	+3.145s
{
	"message" : "The value of the field 'ns' (master_test.cars_test) doesn't match the namespace 'master.cars_test'",
	"stack" : "MongoError: The value of the field 'ns' (master_test.cars_test) doesn't match the namespace 'master.cars_test'\n    at Function.MongoError.create (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/error.js:31:11)\n    at /Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:497:72\n    at authenticateStragglers (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:443:16)\n    at Connection.messageHandler (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:477:5)\n    at Socket. (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/connection.js:321:22)\n    at emitOne (events.js:96:13)\n    at Socket.emit (events.js:188:7)\n    at readableAddChunk (_stream_readable.js:176:18)\n    at Socket.Readable.push (_stream_readable.js:134:10)\n    at TCP.onread (net.js:551:20)",
	"name" : "MongoError",
	"ok" : 0,
	"errmsg" : "The value of the field 'ns' (master_test.cars_test) doesn't match the namespace 'master.cars_test'",
	"code" : 2,
	"codeName" : "BadValue"
}


I fixed it by:

-newIndex.ns = `${dstDb}.${dstCollection}`;

+newIndex.ns = `${srcDb}.${dstCollection}`;

Comments (1)

photo
2

I can't recall the issue. If possible, could you please give me the detailed Repro Steps?


I review the code, the newIndex namespace should be `${dstDb}.${dstCollection}`.

photo
1

dstDb has ns

master.cars_test



cars_test doesn't exist on master, but on master_test


For new index(as I see) I need to have master_test.cars_test

so correct index must be ${srcDb}.${dstCollection}`;


By using this I was able to fix index creation

photo
1

I am copying from master.cars to master_test.cars_test

photo
1

Could you please show me the result of the following script?


use  master_test
db.cars_test.getIndexes();

photo
1

/* 1 */
{
	"v" : NumberInt("2"),
	"key" : {
		"_id" : NumberInt("1")
	},
	"name" : "_id_",
	"ns" : "master_test.cars_test"
},

/* 2 */
{
	"v" : NumberInt("2"),
	"key" : {
		"make_name" : 1
	},
	"name" : "make_name_1",
	"ns" : "master_test.cars_test"
},

/* 3 */
{
	"v" : NumberInt("2"),
	"key" : {
		"car_id" : 1,
		"site_name" : 1
	},
	"name" : "car_id_1_site_name_1",
	"ns" : "master_test.cars_test"
},

/* 4 */
{
	"v" : NumberInt("2"),
	"key" : {
		"updated_at" : 1,
		"currency" : 1,
		"damaged" : 1,
		"tax_paid" : 1,
		"is_sold" : 1,
		"mileage" : 1
	},
	"name" : "updated_at_1_currency_1_damaged_1_tax_paid_1_is_sold_1_mileage_1",
	"ns" : "master_test.cars_test"
}

photo
1

In the previous copy script:


srcDb = "master";

dstDb = "master_test";

dstCollection= "cars_test"


In your result, the value of "ns" field is "master_test.cars_test". It equals to "${dstDb}.${dstCollection}", not "${srcDb}.${dstCollection}". right?


So, newIndex.ns = `${dstDb}.${dstCollection}`;

photo
1

When I use newIndex.ns = `${dstDb}.${dstCollection}`;

I have an error

#script:101:9  	+0.010s Index creating will start in 3 seconds... #script:136:9  	+3.012s create index make_name_1 for master_test.cars_test #script:137:9  	+3.145s { 	"message" : "The value of the field 'ns' (master_test.cars_test) doesn't match the namespace 'master.cars_test'", 	"stack" : "MongoError: The value of the field 'ns' (master_test.cars_test) doesn't match the namespace 'master.cars_test'\n    at Function.MongoError.create (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/error.js:31:11)\n    at /Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:497:72\n    at authenticateStragglers (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:443:16)\n    at Connection.messageHandler (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:477:5)\n    at Socket.<anonymous> (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/connection.js:321:22)\n    at emitOne (events.js:96:13)\n    at Socket.emit (events.js:188:7)\n    at readableAddChunk (_stream_readable.js:176:18)\n    at Socket.Readable.push (_stream_readable.js:134:10)\n    at TCP.onread (net.js:551:20)", 	"name" : "MongoError", 	"ok" : 0, 	"errmsg" : "The value of the field 'ns' (master_test.cars_test) doesn't match the namespace 'master.cars_test'", 	"code" : 2, 	"codeName" : "BadValue" }

When I use


newIndex.ns = `${srcDb}.${dstCollection}`;


Then my index correctly creates

photo
1

I am not very understanding, maybe "delete newIndex.ns" is better. What's your MongoDB version? 3.4, 3.2 or 3.0?

photo
1

I am using 3.4.4(remote machine) and both 3.4.4 and 3.5.9 (local machine)


From local machine I am copying to remote machine. But the same error occurs when copying from 3.4.4 local to 3.4.4 local using the same PC.


I found interesting moment!

It is not reproduced constantly, but for specific cases.

I've created test collection with just 1 index for 1 field.


NOT reproduced when

newIndex.ns = `${dstDb}.${dstCollection}`;

let srcConnection = "localhost_admin";
let srcDb = "master";
let dstConnection = "localhost_admin";
let dstDb = "master";

//idPolicy: overwrite_with_same_id|always_insert_with_new_id|insert_with_new_id_if_id_exists|skip_documents_with_existing_id|abort_if_id_already_exists
let toCopyCollections = [
    { srcCollection: "test", query: {}, projection: {}, dstCollection: "test_test", idPolicy: "overwrite_with_same_id", dropDstCollection: true }
];
...
#script:25:9  	+0.001s
copy docs from localhost_admin:master:test to localhost_admin:master:test_test start...
#script:77:9  	+0.039s
test_test: 1 docs successfully imported, 0 docs failed.
#script:89:9  	+0.050s
copy docs from localhost_admin:master:test to localhost_admin:master:test_test finished.
#script:100:9  	+0.054s
Index creating will start in 3 seconds...
#script:129:9  	+3.056s
create index model_name_1 for master.test_test
#script:130:9  	+3.096s
{
	"createdCollectionAutomatically" : false,
	"numIndexesBefore" : 2,
	"numIndexesAfter" : 2,
	"note" : "all indexes already exist",
	"ok" : 1
}
#script:139:9  	+3.096s
create index complete.
#script:152:9  	+3.096s
1 document(s) of 1 collection(s) have been copied., {
	"test_test" : {
		"nInserted" : 1,
		"nModified" : 0,
		"nSkipped" : 0,
		"failed" : 0
	}
}
#script:157:9  	+3.096s
All documents copied successfully.

But reproduced when

let dstDb = "master_test";

#script:25:9  	+0.002s
copy docs from localhost_admin:master:test to localhost_admin:master_test:test_test start...
#script:77:9  	+0.140s
test_test: 1 docs successfully imported, 0 docs failed.
#script:89:9  	+0.152s
copy docs from localhost_admin:master:test to localhost_admin:master_test:test_test finished.
#script:100:9  	+0.156s
Index creating will start in 3 seconds...
#script:129:9  	+3.156s
create index model_name_1 for master_test.test_test
#script:130:9  	+3.197s
{
	"message" : "The value of the field 'ns' (master_test.test_test) doesn't match the namespace 'master.test_test'",
	"stack" : "MongoError: The value of the field 'ns' (master_test.test_test) doesn't match the namespace 'master.test_test'\n    at Function.MongoError.create (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/error.js:31:11)\n    at /Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:497:72\n    at authenticateStragglers (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:443:16)\n    at Connection.messageHandler (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:477:5)\n    at Socket.<anonymous> (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/connection.js:321:22)\n    at emitOne (events.js:96:13)\n    at Socket.emit (events.js:188:7)\n    at readableAddChunk (_stream_readable.js:176:18)\n    at Socket.Readable.push (_stream_readable.js:134:10)\n    at TCP.onread (net.js:551:20)",
	"name" : "MongoError",
	"ok" : 0,
	"errmsg" : "The value of the field 'ns' (master_test.test_test) doesn't match the namespace 'master.test_test'",
	"code" : 2,
	"codeName" : "BadValue"
}
#script:139:9  	+3.197s
create index complete.
#script:152:9  	+3.198s
1 document(s) of 1 collection(s) have been copied., {
	"test_test" : {
		"nInserted" : 1,
		"nModified" : 0,
		"nSkipped" : 0,
		"failed" : 0
	}
}
#script:157:9  	+3.198s
All documents copied successfully.

photo
1

Here is my one object from test collection

{
	"_id" : ObjectId("59715162861b5b68d1db06eb"),
	"make_name" : "Volkswagen",
	"model_name" : "Golf",
	"car_id" : NumberInt("1"),
	"title" : "title",
	"description" : "description",
	"site_name" : "mobile.de",
	"first_registration" : ISODate("2004-07-01T00:00:00.000Z"),
	"fuel" : "Diesel",
	"mileage" : NumberInt("233000"),
	"category" : "Limousine",
	"horse_power" : NumberInt("105"),
	"cubic_capacity" : NumberInt("1896"),
	"transmission" : "Manual",
	"price" : NumberInt("4500"),
	"currency" : "EUR",
	"negotiable" : false,
	"profit" : NumberInt("1000"),
	"owners_count" : NumberInt("1"),
	"color" : NumberInt("1"),
	"condition" : NumberInt("2"),
	"updated_at" : ISODate("2017-02-15T00:00:00.000Z"),
	"origin_add_date" : ISODate("2017-02-15T00:00:00.000Z"),
	"expiration_date" : null,
	"rear_camera" : false,
	"ABS" : false,
	"four_wheel_drive" : false,
	"bluetooth" : false,
	"board_computer" : false,
	"cd_player" : false,
	"electric_mirrors" : true,
	"electric_windows" : false,
	"parking_assistance" : false,
	"handsfree" : false,
	"guarantee" : false,
	"head_up_display" : false,
	"has_inspection" : false,
	"air_conditioning" : false,
	"alloy_wheel_rims" : false,
	"multi_func_steering_wheel" : false,
	"navigation" : false,
	"non_smoking_car" : false,
	"panorama_roof" : false,
	"particle_filter" : false,
	"rain_sensor" : false,
	"full_service_history" : false,
	"power_steering" : false,
	"sunroof" : false,
	"seat_heating" : false,
	"sports_suspension" : false,
	"sports_seats" : false,
	"pre_heating" : false,
	"start_stop" : false,
	"taxi" : false,
	"tax_paid" : true,
	"cruise_control" : false,
	"climat_control" : false,
	"xenon_headlights" : false,
	"security" : false,
	"sport_package" : false,
	"photos" : null,
	"location" : {
		"country_id" : NumberInt("0"),
		"city_id" : NumberInt("0"),
		"region_id" : NumberInt("10")
	},
	"url" : "testcar.ua",
	"business" : true,
	"damaged" : false,
	"options_downloaded" : true,
	"seats_ventilation" : false,
	"version" : null,
	"on_repair" : false,
	"is_sold" : false
}
And index

/* 1 */
{
	"v" : NumberInt("2"),
	"key" : {
		"_id" : NumberInt("1")
	},
	"name" : "_id_",
	"ns" : "master.test"
},

/* 2 */
{
	"v" : NumberInt("2"),
	"key" : {
		"model_name" : 1
	},
	"name" : "model_name_1",
	"ns" : "master.test"
}

photo
1

I guess this problem may be related to the special database "master".


Could you please delete the newIndex.ns and try it again?


-newIndex.ns = `${dstDb}.${dstCollection}`;


delete newIndex.ns;

photo
1

#script:25:9  	+0.002s
copy docs from localhost_admin:master:test to localhost_admin:master_test:test_test start...
#script:77:9  	+0.053s
test_test: 1 docs successfully imported, 0 docs failed.
#script:89:9  	+0.065s
copy docs from localhost_admin:master:test to localhost_admin:master_test:test_test finished.
#script:100:9  	+0.070s
Index creating will start in 3 seconds...
#script:129:9  	+3.071s
create index model_name_1 for master_test.test_test
#script:130:9  	+3.110s
{
	"createdCollectionAutomatically" : false,
	"numIndexesBefore" : 2,
	"numIndexesAfter" : 2,
	"note" : "all indexes already exist",
	"ok" : 1
}
#script:139:9  	+3.110s
create index complete.
#script:152:9  	+3.111s
1 document(s) of 1 collection(s) have been copied., {
	"test_test" : {
		"nInserted" : 1,
		"nModified" : 0,
		"nSkipped" : 0,
		"failed" : 0
	}
}
#script:157:9  	+3.111s
All documents copied successfully.
db.test_test.getIndexes()

{
	"v" : NumberInt("2"),
	"key" : {
		"_id" : NumberInt("1")
	},
	"name" : "_id_",
	"ns" : "master_test.test_test"
}

photo
1

Seems that indexes are not presented in copied collection when using


-newIndex.ns = `${dstDb}.${dstCollection}`;


+delete newIndex.ns;

photo
2

In your result:


{ "createdCollectionAutomatically" : false,

"numIndexesBefore" : 2,

"numIndexesAfter" : 2,

"note" : "all indexes already exist",

"ok" : 1 }


The "numIndexesAfter" is 2, but db.test_test.getIndexes().length===1? so strange?

photo
1

I made a video, just changing a database to copy to and index creation is failed.https://youtu.be/xG0BMwiBZfM.

photo
2

Thanks for a lot. I can recall the issue locally. I are studying the issue.

photo
2

I got it. That's a db context issue. I will fix the script template in the next update.


In your video, you change "dstDb" to "master_test" by code, but the database context is still be the old "dstDb" "master".


Fixed:


Add a line,


let dstDb = "master_test";


+ use(dstDb)


b752d5f1f3fbfee19bd18fbb60f1b9d7

photo
1

Thank you very much!

photo
1

Hello


Seems I'm getting a different error now, don't know why this happens


#script:8:1  	+0.004s
switched to db (dstDb);
#script:9:1  	+0.004s
Using db: master
#script:28:9  	+0.004s
copy docs from localhost_admin:master_test:cars_test to Working AMAZON root:master:cars_1 start...
#script:80:9  	+7.618s
cars_1: 5000 docs successfully imported, 0 docs failed.
#script:80:9  	+13.740s
cars_1: 10000 docs successfully imported, 0 docs failed.
#script:80:9  	+16.511s
cars_1: 15000 docs successfully imported, 0 docs failed.
#script:80:9  	+21.879s
cars_1: 20000 docs successfully imported, 0 docs failed.
#script:80:9  	+24.284s
cars_1: 25000 docs successfully imported, 0 docs failed.
#script:80:9  	+29.619s
cars_1: 30000 docs successfully imported, 0 docs failed.
#script:80:9  	+32.435s
cars_1: 35000 docs successfully imported, 0 docs failed.
#script:80:9  	+36.878s
cars_1: 40000 docs successfully imported, 0 docs failed.
#script:80:9  	+39.505s
cars_1: 45000 docs successfully imported, 0 docs failed.
#script:80:9  	+44.886s
cars_1: 50000 docs successfully imported, 0 docs failed.
#script:80:9  	+48.489s
cars_1: 55000 docs successfully imported, 0 docs failed.
#script:80:9  	+53.488s
cars_1: 60000 docs successfully imported, 0 docs failed.
#script:80:9  	+58.959s
cars_1: 65000 docs successfully imported, 0 docs failed.
#script:80:9  	+63.412s
cars_1: 70000 docs successfully imported, 0 docs failed.
#script:80:9  	+71.027s
cars_1: 75000 docs successfully imported, 0 docs failed.
#script:80:9  	+77.076s
cars_1: 80000 docs successfully imported, 0 docs failed.
#script:80:9  	+80.343s
cars_1: 85000 docs successfully imported, 0 docs failed.
#script:80:9  	+84.640s
cars_1: 90000 docs successfully imported, 0 docs failed.
#script:80:9  	+88.736s
cars_1: 95000 docs successfully imported, 0 docs failed.
#script:80:9  	+92.624s
cars_1: 100000 docs successfully imported, 0 docs failed.
#script:80:9  	+96.962s
cars_1: 105000 docs successfully imported, 0 docs failed.
#script:80:9  	+100.490s
cars_1: 110000 docs successfully imported, 0 docs failed.
#script:80:9  	+107.939s
cars_1: 115000 docs successfully imported, 0 docs failed.
#script:80:9  	+111.095s
cars_1: 120000 docs successfully imported, 0 docs failed.
#script:80:9  	+118.569s
cars_1: 125000 docs successfully imported, 0 docs failed.
#script:80:9  	+124.684s
cars_1: 130000 docs successfully imported, 0 docs failed.
#script:80:9  	+131.990s
cars_1: 135000 docs successfully imported, 0 docs failed.
#script:80:9  	+134.770s
cars_1: 140000 docs successfully imported, 0 docs failed.
#script:80:9  	+140.386s
cars_1: 145000 docs successfully imported, 0 docs failed.
#script:80:9  	+143.110s
cars_1: 150000 docs successfully imported, 0 docs failed.
#script:80:9  	+150.539s
cars_1: 155000 docs successfully imported, 0 docs failed.
#script:80:9  	+156.584s
cars_1: 160000 docs successfully imported, 0 docs failed.
#script:80:9  	+163.815s
cars_1: 165000 docs successfully imported, 0 docs failed.
#script:80:9  	+167.876s
cars_1: 170000 docs successfully imported, 0 docs failed.
#script:80:9  	+172.678s
cars_1: 175000 docs successfully imported, 0 docs failed.
#script:80:9  	+178.405s
cars_1: 180000 docs successfully imported, 0 docs failed.
#script:80:9  	+181.269s
cars_1: 185000 docs successfully imported, 0 docs failed.
#script:80:9  	+186.761s
cars_1: 190000 docs successfully imported, 0 docs failed.
#script:80:9  	+189.857s
cars_1: 195000 docs successfully imported, 0 docs failed.
#script:80:9  	+194.900s
cars_1: 200000 docs successfully imported, 0 docs failed.
#script:80:9  	+197.915s
cars_1: 205000 docs successfully imported, 0 docs failed.
#script:80:9  	+204.043s
cars_1: 210000 docs successfully imported, 0 docs failed.
#script:80:9  	+210.263s
cars_1: 215000 docs successfully imported, 0 docs failed.
#script:80:9  	+214.313s
cars_1: 220000 docs successfully imported, 0 docs failed.
#script:80:9  	+219.793s
cars_1: 225000 docs successfully imported, 0 docs failed.
#script:80:9  	+224.234s
cars_1: 230000 docs successfully imported, 0 docs failed.
#script:80:9  	+226.867s
cars_1: 235000 docs successfully imported, 0 docs failed.
#script:80:9  	+229.875s
cars_1: 240000 docs successfully imported, 0 docs failed.
#script:80:9  	+234.521s
cars_1: 245000 docs successfully imported, 0 docs failed.
#script:80:9  	+237.408s
cars_1: 250000 docs successfully imported, 0 docs failed.
#script:80:9  	+242.454s
cars_1: 255000 docs successfully imported, 0 docs failed.
#script:80:9  	+244.755s
cars_1: 260000 docs successfully imported, 0 docs failed.
#script:80:9  	+249.787s
cars_1: 265000 docs successfully imported, 0 docs failed.
#script:80:9  	+253.249s
cars_1: 270000 docs successfully imported, 0 docs failed.
#script:80:9  	+258.591s
cars_1: 275000 docs successfully imported, 0 docs failed.
#script:80:9  	+265.647s
cars_1: 280000 docs successfully imported, 0 docs failed.
#script:80:9  	+269.126s
cars_1: 285000 docs successfully imported, 0 docs failed.
#script:80:9  	+274.332s
cars_1: 290000 docs successfully imported, 0 docs failed.
#script:80:9  	+276.932s
cars_1: 295000 docs successfully imported, 0 docs failed.
#script:80:9  	+282.520s
cars_1: 300000 docs successfully imported, 0 docs failed.
#script:80:9  	+286.153s
cars_1: 305000 docs successfully imported, 0 docs failed.
#script:80:9  	+290.102s
cars_1: 310000 docs successfully imported, 0 docs failed.
#script:80:9  	+292.477s
cars_1: 315000 docs successfully imported, 0 docs failed.
#script:80:9  	+296.398s
cars_1: 316577 docs successfully imported, 0 docs failed.
#script:92:9  	+296.411s
copy docs from localhost_admin:master_test:cars_test to Working AMAZON root:master:cars_1 finished.
#script:103:9  	+296.417s
Index creating will start in 3 seconds...
#script:132:9  	+299.417s
remove index make_name_1 for master.cars_1
#script:133:9  	+299.419s
{
	"message" : "ns not found",
	"stack" : "MongoError: ns not found\n    at Function.MongoError.create (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/error.js:31:11)\n    at /Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:497:72\n    at authenticateStragglers (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:443:16)\n    at Connection.messageHandler (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:477:5)\n    at Socket. (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/connection.js:321:22)\n    at emitOne (events.js:96:13)\n    at Socket.emit (events.js:188:7)\n    at readableAddChunk (_stream_readable.js:176:18)\n    at Socket.Readable.push (_stream_readable.js:134:10)\n    at TCP.onread (net.js:551:20)",
	"name" : "MongoError",
	"ok" : 0,
	"errmsg" : "ns not found",
	"code" : 26,
	"codeName" : "NamespaceNotFound"
}
#script:137:9  	+299.419s
create index make_name_1 for master.cars_1
#script:138:9  	+299.419s
{
	"message" : "The value of the field 'ns' (master.cars_1) doesn't match the namespace '(dstDb);.cars_1'",
	"stack" : "MongoError: The value of the field 'ns' (master.cars_1) doesn't match the namespace '(dstDb);.cars_1'\n    at Function.MongoError.create (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/error.js:31:11)\n    at /Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:497:72\n    at authenticateStragglers (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:443:16)\n    at Connection.messageHandler (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:477:5)\n    at Socket. (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/connection.js:321:22)\n    at emitOne (events.js:96:13)\n    at Socket.emit (events.js:188:7)\n    at readableAddChunk (_stream_readable.js:176:18)\n    at Socket.Readable.push (_stream_readable.js:134:10)\n    at TCP.onread (net.js:551:20)",
	"name" : "MongoError",
	"ok" : 0,
	"errmsg" : "The value of the field 'ns' (master.cars_1) doesn't match the namespace '(dstDb);.cars_1'",
	"code" : 2,
	"codeName" : "BadValue"
}
#script:132:9  	+299.419s
remove index car_id_1_site_name_1 for master.cars_1
#script:133:9  	+299.420s
{
	"message" : "ns not found",
	"stack" : "MongoError: ns not found\n    at Function.MongoError.create (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/error.js:31:11)\n    at /Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:497:72\n    at authenticateStragglers (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:443:16)\n    at Connection.messageHandler (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:477:5)\n    at Socket. (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/connection.js:321:22)\n    at emitOne (events.js:96:13)\n    at Socket.emit (events.js:188:7)\n    at readableAddChunk (_stream_readable.js:176:18)\n    at Socket.Readable.push (_stream_readable.js:134:10)\n    at TCP.onread (net.js:551:20)",
	"name" : "MongoError",
	"ok" : 0,
	"errmsg" : "ns not found",
	"code" : 26,
	"codeName" : "NamespaceNotFound"
}
#script:137:9  	+299.420s
create index car_id_1_site_name_1 for master.cars_1
#script:138:9  	+299.420s
{
	"message" : "The value of the field 'ns' (master.cars_1) doesn't match the namespace '(dstDb);.cars_1'",
	"stack" : "MongoError: The value of the field 'ns' (master.cars_1) doesn't match the namespace '(dstDb);.cars_1'\n    at Function.MongoError.create (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/error.js:31:11)\n    at /Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:497:72\n    at authenticateStragglers (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:443:16)\n    at Connection.messageHandler (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:477:5)\n    at Socket. (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/connection.js:321:22)\n    at emitOne (events.js:96:13)\n    at Socket.emit (events.js:188:7)\n    at readableAddChunk (_stream_readable.js:176:18)\n    at Socket.Readable.push (_stream_readable.js:134:10)\n    at TCP.onread (net.js:551:20)",
	"name" : "MongoError",
	"ok" : 0,
	"errmsg" : "The value of the field 'ns' (master.cars_1) doesn't match the namespace '(dstDb);.cars_1'",
	"code" : 2,
	"codeName" : "BadValue"
}
#script:132:9  	+299.420s
remove index updated_at_1_currency_1_damaged_1_tax_paid_1_is_sold_1_mileage_1 for master.cars_1
#script:133:9  	+299.421s
{
	"message" : "ns not found",
	"stack" : "MongoError: ns not found\n    at Function.MongoError.create (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/error.js:31:11)\n    at /Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:497:72\n    at authenticateStragglers (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:443:16)\n    at Connection.messageHandler (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:477:5)\n    at Socket. (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/connection.js:321:22)\n    at emitOne (events.js:96:13)\n    at Socket.emit (events.js:188:7)\n    at readableAddChunk (_stream_readable.js:176:18)\n    at Socket.Readable.push (_stream_readable.js:134:10)\n    at TCP.onread (net.js:551:20)",
	"name" : "MongoError",
	"ok" : 0,
	"errmsg" : "ns not found",
	"code" : 26,
	"codeName" : "NamespaceNotFound"
}
#script:137:9  	+299.421s
create index updated_at_1_currency_1_damaged_1_tax_paid_1_is_sold_1_mileage_1 for master.cars_1
#script:138:9  	+299.421s
{
	"message" : "The value of the field 'ns' (master.cars_1) doesn't match the namespace '(dstDb);.cars_1'",
	"stack" : "MongoError: The value of the field 'ns' (master.cars_1) doesn't match the namespace '(dstDb);.cars_1'\n    at Function.MongoError.create (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/error.js:31:11)\n    at /Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:497:72\n    at authenticateStragglers (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:443:16)\n    at Connection.messageHandler (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:477:5)\n    at Socket. (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/connection.js:321:22)\n    at emitOne (events.js:96:13)\n    at Socket.emit (events.js:188:7)\n    at readableAddChunk (_stream_readable.js:176:18)\n    at Socket.Readable.push (_stream_readable.js:134:10)\n    at TCP.onread (net.js:551:20)",
	"name" : "MongoError",
	"ok" : 0,
	"errmsg" : "The value of the field 'ns' (master.cars_1) doesn't match the namespace '(dstDb);.cars_1'",
	"code" : 2,
	"codeName" : "BadValue"
}
#script:147:9  	+299.421s
create index complete.
#script:160:9  	+299.422s
316577 document(s) of 1 collection(s) have been copied., {
	"cars_1" : {
		"nInserted" : 316577,
		"nModified" : 0,
		"nSkipped" : 0,
		"failed" : 0
	}
}
#script:165:9  	+299.422s
All documents copied successfully.

I use a script you suggested

const BATCH_SIZE = 5000;

let srcConnection = "localhost_admin";
let srcDb = "master_test";
let dstConnection = "Working AMAZON root";
let dstDb = "master";

use (dstDb);
print("Using db: " + dstDb);

//idPolicy: overwrite_with_same_id|always_insert_with_new_id|insert_with_new_id_if_id_exists|skip_documents_with_existing_id|abort_if_id_already_exists
let toCopyCollections = [
    { srcCollection: "cars_test", query: {}, projection: {}, dstCollection: "cars_1", idPolicy: "overwrite_with_same_id", dropDstCollection: true }
];

// if have more indexes than default index (_id), create index after copy data.
let shouldCreateIndex = true;

let totalCopyResult = {
    result: {},
    fails: [],
}

function copyCollection(params) {
    let {srcCollection, dstCollection, query, projection, idPolicy, dropDstCollection} = params;
    let continueRead = true;

    console.log(`copy docs from ${srcConnection}:${srcDb}:${srcCollection} to ${dstConnection}:${dstDb}:${dstCollection} start...`);

    let isSameCollection = srcConnection === dstConnection && srcDb === dstDb && srcCollection === dstCollection;

    if (isSameCollection) {
        if (toCopyCollections.length === 1)
            shouldCreateIndex = false;
        else
            params.shouldCreateIndex = false;
    }

    if (dropDstCollection) {
        // srcCollection is same to dstCollection, can not drop dstCollection (equal to drop srcCollection)
        // drop dst collection and copy from same collection, means nothing to do.
        if (isSameCollection) return;

        mb.dropCollection({ connection: dstConnection, db: dstDb, collection: dstCollection });
    }

    totalCopyResult.result[dstCollection] = {
        nInserted: 0,
        nModified: 0,
        nSkipped: 0,
        failed: 0,
    };
    let collectionRst = totalCopyResult.result[dstCollection];

    let limitReadCount = Number.MAX_SAFE_INTEGER;
    if (isSameCollection)
        limitReadCount = mb.runScript({ connection: srcConnection, db: srcDb, script: `db.getCollection("${srcCollection}").find(${tojson(query)}).count()` })

    let skip = 0;
    while (continueRead) {
        let limit = limitReadCount > BATCH_SIZE ? BATCH_SIZE : limitReadCount;
        let docs = mb.readFromDb({ connection: srcConnection, db: srcDb, collection: srcCollection, query, projection, skip, limit });
        let readLength = docs.length;
        skip += readLength;
        limitReadCount -= readLength;

        if (readLength < BATCH_SIZE)
            continueRead = false;

        if (readLength) {
            let copyResult = mb.writeToDb({ connection: dstConnection, db: dstDb, collection: dstCollection, docs, idPolicy });
            let failed = copyResult.errors.length;
            let success = copyResult.nInserted + copyResult.nModified;

            collectionRst.nInserted += copyResult.nInserted;
            collectionRst.nModified += copyResult.nModified;
            collectionRst.nSkipped += copyResult.nSkipped;
            collectionRst.failed += failed;

            console.log(`${dstCollection}: ${collectionRst.nInserted + collectionRst.nModified} docs successfully imported, ${collectionRst.failed} docs failed.`);

            if (failed) {
                console.log("Failed objects", copyResult.errors);
            }

            totalCopyResult.fails = [...totalCopyResult.fails, ...copyResult.errors];
        }

        sleep(10)
    }

    console.log(`copy docs from ${srcConnection}:${srcDb}:${srcCollection} to ${dstConnection}:${dstDb}:${dstCollection} finished.`);
}

toCopyCollections.forEach(it => copyCollection(it));

if (shouldCreateIndex) {
    let indexCreationPrompted = false;
    function indexCreationPrompt() {
        if (indexCreationPrompted) return;

        let waitTime = 3;
        console.log(`Index creating will start in ${waitTime} seconds...`)
        sleep(1000 * waitTime);
        indexCreationPrompted = true;
    }

    let srcCollections = toCopyCollections.filter(it => it["shouldCreateIndex"] !== false).map(it => it.srcCollection)
    srcCollections.forEach(it => {
        let indexes = mb.runScript({ connection: srcConnection, db: srcDb, script: `db.getCollection("${it}").getIndexes();` });
        if (indexes.length > 1) {
            let toCopyCollection = _.find(toCopyCollections, { srcCollection: it });
            if (!toCopyCollection) return;

            let dstCollection = toCopyCollection.dstCollection;

            indexes.forEach(index => {
                if (index.name === "_id_") return;

                indexCreationPrompt();

                let newIndex = _.cloneDeep(index);

                // remove index version and engine info, these info may fail createIndexes operator.
                delete newIndex.v;
                delete newIndex.textIndexVersion;
                delete newIndex["2dsphereIndexVersion"];
                delete newIndex.storageEngine;

                newIndex.ns = `${dstDb}.${dstCollection}`;

                console.log(`remove index ${newIndex.name} for ${dstDb}.${dstCollection}`);
                console.log(db.runCommand({
                    dropIndexes: dstCollection,
                    index: "*"
                }));
                console.log(`create index ${newIndex.name} for ${dstDb}.${dstCollection}`);
                console.log(db.runCommand({
                    createIndexes: dstCollection,
                    indexes: [newIndex]
                }));
            })
        }
    });

    if (indexCreationPrompted)
        console.log("create index complete.")
}

if (totalCopyResult.result) {
    let successed = 0;
    let failed = 0;
    let collections = _.keys(totalCopyResult.result);
    collections.forEach((key) => {
        let obj = totalCopyResult.result[key];
        successed += obj.nInserted + obj.nModified;
        failed += obj.failed;
    });

    console.log(`${successed} document(s) of ${collections.length} collection(s) have been copied.`, totalCopyResult.result);

    if (failed) {
        console.log(`${failed} document(s) haven't been copied, please check failed list below.`);
    } else {
        console.log("All documents copied successfully.");
    }
}

if (totalCopyResult.fails.length) {
    console.log("All failed objects", totalCopyResult.fails);
}

Could you please help me?

photo
1

Ok, found, I need to use just

use master


NOT:

use (master);

user ("master");

photo
1

use("master");


or


use master

photo
Leave a Comment
 
Attach a file