Bug in copying collection
Cannot Reproduce
I have 2 DB and 2 collections:
master cars
master_test cars_test
Auto generated script looks like this
const BATCH_SIZE = 10000;
let srcConnection = "localhost_admin";
let srcDb = "master";
let dstConnection = "localhost_admin";
let dstDb = "master_test";
//idPolicy: overwrite_with_same_id|always_insert_with_new_id|insert_with_new_id_if_id_exists|skip_documents_with_existing_id|abort_if_id_already_exists
let toCopyCollections = [
{ srcCollection: "cars", query: {}, projection: {}, dstCollection: "cars_test", idPolicy: "overwrite_with_same_id", dropDstCollection: true }
];
// if have more indexes than default index (_id), create index after copy data.
let shouldCreateIndex = true;
let totalCopyResult = {
result: {},
fails: [],
}
function copyCollection(params) {
return
let {srcCollection, dstCollection, query, projection, idPolicy, dropDstCollection} = params;
let continueRead = true;
console.log(`copy docs from ${srcConnection}:${srcDb}:${srcCollection} to ${dstConnection}:${dstDb}:${dstCollection} start...`);
let isSameCollection = srcConnection === dstConnection && srcDb === dstDb && srcCollection === dstCollection;
if (isSameCollection) {
if (toCopyCollections.length === 1)
shouldCreateIndex = false;
else
params.shouldCreateIndex = false;
}
if (dropDstCollection) {
// srcCollection is same to dstCollection, can not drop dstCollection (equal to drop srcCollection)
// drop dst collection and copy from same collection, means nothing to do.
if (isSameCollection) return;
mb.dropCollection({ connection: dstConnection, db: dstDb, collection: dstCollection });
}
totalCopyResult.result[dstCollection] = {
nInserted: 0,
nModified: 0,
nSkipped: 0,
failed: 0,
};
let collectionRst = totalCopyResult.result[dstCollection];
let limitReadCount = Number.MAX_SAFE_INTEGER;
if (isSameCollection)
limitReadCount = mb.runScript({ connection: srcConnection, db: srcDb, script: `db.getCollection("${srcCollection}").find(${tojson(query)}).count()` })
let skip = 0;
while (continueRead) {
let limit = limitReadCount > BATCH_SIZE ? BATCH_SIZE : limitReadCount;
let docs = mb.readFromDb({ connection: srcConnection, db: srcDb, collection: srcCollection, query, projection, skip, limit });
let readLength = docs.length;
skip += readLength;
limitReadCount -= readLength;
if (readLength < BATCH_SIZE)
continueRead = false;
if (readLength) {
let copyResult = mb.writeToDb({ connection: dstConnection, db: dstDb, collection: dstCollection, docs, idPolicy });
let failed = copyResult.errors.length;
let success = copyResult.nInserted + copyResult.nModified;
collectionRst.nInserted += copyResult.nInserted;
collectionRst.nModified += copyResult.nModified;
collectionRst.nSkipped += copyResult.nSkipped;
collectionRst.failed += failed;
console.log(`${dstCollection}: ${collectionRst.nInserted + collectionRst.nModified} docs successfully imported, ${collectionRst.failed} docs failed.`);
if (failed) {
console.log("Failed objects", copyResult.errors);
}
totalCopyResult.fails = [...totalCopyResult.fails, ...copyResult.errors];
}
sleep(10)
}
console.log(`copy docs from ${srcConnection}:${srcDb}:${srcCollection} to ${dstConnection}:${dstDb}:${dstCollection} finished.`);
}
toCopyCollections.forEach(it => copyCollection(it));
if (shouldCreateIndex) {
let indexCreationPrompted = false;
function indexCreationPrompt() {
if (indexCreationPrompted) return;
let waitTime = 3;
console.log(`Index creating will start in ${waitTime} seconds...`)
sleep(1000 * waitTime);
indexCreationPrompted = true;
}
let srcCollections = toCopyCollections.filter(it => it["shouldCreateIndex"] !== false).map(it => it.srcCollection)
srcCollections.forEach(it => {
let indexes = mb.runScript({ connection: srcConnection, db: srcDb, script: `db.getCollection("${it}").getIndexes();` });
if (indexes.length > 1) {
let toCopyCollection = _.find(toCopyCollections, { srcCollection: it });
if (!toCopyCollection) return;
let dstCollection = toCopyCollection.dstCollection;
indexes.forEach(index => {
if (index.name === "_id_") return;
indexCreationPrompt();
let newIndex = _.cloneDeep(index);
// remove index version and engine info, these info may fail createIndexes operator.
delete newIndex.v;
delete newIndex.textIndexVersion;
delete newIndex["2dsphereIndexVersion"];
delete newIndex.storageEngine;
newIndex.ns = `${dstDb}.${dstCollection}`;
console.log(`create index ${newIndex.name} for ${dstDb}.${dstCollection}`);
console.log(db.runCommand({
createIndexes: dstCollection,
indexes: [newIndex]
}));
})
}
});
if (indexCreationPrompted)
console.log("create index complete.")
}
if (totalCopyResult.result) {
let successed = 0;
let failed = 0;
let collections = _.keys(totalCopyResult.result);
collections.forEach((key) => {
let obj = totalCopyResult.result[key];
successed += obj.nInserted + obj.nModified;
failed += obj.failed;
});
console.log(`${successed} document(s) of ${collections.length} collection(s) have been copied.`, totalCopyResult.result);
if (failed) {
console.log(`${failed} document(s) haven't been copied, please check failed list below.`);
} else {
console.log("All documents copied successfully.");
}
}
if (totalCopyResult.fails.length) {
console.log("All failed objects", totalCopyResult.fails);
}
However, it produces errors
#script:101:9 +0.010s
Index creating will start in 3 seconds...
#script:136:9 +3.012s
create index make_name_1 for master_test.cars_test
#script:137:9 +3.145s
{
"message" : "The value of the field 'ns' (master_test.cars_test) doesn't match the namespace 'master.cars_test'",
"stack" : "MongoError: The value of the field 'ns' (master_test.cars_test) doesn't match the namespace 'master.cars_test'\n at Function.MongoError.create (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/error.js:31:11)\n at /Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:497:72\n at authenticateStragglers (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:443:16)\n at Connection.messageHandler (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/pool.js:477:5)\n at Socket.<anonymous> (/Applications/mongobooster.app/Contents/Resources/app.asar/node_modules/mongodb-core/lib/connection/connection.js:321:22)\n at emitOne (events.js:96:13)\n at Socket.emit (events.js:188:7)\n at readableAddChunk (_stream_readable.js:176:18)\n at Socket.Readable.push (_stream_readable.js:134:10)\n at TCP.onread (net.js:551:20)",
"name" : "MongoError",
"ok" : 0,
"errmsg" : "The value of the field 'ns' (master_test.cars_test) doesn't match the namespace 'master.cars_test'",
"code" : 2,
"codeName" : "BadValue"
} I fixed it by:
-newIndex.ns = `${dstDb}.${dstCollection}`;
+newIndex.ns = `${srcDb}.${dstCollection}`;
The same problem 
I can't recall the issue. If possible, could you please give me the detailed Repro Steps?
I review the code, the newIndex namespace should be `${dstDb}.${dstCollection}`.
I can't recall the issue. If possible, could you please give me the detailed Repro Steps?
I review the code, the newIndex namespace should be `${dstDb}.${dstCollection}`.
Replies have been locked on this page!