can't copy collections
Solved
When I try and copy a collection, I get an error message. This was working in the previous version
const BATCH_SIZE = 2000;
let srcConnection = "matchminer";
let srcDb = "matchengine_test";
let dstConnection = "matchminer";
let dstDb = "dev";
use(dstDb);
//idPolicy: overwrite_with_same_id|always_insert_with_new_id|insert_with_new_id_if_id_exists|skip_documents_with_existing_id|abort_if_id_already_exists
let toCopyCollections = [
{ srcCollection: "trial", query: {}, projection: {}, dstCollection: "trial", idPolicy: "overwrite_with_same_id", dropDstCollection: false }
];
// if have more indexes than default index (_id), create index after copy data.
let shouldCreateIndex = true;
let totalCopyResult = {
result: {},
fails: [],
}
function copyCollection(params) {
let { srcCollection, dstCollection, query, projection, idPolicy, dropDstCollection } = params;
let continueRead = true;
console.log(`copy docs from ${srcConnection}:${srcDb}:${srcCollection} to ${dstConnection}:${dstDb}:${dstCollection} start...`);
let isSameCollection = srcConnection === dstConnection && srcDb === dstDb && srcCollection === dstCollection;
if (isSameCollection) {
if (toCopyCollections.length === 1)
shouldCreateIndex = false;
else
params.shouldCreateIndex = false;
}
if (dropDstCollection) {
// srcCollection is same to dstCollection, can not drop dstCollection (equal to drop srcCollection)
// drop dst collection and copy from same collection, means nothing to do.
if (isSameCollection) return;
mb.dropCollection({ connection: dstConnection, db: dstDb, collection: dstCollection });
}
totalCopyResult.result[dstCollection] = {
nInserted: 0,
nModified: 0,
nSkipped: 0,
failed: 0,
};
let collectionRst = totalCopyResult.result[dstCollection];
let limitReadCount = Number.MAX_SAFE_INTEGER;
if (isSameCollection)
limitReadCount = mb.runScript({ connection: srcConnection, db: srcDb, script: `db.getCollection("${srcCollection}").find(${tojson(query)}).count()` })
let skip = 0;
while (continueRead) {
let limit = limitReadCount > BATCH_SIZE ? BATCH_SIZE : limitReadCount;
let docs = mb.readFromDb({ connection: srcConnection, db: srcDb, collection: srcCollection, query, projection, skip, limit });
let readLength = docs.length;
skip += readLength;
limitReadCount -= readLength;
if (readLength < BATCH_SIZE)
continueRead = false;
if (readLength) {
let copyResult = mb.writeToDb({ connection: dstConnection, db: dstDb, collection: dstCollection, docs, idPolicy });
let failed = copyResult.errors.length;
let success = copyResult.nInserted + copyResult.nModified;
collectionRst.nInserted += copyResult.nInserted;
collectionRst.nModified += copyResult.nModified;
collectionRst.nSkipped += copyResult.nSkipped;
collectionRst.failed += failed;
console.log(`${dstCollection}: ${collectionRst.nInserted + collectionRst.nModified} docs successfully imported, ${collectionRst.failed} docs failed.`);
if (failed) {
console.log("Failed objects", copyResult.errors);
}
totalCopyResult.fails = [...totalCopyResult.fails, ...copyResult.errors];
}
sleep(10)
}
console.log(`copy docs from ${srcConnection}:${srcDb}:${srcCollection} to ${dstConnection}:${dstDb}:${dstCollection} finished.`);
}
toCopyCollections.forEach(it => copyCollection(it));
if (shouldCreateIndex) {
let indexCreationPrompted = false;
function indexCreationPrompt() {
if (indexCreationPrompted) return;
let waitTime = 3;
console.log(`Index creating will start in ${waitTime} seconds...`)
sleep(1000 * waitTime);
indexCreationPrompted = true;
}
let srcCollections = toCopyCollections.filter(it => it["shouldCreateIndex"] !== false).map(it => it.srcCollection)
srcCollections.forEach(it => {
let indexes = mb.runScript({ connection: srcConnection, db: srcDb, script: `db.getCollection("${it}").getIndexes();` });
if (indexes.length > 1) {
let toCopyCollection = _.find(toCopyCollections, { srcCollection: it });
if (!toCopyCollection) return;
let dstCollection = toCopyCollection.dstCollection;
indexes.forEach(index => {
if (index.name === "_id_") return;
indexCreationPrompt();
let newIndex = _.cloneDeep(index);
// remove index version and engine info, these info may fail createIndexes operator.
delete newIndex.v;
delete newIndex.textIndexVersion;
delete newIndex["2dsphereIndexVersion"];
delete newIndex.storageEngine;
newIndex.ns = `${dstDb}.${dstCollection}`;
console.log(`create index ${newIndex.name} for ${dstDb}.${dstCollection}`);
console.log(db.runCommand({
createIndexes: dstCollection,
indexes: [newIndex]
}));
})
}
});
if (indexCreationPrompted)
console.log("create index complete.")
}
if (totalCopyResult.result) {
let successed = 0;
let failed = 0;
let collections = _.keys(totalCopyResult.result);
collections.forEach((key) => {
let obj = totalCopyResult.result[key];
successed += obj.nInserted + obj.nModified;
failed += obj.failed;
});
console.log(`${successed} document(s) of ${collections.length} collection(s) have been copied.`, totalCopyResult.result);
if (failed) {
console.log(`${failed} document(s) haven't been copied, please check failed list below.`);
} else {
console.log("All documents copied successfully.");
}
}
if (totalCopyResult.fails.length) {
console.log("All failed objects", totalCopyResult.fails);
}
Error
{
"message" : "Cannot read property 'length' of undefined",
"stack" : "script:71:42" +
"script:94:33" +
"script:94:19"
}
The same problem
Thank you for your feedback.
We have worked out a hotfix to resolve this issue, please download and give it a try.
Download page: https://nosqlbooster.com/downloads
Thank you for your feedback.
We have worked out a hotfix to resolve this issue, please download and give it a try.
Download page: https://nosqlbooster.com/downloads
Resolved in 5.1.12
Resolved in 5.1.12
Replies have been locked on this page!