can't copy collections

Ethan Siegel shared this problem 2 years ago
Solved

When I try and copy a collection, I get an error message. This was working in the previous version

const BATCH_SIZE = 2000;

let srcConnection = "matchminer";
let srcDb = "matchengine_test";
let dstConnection = "matchminer";
let dstDb = "dev";

use(dstDb);

//idPolicy: overwrite_with_same_id|always_insert_with_new_id|insert_with_new_id_if_id_exists|skip_documents_with_existing_id|abort_if_id_already_exists
let toCopyCollections = [
    { srcCollection: "trial", query: {}, projection: {}, dstCollection: "trial", idPolicy: "overwrite_with_same_id", dropDstCollection: false }
];

// if have more indexes than default index (_id), create index after copy data.
let shouldCreateIndex = true;

let totalCopyResult = {
    result: {},
    fails: [],
}

function copyCollection(params) {
    let { srcCollection, dstCollection, query, projection, idPolicy, dropDstCollection } = params;
    let continueRead = true;

    console.log(`copy docs from ${srcConnection}:${srcDb}:${srcCollection} to ${dstConnection}:${dstDb}:${dstCollection} start...`);

    let isSameCollection = srcConnection === dstConnection && srcDb === dstDb && srcCollection === dstCollection;

    if (isSameCollection) {
        if (toCopyCollections.length === 1)
            shouldCreateIndex = false;
        else
            params.shouldCreateIndex = false;
    }

    if (dropDstCollection) {
        // srcCollection is same to dstCollection, can not drop dstCollection (equal to drop srcCollection)
        // drop dst collection and copy from same collection, means nothing to do.
        if (isSameCollection) return;

        mb.dropCollection({ connection: dstConnection, db: dstDb, collection: dstCollection });
    }

    totalCopyResult.result[dstCollection] = {
        nInserted: 0,
        nModified: 0,
        nSkipped: 0,
        failed: 0,
    };
    let collectionRst = totalCopyResult.result[dstCollection];

    let limitReadCount = Number.MAX_SAFE_INTEGER;
    if (isSameCollection)
        limitReadCount = mb.runScript({ connection: srcConnection, db: srcDb, script: `db.getCollection("${srcCollection}").find(${tojson(query)}).count()` })

    let skip = 0;
    while (continueRead) {
        let limit = limitReadCount > BATCH_SIZE ? BATCH_SIZE : limitReadCount;
        let docs = mb.readFromDb({ connection: srcConnection, db: srcDb, collection: srcCollection, query, projection, skip, limit });
        let readLength = docs.length;
        skip += readLength;
        limitReadCount -= readLength;

        if (readLength < BATCH_SIZE)
            continueRead = false;

        if (readLength) {
            let copyResult = mb.writeToDb({ connection: dstConnection, db: dstDb, collection: dstCollection, docs, idPolicy });
            let failed = copyResult.errors.length;
            let success = copyResult.nInserted + copyResult.nModified;

            collectionRst.nInserted += copyResult.nInserted;
            collectionRst.nModified += copyResult.nModified;
            collectionRst.nSkipped += copyResult.nSkipped;
            collectionRst.failed += failed;

            console.log(`${dstCollection}: ${collectionRst.nInserted + collectionRst.nModified} docs successfully imported, ${collectionRst.failed} docs failed.`);

            if (failed) {
                console.log("Failed objects", copyResult.errors);
            }

            totalCopyResult.fails = [...totalCopyResult.fails, ...copyResult.errors];
        }

        sleep(10)
    }

    console.log(`copy docs from ${srcConnection}:${srcDb}:${srcCollection} to ${dstConnection}:${dstDb}:${dstCollection} finished.`);
}

toCopyCollections.forEach(it => copyCollection(it));

if (shouldCreateIndex) {
    let indexCreationPrompted = false;
    function indexCreationPrompt() {
        if (indexCreationPrompted) return;

        let waitTime = 3;
        console.log(`Index creating will start in ${waitTime} seconds...`)
        sleep(1000 * waitTime);
        indexCreationPrompted = true;
    }

    let srcCollections = toCopyCollections.filter(it => it["shouldCreateIndex"] !== false).map(it => it.srcCollection)
    srcCollections.forEach(it => {
        let indexes = mb.runScript({ connection: srcConnection, db: srcDb, script: `db.getCollection("${it}").getIndexes();` });
        if (indexes.length > 1) {
            let toCopyCollection = _.find(toCopyCollections, { srcCollection: it });
            if (!toCopyCollection) return;

            let dstCollection = toCopyCollection.dstCollection;

            indexes.forEach(index => {
                if (index.name === "_id_") return;

                indexCreationPrompt();

                let newIndex = _.cloneDeep(index);

                // remove index version and engine info, these info may fail createIndexes operator.
                delete newIndex.v;
                delete newIndex.textIndexVersion;
                delete newIndex["2dsphereIndexVersion"];
                delete newIndex.storageEngine;

                newIndex.ns = `${dstDb}.${dstCollection}`;

                console.log(`create index ${newIndex.name} for ${dstDb}.${dstCollection}`);
                console.log(db.runCommand({
                    createIndexes: dstCollection,
                    indexes: [newIndex]
                }));
            })
        }
    });

    if (indexCreationPrompted)
        console.log("create index complete.")
}

if (totalCopyResult.result) {
    let successed = 0;
    let failed = 0;
    let collections = _.keys(totalCopyResult.result);
    collections.forEach((key) => {
        let obj = totalCopyResult.result[key];
        successed += obj.nInserted + obj.nModified;
        failed += obj.failed;
    });

    console.log(`${successed} document(s) of ${collections.length} collection(s) have been copied.`, totalCopyResult.result);

    if (failed) {
        console.log(`${failed} document(s) haven't been copied, please check failed list below.`);
    } else {
        console.log("All documents copied successfully.");
    }
}

if (totalCopyResult.fails.length) {
    console.log("All failed objects", totalCopyResult.fails);
}



Error
{
	"message" : "Cannot read property 'length' of undefined",
	"stack" : "script:71:42" +
              "script:94:33" +
              "script:94:19"
}

Replies (2)

photo
2

Thank you for your feedback.

We have worked out a hotfix to resolve this issue, please download and give it a try.

Download page: https://nosqlbooster.com/downloads

photo
1

Resolved in 5.1.12

Leave a Comment
 
Attach a file