Export to JSON, BSON, CSV fails

Daniel Givoni shared this problem 2 months ago
Solved

seems as a problem with cursor instance - "cursor.size is not a function"

error:

{
	"message" : "cursor.size is not a function",
	"stack" : "script:102:25"
}

auto generated script:

import * as path from "path";
import * as fs from "fs";
const promisify = require("bluebird").promisify;
const appendFileAsync = promisify(fs.appendFile);

const BATCH_SIZE = 2000;

let filepath = "/Users/danielgivoni/Downloads/farmer_production.users.csv";
let targetPath = path.dirname(filepath);

if (!fs.existsSync(targetPath))
    require("mkdirp").sync(targetPath);

if (fs.existsSync(filepath))
    fs.unlinkSync(filepath);

const tab = "campain_email_farmers_query.js";

const queryScript = `db.getCollection("users").aggregate(
 [
	{
		"$lookup" : {
			"from" : "farmers",
			"localField" : "FarmerIds",
			"foreignField" : "uniqueId",
			"as" : "farms"
		}
	},
	{
		"$match" : {
			"$and" : [
				{
					"farms" : {
						"$ne" : [
							{
								
							}
						]
					}
				},
				{
					"farms.name" : {
						"$exists" : true
					}
				},
				{
					"farms.stores.emails" : {
						"$ne" : [ ]
					}
				},
				{
					"farms.valid" : {
						"$ne" : false
					}
				}
			]
		}
	},
	{
		"$unwind" : "$identityVerifer"
	},
	{
		"$unwind" : "$farms"
	},
	{
		"$unwind" : "$farms.stores"
	},
	{
		"$unwind" : "$farms.stores.emails"
	},
	{
		"$project" : {
			"telephone" : "$identityVerifer",
			"locale" : 1,
			"farm_name" : "$farms.name",
			"email" : "$farms.stores.emails",
			"_id" : 0,
			"currency" : "$farms.stores.currencyCode"
		}
	}
],
 {
	"$clusterTime" : {
		"clusterTime" : Timestamp(1569397676, 1),
		"signature" : {
			"hash" : BinData(0,"5fFgdo90tggglO5WlKOPjOQ4Ey8="),
			"keyId" : NumberLong("6726444850310283265")
		}
	}
})`;
let skip = 0;
let limit = Number.MAX_SAFE_INTEGER;

let connection = "d<..........>-a0.mlab.com:3<........>";
let db = "farmer_production";

let fields = [];

let isFirstRead = true;

const cursor = mb.getCursorFromQueryScript({ connection, db, queryScript, skip, limit });
const totalCount = cursor.size();
let processedCount = 0;

await(mb.batchReadFromCursor(cursor, BATCH_SIZE, (docs) => {
    return async(() => {
        let readLength = docs.length;
        if (!readLength) {
            return;
        }

        if (_.isEmpty(fields)) {
            fields = mb.tryGetFields(_.sampleSize(docs, 5));
        }

        let csvContent = mb.docsToCSV({ docs, fields, delimiter: ",", withColumnTitle: isFirstRead });
        isFirstRead = false;
        await(appendFileAsync(filepath, csvContent));

        processedCount += readLength;
        const percent = (processedCount / totalCount * 100).toFixed(1);
        console.log(`${percent}%	 ${processedCount}/${totalCount} docs exported to "${path.basename(filepath)}".`);

        sleep(10)
    })();
}));

_.delay(() => mb.openFolder(targetPath), 1000);

sleep(100);
console.log(`\nexport ${processedCount} docs from ${tab} to ${filepath} finished.`);

Comments (6)

photo
1

other exports fail as well:

#script:62:9 +0.953s


export 0 docs from farmer_production:farmers@<........>b.com:<....>6 to /Users/danielgivoni/Downloads/farmer_production.farmers111.json finished.

photo
1

on version 5.1.2 works fine

photo
1

Thank you for your feedback. We have worked out a test build to resolve this issue. Please download and give it a try.

MacOS: https://s3.mongobooster.com/download/releasesv5/nosqlbooster4mongo-5.2.2-beta.1.dmg

photo
1

Could you give me a Linux version, please?

photo
1

i installed the beta version you add a link to:

the cursor error is not showing but the result of export is 0 files:

#script:131:9 +2.015s


export 0 docs from campain_email_farmers_query.js to /Users/danielgivoni/Downloads/farmer_production.users.json finished.

photo
1

Please give me some sample data, I need to recall this issue locally. You can send it to support@nosqlbooster.com.

photo
1

Hi, any update here.

photo
1

Still unable to export full csv report

photo
1

If you're reporting a /1f41e bug, please make sure you include steps to reproduce it. You can send your sample data to support@nosqlbooster.com

  • NoSQLBooster Version
  • Operating System (Windows/Mac/Ubuntu):
  • Expected Behavior
  • Actual Behavior
  • How to Reproduce (include your sample date)
  • Screenshots

photo
1

Issue on version 5.2.1 including this --> https://s3.mongobooster.com/download/releasesv5/nosqlbooster4mongo-5.2.2-beta.1.dmg


447a8da1434390c5c915919af2cdcc5fcb506aab94bbaa99988147a2503ed0c8616152bdd24fdbf17566e42f08777136589e3a89777997c48d2915d4e79ed861f7478a510f3018c1fd0da52315f610ca


The code for export is below

____________________________

import * as path from "path";
import * as fs from "fs";
const promisify = require("bluebird").promisify;
const appendFileAsync = promisify(fs.appendFile);

const BATCH_SIZE = 2000;

let filepath = "/Users/appleuser/Documents/CPLShowRoomBalanceDeducts.csv";
let targetPath = path.dirname(filepath);

if (!fs.existsSync(targetPath))
    require("mkdirp").sync(targetPath);

if (fs.existsSync(filepath))
    fs.unlinkSync(filepath);

const tab = "jive:balancededucts@ds139000-a0.kww98.fleet.mlab.com:39000 (1)";

const queryScript = `db.balancededucts.find({  "agentId" : ObjectId("5be27f8b21ab35652996babb") })
                 .sort({  "_id" : -1 })
                 .limit(200000)`;
let skip = 0;
let limit = 200000;

let connection = "ds139000-a0.kww98.fleet.mlab.com:39000";
let db = "jive";

let fields = [];

let isFirstRead = true;

const cursor = mb.getCursorFromQueryScript({ connection, db, queryScript, skip, limit });
const totalCount = cursor.size ? cursor.size() : cursor.count();
let processedCount = 0;

await(mb.batchReadFromCursor(cursor, BATCH_SIZE, (docs) => {
    return async(() => {
        let readLength = docs.length;
        if (!readLength) {
            return;
        }

        if (_.isEmpty(fields)) {
            fields = mb.tryGetFields(_.sampleSize(docs, 5));
        }

        let csvContent = mb.docsToCSV({ docs, fields, delimiter: ",", withColumnTitle: isFirstRead });
        isFirstRead = false;
        await(appendFileAsync(filepath, csvContent));

        processedCount += readLength;
        const percent = (processedCount / totalCount * 100).toFixed(1);
        console.log(`${percent}%	 ${processedCount}/${totalCount} docs exported to "${path.basename(filepath)}".`);

        sleep(10)
    })();
}));


_.delay(() => mb.openFolder(targetPath), 1000);

sleep(100);
console.log(`\nexport ${processedCount} docs from ${tab} to ${filepath} finished.`);

photo
1

Please replace the `db.balancededucts.find(` with `db.getSiblingDB("jive").balancededucts.find(` and try it again. If it does not work, please send your sample JSON data and detailed repro steps to support@nosqlbooster.com

photo
1

Please can your team address this issue. I rely heavily on your software to generate reports daily.

photo
1

db.balancededucts.find(` with `db.getSiblingDB("jive").balancededucts.find( - doesn't work. It is saying balance deducts property can't be found

This problem (unable to export CSV) only occurs when we choose export option, AND export Current query result. If we export selected results, it exports sucessfully. But not for export Current query result - we get 0 export

photo
1

So, please change back the original code and execute it many times, Is the result of each execution the same? export 0 docs?

photo
1

Please fix this issue or provide link to previous version as this problem only happened after the update

photo
1

I noticed that you all use mlab's MongoDB service. I tried that mlab, does have this problem, but other services such as atlas does not have this problem. I have been able to reproduce the problem and will solve it soon.

You can download 5.1.14 from the following URL:

MacOS: https://nosqlbooster.com/s3/download/releasesv5/nosqlbooster4mongo-5.1.14.dmg

photo
photo
1

I am experiencing the same issue on 5.2.1

photo
1

I am experiencing the same issue on 5.2.1

photo
2

Resolved in 5.2.3