<commit_before>/*

This file is part of databank-couchdb, a CouchDB driver for databank.
Copyright 2013 Kuno Woudt <kuno@frob.nl>

Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License.  You may
obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.  See the License for the specific language governing
permissions and limitations under the License.

*/

var _ = require('underscore');
var step = require('step');
var couchdb = require('couchdb-api');
var databank = require('databank');

var CouchDBDatabank = function(params) {
    this.schema    = params.schema || {};
    this.location  = params.location || "http://localhost:5984";
    this.database  = params.database || null;
    this.clear_database = params._clear_database_for_test_run || false;
    this.connection = null;
};

/* cannot use "/" as separator:
   https://github.com/dominicbarnes/node-couchdb-api/pull/12  */
var SEPARATOR = ':';
var document_key = function (type, id) { return type + SEPARATOR + id; };

/* When running tests, clear the database first, but only do so once. */
var TEST_DATABASE_CLEARED = false;

CouchDBDatabank.prototype = new databank.Databank();

CouchDBDatabank.prototype._error_messages = {
    "clear_test_database": function (location, dbname) {
        return "CouchDB cannot create the test database " + dbname + " at " + location;
    },
    "connect": function (location, dbname) {
        return "CouchDB cannot connect to database " + dbname + " at " + location;
    },
    "delete": function (type, id) {
        return "CouchDB error deleting document " + document_key (type, id);
    },
    "read": function (type, id) {
        return "CouchDB error while reading document " + document_key (type, id);
    },
    "read_document": function (document_id) {
        return "CouchDB error while reading document " + document_id;
    },
    "readAll": function (type, ids) {
        return "CouchDB error while reading all " + type + "documents";
    },
    "save_document": function (document_id) {
        return "CouchDB error while saving document " + document_id;
    },
    "search": function (type, criteria) {
        return "CouchDB error while searching for a " + type;
    }
};

CouchDBDatabank.prototype._error = function (err, verb) {
    if (err instanceof databank.DatabankError)
        return err;

    var self = this;
    var context = Array.prototype.slice.call (arguments, 2);
    var msg = self._error_messages[verb].apply (null, context) +
        ": " + JSON.stringify (err);

    return new databank.DatabankError (msg);
};

CouchDBDatabank.prototype._connected = function(onCompletion) {
    if (!this.connection) {
        onCompletion(new databank.NotConnectedError(), null);
        return false;
    }

    return true;
};

CouchDBDatabank.prototype._new_document = function (type, id) {
    return this.connection.doc (document_key (type, id));
};

CouchDBDatabank.prototype._read_document = function(type, id, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    var document = self._new_document (type, id);
    document.get (function (err, response) {
        if (err && err.error === 'not_found')
        {
            onCompletion (new databank.NoSuchThingError(type, id), null);
        }
        else if (err)
        {
            onCompletion (self._error (err, "read_document", document));
        }
        else
        {
            onCompletion (null, document);
        }
    });
};

CouchDBDatabank.prototype._save_document = function (document, value, onCompletion) {
    var self = this;

    document.body.data = value;
    document.save (function (err, response) {
        if (err && err.error === 'conflict')
        {
            onCompletion(new databank.AlreadyExistsError ());
        }
        else if (err)
        {
            onCompletion(self._error (err, "save_document", document.id))
        }
        else
        {
            onCompletion(null, value);
        }
    });
};

CouchDBDatabank.prototype._clear_test_database = function (params, onCompletion) {
    var self = this;
    if (self.clear_database && !TEST_DATABASE_CLEARED)
    {
        step(
            function () {
                self.connection.drop (this);
            },
            function (err) {
                TEST_DATABASE_CLEARED = true;
                err ? this(err) : self.connection.create (this);
            },
            function () {
                onCompletion ();
            }
        );
    }
    else
    {
        onCompletion ();
    }
};

CouchDBDatabank.prototype.connect = function(params, onCompletion) {
    var self = this;

    if (self.connection) {
        onCompletion(new databank.AlreadyConnectedError(), null);
        return;
    }

    self.connection = couchdb.srv(self.location).db(self.database);

    var verify_connection = function () {
        self.connection.info (function (err, response) {
            err ? onCompletion(self._error (err, "connect", self.location, self.database))
                : onCompletion(null);
        });
    };

    self._clear_test_database (params, function (err) {
        err ? onCompletion(self._error (err, "clear_test_database", self.location, self.database))
            : verify_connection();
    });
};

CouchDBDatabank.prototype.disconnect = function(onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    self.connection = null;
    onCompletion (null);
};

CouchDBDatabank.prototype.create = function(type, id, value, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    // FIXME: onCompletion(new AlreadyExistsError(type, id), null);
    var document = self._new_document (type, id);
    self._save_document (document, value, onCompletion);
};

CouchDBDatabank.prototype.read = function(type, id, onCompletion) {
    var self = this;
    self._read_document (type, id, function (err, document) {
        err ? onCompletion (self._error (err, "read", type, id))
            : onCompletion (null, document.body.data);
    });
};

CouchDBDatabank.prototype.update = function(type, id, value, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    self._read_document (type, id, function (err, document) {
        err ? onCompletion (err)
            : self._save_document (document, value, onCompletion);
    });
};

CouchDBDatabank.prototype.save = function(type, id, value, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    // FIXME: verify that err is a NoSuchThingError.
    self._read_document (type, id, function (err, document) {
        err ? self.create (type, id, value, onCompletion)
            : self.update (type, id, value, onCompletion);
    });
};

CouchDBDatabank.prototype.del = function(type, id, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    self._read_document (type, id, function (err, document) {
        if (!document) {
            onCompletion (err)
        }
        else
        {
            document.del (function (err, response) {
                err ? onCompletion (self._error (err, "delete", type, id))
                    : onCompletion(null);
            });
        }
    });
};

CouchDBDatabank.prototype.search = function(type, criteria, onResult, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    var keys = _(criteria).keys ();

    var map = [
        "function (doc) {",
        "    var isArray = function(o) { return (o instanceof Array || typeof o == \"array\"); };",
        "    var getDottedProperty = null;",
        "    getDottedProperty = function (doc, name) {",
        "        if (!isArray (name)) name = name.split (\".\");",
        "        var key = name.shift ();",
        "        if (!key) return doc;",
        "        if (!doc.hasOwnProperty (key)) return undefined;",
        "        return getDottedProperty (doc[key], name);",
        "    };",
        "    var type = " + JSON.stringify (type + SEPARATOR) + ";",
        "    if (doc._id.substr (0, type.length) !== type) return;",
        "    var criteria = " + JSON.stringify (keys) + ";",
        "    var results = [];",
        "    for (var i in criteria) {",
        "        var value = getDottedProperty(doc.data, criteria[i]);",
        "        if (value === undefined) return;",
        "        results.push (value);",
        "    }",
        "    emit (results, doc);",
        "}"
    ];

    // https://github.com/dominicbarnes/node-couchdb-api/issues/17
    var query_key = _(keys).map (function (key) { return criteria[key] });
    var query = { "qs": { "key": JSON.stringify (query_key) } };

    // FIXME: tempViews are slow.  automatically create permanent view
    // after N requests for the same view?

    self.connection.tempView (map.join ("\n"), null, null, query, function (err, response) {
        if (err)
        {
            onCompletion(self._error (err, "search", type, criteria));
        }

        response.rows.forEach (function (doc) { onResult(doc.value.data); });

        onCompletion(null);
    });
};

CouchDBDatabank.prototype.readAll = function(type, ids, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    var id_map = {};
    ids.forEach (function (id) {
        var docid = document_key (type, id);
        id_map[docid] = id;
    });

    var options = { include_docs: true };
    var docids = _(id_map).keys ();

    self.connection.allDocs (options, docids, function (err, response) {
        if (err) {
            onCompletion (self._error (err, "readAll", type, ids));
        }
        else
        {
            var results = {};
            response.rows.forEach (function (row) {
                results[ id_map[row.key] ] = row.doc ? row.doc.data : null;
            });
            onCompletion(null, results);
        }
    });
};

module.exports = CouchDBDatabank;
<commit_msg>couchdb-api github issues #12 and #17 were fixed in couchdb-api 1.2.2, update our code accordingly.<commit_after>/*

This file is part of databank-couchdb, a CouchDB driver for databank.
Copyright 2013 Kuno Woudt <kuno@frob.nl>

Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License.  You may
obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.  See the License for the specific language governing
permissions and limitations under the License.

*/

var _ = require('underscore');
var step = require('step');
var couchdb = require('couchdb-api');
var databank = require('databank');

var CouchDBDatabank = function(params) {
    this.schema    = params.schema || {};
    this.location  = params.location || "http://localhost:5984";
    this.database  = params.database || null;
    this.clear_database = params._clear_database_for_test_run || false;
    this.connection = null;
};

var SEPARATOR = ':';
var document_key = function (type, id) { return type + SEPARATOR + id; };

/* When running tests, clear the database first, but only do so once. */
var TEST_DATABASE_CLEARED = false;

CouchDBDatabank.prototype = new databank.Databank();

CouchDBDatabank.prototype._error_messages = {
    "clear_test_database": function (location, dbname) {
        return "CouchDB cannot create the test database " + dbname + " at " + location;
    },
    "connect": function (location, dbname) {
        return "CouchDB cannot connect to database " + dbname + " at " + location;
    },
    "delete": function (type, id) {
        return "CouchDB error deleting document " + document_key (type, id);
    },
    "read": function (type, id) {
        return "CouchDB error while reading document " + document_key (type, id);
    },
    "read_document": function (document_id) {
        return "CouchDB error while reading document " + document_id;
    },
    "readAll": function (type, ids) {
        return "CouchDB error while reading all " + type + "documents";
    },
    "save_document": function (document_id) {
        return "CouchDB error while saving document " + document_id;
    },
    "search": function (type, criteria) {
        return "CouchDB error while searching for a " + type;
    }
};

CouchDBDatabank.prototype._error = function (err, verb) {
    if (err instanceof databank.DatabankError)
        return err;

    var self = this;
    var context = Array.prototype.slice.call (arguments, 2);
    var msg = self._error_messages[verb].apply (null, context) +
        ": " + JSON.stringify (err);

    return new databank.DatabankError (msg);
};

CouchDBDatabank.prototype._connected = function(onCompletion) {
    if (!this.connection) {
        onCompletion(new databank.NotConnectedError(), null);
        return false;
    }

    return true;
};

CouchDBDatabank.prototype._new_document = function (type, id) {
    return this.connection.doc (document_key (type, id));
};

CouchDBDatabank.prototype._read_document = function(type, id, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    var document = self._new_document (type, id);
    document.get (function (err, response) {
        if (err && err.error === 'not_found')
        {
            onCompletion (new databank.NoSuchThingError(type, id), null);
        }
        else if (err)
        {
            onCompletion (self._error (err, "read_document", document));
        }
        else
        {
            onCompletion (null, document);
        }
    });
};

CouchDBDatabank.prototype._save_document = function (document, value, onCompletion) {
    var self = this;

    document.body.data = value;
    document.save (function (err, response) {
        if (err && err.error === 'conflict')
        {
            onCompletion(new databank.AlreadyExistsError ());
        }
        else if (err)
        {
            onCompletion(self._error (err, "save_document", document.id))
        }
        else
        {
            onCompletion(null, value);
        }
    });
};

CouchDBDatabank.prototype._clear_test_database = function (params, onCompletion) {
    var self = this;
    if (self.clear_database && !TEST_DATABASE_CLEARED)
    {
        step(
            function () {
                self.connection.drop (this);
            },
            function (err) {
                TEST_DATABASE_CLEARED = true;
                err ? this(err) : self.connection.create (this);
            },
            function () {
                onCompletion ();
            }
        );
    }
    else
    {
        onCompletion ();
    }
};

CouchDBDatabank.prototype.connect = function(params, onCompletion) {
    var self = this;

    if (self.connection) {
        onCompletion(new databank.AlreadyConnectedError(), null);
        return;
    }

    self.connection = couchdb.srv(self.location).db(self.database);

    var verify_connection = function () {
        self.connection.info (function (err, response) {
            err ? onCompletion(self._error (err, "connect", self.location, self.database))
                : onCompletion(null);
        });
    };

    self._clear_test_database (params, function (err) {
        err ? onCompletion(self._error (err, "clear_test_database", self.location, self.database))
            : verify_connection();
    });
};

CouchDBDatabank.prototype.disconnect = function(onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    self.connection = null;
    onCompletion (null);
};

CouchDBDatabank.prototype.create = function(type, id, value, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    // FIXME: onCompletion(new AlreadyExistsError(type, id), null);
    var document = self._new_document (type, id);
    self._save_document (document, value, onCompletion);
};

CouchDBDatabank.prototype.read = function(type, id, onCompletion) {
    var self = this;
    self._read_document (type, id, function (err, document) {
        err ? onCompletion (self._error (err, "read", type, id))
            : onCompletion (null, document.body.data);
    });
};

CouchDBDatabank.prototype.update = function(type, id, value, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    self._read_document (type, id, function (err, document) {
        err ? onCompletion (err)
            : self._save_document (document, value, onCompletion);
    });
};

CouchDBDatabank.prototype.save = function(type, id, value, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    // FIXME: verify that err is a NoSuchThingError.
    self._read_document (type, id, function (err, document) {
        err ? self.create (type, id, value, onCompletion)
            : self.update (type, id, value, onCompletion);
    });
};

CouchDBDatabank.prototype.del = function(type, id, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    self._read_document (type, id, function (err, document) {
        if (!document) {
            onCompletion (err)
        }
        else
        {
            document.del (function (err, response) {
                err ? onCompletion (self._error (err, "delete", type, id))
                    : onCompletion(null);
            });
        }
    });
};

CouchDBDatabank.prototype.search = function(type, criteria, onResult, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    var keys = _(criteria).keys ();

    var map = [
        "function (doc) {",
        "    var isArray = function(o) { return (o instanceof Array || typeof o == \"array\"); };",
        "    var getDottedProperty = null;",
        "    getDottedProperty = function (doc, name) {",
        "        if (!isArray (name)) name = name.split (\".\");",
        "        var key = name.shift ();",
        "        if (!key) return doc;",
        "        if (!doc.hasOwnProperty (key)) return undefined;",
        "        return getDottedProperty (doc[key], name);",
        "    };",
        "    var type = " + JSON.stringify (type + SEPARATOR) + ";",
        "    if (doc._id.substr (0, type.length) !== type) return;",
        "    var criteria = " + JSON.stringify (keys) + ";",
        "    var results = [];",
        "    for (var i in criteria) {",
        "        var value = getDottedProperty(doc.data, criteria[i]);",
        "        if (value === undefined) return;",
        "        results.push (value);",
        "    }",
        "    emit (results, doc);",
        "}"
    ];

    var query = { "key": _(keys).map (function (key) { return criteria[key] }) };

    // FIXME: tempViews are slow.  automatically create permanent view
    // after N requests for the same view?

    self.connection.tempView (map.join ("\n"), null, query, function (err, response) {
        if (err)
        {
            onCompletion(self._error (err, "search", type, criteria));
        }

        response.rows.forEach (function (doc) { onResult(doc.value.data); });

        onCompletion(null);
    });
};

CouchDBDatabank.prototype.readAll = function(type, ids, onCompletion) {
    var self = this;
    if (!self._connected (onCompletion))
        return;

    var id_map = {};
    ids.forEach (function (id) {
        var docid = document_key (type, id);
        id_map[docid] = id;
    });

    var options = { include_docs: true };
    var docids = _(id_map).keys ();

    self.connection.allDocs (options, docids, function (err, response) {
        if (err) {
            onCompletion (self._error (err, "readAll", type, ids));
        }
        else
        {
            var results = {};
            response.rows.forEach (function (row) {
                results[ id_map[row.key] ] = row.doc ? row.doc.data : null;
            });
            onCompletion(null, results);
        }
    });
};

module.exports = CouchDBDatabank;
<|endoftext|>


<commit_before>
'use strict';

var path = require('path')
	, util = require('util')
	, EventEmitter = require('events').EventEmitter
	, request = require('request')
	, _ = require('lodash')
	, cheerio = require('cheerio')
	, fs = require('fs')
	, charsetParser = require('charset-parser')
	, Bottleneck = require('bottleneckp')
	, seenreq = require('seenreq')
	, iconvLite = require('iconv-lite')
	, typeis = require('type-is').is;

var whacko=null, level, levels = ['silly','debug','verbose','info','warn','error','critical'];
try{
	whacko = require('whacko');
}catch(e){
	e.code;
}

function defaultLog(){    //2016-11-24T12:22:55.639Z - debug:
	if( levels.indexOf(arguments[0]) >= levels.indexOf(level) )
		console.log(new Date().toJSON()+' - '+ arguments[0] +': CRAWLER %s', util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
}

function checkJQueryNaming (options) {
	if ('jquery' in options) {
		options.jQuery = options.jquery;
		delete options.jquery;
	}
	return options;
}

function readJqueryUrl (url, callback) {
	if (url.match(/^(file:\/\/|\w+:|\/)/)) {
		fs.readFile(url.replace(/^file:\/\//,''),'utf-8', function(err,jq) {
			callback(err, jq);
		});
	} else {
		callback(null, url);
	}
}

function contentType(res){
	return get(res,'content-type').split(';').filter(item => item.trim().length !== 0).join(';');
}

function get(res,field){
	return res.headers[field.toLowerCase()] || '';
}

var log = defaultLog;

function Crawler (options) {
	var self = this;

	options = options||{};
	if(['onDrain','cache'].some(key => key in options)){
		throw new Error('Support for "onDrain", "cache" has been removed! For more details, see https://github.com/bda-research/node-crawler');
	}

	self.init(options);
}
// augment the prototype for node events using util.inherits
util.inherits(Crawler, EventEmitter);

Crawler.prototype.init = function init (options) {
	var self = this;

	var defaultOptions = {
		autoWindowClose:        true,
		forceUTF8:              true,
		gzip:                   true,
		incomingEncoding:       null,
		jQuery:                 true,
		maxConnections:         10,
		method:                 'GET',
		priority:               5,
		priorityRange:          10,
		rateLimit:             0,
		referer:                false,
		retries:                3,
		retryTimeout:           10000,
		timeout:                15000,
		skipDuplicates:         false,
		rotateUA:               false,
		homogeneous:            false
	};

	//return defaultOptions with overriden properties from options.
	self.options = _.extend(defaultOptions, options);

	// you can use jquery or jQuery
	self.options = checkJQueryNaming(self.options);

	// Don't make these options persist to individual queries
	self.globalOnlyOptions = ['maxConnections', 'rateLimit', 'priorityRange', 'homogeneous', 'skipDuplicates', 'rotateUA'];

	self.limiters = new Bottleneck.Cluster(self.options.maxConnections,self.options.rateLimit,self.options.priorityRange, self.options.priority, self.options.homogeneous);
	self.seen = new seenreq(self.options.seenreq);
	level = self.options.debug?'debug':'info';

	if(self.options.logger)
		log = self.options.logger.log.bind(self.options.logger);

	self.log = log;
	self.on('_release', function(){
		log('debug','Queue size: %d',this.queueSize);

		if(this.limiters.empty)
			return this.emit('drain');
	});
};

Crawler.prototype.setLimiterProperty = function setLimiterProperty (limiter, property, value) {
	var self = this;

	switch(property) {
	case 'rateLimit': self.limiters.key(limiter).setRateLimit(value);break;
	default: break;
	}
};

Crawler.prototype._inject = function _inject (response, options, callback) {
	var $;

	if (options.jQuery === 'whacko') {
		if(!whacko){
			throw new Error('Please install whacko by your own since `crawler` detected you specify explicitly');
		}

		$ = whacko.load(response.body);
		callback(null, response, options, $);
	}else if (options.jQuery === 'cheerio' || options.jQuery.name === 'cheerio' || options.jQuery === true) {
		var defaultCheerioOptions = {
			normalizeWhitespace: false,
			xmlMode: false,
			decodeEntities: true
		};
		var cheerioOptions = options.jQuery.options || defaultCheerioOptions;
		$ = cheerio.load(response.body, cheerioOptions);

		callback(null, response, options, $);
	}else if (options.jQuery.jsdom) {
		var jsdom = options.jQuery.jsdom;
		var scriptLocation = path.resolve(__dirname, '../vendor/jquery-2.1.1.min.js');

		//Use promises
		readJqueryUrl(scriptLocation, function(err, jquery) {
			try {
				jsdom.env({
					url: options.uri,
					html: response.body,
					src: [jquery],
					done: function (errors, window) {
						$ = window.jQuery;
						callback(errors, response, options, $);

						try {
							window.close();
							window = null;
						} catch (err) {
							log('error',err);
						}

					}
				});
			} catch (e) {
				options.callback(e,{options}, options.release);
			}
		});
	}
	// Jquery is set to false are not set
	else {
		callback(null, response, options);
	}
};

Crawler.prototype.isIllegal = function isIllegal (options) {
	if(_.isNull(options) || _.isUndefined(options) || (!_.isString(options) && !_.isPlainObject(options))) {
		return true;
	}
	return false;
};

Crawler.prototype.direct = function direct (options) {
	var self = this;

	if(self.isIllegal(options) || !_.isPlainObject(options)) {
		return log('warn','Illegal queue option: ', JSON.stringify(options));
	}

	if(!('callback' in options) || !_.isFunction(options.callback)) {
		return log('warn','must specify callback function when using sending direct request with crawler');
	}

	options = checkJQueryNaming(options);

	// direct request does not follow the global preRequest
	options.preRequest = options.preRequest || null;

	_.defaults(options, self.options);

	// direct request is not allowed to retry
	options.retries = 0;

	// direct request does not emit event:'request' by default
	options.skipEventRequest = _.isBoolean(options.skipEventRequest) ? options.skipEventRequest : true;

	self.globalOnlyOptions.forEach(globalOnlyOption=>delete options[globalOnlyOption]);

	self._buildHttpRequest(options);
};

Crawler.prototype.queue = function queue (options) {
	var self = this;

	// Did you get a single object or string? Make it compatible.
	options = _.isArray(options) ? options : [options];

	options = _.flattenDeep(options);

	for(var i = 0; i < options.length; ++i) {
		if(self.isIllegal(options[i])) {
			log('warn','Illegal queue option: ', JSON.stringify(options[i]));
			continue;
		}
		self._pushToQueue(
			_.isString(options[i]) ? {uri: options[i]} : options[i]
		);
	}
};

Crawler.prototype._pushToQueue = function _pushToQueue (options) {
	var self = this;

	// you can use jquery or jQuery
	options = checkJQueryNaming(options);

	_.defaults(options, self.options);
	options.headers = _.assign({}, self.options.headers, options.headers);

	// Remove all the global options from our options
	// TODO we are doing this for every _pushToQueue, find a way to avoid this
	self.globalOnlyOptions.forEach(globalOnlyOption=>delete options[globalOnlyOption]);

	// If duplicate skipping is enabled, avoid queueing entirely for URLs we already crawled
	if (self.options.skipDuplicates && self.seen.exists(options, options.seenreq)) {
		return;
	}

	self.emit('schedule',options);

	self.limiters.key(options.limiter||'default').submit(options.priority,function(done, limiter){
		options.release = function(){ done();self.emit('_release'); };
		if(!options.callback)
			options.callback = options.release;

		if (limiter) {
			self.emit('limiterChange', options, limiter);
		}

		if (options.html) {
			self._onContent(null, options, {body:options.html,headers:{'content-type':'text/html'}});
		} else if (typeof options.uri === 'function') {
			options.uri(function(uri) {
				options.uri = uri;
				self._buildHttpRequest(options);
			});
		} else {
			self._buildHttpRequest(options);
		}
	});
};

Crawler.prototype._buildHttpRequest = function _buildHTTPRequest (options) {
	var self = this;

	log('debug',options.method+' '+options.uri);
	if(options.proxy)
		log('debug','Use proxy: %s', options.proxy);

	// Cloning keeps the opts parameter clean:
	// - some versions of "request" apply the second parameter as a
	// property called "callback" to the first parameter
	// - keeps the query object fresh in case of a retry

	var ropts = _.assign({},options);

	if (!ropts.headers) { ropts.headers={}; }
	if (ropts.forceUTF8) {ropts.encoding=null;}
	// specifying json in request will have request sets body to JSON representation of value and
	// adds Content-type: application/json header. Additionally, parses the response body as JSON
	// so the response will be JSON object, no need to deal with encoding
	if (ropts.json) {options.encoding=null;}
	if (ropts.userAgent) {
		if(self.options.rotateUA && _.isArray(ropts.userAgent)){
			ropts.headers['User-Agent'] = ropts.userAgent[0];
			// If "rotateUA" is true, rotate User-Agent
			options.userAgent.push(options.userAgent.shift());
		}else{
			ropts.headers['User-Agent'] = ropts.userAgent;
		}
	}

	if (ropts.referer) {
		ropts.headers.Referer = ropts.referer;
	}

	if (ropts.proxies && ropts.proxies.length) {
		ropts.proxy = ropts.proxies[0];
	}

	var doRequest = function(err) {
		if(err) {
			err.message = 'Error in preRequest' + (err.message ? ', '+err.message : err.message);
			switch(err.op) {
			case 'retry': log('debug', err.message + ', retry ' + options.uri);self._onContent(err,options);break;
			case 'fail': log('debug', err.message + ', fail ' + options.uri);options.callback(err,{options:options},options.release);break;
			case 'abort': log('debug', err.message + ', abort ' + options.uri);options.release();break;
			case 'queue': log('debug', err.message + ', queue ' + options.uri);self.queue(options);options.release();break;
			default: log('debug', err.message + ', retry ' + options.uri);self._onContent(err,options);break;
			}
			return;
		}

		if(ropts.skipEventRequest !== true) {
			self.emit('request',ropts);
		}

		var requestArgs = ['uri','url','qs','method','headers','body','form','formData','json','multipart','followRedirect','followAllRedirects', 'maxRedirects','encoding','pool','timeout','proxy','auth','oauth','strictSSL','jar','aws','gzip','time','tunnel','proxyHeaderWhiteList','proxyHeaderExclusiveList','localAddress','forever', 'agent'];

		request(_.pick.apply(self,[ropts].concat(requestArgs)), function(error,response) {
			if (error) {
				return self._onContent(error, options);
			}

			self._onContent(error,options,response);
		});
	};

	if (options.preRequest && _.isFunction(options.preRequest)) {
		options.preRequest(ropts, doRequest);
	} else {
		doRequest();
	}
};

Crawler.prototype._onContent = function _onContent (error, options, response) {
	var self = this;

	if (error) {
		log('error','Error '+error+' when fetching '+ (options.uri||options.url)+(options.retries?' ('+options.retries+' retries left)':''));

		if (options.retries) {
			self.options.skipDuplicates = false;
			setTimeout(function() {
				options.retries--;
				self.queue(options);
				options.release();
			},options.retryTimeout);
		} else{
			options.callback(error,{options:options},options.release);
		}

		return;
	}

	if (!response.body) { response.body=''; }

	log('debug','Got '+(options.uri||'html')+' ('+response.body.length+' bytes)...');

	try{
		self._doEncoding(options,response);
	}catch(e){
		log('error',e);
		return options.callback(e,{options:options},options.release);
	}

	response.options = options;

	if(options.method === 'HEAD' || !options.jQuery){
		return options.callback(null,response,options.release);
	}

	var injectableTypes = ['html','xhtml','text/xml', 'application/xml', '+xml'];
	if (!options.html && !typeis(contentType(response), injectableTypes)){
		log('warn','response body is not HTML, skip injecting. Set jQuery to false to suppress this message');
		return options.callback(null,response,options.release);
	}

	log('debug','Injecting');

	self._inject(response, options, self._injected.bind(self));
};

Crawler.prototype._injected = function(errors, response, options, $){
	log('debug','Injected');

	response.$ = $;
	options.callback(errors, response, options.release);
};

Crawler.prototype._doEncoding = function(options,response){
	var self = this;

	if(options.encoding === null){
		return;
	}

	if (options.forceUTF8) {
		var charset = options.incomingEncoding || self._parseCharset(response);
		response.charset = charset;
		log('debug','Charset ' + charset);

		if (charset !== 'utf-8' && charset !== 'ascii') {// convert response.body into 'utf-8' encoded buffer
			response.body = iconvLite.decode(response.body, charset);
		}
	}

	response.body = response.body.toString();
};

Crawler.prototype._parseCharset = function(res){
	var charset = charsetParser(contentType(res));
	if(charset)
		return charset;

	if(!typeis(contentType(res), ['html'])){
		log('debug','Charset not detected in response headers, please specify using `incomingEncoding`, use `utf-8` by default');
		return 'utf-8';
	}

	var body = res.body instanceof Buffer?res.body.toString():res.body;
	charset = charsetParser(contentType(res),body,'utf-8');

	return charset;
};

Object.defineProperty(Crawler.prototype,'queueSize',{
	get:function(){
		return this.limiters.unfinishedClients;
	}
});

module.exports = Crawler;
<commit_msg>Simplified the isIllegal method of the Crawler class<commit_after>
'use strict';

var path = require('path')
	, util = require('util')
	, EventEmitter = require('events').EventEmitter
	, request = require('request')
	, _ = require('lodash')
	, cheerio = require('cheerio')
	, fs = require('fs')
	, charsetParser = require('charset-parser')
	, Bottleneck = require('bottleneckp')
	, seenreq = require('seenreq')
	, iconvLite = require('iconv-lite')
	, typeis = require('type-is').is;

var whacko=null, level, levels = ['silly','debug','verbose','info','warn','error','critical'];
try{
	whacko = require('whacko');
}catch(e){
	e.code;
}

function defaultLog(){    //2016-11-24T12:22:55.639Z - debug:
	if( levels.indexOf(arguments[0]) >= levels.indexOf(level) )
		console.log(new Date().toJSON()+' - '+ arguments[0] +': CRAWLER %s', util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
}

function checkJQueryNaming (options) {
	if ('jquery' in options) {
		options.jQuery = options.jquery;
		delete options.jquery;
	}
	return options;
}

function readJqueryUrl (url, callback) {
	if (url.match(/^(file:\/\/|\w+:|\/)/)) {
		fs.readFile(url.replace(/^file:\/\//,''),'utf-8', function(err,jq) {
			callback(err, jq);
		});
	} else {
		callback(null, url);
	}
}

function contentType(res){
	return get(res,'content-type').split(';').filter(item => item.trim().length !== 0).join(';');
}

function get(res,field){
	return res.headers[field.toLowerCase()] || '';
}

var log = defaultLog;

function Crawler (options) {
	var self = this;

	options = options||{};
	if(['onDrain','cache'].some(key => key in options)){
		throw new Error('Support for "onDrain", "cache" has been removed! For more details, see https://github.com/bda-research/node-crawler');
	}

	self.init(options);
}
// augment the prototype for node events using util.inherits
util.inherits(Crawler, EventEmitter);

Crawler.prototype.init = function init (options) {
	var self = this;

	var defaultOptions = {
		autoWindowClose:        true,
		forceUTF8:              true,
		gzip:                   true,
		incomingEncoding:       null,
		jQuery:                 true,
		maxConnections:         10,
		method:                 'GET',
		priority:               5,
		priorityRange:          10,
		rateLimit:             0,
		referer:                false,
		retries:                3,
		retryTimeout:           10000,
		timeout:                15000,
		skipDuplicates:         false,
		rotateUA:               false,
		homogeneous:            false
	};

	//return defaultOptions with overriden properties from options.
	self.options = _.extend(defaultOptions, options);

	// you can use jquery or jQuery
	self.options = checkJQueryNaming(self.options);

	// Don't make these options persist to individual queries
	self.globalOnlyOptions = ['maxConnections', 'rateLimit', 'priorityRange', 'homogeneous', 'skipDuplicates', 'rotateUA'];

	self.limiters = new Bottleneck.Cluster(self.options.maxConnections,self.options.rateLimit,self.options.priorityRange, self.options.priority, self.options.homogeneous);
	self.seen = new seenreq(self.options.seenreq);
	level = self.options.debug?'debug':'info';

	if(self.options.logger)
		log = self.options.logger.log.bind(self.options.logger);

	self.log = log;
	self.on('_release', function(){
		log('debug','Queue size: %d',this.queueSize);

		if(this.limiters.empty)
			return this.emit('drain');
	});
};

Crawler.prototype.setLimiterProperty = function setLimiterProperty (limiter, property, value) {
	var self = this;

	switch(property) {
        case 'rateLimit': self.limiters.key(limiter).setRateLimit(value);break;
        default: break;
	}
};

Crawler.prototype._inject = function _inject (response, options, callback) {
	var $;

	if (options.jQuery === 'whacko') {
		if(!whacko){
			throw new Error('Please install whacko by your own since `crawler` detected you specify explicitly');
		}

		$ = whacko.load(response.body);
		callback(null, response, options, $);
	}else if (options.jQuery === 'cheerio' || options.jQuery.name === 'cheerio' || options.jQuery === true) {
		var defaultCheerioOptions = {
			normalizeWhitespace: false,
			xmlMode: false,
			decodeEntities: true
		};
		var cheerioOptions = options.jQuery.options || defaultCheerioOptions;
		$ = cheerio.load(response.body, cheerioOptions);

		callback(null, response, options, $);
	}else if (options.jQuery.jsdom) {
		var jsdom = options.jQuery.jsdom;
		var scriptLocation = path.resolve(__dirname, '../vendor/jquery-2.1.1.min.js');

		//Use promises
		readJqueryUrl(scriptLocation, function(err, jquery) {
			try {
				jsdom.env({
					url: options.uri,
					html: response.body,
					src: [jquery],
					done: function (errors, window) {
						$ = window.jQuery;
						callback(errors, response, options, $);

						try {
							window.close();
							window = null;
						} catch (err) {
							log('error',err);
						}

					}
				});
			} catch (e) {
				options.callback(e,{options}, options.release);
			}
		});
	}
	// Jquery is set to false are not set
	else {
		callback(null, response, options);
	}
};

Crawler.prototype.isIllegal = function isIllegal (options) {
	return (_.isNull(options) || _.isUndefined(options) || (!_.isString(options) && !_.isPlainObject(options)));
};

Crawler.prototype.direct = function direct (options) {
	var self = this;

	if(self.isIllegal(options) || !_.isPlainObject(options)) {
		return log('warn','Illegal queue option: ', JSON.stringify(options));
	}

	if(!('callback' in options) || !_.isFunction(options.callback)) {
		return log('warn','must specify callback function when using sending direct request with crawler');
	}

	options = checkJQueryNaming(options);

	// direct request does not follow the global preRequest
	options.preRequest = options.preRequest || null;

	_.defaults(options, self.options);

	// direct request is not allowed to retry
	options.retries = 0;

	// direct request does not emit event:'request' by default
	options.skipEventRequest = _.isBoolean(options.skipEventRequest) ? options.skipEventRequest : true;

	self.globalOnlyOptions.forEach(globalOnlyOption=>delete options[globalOnlyOption]);

	self._buildHttpRequest(options);
};

Crawler.prototype.queue = function queue (options) {
	var self = this;

	// Did you get a single object or string? Make it compatible.
	options = _.isArray(options) ? options : [options];

	options = _.flattenDeep(options);

	for(var i = 0; i < options.length; ++i) {
		if(self.isIllegal(options[i])) {
			log('warn','Illegal queue option: ', JSON.stringify(options[i]));
			continue;
		}
		self._pushToQueue(
			_.isString(options[i]) ? {uri: options[i]} : options[i]
		);
	}
};

Crawler.prototype._pushToQueue = function _pushToQueue (options) {
	var self = this;

	// you can use jquery or jQuery
	options = checkJQueryNaming(options);

	_.defaults(options, self.options);
	options.headers = _.assign({}, self.options.headers, options.headers);

	// Remove all the global options from our options
	// TODO we are doing this for every _pushToQueue, find a way to avoid this
	self.globalOnlyOptions.forEach(globalOnlyOption=>delete options[globalOnlyOption]);

	// If duplicate skipping is enabled, avoid queueing entirely for URLs we already crawled
	if (self.options.skipDuplicates && self.seen.exists(options, options.seenreq)) {
		return;
	}

	self.emit('schedule',options);

	self.limiters.key(options.limiter||'default').submit(options.priority,function(done, limiter){
		options.release = function(){ done();self.emit('_release'); };
		if(!options.callback)
			options.callback = options.release;

		if (limiter) {
			self.emit('limiterChange', options, limiter);
		}

		if (options.html) {
			self._onContent(null, options, {body:options.html,headers:{'content-type':'text/html'}});
		} else if (typeof options.uri === 'function') {
			options.uri(function(uri) {
				options.uri = uri;
				self._buildHttpRequest(options);
			});
		} else {
			self._buildHttpRequest(options);
		}
	});
};

Crawler.prototype._buildHttpRequest = function _buildHTTPRequest (options) {
	var self = this;

	log('debug',options.method+' '+options.uri);
	if(options.proxy)
		log('debug','Use proxy: %s', options.proxy);

	// Cloning keeps the opts parameter clean:
	// - some versions of "request" apply the second parameter as a
	// property called "callback" to the first parameter
	// - keeps the query object fresh in case of a retry

	var ropts = _.assign({},options);

	if (!ropts.headers) { ropts.headers={}; }
	if (ropts.forceUTF8) {ropts.encoding=null;}
	// specifying json in request will have request sets body to JSON representation of value and
	// adds Content-type: application/json header. Additionally, parses the response body as JSON
	// so the response will be JSON object, no need to deal with encoding
	if (ropts.json) {options.encoding=null;}
	if (ropts.userAgent) {
		if(self.options.rotateUA && _.isArray(ropts.userAgent)){
			ropts.headers['User-Agent'] = ropts.userAgent[0];
			// If "rotateUA" is true, rotate User-Agent
			options.userAgent.push(options.userAgent.shift());
		}else{
			ropts.headers['User-Agent'] = ropts.userAgent;
		}
	}

	if (ropts.referer) {
		ropts.headers.Referer = ropts.referer;
	}

	if (ropts.proxies && ropts.proxies.length) {
		ropts.proxy = ropts.proxies[0];
	}

	var doRequest = function(err) {
		if(err) {
			err.message = 'Error in preRequest' + (err.message ? ', '+err.message : err.message);
			switch(err.op) {
			case 'retry': log('debug', err.message + ', retry ' + options.uri);self._onContent(err,options);break;
			case 'fail': log('debug', err.message + ', fail ' + options.uri);options.callback(err,{options:options},options.release);break;
			case 'abort': log('debug', err.message + ', abort ' + options.uri);options.release();break;
			case 'queue': log('debug', err.message + ', queue ' + options.uri);self.queue(options);options.release();break;
			default: log('debug', err.message + ', retry ' + options.uri);self._onContent(err,options);break;
			}
			return;
		}

		if(ropts.skipEventRequest !== true) {
			self.emit('request',ropts);
		}

		var requestArgs = ['uri','url','qs','method','headers','body','form','formData','json','multipart','followRedirect','followAllRedirects', 'maxRedirects','encoding','pool','timeout','proxy','auth','oauth','strictSSL','jar','aws','gzip','time','tunnel','proxyHeaderWhiteList','proxyHeaderExclusiveList','localAddress','forever', 'agent'];

		request(_.pick.apply(self,[ropts].concat(requestArgs)), function(error,response) {
			if (error) {
				return self._onContent(error, options);
			}

			self._onContent(error,options,response);
		});
	};

	if (options.preRequest && _.isFunction(options.preRequest)) {
		options.preRequest(ropts, doRequest);
	} else {
		doRequest();
	}
};

Crawler.prototype._onContent = function _onContent (error, options, response) {
	var self = this;

	if (error) {
		log('error','Error '+error+' when fetching '+ (options.uri||options.url)+(options.retries?' ('+options.retries+' retries left)':''));

		if (options.retries) {
			self.options.skipDuplicates = false;
			setTimeout(function() {
				options.retries--;
				self.queue(options);
				options.release();
			},options.retryTimeout);
		} else{
			options.callback(error,{options:options},options.release);
		}

		return;
	}

	if (!response.body) { response.body=''; }

	log('debug','Got '+(options.uri||'html')+' ('+response.body.length+' bytes)...');

	try{
		self._doEncoding(options,response);
	}catch(e){
		log('error',e);
		return options.callback(e,{options:options},options.release);
	}

	response.options = options;

	if(options.method === 'HEAD' || !options.jQuery){
		return options.callback(null,response,options.release);
	}

	var injectableTypes = ['html','xhtml','text/xml', 'application/xml', '+xml'];
	if (!options.html && !typeis(contentType(response), injectableTypes)){
		log('warn','response body is not HTML, skip injecting. Set jQuery to false to suppress this message');
		return options.callback(null,response,options.release);
	}

	log('debug','Injecting');

	self._inject(response, options, self._injected.bind(self));
};

Crawler.prototype._injected = function(errors, response, options, $){
	log('debug','Injected');

	response.$ = $;
	options.callback(errors, response, options.release);
};

Crawler.prototype._doEncoding = function(options,response){
	var self = this;

	if(options.encoding === null){
		return;
	}

	if (options.forceUTF8) {
		var charset = options.incomingEncoding || self._parseCharset(response);
		response.charset = charset;
		log('debug','Charset ' + charset);

		if (charset !== 'utf-8' && charset !== 'ascii') {// convert response.body into 'utf-8' encoded buffer
			response.body = iconvLite.decode(response.body, charset);
		}
	}

	response.body = response.body.toString();
};

Crawler.prototype._parseCharset = function(res){
	var charset = charsetParser(contentType(res));
	if(charset)
		return charset;

	if(!typeis(contentType(res), ['html'])){
		log('debug','Charset not detected in response headers, please specify using `incomingEncoding`, use `utf-8` by default');
		return 'utf-8';
	}

	var body = res.body instanceof Buffer?res.body.toString():res.body;
	charset = charsetParser(contentType(res),body,'utf-8');

	return charset;
};

Object.defineProperty(Crawler.prototype,'queueSize',{
	get:function(){
		return this.limiters.unfinishedClients;
	}
});

module.exports = Crawler;
<|endoftext|>


<commit_before>var https = require('https'),
  url = require('url'),
  util = require('util'),
  package_json = require('../package.json');

function Databox(config) {
  this.config = config || {};

  if (config.hasOwnProperty('push_token'))
    this.push_token = config.push_token;

  this.push_token = process.env.DATABOX_PUSH_TOKEN || this.push_token;

  if (typeof(this.push_token) == "undefined")
    throw new Error("Missing push_token.");

  this.config.push_host = 'https://push.databox.com/';
  this.config.user_agent = util.format('databox-js/%s', package_json.version);
  this.config.accept = util.format('application/vnd.databox.v%s+json', package_json.version.split('.')[0]);
}

Databox.prototype._pushJSONRequest = function (i_options, data, cb) {
  if (typeof(i_options) == "undefined") i_options = {};

  var options = {
    hostname: url.parse(this.config.push_host).hostname,
    port: url.parse(this.config.push_host).port || 443,
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'User-Agent': this.config.user_agent,
      'Accept': this.config.accept
    },
    auth: util.format("%s:%s", this.config.push_token, '')
  };

  for (var k in i_options) {
    if (i_options.hasOwnProperty(k)) options[k] = i_options[k];
  }

  var jsonData = null;
  if (data !== null) {
    jsonData = JSON.stringify({data: data});
    options.headers['Content-Length'] = jsonData.length;
  }

  var request = https
    .request(options, function (response) {
      var bf = '';
      response.on('data', function (data) {
        bf += data;
        if (bf.length > 1e6) request.connection.destroy();
      });

      response.on('end', function (data) {
        if (typeof(cb) != "undefined") {

          try {
            outObject = JSON.parse(bf);
            cb(outObject);
          } catch (e) {
            var p = 1;
            cb(e);
          }

          outObject = null;
          bf = null;
        }
      });
    });

  if (data !== null) request.write(jsonData);
  request.end();

  return request;
};

Databox.prototype._processKPI = function (kpi) {
  var out = {};

  out[util.format("$%s", kpi.key)] = kpi.value;

  if (kpi.hasOwnProperty("date") || typeof(kpi.date) != "undefined") {
    out.date = kpi.date;
    //TODO: If it's some kind of Date/Time instance do "casting".
  }

  if (kpi.hasOwnProperty("unit") || typeof(kpi.unit) != "undefined") {
    out.unit = kpi.unit;
  }

  if (kpi.hasOwnProperty('attributes') || typeof(kpi.attributes) != "undefined") {
    for (var k in kpi.attributes)
      out[k] = kpi.attributes[k];
  }

  return out;
};

Databox.prototype.getPush = function (sha, cb) {
  var path = (sha instanceof Array) ? '/lastpushes?id=' + sha.join(',') : '/lastpushes/' + sha;

  return this._pushJSONRequest({
    method: 'GET',
    path: path
  }, null, cb);
};

Databox.prototype.lastPushes = function (n, cb) {
  return this._pushJSONRequest({
    method: 'GET',
    path: util.format("/lastpushes?limit=%d", parseInt(n, 10))
  }, null, cb);
};

Databox.prototype.lastPush = function (cb) {
  return this.lastPushes(1, cb);
};

Databox.prototype.push = function (kpi, cb) {
  return this._pushJSONRequest('/',
    [kpi].map(this._processKPI),
    cb
  );
};

Databox.prototype.insertAll = function (kpis, cb) {
  return this._pushJSONRequest('/',
    kpis.map(this._processKPI),
    cb
  );
};

Databox.prototype.metrics = function (cb) {
  return this._pushJSONRequest({
    method: 'GET',
    path: '/metrickeys'
  }, null, cb);
};

Databox.prototype.purge = function (cb) {
  return this._pushJSONRequest({
    method: 'DELETE',
    path: '/data'
  }, null, cb);
};

module.exports = Databox;
<commit_msg>Fix error on JSON request with certain unicode chars<commit_after>var https = require('https'),
  url = require('url'),
  util = require('util'),
  package_json = require('../package.json');

function Databox(config) {
  this.config = config || {};

  if (config.hasOwnProperty('push_token'))
    this.push_token = config.push_token;

  this.push_token = process.env.DATABOX_PUSH_TOKEN || this.push_token;

  if (typeof(this.push_token) == "undefined")
    throw new Error("Missing push_token.");

  this.config.push_host = 'https://push.databox.com/';
  this.config.user_agent = util.format('databox-js/%s', package_json.version);
  this.config.accept = util.format('application/vnd.databox.v%s+json', package_json.version.split('.')[0]);
}

Databox.prototype._pushJSONRequest = function (i_options, data, cb) {
  if (typeof(i_options) == "undefined") i_options = {};

  var options = {
    hostname: url.parse(this.config.push_host).hostname,
    port: url.parse(this.config.push_host).port || 443,
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'User-Agent': this.config.user_agent,
      'Accept': this.config.accept
    },
    auth: util.format("%s:%s", this.config.push_token, '')
  };

  for (var k in i_options) {
    if (i_options.hasOwnProperty(k)) options[k] = i_options[k];
  }

  var jsonData = null;
  if (data !== null) {
    jsonData = JSON.stringify({data: data});
    // When any of the values represented inside the Object being stringified,
    // contain some UNICODE characters, produces the return of the following Error:
    // {message : "Invalid request body - JSON parse error", type : "invalid_json"}
    //
    // The proposed patch, converts each string char to its (Hex) Unicode representation
    // An example, using a Spanish word -with a conflictive accent mark in the 'o'- :
    // "Atención" => "\u0041\u0074\u0065\u006e\u0063\u0069\u00f3\u006e"
    // This disambiguation prevents the unexpected error on the request sending.
    // <-- BEGINING OF THE PATCH
    jsonData  = jsonData.replace(/[\u007F-\uFFFF]/g, function(chr) {
      return "\\u" + ("0000" + chr.charCodeAt(0).toString(16)).substr(-4);
    });
    // END OF THE PATCH -->
    options.headers['Content-Length'] = jsonData.length;
  }

  var request = https
    .request(options, function (response) {
      var bf = '';
      response.on('data', function (data) {
        bf += data;
        if (bf.length > 1e6) request.connection.destroy();
      });

      response.on('end', function (data) {
        if (typeof(cb) != "undefined") {

          try {
            outObject = JSON.parse(bf);
            cb(outObject);
          } catch (e) {
            var p = 1;
            cb(e);
          }

          outObject = null;
          bf = null;
        }
      });
    });

  if (data !== null) request.write(jsonData);
  request.end();

  return request;
};

Databox.prototype._processKPI = function (kpi) {
  var out = {};

  out[util.format("$%s", kpi.key)] = kpi.value;

  if (kpi.hasOwnProperty("date") || typeof(kpi.date) != "undefined") {
    out.date = kpi.date;
    //TODO: If it's some kind of Date/Time instance do "casting".
  }

  if (kpi.hasOwnProperty("unit") || typeof(kpi.unit) != "undefined") {
    out.unit = kpi.unit;
  }

  if (kpi.hasOwnProperty('attributes') || typeof(kpi.attributes) != "undefined") {
    for (var k in kpi.attributes)
      out[k] = kpi.attributes[k];
  }

  return out;
};

Databox.prototype.getPush = function (sha, cb) {
  var path = (sha instanceof Array) ? '/lastpushes?id=' + sha.join(',') : '/lastpushes/' + sha;

  return this._pushJSONRequest({
    method: 'GET',
    path: path
  }, null, cb);
};

Databox.prototype.lastPushes = function (n, cb) {
  return this._pushJSONRequest({
    method: 'GET',
    path: util.format("/lastpushes?limit=%d", parseInt(n, 10))
  }, null, cb);
};

Databox.prototype.lastPush = function (cb) {
  return this.lastPushes(1, cb);
};

Databox.prototype.push = function (kpi, cb) {
  return this._pushJSONRequest('/',
    [kpi].map(this._processKPI),
    cb
  );
};

Databox.prototype.insertAll = function (kpis, cb) {
  return this._pushJSONRequest('/',
    kpis.map(this._processKPI),
    cb
  );
};

Databox.prototype.metrics = function (cb) {
  return this._pushJSONRequest({
    method: 'GET',
    path: '/metrickeys'
  }, null, cb);
};

Databox.prototype.purge = function (cb) {
  return this._pushJSONRequest({
    method: 'DELETE',
    path: '/data'
  }, null, cb);
};

module.exports = Databox;
<|endoftext|>


<commit_before>var thunky = require('thunky')
var pump = require('pump')
var through = require('through2')
var subleveldown = require('subleveldown')
var mutexify = require('mutexify')
var lexint = require('lexicographic-integer')
var duplexify = require('duplexify')
var messages = require('./messages')

var noop = function () {}

var KEYS = 'k!'
var BRANCHES = 'b!'

var Dataset = function (dat, name, branch) {
  if (!(this instanceof Dataset)) return new Dataset(dat, name, branch)

  var self = this

  this.name = name
  this.branch = branch
  this.state = null
  this.db = null
  this.dat = dat
  this.lock = mutexify()

  this._checkout = null

  this.open = thunky(function (cb) {
    dat.open(function (err) {
      if (err) return cb(err)
      self._init(branch)
      cb(null, self, dat)
    })
  })

  this.openWrite = thunky(function (cb) {
    self.open(function (err, self, dat) {
      if (err) return cb(err)
      if (self.db) return cb(null, self, dat)

      dat.log.add(null, messages.Commit.encode({type: 'init', dataset: name}), function (err, node) {
        if (err) return cb(err)

        dat._flush(node, function (err) {
          if (err) return cb(err)
          self._init(branch)
          cb(null, self, dat)
        })
      })
    })
  })
}

Dataset.prototype.__defineGetter__('head', function () {
  return this._checkout || (this.state && this.state.head.key)
})

Dataset.prototype.del = function (key, cb) {
  if (!cb) cb = noop

  this.openWrite(function (err, self, dat) {
    if (err) return cb(err)

    var del = {
      type: 'del',
      dataset: self.name,
      key: key
    }

    self._commit(dat, del, cb)
  })
}

Dataset.prototype.put = function (key, value, opts, cb) {
  if (typeof opts === 'function') return this.put(key, value, null, opts)
  if (!cb) cb = noop

  this.openWrite(function (err, self, dat) {
    if (err) return cb(err)

    var put = {
      type: 'put',
      dataset: self.name,
      key: key,
      value: value
    }

    self._commit(dat, put, cb)
  })
}

Dataset.prototype._commit = function (dat, node, cb) {
  var self = this

  this.lock(function (release) {
    var checkout = self._checkout && self._checkout !== self.state.head.key

    var commit = function (head) {
      dat.log.add(head, messages.Commit.encode(node), function (err, node) {
        if (err) return release(cb, err)
        dat._flush(node, function (err) {
          if (err) return release(cb, err)
          if (checkout) self._init(head.key)
          self._checkout = null
          release(cb, null, node) // node is not correct per the api but for now it's ok
        })
      })
    }

    if (!checkout) return commit(self.state.head)

    self.dat.log.get(self._checkout, function (err, head) {
      if (err) return cb(err)
      commit(head)
    })
  })
}

var notFound = function (key) {
  var err = new Error(key + ' not found') // todo: fix error
  err.notFound = true
  err.status = 404
  return err
}

Dataset.prototype.checkout = function (hash) {
  this._checkout = hash
  return this
}

Dataset.prototype.get = function (key, opts, cb) {
  if (typeof opts === 'function') return this.get(key, null, opts)
  if (!cb) cb = noop
  if (!opts) opts = {}

  var checkout = opts.checkout || this._checkout

  this.open(function (err, self, dat) {
    if (err) return cb(err)
    if (!self.state) return cb(notFound(key))

    if (checkout) return self._getCheckout(key, checkout, opts, cb)

    self.db.get('0!' + key, function (err, hash) {
      if (err) return cb(err)
      if (hash === ' ') return cb(notFound(key))

      dat.log.get(hash, function (err, node) {
        if (err) return cb(err)
        cb(null, messages.Commit.decode(node.value).value)
      })
    })
  })
}

Dataset.prototype.createWriteStream = function () {
  return through.obj(function (data, enc, cb) {
    if (data.type === 'del') {
      this.del(data.key, function (err) {
        cb(err)
      })
    } else {
      this.put(data.key, data.value, function (err) {
        cb(err)
      })
    }
  })
}

Dataset.prototype.createReadStream = function (opts) {
  if (!opts) opts = {}

  var result = duplexify.obj()

  this.open(function (err, self, dat) {
    if (err) return result.destroy(err)

    if (!self.state) {
      result.setReadable(false)
      result.setWritable(false)
      return
    }

    var prefix = '0!'
    var checkout = opts.checkout || self._checkout
    var sopts = {
      gt: prefix + (opts.gt || ''),
      lt: prefix + (opts.lt || '\xff'),
      lte: opts.lte ? prefix + opts.lte : undefined,
      gte: opts.gte ? prefix + opts.gte : undefined
    }

    var stream = checkout ? self.db.createKeyStream(sopts) : self.db.createValueStream(sopts)

    var write = checkout ?
      function (key, enc, cb) {
        self._getCheckout(key.slice(2), checkout, opts, function (err, node) {
          if (err && !err.notFound) return cb(err)
          if (err) return cb()
          cb(null, node)
        })
      } :
      function (hash, enc, cb) {
        if (hash === ' ') return cb()
        dat.log.get(hash, function (err, node) {
          if (err) return cb(err)
          cb(null, messages.Commit.decode(node.value))
        })
      }

    result.setReadable(pump(stream, through.obj(write)))
    result.setWritable(false)
  })

  return result
}

var getOne = function (stream, cb) {
  var result = null
  stream.on('error', cb)
  stream.on('data', function (data) {
    result = data
  })
  stream.on('end', function () {
    stream.removeListener('error', cb)
    cb(null, result)
  })
}

Dataset.prototype._getCheckout = function (key, rev, opts, cb) {
  var self = this

  this.dat.log.get(rev, function (err, node) {
    if (err) return cb(err)

    var revs = self.db.createValueStream({
      gt: '1!' + key + '!',
      lte: '1!' + key + '!' + lexint.pack(node.change, 'hex'),
      reverse: true,
      limit: 1
    })

    getOne(revs, function (err, hash) {
      if (err) return cb(err)
      if (hash === ' ' || !hash) return cb(notFound(key))
      self.dat.log.get(hash, function (err, node) {
        if (err) return cb(err)
        cb(null, messages.Commit.decode(node.value))
      })
    })
  })
}

Dataset.prototype._init = function (branch) {
  var dat = this.dat
  var name = this.name

  var reduce = function (result, key) {
    if (!result) return key
    return dat._index[name][result].root.change > dat._index[name][key].root.change ? key : result
  }

  if (!branch) branch = Object.keys(dat._index[name] || {}).reduce(reduce, null)
  this.state = dat._index[name] && dat._index[name][branch]
  if (this.state) this.db = subleveldown(dat._data, this.state.root.key)
}

module.exports = Dataset
<commit_msg>remove dead code<commit_after><|endoftext|>


<commit_before>const mongoose = require('mongoose');
const Promise = mongoose.Promise = require('bluebird');
const bcrypt = require('bcrypt');
const eachAsync = require('each-async');
const db = require('./common');

const User = mongoose.models.User;
const Team = mongoose.models.Team;
const TeamUser = mongoose.models.TeamUser;
const TeamEvent = mongoose.models.TeamEvent;
const Program = mongoose.models.Program;
const Event = mongoose.models.Event;
const Env = mongoose.models.Env;
const Invoice = mongoose.models.Invoice;
const InvoicingOrg = mongoose.models.InvoicingOrg;

const dbUser = require('./User');
const dbTeam = require('./Team');

const exp = {};

module.exports = exp;

exp.testSeed = function() {
    return new Promise(async function (fulfill, reject) {
        try {
            await createAdmin('admin', 'admin');
            if (process.env.ENV != 'dev') {
                console.log("Not DEV environment - skipping seed");
                return fulfill(true);

            }
            const data = {};
            await db.deleteAll();
            await createAdmin('admin', 'admin');
            await exp.testSeedEnv(data);
            await exp.testSeedUsers(data);
            await exp.testSeedPrograms(data);
            await exp.testSeedInvoicingOrgs(data);
            await exp.testSeedEvents(data);
            await exp.testSeedTeams(data);
            await exp.testSeedTeamsCoaches(data);
            await exp.testSeedMembers(data);
            await exp.testSeedInvoices(data);
            await exp.testSeedTeamEvents(data);
            console.log('==== TESTING');
            let myId = '00000000010000000002000' + '0';
            let teamId = '00000000010000000002000' + '0';
            let coachId = '00000000010000000002000' + '1';

            console.log("===== TEAM DETAILS START");
            let t = await dbTeam.getTeamDetails(myId, teamId);
            console.log("===== TEAM DETAILS RESULT");
            console.log(t);

            console.log("===== TEAM COACHES START");
            let tc = await dbTeam.getTeamCoaches(myId, teamId);
            console.log("===== TEAM COACHES RESULT");
            console.log(tc);

            console.log("===== TEAM MEMBERS START");
            let tm = await dbTeam.getTeamMembers(myId, teamId);
            console.log("===== TEAM MEMBERS RESULT");
            console.log(tm);
            return fulfill(true);
        } catch (err) {
            return reject(err);
        }

    })

};

function createAdmin(login, password){
    return new Promise(async function (fulfill, reject) {
        console.log("Creating admin user");
        try {
            var u = await User.findOneActive({username: login});
            if (!u) {
                const s = await bcrypt.genSalt(1);
                const h = await bcrypt.hash(password, s);
                u = await User.create(
                    {
                        username: 'admin',
                        passwordHash: h,
                        salt: s,
                        fullName: 'System admin',
                        isAdmin: true,
                        isSuperAdmin: true
                    });
                console.log("Admin user created");
            } else {
                console.log("Admin user already exists")
            }
            return fulfill(u);
        } catch(err) {
            return reject(err);
        }
    })
}

exp.testSeedEnv = function(data){
    return new Promise(function (fulfill, reject) {
        if (!data) data = {};
        console.log("Seeding env");
        try {
            const e = new Env();

            e.save();
            return fulfill(true);
        } catch (err) {
            return reject(err);
        }
    });

};

exp.testSeedTeams = function(data){
    return new Promise(function (fulfill, reject) {
        if (!data) data = {};
        data.teams = [];
        console.log("Seeding teams");
        eachAsync(
            [
                Team.testData({},'1'),
                Team.testData({},'2'),
                Team.testData({},'3'),
                Team.testData({},'4'),
                Team.testData({},'5')
            ],
            async function (item, index, done) {
                try {
                    const p = await Program.findOneActive();
                    const team = await Team.create(Object.assign(item, {_id:'00000000010000000002000'+index, programId:p._id}));
                    console.log("Team created: " + team.name + "===" + team.id);
                    data.teams.push({id: team.id, name: team.name});
                    return done();
                } catch (err) {
                    reject(err);
                }
            },
            function (err) {
                if (err) reject(err);
                console.log("Finished teams");
                fulfill(data);
            })
    });

};

exp.testSeedUsers = async function(data){
    return new Promise(function (fulfill, reject){
        if (!data) data = {};
        data.users = [];
        eachAsync(
            [
                ['user1','pwd1'],
                ['user2','pwd2'],
                ['user3','pwd3'],
                ['user4','pwd4'],
                ['user5','pwd5']
            ],
            async function (item, index, done) {
                const s = await bcrypt.genSalt(1);
                const h = await bcrypt.hash(item[1],s);
                const user = await User.create(
                    {
                        _id:'00000000010000000002000'+index,
                        username: item[0],
                        passwordHash: h,
                        salt:s,
                        fullName: 'Full '+item[0],
                        email: item[0]+'@email'
                    });
                console.log("User created: " + user.username + "===" + user.id);
                data.users.push({id: user.id, username: user.username});
                return done();
            },
            function (err) {
                if (err) reject(err);
                console.log("Finished users");
                fulfill(data);
            })

    })
};

exp.testSeedTeamsCoaches = async function(data) {
    return new Promise(function (fulfill, reject) {
        if (!data) data = {};
        console.log("Assigning coaches");
        eachAsync(
            data.teams,
            async function (item, index, done) {
                const ut = await TeamUser.create(
                    {
                        teamId: item.id,
                        userId: data.users[index%data.users.length].id,
                        role: 'coach'
                    }
                );
                console.log("Coach for team",item.name,' is ',data.users[index%data.users.length].username);
                return done();
            },
            function (err){
                if (err) reject(err);
                console.log("Finished coaches");
                fulfill(data)
            }
        )

    });
};

exp.testSeedMembers = async function(data){
    return new Promise(
        function (fulfill, reject) {
            console.log('Seeding members');
            if (!data) data = {};
            eachAsync(
                data.teams,
                async function (item, index, done) {
                    for (u of [1, 2, 3, 4, 5, 6]) {
                        let uId = '2'+index+'000000010000000002000' + u;
                        let uName = 'user' + u + '-' + item.name;
                        const user = await User.create(
                            {
                                _id: uId,
                                fullName: 'Full '+uName
                            });
                        await TeamUser.create(
                            {
                                userId: uId,
                                teamId: item.id,
                                role: 'member'
                            }
                        );
                        console.log('Member ', uName, ' team ', item.name);

                    }

                    console.log("Members assigned to team: ", item.name);
                    return done();
                },
                function (err) {
                    if (err) reject(err);
                    console.log("Finished assigning team members");
                    fulfill(data);
                }
            )

        }
    )
};

exp.testSeedPrograms = function(data){
    return new Promise(
        async function (fulfill, reject) {
            console.log('Seeding programs');
            if (!data) data = {};
            data.programs = [];
            console.log(data);
            eachAsync(
                [
                    {name:'FLL 2017/2018 Slovensko', user:'user1'}
                ],
                async function (item, index, done) {
                    const u = await User.findOneActive({username:item.user});
                    const prog = await Program.create(
                        {
                            name: item.name,
                            managers: [u.id]
                        }
                    );
                    console.log('Program', prog.name, ' id ', prog._id);
                    item.id = prog.id;
                    data.programs.push({id: prog.id, name: prog.name});
                    return done();
                },
                function (err) {
                    if (err) reject(err);
                    console.log("Finished creating programs");
                    fulfill(data);
                }
            )

        }
    )
};

exp.testSeedEvents = function(data){
    return new Promise(
        async function (fulfill, reject) {
            console.log('Seeding program events');
            if (!data.programs) data.programs = [
                {name:'FLL 2017/2018 Slovensko', user:'user1'}
            ];
            console.log(data);
            data.events = [];
            eachAsync(
                data.programs,
                async function (item, index, done) {

                    const e = await Event.create({
                        name: "Event "+index+" "+item.name,
                        startDate: new Date(),
                        programId: item.id,
                        invoicingOrg: data.invoicingOrgs[0].id
                    });
                    console.log('Event', e.name, ' id ', e.id);
                    data.events.push({id: e.id, name: e.name, programId:e.programId});
                    return done();
                },
                function (err) {
                    if (err) reject(err);
                    console.log("Finished creating events");
                    fulfill(data);
                }
            )

        }
    )
};

exp.testSeedTeamEvents = function(data){
    return new Promise(
        async function (fulfill, reject) {
            console.log('Seeding team events');
            console.log(data);
            data.teamEvents = [];
            eachAsync(
                data.events,
                async function (item, index, done) {

                    const e = await TeamEvent.create({

                        registeredOn: new Date(),
                        programId: item.programId,
                        eventId: item.id,
                        teamId: data.teams[index%data.teams.length].id,
                        teamNumber: "TeamNum"+index
                    });
                    console.log('TeamEvent', e.teamNumber, ' id ', e.id);
                    data.teamEvents.push({id: e.id, teamId: e.teamId, teamNumber:e.teamNumber, eventId:e.eventId});
                    return done();
                },
                function (err) {
                    if (err) reject(err);
                    console.log("Finished creating team registration for events");
                    fulfill(data);
                }
            )

        }
    )
};

exp.testSeedInvoices = function(data){
    return new Promise(
        async function (fulfill, reject) {
            console.log('Seeding invoices');
            console.log(data);
            data.invoices = [];
            eachAsync(
                data.teams,
                async function (item, index, done) {
                    let uId = '00000000010000000002000' + index;
                    const e = await Invoice.create({
                        _id : uId,
                        number: "FA "+index+" "+item.name,
                        type: index%2?"I":"P",
                        team: item.id,
                        invoicingOrg: data.invoicingOrgs[0].id
                    });
                    Invoice.testData(e,index);
                    e.save();
                    console.log('Invoice', e.number, ' id ', e.id);
                    data.invoices.push({id: e.id, number: e.number, team: e.team});
                    return done();
                },
                function (err) {
                    if (err) reject(err);
                    console.log("Finished seeding invoices");
                    fulfill(data);
                }
            )

        }
    )
};

exp.testSeedInvoicingOrgs = function(data){
    return new Promise(
        async function (fulfill, reject) {
            console.log('Seeding invocing orgs');
            console.log(data);
            data.invoicingOrgs = [];
            const dt = [
                {
                    org: {
                        name: "FLLSK 0001"
                    },
                    nextInvNumber: 5,
                    nextNTInvNumber: 1011,
                    invNumPrefix: "FA",
                    ntInvNumPrefix: "PF",
                    dueDays:21

                }
            ];
            eachAsync(
                dt,
                async function (item, index, done) {
                    //let uId = '00000000010000000002000' + index;
                    const io = await InvoicingOrg.create(item);
                    InvoicingOrg.testData(io,index);
                    io.save();
                    console.log('InvoicingOrg', io.org.name, ' id ', io.id);
                    data.invoicingOrgs.push({id: io.id, name: io.org.name});
                    return done();
                },
                function (err) {
                    if (err) reject(err);
                    console.log("Finished seeding invoicing orgs");
                    fulfill(data);
                }
            )

        }
    )
};
<commit_msg>fixing seed data<commit_after>const mongoose = require('mongoose');
const Promise = mongoose.Promise = require('bluebird');
const bcrypt = require('bcrypt');
const eachAsync = require('each-async');
const db = require('./common');

const User = mongoose.models.User;
const Team = mongoose.models.Team;
const TeamUser = mongoose.models.TeamUser;
const TeamEvent = mongoose.models.TeamEvent;
const Program = mongoose.models.Program;
const Event = mongoose.models.Event;
const Env = mongoose.models.Env;
const Invoice = mongoose.models.Invoice;
const InvoicingOrg = mongoose.models.InvoicingOrg;

const dbUser = require('./User');
const dbTeam = require('./Team');

const exp = {};

module.exports = exp;

exp.testSeed = function() {
    return new Promise(async function (fulfill, reject) {
        try {
            await createAdmin('admin', 'admin');
            if (process.env.ENV != 'dev') {
                console.log("Not DEV environment - skipping seed");
                return fulfill(true);

            }
            const data = {};
            await db.deleteAll();
            await createAdmin('admin', 'admin');
            await exp.testSeedEnv(data);
            await exp.testSeedUsers(data);
            await exp.testSeedPrograms(data);
            await exp.testSeedInvoicingOrgs(data);
            await exp.testSeedEvents(data);
            await exp.testSeedTeams(data);
            await exp.testSeedTeamsCoaches(data);
            await exp.testSeedMembers(data);
            await exp.testSeedInvoices(data);
            await exp.testSeedTeamEvents(data);
            console.log('==== TESTING');
            let myId = '00000000010000000002000' + '0';
            let teamId = '00000000010000000002000' + '0';
            let coachId = '00000000010000000002000' + '1';

            console.log("===== TEAM DETAILS START");
            let t = await dbTeam.getTeamDetails(myId, teamId);
            console.log("===== TEAM DETAILS RESULT");
            console.log(t);

            console.log("===== TEAM COACHES START");
            let tc = await dbTeam.getTeamCoaches(myId, teamId);
            console.log("===== TEAM COACHES RESULT");
            console.log(tc);

            console.log("===== TEAM MEMBERS START");
            let tm = await dbTeam.getTeamMembers(myId, teamId);
            console.log("===== TEAM MEMBERS RESULT");
            console.log(tm);
            return fulfill(true);
        } catch (err) {
            return reject(err);
        }

    })

};

function createAdmin(login, password){
    return new Promise(async function (fulfill, reject) {
        console.log("Creating admin user");
        try {
            var u = await User.findOneActive({username: login});
            if (!u) {
                const s = await bcrypt.genSalt(1);
                const h = await bcrypt.hash(password, s);
                u = await User.create(
                    {
                        username: 'admin',
                        passwordHash: h,
                        salt: s,
                        fullName: 'System admin',
                        isAdmin: true,
                        isSuperAdmin: true,
                        email: "admin@admin.admin"
                    });
                console.log("Admin user created");
            } else {
                console.log("Admin user already exists")
            }
            return fulfill(u);
        } catch(err) {
            return reject(err);
        }
    })
}

exp.testSeedEnv = function(data){
    return new Promise(function (fulfill, reject) {
        if (!data) data = {};
        console.log("Seeding env");
        try {
            const e = new Env();

            e.save();
            return fulfill(true);
        } catch (err) {
            return reject(err);
        }
    });

};

exp.testSeedTeams = function(data){
    return new Promise(function (fulfill, reject) {
        if (!data) data = {};
        data.teams = [];
        console.log("Seeding teams");
        eachAsync(
            [
                Team.testData({},'1'),
                Team.testData({},'2'),
                Team.testData({},'3'),
                Team.testData({},'4'),
                Team.testData({},'5')
            ],
            async function (item, index, done) {
                try {
                    const p = await Program.findOneActive();
                    const team = await Team.create(Object.assign(item, {_id:'00000000010000000002000'+index, programId:p._id}));
                    console.log("Team created: " + team.name + "===" + team.id);
                    data.teams.push({id: team.id, name: team.name});
                    return done();
                } catch (err) {
                    reject(err);
                }
            },
            function (err) {
                if (err) reject(err);
                console.log("Finished teams");
                fulfill(data);
            })
    });

};

exp.testSeedUsers = async function(data){
    return new Promise(function (fulfill, reject){
        if (!data) data = {};
        data.users = [];
        eachAsync(
            [
                ['user1','pwd1'],
                ['user2','pwd2'],
                ['user3','pwd3'],
                ['user4','pwd4'],
                ['user5','pwd5']
            ],
            async function (item, index, done) {
                const s = await bcrypt.genSalt(1);
                const h = await bcrypt.hash(item[1],s);
                const user = await User.create(
                    {
                        _id:'00000000010000000002000'+index,
                        username: item[0],
                        passwordHash: h,
                        salt:s,
                        fullName: 'Full '+item[0],
                        email: item[0]+'@email.huhu'
                    });
                console.log("User created: " + user.username + "===" + user.id);
                data.users.push({id: user.id, username: user.username});
                return done();
            },
            function (err) {
                if (err) reject(err);
                console.log("Finished users");
                fulfill(data);
            })

    })
};

exp.testSeedTeamsCoaches = async function(data) {
    return new Promise(function (fulfill, reject) {
        if (!data) data = {};
        console.log("Assigning coaches");
        eachAsync(
            data.teams,
            async function (item, index, done) {
                const ut = await TeamUser.create(
                    {
                        teamId: item.id,
                        userId: data.users[index%data.users.length].id,
                        role: 'coach'
                    }
                );
                console.log("Coach for team",item.name,' is ',data.users[index%data.users.length].username);
                return done();
            },
            function (err){
                if (err) reject(err);
                console.log("Finished coaches");
                fulfill(data)
            }
        )

    });
};

exp.testSeedMembers = async function(data){
    return new Promise(
        function (fulfill, reject) {
            console.log('Seeding members');
            if (!data) data = {};
            eachAsync(
                data.teams,
                async function (item, index, done) {
                    for (let u of [1, 2, 3, 4, 5, 6]) {
                        let uId = '2'+index+'000000010000000002000' + u;
                        let uName = 'user' + u + '-' + index;
                        console.log("Creating member uid=",uId,"username=",uName);
                        const user = await User.create(
                            {
                                _id: uId,
                                fullName: 'Full '+uName,
                                username: uName,
                                email:"user"+u+index+"@test.test"
                            });
                        await TeamUser.create(
                            {
                                userId: uId,
                                teamId: item.id,
                                role: 'member'
                            }
                        );
                        console.log('Member ', uName, ' team ', item.name);

                    }

                    console.log("Members assigned to team: ", item.name);
                    return done();
                },
                function (err) {
                    if (err) reject(err);
                    console.log("Finished assigning team members");
                    fulfill(data);
                }
            )

        }
    )
};

exp.testSeedPrograms = function(data){
    return new Promise(
        async function (fulfill, reject) {
            console.log('Seeding programs');
            if (!data) data = {};
            data.programs = [];
            console.log(data);
            eachAsync(
                [
                    {name:'FLL 2017/2018 Slovensko', user:'user1'}
                ],
                async function (item, index, done) {
                    const u = await User.findOneActive({username:item.user});
                    const prog = await Program.create(
                        {
                            name: item.name,
                            managers: [u.id]
                        }
                    );
                    console.log('Program', prog.name, ' id ', prog._id);
                    item.id = prog.id;
                    data.programs.push({id: prog.id, name: prog.name});
                    return done();
                },
                function (err) {
                    if (err) reject(err);
                    console.log("Finished creating programs");
                    fulfill(data);
                }
            )

        }
    )
};

exp.testSeedEvents = function(data){
    return new Promise(
        async function (fulfill, reject) {
            console.log('Seeding program events');
            if (!data.programs) data.programs = [
                {name:'FLL 2017/2018 Slovensko', user:'user1'}
            ];
            console.log(data);
            data.events = [];
            eachAsync(
                data.programs,
                async function (item, index, done) {

                    const e = await Event.create({
                        name: "Event "+index+" "+item.name,
                        startDate: new Date(),
                        programId: item.id,
                        invoicingOrg: data.invoicingOrgs[0].id
                    });
                    console.log('Event', e.name, ' id ', e.id);
                    data.events.push({id: e.id, name: e.name, programId:e.programId});
                    return done();
                },
                function (err) {
                    if (err) reject(err);
                    console.log("Finished creating events");
                    fulfill(data);
                }
            )

        }
    )
};

exp.testSeedTeamEvents = function(data){
    return new Promise(
        async function (fulfill, reject) {
            console.log('Seeding team events');
            console.log(data);
            data.teamEvents = [];
            eachAsync(
                data.events,
                async function (item, index, done) {

                    const e = await TeamEvent.create({

                        registeredOn: new Date(),
                        programId: item.programId,
                        eventId: item.id,
                        teamId: data.teams[index%data.teams.length].id,
                        teamNumber: "TeamNum"+index
                    });
                    console.log('TeamEvent', e.teamNumber, ' id ', e.id);
                    data.teamEvents.push({id: e.id, teamId: e.teamId, teamNumber:e.teamNumber, eventId:e.eventId});
                    return done();
                },
                function (err) {
                    if (err) reject(err);
                    console.log("Finished creating team registration for events");
                    fulfill(data);
                }
            )

        }
    )
};

exp.testSeedInvoices = function(data){
    return new Promise(
        async function (fulfill, reject) {
            console.log('Seeding invoices');
            console.log(data);
            data.invoices = [];
            eachAsync(
                data.teams,
                async function (item, index, done) {
                    let uId = '00000000010000000002000' + index;
                    const e = await Invoice.create({
                        _id : uId,
                        number: "FA "+index+" "+item.name,
                        type: index%2?"I":"P",
                        team: item.id,
                        invoicingOrg: data.invoicingOrgs[0].id
                    });
                    Invoice.testData(e,index);
                    e.save();
                    console.log('Invoice', e.number, ' id ', e.id);
                    data.invoices.push({id: e.id, number: e.number, team: e.team});
                    return done();
                },
                function (err) {
                    if (err) reject(err);
                    console.log("Finished seeding invoices");
                    fulfill(data);
                }
            )

        }
    )
};

exp.testSeedInvoicingOrgs = function(data){
    return new Promise(
        async function (fulfill, reject) {
            console.log('Seeding invocing orgs');
            console.log(data);
            data.invoicingOrgs = [];
            const dt = [
                {
                    org: {
                        name: "FLLSK 0001"
                    },
                    nextInvNumber: 5,
                    nextNTInvNumber: 1011,
                    invNumPrefix: "FA",
                    ntInvNumPrefix: "PF",
                    dueDays:21

                }
            ];
            eachAsync(
                dt,
                async function (item, index, done) {
                    //let uId = '00000000010000000002000' + index;
                    const io = await InvoicingOrg.create(item);
                    InvoicingOrg.testData(io,index);
                    io.save();
                    console.log('InvoicingOrg', io.org.name, ' id ', io.id);
                    data.invoicingOrgs.push({id: io.id, name: io.org.name});
                    return done();
                },
                function (err) {
                    if (err) reject(err);
                    console.log("Finished seeding invoicing orgs");
                    fulfill(data);
                }
            )

        }
    )
};
<|endoftext|>