<commit_before>'use strict';

const DareError = require('./utils/error');
const fieldReducer = require('./utils/field_reducer');
const checkFormat = require('./utils/unwrap_field');
const checkKey = require('./utils/validate_field');
const checkTableAlias = require('./utils/validate_alias');
const formatDateTime = require('./utils/format_datetime');

module.exports = function(options) {
	return new Promise((accept, reject) => {
		const fn = format_request.call(this, options);
		if (fn && fn.then) {
			fn.then(accept, reject);
		}
		accept(fn);
	});
};

function format_request(options = {}) {

	// Use the alias to find the real table name
	if (!options.alias) {
		const alias = options.table;
		options.alias = alias;
		options.table = this.table_alias_handler(alias);
	}

	// Reject when the table is not recognised
	if (!options.table) {
		throw new DareError(DareError.INVALID_REFERENCE, `Unrecognized reference '${options.table}'`);
	}

	// Call bespoke table handler
	const method = this.options.method;
	const table = options.table;
	const handlers = this.options[method] || {};
	let handler;

	if (table in handlers) {
		handler = handlers[table];
	}
	else if ('default' in handlers) {
		handler = handlers.default;
	}
	if (handler) {
		const fn = handler.call(this, options);
		if (fn && fn.then) {
			return fn.then(format_specs.bind(this, options));
		}
	}

	return format_specs.call(this, options);
}

function format_specs(options) {

	const schema = this.options.schema || {};
	const table_schema = schema[options.table] || {};

	const joined = {};
	const filters = [];

	// Format filters
	{
		const filter = options.filter || {};

		// filter must be an object with key=>values
		if (typeof filter !== 'object') {
			throw new DareError(DareError.INVALID_REFERENCE, `The filter '${filter}' is invalid.`);
		}

		// Explore the filter for any table joins
		for (let key in filter) {

			const value = filter[key];

			if (value && typeof value === 'object' && !Array.isArray(value)) {

				// Check this is a path
				checkTableAlias(key);

				// Add it to the join table
				joined[key] = joined[key] || {};
				joined[key].filter = Object.assign(joined[key].filter || {}, value);
			}
			else {

				let negate = false;

				// Does this have a negate operator?
				if (key.substring(0, 1) === '-') {

					// Mark as negative filter
					negate = true;

					// Strip the key
					key = key.substring(1);
				}

				// Check this is a path
				checkKey(key);

				const type = table_schema[key] && table_schema[key].type;
				filters.push(prepCondition(key, value, type, negate));
			}
		}
	}

	// Set the prefix if not already
	options.field_alias_path = options.field_alias_path || '';

	// Format fields
	let fields = options.fields;
	if (fields) {

		// Fields must be an array, or a dictionary (aka object)
		if (typeof fields !== 'object') {
			throw new DareError(DareError.INVALID_REFERENCE, `The field definition '${fields}' is invalid.`);
		}

		// Make the fields an array
		if (!Array.isArray(fields)) {
			fields = [fields];
		}

		// Filter out child fields
		fields = fields.reduce(fieldReducer.call(this, options.field_alias_path, joined, table_schema), []);
	}

	// Format conditional joins
	if (options.join) {

		const _join = {};
		const join = options.join;

		// filter must be an object with key=>values
		if (typeof join !== 'object') {
			throw new DareError(DareError.INVALID_REFERENCE, `The join '${join}' is invalid.`);
		}

		// Explore the filter for any table joins
		for (const key in join) {

			const value = join[key];

			if (value && typeof value === 'object' && !Array.isArray(value)) {

				// Check this is a path
				checkTableAlias(key);

				// Add it to the join table
				joined[key] = joined[key] || {};
				joined[key].join = Object.assign(joined[key].join || {}, value);
			}
			else {
				_join[key] = value;
			}
		}

		// Set the reduced condtions
		options.join = _join;
	}

	// Update the joined tables
	options.joined = joined;
	this.table_handler(options);
	delete options.joined;

	// Update the filters to be an array
	options._filter = filters.length ? filters : null;

	// Update the fields
	options.fields = fields;

	// Join
	if (options.join) {

		const _join = [];
		const join = options.join;

		for (let key in join) {

			const value = join[key];

			let negate = false;

			// Does this have a negate operator?
			if (key.substring(0, 1) === '-') {

				// Mark as negative filter
				negate = true;

				// Strip the key
				key = key.substring(1);
			}

			// Check this is a path
			checkKey(key);

			const type = table_schema[key] && table_schema[key].type;
			_join.push(prepCondition(key, value, type, negate));
		}
		options._join = _join;
	}

	// Groupby
	// If the content is grouped
	if (options.groupby) {
		// Explode the group formatter...
		checkFormat(options.groupby);
	}

	// Orderby
	// If the content is ordered
	if (options.orderby) {

		let a = options.orderby;

		if (typeof a === 'string') {
			a = a.split(/\s*,\s*/);
		}
		else if (!Array.isArray(a)) {
			a = [a];
		}

		a.forEach(def => {

			if (typeof def === 'string') {
				def = def.replace(/\s*(DESC|ASC)$/i, '');
			}

			// Check format
			checkFormat(def);
		});

		options.orderby = a;
	}

	// Set default limit
	limit(options);

	// Joins
	{
		const joins = options.joins || [];

		// Add additional joins which have been derived from nested fields and filters...
		for (const alias in joined) {

			// Furnish the join table a little more...
			const join_object = Object.assign(joined[alias], {
				alias,
				field_alias_path: `${options.field_alias_path + alias}.`
			});

			if (!join_object.table) {
				join_object.table = this.table_alias_handler(alias);
			}

			// Do the smart bit...
			// Augment the join object, with additional 'conditions'
			const new_join_object = this.join_handler(join_object, options.table);

			// Reject if the join handler returned a falsy value
			if (!new_join_object) {
				throw new DareError(DareError.INVALID_REFERENCE, `Could not understand field '${alias}'`);
			}

			// Help the GET parser

			// Does this contain a nested filter?
			join_object.has_filter = new_join_object.has_filter = !!join_object.filter;

			// Does this contain nested fields
			join_object.has_fields = new_join_object.has_fields = !!(Array.isArray(join_object.fields) ? join_object.fields.length : join_object.fields);

			// Update the request with this table join
			joins.push(new_join_object);
		}

		// Loop through the joins array
		if (joins.length) {
			// Loop through the joins and pass through the formatter
			return Promise.all(joins.map(join_object => format_request.call(this, join_object)))
				.then(a => {
					options._joins = a;
					return options;
				});
		}
	}

	return options;
}

function limit(opts) {

	if (opts.limit === undefined) {
		opts.limit = 1;
		opts.single = true;
	}

	else {
		let limit = opts.limit;
		if (typeof limit === 'string' && limit.match(/^\d+$/)) {
			limit = +opts.limit;
		}
		if (isNaN(limit) || limit > 10000 || limit < 1) {
			throw new DareError(DareError.INVALID_LIMIT, `Out of bounds limit value: '${limit}'`);
		}
	}

	let start = opts.start;

	if (start !== undefined) {
		if (typeof start === 'string' && start.match(/^\d+$/)) {
			start = +opts.start;
		}
		if (typeof start !== 'number' || isNaN(start) || start < 0) {
			throw new DareError(DareError.INVALID_START, `Out of bounds start value: '${start}'`);
		}
		opts.start = start;
	}
}

function prepCondition(field, value, type, negate) {

	if (type === 'datetime') {
		value = formatDateTime(value);
	}

	// Set the default negate operator, if appropriate
	negate = negate ? 'NOT ' : '';

	// Range
	// A range is denoted by two dots, e.g 1..10
	let condition;
	let values;
	const a = (typeof value === 'string') && value.split('..');

	if (a.length === 2) {

		if (a[0] && a[1]) {
			condition = 'BETWEEN ? AND ?';
			values = a;
		}
		else if (a[0]) {
			condition = '?? > ?';
			values = [a[0]];
		}
		else {
			condition = '?? < ?';
			values = [a[1]];
		}
	}

	// Not match
	else if (typeof value === 'string' && value[0] === '!') {
		condition = 'LIKE ?';
		values = [value.slice(1)];
		negate = 'NOT ';
	}

	// String partial match
	else if (typeof value === 'string' && value.match('%')) {
		condition = 'LIKE ?';
		values = [value];
	}

	// Null
	else if (value === null) {
		condition = `IS ${negate}NULL`;
		values = [];
		negate = ''; // already negated
	}

	// Add to the array of items
	else if (Array.isArray(value)) {
		condition = `IN (${value.map(() => '?')})`;
		values = value;
	}

	else {
		condition = '= ?';
		values = [value];
		negate = negate ? '!' : '';
	}

	return [field, negate + condition, values];
}
<commit_msg>Remove trailing space<commit_after>'use strict';

const DareError = require('./utils/error');
const fieldReducer = require('./utils/field_reducer');
const checkFormat = require('./utils/unwrap_field');
const checkKey = require('./utils/validate_field');
const checkTableAlias = require('./utils/validate_alias');
const formatDateTime = require('./utils/format_datetime');

module.exports = function(options) {
	return new Promise((accept, reject) => {
		const fn = format_request.call(this, options);
		if (fn && fn.then) {
			fn.then(accept, reject);
		}
		accept(fn);
	});
};

function format_request(options = {}) {

	// Use the alias to find the real table name
	if (!options.alias) {
		const alias = options.table;
		options.alias = alias;
		options.table = this.table_alias_handler(alias);
	}

	// Reject when the table is not recognised
	if (!options.table) {
		throw new DareError(DareError.INVALID_REFERENCE, `Unrecognized reference '${options.table}'`);
	}

	// Call bespoke table handler
	const method = this.options.method;
	const table = options.table;
	const handlers = this.options[method] || {};
	let handler;

	if (table in handlers) {
		handler = handlers[table];
	}
	else if ('default' in handlers) {
		handler = handlers.default;
	}
	if (handler) {
		const fn = handler.call(this, options);
		if (fn && fn.then) {
			return fn.then(format_specs.bind(this, options));
		}
	}

	return format_specs.call(this, options);
}

function format_specs(options) {

	const schema = this.options.schema || {};
	const table_schema = schema[options.table] || {};

	const joined = {};
	const filters = [];

	// Format filters
	{
		const filter = options.filter || {};

		// filter must be an object with key=>values
		if (typeof filter !== 'object') {
			throw new DareError(DareError.INVALID_REFERENCE, `The filter '${filter}' is invalid.`);
		}

		// Explore the filter for any table joins
		for (let key in filter) {

			const value = filter[key];

			if (value && typeof value === 'object' && !Array.isArray(value)) {

				// Check this is a path
				checkTableAlias(key);

				// Add it to the join table
				joined[key] = joined[key] || {};
				joined[key].filter = Object.assign(joined[key].filter || {}, value);
			}
			else {

				let negate = false;

				// Does this have a negate operator?
				if (key.substring(0, 1) === '-') {

					// Mark as negative filter
					negate = true;

					// Strip the key
					key = key.substring(1);
				}

				// Check this is a path
				checkKey(key);

				const type = table_schema[key] && table_schema[key].type;
				filters.push(prepCondition(key, value, type, negate));
			}
		}
	}

	// Set the prefix if not already
	options.field_alias_path = options.field_alias_path || '';

	// Format fields
	let fields = options.fields;
	if (fields) {

		// Fields must be an array, or a dictionary (aka object)
		if (typeof fields !== 'object') {
			throw new DareError(DareError.INVALID_REFERENCE, `The field definition '${fields}' is invalid.`);
		}

		// Make the fields an array
		if (!Array.isArray(fields)) {
			fields = [fields];
		}

		// Filter out child fields
		fields = fields.reduce(fieldReducer.call(this, options.field_alias_path, joined, table_schema), []);
	}

	// Format conditional joins
	if (options.join) {

		const _join = {};
		const join = options.join;

		// filter must be an object with key=>values
		if (typeof join !== 'object') {
			throw new DareError(DareError.INVALID_REFERENCE, `The join '${join}' is invalid.`);
		}

		// Explore the filter for any table joins
		for (const key in join) {

			const value = join[key];

			if (value && typeof value === 'object' && !Array.isArray(value)) {

				// Check this is a path
				checkTableAlias(key);

				// Add it to the join table
				joined[key] = joined[key] || {};
				joined[key].join = Object.assign(joined[key].join || {}, value);
			}
			else {
				_join[key] = value;
			}
		}

		// Set the reduced condtions
		options.join = _join;
	}

	// Update the joined tables
	options.joined = joined;
	this.table_handler(options);
	delete options.joined;

	// Update the filters to be an array
	options._filter = filters.length ? filters : null;

	// Update the fields
	options.fields = fields;

	// Join
	if (options.join) {

		const _join = [];
		const join = options.join;

		for (let key in join) {

			const value = join[key];

			let negate = false;

			// Does this have a negate operator?
			if (key.substring(0, 1) === '-') {

				// Mark as negative filter
				negate = true;

				// Strip the key
				key = key.substring(1);
			}

			// Check this is a path
			checkKey(key);

			const type = table_schema[key] && table_schema[key].type;
			_join.push(prepCondition(key, value, type, negate));
		}
		options._join = _join;
	}

	// Groupby
	// If the content is grouped
	if (options.groupby) {
		// Explode the group formatter...
		checkFormat(options.groupby);
	}

	// Orderby
	// If the content is ordered
	if (options.orderby) {

		let a = options.orderby;

		if (typeof a === 'string') {
			a = a.split(/\s*,\s*/);
		}
		else if (!Array.isArray(a)) {
			a = [a];
		}

		a.forEach(def => {

			if (typeof def === 'string') {
				def = def.replace(/\s*(DESC|ASC)$/i, '');
			}

			// Check format
			checkFormat(def);
		});

		options.orderby = a;
	}

	// Set default limit
	limit(options);

	// Joins
	{
		const joins = options.joins || [];

		// Add additional joins which have been derived from nested fields and filters...
		for (const alias in joined) {

			// Furnish the join table a little more...
			const join_object = Object.assign(joined[alias], {
				alias,
				field_alias_path: `${options.field_alias_path + alias}.`
			});

			if (!join_object.table) {
				join_object.table = this.table_alias_handler(alias);
			}

			// Do the smart bit...
			// Augment the join object, with additional 'conditions'
			const new_join_object = this.join_handler(join_object, options.table);

			// Reject if the join handler returned a falsy value
			if (!new_join_object) {
				throw new DareError(DareError.INVALID_REFERENCE, `Could not understand field '${alias}'`);
			}

			// Help the GET parser

			// Does this contain a nested filter?
			join_object.has_filter = new_join_object.has_filter = !!join_object.filter;

			// Does this contain nested fields
			join_object.has_fields = new_join_object.has_fields = !!(Array.isArray(join_object.fields) ? join_object.fields.length : join_object.fields);

			// Update the request with this table join
			joins.push(new_join_object);
		}

		// Loop through the joins array
		if (joins.length) {
			// Loop through the joins and pass through the formatter
			return Promise.all(joins.map(join_object => format_request.call(this, join_object)))
				.then(a => {
					options._joins = a;
					return options;
				});
		}
	}

	return options;
}

function limit(opts) {

	if (opts.limit === undefined) {
		opts.limit = 1;
		opts.single = true;
	}

	else {
		let limit = opts.limit;
		if (typeof limit === 'string' && limit.match(/^\d+$/)) {
			limit = +opts.limit;
		}
		if (isNaN(limit) || limit > 10000 || limit < 1) {
			throw new DareError(DareError.INVALID_LIMIT, `Out of bounds limit value: '${limit}'`);
		}
	}

	let start = opts.start;

	if (start !== undefined) {
		if (typeof start === 'string' && start.match(/^\d+$/)) {
			start = +opts.start;
		}
		if (typeof start !== 'number' || isNaN(start) || start < 0) {
			throw new DareError(DareError.INVALID_START, `Out of bounds start value: '${start}'`);
		}
		opts.start = start;
	}
}

function prepCondition(field, value, type, negate) {

	if (type === 'datetime') {
		value = formatDateTime(value);
	}

	// Set the default negate operator, if appropriate
	negate = negate ? 'NOT ' : '';

	// Range
	// A range is denoted by two dots, e.g 1..10
	let condition;
	let values;
	const a = (typeof value === 'string') && value.split('..');

	if (a.length === 2) {

		if (a[0] && a[1]) {
			condition = 'BETWEEN ? AND ?';
			values = a;
		}
		else if (a[0]) {
			condition = '?? > ?';
			values = [a[0]];
		}
		else {
			condition = '?? < ?';
			values = [a[1]];
		}
	}

	// Not match
	else if (typeof value === 'string' && value[0] === '!') {
		condition = 'LIKE ?';
		values = [value.slice(1)];
		negate = 'NOT ';
	}

	// String partial match
	else if (typeof value === 'string' && value.match('%')) {
		condition = 'LIKE ?';
		values = [value];
	}

	// Null
	else if (value === null) {
		condition = `IS ${negate}NULL`;
		values = [];
		negate = ''; // already negated
	}

	// Add to the array of items
	else if (Array.isArray(value)) {
		condition = `IN (${value.map(() => '?')})`;
		values = value;
	}

	else {
		condition = '= ?';
		values = [value];
		negate = negate ? '!' : '';
	}

	return [field, negate + condition, values];
}
<|endoftext|>


<commit_before>var db = require("./db");
var time = require("./time");
var util = require("./util");
var async = require("async");

/*
 * Object to hold all timer type handlers.
 */
var timer_type_handlers = {};

/*
 * Load all timers
 */
exports.load  = function(ret_handler){

	db.criteria("timers", {}, {etime:1}, function( err, arr_timers ){

		arr_timers.forEach(function(timer_data, index, array) {

	    	start_timer( timer_data );
		});

		if(ret_handler)
			ret_handler(arr_timers.length);
	});
}

/*
 * Add a timer type.
 */
exports.add_timer_type = function( type_name, timer_type_handler ){

	timer_type_handlers[type_name] = timer_type_handler;
}

/*
 * Add a new timer
 */
var start_timer = exports.start_timer = function( timer_data ){

	if( typeof timer_data != "object" || (!timer_data.etime || !timer_data.type_name) ){

		throw new Error("timer_data not correct, must contain: {etime,type_name}");
	}

	var interval = timer_data.etime - time.now();

	if(interval > 0){

		async.series(
		[
			function(next){

				timer_data._id?
					next():
					db.save("timers", timer_data, function(err,val){

						if(err)
							throw err;
						else
							next();
					});
			},
			function(next){

				setTimeout(function(){
					if(timer_type_handlers[timer_data.type_name ])
						timer_type_handlers[timer_data.type_name ]( timer_data.data );
					else
						console.log("warn: timer type '%s' not registered",timer_data.type_name);

					db.removeById("timers",""+timer_data._id, function(err){

						if(err)
							throw err;

					});
				},interval);

			}

		]);
	}else{

		if(timer_type_handlers[timer_data.type_name ])
			timer_type_handlers[timer_data.type_name ]( timer_data.data );
		else
			console.log("warn: timer type %s not registered",timer_data.type_name);
	}

}

<commit_msg>minor change to get easier test from test-timers.js.<commit_after>var db = require("./db");
var time = require("./time");
var util = require("./util");
var async = require("async");

/*
 * Object to hold all timer type handlers.
 */
var timer_type_handlers = {};

/*
 * Load all timers
 */
exports.load  = function(ret_handler){

	db.criteria("timers", {}, {etime:1}, function( err, arr_timers ){

		arr_timers.forEach(function(timer_data, index, array) {

	    	exports.start_timer( timer_data );
		});

		if(ret_handler)
			ret_handler(arr_timers.length);
	});
}

/*
 * Add a timer type.
 */
exports.add_timer_type = function( type_name, timer_type_handler ){

	timer_type_handlers[type_name] = timer_type_handler;
}

/*
 * Add a new timer
 */
exports.start_timer = function( timer_data ){

	if( typeof timer_data != "object" || (!timer_data.etime || !timer_data.type_name) ){

		throw new Error("timer_data not correct, must contain: {etime,type_name}");
	}

	var interval = timer_data.etime - time.now();

	if(interval > 0){

		async.series(
		[
			function(next){

				timer_data._id?
					next():
					db.save("timers", timer_data, function(err,val){

						if(err)
							throw err;
						else
							next();
					});
			},
			function(next){

				setTimeout(function(){
					if(timer_type_handlers[timer_data.type_name ])
						timer_type_handlers[timer_data.type_name ]( timer_data.data );
					else
						console.log("warn: timer type '%s' not registered",timer_data.type_name);

					db.removeById("timers",""+timer_data._id, function(err){

						if(err)
							throw err;

					});
				},interval);

			}

		]);
	}else{

		if(timer_type_handlers[timer_data.type_name ])
			timer_type_handlers[timer_data.type_name ]( timer_data.data );
		else
			console.log("warn: timer type %s not registered",timer_data.type_name);
	}

}

<|endoftext|>


<commit_before>'use strict';

var googleapis = require('googleapis');
var rarity = require('rarity');
var async = require('async');

var config = require('../config/configuration.js');
var retrieveEvents = require('./helpers/retrieve.js');

module.exports = function updateAccount(serviceData, cursor, queues, cb) {
  // Retrieve all events since last call
  var newCursor = new Date();

  async.waterfall([
    function refreshTokens(cb) {
      var oauth2Client = new googleapis.auth.OAuth2(config.googleId, config.googleSecret, serviceData.callbackUrl);
      oauth2Client.refreshToken_(serviceData.tokens.refresh_token, rarity.carryAndSlice([oauth2Client], 3, cb));
    },
    function callRetrieveEvents(oauth2Client, tokens, cb) {
      if(typeof tokens !== 'object' || !tokens.access_token) {
        return cb(new Error("Can't refresh tokens"));
      }

      tokens.refresh_token = serviceData.tokens.refresh_token;
      serviceData.tokens = tokens;
      oauth2Client.credentials = tokens;
      retrieveEvents(oauth2Client, cursor, cb);
    },
    function handleEvents(events, cb) {
      // Handle events and identifier => htmlLink
      events.forEach(function(event) {
        event.identifier = event.htmlLink;

        if(event.status === 'cancelled') {
          // On first run, we can safely skip the DELETION as there is no documents on API to be removed
          if(cursor) {
            queues.deletion.push(event);
          }
        }
        else {
          queues.addition.push(event);
        }
      });

      cb(null, newCursor, serviceData);
    }
  ], cb);
};
<commit_msg>Send TokenError when we need a refresh<commit_after>'use strict';

var googleapis = require('googleapis');
var rarity = require('rarity');
var async = require('async');
var TokenError = require('anyfetch-provider').TokenError;

var config = require('../config/configuration.js');
var retrieveEvents = require('./helpers/retrieve.js');

module.exports = function updateAccount(serviceData, cursor, queues, cb) {
  // Retrieve all events since last call
  var newCursor = new Date();

  async.waterfall([
    function refreshTokens(cb) {
      var oauth2Client = new googleapis.auth.OAuth2(config.googleId, config.googleSecret, serviceData.callbackUrl);
      oauth2Client.refreshToken_(serviceData.tokens.refresh_token, function(err, tokens) {
        if(err) {
          if(err.toString().match(/token/i)) {
            return cb(new TokenError());
          }
          return cb(err);
        }

        if(typeof tokens !== 'object' || !tokens.access_token) {
          return cb(new TokenError());
        }

        tokens.refresh_token = serviceData.tokens.refresh_token;
        serviceData.tokens = tokens;
        oauth2Client.credentials = tokens;
        retrieveEvents(oauth2Client, cursor, cb);
      });
    },
    function handleEvents(events, cb) {
      // Handle events and identifier => htmlLink
      events.forEach(function(event) {
        event.identifier = event.htmlLink;

        if(event.status === 'cancelled') {
          // On first run, we can safely skip the DELETION as there is no documents on API to be removed
          if(cursor) {
            queues.deletion.push(event);
          }
        }
        else {
          queues.addition.push(event);
        }
      });

      cb(null, newCursor, serviceData);
    }
  ], cb);
};
<|endoftext|>


<commit_before>/**
 * @fileOverview Defines client side API in which files can be uploaded.
 */

Slingshot = {};

/**
 *
 * @param {string} directive - Name of server-directive to use.
 * @param {object} [metaData] - Data to be sent to directive.
 * @constructor
 */

Slingshot.Upload = function (directive, metaData) {

  if (!window.File || !window.FormData) {
    throw new Error("Browser does not support HTML5 uploads");
  }

  var self = this,
      loaded = new ReactiveVar(),
      total = new ReactiveVar(),
      status = new ReactiveVar("idle"),
      dataUri = new ReactiveVar();

  function buildFormData() {
    var formData = new window.FormData();

    _.each(self.instructions.postData, function (field) {
      formData.append(field.name, field.value);
    });

    formData.append("file", self.file);

    return formData;
  }

  _.extend(self, {

    /**
     * @returns {string}
     */

    status: function () {
      return status.get();
    },

    /**
     * @returns {number}
     */

    progress: function () {
      return self.uploaded() / total.get();
    },

    /**
     * @returns {number}
     */

    uploaded: function () {
      return loaded.get();
    },

    /**
     * @param {File} file
     * @param {Function} [callback]
     * @returns {Slingshot.Upload}
     */

    send: function (file, callback) {
      check(file, window.File);

      self.file = file;

      self.request(file, function (error, instructions) {
        if (error) {
          return callback(error);
        }

        self.instructions = instructions;

        self.transfer(file, callback);
      });

      return self;
    },

    /**
     * @param {Function} [callback]
     * @returns {Slingshot.Upload}
     */

    request: function (callback) {
      status.set("authorizing");
      Meteor.call("slingshot/uploadRequest", directive, metaData,
        _.pick(self.file, "name", "size", "type"), function (error,
                                                             instructions) {
          status.set(error ? "failed" : "authorized");

          callback(error, instructions);
        });

      return self;
    },

    /**
     * @param {Function} [callback]
     *
     * @returns {Slingshot.Upload}
     */

    transfer: function (callback) {
      if (status.curValue !== "authorized") {
        throw new Error("Cannot transfer file at upload status: " +
          status.curValue);
      }

      status.set("transferring");
      loaded.set(0);

      var xhr = new XMLHttpRequest();

      xhr.upload.addEventListener("progress", function (event) {
        if (event.lengthComputable) {
          loaded.set(event.loaded);
          total.set(event.total);
        }
      }, false);

      xhr.addEventListener("load", function () {
        status.set("done");
        loaded.set(total.get());
        callback(null, self.instructions.download);
      });

      xhr.addEventListener("error", function () {
        status.set("failed");
        callback(new Error("Upload failed - see console logs"));
      });

      xhr.addEventListener("abort", function () {
        status.set("aborted");
        callback(new Error("The upload has been cancelled"));
      });

      xhr.open("POST", self.instructions.upload, true);
      xhr.send(buildFormData());

      return self;
    },

    /**
     * Latency compensated url of the file to be uploaded.
     *
     * @returns {string}
     */

    url: function () {
      var status = self.status();

      if (self.instructions && status === "done") {
        return self.instructions.download;
      }

      var url = dataUri.get();

      if (url)
        return url;

      if (Tracker.active && self.file && status !== "failed" &&
        status !== "aborted" && window.FileReader) {
          var reader = new window.FileReader();

          reader.onloadend = function () {
            dataUri.set(reader.result);
          };

          reader.readAsDataURL(self.file);
      }
    }
  });
};
<commit_msg>Fixed memory leak for latency compensated url reading.<commit_after>/**
 * @fileOverview Defines client side API in which files can be uploaded.
 */

Slingshot = {};

/**
 *
 * @param {string} directive - Name of server-directive to use.
 * @param {object} [metaData] - Data to be sent to directive.
 * @constructor
 */

Slingshot.Upload = function (directive, metaData) {

  if (!window.File || !window.FormData) {
    throw new Error("Browser does not support HTML5 uploads");
  }

  var self = this,
      loaded = new ReactiveVar(),
      total = new ReactiveVar(),
      status = new ReactiveVar("idle"),
      dataUri = new ReactiveVar();

  function buildFormData() {
    var formData = new window.FormData();

    _.each(self.instructions.postData, function (field) {
      formData.append(field.name, field.value);
    });

    formData.append("file", self.file);

    return formData;
  }

  _.extend(self, {

    /**
     * @returns {string}
     */

    status: function () {
      return status.get();
    },

    /**
     * @returns {number}
     */

    progress: function () {
      return self.uploaded() / total.get();
    },

    /**
     * @returns {number}
     */

    uploaded: function () {
      return loaded.get();
    },

    /**
     * @param {File} file
     * @param {Function} [callback]
     * @returns {Slingshot.Upload}
     */

    send: function (file, callback) {
      check(file, window.File);

      self.file = file;

      self.request(file, function (error, instructions) {
        if (error) {
          return callback(error);
        }

        self.instructions = instructions;

        self.transfer(file, callback);
      });

      return self;
    },

    /**
     * @param {Function} [callback]
     * @returns {Slingshot.Upload}
     */

    request: function (callback) {
      status.set("authorizing");
      Meteor.call("slingshot/uploadRequest", directive, metaData,
        _.pick(self.file, "name", "size", "type"), function (error,
                                                             instructions) {
          status.set(error ? "failed" : "authorized");

          callback(error, instructions);
        });

      return self;
    },

    /**
     * @param {Function} [callback]
     *
     * @returns {Slingshot.Upload}
     */

    transfer: function (callback) {
      if (status.curValue !== "authorized") {
        throw new Error("Cannot transfer file at upload status: " +
          status.curValue);
      }

      status.set("transferring");
      loaded.set(0);

      var xhr = new XMLHttpRequest();

      xhr.upload.addEventListener("progress", function (event) {
        if (event.lengthComputable) {
          loaded.set(event.loaded);
          total.set(event.total);
        }
      }, false);

      xhr.addEventListener("load", function () {
        status.set("done");
        loaded.set(total.get());
        callback(null, self.instructions.download);
      });

      xhr.addEventListener("error", function () {
        status.set("failed");
        callback(new Error("Upload failed - see console logs"));
      });

      xhr.addEventListener("abort", function () {
        status.set("aborted");
        callback(new Error("The upload has been cancelled"));
      });

      xhr.open("POST", self.instructions.upload, true);
      xhr.send(buildFormData());

      return self;
    },

    /**
     * Latency compensated url of the file to be uploaded.
     *
     * @returns {string}
     */

    url: function () {
      var status = self.status();

      if (self.instructions && status === "done") {
        dataUri.set();
        return self.instructions.download;
      }

      var url = dataUri.get();

      if (url)
        return url;

      if (Tracker.active && self.file && status !== "failed" &&
        status !== "aborted" && window.FileReader) {
          var reader = new window.FileReader();

          reader.onloadend = function () {
            dataUri.set(reader.result);
          };

          reader.readAsDataURL(self.file);
      }
    }
  });
};
<|endoftext|>


<commit_before>'use strict';

const { fork, spawn } = require('child_process');
const path = require('path');
const trimNewlines = require('trim-newlines');
const debugLog = require('./debugLog');
const { createUniqueId } = require('./utils');

const handlerCache = {};
const messageCallbacks = {};

function runProxyHandler(funOptions, options) {
  return (event, context) => {
    const args = ['invoke', 'local', '-f', funOptions.funName];
    const stage = options.s || options.stage;

    if (stage) args.push('-s', stage);

    // Use path to binary if provided, otherwise assume globally-installed
    const binPath = options.b || options.binPath;
    const cmd = binPath || 'sls';

    const process = spawn(cmd, args, {
      cwd: funOptions.servicePath,
      shell: true,
      stdio: ['pipe', 'pipe', 'pipe'],
    });

    process.stdin.write(`${JSON.stringify(event)}\n`);
    process.stdin.end();

    let results = '';
    let hasDetectedJson = false;

    process.stdin.on('data', (data) => {
      let str = data.toString('utf8');

      if (hasDetectedJson) {
        // Assumes that all data after matching the start of the
        // JSON result is the rest of the context result.
        results += trimNewlines(str);
      } else {
        // Search for the start of the JSON result
        // https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-output-format
        const match = /{[\r\n]?\s*"isBase64Encoded"|{[\r\n]?\s*"statusCode"|{[\r\n]?\s*"headers"|{[\r\n]?\s*"body"|{[\r\n]?\s*"principalId"/.exec(
          str,
        );
        if (match && match.index > -1) {
          // The JSON result was in this chunk so slice it out
          hasDetectedJson = true;
          results += trimNewlines(str.slice(match.index));
          str = str.slice(0, match.index);
        }

        if (str.length > 0) {
          // The data does not look like JSON and we have not
          // detected the start of JSON, so write the
          // output to the console instead.
          console.log('Proxy Handler could not detect JSON:', str);
        }
      }
    });

    process.stderr.on('data', (data) => {
      context.fail(data);
    });

    process.on('close', (code) => {
      if (code.toString() === '0') {
        try {
          context.succeed(JSON.parse(results));
        } catch (ex) {
          context.fail(results);
        }
      } else {
        context.succeed(code, results);
      }
    });
  };
}

exports.getFunctionOptions = function getFunctionOptions(
  fun,
  funName,
  servicePath,
  serviceRuntime,
) {
  // Split handler into method name and path i.e. handler.run
  // Support nested paths i.e. ./src/somefolder/.handlers/handler.run
  const lastIndexOfDelimiter = fun.handler.lastIndexOf('.');
  const handlerPath = fun.handler.substr(0, lastIndexOfDelimiter);
  const handlerName = fun.handler.substr(lastIndexOfDelimiter + 1);

  return {
    funName,
    funTimeout: (fun.timeout || 30) * 1000,
    handlerName, // i.e. run
    handlerPath: path.join(servicePath, handlerPath),
    memorySize: fun.memorySize,
    runtime: fun.runtime || serviceRuntime,
  };
};

exports.createExternalHandler = function createExternalHandler(
  funOptions,
  options,
) {
  let handlerContext = handlerCache[funOptions.handlerPath];

  function handleFatal(error) {
    debugLog(`External handler received fatal error ${JSON.stringify(error)}`);
    handlerContext.inflight.forEach((id) => messageCallbacks[id](error));
    handlerContext.inflight.clear();
    delete handlerCache[funOptions.handlerPath];
  }

  if (!handlerContext) {
    debugLog(`Loading external handler... (${funOptions.handlerPath})`);

    const helperPath = path.resolve(__dirname, 'ipcHelper.js');
    const env = {};
    for (const key of Object.getOwnPropertyNames(process.env)) {
      if (process.env[key] !== undefined && process.env[key] !== 'undefined')
        env[key] = process.env[key];
    }

    const ipcProcess = fork(helperPath, [funOptions.handlerPath], {
      env,
      stdio: [0, 1, 2, 'ipc'],
    });

    handlerContext = { process: ipcProcess, inflight: new Set() };

    if (options.skipCacheInvalidation) {
      handlerCache[funOptions.handlerPath] = handlerContext;
    }

    ipcProcess.on('message', (message) => {
      debugLog(`External handler received message ${JSON.stringify(message)}`);
      if (message.id && messageCallbacks[message.id]) {
        messageCallbacks[message.id](message.error, message.ret);
        handlerContext.inflight.delete(message.id);
        delete messageCallbacks[message.id];
      } else if (message.error) {
        // Handler died!
        handleFatal(message.error);
      }

      if (!options.skipCacheInvalidation) {
        handlerContext.process.kill();
        delete handlerCache[funOptions.handlerPath];
      }
    });

    ipcProcess.on('error', (error) => handleFatal(error));
    ipcProcess.on('exit', (code) =>
      handleFatal(`Handler process exited with code ${code}`),
    );
  } else {
    debugLog(`Using existing external handler for ${funOptions.handlerPath}`);
  }

  return (event, context, done) => {
    const id = createUniqueId();
    messageCallbacks[id] = done;
    handlerContext.inflight.add(id);
    handlerContext.process.send(
      Object.assign({}, funOptions, { id, event, context }),
    );
  };
};

// Create a function handler
// The function handler is used to simulate Lambda functions
exports.createHandler = function createHandler(funOptions, options) {
  if (options.useSeparateProcesses) {
    return this.createExternalHandler(funOptions, options);
  }

  if (!options.skipCacheInvalidation) {
    debugLog('Invalidating cache...');

    for (const key in require.cache) {
      // Require cache invalidation, brutal and fragile.
      // Might cause errors, if so please submit an issue.
      if (!key.match(options.cacheInvalidationRegex || /node_modules/))
        delete require.cache[key];
    }
    const currentFilePath = __filename;
    if (
      require.cache[currentFilePath] &&
      require.cache[currentFilePath].children
    ) {
      const nextChildren = [];

      require.cache[currentFilePath].children.forEach((moduleCache) => {
        if (
          moduleCache.filename.match(
            options.cacheInvalidationRegex || /node_modules/,
          )
        ) {
          nextChildren.push(moduleCache);
        }
      });

      require.cache[currentFilePath].children = nextChildren;
    }
  }

  debugLog(`Loading handler... (${funOptions.handlerPath})`);

  let handler = null;

  if (funOptions.runtime.startsWith('nodejs')) {
    handler = require(funOptions.handlerPath)[funOptions.handlerName];
  } else {
    handler = runProxyHandler(funOptions, options);
  }

  if (typeof handler !== 'function') {
    throw new Error(
      `Serverless-offline: handler for '${funOptions.funName}' is not a function`,
    );
  }

  return handler;
};

exports.cleanup = function cleanup() {
  for (const key in handlerCache) {
    handlerCache[key].process.kill();
  }
};
<commit_msg>Revert e8306dd, fixes #723, fixes #728, fixes #730<commit_after>'use strict';

const { fork, spawn } = require('child_process');
const path = require('path');
const trimNewlines = require('trim-newlines');
const debugLog = require('./debugLog');
const { createUniqueId } = require('./utils');

const handlerCache = {};
const messageCallbacks = {};

function runProxyHandler(funOptions, options) {
  return (event, context) => {
    const args = ['invoke', 'local', '-f', funOptions.funName];
    const stage = options.s || options.stage;

    if (stage) args.push('-s', stage);

    // Use path to binary if provided, otherwise assume globally-installed
    const binPath = options.b || options.binPath;
    const cmd = binPath || 'sls';

    const process = spawn(cmd, args, {
      cwd: funOptions.servicePath,
      shell: true,
      stdio: ['pipe', 'pipe', 'pipe'],
    });

    process.stdin.write(`${JSON.stringify(event)}\n`);
    process.stdin.end();

    let results = '';
    let hasDetectedJson = false;

    process.stdout.on('data', (data) => {
      let str = data.toString('utf8');

      if (hasDetectedJson) {
        // Assumes that all data after matching the start of the
        // JSON result is the rest of the context result.
        results += trimNewlines(str);
      } else {
        // Search for the start of the JSON result
        // https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-output-format
        const match = /{[\r\n]?\s*"isBase64Encoded"|{[\r\n]?\s*"statusCode"|{[\r\n]?\s*"headers"|{[\r\n]?\s*"body"|{[\r\n]?\s*"principalId"/.exec(
          str,
        );
        if (match && match.index > -1) {
          // The JSON result was in this chunk so slice it out
          hasDetectedJson = true;
          results += trimNewlines(str.slice(match.index));
          str = str.slice(0, match.index);
        }

        if (str.length > 0) {
          // The data does not look like JSON and we have not
          // detected the start of JSON, so write the
          // output to the console instead.
          console.log('Proxy Handler could not detect JSON:', str);
        }
      }
    });

    process.stderr.on('data', (data) => {
      context.fail(data);
    });

    process.on('close', (code) => {
      if (code.toString() === '0') {
        try {
          context.succeed(JSON.parse(results));
        } catch (ex) {
          context.fail(results);
        }
      } else {
        context.succeed(code, results);
      }
    });
  };
}

exports.getFunctionOptions = function getFunctionOptions(
  fun,
  funName,
  servicePath,
  serviceRuntime,
) {
  // Split handler into method name and path i.e. handler.run
  // Support nested paths i.e. ./src/somefolder/.handlers/handler.run
  const lastIndexOfDelimiter = fun.handler.lastIndexOf('.');
  const handlerPath = fun.handler.substr(0, lastIndexOfDelimiter);
  const handlerName = fun.handler.substr(lastIndexOfDelimiter + 1);

  return {
    funName,
    funTimeout: (fun.timeout || 30) * 1000,
    handlerName, // i.e. run
    handlerPath: path.join(servicePath, handlerPath),
    memorySize: fun.memorySize,
    runtime: fun.runtime || serviceRuntime,
  };
};

exports.createExternalHandler = function createExternalHandler(
  funOptions,
  options,
) {
  let handlerContext = handlerCache[funOptions.handlerPath];

  function handleFatal(error) {
    debugLog(`External handler received fatal error ${JSON.stringify(error)}`);
    handlerContext.inflight.forEach((id) => messageCallbacks[id](error));
    handlerContext.inflight.clear();
    delete handlerCache[funOptions.handlerPath];
  }

  if (!handlerContext) {
    debugLog(`Loading external handler... (${funOptions.handlerPath})`);

    const helperPath = path.resolve(__dirname, 'ipcHelper.js');
    const env = {};
    for (const key of Object.getOwnPropertyNames(process.env)) {
      if (process.env[key] !== undefined && process.env[key] !== 'undefined')
        env[key] = process.env[key];
    }

    const ipcProcess = fork(helperPath, [funOptions.handlerPath], {
      env,
      stdio: [0, 1, 2, 'ipc'],
    });

    handlerContext = { process: ipcProcess, inflight: new Set() };

    if (options.skipCacheInvalidation) {
      handlerCache[funOptions.handlerPath] = handlerContext;
    }

    ipcProcess.on('message', (message) => {
      debugLog(`External handler received message ${JSON.stringify(message)}`);
      if (message.id && messageCallbacks[message.id]) {
        messageCallbacks[message.id](message.error, message.ret);
        handlerContext.inflight.delete(message.id);
        delete messageCallbacks[message.id];
      } else if (message.error) {
        // Handler died!
        handleFatal(message.error);
      }

      if (!options.skipCacheInvalidation) {
        handlerContext.process.kill();
        delete handlerCache[funOptions.handlerPath];
      }
    });

    ipcProcess.on('error', (error) => handleFatal(error));
    ipcProcess.on('exit', (code) =>
      handleFatal(`Handler process exited with code ${code}`),
    );
  } else {
    debugLog(`Using existing external handler for ${funOptions.handlerPath}`);
  }

  return (event, context, done) => {
    const id = createUniqueId();
    messageCallbacks[id] = done;
    handlerContext.inflight.add(id);
    handlerContext.process.send(
      Object.assign({}, funOptions, { id, event, context }),
    );
  };
};

// Create a function handler
// The function handler is used to simulate Lambda functions
exports.createHandler = function createHandler(funOptions, options) {
  if (options.useSeparateProcesses) {
    return this.createExternalHandler(funOptions, options);
  }

  if (!options.skipCacheInvalidation) {
    debugLog('Invalidating cache...');

    for (const key in require.cache) {
      // Require cache invalidation, brutal and fragile.
      // Might cause errors, if so please submit an issue.
      if (!key.match(options.cacheInvalidationRegex || /node_modules/))
        delete require.cache[key];
    }
    const currentFilePath = __filename;
    if (
      require.cache[currentFilePath] &&
      require.cache[currentFilePath].children
    ) {
      const nextChildren = [];

      require.cache[currentFilePath].children.forEach((moduleCache) => {
        if (
          moduleCache.filename.match(
            options.cacheInvalidationRegex || /node_modules/,
          )
        ) {
          nextChildren.push(moduleCache);
        }
      });

      require.cache[currentFilePath].children = nextChildren;
    }
  }

  debugLog(`Loading handler... (${funOptions.handlerPath})`);

  let handler = null;

  if (funOptions.runtime.startsWith('nodejs')) {
    handler = require(funOptions.handlerPath)[funOptions.handlerName];
  } else {
    handler = runProxyHandler(funOptions, options);
  }

  if (typeof handler !== 'function') {
    throw new Error(
      `Serverless-offline: handler for '${funOptions.funName}' is not a function`,
    );
  }

  return handler;
};

exports.cleanup = function cleanup() {
  for (const key in handlerCache) {
    handlerCache[key].process.kill();
  }
};
<|endoftext|>


<commit_before>(function() {
	if( typeof window.addEventListener === 'function' ) {
		Reveal.addEventListener("slidechanged", function (event) {
			if (event.previousSlide.querySelector('.gallery') || document.querySelector('.reveal > .gallery')) {
				Gallery.stop();
			}

			Gallery.start(event.currentSlide);
		});

		// during initial load
		if (Reveal.getCurrentSlide()) {
			Gallery.start(Reveal.getCurrentSlide());
		}
	}
})();<commit_msg>Fix plugin to pass right node in<commit_after>(function() {
	if( typeof window.addEventListener === 'function' ) {
		Reveal.addEventListener("slidechanged", function (event) {
			if (event.previousSlide.querySelector('.gallery') || document.querySelector('.reveal > .gallery')) {
				Gallery.stop();
			}

			var galleryNode = event.currentSlide.querySelector('.gallery');
			if (galleryNode) {
				Gallery.start(galleryNode);
			}

		});

		// during initial load
		if (Reveal.getCurrentSlide()) {
			var galleryNode = Reveal.getCurrentSlide().querySelector('.gallery');
			if (galleryNode) {
				Gallery.start(galleryNode);
			}
		}
	}
})();<|endoftext|>


<commit_before>var impromptu = require('../lib/impromptu')
var minimist = require('minimist')
var domain = require('domain').create()

var argv = minimist(process.argv.slice(2), {
  defaults: {
    logfile: true,
    foreground: false
  },
  alias: {
    h: 'help',
    v: 'version'
  }
})

impromptu.state.set('verbosity', argv.verbosity)

impromptu.log.defaultDestinations.server = argv.foreground
impromptu.log.defaultDestinations.file = argv.logfile

domain.on('error', function (err) {
  try {
    var killtimer = setTimeout(function () {
      process.exit()
    }, 5000)
    killtimer.unref()

    if (process.connected) {
      process.send({
        type: 'end',
        data: impromptu.fallback()
      })
    }

    impromptu.log.error('worker error', err)

    if (process.connected) {
      process.disconnect()
    } else {
      process.kill()
    }
  } catch (e) {
    process.kill()
  }
})

domain.run(function () {
  impromptu.load()
})

var parseEnv = function(printenvOutput) {
  var env = {}
  if (printenvOutput) {
    var pairs = printenvOutput.split(/(?:^|\n)([a-z0-9_]+)=/i)

    // Remove the first blank match.
    pairs.shift()

    for (var index = 0; index < pairs.length; index += 2) {
      var key = pairs[index]
      env[key] = pairs[index + 1]
    }
  }
  return env
}

process.on('message', function(message) {
  if (message.type === 'env') {
    domain.run(function () {
      buildPrompt(message.data)
    })
  } else if (message.type === 'test') {
    runTests()
  }
})

var buildPrompt = function(envString) {
  var env = parseEnv(envString)

  if (env.IMPROMPTU_SHELL) {
    impromptu.state.set('shell', env.IMPROMPTU_SHELL)
  }

  // Overload the environment.
  process.env = env

  // Update the current working directory.
  try {
    process.chdir(env.PWD)
  } catch (e) {}

  impromptu.build(function(err, results) {
    // Send back the generated prompt.
    // If no prompt is generated, we fall back to the environment's existing prompt.
    // As a result, by registering no prompt sections, Impromptu can be used strictly
    // for its background updating capabilities. If PS1 exists, falls back to "`pwd` $".
    process.send({
      type: 'end',
      data: results || env.PS1 || impromptu.fallback()
    })

    // Run the background update.
    // We synchronously perform the background update to optimize for speed of prompt
    // generation. Reusing the process allows us to conserve memory while the socket
    // server is idling.
    impromptu.state.set('refreshCache', true)

    // Rebuild the prompt to refresh the cache.
    impromptu.prompt.build(function(err, results) {
      process.exit()
    })

    // Set a time limit for background updates.
    // TODO: Add more robust process management/collection to the server.
    var killtimer = setTimeout(function() {
      process.exit()
    }, 60000)
    // But don't keep the process open just for that!
    killtimer.unref()
  })
}

// Tests
// FYI: This is ridiculous.
var runTests = function() {
  require('coffee-script')

  var path = require('path')
  var fs = require('fs')
  var Mocha = require('mocha')

  // Allow unlimited listeners for tests.
  process.setMaxListeners(0)

  var testDir = path.resolve(__dirname, '../test')
  var files = fs.readdirSync(testDir).filter(function(f) {
    return f.match(/\.js$/)
  })

  var mocha = new Mocha({
    reporter: 'spec'
  })

  for (var i = 0; i < files.length; i++) {
    var file = files[i]
    mocha.addFile(path.resolve(testDir, file))
  }

  mocha.run(function(failures) {
    var exitCode = failures === 0 ? 0 : 1

    process.send({
      type: 'shutdown',
      code: exitCode
    })
    process.exit(failures)
  })
}
<commit_msg>Properly log worker errors in the build method.<commit_after>var impromptu = require('../lib/impromptu')
var minimist = require('minimist')
var domain = require('domain').create()

var argv = minimist(process.argv.slice(2), {
  defaults: {
    logfile: true,
    foreground: false
  },
  alias: {
    h: 'help',
    v: 'version'
  }
})

impromptu.state.set('verbosity', argv.verbosity)

impromptu.log.defaultDestinations.server = argv.foreground
impromptu.log.defaultDestinations.file = argv.logfile

domain.on('error', function (err) {
  try {
    var killtimer = setTimeout(function () {
      process.exit()
    }, 5000)
    killtimer.unref()

    if (process.connected) {
      process.send({
        type: 'end',
        data: impromptu.fallback()
      })
    }

    impromptu.log.error('worker error', err)

    if (process.connected) {
      process.disconnect()
    } else {
      process.kill()
    }
  } catch (e) {
    process.kill()
  }
})

domain.run(function () {
  impromptu.load()
})

var parseEnv = function(printenvOutput) {
  var env = {}
  if (printenvOutput) {
    var pairs = printenvOutput.split(/(?:^|\n)([a-z0-9_]+)=/i)

    // Remove the first blank match.
    pairs.shift()

    for (var index = 0; index < pairs.length; index += 2) {
      var key = pairs[index]
      env[key] = pairs[index + 1]
    }
  }
  return env
}

process.on('message', function(message) {
  if (message.type === 'env') {
    domain.run(function () {
      buildPrompt(message.data)
    })
  } else if (message.type === 'test') {
    runTests()
  }
})

var buildPrompt = function(envString) {
  var env = parseEnv(envString)

  if (env.IMPROMPTU_SHELL) {
    impromptu.state.set('shell', env.IMPROMPTU_SHELL)
  }

  // Overload the environment.
  process.env = env

  // Update the current working directory.
  try {
    process.chdir(env.PWD)
  } catch (e) {}

  impromptu.build(function(err, results) {
    // Send back the generated prompt.
    // If no prompt is generated, we fall back to the environment's existing prompt.
    // As a result, by registering no prompt sections, Impromptu can be used strictly
    // for its background updating capabilities. If PS1 exists, falls back to "`pwd` $".
    process.send({
      type: 'end',
      data: results || env.PS1 || impromptu.fallback()
    })

    if (err) impromptu.log.error('worker error', err)

    // Run the background update.
    // We synchronously perform the background update to optimize for speed of prompt
    // generation. Reusing the process allows us to conserve memory while the socket
    // server is idling.
    impromptu.state.set('refreshCache', true)

    // Rebuild the prompt to refresh the cache.
    impromptu.prompt.build(function(err, results) {
      process.exit()
    })

    // Set a time limit for background updates.
    // TODO: Add more robust process management/collection to the server.
    var killtimer = setTimeout(function() {
      process.exit()
    }, 60000)
    // But don't keep the process open just for that!
    killtimer.unref()
  })
}

// Tests
// FYI: This is ridiculous.
var runTests = function() {
  require('coffee-script')

  var path = require('path')
  var fs = require('fs')
  var Mocha = require('mocha')

  // Allow unlimited listeners for tests.
  process.setMaxListeners(0)

  var testDir = path.resolve(__dirname, '../test')
  var files = fs.readdirSync(testDir).filter(function(f) {
    return f.match(/\.js$/)
  })

  var mocha = new Mocha({
    reporter: 'spec'
  })

  for (var i = 0; i < files.length; i++) {
    var file = files[i]
    mocha.addFile(path.resolve(testDir, file))
  }

  mocha.run(function(failures) {
    var exitCode = failures === 0 ? 0 : 1

    process.send({
      type: 'shutdown',
      code: exitCode
    })
    process.exit(failures)
  })
}
<|endoftext|>


<commit_before>const fs = require('fs');
const path = require('path');
const childProcess = require('child_process');

const DATA_TAG = 'WORKER_JSON_MESSAGE';
const DATA_TAG_REG = new RegExp(`<${DATA_TAG}>(.*?)</${DATA_TAG}>`);

const decode = content => {
    let match = String(content).match(DATA_TAG_REG);
    if (match) {
        return JSON.parse(match[1]);
    }
};

const encode = data => {
    let json = JSON.stringify(data);
    json = `<${DATA_TAG}>${json}</${DATA_TAG}>`;
    return json;
};

/**
 * 使用子进程运行命令
 * @param   {string}    command     命令
 * @param   {Object}    options     进程选项
 * @see https://nodejs.org/api/child_process.html#child_process_child_process_exec_command_options_callback
 */
const worker = (command, options = {}) => {

    let run = (resolve, reject) => {
        let child = childProcess.exec(command, options);
        let timer = null;
        let callback = (errors, data) => {
            clearTimeout(timer);
            if (errors) {
                reject(errors);
            } else {
                resolve(data);
            }
        };

        child.stdout.on('data', content => {
            let data = decode(content);
            if (data) {
                callback(null, data);
            } else {
                process.stdout.write(content);
            }
        });

        child.stderr.on('data', content => {
            process.stderr.write(content);
        });

        child.on('exit', code => {
            clearTimeout(timer);
            if (code === 0) {
                callback(null);
            } else {
                callback(new Error(`"${command}": child process exited with code ${code}.`));
            }
        });

        // 解决 node timeout 不会走异常流程的问题
        if (options.timeout) {
            timer = setTimeout(() => {
                reject(new Error(`"${command}": child process timeout.`));
                child.kill();
            }, options.timeout);
        }
    };

    return new Promise((...promiseParams) => {

        if (options.env && options.env.PATH) {
            // 将 node_modules/.bin 加入环境变量，优先使用本地模块执行命令
            let localBin = path.join(process.cwd(), 'node_modules', '.bin');
            if (options.env.PATH.includes(localBin)) {
                run(...promiseParams);
            } else {
                fs.access(localBin, errors => {
                    if (!errors) {
                        options.env.PATH = localBin + ':' + options.env.PATH;
                    }
                    run(...promiseParams);
                });
            }
        } else {
            run(...promiseParams);
        }
    });
};

/**
 * 向父进程发送消息
 * @param   {Object}    data    JSON 数据
 */
worker.send = data => {
    process.stdout.write(encode(data));
};

module.exports = worker;<commit_msg>程序返回结果立即杀进程<commit_after>const fs = require('fs');
const path = require('path');
const childProcess = require('child_process');

const DATA_TAG = 'WORKER_JSON_MESSAGE';
const DATA_TAG_REG = new RegExp(`<${DATA_TAG}>(.*?)</${DATA_TAG}>`);

const decode = content => {
    let match = String(content).match(DATA_TAG_REG);
    if (match) {
        return JSON.parse(match[1]);
    }
};

const encode = data => {
    let json = JSON.stringify(data);
    json = `<${DATA_TAG}>${json}</${DATA_TAG}>`;
    return json;
};

/**
 * 使用子进程运行命令
 * @param   {string}    command     命令
 * @param   {Object}    options     进程选项
 * @see https://nodejs.org/api/child_process.html#child_process_child_process_exec_command_options_callback
 */
const worker = (command, options = {}) => {

    let run = (resolve, reject) => {
        let child = childProcess.exec(command, options);
        let timer = null;
        let callback = (errors, data) => {
            clearTimeout(timer);
            if (errors) {
                reject(errors);
            } else {
                resolve(data);
            }
            child.kill();
        };

        child.stdout.on('data', content => {
            let data = decode(content);
            if (data) {
                callback(null, data);
            } else {
                process.stdout.write(content);
            }
        });

        child.stderr.on('data', content => {
            process.stderr.write(content);
        });

        child.on('exit', code => {
            if (code === 0) {
                callback(null);
            } else {
                callback(new Error(`"${command}": child process exited with code ${code}.`));
            }
        });

        // 解决 node timeout 不会走异常流程的问题
        if (options.timeout) {
            timer = setTimeout(() => {
                callback(new Error(`"${command}": child process timeout.`));
            }, options.timeout);
        }
    };

    return new Promise((...promiseParams) => {

        if (options.env && options.env.PATH) {
            // 将 node_modules/.bin 加入环境变量，优先使用本地模块执行命令
            let localBin = path.join(process.cwd(), 'node_modules', '.bin');
            if (options.env.PATH.includes(localBin)) {
                run(...promiseParams);
            } else {
                fs.access(localBin, errors => {
                    if (!errors) {
                        options.env.PATH = localBin + ':' + options.env.PATH;
                    }
                    run(...promiseParams);
                });
            }
        } else {
            run(...promiseParams);
        }
    });
};

/**
 * 向父进程发送消息
 * @param   {Object}    data    JSON 数据
 */
worker.send = data => {
    process.stdout.write(encode(data));
};

module.exports = worker;<|endoftext|>


<commit_before>'use strict';

const puppeteer = require('puppeteer');
const cheerio = require('cheerio');
const url = require('url');
const logger = require('../logger');

/**
 * @param {string} userAgent user agent
 * @param {object} puppeteer puppeteer options
 * @param {object} tbs extra options for TBS request parameter
 */
class GoogleScraper {
  constructor({
    userAgent = 'Mozilla/5.0 (X11; Linux i686; rv:64.0) Gecko/20100101 Firefox/64.0',
    scrollDelay = 500,
    puppeteer = {},
    tbs = {},
  } = {}) {
    this.userAgent = userAgent;
    this.scrollDelay = scrollDelay;
    this.puppeteerOptions = puppeteer;
    this.tbs = this._parseRequestParameters(tbs);
  }

  _parseRequestParameters(tbs) {
    if (tbs === undefined) {
      return '';
    }

    const options = Object.keys(tbs)
      .filter((key) => tbs[key])
      .map((key) => `${key}:${tbs[key]}`)
      .join(',');
    return encodeURIComponent(options);
  }

  async scrape(searchQuery, limit = 100) {
    if (searchQuery === undefined || searchQuery === '') {
      throw new Error('Invalid search query provided');
    }
    const query = `https://www.google.com/search?q=${searchQuery}&source=lnms&tbm=isch&sa=X&tbs=${this.tbs}`;

    logger.info(`Start Google search for "${searchQuery}"`);
    const browser = await puppeteer.launch({
      ...this.puppeteerOptions,
    });
    const page = await browser.newPage();
    await page.setBypassCSP(true);
    await page.goto(query, {
      waitUntil: 'networkidle0',
    });
    await page.setViewport({ width: 1920, height: 1080 });
    await page.setUserAgent(this.userAgent);

    let results = [];
    let previousCount = -1;
    while (results.length < limit) {
      await this._scrollToEnd(page);
      await this._clickAllImages(page);
      await page
        .waitForTimeout("#islrg a[href^='/imgres']", { timeout: 1000 }) // Wait for the selector to appear in page.
        .catch(() => logger.debug('No results on this page')); // Unblock the flow

      const html = await page.content();
      const links = this._parseLinksFromHTML(html);
      previousCount = results.length;
      results = links.slice(0, limit);
      if (previousCount === results.length) {
        logger.debug('End of the page is reached');
        break;
      }

      logger.debug(`Got ${results.length} results so far`);
    }

    await browser.close();
    return results;
  }

  /**
   * Scroll to the end of the page.
   * @param {page} Puppeteer page to scroll
   */
  async _scrollToEnd(page) {
    logger.debug('Scrolling to the end of the page');

    const isScrollable = await this._isScrollable(page);
    if (!isScrollable) {
      logger.debug('No results on this page');
      return;
    }

    const buttonIsVisible = await this._isButtonVisible(page);
    await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
    logger.debug(`Scrolled to bottom of the page`);

    if (buttonIsVisible) {
      await page.click("#islmp input[type='button']");
      logger.debug('Clicked on show more results');
    }

    await page.waitForTimeout(this.scrollDelay);
    await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
  }

  _isScrollable(page) {
    return page.evaluate(() => {
      return document.querySelector("#islmp input[type='button']") !== null;
    });
  }

  _isButtonVisible(page) {
    return page.evaluate(() => {
      function isVisible(e) {
        return !!(e.offsetWidth || e.offsetHeight || e.getClientRects().length);
      }
      return isVisible(document.querySelector("#islmp input[type='button']"));
    });
  }

  _getInfiniteScrollStatus(page) {
    return page.evaluate(() => {
      let status = document.querySelector('#islmp div[data-endedmessage] > div:last-child')
        .innerText;
      return status;
    });
  }

  async _clickAllImages(page) {
    logger.debug('Scrolling to the end of the page');
    return page.evaluate(() => {
      let elements = document.querySelectorAll('#islrg img');

      function rightClick(element) {
        return new Promise((resolve) => {
          let event = new MouseEvent('mousedown', {
            bubbles: true,
            cancelable: false,
            view: window,
            button: 2,
            buttons: 2,
            clientX: element.getBoundingClientRect().x,
            clientY: element.getBoundingClientRect().y,
          });
          element.dispatchEvent(event);
          resolve();
        });
      }

      async function rightClickAll(elements) {
        for (const element of elements) {
          await rightClick(element);
        }
      }
      rightClickAll(elements);
    });
  }

  _parseLinksFromHTML(html) {
    let links = [];

    let $ = cheerio.load(html);

    $("#islrg a[href^='/imgres']").each(function (i, elem) {
      let link = $(this).attr('href');
      let parsedLink = url.parse(link, { parseQueryString: true });
      let imageurl = parsedLink.query.imgurl;
      let source = parsedLink.query.imgrefurl;
      links.push({ url: imageurl, source });
    });

    return links;
  }
}

module.exports = GoogleScraper;
<commit_msg>Added support for safe search on google (no NSFW content)<commit_after>'use strict';

const puppeteer = require('puppeteer');
const cheerio = require('cheerio');
const url = require('url');
const logger = require('../logger');

/**
 * @param {string} userAgent user agent
 * @param {object} puppeteer puppeteer options
 * @param {object} tbs extra options for TBS request parameter
 */
class GoogleScraper {
  constructor({
    userAgent = 'Mozilla/5.0 (X11; Linux i686; rv:64.0) Gecko/20100101 Firefox/64.0',
    scrollDelay = 500,
    puppeteer = {},
    tbs = {},
    safe = false,
  } = {}) {
    this.userAgent = userAgent;
    this.scrollDelay = scrollDelay;
    this.puppeteerOptions = puppeteer;
    this.tbs = this._parseRequestParameters(tbs);
    this.safe = this._isQuerySafe(safe);
  }

  _parseRequestParameters(tbs) {
    if (tbs === undefined) {
      return '';
    }

    const options = Object.keys(tbs)
      .filter((key) => tbs[key])
      .map((key) => `${key}:${tbs[key]}`)
      .join(',');
    return encodeURIComponent(options);
  }

  _isQuerySafe(safe) {
    if (safe === true) {
      return '&safe=active';
    } else return '';
  }

  async scrape(searchQuery, limit = 100) {
    if (searchQuery === undefined || searchQuery === '') {
      throw new Error('Invalid search query provided');
    }
    const query = `https://www.google.com/search?q=${searchQuery}&source=lnms&tbm=isch${this.safe}&sa=X&tbs=${this.tbs}`;

    logger.info(`Start Google search for "${searchQuery}"`);
    const browser = await puppeteer.launch({
      ...this.puppeteerOptions,
    });
    const page = await browser.newPage();
    await page.setBypassCSP(true);
    await page.goto(query, {
      waitUntil: 'networkidle0',
    });
    await page.setViewport({ width: 1920, height: 1080 });
    await page.setUserAgent(this.userAgent);

    let results = [];
    let previousCount = -1;
    while (results.length < limit) {
      await this._scrollToEnd(page);
      await this._clickAllImages(page);
      await page
        .waitForTimeout("#islrg a[href^='/imgres']", { timeout: 1000 }) // Wait for the selector to appear in page.
        .catch(() => logger.debug('No results on this page')); // Unblock the flow

      const html = await page.content();
      const links = this._parseLinksFromHTML(html);
      previousCount = results.length;
      results = links.slice(0, limit);
      if (previousCount === results.length) {
        logger.debug('End of the page is reached');
        break;
      }

      logger.debug(`Got ${results.length} results so far`);
    }

    await browser.close();
    return results;
  }

  /**
   * Scroll to the end of the page.
   * @param {page} Puppeteer page to scroll
   */
  async _scrollToEnd(page) {
    logger.debug('Scrolling to the end of the page');

    const isScrollable = await this._isScrollable(page);
    if (!isScrollable) {
      logger.debug('No results on this page');
      return;
    }

    const buttonIsVisible = await this._isButtonVisible(page);
    await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
    logger.debug(`Scrolled to bottom of the page`);

    if (buttonIsVisible) {
      await page.click("#islmp input[type='button']");
      logger.debug('Clicked on show more results');
    }

    await page.waitForTimeout(this.scrollDelay);
    await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
  }

  _isScrollable(page) {
    return page.evaluate(() => {
      return document.querySelector("#islmp input[type='button']") !== null;
    });
  }

  _isButtonVisible(page) {
    return page.evaluate(() => {
      function isVisible(e) {
        return !!(e.offsetWidth || e.offsetHeight || e.getClientRects().length);
      }
      return isVisible(document.querySelector("#islmp input[type='button']"));
    });
  }

  _getInfiniteScrollStatus(page) {
    return page.evaluate(() => {
      let status = document.querySelector('#islmp div[data-endedmessage] > div:last-child')
        .innerText;
      return status;
    });
  }

  async _clickAllImages(page) {
    logger.debug('Scrolling to the end of the page');
    return page.evaluate(() => {
      let elements = document.querySelectorAll('#islrg img');

      function rightClick(element) {
        return new Promise((resolve) => {
          let event = new MouseEvent('mousedown', {
            bubbles: true,
            cancelable: false,
            view: window,
            button: 2,
            buttons: 2,
            clientX: element.getBoundingClientRect().x,
            clientY: element.getBoundingClientRect().y,
          });
          element.dispatchEvent(event);
          resolve();
        });
      }

      async function rightClickAll(elements) {
        for (const element of elements) {
          await rightClick(element);
        }
      }
      rightClickAll(elements);
    });
  }

  _parseLinksFromHTML(html) {
    let links = [];

    let $ = cheerio.load(html);

    $("#islrg a[href^='/imgres']").each(function (i, elem) {
      let link = $(this).attr('href');
      let parsedLink = url.parse(link, { parseQueryString: true });
      let imageurl = parsedLink.query.imgurl;
      let source = parsedLink.query.imgrefurl;
      links.push({ url: imageurl, source });
    });

    return links;
  }
}

module.exports = GoogleScraper;
<|endoftext|>


<commit_before>'use strict';

var bunyan = require('bunyan');
var cluster = require('cluster');
var EventEmitter = require('events').EventEmitter;
var removeValue = require('./remove-value');
var prependListener = require('./prepend-listener');

// Exports

module.exports = {
    run: run,
    attach: attach,
    serverStatus: serverStatus,
    shutdown: shutdown
};

// Config

var config;

// State

var httpServer;
var isRunning = true;
var activeConnections = [];
var activeRequests = [];
var workerStatus = {
    status: null,
    startTime: new Date(),
    connections: 0,
    requests: 0,
    bytesRead: 0,
    bytesWritten: 0,
    pid: process.pid,
    generation: null,
    activeConnections: null
};

// Logging

var log = null;

// Server status listener

var serverStatusEvent = new EventEmitter();
serverStatusEvent.fetch = function () {
    if (! cluster.isMaster) process.send({event: 'serverStatus'});
};

/**
 * Run the cluster worker
 *
 * Options:
 * - shutdownTimeout: worker shutdown timeout (default: 30000)
 * - log: bunyan instance for logging
 *
 * @param {Object} opts
 * @api public
 */
function run(opts) {
    config = opts;

    if (opts.log) {
        log = opts.log.child({'component': 'worker'});
    } else {
        log = bunyan.createLogger({name: 'flora-cluster', 'component': 'worker'});
    }

    // Setup process events

    process.on('message', function onMessage(message) {
        if (!message.event) return;
        //console.log('worker/%d: message from master: %s', process.pid, message.event);

        process.emit('flora::' + message.event, message);
    });

    process.on('flora::shutdown', shutdown);

    process.on('flora::status', sendStatus);

    process.on('flora::serverStatus', function onServerStatus(message) {
        serverStatusEvent.emit('serverStatus', message.serverStatus);
    });

    // Setup signals

    process.on('SIGTERM', function onSigTerm() {
        log.info('received SIGTERM, exiting immediately');
        process.exit(0);
    });

    process.on('SIGINT', function onSigInt() {
        log.info('received SIGINT, exiting immediately');
        process.exit(0);
    });

    process.on('SIGQUIT', function onSigQuit() {
        log.info('received SIGQUIT, shutting down gracefully');
        shutdown();
    });

    // Handle uncaughtExceptions

    process.on('uncaughtException', function onUncaughtException(err) {
        log.error(err, 'uncaught Exception, shutting down gracefully');
        shutdown();
    });
}

/**
 * Attach to an exising HTTP server
 *
 * @param {http.Server} server
 * @api public
 */
function attach(server) {
    httpServer = server;
    prependListener(httpServer, 'request', onRequest);
    prependListener(httpServer, 'connection', onConnection);
    httpServer.on('close', onCloseServer);
    // TODO: httpServer.setTimeout(...)
}

/**
 * Shutdown the worker
 *
 * @api public
 */
function shutdown() {
    if (!isRunning) return;

    isRunning = false;

    if (!httpServer) {
        if (!cluster.isMaster) cluster.worker.disconnect();
        return;
    }

    try {
        // make sure we close down within given timeout:
        var shutdownTimeout = setTimeout(function onShutdownTimeout() {
            log.info('exit after timeout');
            process.exit(1);
        }, config.shutdownTimeout);

        // but don't keep the process open just for that:
        shutdownTimeout.unref();

        // notify master that we are shutting down (so new workers can be spawned immediately):
        if (!cluster.isMaster) process.send({event: 'shutdown'});

        // stop accepting new connections:
        try {
            httpServer.close();
        } catch (err) {}

        // close idle keep-alive connections:
        activeConnections.forEach(function (connection) {
            if (connection.flora.isIdle) {
                connection.end();
            }
        });

        sendStatus();
    } catch (e) {
        log.error(e, 'error on graceful shutdown');
    }
}

function onRequest(request, response) {
    workerStatus.requests++;
    request.connection.flora.requests++;
    request.connection.flora.isIdle = false;
    request.connection.flora.idleSince = null;

    request.flora = request.flora || {};
    request.flora.status = 'processing';
    request.flora.startTime = new Date();

    // remember connection-info for aborted requests:
    request.flora.connectionInfo = {
        remoteAddress: request.connection.remoteAddress,
        remotePort: request.connection.remotePort,
        localAddress: request.connection.localAddress,
        localPort: request.connection.localPort
    };

    activeRequests.push(request);

    // inject our "end" method to track "sending" status
    var _end = response.end;
    response.end = function () {
        if (request.flora.status === 'aborted') {
            removeValue(activeRequests, request);
        } else {
            request.flora.status = 'sending';
        }
        return _end.apply(this, arguments);
    };

    // TODO: Set different timeout for running requests than for idle keep-alive-connections

    response.on('finish', function onFinishRequest() {
        removeValue(activeRequests, request);

        request.connection.flora.isIdle = true;
        request.connection.flora.idleSince = new Date();
        request.flora.status = 'finished';

        updateStats(request.connection);

        if (!isRunning) {
            // close keep-alive-connection:
            request.connection.end();
        }
    });

    response.on('close', function onCloseRequest() {
        if (request.flora.status === 'sending') {
            removeValue(activeRequests, request);
        }

        request.flora.status = 'aborted';

        updateStats(request.connection);
    });
}

function onConnection(connection) {
    connection.setNoDelay(true);
    connection.setKeepAlive(true);

    workerStatus.connections++;
    connection.flora = connection.flora || {};
    connection.flora.startTime = new Date();
    connection.flora.requests = 0;
    connection.flora.isIdle = true;
    connection.flora.idleSince = connection.flora.startTime;

    activeConnections.push(connection);

    connection.on('close', function onCloseConnection() {
        updateStats(connection);
        removeValue(activeConnections, connection);
    });
}

function onCloseServer() {
    sendStatus();

    // if all references are closed properly, worker exits here,
    // otherwise the kill-timeout from shutdown() should catch it:
    if (!cluster.isMaster) cluster.worker.disconnect();
}

function updateStats(connection) {
    // bytesRead/bytesWritten is per connection, not per request (keep-alive!):
    workerStatus.bytesRead += connection.bytesRead -
        (connection.flora._previousBytesRead || 0);
    workerStatus.bytesWritten += connection.bytesWritten -
        (connection.flora._previousBytesWritten || 0);

    connection.flora._previousBytesRead = connection.bytesRead;
    connection.flora._previousBytesWritten = connection.bytesWritten;
}

function sendStatus() {
    var status = getStatus();

    if (! cluster.isMaster) {
        process.send({
            event: 'status',
            status: status
        });
    }
}

function getStatus() {
    var connections = [];

    activeRequests.forEach(function (request) {
        request.flora._isOrphaned = true;
    });

    activeConnections.forEach(function (connection) {
        var matchingRequest = null;
        activeRequests.forEach(function (request) {
            if (request.connection !== connection) return;

            request.flora._isOrphaned = false;

            matchingRequest = {
                status: request.flora.status,
                startTime: request.flora.startTime,
                method: request.method,
                url: request.url,
                httpVersion: request.httpVersion
            };
        });

        connections.push({
            status: 'connected',
            startTime: connection.flora.startTime,
            remoteAddress: connection.remoteAddress,
            remotePort: connection.remotePort,
            localAddress: connection.localAddress,
            localPort: connection.localPort,
            bytesRead: connection.bytesRead,
            bytesWritten: connection.bytesWritten,
            bufferSize: connection.bufferSize,
            isIdle: connection.flora.isIdle,
            idleSince: connection.flora.idleSince,
            requests: connection.flora.requests,
            request: matchingRequest
        });
    });

    activeRequests.forEach(function (request) {
        if (!request.flora._isOrphaned) return;

        connections.push({
            status: 'closed',
            startTime: request.connection.flora.startTime,
            remoteAddress: request.flora.connectionInfo.remoteAddress,
            remotePort: request.flora.connectionInfo.remotePort,
            localAddress: request.flora.connectionInfo.localAddress,
            localPort: request.flora.connectionInfo.localPort,
            bytesRead: request.connection.bytesRead,
            bytesWritten: request.connection.bytesWritten,
            isIdle: request.connection.flora.isIdle,
            idleSince: request.connection.flora.idleSince,
            requests: request.connection.flora.requests,
            request: {
                status: request.flora.status,
                startTime: request.flora.startTime,
                method: request.method,
                url: request.url,
                httpVersion: request.httpVersion
            }
        });
    });

    var status = {}, key;
    for (key in workerStatus) {
        status[key] = workerStatus[key];
    }

    status.status = isRunning ? 'running' : 'shutdown';
    status.activeConnections = connections;

    return status;
}

/**
 * Retrieve the cluster status
 *
 * @param {Function} callback
 * @api public
 */
function serverStatus(callback) {
    if (cluster.isMaster) {
        callback(null, getStatus());
        return;
    }
    serverStatusEvent.fetch();
    serverStatusEvent.once('serverStatus', function onServerStatus(serverStatus) {
        callback(null, serverStatus);
    });
}
<commit_msg>Whitespace<commit_after>'use strict';

var bunyan = require('bunyan');
var cluster = require('cluster');
var EventEmitter = require('events').EventEmitter;
var removeValue = require('./remove-value');
var prependListener = require('./prepend-listener');

// Exports

module.exports = {
    run: run,
    attach: attach,
    serverStatus: serverStatus,
    shutdown: shutdown
};

// Config

var config;

// State

var httpServer;
var isRunning = true;
var activeConnections = [];
var activeRequests = [];
var workerStatus = {
    status: null,
    startTime: new Date(),
    connections: 0,
    requests: 0,
    bytesRead: 0,
    bytesWritten: 0,
    pid: process.pid,
    generation: null,
    activeConnections: null
};

// Logging

var log = null;

// Server status listener

var serverStatusEvent = new EventEmitter();
serverStatusEvent.fetch = function () {
    if (!cluster.isMaster) process.send({event: 'serverStatus'});
};

/**
 * Run the cluster worker
 *
 * Options:
 * - shutdownTimeout: worker shutdown timeout (default: 30000)
 * - log: bunyan instance for logging
 *
 * @param {Object} opts
 * @api public
 */
function run(opts) {
    config = opts;

    if (opts.log) {
        log = opts.log.child({'component': 'worker'});
    } else {
        log = bunyan.createLogger({name: 'flora-cluster', 'component': 'worker'});
    }

    // Setup process events

    process.on('message', function onMessage(message) {
        if (!message.event) return;
        //console.log('worker/%d: message from master: %s', process.pid, message.event);

        process.emit('flora::' + message.event, message);
    });

    process.on('flora::shutdown', shutdown);

    process.on('flora::status', sendStatus);

    process.on('flora::serverStatus', function onServerStatus(message) {
        serverStatusEvent.emit('serverStatus', message.serverStatus);
    });

    // Setup signals

    process.on('SIGTERM', function onSigTerm() {
        log.info('received SIGTERM, exiting immediately');
        process.exit(0);
    });

    process.on('SIGINT', function onSigInt() {
        log.info('received SIGINT, exiting immediately');
        process.exit(0);
    });

    process.on('SIGQUIT', function onSigQuit() {
        log.info('received SIGQUIT, shutting down gracefully');
        shutdown();
    });

    // Handle uncaughtExceptions

    process.on('uncaughtException', function onUncaughtException(err) {
        log.error(err, 'uncaught Exception, shutting down gracefully');
        shutdown();
    });
}

/**
 * Attach to an exising HTTP server
 *
 * @param {http.Server} server
 * @api public
 */
function attach(server) {
    httpServer = server;
    prependListener(httpServer, 'request', onRequest);
    prependListener(httpServer, 'connection', onConnection);
    httpServer.on('close', onCloseServer);
    // TODO: httpServer.setTimeout(...)
}

/**
 * Shutdown the worker
 *
 * @api public
 */
function shutdown() {
    if (!isRunning) return;

    isRunning = false;

    if (!httpServer) {
        if (!cluster.isMaster) cluster.worker.disconnect();
        return;
    }

    try {
        // make sure we close down within given timeout:
        var shutdownTimeout = setTimeout(function onShutdownTimeout() {
            log.info('exit after timeout');
            process.exit(1);
        }, config.shutdownTimeout);

        // but don't keep the process open just for that:
        shutdownTimeout.unref();

        // notify master that we are shutting down (so new workers can be spawned immediately):
        if (!cluster.isMaster) process.send({event: 'shutdown'});

        // stop accepting new connections:
        try {
            httpServer.close();
        } catch (err) {}

        // close idle keep-alive connections:
        activeConnections.forEach(function (connection) {
            if (connection.flora.isIdle) {
                connection.end();
            }
        });

        sendStatus();
    } catch (e) {
        log.error(e, 'error on graceful shutdown');
    }
}

function onRequest(request, response) {
    workerStatus.requests++;
    request.connection.flora.requests++;
    request.connection.flora.isIdle = false;
    request.connection.flora.idleSince = null;

    request.flora = request.flora || {};
    request.flora.status = 'processing';
    request.flora.startTime = new Date();

    // remember connection-info for aborted requests:
    request.flora.connectionInfo = {
        remoteAddress: request.connection.remoteAddress,
        remotePort: request.connection.remotePort,
        localAddress: request.connection.localAddress,
        localPort: request.connection.localPort
    };

    activeRequests.push(request);

    // inject our "end" method to track "sending" status
    var _end = response.end;
    response.end = function () {
        if (request.flora.status === 'aborted') {
            removeValue(activeRequests, request);
        } else {
            request.flora.status = 'sending';
        }
        return _end.apply(this, arguments);
    };

    // TODO: Set different timeout for running requests than for idle keep-alive-connections

    response.on('finish', function onFinishRequest() {
        removeValue(activeRequests, request);

        request.connection.flora.isIdle = true;
        request.connection.flora.idleSince = new Date();
        request.flora.status = 'finished';

        updateStats(request.connection);

        if (!isRunning) {
            // close keep-alive-connection:
            request.connection.end();
        }
    });

    response.on('close', function onCloseRequest() {
        if (request.flora.status === 'sending') {
            removeValue(activeRequests, request);
        }

        request.flora.status = 'aborted';

        updateStats(request.connection);
    });
}

function onConnection(connection) {
    connection.setNoDelay(true);
    connection.setKeepAlive(true);

    workerStatus.connections++;
    connection.flora = connection.flora || {};
    connection.flora.startTime = new Date();
    connection.flora.requests = 0;
    connection.flora.isIdle = true;
    connection.flora.idleSince = connection.flora.startTime;

    activeConnections.push(connection);

    connection.on('close', function onCloseConnection() {
        updateStats(connection);
        removeValue(activeConnections, connection);
    });
}

function onCloseServer() {
    sendStatus();

    // if all references are closed properly, worker exits here,
    // otherwise the kill-timeout from shutdown() should catch it:
    if (!cluster.isMaster) cluster.worker.disconnect();
}

function updateStats(connection) {
    // bytesRead/bytesWritten is per connection, not per request (keep-alive!):
    workerStatus.bytesRead += connection.bytesRead -
        (connection.flora._previousBytesRead || 0);
    workerStatus.bytesWritten += connection.bytesWritten -
        (connection.flora._previousBytesWritten || 0);

    connection.flora._previousBytesRead = connection.bytesRead;
    connection.flora._previousBytesWritten = connection.bytesWritten;
}

function sendStatus() {
    var status = getStatus();

    if (!cluster.isMaster) {
        process.send({
            event: 'status',
            status: status
        });
    }
}

function getStatus() {
    var connections = [];

    activeRequests.forEach(function (request) {
        request.flora._isOrphaned = true;
    });

    activeConnections.forEach(function (connection) {
        var matchingRequest = null;
        activeRequests.forEach(function (request) {
            if (request.connection !== connection) return;

            request.flora._isOrphaned = false;

            matchingRequest = {
                status: request.flora.status,
                startTime: request.flora.startTime,
                method: request.method,
                url: request.url,
                httpVersion: request.httpVersion
            };
        });

        connections.push({
            status: 'connected',
            startTime: connection.flora.startTime,
            remoteAddress: connection.remoteAddress,
            remotePort: connection.remotePort,
            localAddress: connection.localAddress,
            localPort: connection.localPort,
            bytesRead: connection.bytesRead,
            bytesWritten: connection.bytesWritten,
            bufferSize: connection.bufferSize,
            isIdle: connection.flora.isIdle,
            idleSince: connection.flora.idleSince,
            requests: connection.flora.requests,
            request: matchingRequest
        });
    });

    activeRequests.forEach(function (request) {
        if (!request.flora._isOrphaned) return;

        connections.push({
            status: 'closed',
            startTime: request.connection.flora.startTime,
            remoteAddress: request.flora.connectionInfo.remoteAddress,
            remotePort: request.flora.connectionInfo.remotePort,
            localAddress: request.flora.connectionInfo.localAddress,
            localPort: request.flora.connectionInfo.localPort,
            bytesRead: request.connection.bytesRead,
            bytesWritten: request.connection.bytesWritten,
            isIdle: request.connection.flora.isIdle,
            idleSince: request.connection.flora.idleSince,
            requests: request.connection.flora.requests,
            request: {
                status: request.flora.status,
                startTime: request.flora.startTime,
                method: request.method,
                url: request.url,
                httpVersion: request.httpVersion
            }
        });
    });

    var status = {}, key;
    for (key in workerStatus) {
        status[key] = workerStatus[key];
    }

    status.status = isRunning ? 'running' : 'shutdown';
    status.activeConnections = connections;

    return status;
}

/**
 * Retrieve the cluster status
 *
 * @param {Function} callback
 * @api public
 */
function serverStatus(callback) {
    if (cluster.isMaster) {
        callback(null, getStatus());
        return;
    }
    serverStatusEvent.fetch();
    serverStatusEvent.once('serverStatus', function onServerStatus(serverStatus) {
        callback(null, serverStatus);
    });
}
<|endoftext|>