<commit_before>/*
 * Module dependencies.
 */

var request = require('./request'),
    Arguments = require('./arguments'),
    fs = require('fs'),
    p = require('path');

/**
 * API Request Constructor.
 *
 * Creating an instance of the API Request will enable the RESTful access
 * to a PhoneGap Build API.
 *
 * Options:
 *
 *   - `options` {Object} defines the API server that has been authenticated.
 *   - `api.token` {String} is the user token for authentication.
 *   - `api.protocol` {String} is the protocol, e.g. 'http' or 'https'.
 *   - `api.host` {String} is the host address, e.g. 'build.phonegap.com'.
 *   - `api.port` {String} is the port, e.g. '80' or '443'.
 *   - `api.path` {String} is the api path, e.g. '/api/v1'
 *   - [`api.path`] {String} is a proxy to forward requests, e.g. 'http://myproxy.com'
 */

module.exports = function(options) {
    // require options argument
    if (!options) throw new Error('missing options argument');
    if (!options.token) throw new Error('missing options.token argument');
    if (!options.protocol) throw new Error('missing options.protocol argument');
    if (!options.host) throw new Error('missing options.host argument');
    if (!options.port) throw new Error('missing options.port argument');
    if (!options.path) throw new Error('missing options.path argument');

    /**
     * API Request.
     *
     * Sends an HTTP request to the PhoneGap Build API.
     *
     * By default, the request method is `GET` but can be changed
     * in the `options` parameters (e.g. `{ method: 'POST' }`).
     *
     * Options:
     *
     *   - `path` {String} is a resource path (e.g. `"/apps"`).
     *   - `[options]` {Object} is a request options object.
     *   - `[callback]` {Function} is trigger after the request
     *     - `e` {Error} is null unless there is an error
     *     - `data` {Object} is the JSON response.
     */

    var api = function(path, options, callback) {
        var args = new Arguments(path, options, callback);
        args.options.method = args.options.method || 'GET';
        return api.request(args.path, args.options, args.callback);
    };

    /**
     * GET API Request.
     *
     * Options:
     *
     *   - `path` {String} is a resource path (e.g. `"/apps"`).
     *   - `[options]` {Object} is a request options object.
     *   - `[callback]` {Function} is trigger after the request
     *     - `e` {Error} is null unless there is an error
     *     - `data` {Object} is the JSON response.
     */

    api.get = function(path, options, callback) {
        var args = new Arguments(path, options, callback);
        args.options.method = 'GET';
        return api.request(args.path, args.options, args.callback);
    };

    /**
     * POST API Request.
     *
     * Options:
     *
     *   - `path` {String} is a resource path (e.g. `"/apps"`).
     *   - `[options]` {Object} is a request options object.
     *   - `[callback]` {Function} is trigger after the request
     *     - `e` {Error} is null unless there is an error
     *     - `data` {Object} is the JSON response.
     */

    api.post = function(path, options, callback) {
        var args = new Arguments(path, options, callback);
        args.options.method = 'POST';
        return api.request(args.path, args.options, args.callback);
    };

    /**
     * PUT API Request.
     *
     * Options:
     *
     *   - `path` {String} is a resource path (e.g. `"/apps"`).
     *   - `[options]` {Object} is a request options object.
     *   - `[callback]` {Function} is trigger after the request
     *     - `e` {Error} is null unless there is an error
     *     - `data` {Object} is the JSON response.
     */

    api.put = function(path, options, callback) {
        var args = new Arguments(path, options, callback);
        args.options.method = 'PUT';
        return api.request(args.path, args.options, args.callback);
    };

    /**
     * DELETE API Request.
     *
     * Options:
     *
     *   - `path` {String} is a resource path (e.g. `"/apps"`).
     *   - `[options]` {Object} is a request options object.
     *   - `[callback]` {Function} is trigger after the request
     *     - `e` {Error} is null unless there is an error
     *     - `data` {Object} is the JSON response.
     */

    api.del = function(path, options, callback) {
        var args = new Arguments(path, options, callback);
        args.options.method = 'DELETE';
        return api.request(args.path, args.options, args.callback);
    };

    /**
     * API Request Defaults.
     *
     * The method simply forwards to [requests](https://github.com/mikeal/request)
     * default method.
     */

    api.defaults = function(options, requester) {
        return request.defaults(options, requester);
    };

    /*
     * API Configuration.
     *
     * Internal configuration values for constructing an API request.
     *
     * Values:
     *
     *   - `api.token` {String} is the user token for authentication.
     *   - `api.protocol` {String} is the protocol, e.g. 'http' or 'https'.
     *   - `api.host` {String} is the host address, e.g. 'build.phonegap.com'.
     *   - `api.port` {String} is the port, e.g. '80' or '443'.
     *   - `api.path` {String} is the api path, e.g. '/api/v1'
     *   - [`api.proxy`] {String} is proxy for requests through, e.g. 'http://myproxy.com'
     */

    api.token = options.token;
    api.protocol = options.protocol;
    api.host = options.host;
    api.port = options.port;
    api.path = options.path;
    api.defaults.version = options.version;
    api.proxy = options.proxy;

    /*
     * API Request Handler.
     *
     * Private method that handles all types of API requests.
     *
     * Options:
     *
     *   - `path` {String} is a resource path (e.g. `"/apps"`).
     *   - `[options]` {Object} is a request options object.
     *   - `[callback]` {Function} is trigger after the request
     *     - `e` {Error} is null unless there is an error
     *     - `data` {Object} is the JSON response.
     */

    api.request = function(path, options, callback) {
        // require arguments
        if (!path) throw new Error('missing path argument');
        if (!options) throw new Error('missing options argument');

        // optional arguments
        callback = callback || function() {};

        var args = new Arguments(path, callback);
        var uri = api.protocol + '//' +
                  api.host + ':' +
                  api.port +
                  api.path +
                  args.path + '?auth_token=' + api.token;

        // hijack the form option.
        // this is a clean interface to define form data, but the
        // phonegap build api does not support encoded querystring data.
        var data = options.form;
        options.form = undefined;

        // use proxy when provided
        options.proxy = api.proxy;

        var r = request.send(uri, options, function(e, res, body) {
            if (e) {
                // error in request
                args.callback(e);
            }
            else if (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 202) {
                // provide a default message when none is provided
                body = body || 'server returned status code ' + res.statusCode;

                // eror in response
                args.callback(new Error(body));
            }
            else {
                try {
                    body = JSON.parse(body);
                }
                catch(err) {
                    // continue but data untouched (likely image data)
                }

                if (typeof body.error === 'string') {
                    // api response includes an error
                    args.callback(new Error(body.error));
                }
                else {
                    // api response is successful
                    args.callback(null, body);
                }
            }
        });

        // form-data support as content-type multipart/form-data
        if (data) {
            var form = r.form(),
                key,
                value;

            // handle each key in the form object:
            //   - `data` {Object} is handled as JSON.
            //   - `*` {String} is handled as a file path.
            for(key in data) {
                if (key === 'data') {
                    value = JSON.stringify(data[key]);
                }
                else {
                    value = fs.createReadStream(p.normalize(data[key]));
                }

                form.append(key, value);
            }
        }

        return r;
    };

    return api;
};

module.exports.prototype = {
};
<commit_msg>[lib] Fix proxy option typo in comment.<commit_after>/*
 * Module dependencies.
 */

var request = require('./request'),
    Arguments = require('./arguments'),
    fs = require('fs'),
    p = require('path');

/**
 * API Request Constructor.
 *
 * Creating an instance of the API Request will enable the RESTful access
 * to a PhoneGap Build API.
 *
 * Options:
 *
 *   - `options` {Object} defines the API server that has been authenticated.
 *   - `api.token` {String} is the user token for authentication.
 *   - `api.protocol` {String} is the protocol, e.g. 'http' or 'https'.
 *   - `api.host` {String} is the host address, e.g. 'build.phonegap.com'.
 *   - `api.port` {String} is the port, e.g. '80' or '443'.
 *   - `api.path` {String} is the api path, e.g. '/api/v1'
 *   - [`api.proxy`] {String} is a proxy to forward requests, e.g. 'http://myproxy.com'
 */

module.exports = function(options) {
    // require options argument
    if (!options) throw new Error('missing options argument');
    if (!options.token) throw new Error('missing options.token argument');
    if (!options.protocol) throw new Error('missing options.protocol argument');
    if (!options.host) throw new Error('missing options.host argument');
    if (!options.port) throw new Error('missing options.port argument');
    if (!options.path) throw new Error('missing options.path argument');

    /**
     * API Request.
     *
     * Sends an HTTP request to the PhoneGap Build API.
     *
     * By default, the request method is `GET` but can be changed
     * in the `options` parameters (e.g. `{ method: 'POST' }`).
     *
     * Options:
     *
     *   - `path` {String} is a resource path (e.g. `"/apps"`).
     *   - `[options]` {Object} is a request options object.
     *   - `[callback]` {Function} is trigger after the request
     *     - `e` {Error} is null unless there is an error
     *     - `data` {Object} is the JSON response.
     */

    var api = function(path, options, callback) {
        var args = new Arguments(path, options, callback);
        args.options.method = args.options.method || 'GET';
        return api.request(args.path, args.options, args.callback);
    };

    /**
     * GET API Request.
     *
     * Options:
     *
     *   - `path` {String} is a resource path (e.g. `"/apps"`).
     *   - `[options]` {Object} is a request options object.
     *   - `[callback]` {Function} is trigger after the request
     *     - `e` {Error} is null unless there is an error
     *     - `data` {Object} is the JSON response.
     */

    api.get = function(path, options, callback) {
        var args = new Arguments(path, options, callback);
        args.options.method = 'GET';
        return api.request(args.path, args.options, args.callback);
    };

    /**
     * POST API Request.
     *
     * Options:
     *
     *   - `path` {String} is a resource path (e.g. `"/apps"`).
     *   - `[options]` {Object} is a request options object.
     *   - `[callback]` {Function} is trigger after the request
     *     - `e` {Error} is null unless there is an error
     *     - `data` {Object} is the JSON response.
     */

    api.post = function(path, options, callback) {
        var args = new Arguments(path, options, callback);
        args.options.method = 'POST';
        return api.request(args.path, args.options, args.callback);
    };

    /**
     * PUT API Request.
     *
     * Options:
     *
     *   - `path` {String} is a resource path (e.g. `"/apps"`).
     *   - `[options]` {Object} is a request options object.
     *   - `[callback]` {Function} is trigger after the request
     *     - `e` {Error} is null unless there is an error
     *     - `data` {Object} is the JSON response.
     */

    api.put = function(path, options, callback) {
        var args = new Arguments(path, options, callback);
        args.options.method = 'PUT';
        return api.request(args.path, args.options, args.callback);
    };

    /**
     * DELETE API Request.
     *
     * Options:
     *
     *   - `path` {String} is a resource path (e.g. `"/apps"`).
     *   - `[options]` {Object} is a request options object.
     *   - `[callback]` {Function} is trigger after the request
     *     - `e` {Error} is null unless there is an error
     *     - `data` {Object} is the JSON response.
     */

    api.del = function(path, options, callback) {
        var args = new Arguments(path, options, callback);
        args.options.method = 'DELETE';
        return api.request(args.path, args.options, args.callback);
    };

    /**
     * API Request Defaults.
     *
     * The method simply forwards to [requests](https://github.com/mikeal/request)
     * default method.
     */

    api.defaults = function(options, requester) {
        return request.defaults(options, requester);
    };

    /*
     * API Configuration.
     *
     * Internal configuration values for constructing an API request.
     *
     * Values:
     *
     *   - `api.token` {String} is the user token for authentication.
     *   - `api.protocol` {String} is the protocol, e.g. 'http' or 'https'.
     *   - `api.host` {String} is the host address, e.g. 'build.phonegap.com'.
     *   - `api.port` {String} is the port, e.g. '80' or '443'.
     *   - `api.path` {String} is the api path, e.g. '/api/v1'
     *   - [`api.proxy`] {String} is proxy for requests through, e.g. 'http://myproxy.com'
     */

    api.token = options.token;
    api.protocol = options.protocol;
    api.host = options.host;
    api.port = options.port;
    api.path = options.path;
    api.defaults.version = options.version;
    api.proxy = options.proxy;

    /*
     * API Request Handler.
     *
     * Private method that handles all types of API requests.
     *
     * Options:
     *
     *   - `path` {String} is a resource path (e.g. `"/apps"`).
     *   - `[options]` {Object} is a request options object.
     *   - `[callback]` {Function} is trigger after the request
     *     - `e` {Error} is null unless there is an error
     *     - `data` {Object} is the JSON response.
     */

    api.request = function(path, options, callback) {
        // require arguments
        if (!path) throw new Error('missing path argument');
        if (!options) throw new Error('missing options argument');

        // optional arguments
        callback = callback || function() {};

        var args = new Arguments(path, callback);
        var uri = api.protocol + '//' +
                  api.host + ':' +
                  api.port +
                  api.path +
                  args.path + '?auth_token=' + api.token;

        // hijack the form option.
        // this is a clean interface to define form data, but the
        // phonegap build api does not support encoded querystring data.
        var data = options.form;
        options.form = undefined;

        // use proxy when provided
        options.proxy = api.proxy;

        var r = request.send(uri, options, function(e, res, body) {
            if (e) {
                // error in request
                args.callback(e);
            }
            else if (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 202) {
                // provide a default message when none is provided
                body = body || 'server returned status code ' + res.statusCode;

                // eror in response
                args.callback(new Error(body));
            }
            else {
                try {
                    body = JSON.parse(body);
                }
                catch(err) {
                    // continue but data untouched (likely image data)
                }

                if (typeof body.error === 'string') {
                    // api response includes an error
                    args.callback(new Error(body.error));
                }
                else {
                    // api response is successful
                    args.callback(null, body);
                }
            }
        });

        // form-data support as content-type multipart/form-data
        if (data) {
            var form = r.form(),
                key,
                value;

            // handle each key in the form object:
            //   - `data` {Object} is handled as JSON.
            //   - `*` {String} is handled as a file path.
            for(key in data) {
                if (key === 'data') {
                    value = JSON.stringify(data[key]);
                }
                else {
                    value = fs.createReadStream(p.normalize(data[key]));
                }

                form.append(key, value);
            }
        }

        return r;
    };

    return api;
};

module.exports.prototype = {
};
<|endoftext|>


<commit_before>/**
 * Helper "class" for accessing MediaWiki API and handling cookie-based session
 */
module.exports = (function() {
	// introspect package.json to get module version
	var VERSION = require('../package').version;

	var
		// @see https://github.com/caolan/async
		async = require('async'),
		// @see https://github.com/mikeal/request
		request = require('request');

	// import deferred-js
	// @see https://github.com/heavylifters/deferred-js
	var Deferred = require('./deferred/deferred').Deferred;

	var api = function(options) {
		this.protocol = options.protocol || 'http';
		this.port = options.port || (this.protocol === 'https' ? 443 : 80);
		this.server = options.server;
		this.path = options.path;
		this.proxy = options.proxy;
		this.jar = request.jar(); // create new cookie jar for each instance

		this.debug = options.debug;

		// set up logging
		var winston = require('winston');
		this.logger = new (winston.Logger)();

		if (!options.silent) {
			// console logging
			this.logger.add(winston.transports.Console, {
				level: this.debug ? 'debug' : 'info'
			});
			this.logger.cli();

			// file logging
			var path = require('path'),
				fs = require('fs'),
				logDir = path.dirname(process.argv[1]) + '/log/',
				logFile = logDir + path.basename(process.argv[1], '.js') + '.log';

			if (fs.existsSync(logDir)) {
				this.logger.add(winston.transports.File, {
					colorize: true,
					filename: logFile,
					json: false,
					level: this.debug ? 'debug' : 'info'
				});
			}
		}

		// requests queue
		var concurrency = options.concurrency || 3; // how many tasks (i.e. requests) to run in parallel
		this.queue = async.queue(function(task, callback) {
			// process the task
			callback();
		}, concurrency);
		this.tasksCnt = 0;

		// HTTP client
		this.formatUrl = require('url').format;

		this.userAgent = options.userAgent || ('nodemw v' + VERSION + ' (node.js ' + process.version + '; ' + process.platform + ' ' + process.arch + ')');
		this.version = VERSION;

		// debug info
		this.info(process.argv.join(' '));
		this.info(this.userAgent);
		this.info('Using <' + this.protocol + '://' + this.server + ':' + this.port + this.path + '/api.php> as API entry point');
		this.info('----');
	};

	var doRequest = function(params, callback, method, promise) {
		var self = this,
			// store requested action - will be used when parsing a response
			actionName = params.action,
			// "request" options
			options = {
				method: method || 'GET',
				proxy: this.proxy || false,
				jar: this.jar,
				headers: {
					'User-Agent': this.userAgent
				}
			};

		// HTTP request parameters
		params = params || {};

		// force JSON format
		params.format = 'json';

		// handle uploads
		if (method === 'UPLOAD') {
			options.method = 'POST';

			var CRLF = "\r\n",
				postBody = [],
				boundary = 'nodemw' + Math.random().toString().substr(2);

			// encode each field
			Object.keys(params).forEach(function(fieldName) {
				var value = params[fieldName];

				postBody.push("--" + boundary);
				postBody.push(CRLF);

				if (typeof value === 'string') {
					// properly encode UTF8 in binary-safe POST data
					postBody.push("Content-Disposition: form-data; name=\"" + fieldName + "\"");
					postBody.push(CRLF);
					postBody.push(CRLF);
					postBody.push(new Buffer(value, 'utf8'));
				}
				else {
					// send attachment
					postBody.push("Content-Disposition: form-data; name=\"" + fieldName + "\"; filename=\"foo\"");
					postBody.push(CRLF);
					postBody.push(CRLF);
					postBody.push(value);
				}

				postBody.push(CRLF);
			});

			postBody.push("--" + boundary + "--");

			// encode post data
			options.headers['content-type'] = 'multipart/form-data; boundary=' + boundary;
			options.body = postBody;

			params = {};
		}

		// form an URL to API
		options.url = this.formatUrl({
			protocol: this.protocol,
			port: this.port,
			hostname: this.server,
			pathname: this.path + '/api.php',
			query: (options.method === 'GET') ? params : {}
		});

		// POST all parameters (avoid "request string too long" errors)
		if (method === 'POST') {
			options.form = params;
		}

		this.logger.debug('API action: %s', actionName);
		this.logger.debug('%d/%d: %s <%s>', promise.taskId, this.tasksCnt, options.method, options.url);
		if (options.form) this.logger.debug('POST fields: %s', Object.keys(options.form).join(', '));

		request(options, function(error, response, body) {
			if (error) {
				self.logger.error('Request to API failed: %s', error);

				promise.reject({
					err:'Request to API failed: ' + error,
					info: error
				});
			}

			if (response.statusCode !== 200) {
				self.logger.error('Request to API failed: HTTP status code was %d for <%s>', response.statusCode, options.url);
				self.logger.data(new Error().stack);

				promise.reject({
					err:'Request to API failed: HTTP status code was ' + response.statusCode,
					info: response
				});
			}

			// parse response
			var data,
				info,
				next;

			try {
				data = JSON.parse(body);
				info = data && data[actionName];
				next = data && data['query-continue'] && data['query-continue'][params.list];
			}
			catch(e) {
				self.logger.error('Error parsing JSON response: %s',  body);
				self.logger.data(body);

				promise.reject({
					err: 'Error parsing JSON response',
					info: data.body
				});
			}

			//if (!callback) data.error = {info: 'foo'}; // debug

			if (data && !data.error) {
				if (typeof callback === 'function') {
					callback && callback(info, next, data);
				}
				promise.resolve({
					info: info,
					next: next,
					data: data
				});
			}
			else {
				self.logger.error('Error returned by API: %s',  data.error.info);
				self.logger.data(data.error);

				promise.reject({
					err: 'Error returned by API: ' + data.error.info,
					info: data.error
				});
			}
		});

		return promise;
	};

	// add a function to the queue
	var pushToQueue = function(fn) {
		var promise = new Deferred(),
			task = {
				fn: fn,
				promise: promise
			};

		this.tasksCnt++;
		promise.taskId = this.tasksCnt;

		// add to the queue
		this.queue.push(task, function(err) {
			if (err) {
				promise.reject(err);
				return;
			}

			// run the task that will resolve / reject given promise
			task.fn(task.promise);
		});

		return promise;
	};

	// public interface
	api.prototype = {
		log: function() {
			this.logger.log.apply(this.logger, arguments);
		},

		info: function() {
			this.logger.info.apply(this.logger, arguments);
		},

		warn: function(msg, extra) {
			this.logger.warn.apply(this.logger, arguments);
		},

		error: function(msg, extra) {
			this.logger.error.apply(this.logger, arguments);
		},

		// adds request to the queue and returns a promise
		call: function(params, callback, method) {
			var self = this;

			// add a request to the queue
			return pushToQueue.call(this, function(promise) {
				doRequest.apply(self, [params, callback, method, promise]);
			});
		},

		// fetch an external resource
		fetchUrl: function(url, encoding) {
			var self = this;
			encoding = encoding || 'utf-8';

			this.info("Fetching <%s> (as %s)...", url, encoding);

			// add a request to the queue
			return pushToQueue.call(this, function(promise) {
				request({
					url: url,
					encoding: (encoding === 'binary') ? null : encoding
				}, function (error, response, body) {
					if (!error && response.statusCode === 200) {
						self.info('Fetched %s kB', (body.length/1024).toFixed(2));
						promise.resolve(body);
					}
					else {
						self.error('Failed to fetch <' + url + '>');
						promise.reject({
							err: 'fetchUrl failed with: ' + error,
							info: error
						});
					}
				});
			});
		}
	};

	return api;
}());
<commit_msg>Exception thrown when ECONNRESET is not caught<commit_after>/**
 * Helper "class" for accessing MediaWiki API and handling cookie-based session
 */
module.exports = (function() {
	// introspect package.json to get module version
	var VERSION = require('../package').version;

	var
		// @see https://github.com/caolan/async
		async = require('async'),
		// @see https://github.com/mikeal/request
		request = require('request');

	// import deferred-js
	// @see https://github.com/heavylifters/deferred-js
	var Deferred = require('./deferred/deferred').Deferred;

	var api = function(options) {
		this.protocol = options.protocol || 'http';
		this.port = options.port || (this.protocol === 'https' ? 443 : 80);
		this.server = options.server;
		this.path = options.path;
		this.proxy = options.proxy;
		this.jar = request.jar(); // create new cookie jar for each instance

		this.debug = options.debug;

		// set up logging
		var winston = require('winston');
		this.logger = new (winston.Logger)();

		if (!options.silent) {
			// console logging
			this.logger.add(winston.transports.Console, {
				level: this.debug ? 'debug' : 'info'
			});
			this.logger.cli();

			// file logging
			var path = require('path'),
				fs = require('fs'),
				logDir = path.dirname(process.argv[1]) + '/log/',
				logFile = logDir + path.basename(process.argv[1], '.js') + '.log';

			if (fs.existsSync(logDir)) {
				this.logger.add(winston.transports.File, {
					colorize: true,
					filename: logFile,
					json: false,
					level: this.debug ? 'debug' : 'info'
				});
			}
		}

		// requests queue
		var concurrency = options.concurrency || 3; // how many tasks (i.e. requests) to run in parallel
		this.queue = async.queue(function(task, callback) {
			// process the task
			callback();
		}, concurrency);
		this.tasksCnt = 0;

		// HTTP client
		this.formatUrl = require('url').format;

		this.userAgent = options.userAgent || ('nodemw v' + VERSION + ' (node.js ' + process.version + '; ' + process.platform + ' ' + process.arch + ')');
		this.version = VERSION;

		// debug info
		this.info(process.argv.join(' '));
		this.info(this.userAgent);
		this.info('Using <' + this.protocol + '://' + this.server + ':' + this.port + this.path + '/api.php> as API entry point');
		this.info('----');
	};

	var doRequest = function(params, callback, method, promise) {
		var self = this,
			// store requested action - will be used when parsing a response
			actionName = params.action,
			// "request" options
			options = {
				method: method || 'GET',
				proxy: this.proxy || false,
				jar: this.jar,
				headers: {
					'User-Agent': this.userAgent
				}
			};

		// HTTP request parameters
		params = params || {};

		// force JSON format
		params.format = 'json';

		// handle uploads
		if (method === 'UPLOAD') {
			options.method = 'POST';

			var CRLF = "\r\n",
				postBody = [],
				boundary = 'nodemw' + Math.random().toString().substr(2);

			// encode each field
			Object.keys(params).forEach(function(fieldName) {
				var value = params[fieldName];

				postBody.push("--" + boundary);
				postBody.push(CRLF);

				if (typeof value === 'string') {
					// properly encode UTF8 in binary-safe POST data
					postBody.push("Content-Disposition: form-data; name=\"" + fieldName + "\"");
					postBody.push(CRLF);
					postBody.push(CRLF);
					postBody.push(new Buffer(value, 'utf8'));
				}
				else {
					// send attachment
					postBody.push("Content-Disposition: form-data; name=\"" + fieldName + "\"; filename=\"foo\"");
					postBody.push(CRLF);
					postBody.push(CRLF);
					postBody.push(value);
				}

				postBody.push(CRLF);
			});

			postBody.push("--" + boundary + "--");

			// encode post data
			options.headers['content-type'] = 'multipart/form-data; boundary=' + boundary;
			options.body = postBody;

			params = {};
		}

		// form an URL to API
		options.url = this.formatUrl({
			protocol: this.protocol,
			port: this.port,
			hostname: this.server,
			pathname: this.path + '/api.php',
			query: (options.method === 'GET') ? params : {}
		});

		// POST all parameters (avoid "request string too long" errors)
		if (method === 'POST') {
			options.form = params;
		}

		this.logger.debug('API action: %s', actionName);
		this.logger.debug('%d/%d: %s <%s>', promise.taskId, this.tasksCnt, options.method, options.url);
		if (options.form) this.logger.debug('POST fields: %s', Object.keys(options.form).join(', '));

		request(options, function(error, response, body) {
			response = response || {};

			if (error) {
				self.logger.error('Request to API failed: %s', error);

				promise.reject({
					err:'Request to API failed: ' + error,
					info: error
				});
			}

			if (response.statusCode !== 200) {
				self.logger.error('Request to API failed: HTTP status code was %d for <%s>', response.statusCode || 'unknown', options.url);
				self.logger.data(new Error().stack);

				promise.reject({
					err:'Request to API failed: HTTP status code was ' + response.statusCode,
					info: response
				});
			}

			// parse response
			var data,
				info,
				next;

			try {
				data = JSON.parse(body);
				info = data && data[actionName];
				next = data && data['query-continue'] && data['query-continue'][params.list];
			}
			catch(e) {
				self.logger.error('Error parsing JSON response: %s',  body);
				self.logger.data(body);

				promise.reject({
					err: 'Error parsing JSON response',
					info: data.body
				});
			}

			//if (!callback) data.error = {info: 'foo'}; // debug

			if (data && !data.error) {
				if (typeof callback === 'function') {
					callback && callback(info, next, data);
				}
				promise.resolve({
					info: info,
					next: next,
					data: data
				});
			}
			else {
				self.logger.error('Error returned by API: %s',  data.error.info);
				self.logger.data(data.error);

				promise.reject({
					err: 'Error returned by API: ' + data.error.info,
					info: data.error
				});
			}
		});

		return promise;
	};

	// add a function to the queue
	var pushToQueue = function(fn) {
		var promise = new Deferred(),
			task = {
				fn: fn,
				promise: promise
			};

		this.tasksCnt++;
		promise.taskId = this.tasksCnt;

		// add to the queue
		this.queue.push(task, function(err) {
			if (err) {
				promise.reject(err);
				return;
			}

			// run the task that will resolve / reject given promise
			task.fn(task.promise);
		});

		return promise;
	};

	// public interface
	api.prototype = {
		log: function() {
			this.logger.log.apply(this.logger, arguments);
		},

		info: function() {
			this.logger.info.apply(this.logger, arguments);
		},

		warn: function(msg, extra) {
			this.logger.warn.apply(this.logger, arguments);
		},

		error: function(msg, extra) {
			this.logger.error.apply(this.logger, arguments);
		},

		// adds request to the queue and returns a promise
		call: function(params, callback, method) {
			var self = this;

			// add a request to the queue
			return pushToQueue.call(this, function(promise) {
				doRequest.apply(self, [params, callback, method, promise]);
			});
		},

		// fetch an external resource
		fetchUrl: function(url, encoding) {
			var self = this;
			encoding = encoding || 'utf-8';

			this.info("Fetching <%s> (as %s)...", url, encoding);

			// add a request to the queue
			return pushToQueue.call(this, function(promise) {
				request({
					url: url,
					encoding: (encoding === 'binary') ? null : encoding
				}, function (error, response, body) {
					if (!error && response.statusCode === 200) {
						self.info('Fetched %s kB', (body.length/1024).toFixed(2));
						promise.resolve(body);
					}
					else {
						self.error('Failed to fetch <' + url + '>');
						promise.reject({
							err: 'fetchUrl failed with: ' + error,
							info: error
						});
					}
				});
			});
		}
	};

	return api;
}());
<|endoftext|>


<commit_before>'use strict'

const pool = require('./pool');
const express = require('express');
const bodyParser = require('body-parser');
const multer = require('multer'); // v1.0.5
const upload = multer(); // for parsing multipart/form-data

let app = express();
const text_parser = bodyParser.text();

const port = 80 ;

let verbose = true;
const log = console.error.bind(console, "[api.js] ")
const trace = (blabla) => {
	if(verbose) log(blabla);
};

pool.init(8, null, (err, res) => {
	if(err) trace(err);
	else trace('pool ready');
});

app.get('/', function (request, response) {
	response.send('RExpress 0.0.1')
})
// call a function w/ form-data (safe)
.post('/R/:function', upload.array(), function (request, response) {

	let call = request.params.function + '(' ;
	for(let key in request.body){
		call += key + "=" + request.body[key] + ','
	}
	call = call.substring(0, call.length - 1); // delete last comma
	call += ")" ;
	pool.submit_job(call, (err, res) => {
		response.send(res);
	})
})
// execute the R program given in body
// -- very unsafe but useful in testing purposes
.post('/R', text_parser, function (request, response) {

	console.log(request.body);
	pool.submit_job(request.body, (err, res) => {
		response.send(res);
	})
})

app.listen(port);
<commit_msg>ADD automatic loading of .R scripts<commit_after>'use strict'

const pool = require('./pool');
const fs = require('fs');
const express = require('express');
const bodyParser = require('body-parser');
const multer = require('multer'); // v1.0.5
const upload = multer(); // for parsing multipart/form-data

let app = express();
const text_parser = bodyParser.text();

const port = 80 ;

let verbose = true;
const log = console.error.bind(console, "[api.js] ")
const trace = (blabla) => {
	if(verbose) log(blabla);
};

// get custom libraries
fs.readdir('../R/', (err, files) =>{

	let sources = "" ;
	for(let file of files){
		sources += 'source("../R/' + file + '")\n' ;
	}
	// builds the R interpreters pool -- 8 workers ; ../R folder
	pool.init(8, sources, (err, res) => {
		if(err) trace(err);
		else trace('pool ready');
	});
})

app.get('/', function (request, response) {
	response.send('RExpress 0.0.1')
})
// call a function w/ form-data (safe)
.post('/R/:function', upload.array(), function (request, response) {

	let call = request.params.function + '(' ;
	for(let key in request.body){
		call += key + "=" + request.body[key] + ','
	}
	call = call.substring(0, call.length - 1); // delete last comma
	call += ")" ;
	pool.submit_job(call, (err, res) => {
		response.send(res);
	})
})
// execute the R program given in body
// -- very unsafe but useful in testing purposes
.post('/R', text_parser, function (request, response) {

	console.log(request.body);
	pool.submit_job(request.body, (err, res) => {
		response.send(res);
	})
})

app.listen(port);
<|endoftext|>


<commit_before>"use strict";

var _ = require('underscore');
var async = require('async');
var Redis = require('redis');

module.exports = function(opts){

  opts = opts || {};

  var env = process.env.NODE_ENV || 'development';

  if(!opts.redis){
    opts.redis = {
      host: '127.0.0.1',
      port: 6379,
    };
  }

  if(!opts.prefix){
    opts.prefix = 'df:' + env;
  }

  var fns;

  if(opts.hasOwnProperty('fns')){
    fns = opts.fns;
  } else {
    fns = require('./fns');
  }

  var redis = Redis.createClient(
    opts.redis.port,
    opts.redis.host
  );

  var api = {};

  // cleanup for exit
  api.quit = function(next) {
    redis.quit();
    next();
  };

  // Delete all nodes and other data (eg queues) from the topology

  // Delete all nodes. Reset the database and topology to pristine state
  api.reset = function(done) {

    var nodes;

    // delete nodes one by one to emit del messages
    var all = function(next){
      api.all(function(err, n){
        nodes = _.toArray(n);
        next();
      });
    };

    var del = function(x, next){
      api.del(x.id, next);
    };

    var delAll = function(next){
      async.eachSeries(nodes, del, next);
    };

    // delete remaining keys to clear queue
    var deleteKeys = function(next){
      redis.keys(
        opts.prefix + ':*',
        function(err, keys){
          _.each(keys, function(key){
            redis.del(key);
          });
          next();
        });
    };

    async.series(
      [all, delAll, deleteKeys],
      done
    );

  };

  // make fns available (worker uses these)
  api.fns = function(x){
    return fns[x];
  };

  // a node's sources are persisted as a Redis set

  // each node in the sources list has a set of targets. the source
  // node id is maintained in that set

  // Set attrs on a node by persisting them them in Redis as a hash

  // set attrs on a node
  api.setAttrs = function(id, attrs, done){
    redis.hmset(
      opts.prefix + ':attrs:' + id,
      attrs,
      done
    );
  };

  api.getAttrs = function(id, done){
    redis.hgetall(
      opts.prefix + ':attrs:' + id,
      function(err, res){
        if(!res){
          res = {};
        }
        done(false, res);
      });
  };

  // set with #attrs(id, {}, done) or get with #attrs(id, done)

  api.attrs = function(id, attrs, done){
    if(arguments.length === 2){
      done = attrs;
      return api.getAttrs(id, done);
    }
    return api.setAttrs(id, attrs, done);
  };

  // used when deleting a node
  api.delAttrs = function(id, done){
    redis.del(
      opts.prefix + ':attrs:' + id,
      done
    );
  };

  // add a source to a node
  api.addSource = function(id, source_id, done){

    async.parallel([
      function(next){
        redis.sadd(
          opts.prefix + ':sources:' + id,
          source_id,
          next
        );
      },
      function(next){
        redis.sadd(
          opts.prefix + ':targets:' + source_id,
          id,
          next
        );
      }], function(){
        api.emit('addSource', id, [source_id], done);
      });
  };

  // remove a source from a node
  api.delSource = function(id, source_id, done){
    redis.srem(
      opts.prefix + ':sources:' + id,
      source_id
    );
    redis.srem(
      opts.prefix + ':targets:' + source_id,
      id
    );
    return done();
  };

  // get all sources a node has
  api.getSources = function(id, done){
    redis.smembers(
      opts.prefix + ':sources:' + id,
      function(err, sources){
        if(!sources){
          sources = [];
        }
        return done(false, sources);
      });
  };

  // remove all sources from a node
  api.delSources = function(id, done){
    this.getSources(
      id,
      function(err, sources){
        if(!sources){
          return done();
        }
        async.eachSeries(
          sources,
          function(source_id, next){
            api.delSource(id, source_id, next);
          },
          done
        );
      });
  };

  api.getTargets = function(id, done){
    redis.smembers(
      opts.prefix + ':targets:' + id,
      function(err, targets){
        if(!targets){
          targets = [];
        }
        return done(false, targets);
      });
  };

  // set/get in Redis last message output by node
  api.setData = function(id, data, done){
    redis.set(
      opts.prefix + ':data:' + id,
      JSON.stringify(data),
      function(){
        api.emit('setData', id, data, done);
      }
    );
  };

  api.getData = function(id, done){
    redis.get(
      opts.prefix + ':data:' + id,
      function(err, res){
        var data = null;
        if(res){
          data = JSON.parse(res);
        }
        done(false, data);
      });
  };

  // remove any existing sources a node has and replace with provided
  // set
  api.setSources = function(id, sources, done){
    this.delSources(id, function(){
      if(!sources){
        return done();
      }
      async.eachSeries(
        sources,
        function(source_id, next){
          api.addSource(id, source_id, next);
        },
        done
      );
    });
  };

  // load a node from Redis
  // Get a single node. Callback provides null if doesn't exist

  api.get = function(id, done){
    var node = {};

    redis.get(
      opts.prefix + ':nodes:' + id,
      function(err, res){

        if(err){
          console.log('ERROR', err);
          return done(err);
        }

        if(!res){
          return done(null, null);
        }

        res = JSON.parse(res);
        node.id = res.id;
        node.fn = res.fn;
        node.x = res.x;
        node.y = res.y;

        async.parallel([
          function(next){
            api.getSources(id, function(err, sources){
              node.sources = sources;
              next();
            });
          },
          function(next){
            api.getAttrs(id, function(err, res){
              node.attrs = res;
              next();
            });
          },
          function(next){
            api.getData(id, function(err, data){
              node.data = data;
              next();
            });
          }
        ], function(err){
          if(err){
            console.log(err);
            return done(err, null);
          }
          done(false, node);
        });
      });

  };

  // get all nodes in the topology
  api.all = function(done){

    var keys = [];
    var nodes = {};

    async.series([
      function(next){
        redis.keys(
          opts.prefix + ':nodes:*',
          function(err, res){
            if(res){
              keys = res;
            }
            next();
          });
      },
      function(next){
        async.eachSeries(
          keys,
          function(key, next){
            var id = _.last(key.split(':'));
            api.get(id, function(err, node){
              nodes[node.id] = node;
              next();
            });
          }, function(){
            next();
          });
      }], function(){
        done(false, nodes);
      });
  };

  // add a node to the topology
  api.add = function(node, done){

    this.get(
      node.id,
      function(err, n){

        // Duplicate node ids generate an error

        if(n){
          return done(new Error('duplicate-id'));
        }

        if(!node.hasOwnProperty('sources')){
          node.sources = [];
        }

        if(!node.hasOwnProperty('attrs')){
          node.attrs = {};
        }

        if(!node.hasOwnProperty('x')){
          node.x = null;
        }

        if(!node.hasOwnProperty('y')){
          node.y = null;
        }

        if(!node.hasOwnProperty('fn') || !node.fn){
          node.fn = 'thru';
        }

        if(!fns.hasOwnProperty(node.fn)){
          node.fn = 'thru';
        }

        // copy default attrs from fn definition

        var fn = fns[node.fn];

        if(fn.hasOwnProperty('attrs')){
          _.each(fn.attrs, function(v, k){
            if(!node.attrs.hasOwnProperty(k)){
              node.attrs[k] = v;
            }
          });
        }

        async.series([

          function(next){
            // save node. id and fn are immutable
            var key = opts.prefix + ':nodes:' + node.id;
            var val = {
              id: node.id,
              fn: node.fn,
              x: node.x,
              y: node.y
            };
            redis.set(key, JSON.stringify(val));
            next();
          },

          function(next){
            // set the inital sources on the node
            api.setSources(node.id, node.sources, next);
          },

          function(next){
            // init node
            if(!fn.hasOwnProperty('init')){
              return next();
            }

            // allow init to run. It may set some attrs
            fn.attrs = node.attrs;
            fn.init(next);
          },

          function(next){
            // set the initial attrs on the node
            api.setAttrs(node.id, node.attrs, next);
          }

        ], function(){
          api.emit('add', node.id, node, done);
        });
      });

  };

  // change a nodes's properties (just x and y for now)
  api.set = function(id, newVals, done){

    api.get(
      id,
      function(err, node){

        if(!node){
          return done();
        }

        var keys = ['x','y'];
        var vals = {};
        _.each(keys, function(key){
          if(newVals.hasOwnProperty(key)){
            vals[key] = newVals[key];
          }
        });

        var set = function(next){
          var key = opts.prefix + ':nodes:' + node.id;
          var val = {
            id: node.id,
            fn: node.fn,
            x: vals.x,
            y: vals.y
          };

          redis.set(
            key,
            JSON.stringify(val),
            next
          );
        };

        async.series([
          set
        ], function(){
          api.emit('set', id, vals, done);
        });
      });
  };

  // remove a node from the topology
  api.del = function(id, done){

    this.get(
      id,
      function(err, n){

        if(!n){
          return done();
        }

        async.series([

          function(next){
            // id and fn are immutable
            var key = opts.prefix + ':nodes:' + id;
            redis.del(key);
            next();
          },

          function(next){
            api.delSources(id, next);
          },

          function(next){
            api.delAttrs(id, next);
          }

        ], function(){
          api.emit('del', id, done);
        });
      });

  };

  // inject a message in to a node

  // message handler does process then emits event (redis pub) and
  // distributes message to targets (redis queue)
  api.inject = function(id, data, done){

    var message = {
      at: new Date().getTime(),
      id: id,
      data: data
    };

    redis.lpush(
      opts.prefix + ':queue',
      JSON.stringify(message),
      function(err){
        return done();
      });
  };

  // delete all messages off queue
  api.purge = function(done){

    redis.del(
      opts.prefix + ':queue',
      function(err){
        if(done){
          return done(null);
        }
      });
  };

  api.extract = function(done){

    redis.rpop(
      opts.prefix + ':queue',
      function(err, reply){

        if(err){
          console.log('#run error', err);
          return done(err, null, null);
        }

        if (!reply){
          // timed out, no message
          return done(null, null, null);
        }

        var id, msg, input;

        try {
          msg = JSON.parse(reply);
          // if brpop use this instead
          //input = JSON.parse(reply[1]);
        } catch (e) {
          // bad json
          console.log('bad json', e, reply);
          return done(err, null, null);
        }

        return done(null, msg.id, msg.data);

      });
  };

  api.process = function(id, data, done){

    api.get(
      id,
      function(err, node){

        if(!node){
          return done(new Error('unknown-node'));
        }

        // processing method from the node's function
        var fn = api.fns(node.fn).process;

        // process with the node's attrs and incoming data
        fn(
          node.attrs,
          data,
          function(err, attrs, output){

            // no attrs provided - don't change them
            if(typeof attrs === 'undefined'){
              return done();
            }

            // no specific output data from fn -- no message to pass to
            // nodes that source this one, just save the node's
            // (possibly changed) state and get out of here
            if(typeof output === 'undefined'){
              return api.setAttrs(id, attrs, done);
            }

            // there is some output. save it and pass on to connected nodes

            async.series([

              // save the node's new state
              function(next){
                api.setAttrs(id, attrs, function(){
                  next();
                });
              },

              // latch the output on the node
              function(next){
                api.setData(id, output, function(){
                  next();
                });
              },

              // distribute to connected nodes via the queue
              function(next){

                api.getTargets(
                  id,
                  function(err, targets){
                    if(targets.length === 0){
                      return next();
                    }
                    async.eachSeries(
                      targets,
                      function(target_id, done){
                        api.inject(target_id, output, done);
                      }, function(){
                        next();
                      });
                  });
              }
            ], function(){
              done(false, output);
            });
          });
      });

  };

  // if exists process one message from the queue
  api.tick = function(done){
    // pull a message off the queue
    api.extract(function(err, id, data){
      // nothing on queue
      if(!id){
        return done(null, null, null);
      }
      // id is node id to pass the data to
      api.process(
        id,
        data,
        function(err, output){
          return done(null, id, output);
        });
    });
  };

  // events stream

  api.emit = function(method, id, args, done){

    if (typeof args === 'function'){
      done = args;
      args = false;
    }

    //console.log('EMIT:', method, id, JSON.stringify(args));

    redis.publish(
      opts.prefix + ':events:' + id + ':' + method,
      JSON.stringify(args),
      done
    );
  };

  // list of events listeners
  var list = [];

  api.listener = function(id, handler){
    var key;

    // if no id supplied, then get all events
    if(typeof id === 'function'){
      handler = id;
      id = false;
      key = opts.prefix + ':events:*';
    } else {
      key = opts.prefix + ':events:' + id + ':*';
    }

    var sub = Redis.createClient(
      opts.redis.port,
      opts.redis.host
    );

    sub.psubscribe(key);
    sub.on('pmessage', function(pattern, channel, msg){
      //console.log('**', msg, typeof msg);
      var args;
      if(msg && typeof msg !== 'undefined' && msg !== 'undefined'){
        args = JSON.parse(msg);
      }
      console.log('####', channel);
      // key may have prefixes, so count from the end
      // e.g: df:test:events:my-node-id:setData
      var keys = channel.split(':');
      var id = keys[keys.length-2];
      var method = keys[keys.length-1];
      //console.log(method, id, args);
      handler(method, id, args);
    });

    var quit = function(){
      sub.quit();
    };

    return {
      quit: quit
    };

  };

  api.worker = require('./worker');

  return api;

};
<commit_msg>Cleanup<commit_after>"use strict";

var _ = require('underscore');
var async = require('async');
var Redis = require('redis');

module.exports = function(opts){

  opts = opts || {};

  var env = process.env.NODE_ENV || 'development';

  if(!opts.redis){
    opts.redis = {
      host: '127.0.0.1',
      port: 6379,
    };
  }

  if(!opts.prefix){
    opts.prefix = 'df:' + env;
  }

  var fns;

  if(opts.hasOwnProperty('fns')){
    fns = opts.fns;
  } else {
    fns = require('./fns');
  }

  var redis = Redis.createClient(
    opts.redis.port,
    opts.redis.host
  );

  var api = {};

  // cleanup for exit
  api.quit = function(next) {
    redis.quit();
    next();
  };

  // Delete all nodes and other data (eg queues) from the topology

  // Delete all nodes. Reset the database and topology to pristine state
  api.reset = function(done) {

    var nodes;

    // delete nodes one by one to emit del messages
    var all = function(next){
      api.all(function(err, n){
        nodes = _.toArray(n);
        next();
      });
    };

    var del = function(x, next){
      api.del(x.id, next);
    };

    var delAll = function(next){
      async.eachSeries(nodes, del, next);
    };

    // delete remaining keys to clear queue
    var deleteKeys = function(next){
      redis.keys(
        opts.prefix + ':*',
        function(err, keys){
          _.each(keys, function(key){
            redis.del(key);
          });
          next();
        });
    };

    async.series(
      [all, delAll, deleteKeys],
      done
    );

  };

  // make fns available (worker uses these)
  api.fns = function(x){
    return fns[x];
  };

  // a node's sources are persisted as a Redis set

  // each node in the sources list has a set of targets. the source
  // node id is maintained in that set

  // Set attrs on a node by persisting them them in Redis as a hash

  // set attrs on a node
  api.setAttrs = function(id, attrs, done){
    redis.hmset(
      opts.prefix + ':attrs:' + id,
      attrs,
      done
    );
  };

  api.getAttrs = function(id, done){
    redis.hgetall(
      opts.prefix + ':attrs:' + id,
      function(err, res){
        if(!res){
          res = {};
        }
        done(false, res);
      });
  };

  // set with #attrs(id, {}, done) or get with #attrs(id, done)

  api.attrs = function(id, attrs, done){
    if(arguments.length === 2){
      done = attrs;
      return api.getAttrs(id, done);
    }
    return api.setAttrs(id, attrs, done);
  };

  // used when deleting a node
  api.delAttrs = function(id, done){
    redis.del(
      opts.prefix + ':attrs:' + id,
      done
    );
  };

  // add a source to a node
  api.addSource = function(id, source_id, done){

    async.parallel([
      function(next){
        redis.sadd(
          opts.prefix + ':sources:' + id,
          source_id,
          next
        );
      },
      function(next){
        redis.sadd(
          opts.prefix + ':targets:' + source_id,
          id,
          next
        );
      }], function(){
        api.emit('addSource', id, [source_id], done);
      });
  };

  // remove a source from a node
  api.delSource = function(id, source_id, done){
    redis.srem(
      opts.prefix + ':sources:' + id,
      source_id
    );
    redis.srem(
      opts.prefix + ':targets:' + source_id,
      id
    );
    return done();
  };

  // get all sources a node has
  api.getSources = function(id, done){
    redis.smembers(
      opts.prefix + ':sources:' + id,
      function(err, sources){
        if(!sources){
          sources = [];
        }
        return done(false, sources);
      });
  };

  // remove all sources from a node
  api.delSources = function(id, done){
    this.getSources(
      id,
      function(err, sources){
        if(!sources){
          return done();
        }
        async.eachSeries(
          sources,
          function(source_id, next){
            api.delSource(id, source_id, next);
          },
          done
        );
      });
  };

  api.getTargets = function(id, done){
    redis.smembers(
      opts.prefix + ':targets:' + id,
      function(err, targets){
        if(!targets){
          targets = [];
        }
        return done(false, targets);
      });
  };

  // set/get in Redis last message output by node
  api.setData = function(id, data, done){
    redis.set(
      opts.prefix + ':data:' + id,
      JSON.stringify(data),
      function(){
        api.emit('setData', id, data, done);
      }
    );
  };

  api.getData = function(id, done){
    redis.get(
      opts.prefix + ':data:' + id,
      function(err, res){
        var data = null;
        if(res){
          data = JSON.parse(res);
        }
        done(false, data);
      });
  };

  // remove any existing sources a node has and replace with provided
  // set
  api.setSources = function(id, sources, done){
    this.delSources(id, function(){
      if(!sources){
        return done();
      }
      async.eachSeries(
        sources,
        function(source_id, next){
          api.addSource(id, source_id, next);
        },
        done
      );
    });
  };

  // load a node from Redis
  // Get a single node. Callback provides null if doesn't exist

  api.get = function(id, done){
    var node = {};

    redis.get(
      opts.prefix + ':nodes:' + id,
      function(err, res){

        if(err){
          console.log('ERROR', err);
          return done(err);
        }

        if(!res){
          return done(null, null);
        }

        res = JSON.parse(res);
        node.id = res.id;
        node.fn = res.fn;
        node.x = res.x;
        node.y = res.y;

        async.parallel([
          function(next){
            api.getSources(id, function(err, sources){
              node.sources = sources;
              next();
            });
          },
          function(next){
            api.getAttrs(id, function(err, res){
              node.attrs = res;
              next();
            });
          },
          function(next){
            api.getData(id, function(err, data){
              node.data = data;
              next();
            });
          }
        ], function(err){
          if(err){
            console.log(err);
            return done(err, null);
          }
          done(false, node);
        });
      });

  };

  // get all nodes in the topology
  api.all = function(done){

    var keys = [];
    var nodes = {};

    async.series([
      function(next){
        redis.keys(
          opts.prefix + ':nodes:*',
          function(err, res){
            if(res){
              keys = res;
            }
            next();
          });
      },
      function(next){
        async.eachSeries(
          keys,
          function(key, next){
            var id = _.last(key.split(':'));
            api.get(id, function(err, node){
              nodes[node.id] = node;
              next();
            });
          }, function(){
            next();
          });
      }], function(){
        done(false, nodes);
      });
  };

  // add a node to the topology
  api.add = function(node, done){

    this.get(
      node.id,
      function(err, n){

        // Duplicate node ids generate an error

        if(n){
          return done(new Error('duplicate-id'));
        }

        if(!node.hasOwnProperty('sources')){
          node.sources = [];
        }

        if(!node.hasOwnProperty('attrs')){
          node.attrs = {};
        }

        if(!node.hasOwnProperty('x')){
          node.x = null;
        }

        if(!node.hasOwnProperty('y')){
          node.y = null;
        }

        if(!node.hasOwnProperty('fn') || !node.fn){
          node.fn = 'thru';
        }

        if(!fns.hasOwnProperty(node.fn)){
          node.fn = 'thru';
        }

        // copy default attrs from fn definition

        var fn = fns[node.fn];

        if(fn.hasOwnProperty('attrs')){
          _.each(fn.attrs, function(v, k){
            if(!node.attrs.hasOwnProperty(k)){
              node.attrs[k] = v;
            }
          });
        }

        async.series([

          function(next){
            // save node. id and fn are immutable
            var key = opts.prefix + ':nodes:' + node.id;
            var val = {
              id: node.id,
              fn: node.fn,
              x: node.x,
              y: node.y
            };
            redis.set(key, JSON.stringify(val));
            next();
          },

          function(next){
            // set the inital sources on the node
            api.setSources(node.id, node.sources, next);
          },

          function(next){
            // init node
            if(!fn.hasOwnProperty('init')){
              return next();
            }

            // allow init to run. It may set some attrs
            fn.attrs = node.attrs;
            fn.init(next);
          },

          function(next){
            // set the initial attrs on the node
            api.setAttrs(node.id, node.attrs, next);
          }

        ], function(){
          api.emit('add', node.id, node, done);
        });
      });

  };

  // change a nodes's properties (just x and y for now)
  api.set = function(id, newVals, done){

    api.get(
      id,
      function(err, node){

        if(!node){
          return done();
        }

        var keys = ['x','y'];
        var vals = {};
        _.each(keys, function(key){
          if(newVals.hasOwnProperty(key)){
            vals[key] = newVals[key];
          }
        });

        var set = function(next){
          var key = opts.prefix + ':nodes:' + node.id;
          var val = {
            id: node.id,
            fn: node.fn,
            x: vals.x,
            y: vals.y
          };

          redis.set(
            key,
            JSON.stringify(val),
            next
          );
        };

        async.series([
          set
        ], function(){
          api.emit('set', id, vals, done);
        });
      });
  };

  // remove a node from the topology
  api.del = function(id, done){

    this.get(
      id,
      function(err, n){

        if(!n){
          return done();
        }

        async.series([

          function(next){
            // id and fn are immutable
            var key = opts.prefix + ':nodes:' + id;
            redis.del(key);
            next();
          },

          function(next){
            api.delSources(id, next);
          },

          function(next){
            api.delAttrs(id, next);
          }

        ], function(){
          api.emit('del', id, done);
        });
      });

  };

  // inject a message in to a node

  // message handler does process then emits event (redis pub) and
  // distributes message to targets (redis queue)
  api.inject = function(id, data, done){

    var message = {
      at: new Date().getTime(),
      id: id,
      data: data
    };

    redis.lpush(
      opts.prefix + ':queue',
      JSON.stringify(message),
      function(err){
        return done();
      });
  };

  // delete all messages off queue
  api.purge = function(done){

    redis.del(
      opts.prefix + ':queue',
      function(err){
        if(done){
          return done(null);
        }
      });
  };

  api.extract = function(done){

    redis.rpop(
      opts.prefix + ':queue',
      function(err, reply){

        if(err){
          console.log('#run error', err);
          return done(err, null, null);
        }

        if (!reply){
          // timed out, no message
          return done(null, null, null);
        }

        var id, msg, input;

        try {
          msg = JSON.parse(reply);
          // if brpop use this instead
          //input = JSON.parse(reply[1]);
        } catch (e) {
          // bad json
          console.log('bad json', e, reply);
          return done(err, null, null);
        }

        return done(null, msg.id, msg.data);

      });
  };

  api.process = function(id, data, done){

    api.get(
      id,
      function(err, node){

        if(!node){
          return done(new Error('unknown-node'));
        }

        // processing method from the node's function
        var fn = api.fns(node.fn).process;

        // process with the node's attrs and incoming data
        fn(
          node.attrs,
          data,
          function(err, attrs, output){

            // no attrs provided - don't change them
            if(typeof attrs === 'undefined'){
              return done();
            }

            // no specific output data from fn -- no message to pass to
            // nodes that source this one, just save the node's
            // (possibly changed) state and get out of here
            if(typeof output === 'undefined'){
              return api.setAttrs(id, attrs, done);
            }

            // there is some output. save it and pass on to connected nodes

            async.series([

              // save the node's new state
              function(next){
                api.setAttrs(id, attrs, function(){
                  next();
                });
              },

              // latch the output on the node
              function(next){
                api.setData(id, output, function(){
                  next();
                });
              },

              // distribute to connected nodes via the queue
              function(next){

                api.getTargets(
                  id,
                  function(err, targets){
                    if(targets.length === 0){
                      return next();
                    }
                    async.eachSeries(
                      targets,
                      function(target_id, done){
                        api.inject(target_id, output, done);
                      }, function(){
                        next();
                      });
                  });
              }
            ], function(){
              done(false, output);
            });
          });
      });

  };

  // if exists process one message from the queue
  api.tick = function(done){
    // pull a message off the queue
    api.extract(function(err, id, data){
      // nothing on queue
      if(!id){
        return done(null, null, null);
      }
      // id is node id to pass the data to
      api.process(
        id,
        data,
        function(err, output){
          return done(null, id, output);
        });
    });
  };

  // events stream

  api.emit = function(method, id, args, done){

    if (typeof args === 'function'){
      done = args;
      args = false;
    }

    //console.log('EMIT:', method, id, JSON.stringify(args));

    redis.publish(
      opts.prefix + ':events:' + id + ':' + method,
      JSON.stringify(args),
      done
    );
  };

  // list of events listeners
  var list = [];

  api.listener = function(id, handler){
    var key;

    // if no id supplied, then get all events (global listener)

    if(typeof id === 'function'){
      handler = id;
      id = false;
      key = opts.prefix + ':events:*';
    } else {
      key = opts.prefix + ':events:' + id + ':*';
    }

    var sub = Redis.createClient(
      opts.redis.port,
      opts.redis.host
    );

    sub.psubscribe(key);

    sub.on('pmessage', function(pattern, channel, msg){
      //console.log('**', msg, typeof msg);
      var args;
      if(msg && typeof msg !== 'undefined' && msg !== 'undefined'){
        args = JSON.parse(msg);
      }

      // key may have prefixes, so count from the end
      // e.g: df:test:events:my-node-id:setData
      var keys = channel.split(':');
      var id = keys[keys.length-2];
      var method = keys[keys.length-1];
      handler(method, id, args);
    });

    var quit = function(){
      sub.quit();
    };

    return {
      quit: quit
    };

  };

  api.worker = require('./worker');

  return api;

};
<|endoftext|>


<commit_before>const express = require('express');
const Rule = require('./rule.js');
const logger = require('./logger.js');
const Acme = require('./acme');
const http = require('http');
const https = require('https');
const config = require('rc')('lroxy', {
  email: '',
  httpPort: 8080,
  httpsPort: 8443,
  rules: [],
});

const app = express();
const rules = [];
for (const rule of config.rules) {
  rules.push(new Rule(rule));
}
const names = {};
for (const rule of rules) {
  names[rule.getFromHost()] = true;
}
delete names[config.commonName];

const acme = new Acme(config.email, config.commonName, Object.keys(names));

app.use((req, res, next) => {
  next();
  logger.info('Handled request.', {
    remoteAddress: req.connection.remoteAddress,
    method: req.method,
    referer: req.headers.referer,
    userAgent: req.headers['user-agent'],
    host: req.headers.host,
    url: req.url,
    statusCode: res.statusCode,
    proxyTo: res.proxyToAddress,
  });
});

app.get('/.well-known/acme-challenge/:token', (req, res) => {
  const challenge = acme.getChallenge(req.params.token);
  res.write(challenge);
  res.end();
});

app.use((req, res) => {
  if (!req.secure) {
    return res.redirect("https://" + req.headers.host + req.url);
  }

  for (const rule of rules) {
    if (rule.match(req)) {
      rule.handle(req, res);
      return;
    }
  }

  // By default, reply service unavaliable if there is no matching reverse
  // proxy rule.
  res.writeHead(503, 'Service Unavaliable');
  res.end('Service Unavaliable');
});

module.exports = async function() {
  http.createServer(app).listen(config.httpPort);
  logger.info(`lroxy (http) started on ${config.httpPort}.`);
  const sslInfo = await acme.run();
  https.createServer({
    key: sslInfo.key,
    cert: sslInfo.cert,
  }, app).listen(config.httpsPort);
  logger.info(`lroxy (https) started on ${config.httpsPort}.`);
};
<commit_msg>Allow lroxy to run with HTTP only.<commit_after>const express = require('express');
const Rule = require('./rule.js');
const logger = require('./logger.js');
const Acme = require('./acme');
const http = require('http');
const https = require('https');
const config = require('rc')('lroxy', {
  email: null,
  commonName: null,
  httpPort: 8080,
  httpsPort: null,
  rules: [],
});

const app = express();
const rules = [];
for (const rule of config.rules) {
  rules.push(new Rule(rule));
}
const names = {};
for (const rule of rules) {
  names[rule.getFromHost()] = true;
}
delete names[config.commonName];

const acme = new Acme(config.email, config.commonName, Object.keys(names));

app.use((req, res, next) => {
  next();
  logger.info('Handled request.', {
    remoteAddress: req.connection.remoteAddress,
    method: req.method,
    referer: req.headers.referer,
    userAgent: req.headers['user-agent'],
    url: `${req.secure ? 'https' : 'http'}://${req.hostname}${req.url}`,
    statusCode: res.statusCode,
    proxyTo: res.proxyToAddress,
  });
});

app.get('/.well-known/acme-challenge/:token', (req, res) => {
  const challenge = acme.getChallenge(req.params.token);
  res.write(challenge);
  res.end();
});

app.use((req, res) => {
  if (config.httpsPort != null && !req.secure) {
    return res.redirect('https://' + req.headers.host + req.url);
  }

  for (const rule of rules) {
    if (rule.match(req)) {
      rule.handle(req, res);
      return;
    }
  }

  // By default, reply service unavaliable if there is no matching reverse
  // proxy rule.
  res.writeHead(503, 'Service Unavaliable');
  res.end('Service Unavaliable');
});

module.exports = async function() {
  http.createServer(app).listen(config.httpPort);
  logger.info(`lroxy (http) started on ${config.httpPort}.`);

  if (config.httpsPort != null) {
    const sslInfo = await acme.run();
    https.createServer({
      key: sslInfo.key,
      cert: sslInfo.cert,
    }, app).listen(config.httpsPort);
    logger.info(`lroxy (https) started on ${config.httpsPort}.`);
  }
};
<|endoftext|>