<commit_before>/*
 * CouchDB Worker
 *
 * A worker module that manages state.
 *
 * Author: Johannes J. Schmidt
 * (c) null2 GmbH, 2012
 * MIT Licensed
 */

var request = require("request");
var extend = require("node.extend");

module.exports = Worker;

function Worker(config, db) {
  if (typeof config !== 'object') throw('I need a config!');

  // worker name
  if (!config.name) throw('I need a name!');
  this.name = config.name;

  // server
  if (!config.server) throw('I need a server!');
  this.server = config.server;
  // strip username and password for log messages
  this.server_stripped = this.server.replace(/:\/\/.+:.+@/, '://');

  // processor
  if (typeof config.processor !== 'object') throw('I need a processor!');
  if (typeof config.processor.process !== 'function') throw('I need a processor.process function!');
  this.processor = config.processor;

  // config and status id
  if (config.config_id && typeof config.config_id !== 'string') throw('config_id must be a string!');
  this.config_id = config.config_id || 'worker-config/' + this.name;
  if (config.status_id && typeof config.status_id !== 'string') throw('status_id must be a string!');
  this.status_id = config.status_id || 'worker-status/' + this.name;

  // batch size
  if (config.batch_size && typeof config.batch_size !== 'number') throw('batch_size must be a number!');
  this.batch_size = typeof config.batch_size === 'number' ? config.batch_size : 10;

  // timeout
  if (config.timeout && typeof config.timeout !== 'number') throw('timeout must be a number!');
  this.timeout = typeof config.timeout === 'number' ? config.timeout : 60000;

  // since
  if (config.since && typeof config.since !== 'number') throw('since must be a number!');
  this.since = config.since || 0;

  this.db = db;

  console.log(this.name + ' worker listening on ' + this.server_stripped + '/' + this.db);

  // initially get config
  request({
    url: this.server
      + '/' + encodeURIComponent(this.db)
      + '/' + encodeURIComponent(this.config_id),
    json: true
  }, function(error, resp, doc) {
    if (error !== null) {
      console.error(this.server_stripped + '/' + this.db + ' Error fetching config: ', error);
      return;
    }

    if (resp.statusCode < 400 && typeof doc === 'object') {
      this.config = doc;
    }

    // initially get status
    request({
      url: this.server
        + '/' + encodeURIComponent(this.db)
        + '/' + encodeURIComponent(this.status_id),
      json: true
    }, function(error, resp, doc) {
      if (error !== null) {
        console.error(this.server_stripped + '/' + this.db + ' Error fetching status: ', error);
        return;
      }

      if (resp.statusCode < 400 && typeof doc === 'object') {
        this.status = doc;
      }

      this._listen();
    }.bind(this));
  }.bind(this));
};

// request next batch of changes
Worker.prototype._listen = function() {
  request({
    url: this.server
      + '/' + encodeURIComponent(this.db)
      + '/_changes?include_docs=true&feed=longpoll'
      + '&timeout=' + this.timeout
      + '&limit=' + this.batch_size
      // pick the highest seq we have: status.last_seq, since or last_seq
      + '&since=' + Math.max((this.status && this.status.last_seq) || 0, this.since || 0, this.last_seq || 0),
    json: true
  }, this._onchanges.bind(this));
};

// changes callback
Worker.prototype._onchanges = function(error, resp, changes) {
  // TODO: retry, like follow does
  if (this._checkResponse(error, resp, changes, 'get changes feed')) return;

  this.docs_processed = 0;
  this.docs_checked = 0;
  this.changes = changes.results;

  // remember update seq
  this.last_seq = changes.last_seq;

  // start processing changes
  this._nextChange();
};

// get change from change results
// remove from changes results
// process it
// if no more changes left, call process done
Worker.prototype._nextChange = function() {
  var change = this.changes.pop();

  if (!change) {
    return this._changesDone();
  }

  // config document
  if (change.id === this.config_id) {
    this.config = change.doc;
    if (change.deleted) delete this.config;
    return this._nextChange();
  }

  // status document
  if (change.id === this.status_id) {
    // update status
    this.status = change.doc;
    if (change.deleted) delete this.status;
    return this._nextChange();
  }

  if (!change.deleted && this._check(change.doc)) {
    // process doc
    this._process(change.doc);
  } else {
    // ignore doc
    this._nextChange();
  }
};

// check a doc if it needs processing
// that is if we have a config doc
// and the processor function returns true for that doc
Worker.prototype._check = function(doc) {
  this.docs_checked++;

  // ignore design documents
  if (doc._id.match(/^_design\//)) return false;

  // ignore if we do not have a config doc
  if (!this.config) return false;

  // ignore if we already processed that doc
  if (doc.worker_status && doc.worker_status[this.name]) return false;

  // default is true for all docs
  if (typeof this.processor.check !== 'function') return true;

  return this.processor.check.call(this, doc);
};

// process a doc
Worker.prototype._process = function(doc) {
  this._log('start: ' + doc._id);

  this._setWorkerStatus(doc, 'triggered');

  // save doc
  request({
    url: this.server
      + '/' + encodeURIComponent(this.db)
      + '/' + encodeURIComponent(doc._id),
    method: 'PUT',
    body: JSON.stringify(doc),
    json: true
  }, function(error, resp, data) {
    // ignore on errors, we will get this doc again as a change
    if (this._checkResponse(error, resp, data, 'grab doc')) return this._nextChange();

    // update doc's rev
    doc._rev = data.rev;

    // process doc
    this.processor.process.call(this, doc, function(error, attributes) {
      this._processDone(error, doc, attributes);
    }.bind(this));
  }.bind(this));
};

// insert worker status into doc
Worker.prototype._setWorkerStatus = function(doc, status, error) {
  doc.worker_status || (doc.worker_status = {});
  doc.worker_status[this.name] = {
    status: status
  };
  if (error) doc.worker_status[this.name].error = error;
};

// finish processing of one doc
Worker.prototype._processDone = function(error, doc, attributes) {
  this.docs_processed++;

  this._commit(doc, attributes, error ? 'error' : 'completed', error);
};

// TODO: implement resolveConflict
Worker.prototype._commit = function(doc, attributes, status, processError) {
  this._setWorkerStatus(doc, status, processError);
  this._mergeResult(doc, attributes);

  // save doc
  request({
    url: this.server
      + '/' + encodeURIComponent(this.db)
      + '/' + encodeURIComponent(doc._id),
    method: 'PUT',
    body: JSON.stringify(doc),
    json: true
  }, function(error, resp, data) {
    // reset on errors
    if (this._checkResponse(error, resp, data, 'commit doc')) return this._reset(newDoc);

    // update doc's rev
    doc._rev = data.rev;

    this._log('finish: ' + doc._id);

    // continue
    this._nextChange();
  }.bind(this));
};

// reset state
Worker.prototype._reset = function(doc) {
  request({
    url: this.server
      + '/' + encodeURIComponent(this.db)
      + '/' + encodeURIComponent(doc._id),
    json: true
  }, function(error, resp, newDoc) {
    // ignore on error
    // TODO: retry on errors
    if (this._checkResponse(error, resp, data, 'fetch doc')) return this._nextChange();

    // reset the worker state
    if (doc.worker_status) delete doc.worker_status[this.name];

    // save doc
    request({
      url: this.server
        + '/' + encodeURIComponent(this.db)
        + '/' + encodeURIComponent(doc._id),
      method: 'PUT',
      body: JSON.stringify(doc),
      json: true
    }, function(error, resp, data) {
      // retry to reset on errors
      // TODO: do not try in that endless loop
      if (this._checkResponse(error, resp, data, 'commit doc')) return this._reset(doc);

      this._log('reset: ' + doc._id);

      // continue
      this._nextChange();
    }.bind(this));
  }.bind(this));
};

// merge the result of a processor's process function
Worker.prototype._mergeResult = function(doc, attributes) {
  if (typeof attributes !== 'object') {
    console.error("processor.process done function called with non-object parameter!");
    console.error(attributes);
    return;
  }

  extend(doc, attributes);
};

// called when batch done
// store update seq
// listen again
Worker.prototype._changesDone = function() {
  // only store status if we checked docs something
  // (ignore changes of status and config docs)
  if (!this.docs_checked) return this._listen();

  // update status information
  this.status || (this.status = { _id: this.status_id });
  this.status.last_seq = this.last_seq;
  this.status.docs_checked || (this.status.docs_checked = 0);
  this.status.docs_checked += this.docs_checked;
  this.status.docs_processed || (this.status.docs_processed = 0);
  this.status.docs_processed += this.docs_processed;

  // store status
  request({
    url: this.server
      + '/' + encodeURIComponent(this.db)
      + '/' + encodeURIComponent(this.status_id),
    method: 'PUT',
    body: JSON.stringify(this.status),
    json: true
  }, function(error, resp, data) {
    if (this._checkResponse(error, resp, data, 'store status')) return;

    // update status rev
    this.status._rev = data.rev;

    // start listening for next changes
    this._listen();
  }.bind(this));
};

// check response and log errors
Worker.prototype._checkResponse = function(error, resp, data, msg) {
  msg || (msg = 'in response');

  if (error !== null) {
    console.error(this.server_stripped + '/' + this.db + ' Error ' + msg + ': ', error);
    return {
      msg: msg,
      error: error
    };
  }
  if (resp.statusCode >= 400 || typeof data !== 'object') {
    console.error(this.server_stripped + '/' + this.db + ' Response Error ' + msg + ': ' + resp.statusCode, data);
    return {
      msg: msg,
      status: resp.statusCode,
      error: data
    };
  }

  return null;
};

// log messages with server and db
Worker.prototype._log = function(msg) {
  console.log(this.server_stripped + '/' + this.db + ' ' + msg);
};
<commit_msg>enforce processor.check to be a function if given<commit_after>/*
 * CouchDB Worker
 *
 * A worker module that manages state.
 *
 * Author: Johannes J. Schmidt
 * (c) null2 GmbH, 2012
 * MIT Licensed
 */

var request = require("request");
var extend = require("node.extend");

module.exports = Worker;

function Worker(config, db) {
  if (typeof config !== 'object') throw('I need a config!');

  // worker name
  if (!config.name) throw('I need a name!');
  this.name = config.name;

  // server
  if (!config.server) throw('I need a server!');
  this.server = config.server;
  // strip username and password for log messages
  this.server_stripped = this.server.replace(/:\/\/.+:.+@/, '://');

  // processor
  if (typeof config.processor !== 'object') throw('I need a processor!');
  if (typeof config.processor.process !== 'function') throw('I need a processor.process function!');
  if (config.processor.check && typeof config.processor.process !== 'function') throw('processor.check must be a function if given!');
  this.processor = config.processor;

  // config and status id
  if (config.config_id && typeof config.config_id !== 'string') throw('config_id must be a string!');
  this.config_id = config.config_id || 'worker-config/' + this.name;
  if (config.status_id && typeof config.status_id !== 'string') throw('status_id must be a string!');
  this.status_id = config.status_id || 'worker-status/' + this.name;

  // batch size
  if (config.batch_size && typeof config.batch_size !== 'number') throw('batch_size must be a number!');
  this.batch_size = typeof config.batch_size === 'number' ? config.batch_size : 10;

  // timeout
  if (config.timeout && typeof config.timeout !== 'number') throw('timeout must be a number!');
  this.timeout = typeof config.timeout === 'number' ? config.timeout : 60000;

  // since
  if (config.since && typeof config.since !== 'number') throw('since must be a number!');
  this.since = config.since || 0;

  this.db = db;

  console.log(this.name + ' worker listening on ' + this.server_stripped + '/' + this.db);

  // initially get config
  request({
    url: this.server
      + '/' + encodeURIComponent(this.db)
      + '/' + encodeURIComponent(this.config_id),
    json: true
  }, function(error, resp, doc) {
    if (error !== null) {
      console.error(this.server_stripped + '/' + this.db + ' Error fetching config: ', error);
      return;
    }

    if (resp.statusCode < 400 && typeof doc === 'object') {
      this.config = doc;
    }

    // initially get status
    request({
      url: this.server
        + '/' + encodeURIComponent(this.db)
        + '/' + encodeURIComponent(this.status_id),
      json: true
    }, function(error, resp, doc) {
      if (error !== null) {
        console.error(this.server_stripped + '/' + this.db + ' Error fetching status: ', error);
        return;
      }

      if (resp.statusCode < 400 && typeof doc === 'object') {
        this.status = doc;
      }

      this._listen();
    }.bind(this));
  }.bind(this));
};

// request next batch of changes
Worker.prototype._listen = function() {
  request({
    url: this.server
      + '/' + encodeURIComponent(this.db)
      + '/_changes?include_docs=true&feed=longpoll'
      + '&timeout=' + this.timeout
      + '&limit=' + this.batch_size
      // pick the highest seq we have: status.last_seq, since or last_seq
      + '&since=' + Math.max((this.status && this.status.last_seq) || 0, this.since || 0, this.last_seq || 0),
    json: true
  }, this._onchanges.bind(this));
};

// changes callback
Worker.prototype._onchanges = function(error, resp, changes) {
  // TODO: retry, like follow does
  if (this._checkResponse(error, resp, changes, 'get changes feed')) return;

  this.docs_processed = 0;
  this.docs_checked = 0;
  this.changes = changes.results;

  // remember update seq
  this.last_seq = changes.last_seq;

  // start processing changes
  this._nextChange();
};

// get change from change results
// remove from changes results
// process it
// if no more changes left, call process done
Worker.prototype._nextChange = function() {
  var change = this.changes.pop();

  if (!change) {
    return this._changesDone();
  }

  // config document
  if (change.id === this.config_id) {
    this.config = change.doc;
    if (change.deleted) delete this.config;
    return this._nextChange();
  }

  // status document
  if (change.id === this.status_id) {
    // update status
    this.status = change.doc;
    if (change.deleted) delete this.status;
    return this._nextChange();
  }

  if (!change.deleted && this._check(change.doc)) {
    // process doc
    this._process(change.doc);
  } else {
    // ignore doc
    this._nextChange();
  }
};

// check a doc if it needs processing
// that is if we have a config doc
// and the processor function returns true for that doc
Worker.prototype._check = function(doc) {
  this.docs_checked++;

  // ignore design documents
  if (doc._id.match(/^_design\//)) return false;

  // ignore if we do not have a config doc
  if (!this.config) return false;

  // ignore if we already processed that doc
  if (doc.worker_status && doc.worker_status[this.name]) return false;

  // default is true for all docs
  if (typeof this.processor.check !== 'function') return true;

  return this.processor.check.call(this, doc);
};

// process a doc
Worker.prototype._process = function(doc) {
  this._log('start: ' + doc._id);

  this._setWorkerStatus(doc, 'triggered');

  // save doc
  request({
    url: this.server
      + '/' + encodeURIComponent(this.db)
      + '/' + encodeURIComponent(doc._id),
    method: 'PUT',
    body: JSON.stringify(doc),
    json: true
  }, function(error, resp, data) {
    // ignore on errors, we will get this doc again as a change
    if (this._checkResponse(error, resp, data, 'grab doc')) return this._nextChange();

    // update doc's rev
    doc._rev = data.rev;

    // process doc
    this.processor.process.call(this, doc, function(error, attributes) {
      this._processDone(error, doc, attributes);
    }.bind(this));
  }.bind(this));
};

// insert worker status into doc
Worker.prototype._setWorkerStatus = function(doc, status, error) {
  doc.worker_status || (doc.worker_status = {});
  doc.worker_status[this.name] = {
    status: status
  };
  if (error) doc.worker_status[this.name].error = error;
};

// finish processing of one doc
Worker.prototype._processDone = function(error, doc, attributes) {
  this.docs_processed++;

  this._commit(doc, attributes, error ? 'error' : 'completed', error);
};

// TODO: implement resolveConflict
Worker.prototype._commit = function(doc, attributes, status, processError) {
  this._setWorkerStatus(doc, status, processError);
  this._mergeResult(doc, attributes);

  // save doc
  request({
    url: this.server
      + '/' + encodeURIComponent(this.db)
      + '/' + encodeURIComponent(doc._id),
    method: 'PUT',
    body: JSON.stringify(doc),
    json: true
  }, function(error, resp, data) {
    // reset on errors
    if (this._checkResponse(error, resp, data, 'commit doc')) return this._reset(newDoc);

    // update doc's rev
    doc._rev = data.rev;

    this._log('finish: ' + doc._id);

    // continue
    this._nextChange();
  }.bind(this));
};

// reset state
Worker.prototype._reset = function(doc) {
  request({
    url: this.server
      + '/' + encodeURIComponent(this.db)
      + '/' + encodeURIComponent(doc._id),
    json: true
  }, function(error, resp, newDoc) {
    // ignore on error
    // TODO: retry on errors
    if (this._checkResponse(error, resp, data, 'fetch doc')) return this._nextChange();

    // reset the worker state
    if (doc.worker_status) delete doc.worker_status[this.name];

    // save doc
    request({
      url: this.server
        + '/' + encodeURIComponent(this.db)
        + '/' + encodeURIComponent(doc._id),
      method: 'PUT',
      body: JSON.stringify(doc),
      json: true
    }, function(error, resp, data) {
      // retry to reset on errors
      // TODO: do not try in that endless loop
      if (this._checkResponse(error, resp, data, 'commit doc')) return this._reset(doc);

      this._log('reset: ' + doc._id);

      // continue
      this._nextChange();
    }.bind(this));
  }.bind(this));
};

// merge the result of a processor's process function
Worker.prototype._mergeResult = function(doc, attributes) {
  if (typeof attributes !== 'object') {
    console.error("processor.process done function called with non-object parameter!");
    console.error(attributes);
    return;
  }

  extend(doc, attributes);
};

// called when batch done
// store update seq
// listen again
Worker.prototype._changesDone = function() {
  // only store status if we checked docs something
  // (ignore changes of status and config docs)
  if (!this.docs_checked) return this._listen();

  // update status information
  this.status || (this.status = { _id: this.status_id });
  this.status.last_seq = this.last_seq;
  this.status.docs_checked || (this.status.docs_checked = 0);
  this.status.docs_checked += this.docs_checked;
  this.status.docs_processed || (this.status.docs_processed = 0);
  this.status.docs_processed += this.docs_processed;

  // store status
  request({
    url: this.server
      + '/' + encodeURIComponent(this.db)
      + '/' + encodeURIComponent(this.status_id),
    method: 'PUT',
    body: JSON.stringify(this.status),
    json: true
  }, function(error, resp, data) {
    if (this._checkResponse(error, resp, data, 'store status')) return;

    // update status rev
    this.status._rev = data.rev;

    // start listening for next changes
    this._listen();
  }.bind(this));
};

// check response and log errors
Worker.prototype._checkResponse = function(error, resp, data, msg) {
  msg || (msg = 'in response');

  if (error !== null) {
    console.error(this.server_stripped + '/' + this.db + ' Error ' + msg + ': ', error);
    return {
      msg: msg,
      error: error
    };
  }
  if (resp.statusCode >= 400 || typeof data !== 'object') {
    console.error(this.server_stripped + '/' + this.db + ' Response Error ' + msg + ': ' + resp.statusCode, data);
    return {
      msg: msg,
      status: resp.statusCode,
      error: data
    };
  }

  return null;
};

// log messages with server and db
Worker.prototype._log = function(msg) {
  console.log(this.server_stripped + '/' + this.db + ' ' + msg);
};
<|endoftext|>


<commit_before>var _ = require('lodash'),
	aws = require('aws-sdk'),
	async = require('async'),
	config = require('./config').Config,
	fs = require('fs'),
	request = require('request'),
	Thumbnailer = require('./thumbnailer').Thumbnailer;

/**
 * Initialize the Worker
 *
 * @param object opts Worker configuration. Optional.
 */
function Worker(opts) {
	_.extend(this, {
		grabber: null,
		saver: null,
		logger: require('./logger')
	}, opts);

	this.sqs = new aws.SQS({
		accessKeyId: config.get('awsKey'),
		secretAccessKey: config.get('awsSecret'),
		region: config.get('awsRegion')
	});

	config.set('sqsQueueUrl', this.sqs.endpoint.protocol + '//' + this.sqs.endpoint.hostname + '/' + config.get('sqsQueue'));
}

/**
 * Start the worker
 */
Worker.prototype.start = function() {
	this._processSQSMessage();
};

/**
 * Process the next message in the queue
 */
Worker.prototype._processSQSMessage = function() {
	var _this = this;

	this.logger.info('wait for message on ' + config.get('sqsQueue'));

	this.sqs.receiveMessage( { QueueUrl: config.get('sqsQueueUrl'), MaxNumberOfMessages: 1 }, function (err, job) {
		if (err) {
			_this.logger.error(err);
			_this._nextJob();
			return;
		}

		if (!job.Messages || job.Messages.length === 0) {
			_this._nextJob();
			return;
		}

		// Handle the message we pulled off the queue.
		var handle = job.Messages[0].ReceiptHandle,
			body = null;

		try { // a JSON string message body is accepted.
			body = JSON.parse( job.Messages[0].Body );
		} catch(e) {
			if (e instanceof SyntaxError) {
				// a Base64 encoded JSON string message body is also accepted.
				body = JSON.parse( new Buffer(job.Messages[0].Body, 'base64').toString( 'binary' ) );
			} else {
				// TODO: figure out if throwing is actually
				// the right thing to do here.
				throw e;
			}
		}

		_this._runJob(handle, body, function() {
			_this._nextJob();
		});
	});
};

/**
 * asynchronously enqueue reading the next thumbnail
 * job off of SQS.
 */
Worker.prototype._nextJob = function() {
	var _this = this;

	process.nextTick(function() {
		_this._processSQSMessage();
	});
};

/**
 * Process a job from the queue
 *
 * @param string handle The SQS message handle
 * @param object job The job parameters
 * @param function callback The callback function
 */
Worker.prototype._runJob = function(handle, job, callback) {
	/**
		job = {
			"original": "/foo/awesome.jpg",
			// OR:
			"resources": [
			// List of resources to download
			],
			"prefix": "/foo/awesome",
			"descriptions": [{
				"suffix": "small",
				"width": 64,
				"height": 64
			}],
		}
	*/

	// handle legacy, 'original' key vs. 'resources'.
	if (job.original) job.resources = [job.original];

	// if the prefix is missing, default to the filename.
	if (!job.perfix) job.prefix = job.resources[0].split('.').slice(0, -1).join('.');

	var _this = this;

	async.waterfall([
		function(done) {
			async.mapLimit(job.resources, 5, function(resource, done) {
				_this._downloadFromS3(job.bucket, job.region, resource, done);
			}, done);
		},
		function(s3Downloads, done) {
			_this._createThumbnails(s3Downloads, job, function(err, uploadedFiles) {
				async.forEach(_.pluck(s3Downloads, 'localPath'), fs.unlink, function(errUnlink) {
					if (errUnlink) {
						_this.logger.warn("WARNING: failed to delete temporary file " + errUnlink.path);
					}
					done(err, uploadedFiles);
				});
			});
		},
		function(uploadedFiles, done) {
			job.output = uploadedFiles;
			_this._notify(job, done);
		}
	], function(err) {
		if (!err) _this._deleteJob(handle);
		else _this.logger.error(err.message);
		callback();
	});
};

/**
 * Download the image from S3
 *
 * @param string remoteImagePath The s3 path to the image
 * @param function callback The callback function
 */
Worker.prototype._downloadFromS3 = function(bucket, region, remoteImagePath, callback) {
	// allow a default bucket to be overridden.
	bucket = bucket || config.get('s3Bucket');
	region = region || config.get('awsRegion');

	this.grabber.download(bucket, region, remoteImagePath, function(err, localPath, metadata) {
		// Leave the job in the queue if an error occurs.
		if (err) return callback(err);
		callback(null, {localPath: localPath, metadata: metadata});
	});
};

/**
 * Create thumbnails for the image
 *
 * @param string localPath The local path to store the images
 * @param object job The job description
 * @param function callback The callback function
 */
Worker.prototype._createThumbnails = function(s3Downloads, job, callback) {
	var _this = this,
		localPaths = _.pluck(s3Downloads, 'localPath'),
		work = [],
		bucket = job.bucket || config.get('s3Bucket'),
		region = job.region || config.get('awsRegion');

	// Create thumbnailing work for each thumbnail description.
	job.descriptions.forEach(function(description) {
		work.push(function(done) {

			var remoteImagePath = _this._thumbnailKey(job.prefix, description.suffix, description.format);

			(new Thumbnailer()).execute(description, localPaths, function(err, convertedImagePath) {

				if (err) {
					_this.logger.error(err);
					done();
				} else {
					_this._saveThumbnailToS3(bucket, region, convertedImagePath, remoteImagePath, s3Downloads[0].metadata, function(err) {
						if (err) _this.logger.error(err);
						done(null, remoteImagePath);
					});
				}

			});

		});
	});

	// perform thumbnailing in parallel.
	async.parallel(work, callback);
};

/**
 * Save the thumbnail to S3
 *
 * @param string convertedImagePath The local path to the image
 * @param string remoteImagePath The S3 path for the image
 * @param function callback The callback function
 */
Worker.prototype._saveThumbnailToS3 = function(bucket, region, convertedImagePath, remoteImagePath, metadata, callback) {
	this.saver.save(bucket, region, convertedImagePath, remoteImagePath, metadata, function(err) {
		fs.unlink(convertedImagePath, function() {
			callback(err);
		});
	});
};

/**
 * Generate a path for this thumbnail
 *
 * @param string original The original image path
 * @param string suffix The thumbnail suffix. e.g. "small"
 * @param string format The thumbnail format. e.g. "jpg". Optional.
 */
Worker.prototype._thumbnailKey = function(prefix, suffix, format) {
	return prefix + '_' + suffix + '.' + (format || 'jpg');
};

/**
 * Remove a job from the queue
 *
 * @param string handle The SQS message handle
 */
Worker.prototype._deleteJob = function(handle) {
	var _this = this;

	this.sqs.deleteMessage({QueueUrl: config.get('sqsQueueUrl'), ReceiptHandle: handle}, function(err, resp) {
		if (err) {
			_this.logger.error("error deleting thumbnail job " + handle, err);
			return;
		}
		_this.logger.info('deleted thumbnail job ' + handle);
	});
};

/**
 * Call notification url
 *
 * @param string job: the body of the SQS job.
 */
Worker.prototype._notify = function(job, cb) {
	var _this = this;

  if (!job.notify) return cb();

	var options = {
		method: "POST",
		url: job.notify,
		json: true,
		body: job
	};

	request.post(options, function(err) {
		if (!err) _this.logger.error('notified:', job.notify);
		return cb();
	});
};

exports.Worker = Worker;
<commit_msg>Corrected typo in worker preventing custom 'prefix' from being used.<commit_after>var _ = require('lodash'),
	aws = require('aws-sdk'),
	async = require('async'),
	config = require('./config').Config,
	fs = require('fs'),
	request = require('request'),
	Thumbnailer = require('./thumbnailer').Thumbnailer;

/**
 * Initialize the Worker
 *
 * @param object opts Worker configuration. Optional.
 */
function Worker(opts) {
	_.extend(this, {
		grabber: null,
		saver: null,
		logger: require('./logger')
	}, opts);

	this.sqs = new aws.SQS({
		accessKeyId: config.get('awsKey'),
		secretAccessKey: config.get('awsSecret'),
		region: config.get('awsRegion')
	});

	config.set('sqsQueueUrl', this.sqs.endpoint.protocol + '//' + this.sqs.endpoint.hostname + '/' + config.get('sqsQueue'));
}

/**
 * Start the worker
 */
Worker.prototype.start = function() {
	this._processSQSMessage();
};

/**
 * Process the next message in the queue
 */
Worker.prototype._processSQSMessage = function() {
	var _this = this;

	this.logger.info('wait for message on ' + config.get('sqsQueue'));

	this.sqs.receiveMessage( { QueueUrl: config.get('sqsQueueUrl'), MaxNumberOfMessages: 1 }, function (err, job) {
		if (err) {
			_this.logger.error(err);
			_this._nextJob();
			return;
		}

		if (!job.Messages || job.Messages.length === 0) {
			_this._nextJob();
			return;
		}

		// Handle the message we pulled off the queue.
		var handle = job.Messages[0].ReceiptHandle,
			body = null;

		try { // a JSON string message body is accepted.
			body = JSON.parse( job.Messages[0].Body );
		} catch(e) {
			if (e instanceof SyntaxError) {
				// a Base64 encoded JSON string message body is also accepted.
				body = JSON.parse( new Buffer(job.Messages[0].Body, 'base64').toString( 'binary' ) );
			} else {
				// TODO: figure out if throwing is actually
				// the right thing to do here.
				throw e;
			}
		}

		_this._runJob(handle, body, function() {
			_this._nextJob();
		});
	});
};

/**
 * asynchronously enqueue reading the next thumbnail
 * job off of SQS.
 */
Worker.prototype._nextJob = function() {
	var _this = this;

	process.nextTick(function() {
		_this._processSQSMessage();
	});
};

/**
 * Process a job from the queue
 *
 * @param string handle The SQS message handle
 * @param object job The job parameters
 * @param function callback The callback function
 */
Worker.prototype._runJob = function(handle, job, callback) {
	/**
		job = {
			"original": "/foo/awesome.jpg",
			// OR:
			"resources": [
			// List of resources to download
			],
			"prefix": "/foo/awesome",
			"descriptions": [{
				"suffix": "small",
				"width": 64,
				"height": 64
			}],
		}
	*/

	// handle legacy, 'original' key vs. 'resources'.
	if (job.original) job.resources = [job.original];

	// if the prefix is missing, default to the filename.
	if (!job.prefix) job.prefix = job.resources[0].split('.').slice(0, -1).join('.');

	var _this = this;

	async.waterfall([
		function(done) {
			async.mapLimit(job.resources, 5, function(resource, done) {
				_this._downloadFromS3(job.bucket, job.region, resource, done);
			}, done);
		},
		function(s3Downloads, done) {
			_this._createThumbnails(s3Downloads, job, function(err, uploadedFiles) {
				async.forEach(_.pluck(s3Downloads, 'localPath'), fs.unlink, function(errUnlink) {
					if (errUnlink) {
						_this.logger.warn("WARNING: failed to delete temporary file " + errUnlink.path);
					}
					done(err, uploadedFiles);
				});
			});
		},
		function(uploadedFiles, done) {
			job.output = uploadedFiles;
			_this._notify(job, done);
		}
	], function(err) {
		if (!err) _this._deleteJob(handle);
		else _this.logger.error(err.message);
		callback();
	});
};

/**
 * Download the image from S3
 *
 * @param string remoteImagePath The s3 path to the image
 * @param function callback The callback function
 */
Worker.prototype._downloadFromS3 = function(bucket, region, remoteImagePath, callback) {
	// allow a default bucket to be overridden.
	bucket = bucket || config.get('s3Bucket');
	region = region || config.get('awsRegion');

	this.grabber.download(bucket, region, remoteImagePath, function(err, localPath, metadata) {
		// Leave the job in the queue if an error occurs.
		if (err) return callback(err);
		callback(null, {localPath: localPath, metadata: metadata});
	});
};

/**
 * Create thumbnails for the image
 *
 * @param string localPath The local path to store the images
 * @param object job The job description
 * @param function callback The callback function
 */
Worker.prototype._createThumbnails = function(s3Downloads, job, callback) {
	var _this = this,
		localPaths = _.pluck(s3Downloads, 'localPath'),
		work = [],
		bucket = job.bucket || config.get('s3Bucket'),
		region = job.region || config.get('awsRegion');

	// Create thumbnailing work for each thumbnail description.
	job.descriptions.forEach(function(description) {
		work.push(function(done) {

			var remoteImagePath = _this._thumbnailKey(job.prefix, description.suffix, description.format);

			(new Thumbnailer()).execute(description, localPaths, function(err, convertedImagePath) {

				if (err) {
					_this.logger.error(err);
					done();
				} else {
					_this._saveThumbnailToS3(bucket, region, convertedImagePath, remoteImagePath, s3Downloads[0].metadata, function(err) {
						if (err) _this.logger.error(err);
						done(null, remoteImagePath);
					});
				}

			});

		});
	});

	// perform thumbnailing in parallel.
	async.parallel(work, callback);
};

/**
 * Save the thumbnail to S3
 *
 * @param string convertedImagePath The local path to the image
 * @param string remoteImagePath The S3 path for the image
 * @param function callback The callback function
 */
Worker.prototype._saveThumbnailToS3 = function(bucket, region, convertedImagePath, remoteImagePath, metadata, callback) {
	this.saver.save(bucket, region, convertedImagePath, remoteImagePath, metadata, function(err) {
		fs.unlink(convertedImagePath, function() {
			callback(err);
		});
	});
};

/**
 * Generate a path for this thumbnail
 *
 * @param string original The original image path
 * @param string suffix The thumbnail suffix. e.g. "small"
 * @param string format The thumbnail format. e.g. "jpg". Optional.
 */
Worker.prototype._thumbnailKey = function(prefix, suffix, format) {
	return prefix + '_' + suffix + '.' + (format || 'jpg');
};

/**
 * Remove a job from the queue
 *
 * @param string handle The SQS message handle
 */
Worker.prototype._deleteJob = function(handle) {
	var _this = this;

	this.sqs.deleteMessage({QueueUrl: config.get('sqsQueueUrl'), ReceiptHandle: handle}, function(err, resp) {
		if (err) {
			_this.logger.error("error deleting thumbnail job " + handle, err);
			return;
		}
		_this.logger.info('deleted thumbnail job ' + handle);
	});
};

/**
 * Call notification url
 *
 * @param string job: the body of the SQS job.
 */
Worker.prototype._notify = function(job, cb) {
	var _this = this;

  if (!job.notify) return cb();

	var options = {
		method: "POST",
		url: job.notify,
		json: true,
		body: job
	};

	request.post(options, function(err) {
		if (!err) _this.logger.error('notified:', job.notify);
		return cb();
	});
};

exports.Worker = Worker;
<|endoftext|>


<commit_before>var
	beanstalk   = require('./client'),
	events      = require('events'),
	util        = require('util'),
	yaml        = require('js-yaml')
	;

/*
Events emitted:

error: payload is error; execution is halted
close: no payload

warning: payload is object with error information; execution continues
info: payload is object with action info

started: no payload
stopped: no payload

job.reserved: job has been reserved; payload is job id
job.handled: payload is object with job info
job.deleted: payload is jobid
job.buried: payload is jobid
*/

var FiveBeansWorker = function(options)
{
	events.EventEmitter.call(this);

	this.id            = options.id;
	this.host          = options.host;
	this.port          = options.port;
	this.handlers      = options.handlers;
	this.ignoreDefault = options.ignoreDefault;
	this.stopped       = false;

	this.client = null;
};
util.inherits(FiveBeansWorker, events.EventEmitter);

FiveBeansWorker.prototype.start = function(tubes)
{
	var self = this;
	this.stopped = false;

	this.on('next', this.doNext.bind(this));

	function finishedStarting()
	{
		self.emit('started');
		self.emit('next');
	}

	this.client = new beanstalk(this.host, this.port);

	this.client.on('connect', function()
	{
		self.emitInfo('connected to beanstalkd at '+ self.host + ':'+self.port);
		self.watch(tubes, function()
		{
			if (tubes && tubes.length && self.ignoreDefault)
			{
				self.ignore(['default'], function()
				{
					finishedStarting();
				});
			}
			else
			{
				finishedStarting();
			}
		});
	});

	this.client.on('error', function(err)
	{
		self.emitWarning({message: 'beanstalkd connection error', error: err});
		self.emit('error', err);
	});

	this.client.on('close', function()
	{
		self.emitInfo('beanstalkd connection closed');
		self.emit('close');
	});

	this.client.connect();
};

FiveBeansWorker.prototype.watch = function(tubes, callback)
{
	var self = this;
	var tube;
	if (tubes && (tube = tubes[0]))
	{
		self.emitInfo('watching tube ' + tube);
		self.client.watch(tube, function(err)
		{
			if (err) self.emitWarning({ message: 'error watching tube', tube: tube, error: err });
			self.watch(tubes.slice(1), callback);
		});
	}
	else
		callback();
};

FiveBeansWorker.prototype.ignore = function(tubes, callback)
{
	var self = this;
	var tube;
	if (tubes && (tube = tubes[0]))
	{
		self.emitInfo('ignoring tube '+tube);
		self.client.ignore(tube, function(err)
		{
			if (err) self.emitWarning({ message: 'error ignoring tube', tube: tube, error: err });
			self.ignore(tubes.slice(1), callback);
		});
	}
	else
		callback();
};

FiveBeansWorker.prototype.stop = function()
{
	this.emitInfo('stopping...');
	this.stopped = true;
};

FiveBeansWorker.prototype.doNext = function()
{
	var self = this;
	if (self.stopped)
	{
		self.client.end();
		self.emitInfo('stopped');
		self.emit('stopped');
		return;
	}

	self.client.reserve_with_timeout(1, function(err, jobID, payload)
	{
		if (err)
		{
			if ('TIMED_OUT' !== err)
				self.emitWarning({ message: 'error reserving job', error: err });
			self.emit('next');
		}
		else
		{
			self.client.stats_job(jobID, function(err, response)
			{
				if (err)
					self.emitWarning({ message: 'unable to receive stats', error: err });

				self.stats = response;

				var job = null;
				try { job = JSON.parse(payload); }
				catch (e) { self.emitWarning({ message: 'parsing job JSON', id: jobID, error: e }); }

				try { self.emit('job.reserved', jobID); }
				catch (e) { return self.buryAndMoveOn(jobID); }

				if (!job || (typeof job !== 'object'))
					self.buryAndMoveOn(jobID);
				else if (job instanceof Array)
					self.runJob(jobID, job[1]);
				else
					self.runJob(jobID, job);
			});
		}
	});
};

FiveBeansWorker.prototype.runJob = function(jobID, job)
{
	var self = this;
	var handler = this.lookupHandler(job.type);
	if (job.type === undefined)
	{
		self.emitWarning({ message: 'no job type', id: jobID, job: job });
		self.deleteAndMoveOn(jobID);
	}
	else if (!handler)
	{
		self.emitWarning({ message: 'no handler found', id: jobID, type: job.type });
		self.buryAndMoveOn(jobID);
	}
	else
	{
		self.callHandler(handler, jobID, job.payload);
	}
};

FiveBeansWorker.prototype.lookupHandler = function(type)
{
	return this.handlers[type];
};

FiveBeansWorker.prototype.callHandler = function(handler, jobID, jobdata)
{
	var self = this;
	var start = new Date().getTime();
	this.currentJob = jobID;
	this.currentHandler = handler;

	var timeout = setTimeout(function()
	{
		self.emit('job.timedOut', { id: jobID, handler: handler, payload: jobdata, type: handler.type, priority: self.stats.pri });
	}, self.stats ? self.stats.ttr*1000 - 1000 : 60000);

	try
	{
		handler.work(jobdata, function(action, delay)
		{
			clearTimeout(timeout);

			var elapsed = new Date().getTime() - start;
			action = action || 'success';

			self.emit('job.handled', { id: jobID, type: handler.type, elapsed: elapsed, action: action });

			switch (action)
			{
			case 'success':
				self.deleteAndMoveOn(jobID);
				break;

			case 'release':
				self.releaseAndMoveOn(jobID, delay);
				break;

			case 'bury':
				self.buryAndMoveOn(jobID);
				break;

			default:
				self.buryAndMoveOn(jobID);
				break;
			}
		});
	}
	catch (e)
	{
		clearTimeout(timeout);

		self.emitWarning({ message: 'exception in job handler', id: jobID, handler: handler.type, error: e });
		self.buryAndMoveOn(jobID);
	}
};

FiveBeansWorker.prototype.buryAndMoveOn = function(jobID)
{
	var self = this;
	self.client.bury(jobID, beanstalk.LOWEST_PRIORITY, function(err)
	{
		if (err) self.emitWarning({ message: 'error burying', id: jobID, error: err });
		self.emit('job.buried', jobID);
		self.emit('next');
	});
};

FiveBeansWorker.prototype.releaseAndMoveOn = function(jobID, delay)
{
	var self = this;
	if (delay === undefined) delay = 30;

	self.client.release(jobID, beanstalk.LOWEST_PRIORITY, delay, function(err)
	{
		if (err) self.emitWarning({ message: 'error releasing', id: jobID, error: err });
		self.emit('job.released', jobID);
		self.emit('next');
	});
};

FiveBeansWorker.prototype.deleteAndMoveOn = function(jobID)
{
	var self = this;
	self.client.destroy(jobID, function(err)
	{
		if (err) self.emitWarning({ message: 'error deleting', id: jobID, error: err });
		self.emit('job.deleted', jobID);
		self.emit('next');
	});
};

FiveBeansWorker.prototype.emitInfo = function(message)
{
	this.emit('info',
	{
		clientid: this.id,
		message:  message,
	});
	// console.log(message);
};

FiveBeansWorker.prototype.emitWarning = function(data)
{
	data.clientid = this.id;
	this.emit('warning', data);
};

module.exports = FiveBeansWorker;
<commit_msg>detect when a job is rerun after an unexpected failure, and allow handlers to treat them specially<commit_after>var
	beanstalk   = require('./client'),
	events      = require('events'),
	util        = require('util'),
	yaml        = require('js-yaml')
	;

/*
Events emitted:

error: payload is error; execution is halted
close: no payload

warning: payload is object with error information; execution continues
info: payload is object with action info

started: no payload
stopped: no payload

job.reserved: job has been reserved; payload is job id
job.handled: payload is object with job info
job.deleted: payload is jobid
job.buried: payload is jobid
*/

var FiveBeansWorker = function(options)
{
	events.EventEmitter.call(this);

	this.id            = options.id;
	this.host          = options.host;
	this.port          = options.port;
	this.handlers      = options.handlers;
	this.ignoreDefault = options.ignoreDefault;
	this.stopped       = false;

	this.client = null;
};
util.inherits(FiveBeansWorker, events.EventEmitter);

FiveBeansWorker.prototype.start = function(tubes)
{
	var self = this;
	this.stopped = false;

	this.on('next', this.doNext.bind(this));

	function finishedStarting()
	{
		self.emit('started');
		self.emit('next');
	}

	this.client = new beanstalk(this.host, this.port);

	this.client.on('connect', function()
	{
		self.emitInfo('connected to beanstalkd at '+ self.host + ':'+self.port);
		self.watch(tubes, function()
		{
			if (tubes && tubes.length && self.ignoreDefault)
			{
				self.ignore(['default'], function()
				{
					finishedStarting();
				});
			}
			else
			{
				finishedStarting();
			}
		});
	});

	this.client.on('error', function(err)
	{
		self.emitWarning({message: 'beanstalkd connection error', error: err});
		self.emit('error', err);
	});

	this.client.on('close', function()
	{
		self.emitInfo('beanstalkd connection closed');
		self.emit('close');
	});

	this.client.connect();
};

FiveBeansWorker.prototype.watch = function(tubes, callback)
{
	var self = this;
	var tube;
	if (tubes && (tube = tubes[0]))
	{
		self.emitInfo('watching tube ' + tube);
		self.client.watch(tube, function(err)
		{
			if (err) self.emitWarning({ message: 'error watching tube', tube: tube, error: err });
			self.watch(tubes.slice(1), callback);
		});
	}
	else
		callback();
};

FiveBeansWorker.prototype.ignore = function(tubes, callback)
{
	var self = this;
	var tube;
	if (tubes && (tube = tubes[0]))
	{
		self.emitInfo('ignoring tube '+tube);
		self.client.ignore(tube, function(err)
		{
			if (err) self.emitWarning({ message: 'error ignoring tube', tube: tube, error: err });
			self.ignore(tubes.slice(1), callback);
		});
	}
	else
		callback();
};

FiveBeansWorker.prototype.stop = function()
{
	this.emitInfo('stopping...');
	this.stopped = true;
};

FiveBeansWorker.prototype.doNext = function()
{
	var self = this;
	if (self.stopped)
	{
		self.client.end();
		self.emitInfo('stopped');
		self.emit('stopped');
		return;
	}

	self.client.reserve_with_timeout(1, function(err, jobID, payload)
	{
		if (err)
		{
			if ('TIMED_OUT' !== err)
				self.emitWarning({ message: 'error reserving job', error: err });
			self.emit('next');
		}
		else
		{
			self.client.stats_job(jobID, function(err, response)
			{
				if (err)
					self.emitWarning({ message: 'unable to receive stats', error: err });

				self.stats = response;

				var job = null;
				try { job = JSON.parse(payload); }
				catch (e) { self.emitWarning({ message: 'parsing job JSON', id: jobID, error: e }); }

				try { self.emit('job.reserved', jobID); }
				catch (e) { return self.buryAndMoveOn(jobID); }

				if (!job || (typeof job !== 'object'))
					self.buryAndMoveOn(jobID);
				else if (job instanceof Array)
					self.runJob(jobID, job[1]);
				else
					self.runJob(jobID, job);
			});
		}
	});
};

FiveBeansWorker.prototype.runJob = function(jobID, job)
{
	var self = this;
	var handler = this.lookupHandler(job.type);
	if (job.type === undefined)
	{
		self.emitWarning({ message: 'no job type', id: jobID, job: job });
		self.deleteAndMoveOn(jobID);
	}
	else if (!handler)
	{
		self.emitWarning({ message: 'no handler found', id: jobID, type: job.type });
		self.buryAndMoveOn(jobID);
	}
	else
	{
		if (self.stats.reserves - self.stats.releases >= 2 && self.stats.kicks === 0)
		{
			if (handler.retryFailure) self.callHandler(handler, jobID, job.payload, 'retryFailure');
			else self.buryAndMoveOn(jobID);
		}
		else
		{
			self.callHandler(handler, jobID, job.payload, 'work');
		}
	}
};

FiveBeansWorker.prototype.lookupHandler = function(type)
{
	return this.handlers[type];
};

FiveBeansWorker.prototype.callHandler = function(handler, jobID, jobdata, handlerFunction)
{
	var self = this;
	var start = new Date().getTime();
	this.currentJob = jobID;
	this.currentHandler = handler;

	var timeout = setTimeout(function()
	{
		self.emit('job.timedOut', { id: jobID, handler: handler, payload: jobdata, type: handler.type, priority: self.stats.pri });
	}, self.stats ? self.stats.ttr*1000 - 1000 : 60000);

	try
	{
		handler[handlerFunction](jobdata, function(action, delay)
		{
			clearTimeout(timeout);

			var elapsed = new Date().getTime() - start;
			action = action || 'success';

			self.emit('job.handled', { id: jobID, type: handler.type, elapsed: elapsed, action: action });

			switch (action)
			{
			case 'success':
				self.deleteAndMoveOn(jobID);
				break;

			case 'release':
				self.releaseAndMoveOn(jobID, delay);
				break;

			case 'bury':
				self.buryAndMoveOn(jobID);
				break;

			default:
				self.buryAndMoveOn(jobID);
				break;
			}
		});
	}
	catch (e)
	{
		clearTimeout(timeout);

		self.emitWarning({ message: 'exception in job handler', id: jobID, handler: handler.type, error: e });
		self.buryAndMoveOn(jobID);
	}
};

FiveBeansWorker.prototype.buryAndMoveOn = function(jobID)
{
	var self = this;
	self.client.bury(jobID, beanstalk.LOWEST_PRIORITY, function(err)
	{
		if (err) self.emitWarning({ message: 'error burying', id: jobID, error: err });
		self.emit('job.buried', jobID);
		self.emit('next');
	});
};

FiveBeansWorker.prototype.releaseAndMoveOn = function(jobID, delay)
{
	var self = this;
	if (delay === undefined) delay = 30;

	self.client.release(jobID, beanstalk.LOWEST_PRIORITY, delay, function(err)
	{
		if (err) self.emitWarning({ message: 'error releasing', id: jobID, error: err });
		self.emit('job.released', jobID);
		self.emit('next');
	});
};

FiveBeansWorker.prototype.deleteAndMoveOn = function(jobID)
{
	var self = this;
	self.client.destroy(jobID, function(err)
	{
		if (err) self.emitWarning({ message: 'error deleting', id: jobID, error: err });
		self.emit('job.deleted', jobID);
		self.emit('next');
	});
};

FiveBeansWorker.prototype.emitInfo = function(message)
{
	this.emit('info',
	{
		clientid: this.id,
		message:  message,
	});
	// console.log(message);
};

FiveBeansWorker.prototype.emitWarning = function(data)
{
	data.clientid = this.id;
	this.emit('warning', data);
};

module.exports = FiveBeansWorker;
<|endoftext|>


<commit_before>const qs = require('querystring');

const {remote, ipcRenderer: ipc} = require('electron');
const {GitProcess} = require('dugite');

class AverageTracker {
  constructor({limit} = {limit: 10}) {
    // for now this serves a dual purpose - # of values tracked AND # discarded prior to starting avg calculation
    this.limit = limit;
    this.sum = 0;
    this.values = [];
  }

  addValue(value) {
    if (this.values.length >= this.limit) {
      const discardedValue = this.values.shift();
      this.sum -= discardedValue;
    }
    this.values.push(value);
    this.sum += value;
  }

  getAverage() {
    if (this.enoughData()) {
      return this.sum / this.limit;
    } else {
      return null;
    }
  }

  getLimit() {
    return this.limit;
  }

  enoughData() {
    return this.values.length === this.limit;
  }
}

const query = qs.parse(window.location.search.substr(1));
const sourceWebContentsId = remote.getCurrentWindow().webContents.id;
const operationCountLimit = parseInt(query.operationCountLimit, 10);
const averageTracker = new AverageTracker({limit: operationCountLimit});
const childPidsById = new Map();

const destroyRenderer = () => {
  if (!managerWebContents.isDestroyed()) {
    managerWebContents.removeListener('crashed', destroyRenderer);
    managerWebContents.removeListener('destroyed', destroyRenderer);
  }
  const win = remote.BrowserWindow.fromWebContents(remote.getCurrentWebContents());
  if (win && !win.isDestroyed()) {
    win.destroy();
  }
};
const managerWebContentsId = parseInt(query.managerWebContentsId, 10);
const managerWebContents = remote.webContents.fromId(managerWebContentsId);
if (managerWebContents && !managerWebContents.isDestroyed()) {
  managerWebContents.on('crashed', destroyRenderer);
  managerWebContents.on('destroyed', destroyRenderer);
  window.onbeforeunload = () => {
    managerWebContents.removeListener('crashed', destroyRenderer);
    managerWebContents.removeListener('destroyed', destroyRenderer);
  };
}

const channelName = query.channelName;
ipc.on(channelName, (event, {type, data}) => {
  if (type === 'git-exec') {
    const {args, workingDir, options, id} = data;
    if (args) {
      document.getElementById('command').textContent = `git ${args.join(' ')}`;
    }

    options.processCallback = child => {
      childPidsById.set(id, child.pid);

      child.on('error', err => {
        event.sender.sendTo(managerWebContentsId, channelName, {
          sourceWebContentsId,
          type: 'git-spawn-error',
          data: {id, err},
        });
      });

      child.stdin.on('error', err => {
        event.sender.sendTo(managerWebContentsId, channelName, {
          sourceWebContentsId,
          type: 'git-stdin-error',
          data: {id, stdin: options.stdin, err},
        });
      });
    };

    const spawnStart = performance.now();
    GitProcess.exec(args, workingDir, options)
      .then(({stdout, stderr, exitCode}) => {
        const timing = {
          spawnTime: spawnEnd - spawnStart,
          execTime: performance.now() - spawnEnd,
        };
        childPidsById.delete(id);
        event.sender.sendTo(managerWebContentsId, channelName, {
          sourceWebContentsId,
          type: 'git-data',
          data: {
            id,
            average: averageTracker.getAverage(),
            results: {stdout, stderr, exitCode, timing},
          },
        });
      });
    const spawnEnd = performance.now();
    averageTracker.addValue(spawnEnd - spawnStart);

    // TODO: consider using this to avoid duplicate write operations upon crashing.
    // For now we won't do this to avoid clogging up ipc channel
    // event.sender.sendTo(managerWebContentsId, channelName, {sourceWebContentsId, type: 'exec-started', data: {id}});

    if (averageTracker.enoughData() && averageTracker.getAverage() > 20) {
      event.sender.sendTo(managerWebContentsId, channelName, {type: 'slow-spawns'});
    }
  } else if (type === 'git-cancel') {
    const {id} = data;
    const childPid = childPidsById.get(id);
    if (childPid !== undefined) {
      require('tree-kill')(childPid, 'SIGINT', () => {
        event.sender.sendTo(managerWebContentsId, channelName, {
          sourceWebContentsId,
          type: 'git-cancelled',
          data: {id, childPid},
        });
      });
      childPidsById.delete(id);
    }
  } else {
    throw new Error(`Could not identify type ${type}`);
  }
});

ipc.sendTo(managerWebContentsId, channelName, {sourceWebContentsId, type: 'renderer-ready', data: {pid: process.pid}});
<commit_msg>exec() now rejects its promise when the subprocess is killed by a signal<commit_after>const qs = require('querystring');

const {remote, ipcRenderer: ipc} = require('electron');
const {GitProcess} = require('dugite');

class AverageTracker {
  constructor({limit} = {limit: 10}) {
    // for now this serves a dual purpose - # of values tracked AND # discarded prior to starting avg calculation
    this.limit = limit;
    this.sum = 0;
    this.values = [];
  }

  addValue(value) {
    if (this.values.length >= this.limit) {
      const discardedValue = this.values.shift();
      this.sum -= discardedValue;
    }
    this.values.push(value);
    this.sum += value;
  }

  getAverage() {
    if (this.enoughData()) {
      return this.sum / this.limit;
    } else {
      return null;
    }
  }

  getLimit() {
    return this.limit;
  }

  enoughData() {
    return this.values.length === this.limit;
  }
}

const query = qs.parse(window.location.search.substr(1));
const sourceWebContentsId = remote.getCurrentWindow().webContents.id;
const operationCountLimit = parseInt(query.operationCountLimit, 10);
const averageTracker = new AverageTracker({limit: operationCountLimit});
const childPidsById = new Map();

const destroyRenderer = () => {
  if (!managerWebContents.isDestroyed()) {
    managerWebContents.removeListener('crashed', destroyRenderer);
    managerWebContents.removeListener('destroyed', destroyRenderer);
  }
  const win = remote.BrowserWindow.fromWebContents(remote.getCurrentWebContents());
  if (win && !win.isDestroyed()) {
    win.destroy();
  }
};
const managerWebContentsId = parseInt(query.managerWebContentsId, 10);
const managerWebContents = remote.webContents.fromId(managerWebContentsId);
if (managerWebContents && !managerWebContents.isDestroyed()) {
  managerWebContents.on('crashed', destroyRenderer);
  managerWebContents.on('destroyed', destroyRenderer);
  window.onbeforeunload = () => {
    managerWebContents.removeListener('crashed', destroyRenderer);
    managerWebContents.removeListener('destroyed', destroyRenderer);
  };
}

const channelName = query.channelName;
ipc.on(channelName, (event, {type, data}) => {
  if (type === 'git-exec') {
    const {args, workingDir, options, id} = data;
    if (args) {
      document.getElementById('command').textContent = `git ${args.join(' ')}`;
    }

    options.processCallback = child => {
      childPidsById.set(id, child.pid);

      child.on('error', err => {
        event.sender.sendTo(managerWebContentsId, channelName, {
          sourceWebContentsId,
          type: 'git-spawn-error',
          data: {id, err},
        });
      });

      child.stdin.on('error', err => {
        event.sender.sendTo(managerWebContentsId, channelName, {
          sourceWebContentsId,
          type: 'git-stdin-error',
          data: {id, stdin: options.stdin, err},
        });
      });
    };

    const spawnStart = performance.now();
    GitProcess.exec(args, workingDir, options)
      .then(({stdout, stderr, exitCode}) => {
        const timing = {
          spawnTime: spawnEnd - spawnStart,
          execTime: performance.now() - spawnEnd,
        };
        childPidsById.delete(id);
        event.sender.sendTo(managerWebContentsId, channelName, {
          sourceWebContentsId,
          type: 'git-data',
          data: {
            id,
            average: averageTracker.getAverage(),
            results: {stdout, stderr, exitCode, timing},
          },
        });
      }, err => {
        const timing = {
          spawnTime: spawnEnd - spawnStart,
          execTime: performance.now() - spawnEnd,
        };
        childPidsById.delete(id);
        event.sender.sendTo(managerWebContentsId, channelName, {
          sourceWebContentsId,
          type: 'git-data',
          data: {
            id,
            average: averageTracker.getAverage(),
            results: {
              stdout: err.stdout,
              stderr: err.stderr,
              exitCode: err.code,
              signal: err.signal,
              timing,
            },
          },
        });
      });
    const spawnEnd = performance.now();
    averageTracker.addValue(spawnEnd - spawnStart);

    // TODO: consider using this to avoid duplicate write operations upon crashing.
    // For now we won't do this to avoid clogging up ipc channel
    // event.sender.sendTo(managerWebContentsId, channelName, {sourceWebContentsId, type: 'exec-started', data: {id}});

    if (averageTracker.enoughData() && averageTracker.getAverage() > 20) {
      event.sender.sendTo(managerWebContentsId, channelName, {type: 'slow-spawns'});
    }
  } else if (type === 'git-cancel') {
    const {id} = data;
    const childPid = childPidsById.get(id);
    if (childPid !== undefined) {
      require('tree-kill')(childPid, 'SIGINT', () => {
        event.sender.sendTo(managerWebContentsId, channelName, {
          sourceWebContentsId,
          type: 'git-cancelled',
          data: {id, childPid},
        });
      });
      childPidsById.delete(id);
    }
  } else {
    throw new Error(`Could not identify type ${type}`);
  }
});

ipc.sendTo(managerWebContentsId, channelName, {sourceWebContentsId, type: 'renderer-ready', data: {pid: process.pid}});
<|endoftext|>


<commit_before>// WrapUp base class

// core requires

var readFile = require("fs").readFileSync,
	path = require("path")

//@* i love sebmarkbage
var Module = require("module")

// dependencies

var ujs = require("uglify-js"),
	parser = ujs.parser,
	uglify = ujs.uglify,
	emitter = require("emi")

// util

var replaces = function(str, obj){
	for (var key in obj) str = str.replace(new RegExp("@" + key, "g"), obj[key])
	return str
}, camelize = function(self){
	return (self + "").replace(/-\D/g, function(match){
		return match.charAt(1).toUpperCase()
	})
}

// templates

var ipath = path.join(__dirname, "../includes/"),
	boil = "(function(modules){\n"+ readFile(ipath + "require.js", "utf-8") + "\n@wrup\n@globals\n})({@modules})",
	wrup = "window.wrup = function wrup(id){return (id = table[id]) ? require(id) : null}",
	glob = "window.@name = require('@id')",
	req = "require('@id')",
	mods = "'@id': function(require, module, exports, global){@src}"

// WrapUp

var WrapUp = function(){
	this.packages = {}
	this.modules = {}
	this.required = {}
	this.mains = {}

	this.index = 0
}

WrapUp.prototype = new emitter

WrapUp.prototype.scan = function(what, from){

	var modulefull = this.resolve(what, from)
	if (!modulefull){
		this.emit("error:resolve", what, from)
		return null
	}

	if (modulefull.indexOf("/") !== 0){
		this.emit("error:native", modulefull, from)
		return null
	}

	var module = this.modules[modulefull]

	if (module) return module.uid

	var src = readFile(modulefull, "utf-8").toString()

	// same version check for poor people
	// could break complex requires
	// for (var p in this.modules){
	// 	var d = this.modules[p]
	// 	if (d.src === src) return d.uid
	// }

	module = this.modules[modulefull] = {
		uid: (this.index++).toString(36)
	}

	var modulepath = path.dirname(modulefull),
		modulename = path.basename(modulefull, ".js")

	// performing dead-code removal before finding require calls to avoid catching useless requires
	// as well as nuking comments to avoid catching requires within comments.
	try {
		var ast = parser.parse(src)
		ast = uglify.ast_squeeze(ast, {make_seqs: false, dead_code: true})
		src = uglify.gen_code(ast)
	} catch (err){
		this.emit("error:js", modulefull, from, err)
		return null
	}

	var self = this

	module.src = src.replace(/require\(\s*["']([^'"\s]+)["']\s*\)/g, function(match, dependency){
		var k = self.scan(dependency, modulefull)
		return k ? "require('" + k + "')" : "null"
	})

	return module.uid
}

WrapUp.prototype.package = function(pkg){
	var name, main, json, pkgpath
		resolved = path.resolve(pkg),
		basename = path.basename(resolved, ".json")

	// if it is a package.json file, try to find it
	// else append package.json and prey
	var jsonpath = (basename === "package") ? this.resolve(resolved) : this.resolve(path.join(resolved, "package.json"))

	if (jsonpath){
		pkgpath = path.dirname(jsonpath)
		json = require(jsonpath)// just require the json
		name = json.name
		main = this.resolve(pkgpath)
		return this.module(main, name)
	}

	this.emit("error:package", pkg)

	return this
}

WrapUp.prototype.module = function(module, namespace){
	if (namespace == null){ // automatic namespace, uses filename
		var resolved = this.resolve(module)
		if (!resolved) this.emit("error:resolve", module, "you")
		namespace = path.basename(resolved, '.js')
	}
	var id = null
	if (!this.required[namespace]){
		id = this.scan(module)
		if (id) this.required[namespace] = id
	} else {
		this.emit("error:namespace", namespace, module)
	}
	return this
}

WrapUp.prototype.resolve = function(module, from){ //resolve module from cwd or relative to another module.
	from = (from == null) ? path.join(process.cwd(), "wrup") : path.resolve(from)
	var m = new Module(from)
	m.filename = from
	m.paths = Module._nodeModulePaths(path.dirname(from))
	try {
		return Module._resolveFilename(module, m)
	} catch (err){}
	return null
}

WrapUp.prototype.up = WrapUp.prototype.toString = function(options){
	if (options == null) options = {}

	if (options.wrup == null) options.wrup = false
	if (options.globalize == null) options.globalize = true
	if (options.compress == null) options.compress = false

	var flat = [],
		globals = [],
		table = {},
		requires = []

	for (var fullpath in this.modules){
		var mod = this.modules[fullpath]
		flat.push(replaces(mods, {id: mod.uid, src: mod.src}))
	}

	if (flat.length === 0){
		this.emit("error:empty")
		return ""
	}

	for (var ns in this.required){
		globals.push(replaces(glob, {id: this.required[ns], name: camelize(ns)}))
		requires.push(replaces(req, {id: this.required[ns]}))
		table[ns] = this.required[ns]
	}

	if (!options.wrup && !options.globalize)
		this.emit("warning:access")

	var js = replaces(boil, {
			modules: flat.join(","),
			globals: options.globalize ? globals.join("\n") : (options.wrup) ? "" : requires.join("\n"),
			wrup: (options.wrup) ? ", table = " + JSON.stringify(table) + "\n" + wrup : ""
		});

	try {
		var ast = parser.parse(js)
		if (options.compress){
			ast = uglify.ast_mangle(ast)
			ast = uglify.ast_squeeze(ast)
			ast = uglify.ast_lift_variables(ast)
		}
		return uglify.gen_code(ast, {beautify: !options.compress})
	} catch(err){
		this.emit("error:internal", err)
	}

	return ""

}

module.exports = WrapUp
<commit_msg> - better json finding method, if a bit redundant.<commit_after>// WrapUp base class

// core requires

var readFile = require("fs").readFileSync,
	path = require("path")

//@* i love sebmarkbage
var Module = require("module")

// dependencies

var ujs = require("uglify-js"),
	parser = ujs.parser,
	uglify = ujs.uglify,
	emitter = require("emi")

// util

var replaces = function(str, obj){
	for (var key in obj) str = str.replace(new RegExp("@" + key, "g"), obj[key])
	return str
}, camelize = function(self){
	return (self + "").replace(/-\D/g, function(match){
		return match.charAt(1).toUpperCase()
	})
}

// templates

var ipath = path.join(__dirname, "../includes/"),
	boil = "(function(modules){\n"+ readFile(ipath + "require.js", "utf-8") + "\n@wrup\n@globals\n})({@modules})",
	wrup = "window.wrup = function wrup(id){return (id = table[id]) ? require(id) : null}",
	glob = "window.@name = require('@id')",
	req = "require('@id')",
	mods = "'@id': function(require, module, exports, global){@src}"

// WrapUp

var WrapUp = function(){
	this.packages = {}
	this.modules = {}
	this.required = {}
	this.mains = {}

	this.index = 0
}

WrapUp.prototype = new emitter

WrapUp.prototype.scan = function(what, from){

	var modulefull = this.resolve(what, from)
	if (!modulefull){
		this.emit("error:resolve", what, from)
		return null
	}

	if (modulefull.indexOf("/") !== 0){
		this.emit("error:native", modulefull, from)
		return null
	}

	var module = this.modules[modulefull]

	if (module) return module.uid

	var src = readFile(modulefull, "utf-8").toString()

	// same version check for poor people
	// could break complex requires
	// for (var p in this.modules){
	// 	var d = this.modules[p]
	// 	if (d.src === src) return d.uid
	// }

	module = this.modules[modulefull] = {
		uid: (this.index++).toString(36)
	}

	var modulepath = path.dirname(modulefull),
		modulename = path.basename(modulefull, ".js")

	// performing dead-code removal before finding require calls to avoid catching useless requires
	// as well as nuking comments to avoid catching requires within comments.
	try {
		var ast = parser.parse(src)
		ast = uglify.ast_squeeze(ast, {make_seqs: false, dead_code: true})
		src = uglify.gen_code(ast)
	} catch (err){
		this.emit("error:js", modulefull, from, err)
		return null
	}

	var self = this

	module.src = src.replace(/require\(\s*["']([^'"\s]+)["']\s*\)/g, function(match, dependency){
		var k = self.scan(dependency, modulefull)
		return k ? "require('" + k + "')" : "null"
	})

	return module.uid
}

WrapUp.prototype.package = function(pkg){
	var self = this, jsonpath

	var gjs = function(p){
		var jp = (path.basename(p, ".json") === "package") ? self.resolve(p) : self.resolve(path.join(p, "package.json"))
		return (jp && path.basename(jp, ".json") === "package") ? jp : null
	}

	jsonpath = gjs(pkg) || gjs(path.resolve(pkg)) // find json

	if (jsonpath){
		var pkgpath = path.dirname(jsonpath),
			json = require(jsonpath),// just require the json
			name = json.name,
			main = this.resolve(pkgpath)
		return this.module(main, name)
	}

	this.emit("error:package", pkg)

	return this
}

WrapUp.prototype.module = function(module, namespace){
	if (namespace == null){ // automatic namespace, uses filename
		var resolved = this.resolve(module)
		if (!resolved) this.emit("error:resolve", module, "you")
		namespace = path.basename(resolved, '.js')
	}
	var id = null
	if (!this.required[namespace]){
		id = this.scan(module)
		if (id) this.required[namespace] = id
	} else {
		this.emit("error:namespace", namespace, module)
	}
	return this
}

WrapUp.prototype.resolve = function(module, from){ //resolve module from cwd or relative to another module.
	from = (from == null) ? path.join(process.cwd(), "wrup") : path.resolve(from)
	var m = new Module(from)
	m.filename = from
	m.paths = Module._nodeModulePaths(path.dirname(from))
	try {
		return Module._resolveFilename(module, m)
	} catch (err){}
	return null
}

WrapUp.prototype.up = WrapUp.prototype.toString = function(options){
	if (options == null) options = {}

	if (options.wrup == null) options.wrup = false
	if (options.globalize == null) options.globalize = true
	if (options.compress == null) options.compress = false

	var flat = [],
		globals = [],
		table = {},
		requires = []

	for (var fullpath in this.modules){
		var mod = this.modules[fullpath]
		flat.push(replaces(mods, {id: mod.uid, src: mod.src}))
	}

	if (flat.length === 0){
		this.emit("error:empty")
		return ""
	}

	for (var ns in this.required){
		globals.push(replaces(glob, {id: this.required[ns], name: camelize(ns)}))
		requires.push(replaces(req, {id: this.required[ns]}))
		table[ns] = this.required[ns]
	}

	if (!options.wrup && !options.globalize)
		this.emit("warning:access")

	var js = replaces(boil, {
			modules: flat.join(","),
			globals: options.globalize ? globals.join("\n") : (options.wrup) ? "" : requires.join("\n"),
			wrup: (options.wrup) ? ", table = " + JSON.stringify(table) + "\n" + wrup : ""
		});

	try {
		var ast = parser.parse(js)
		if (options.compress){
			ast = uglify.ast_mangle(ast)
			ast = uglify.ast_squeeze(ast)
			ast = uglify.ast_lift_variables(ast)
		}
		return uglify.gen_code(ast, {beautify: !options.compress})
	} catch(err){
		this.emit("error:internal", err)
	}

	return ""

}

module.exports = WrapUp
<|endoftext|>


<commit_before>
/**
 * Module dependencies.
 */

var assert = require('assert');
var inherits = require('util').inherits;
var debug = require('debug')('wave:writer');
var Transform = require('stream').Transform;
var Parser = require('stream-parser');

// for node v0.8.x support, remove after v0.12.x
if (!Transform) Transform = require('readable-stream/transform');

/**
 * Module exports.
 */

module.exports = Writer;

/**
 * RIFF Chunk IDs in Buffers.
 *
 * @api private
 */

var RIFF = new Buffer('RIFF');
var WAVE = new Buffer('WAVE');
var fmt  = new Buffer('fmt ');
var data = new Buffer('data');

/**
 * The max size of the "data" chunk of a WAVE file. This is the max unsigned
 * 32-bit int value, minus 100 bytes (overkill, 44 would be safe) for the header.
 *
 * @api private
 */

var MAX_WAV = 4294967295 - 100;

/**
 * The `Writer` class accepts raw audio data written to it (only PCM audio data is
 * currently supported), and outputs a WAV file with a valid WAVE header at the
 * beginning specifying the formatting information of the audio stream.
 *
 * Note that there's an interesting problem, because the WAVE header also
 * specifies the total byte length of the audio data in the file, and there's no
 * way that we can know this ahead of time. Therefore the WAVE header will contain
 * a byte-length if `0` initially, which most WAVE decoders will know means to
 * just read until `EOF`.
 *
 * Optionally, if you are in a situation where you can seek back to the beginning
 * of the destination of the WAVE file (like writing to a regular file, for
 * example), then you may listen for the `"header"` event which will be emitted
 * _after_ all the data has been written, and you can go back and rewrite the new
 * header with proper audio byte length into the beginning of the destination
 * (though if your destination _is_ a regular file, you should use the the
 * `FileWriter` class instead).
 *
 * @param {Object} opts optional options object
 * @api public
 */

function Writer (opts) {
  if (!(this instanceof Writer)) {
    return new Writer(opts);
  }
  Parser.call(this, opts);

  // TODO: allow/properly handle other WAVE audio formats
  this.endianness = 'LE';
  this.format = 1; // raw PCM
  this.channels = 2;
  this.sampleRate = 44100;
  this.bitDepth = 16;

  if (opts) {
    if (null != opts.format) this.format = opts.format;
    if (null != opts.channels) this.channels = opts.channels;
    if (null != opts.sampleRate) this.sampleRate = opts.sampleRate;
    if (null != opts.bitDepth) this.bitDepth = opts.bitDepth;
  }

  // the first time _transform() is called, _writeHeader() will be run
  //this.immediate(this._writeHeader);
  this._bytes(0, this._writeHeader);
}
inherits(Writer, Transform);

/**
 * Mixin `Parser`.
 */

Parser(Writer.prototype);

/**
 * Writes the WAVE header.
 *
 * @api private
 */

Writer.prototype._writeHeader = function (chunk, write) {
  debug('_writeHeader()');
  var headerLength = 44; // TODO: 44 is only for format 1 (PCM), any other
                         // format will have a variable size...
  var dataLength = this.dataLength;
  if (null == dataLength) dataLength = MAX_WAV;
  var fileSize = dataLength + headerLength;
  var header = new Buffer(headerLength);
  var offset = 0;

  // write the "RIFF" identifier
  RIFF.copy(header, offset);
  offset += RIFF.length;

  // write the file size minus the identifier and this 32-bit int
  header['writeUInt32' + this.endianness](fileSize - 8, offset);
  offset += 4;

  // write the "WAVE" identifier
  WAVE.copy(header, offset);
  offset += WAVE.length;

  // write the "fmt " sub-chunk identifier
  fmt.copy(header, offset);
  offset += fmt.length;

  // write the size of the "fmt " chunk
  // XXX: value of 16 is hard-coded for raw PCM format. other formats have
  // different size.
  header['writeUInt32' + this.endianness](16, offset);
  offset += 4;

  // write the audio format code
  header['writeUInt16' + this.endianness](this.format, offset);
  offset += 2;

  // write the number of channels
  header['writeUInt16' + this.endianness](this.channels, offset);
  offset += 2;

  // write the sample rate
  header['writeUInt32' + this.endianness](this.sampleRate, offset);
  offset += 4;

  // write the byte rate
  var byteRate = this.byteRate;
  if (null == byteRate) {
    byteRate = this.sampleRate * this.channels * this.bitDepth / 8;
  }
  header['writeUInt32' + this.endianness](byteRate, offset);
  offset += 4;

  // write the block align
  var blockAlign = this.blockAlign;
  if (null == blockAlign) {
    blockAlign = this.channels * this.bitDepth / 8;
  }
  header['writeUInt16' + this.endianness](blockAlign, offset);
  offset += 2;

  // write the bits per sample
  header['writeUInt16' + this.endianness](this.bitDepth, offset);
  offset += 2;

  // write the "data" sub-chunk ID
  data.copy(header, offset);
  offset += data.length;

  // write the remaining length of the rest of the data
  header['writeUInt32' + this.endianness](dataLength, offset);
  offset += 4;

  // save the "header" Buffer for the end, we emit the "header" event at the end
  // with the "size" values properly filled out. if this stream is being piped to
  // a file (or anything else seekable), then this correct header should be placed
  // at the very beginning of the file.
  this._header = header;
  this.headerLength = headerLength;

  // flush the header and after that pass-through "dataLength" bytes
  write(header);
  this._passthrough(dataLength, this._onEnd);
};

/**
 * Called for the "end" event of this Writer instance.
 *
 * @api private
 */

Writer.prototype._onEnd = function (write) {
  debug('_onEnd()');
};

/**
 * Emits a "header" event after the readable side of the stream has finished.
 *
 * @api private
 */

Writer.prototype._flush = function (write, done) {
  debug('_flush()');
  done();
  this.dataLength = this.bytesProcessed;
  process.nextTick(this._emitHeader.bind(this));
};

/**
 * Emits the "header" event. This can safely be ignored, or if you are writing
 * this WAVE file to somewhere that is seekable (i.e. the filesystem), then you
 * should write this "header" buffer at the beginning of the file to get the
 * correct file size values in the file. This isn't too important since most audio
 * players look at the file size rather than those byte values in the header, but
 * it's good to when when possible.
 *
 * @api private
 */

Writer.prototype._emitHeader = function () {
  debug('_emitHeader()');
  var dataLength = this.dataLength;
  var headerLength = this.headerLength;
  var header = this._header;

  // write the file length at the beginning of the header
  header['writeUInt32' + this.endianness](dataLength + headerLength - 8, 4);

  // write the data length at the end of the header
  header['writeUInt32' + this.endianness](dataLength, headerLength - 4);

  this.emit('header', header);
};
<commit_msg>writer: fix leftover typo from the 'stream-parser' switch<commit_after>
/**
 * Module dependencies.
 */

var assert = require('assert');
var inherits = require('util').inherits;
var debug = require('debug')('wave:writer');
var Transform = require('stream').Transform;
var Parser = require('stream-parser');

// for node v0.8.x support, remove after v0.12.x
if (!Transform) Transform = require('readable-stream/transform');

/**
 * Module exports.
 */

module.exports = Writer;

/**
 * RIFF Chunk IDs in Buffers.
 *
 * @api private
 */

var RIFF = new Buffer('RIFF');
var WAVE = new Buffer('WAVE');
var fmt  = new Buffer('fmt ');
var data = new Buffer('data');

/**
 * The max size of the "data" chunk of a WAVE file. This is the max unsigned
 * 32-bit int value, minus 100 bytes (overkill, 44 would be safe) for the header.
 *
 * @api private
 */

var MAX_WAV = 4294967295 - 100;

/**
 * The `Writer` class accepts raw audio data written to it (only PCM audio data is
 * currently supported), and outputs a WAV file with a valid WAVE header at the
 * beginning specifying the formatting information of the audio stream.
 *
 * Note that there's an interesting problem, because the WAVE header also
 * specifies the total byte length of the audio data in the file, and there's no
 * way that we can know this ahead of time. Therefore the WAVE header will contain
 * a byte-length if `0` initially, which most WAVE decoders will know means to
 * just read until `EOF`.
 *
 * Optionally, if you are in a situation where you can seek back to the beginning
 * of the destination of the WAVE file (like writing to a regular file, for
 * example), then you may listen for the `"header"` event which will be emitted
 * _after_ all the data has been written, and you can go back and rewrite the new
 * header with proper audio byte length into the beginning of the destination
 * (though if your destination _is_ a regular file, you should use the the
 * `FileWriter` class instead).
 *
 * @param {Object} opts optional options object
 * @api public
 */

function Writer (opts) {
  if (!(this instanceof Writer)) {
    return new Writer(opts);
  }
  Transform.call(this, opts);

  // TODO: allow/properly handle other WAVE audio formats
  this.endianness = 'LE';
  this.format = 1; // raw PCM
  this.channels = 2;
  this.sampleRate = 44100;
  this.bitDepth = 16;

  if (opts) {
    if (null != opts.format) this.format = opts.format;
    if (null != opts.channels) this.channels = opts.channels;
    if (null != opts.sampleRate) this.sampleRate = opts.sampleRate;
    if (null != opts.bitDepth) this.bitDepth = opts.bitDepth;
  }

  // the first time _transform() is called, _writeHeader() will be run
  //this.immediate(this._writeHeader);
  this._bytes(0, this._writeHeader);
}
inherits(Writer, Transform);

/**
 * Mixin `Parser`.
 */

Parser(Writer.prototype);

/**
 * Writes the WAVE header.
 *
 * @api private
 */

Writer.prototype._writeHeader = function (chunk, write) {
  debug('_writeHeader()');
  var headerLength = 44; // TODO: 44 is only for format 1 (PCM), any other
                         // format will have a variable size...
  var dataLength = this.dataLength;
  if (null == dataLength) dataLength = MAX_WAV;
  var fileSize = dataLength + headerLength;
  var header = new Buffer(headerLength);
  var offset = 0;

  // write the "RIFF" identifier
  RIFF.copy(header, offset);
  offset += RIFF.length;

  // write the file size minus the identifier and this 32-bit int
  header['writeUInt32' + this.endianness](fileSize - 8, offset);
  offset += 4;

  // write the "WAVE" identifier
  WAVE.copy(header, offset);
  offset += WAVE.length;

  // write the "fmt " sub-chunk identifier
  fmt.copy(header, offset);
  offset += fmt.length;

  // write the size of the "fmt " chunk
  // XXX: value of 16 is hard-coded for raw PCM format. other formats have
  // different size.
  header['writeUInt32' + this.endianness](16, offset);
  offset += 4;

  // write the audio format code
  header['writeUInt16' + this.endianness](this.format, offset);
  offset += 2;

  // write the number of channels
  header['writeUInt16' + this.endianness](this.channels, offset);
  offset += 2;

  // write the sample rate
  header['writeUInt32' + this.endianness](this.sampleRate, offset);
  offset += 4;

  // write the byte rate
  var byteRate = this.byteRate;
  if (null == byteRate) {
    byteRate = this.sampleRate * this.channels * this.bitDepth / 8;
  }
  header['writeUInt32' + this.endianness](byteRate, offset);
  offset += 4;

  // write the block align
  var blockAlign = this.blockAlign;
  if (null == blockAlign) {
    blockAlign = this.channels * this.bitDepth / 8;
  }
  header['writeUInt16' + this.endianness](blockAlign, offset);
  offset += 2;

  // write the bits per sample
  header['writeUInt16' + this.endianness](this.bitDepth, offset);
  offset += 2;

  // write the "data" sub-chunk ID
  data.copy(header, offset);
  offset += data.length;

  // write the remaining length of the rest of the data
  header['writeUInt32' + this.endianness](dataLength, offset);
  offset += 4;

  // save the "header" Buffer for the end, we emit the "header" event at the end
  // with the "size" values properly filled out. if this stream is being piped to
  // a file (or anything else seekable), then this correct header should be placed
  // at the very beginning of the file.
  this._header = header;
  this.headerLength = headerLength;

  // flush the header and after that pass-through "dataLength" bytes
  write(header);
  this._passthrough(dataLength, this._onEnd);
};

/**
 * Called for the "end" event of this Writer instance.
 *
 * @api private
 */

Writer.prototype._onEnd = function (write) {
  debug('_onEnd()');
};

/**
 * Emits a "header" event after the readable side of the stream has finished.
 *
 * @api private
 */

Writer.prototype._flush = function (write, done) {
  debug('_flush()');
  done();
  this.dataLength = this.bytesProcessed;
  process.nextTick(this._emitHeader.bind(this));
};

/**
 * Emits the "header" event. This can safely be ignored, or if you are writing
 * this WAVE file to somewhere that is seekable (i.e. the filesystem), then you
 * should write this "header" buffer at the beginning of the file to get the
 * correct file size values in the file. This isn't too important since most audio
 * players look at the file size rather than those byte values in the header, but
 * it's good to when when possible.
 *
 * @api private
 */

Writer.prototype._emitHeader = function () {
  debug('_emitHeader()');
  var dataLength = this.dataLength;
  var headerLength = this.headerLength;
  var header = this._header;

  // write the file length at the beginning of the header
  header['writeUInt32' + this.endianness](dataLength + headerLength - 8, 4);

  // write the data length at the end of the header
  header['writeUInt32' + this.endianness](dataLength, headerLength - 4);

  this.emit('header', header);
};
<|endoftext|>


<commit_before>(function () {

var sax;

if (typeof module !== 'undefined' && module.exports) {
  // We're being used in a Node-like environment
  sax = require('sax');
}
else {
  // assume it's attached to the Window object in a browser
  sax = this.sax;

  if (!sax) // no sax for you!
    throw new Error("Expected sax to be defined. Make sure you're including sax.js before this file.");
}

/*
XmlElement is our basic building block. Everything is an XmlElement; even XmlDocument
behaves like an XmlElement by inheriting its attributes and functions.
*/

function XmlElement(tag) {
  // Capture the parser object off of the XmlDocument delegate
  var parser = delegates[delegates.length - 1].parser;

  this.name = tag.name;
  this.attr = tag.attributes || {};
  this.val = "";
  this.children = [];
  this.firstChild = null;
  this.lastChild = null;

  // Assign parse information
  this.line = parser.line;
  this.column = parser.column;
  this.position = parser.position;
  this.startTagPosition = parser.startTagPosition;
}

// SaxParser handlers

XmlElement.prototype._opentag = function(tag) {

  var child = new XmlElement(tag);

  // add to our children array
  this.children.push(child);

  // update first/last pointers
  if (!this.firstChild) this.firstChild = child;
  this.lastChild = child;

  delegates.unshift(child);
};

XmlElement.prototype._closetag = function() {
  delegates.shift();
};

XmlElement.prototype._text = function(text) {
  if (text) this.val += text;
};

XmlElement.prototype._cdata = function(cdata) {
  if (cdata) this.val += cdata;
};

XmlElement.prototype._error = function(err) {
  throw err;
};

// Useful functions

XmlElement.prototype.eachChild = function(iterator, context) {
  for (var i=0, l=this.children.length; i<l; i++)
    if (iterator.call(context, this.children[i], i, this.children) === false) return;
};

XmlElement.prototype.childNamed = function(name) {
  for (var i=0, l=this.children.length; i<l; i++) {
    var child = this.children[i];
    if (child.name === name) return child;
  }
  return undefined;
};

XmlElement.prototype.childrenNamed = function(name) {
  var matches = [];

  for (var i=0, l=this.children.length; i<l; i++)
    if (this.children[i].name === name)
      matches.push(this.children[i]);

  return matches;
};

XmlElement.prototype.childWithAttribute = function(name,value) {
  for (var i=0, l=this.children.length; i<l; i++) {
    var child = this.children[i];
    if ( (value && child.attr[name] === value) || (!value && child.attr[name]) )
      return child;
  }
  return undefined;
};

XmlElement.prototype.descendantWithPath = function(path) {
  var descendant = this;
  var components = path.split('.');

  for (var i=0, l=components.length; i<l; i++)
    if (descendant)
      descendant = descendant.childNamed(components[i]);
    else
      return undefined;

  return descendant;
};

XmlElement.prototype.valueWithPath = function(path) {
  var components = path.split('@');
  var descendant = this.descendantWithPath(components[0]);
  if (descendant)
    return components.length > 1 ? descendant.attr[components[1]] : descendant.val;
  else
    return undefined;
};

// String formatting (for debugging)

XmlElement.prototype.toString = function(options) {
  return this.toStringWithIndent("", options);
};

XmlElement.prototype.toStringWithIndent = function(indent, options) {
  var s = indent + "<" + this.name;
  var linebreak = options && options.compressed ? "" : "\n";

  for (var name in this.attr)
    if (Object.prototype.hasOwnProperty.call(this.attr, name))
        s += " " + name + '="' + encodeURIComponent(this.attr[name]) + '"';

  var finalVal = this.val.trim();
  if (finalVal.indexOf('<') !== -1 || finalVal.indexOf('>') !== -1 || finalVal.indexOf('&') !== -1) {
    if (options && options.cdata) {
      finalVal = "<![CDATA[" + finalVal + "]]>";
    } else {
      finalVal = finalVal.replace(/</g, "<").replace(/>/g, ">").replace(/&/g, '&');
    }
  }

  if (options && options.trimmed && finalVal.length > 25)
    finalVal = finalVal.substring(0,25).trim() + "…";

  if (this.children.length) {
    s += ">" + linebreak;

    var childIndent = indent + (options && options.compressed ? "" : "  ");

    if (finalVal.length)
      s += childIndent + finalVal + linebreak;

    for (var i=0, l=this.children.length; i<l; i++)
      s += this.children[i].toStringWithIndent(childIndent, options) + linebreak;

    s += indent + "</" + this.name + ">";
  }
  else if (finalVal.length) {
    s += ">" + finalVal + "</" + this.name +">";
  }
  else s += "/>";

  return s;
};

/*
XmlDocument is the class we expose to the user; it uses the sax parser to create a hierarchy
of XmlElements.
*/

function XmlDocument(xml) {
  xml && (xml = xml.toString().trim());

  if (!xml)
    throw new Error("No XML to parse!");

  // Expose the parser to the other delegates while the parser is running
  this.parser = sax.parser(true); // strict
  addParserEvents(this.parser);

  // We'll use the file-scoped "delegates" var to remember what elements we're currently
  // parsing; they will push and pop off the stack as we get deeper into the XML hierarchy.
  // It's safe to use a global because JS is single-threaded.
  delegates = [this];

  this.parser.write(xml);

  // Remove the parser as it is no longer needed and should not be exposed to clients
  delete this.parser;
}

// make XmlDocument inherit XmlElement's methods
extend(XmlDocument.prototype, XmlElement.prototype);

XmlDocument.prototype._opentag = function(tag) {
  if (typeof this.children === 'undefined')
    // the first tag we encounter should be the root - we'll "become" the root XmlElement
    XmlElement.call(this,tag);
  else
    // all other tags will be the root element's children
    XmlElement.prototype._opentag.apply(this,arguments);
};

// file-scoped global stack of delegates
var delegates = null;

/*
Helper functions
*/

function addParserEvents(parser) {
  parser.onopentag = parser_opentag;
  parser.onclosetag = parser_closetag;
  parser.ontext = parser_text;
  parser.oncdata = parser_cdata;
  parser.onerror = parser_error;
}

// create these closures and cache them by keeping them file-scoped
function parser_opentag() { delegates[0]._opentag.apply(delegates[0],arguments) }
function parser_closetag() { delegates[0]._closetag.apply(delegates[0],arguments) }
function parser_text() { delegates[0]._text.apply(delegates[0],arguments) }
function parser_cdata() { delegates[0]._cdata.apply(delegates[0],arguments) }
function parser_error() { delegates[0]._error.apply(delegates[0],arguments) }

// a relatively standard extend method
function extend(destination, source) {
  for (var prop in source)
    if (source.hasOwnProperty(prop))
      destination[prop] = source[prop];
}

// Are we being used in a Node-like environment?
if (typeof module !== 'undefined' && module.exports)
    module.exports.XmlDocument = XmlDocument;
else
    this.XmlDocument = XmlDocument;

})();
<commit_msg>Fixed incorrect attribute escaping & escape order<commit_after>(function () {

var sax;

if (typeof module !== 'undefined' && module.exports) {
  // We're being used in a Node-like environment
  sax = require('sax');
}
else {
  // assume it's attached to the Window object in a browser
  sax = this.sax;

  if (!sax) // no sax for you!
    throw new Error("Expected sax to be defined. Make sure you're including sax.js before this file.");
}

/*
XmlElement is our basic building block. Everything is an XmlElement; even XmlDocument
behaves like an XmlElement by inheriting its attributes and functions.
*/

function XmlElement(tag) {
  // Capture the parser object off of the XmlDocument delegate
  var parser = delegates[delegates.length - 1].parser;

  this.name = tag.name;
  this.attr = tag.attributes || {};
  this.val = "";
  this.children = [];
  this.firstChild = null;
  this.lastChild = null;

  // Assign parse information
  this.line = parser.line;
  this.column = parser.column;
  this.position = parser.position;
  this.startTagPosition = parser.startTagPosition;
}

// SaxParser handlers

XmlElement.prototype._opentag = function(tag) {

  var child = new XmlElement(tag);

  // add to our children array
  this.children.push(child);

  // update first/last pointers
  if (!this.firstChild) this.firstChild = child;
  this.lastChild = child;

  delegates.unshift(child);
};

XmlElement.prototype._closetag = function() {
  delegates.shift();
};

XmlElement.prototype._text = function(text) {
  if (text) this.val += text;
};

XmlElement.prototype._cdata = function(cdata) {
  if (cdata) this.val += cdata;
};

XmlElement.prototype._error = function(err) {
  throw err;
};

// Useful functions

XmlElement.prototype.eachChild = function(iterator, context) {
  for (var i=0, l=this.children.length; i<l; i++)
    if (iterator.call(context, this.children[i], i, this.children) === false) return;
};

XmlElement.prototype.childNamed = function(name) {
  for (var i=0, l=this.children.length; i<l; i++) {
    var child = this.children[i];
    if (child.name === name) return child;
  }
  return undefined;
};

XmlElement.prototype.childrenNamed = function(name) {
  var matches = [];

  for (var i=0, l=this.children.length; i<l; i++)
    if (this.children[i].name === name)
      matches.push(this.children[i]);

  return matches;
};

XmlElement.prototype.childWithAttribute = function(name,value) {
  for (var i=0, l=this.children.length; i<l; i++) {
    var child = this.children[i];
    if ( (value && child.attr[name] === value) || (!value && child.attr[name]) )
      return child;
  }
  return undefined;
};

XmlElement.prototype.descendantWithPath = function(path) {
  var descendant = this;
  var components = path.split('.');

  for (var i=0, l=components.length; i<l; i++)
    if (descendant)
      descendant = descendant.childNamed(components[i]);
    else
      return undefined;

  return descendant;
};

XmlElement.prototype.valueWithPath = function(path) {
  var components = path.split('@');
  var descendant = this.descendantWithPath(components[0]);
  if (descendant)
    return components.length > 1 ? descendant.attr[components[1]] : descendant.val;
  else
    return undefined;
};

// String formatting (for debugging)

XmlElement.prototype.toString = function(options) {
  return this.toStringWithIndent("", options);
};

function escapeValue(str, isAttribute) {
  str = str.replace(/&/g, '&').replace(/</g, "<").replace(/>/g, ">");
  if (isAttribute) {
    str = str.replace(/"/g, '"');
  }
  return str;
}

XmlElement.prototype.toStringWithIndent = function(indent, options) {
  var s = indent + "<" + this.name;
  var linebreak = options && options.compressed ? "" : "\n";

  for (var name in this.attr) {
    if (Object.prototype.hasOwnProperty.call(this.attr, name)) {
      s += " " + name + '="' + escapeValue(this.attr[name]) + '"';
    }
  }

  var finalVal = this.val.trim();
  if (finalVal.indexOf('<') !== -1 || finalVal.indexOf('>') !== -1 || finalVal.indexOf('&') !== -1) {
    if (options && options.cdata) {
      finalVal = "<![CDATA[" + finalVal + "]]>";
    } else {
      finalVal = escapeValue(finalVal);
    }
  }

  if (options && options.trimmed && finalVal.length > 25)
    finalVal = finalVal.substring(0,25).trim() + "…";

  if (this.children.length) {
    s += ">" + linebreak;

    var childIndent = indent + (options && options.compressed ? "" : "  ");

    if (finalVal.length)
      s += childIndent + finalVal + linebreak;

    for (var i=0, l=this.children.length; i<l; i++)
      s += this.children[i].toStringWithIndent(childIndent, options) + linebreak;

    s += indent + "</" + this.name + ">";
  }
  else if (finalVal.length) {
    s += ">" + finalVal + "</" + this.name +">";
  }
  else s += "/>";

  return s;
};

/*
XmlDocument is the class we expose to the user; it uses the sax parser to create a hierarchy
of XmlElements.
*/

function XmlDocument(xml) {
  xml && (xml = xml.toString().trim());

  if (!xml)
    throw new Error("No XML to parse!");

  // Expose the parser to the other delegates while the parser is running
  this.parser = sax.parser(true); // strict
  addParserEvents(this.parser);

  // We'll use the file-scoped "delegates" var to remember what elements we're currently
  // parsing; they will push and pop off the stack as we get deeper into the XML hierarchy.
  // It's safe to use a global because JS is single-threaded.
  delegates = [this];

  this.parser.write(xml);

  // Remove the parser as it is no longer needed and should not be exposed to clients
  delete this.parser;
}

// make XmlDocument inherit XmlElement's methods
extend(XmlDocument.prototype, XmlElement.prototype);

XmlDocument.prototype._opentag = function(tag) {
  if (typeof this.children === 'undefined')
    // the first tag we encounter should be the root - we'll "become" the root XmlElement
    XmlElement.call(this,tag);
  else
    // all other tags will be the root element's children
    XmlElement.prototype._opentag.apply(this,arguments);
};

// file-scoped global stack of delegates
var delegates = null;

/*
Helper functions
*/

function addParserEvents(parser) {
  parser.onopentag = parser_opentag;
  parser.onclosetag = parser_closetag;
  parser.ontext = parser_text;
  parser.oncdata = parser_cdata;
  parser.onerror = parser_error;
}

// create these closures and cache them by keeping them file-scoped
function parser_opentag() { delegates[0]._opentag.apply(delegates[0],arguments) }
function parser_closetag() { delegates[0]._closetag.apply(delegates[0],arguments) }
function parser_text() { delegates[0]._text.apply(delegates[0],arguments) }
function parser_cdata() { delegates[0]._cdata.apply(delegates[0],arguments) }
function parser_error() { delegates[0]._error.apply(delegates[0],arguments) }

// a relatively standard extend method
function extend(destination, source) {
  for (var prop in source)
    if (source.hasOwnProperty(prop))
      destination[prop] = source[prop];
}

// Are we being used in a Node-like environment?
if (typeof module !== 'undefined' && module.exports)
    module.exports.XmlDocument = XmlDocument;
else
    this.XmlDocument = XmlDocument;

})();
<|endoftext|>


<commit_before>alert("todo")
<commit_msg>Update yasmij.js<commit_after>alert("todo\n. Temp Hack: Please include all the javascript files in the src directory instead.")
<|endoftext|>