<commit_before>module.exports = function(app, db, env) {
	var url = require('url');
	var ldp = require('./vocab/ldp.js');			// LDP vocabulary
	var media = require('./media.js');				// media types
	var N3 = require('n3');
	var crypto = require('crypto');					// for MD5 (ETags)

	// route any requests matching /r/*
	var resource = app.route(env.context + '*');
	resource.all(function(req, res, next) {
		// all responses should have Link: <ldp:Resource> rel=type
		res.links({
			type: ldp.Resource
		});
		next();
	});

	resource.get(function(req, res, next) {
		console.log('GET ' + req.path);
		db.get(req.fullURL, function(err, triples, interactionModel) {
			if (err) {
				console.log(err.stack);
				res.send(500);
				return;
			}

			if (!triples) {
				res.send(404);
				return;
			}

			res.links({
				type: ldp.RDFSource
			});

			if (interactionModel === ldp.BasicContainer) {
				res.links({
					type: ldp.BasicContainer
				});

				// add in ldp:contains triples
				db.getContainment(req.fullURL, function(err, containment) {
					if (err) {
						console.log(err.stack);
						res.send(500);
						return;
					}

					containment.forEach(function(resource) {
						triples.push({
							subject: req.fullURL,
							predicate: ldp.contains,
							object: resource
						});
					});

					writeTurtle(req, res, triples);
				});
			} else {
				writeTurtle(req, res, triples);
			}
		});
	});

	resource.put(function(req, res, next) {
		console.log('PUT ' + req.path);
		if (!req.is('text/turtle')) {
			res.send(415);
			return;
		}

		parse(req, req.fullURL, function(err, triples, interactionModel) {
			if (err) {
				res.send(400);
				return;
			}

			// get the resource to check if it exists and check its ETag
			db.get(req.fullURL, function(err, originalTriples, originalInteraction) {
				db.put(req.fullURL, null, interactionModel, triples, function(err) {
					if (err) {
						console.log(err.stack);
						res.send(500);
						return;
					}

					res.send(triples ? 204 : 201)
				});
			});
		});
	});

	resource.post(function(req, res, next) {
		console.log('POST ' + req.path);
		if (!req.is('text/turtle')) {
			res.send(415);
			return;
		}

		db.isContainer(req.fullURL, function(err, result) {
			if (err) {
				console.log(err.stack);
				res.send(500);
				return;
			}

			console.log('isContainer: ' + result);

			if (!result) {
				res.send(409);
				return;
			}

			var loc = req.fullURL
						   + ((req.fullURL.substr(-1) == '/') ? '' : '/')
						   + 'res' + Date.now();
			parse(req, loc, function(err, triples, interactionModel) {
				if (err) {
					res.send(400);
					return;
				}

				db.put(loc, req.fullURL, interactionModel, triples, function(err) {
					if (err) {
						console.log(err.stack);
						res.send(500);
						return;
					}

					res.location(loc).send(201);
				});
			});
		});
	});

	resource.delete(function(req, res, next) {
		console.log('DELETE: ' + req.path);
		db.remove(req.fullURL, function(err, result) {
			if (err) {
				console.log(err.stack);
				res.send(500);
				return;
			}

			res.send(result ? 204 : 404);
		});
	});

	resource.options(function(req, res, next) {
		db.get(req.fullURL, function(err, triples, interactionModel) {
			if (err) {
				console.log(err.stack);
				res.send(500);
				return;
			}

			if (!triples) {
				res.send(404);
				return;
			}

			res.links({
				type: ldp.RDFSource
			});

			var allow = 'GET,PUT,DELETE,OPTIONS';
			if (interactionModel === ldp.BasicContainer) {
				res.links({
					type: ldp.BasicContainer
				});

				allow += ',POST';
				res.set('Accept-Post', media.turtle);
			}

			res.set('Allow', allow);
			res.send(200);
		});
	});

	function parse(req, resourceURI, callback) {
		var parser = N3.Parser();
		var triples = [];
		var interactionModel = ldp.RDFSource;
		parser.parse(req.rawBody, function(err, triple) {
			if (err) {
				callback(err);
				return;
			}

			if (triple) {
				// resolve the null relative URI
				if (triple.subject === '') {
					triple.subject = resourceURI;

				}

				// if this triple is <> rdf:type ldp:BasicContainer RDF type,
				// set the interaction model as BasicContainer
				if (triple.subject === resourceURI
					&& triple.predicate === 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'
					&& triple.object === ldp.BasicContainer) {
						interactionModel = ldp.BasicContainer;
				}

				triples.push(triple);
				return;
			}

			callback(null, triples, interactionModel);
		});
	}

	function asTurtle(triples, callback) {
		var writer = N3.Writer();
		writer.addTriples(triples);
		writer.end(callback);
	}

	function getETag(turtle) {
		return 'W/"' + crypto.createHash('md5').update(turtle).digest('hex') + '"';
	}

	function writeTurtle(req, res, triples) {
		asTurtle(triples, function(err, turtle) {
			if (err) {
				res.send(500);
			} else {
				var eTag = getETag(turtle);
				if (req.get('If-None-Match') === eTag) {
					res.send(304);
					return;
				}

				res.writeHead(200, { 'ETag': eTag, 'Content-Type': media.turtle });
				res.end(new Buffer(turtle), 'utf-8');
			}
		});
	}
};
<commit_msg>require If-Match header on PUT<commit_after>module.exports = function(app, db, env) {
	var url = require('url');
	var ldp = require('./vocab/ldp.js');			// LDP vocabulary
	var media = require('./media.js');				// media types
	var N3 = require('n3');
	var crypto = require('crypto');					// for MD5 (ETags)

	// route any requests matching /r/*
	var resource = app.route(env.context + '*');
	resource.all(function(req, res, next) {
		// all responses should have Link: <ldp:Resource> rel=type
		res.links({
			type: ldp.Resource
		});
		next();
	});

	resource.get(function(req, res, next) {
		console.log('GET ' + req.path);
		db.get(req.fullURL, function(err, triples, interactionModel) {
			if (err) {
				console.log(err.stack);
				res.send(500);
				return;
			}

			if (!triples) {
				res.send(404);
				return;
			}

			addLinkHeaders(res, interactionModel);
			addContainment(req, triples, interactionModel, function(err) {
				if (err) {
					console.log(err.stack);
					res.send(500);
					return;
				}

				writeTurtle(req, res, triples);
			});
		});
	});

	resource.put(function(req, res, next) {
		console.log('PUT ' + req.path);
		if (!req.is('text/turtle')) {
			res.send(415);
			return;
		}

		parse(req, req.fullURL, function(err, triples, interactionModel) {
			if (err) {
				res.send(400);
				return;
			}

			// get the resource to check if it exists and check its ETag
			db.get(req.fullURL, function(err, originalTriples, originalInteraction) {
				if (originalTriples) {
					// update
					var ifMatch = req.get('If-Match');
					if (!ifMatch) {
						res.send(428);
						return;
					}

					// add containment triples if necessary to calculate the correct ETag
					// for containers
					addContainment(req, triples, interactionModel, function(err) {
						if (err) {
							console.log(err.stack);
							res.send(500);
							return;
						}

						// calculate the ETag from the text/turtle representation
						asTurtle(triples, function(err, turtle) {
							if (err) {
								console.log(err.stack);
								res.send(500);
								return;
							}

							var eTag = getETag(turtle);
							if (ifMatch !== eTag) {
								res.send(412);
								return;
							}

							// we don't support changing from RDFSource to container or back
							// use the original interaction model
							db.put(req.fullURL, null, originalInteraction, triples, function(err) {
								if (err) {
									console.log(err.stack);
									res.send(500);
									return;
								}

								res.send(originalTriples ? 204 : 201)
							});
						});
					});
				} else {
					// create
					db.put(req.fullURL, null, interactionModel, triples, function(err) {
						if (err) {
							console.log(err.stack);
							res.send(500);
							return;
						}

						res.send(originalTriples ? 204 : 201)
					});
				}
			});
		});
	});

	resource.post(function(req, res, next) {
		console.log('POST ' + req.path);
		if (!req.is('text/turtle')) {
			res.send(415);
			return;
		}

		db.isContainer(req.fullURL, function(err, result) {
			if (err) {
				console.log(err.stack);
				res.send(500);
				return;
			}

			if (!result) {
				res.send(409);
				return;
			}

			var loc = req.fullURL
						   + ((req.fullURL.substr(-1) == '/') ? '' : '/')
						   + 'res' + Date.now();
			parse(req, loc, function(err, triples, interactionModel) {
				if (err) {
					res.send(400);
					return;
				}

				addLinkHeaders(res, interactionModel);

				db.put(loc, req.fullURL, interactionModel, triples, function(err) {
					if (err) {
						console.log(err.stack);
						res.send(500);
						return;
					}

					res.location(loc).send(201);
				});
			});
		});
	});

	resource.delete(function(req, res, next) {
		console.log('DELETE: ' + req.path);
		db.remove(req.fullURL, function(err, result) {
			if (err) {
				console.log(err.stack);
				res.send(500);
				return;
			}

			res.send(result ? 204 : 404);
		});
	});

	resource.options(function(req, res, next) {
		db.get(req.fullURL, function(err, triples, interactionModel) {
			if (err) {
				console.log(err.stack);
				res.send(500);
				return;
			}

			if (!triples) {
				res.send(404);
				return;
			}

			addLinkHeaders(res, interactionModel);

			var allow = 'GET,PUT,DELETE,OPTIONS';
			if (interactionModel === ldp.BasicContainer) {
				res.links({
					type: ldp.BasicContainer
				});

				allow += ',POST';
				res.set('Accept-Post', media.turtle);
			}

			res.set('Allow', allow);
			res.send(200);
		});
	});

	function parse(req, resourceURI, callback) {
		var parser = N3.Parser();
		var triples = [];
		var interactionModel = ldp.RDFSource;
		parser.parse(req.rawBody, function(err, triple) {
			if (err) {
				callback(err);
				return;
			}

			if (triple) {
				// resolve the null relative URI
				if (triple.subject === '') {
					triple.subject = resourceURI;

				}

				// if this triple is <> rdf:type ldp:BasicContainer RDF type,
				// set the interaction model as BasicContainer
				if (triple.subject === resourceURI
					&& triple.predicate === 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'
					&& triple.object === ldp.BasicContainer) {
						interactionModel = ldp.BasicContainer;
				}

				triples.push(triple);
				return;
			}

			callback(null, triples, interactionModel);
		});
	}

	function asTurtle(triples, callback) {
		var writer = N3.Writer();
		writer.addTriples(triples);
		writer.end(callback);
	}

	function getETag(turtle) {
		return 'W/"' + crypto.createHash('md5').update(turtle).digest('hex') + '"';
	}

	function addLinkHeaders(res, interactionModel) {
		res.links({ type: ldp.RDFSource });

		if (interactionModel === ldp.BasicContainer) {
			res.links({ type: ldp.BasicContainer });
		}
	}

	// insert containment triples if necessary
	function addContainment(req, triples, interactionModel, callback) {
		if (interactionModel === ldp.BasicContainer) {
			db.getContainment(req.fullURL, function(err, containment) {
				if (containment) {
					containment.forEach(function(resource) {
						triples.push({
							subject: req.fullURL,
							predicate: ldp.contains,
							object: resource
						});
					});
				}
				callback(err);
			});
		} else {
			callback();
		}
	}

	function writeTurtle(req, res, triples) {
		asTurtle(triples, function(err, turtle) {
			if (err) {
				res.send(500);
			} else {
				var eTag = getETag(turtle);
				if (req.get('If-None-Match') === eTag) {
					res.send(304);
					return;
				}

				res.writeHead(200, { 'ETag': eTag, 'Content-Type': media.turtle });
				res.end(new Buffer(turtle), 'utf-8');
			}
		});
	}
};
<|endoftext|>


<commit_before>/* jslint node: true, esnext: true */
"use strict";

const http = require('http'),
  https = require('https'),
  address = require('network-address'),
  Koa = require('kronos-koa'),
  IO = require('koa-socket'),
  Service = require('kronos-service').Service,
  ReceiveEndpoint = require('kronos-endpoint').ReceiveEndpoint;

const configAttributes = {
  port: {
    needsRestart: true,
    default: 9898
  },
  hostname: {
    needsRestart: true,
    default: address()
  },
  key: {
    needsRestart: true
  },
  cert: {
    needsRestart: true
  },
  timeout: {
    default: 120000
  },
  io: {
    needsRestart: true
  }
};

/**
 * HTTP server
 */
class ServiceKOA extends Service {

  static get name() {
    return "koa";
  }

  get type() {
    return ServiceKOA.name;
  }

  constructor(config, owner) {
    super(config, owner);

    this.koa = new Koa();

    const props = {};

    Object.keys(configAttributes).forEach(name =>
      props[name] = {
        get() {
          return config[name] || configAttributes[name].default;
        }
      });

    Object.defineProperties(this, props);

    const io = new ReceiveEndpoint('io');
    io.receive = request => {
      if (this.koa.io) {
        this.koa.io.broadcast(request.event, request.data);
      }
      return Promise.resolve();
    };

    this.addEndpoint(io);
  }

  get isSecure() {
    return this.key !== undefined;
  }

  get serverOptions() {
    if (this.isSecure) {
      return {
        key: this.key,
        cert: this.cert
      };
    }

    return undefined;
  }

  get scheme() {
    return this.isSecure ? 'https' : 'http';
  }

  get url() {
    return `${this.scheme}://${this.hostname}:${this.port}`;
  }

  /**
   * apply new configuration.
   * if required restarts the server
   */
  configure(config) {
    let needsRestart = false;

    if (config.timeout) {
      this.server.setTimeout(config.timeout);
    }

    Object.keys(configAttributes).forEach(name => {
      if (config[name] !== undefined && this[name] !== config[name]) {
        needsRestart |= configAttributes[name].needsRestart;
        this.config[name] = config[name];
      }
    });

    if (config.io) {
      const io = new IO();
      io.attach(this.koa);
    }

    return needsRestart ? this.restartIfRunning() : Promise.resolve();
  }

  _start() {
    if (!this.server) {
      if (this.isSecure) {
        this.server = https.createServer(this.serverOptions, this.koa.callback());
      } else {
        this.server = http.createServer(this.koa.callback());
      }

      if (this.timeout) {
        this.server.setTimeout(this.timeout);
      }

      return new Promise((fullfill, reject) => {
        this.info(level => `Starting ${this.url}`);

        try {
          this.server.listen(this.port, this.hostname, err => {
            if (err) {
              this.server = undefined;
              reject(err);
            } else {
              fullfill();
            }
          });
        } catch (e) {
          this.error(e);
          reject(e);
        }
      });
    }

    return Promise.resolve();
  }

  _stop() {
    return new Promise((fulfill, reject) => {
      this.info(`Stopping ${this.server}`);

      this.server.close(err => {
        if (err) {
          reject(err);
        } else {
          this.server = undefined;
          fulfill();
        }
      });
    });
  }
}

module.exports.Service = ServiceKOA;

module.exports.registerWithManager = manager => manager.registerServiceFactory(ServiceKOA);
<commit_msg>feat(logging): stop logger now shows the server url<commit_after>/* jslint node: true, esnext: true */
"use strict";

const http = require('http'),
  https = require('https'),
  address = require('network-address'),
  Koa = require('kronos-koa'),
  IO = require('koa-socket'),
  Service = require('kronos-service').Service,
  ReceiveEndpoint = require('kronos-endpoint').ReceiveEndpoint;

const configAttributes = {
  port: {
    needsRestart: true,
    default: 9898
  },
  hostname: {
    needsRestart: true,
    default: address()
  },
  key: {
    needsRestart: true
  },
  cert: {
    needsRestart: true
  },
  timeout: {
    default: 120000
  },
  io: {
    needsRestart: true
  }
};

/**
 * HTTP server
 */
class ServiceKOA extends Service {

  static get name() {
    return "koa";
  }

  get type() {
    return ServiceKOA.name;
  }

  constructor(config, owner) {
    super(config, owner);

    this.koa = new Koa();

    const props = {};

    Object.keys(configAttributes).forEach(name =>
      props[name] = {
        get() {
          return config[name] || configAttributes[name].default;
        }
      });

    Object.defineProperties(this, props);

    const io = new ReceiveEndpoint('io');
    io.receive = request => {
      if (this.koa.io) {
        this.koa.io.broadcast(request.event, request.data);
      }
      return Promise.resolve();
    };

    this.addEndpoint(io);
  }

  get isSecure() {
    return this.key !== undefined;
  }

  get serverOptions() {
    if (this.isSecure) {
      return {
        key: this.key,
        cert: this.cert
      };
    }

    return undefined;
  }

  get scheme() {
    return this.isSecure ? 'https' : 'http';
  }

  get url() {
    return `${this.scheme}://${this.hostname}:${this.port}`;
  }

  /**
   * apply new configuration.
   * if required restarts the server
   */
  configure(config) {
    let needsRestart = false;

    if (config.timeout) {
      this.server.setTimeout(config.timeout);
    }

    Object.keys(configAttributes).forEach(name => {
      if (config[name] !== undefined && this[name] !== config[name]) {
        needsRestart |= configAttributes[name].needsRestart;
        this.config[name] = config[name];
      }
    });

    if (config.io) {
      const io = new IO();
      io.attach(this.koa);
    }

    return needsRestart ? this.restartIfRunning() : Promise.resolve();
  }

  _start() {
    if (!this.server) {
      if (this.isSecure) {
        this.server = https.createServer(this.serverOptions, this.koa.callback());
      } else {
        this.server = http.createServer(this.koa.callback());
      }

      if (this.timeout) {
        this.server.setTimeout(this.timeout);
      }

      return new Promise((fullfill, reject) => {
        this.trace(level => `starting ${this.url}`);

        try {
          this.server.listen(this.port, this.hostname, err => {
            if (err) {
              this.server = undefined;
              reject(err);
            } else {
              fullfill();
            }
          });
        } catch (e) {
          this.error(e);
          reject(e);
        }
      });
    }

    return Promise.resolve();
  }

  _stop() {
    return new Promise((fulfill, reject) => {
      this.trace(level => `stopping ${this.url}`);

      this.server.close(err => {
        if (err) {
          reject(err);
        } else {
          this.server = undefined;
          fulfill();
        }
      });
    });
  }
}

module.exports.Service = ServiceKOA;

module.exports.registerWithManager = manager => manager.registerServiceFactory(ServiceKOA);
<|endoftext|>


<commit_before>/* globals d3, django, moment, nv */
hqDefine("reports/js/aggregate_app_status", function() {
    function setupCharts(data, div) {
        nv.addGraph(function() {
            var chart = nv.models.multiBarChart()
              .transitionDuration(100)
              .showControls(false)
              .reduceXTicks(true)
              .rotateLabels(0)
              .groupSpacing(0.1)
            ;

            chart.yAxis
                .tickFormat(d3.format(',f'));

            d3.select('#' + div + ' svg')
                .datum([data])
                .call(chart);

            nv.utils.windowResize(chart.update);
            return chart;
        });
    }
    $(document).ajaxSuccess(function(event, xhr, settings) {
        if (settings.url.match(/reports\/async\/aggregate_user_status/)) {
            setupCharts($("#submission-data").data("value"), 'submission_dates');
            setupCharts($("#sync-data").data("value"), 'sync_dates');
            $('.chart-toggle').click(function () {
                $(this).parent().children().not(this).removeClass('btn-primary');  // deselect oher buttons
                $(this).addClass('btn-primary');  // select self
                // update data
                setupCharts($("#" + $(this).data('chart-data')).data("value"), $(this).data('chart-div'));
            });
        }
    });
});
<commit_msg>remove globals<commit_after>/* globals d3, nv */
hqDefine("reports/js/aggregate_app_status", function() {
    function setupCharts(data, div) {
        nv.addGraph(function() {
            var chart = nv.models.multiBarChart()
              .transitionDuration(100)
              .showControls(false)
              .reduceXTicks(true)
              .rotateLabels(0)
              .groupSpacing(0.1)
            ;

            chart.yAxis
                .tickFormat(d3.format(',f'));

            d3.select('#' + div + ' svg')
                .datum([data])
                .call(chart);

            nv.utils.windowResize(chart.update);
            return chart;
        });
    }
    $(document).ajaxSuccess(function(event, xhr, settings) {
        if (settings.url.match(/reports\/async\/aggregate_user_status/)) {
            setupCharts($("#submission-data").data("value"), 'submission_dates');
            setupCharts($("#sync-data").data("value"), 'sync_dates');
            $('.chart-toggle').click(function () {
                $(this).parent().children().not(this).removeClass('btn-primary');  // deselect oher buttons
                $(this).addClass('btn-primary');  // select self
                // update data
                setupCharts($("#" + $(this).data('chart-data')).data("value"), $(this).data('chart-div'));
            });
        }
    });
});
<|endoftext|>


<commit_before>const LRU = cjsRequire('lru-cache')
const { BamFile } = cjsRequire('@gmod/bam')

const { Buffer } = cjsRequire('buffer')

const bamIndexedFilesCache = LRU(5)

const BlobFilehandleWrapper = cjsRequire('../../Model/BlobFilehandleWrapper')

class BamSlightlyLazyFeature {

    _get_name() { return this.record.get('name') }
    _get_start() { return this.record.get('start') }
    _get_end() { return this.record.get('end') }
    _get_type() { return 'match'}
    _get_mapping_quality() { return this.record.mappingQuality}
    _get_flags() { return `0x${this.record.flags.toString(16)}`}
    _get_strand() { return this.record.isReverseComplemented() ? -1 : 1 }
    _get_read_group_id() { return this.record.readGroupId }
    _get_qual() { return this.record.get('qual')}
    _get_cigar() { return this.record.get('cigar')}
    _get_seq_id() { return this._store._refIdToName(this.record.sequenceId)}
    _get_qc_failed() { return this.record.isFailedQc()}
    _get_secondary_alignment() { return this.record.isSecondary()}
    _get_supplementary_alignment() { return this.record.isSupplementary()}
    _get_multi_segment_template() { return this.record.isPaired()}
    _get_multi_segment_all_correctly_aligned() { return this.record.isProperlyPaired()}
    _get_multi_segment_next_segment_unmapped() { return this.record.isMateUnmapped()}
    _get_multi_segment_first() { return this.record.isRead1()}
    _get_multi_segment_last() { return this.record.isRead2()}
    _get_multi_segment_next_segment_reversed() { return this.record.isMateReverseComplemented()}
    _get_unmapped() { return this.record.isSegmentUnmapped()}
    _get_next_seq_id() { return this.record.mate ? this._store._refIdToName(this.record.mate.sequenceId) : undefined }
    _get_next_segment_position() { return this.record.mate
        ? ( this._store._refIdToName(this.record.mate.sequenceId)+':'+this.record.mate.alignmentStart) : undefined}
    _get_tags() { console.log('tags'); return this.record.tags }
    _get_seq() { return this.record.getReadBases() }
    _get_md() { return this.record.get('md') }

    constructor(record, store) {
        this.record = record
        this._store = store
    }

    tags() {
        const properties = Object.getOwnPropertyNames(BamSlightlyLazyFeature.prototype)
        return properties
            .filter(prop => /^_get_/.test(prop))
            .map(methodName => methodName.replace('_get_',''))
    }

    id() {
        return this.record.get('id')
    }

    get(field) {
        const methodName = `_get_${field.toLowerCase()}`
        if (this[methodName]) return this[methodName]()
        return undefined
    }

    parent() {}

    children() {}
}

define( [
            'dojo/_base/declare',
            'JBrowse/Errors',
            'JBrowse/Store/SeqFeature',
            'JBrowse/Store/DeferredStatsMixin',
            'JBrowse/Store/DeferredFeaturesMixin',
            'JBrowse/Store/SeqFeature/IndexedStatsEstimationMixin',
            'JBrowse/Model/XHRBlob',
            'JBrowse/Model/SimpleFeature',
        ],
        function(
            declare,
            Errors,
            SeqFeatureStore,
            DeferredStatsMixin,
            DeferredFeaturesMixin,
            IndexedStatsEstimationMixin,
            XHRBlob,
            SimpleFeature,
        ) {

return declare( [ SeqFeatureStore, DeferredStatsMixin, DeferredFeaturesMixin, IndexedStatsEstimationMixin ], {
    constructor: function( args ) {

        let dataBlob
        if (args.bam)
            dataBlob = new BlobFilehandleWrapper(args.bam)
        else if (args.urlTemplate)
            dataBlob = new BlobFilehandleWrapper(new XHRBlob(this.resolveUrl(args.urlTemplate || 'data.bam'), { expectRanges: true }))
        else throw new Error('must provide either `bam` or `urlTemplate`')

        let baiBlob, csiBlob
        if (args.bai)
            baiBlob = new BlobFilehandleWrapper(args.bai)
        else if (args.csi)
            csiBlob = new BlobFilehandleWrapper(args.csi)
        else if (args.baiUrlTemplate)
            baiBlob = new BlobFilehandleWrapper(new XHRBlob(this.resolveUrl(args.baiUrlTemplate)))
        else if (args.csiUrlTemplate)
            csiBlob = new BlobFilehandleWrapper(new XHRBlob(this.resolveUrl(args.csiUrlTemplate)))
        else if (args.urlTemplate)
            baiBlob = new BlobFilehandleWrapper(new XHRBlob(this.resolveUrl(args.urlTemplate+'.bai')))
        else throw new Error('no index provided, must provide a BAI or CSI index')

        this.source = dataBlob.toString()

        // LRU-cache the BAM object so we don't have to re-download the
        // index when we switch chromosomes
        const cacheKey = `data: ${dataBlob}, index: ${csiBlob||baiBlob}`
        this.bam = bamIndexedFilesCache.get(cacheKey)
        if (!this.bam) {
            this.bam = new BamFile({
                bamFilehandle: dataBlob,
                baiFilehandle: baiBlob,
                csiFilehandle: csiBlob
            })

            bamIndexedFilesCache.set(cacheKey, this.bam)
        }

        // pre-download the index before running the statistics estimation so that the stats
        // estimation doesn't time out
        this.bam.hasDataForReferenceSequence(0)
            .then(() => this.bam.getHeader())
            .then((header) => this._setSamHeader(header))
            .then((res) => {
                this._deferred.features.resolve({success:true});
            })
            .then(() => this._estimateGlobalStats())
            .then(stats => {
                this.globalStats = stats;
                this._deferred.stats.resolve({success:true});
            })
            .catch(err => {
                this._deferred.features.reject(err)
                this._deferred.stats.reject(err)
            })

        this.storeTimeout = args.storeTimeout || 3000;
    },

    // process the parsed SAM header from the bam file
    _setSamHeader(samHeader) {
        this._samHeader = {}

        // use the @SQ lines in the header to figure out the
        // mapping between ref seq ID numbers and names
        const refSeqIdToName = []
        const refSeqNameToId = {}
        const sqLines = samHeader.filter(l => l.tag === 'SQ')
        sqLines.forEach((sqLine, seqId) => {
            sqLine.data.forEach(item => {
                if (item.tag === 'SN') {
                    // this is the seq name
                    const seqName = item.value
                    refSeqNameToId[seqName] = seqId
                    refSeqIdToName[seqId] = seqName
                }
            })
        })
        if (refSeqIdToName.length) {
            this._samHeader.refSeqIdToName = refSeqIdToName
            this._samHeader.refSeqNameToId = refSeqNameToId
        }
    },

    _refNameToId(refName) {
        // use info from the SAM header if possible, but fall back to using
        // the ref seq order from when the browser's refseqs were loaded
        if (this._samHeader.refSeqNameToId)
            return this._samHeader.refSeqNameToId[refName]
        else
            return this.browser.getRefSeqNumber(refName)
    },

    _refIdToName(refId) {
        // use info from the SAM header if possible, but fall back to using
        // the ref seq order from when the browser's refseqs were loaded
        if (this._samHeader.refSeqIdToName) {
            return this._samHeader.refSeqIdToName[refId]
        } else {
            let ref = this.browser.getRefSeqById(refId)
            return ref ? ref.name : undefined
        }
    },

    /**
     * Interrogate whether a store has data for a given reference
     * sequence.  Calls the given callback with either true or false.
     */
    hasRefSeq: function( seqName, callback, errorCallback ) {
        seqName = this.browser.regularizeReferenceName( seqName );

        this._deferred.stats
        .then(() => this.bam.hasDataForReferenceSequence(seqName))
        .then(callback, errorCallback)
    },

    // called by getFeatures from the DeferredFeaturesMixin
    _getFeatures: function( query, featCallback, endCallback, errorCallback ) {
        const seqName = query.ref || this.refSeq.name

        this.bam.getRecordsForRange(seqName, query.start + 1, query.end)
            .then(records => {
                for (let i = 0; i < records.length; i+= 1) {
                    featCallback(this._bamRecordToFeature(records[i]))
                }

                endCallback()
            })
            .catch(err => {
                // // map the BamSizeLimitError to JBrowse Errors.DataOverflow
                // if (err instanceof BamSizeLimitError) {
                //     err = new Errors.DataOverflow(err)
                // }

                errorCallback(err)
            })
    },

    _bamRecordToFeature(record) {
        return new BamSlightlyLazyFeature(record, this)
    },

    saveStore: function() {
        return {
            urlTemplate: this.config.bam.url,
            baiUrlTemplate: (this.config.bai||{}).url,
            csiUrlTemplate: (this.config.csi||{}).url
        };
    }

});
});
<commit_msg>Support renameRefSeqs<commit_after>const LRU = cjsRequire('lru-cache')
const { BamFile } = cjsRequire('@gmod/bam')

const { Buffer } = cjsRequire('buffer')

const bamIndexedFilesCache = LRU(5)

const BlobFilehandleWrapper = cjsRequire('../../Model/BlobFilehandleWrapper')

class BamSlightlyLazyFeature {

    _get_name() { return this.record.get('name') }
    _get_start() { return this.record.get('start') }
    _get_end() { return this.record.get('end') }
    _get_type() { return 'match'}
    _get_mapping_quality() { return this.record.mappingQuality}
    _get_flags() { return `0x${this.record.flags.toString(16)}`}
    _get_strand() { return this.record.isReverseComplemented() ? -1 : 1 }
    _get_read_group_id() { return this.record.readGroupId }
    _get_qual() { return this.record.get('qual')}
    _get_cigar() { return this.record.get('cigar')}
    _get_seq_id() { return this._store._refIdToName(this.record.sequenceId)}
    _get_qc_failed() { return this.record.isFailedQc()}
    _get_secondary_alignment() { return this.record.isSecondary()}
    _get_supplementary_alignment() { return this.record.isSupplementary()}
    _get_multi_segment_template() { return this.record.isPaired()}
    _get_multi_segment_all_correctly_aligned() { return this.record.isProperlyPaired()}
    _get_multi_segment_next_segment_unmapped() { return this.record.isMateUnmapped()}
    _get_multi_segment_first() { return this.record.isRead1()}
    _get_multi_segment_last() { return this.record.isRead2()}
    _get_multi_segment_next_segment_reversed() { return this.record.isMateReverseComplemented()}
    _get_unmapped() { return this.record.isSegmentUnmapped()}
    _get_next_seq_id() { return this.record.mate ? this._store._refIdToName(this.record.mate.sequenceId) : undefined }
    _get_next_segment_position() { return this.record.mate
        ? ( this._store._refIdToName(this.record.mate.sequenceId)+':'+this.record.mate.alignmentStart) : undefined}
    _get_tags() { console.log('tags'); return this.record.tags }
    _get_seq() { return this.record.getReadBases() }
    _get_md() { return this.record.get('md') }

    constructor(record, store) {
        this.record = record
        this._store = store
    }

    tags() {
        const properties = Object.getOwnPropertyNames(BamSlightlyLazyFeature.prototype)
        return properties
            .filter(prop => /^_get_/.test(prop))
            .map(methodName => methodName.replace('_get_',''))
    }

    id() {
        return this.record.get('id')
    }

    get(field) {
        const methodName = `_get_${field.toLowerCase()}`
        if (this[methodName]) return this[methodName]()
        return undefined
    }

    parent() {}

    children() {}
}

define( [
            'dojo/_base/declare',
            'JBrowse/Errors',
            'JBrowse/Store/SeqFeature',
            'JBrowse/Store/DeferredStatsMixin',
            'JBrowse/Store/DeferredFeaturesMixin',
            'JBrowse/Store/SeqFeature/IndexedStatsEstimationMixin',
            'JBrowse/Model/XHRBlob',
            'JBrowse/Model/SimpleFeature',
        ],
        function(
            declare,
            Errors,
            SeqFeatureStore,
            DeferredStatsMixin,
            DeferredFeaturesMixin,
            IndexedStatsEstimationMixin,
            XHRBlob,
            SimpleFeature,
        ) {

return declare( [ SeqFeatureStore, DeferredStatsMixin, DeferredFeaturesMixin, IndexedStatsEstimationMixin ], {
    constructor: function( args ) {

        let dataBlob
        if (args.bam)
            dataBlob = new BlobFilehandleWrapper(args.bam)
        else if (args.urlTemplate)
            dataBlob = new BlobFilehandleWrapper(new XHRBlob(this.resolveUrl(args.urlTemplate || 'data.bam'), { expectRanges: true }))
        else throw new Error('must provide either `bam` or `urlTemplate`')

        let baiBlob, csiBlob
        if (args.bai)
            baiBlob = new BlobFilehandleWrapper(args.bai)
        else if (args.csi)
            csiBlob = new BlobFilehandleWrapper(args.csi)
        else if (args.baiUrlTemplate)
            baiBlob = new BlobFilehandleWrapper(new XHRBlob(this.resolveUrl(args.baiUrlTemplate)))
        else if (args.csiUrlTemplate)
            csiBlob = new BlobFilehandleWrapper(new XHRBlob(this.resolveUrl(args.csiUrlTemplate)))
        else if (args.urlTemplate)
            baiBlob = new BlobFilehandleWrapper(new XHRBlob(this.resolveUrl(args.urlTemplate+'.bai')))
        else throw new Error('no index provided, must provide a BAI or CSI index')

        this.source = dataBlob.toString()

        // LRU-cache the BAM object so we don't have to re-download the
        // index when we switch chromosomes
        const cacheKey = `data: ${dataBlob}, index: ${csiBlob||baiBlob}`
        this.bam = bamIndexedFilesCache.get(cacheKey)
        if (!this.bam) {
            this.bam = new BamFile({
                bamFilehandle: dataBlob,
                baiFilehandle: baiBlob,
                csiFilehandle: csiBlob,
                renameRefSeqs: n => this.browser.regularizeReferenceName(n)
            })

            bamIndexedFilesCache.set(cacheKey, this.bam)
        }

        // pre-download the index before running the statistics estimation so that the stats
        // estimation doesn't time out
        this.bam.hasDataForReferenceSequence(0)
            .then(() => this.bam.getHeader())
            .then((header) => this._setSamHeader(header))
            .then((res) => {
                this._deferred.features.resolve({success:true});
            })
            .then(() => this._estimateGlobalStats())
            .then(stats => {
                this.globalStats = stats;
                this._deferred.stats.resolve({success:true});
            })
            .catch(err => {
                this._deferred.features.reject(err)
                this._deferred.stats.reject(err)
            })

        this.storeTimeout = args.storeTimeout || 3000;
    },

    // process the parsed SAM header from the bam file
    _setSamHeader(samHeader) {
        this._samHeader = {}

        // use the @SQ lines in the header to figure out the
        // mapping between ref seq ID numbers and names
        const refSeqIdToName = []
        const refSeqNameToId = {}
        const sqLines = samHeader.filter(l => l.tag === 'SQ')
        sqLines.forEach((sqLine, seqId) => {
            sqLine.data.forEach(item => {
                if (item.tag === 'SN') {
                    // this is the seq name
                    const seqName = item.value
                    refSeqNameToId[seqName] = seqId
                    refSeqIdToName[seqId] = seqName
                }
            })
        })
        if (refSeqIdToName.length) {
            this._samHeader.refSeqIdToName = refSeqIdToName
            this._samHeader.refSeqNameToId = refSeqNameToId
        }
    },

    _refNameToId(refName) {
        // use info from the SAM header if possible, but fall back to using
        // the ref seq order from when the browser's refseqs were loaded
        if (this._samHeader.refSeqNameToId)
            return this._samHeader.refSeqNameToId[refName]
        else
            return this.browser.getRefSeqNumber(refName)
    },

    _refIdToName(refId) {
        // use info from the SAM header if possible, but fall back to using
        // the ref seq order from when the browser's refseqs were loaded
        if (this._samHeader.refSeqIdToName) {
            return this._samHeader.refSeqIdToName[refId]
        } else {
            let ref = this.browser.getRefSeqById(refId)
            return ref ? ref.name : undefined
        }
    },

    /**
     * Interrogate whether a store has data for a given reference
     * sequence.  Calls the given callback with either true or false.
     */
    hasRefSeq: function( seqName, callback, errorCallback ) {
        seqName = this.browser.regularizeReferenceName( seqName );

        this._deferred.stats
        .then(() => this.bam.hasDataForReferenceSequence(seqName))
        .then(callback, errorCallback)
    },

    // called by getFeatures from the DeferredFeaturesMixin
    _getFeatures: function( query, featCallback, endCallback, errorCallback ) {
        let seqName = query.ref || this.refSeq.name
        seqName = this.browser.regularizeReferenceName( seqName );

        this.bam.getRecordsForRange(seqName, query.start + 1, query.end)
            .then(records => {
                for (let i = 0; i < records.length; i+= 1) {
                    featCallback(this._bamRecordToFeature(records[i]))
                }

                endCallback()
            })
            .catch(err => {
                // // map the BamSizeLimitError to JBrowse Errors.DataOverflow
                // if (err instanceof BamSizeLimitError) {
                //     err = new Errors.DataOverflow(err)
                // }

                errorCallback(err)
            })
    },

    _bamRecordToFeature(record) {
        return new BamSlightlyLazyFeature(record, this)
    },

    saveStore: function() {
        return {
            urlTemplate: this.config.bam.url,
            baiUrlTemplate: (this.config.bai||{}).url,
            csiUrlTemplate: (this.config.csi||{}).url
        };
    }

});
});
<|endoftext|>


<commit_before>
var _ = require('lodash'),
  redis = require('redis'),
  createGithubApi = require('./create-github-api.js');

// Session backed by Redis, but that
// attempts to lookup against GHE if
// the token is not found in the DB.
function SessionGithub(opts) {
  _.extend(this, {
    client: redis.createClient(),
    githubHost: 'api.github.com',
    debug: true,
    githubPathPrefix: '/api/v3'
  }, require('@npm/enterprise-configurator').Config(), opts)
}

SessionGithub.prototype.get = function(key, cb) {
  var _this = this;

  this.client.get(key, function(err, data) {
    if (err) cb(err);
    else if (data) cb(null, JSON.parse(data));
    else { // attempt to lookup user in GHE.
      _this._githubLookup(key, cb);
    }
  });
};

SessionGithub.prototype._githubLookup = function(key, cb) {
  var _this = this,
    github = createGithubApi(this),
    // extract the GHE key from the user-<token> string.
    token = key.split('-').splice(1).join('-');

  github.authenticate({
    type: 'oauth',
    token: token
  });

  github.user.get({}, function(err, res) {
    if (err) cb(err);
    else if (res.code < 200 || res.code >= 400) cb(Error('status = ' + res.code));
    else {
      var session = {
        name: res.login,
        email: res.email
      };

      _this.set(key, session, function(err) {
        if (err) cb(err);
        else cb(undefined, session);
      });
    }
  });
};

SessionGithub.prototype.set = function(key, session, cb) {
  this.client.set(key, JSON.stringify(session), cb);
};

module.exports = SessionGithub;
<commit_msg>Add comments to GitHub session implementation<commit_after>
var _ = require('lodash'),
  redis = require('redis'),
  createGithubApi = require('./create-github-api.js');

// Session backed by Redis, but that
// attempts to lookup against GHE if
// the token is not found in the DB.
function SessionGithub(opts) {
  _.extend(this, {
    client: redis.createClient(),
    githubHost: 'api.github.com',
    debug: true,
    githubPathPrefix: '/api/v3'
  }, require('@npm/enterprise-configurator').Config(), opts)
}

SessionGithub.prototype.get = function(key, cb) {
  var _this = this;

  // First check if we have the session in Redis.
  this.client.get(key, function(err, data) {
    if (err) cb(err);
    // If we do, simply return the existing session.
    else if (data) cb(null, JSON.parse(data));
    else {
      // If we don't, talk to GHE instance. This can happen when, for example,
      // user tries logging into a different npm Enterprise instance with an
      // existing, logged in .npmrc.
      _this._githubLookup(key, cb);
    }
  });
};

SessionGithub.prototype._githubLookup = function(key, cb) {
  var _this = this,
    github = createGithubApi(this),
    // extract the GHE key from the user-<token> string.
    token = key.split('-').splice(1).join('-');

  // the token authenticator creates is a GitHub OAuth token, so we can simply
  // try authenticating with it and then fetching the authenticated user.
  github.authenticate({
    type: 'oauth',
    token: token
  });

  github.user.get({}, function(err, res) {
    if (err) cb(err);
    else if (res.code < 200 || res.code >= 400) cb(Error('status = ' + res.code));
    else {
      var session = {
        name: res.login,
        email: res.email
      };

      // If authentication with GitHub succeeded, persist the GitHub login and
      // email in our session store.
      _this.set(key, session, function(err) {
        if (err) cb(err);
        else cb(undefined, session);
      });
    }
  });
};

SessionGithub.prototype.set = function(key, session, cb) {
  this.client.set(key, JSON.stringify(session), cb);
};

module.exports = SessionGithub;
<|endoftext|>


<commit_before>const { EventEmitter } = require('events');
const { exec } = require('child_process');
const defaultShell = require('default-shell');

let spawn;
try {
  spawn = require('child_pty').spawn;
} catch (err) {
  console.error(
    'A native module failed to load. Typically this means ' +
    'you installed the modules incorrectly.\n Use `scripts/install.sh` ' +
    'to trigger the installation.\n ' +
    'More information: https://github.com/zeit/hyperterm/issues/72'
  );
}

const TITLE_POLL_INTERVAL = 500;

module.exports = class Session extends EventEmitter {

  constructor ({ rows, cols: columns, cwd, shell }) {
    super();
    this.pty = spawn(shell || defaultShell, ['--login'], {
      columns,
      rows,
      cwd,
      env: Object.assign({}, process.env, {
        TERM: 'xterm-256color'
      })
    });

    this.pty.stdout.on('data', (data) => {
      this.emit('data', data.toString('utf8'));
    });

    this.pty.on('exit', () => {
      if (!this.ended) {
        this.ended = true;
        this.emit('exit');
      }
    });

    this.shell = defaultShell;
    this.getTitle();
  }

  focus () {
    this.subscribed = true;
    this.getTitle();
  }

  blur () {
    this.subscribed = false;
    clearTimeout(this.titlePoll);
  }

  getTitle () {
    if ('win32' === process.platform) return;
    if (this.fetching) return;
    this.fetching = true;

    let tty = this.pty.stdout.ttyname;
    tty = tty.replace(/^\/dev\/tty/, '');

    // try to exclude grep from the results
    // by grepping for `[s]001` instead of `s001`
    tty = `[${tty[0]}]${tty.substr(1)}`;

    // TODO: limit the concurrency of how many processes we run?
    // TODO: only tested on mac
    exec(`ps uxac | grep ${tty} | head -n 1`, (err, out) => {
      this.fetching = false;
      if (this.ended) return;
      if (err) return;
      let title = out.split(' ').pop();
      if (title) {
        title = title.replace(/^\(/, '');
        title = title.replace(/\)?\n$/, '');
        if (title !== this.lastTitle) {
          this.emit('title', title);
          this.lastTitle = title;
        }
      }

      if (this.subscribed) {
        this.titlePoll = setTimeout(() => this.getTitle(), TITLE_POLL_INTERVAL);
      }
    });
  }

  exit () {
    this.destroy();
  }

  write (data) {
    this.pty.stdin.write(data);
  }

  resize ({ cols: columns, rows }) {
    try {
      this.pty.stdout.resize({ columns, rows });
    } catch (err) {
      console.error(err.stack);
    }
  }

  destroy () {
    try {
      this.pty.kill('SIGHUP');
    } catch (err) {
      console.error('exit error', err.stack);
    }
    this.emit('exit');
    this.ended = true;
    this.blur();
  }

};
<commit_msg>add LANG env variable (#354)<commit_after>const { app } = require('electron');
const { EventEmitter } = require('events');
const { exec } = require('child_process');
const defaultShell = require('default-shell');

let spawn;
try {
  spawn = require('child_pty').spawn;
} catch (err) {
  console.error(
    'A native module failed to load. Typically this means ' +
    'you installed the modules incorrectly.\n Use `scripts/install.sh` ' +
    'to trigger the installation.\n ' +
    'More information: https://github.com/zeit/hyperterm/issues/72'
  );
}

const TITLE_POLL_INTERVAL = 500;

module.exports = class Session extends EventEmitter {

  constructor ({ rows, cols: columns, cwd, shell }) {
    super();
    this.pty = spawn(shell || defaultShell, ['--login'], {
      columns,
      rows,
      cwd,
      env: Object.assign({}, process.env, {
        LANG: app.getLocale(),
        TERM: 'xterm-256color'
      })
    });

    this.pty.stdout.on('data', (data) => {
      this.emit('data', data.toString('utf8'));
    });

    this.pty.on('exit', () => {
      if (!this.ended) {
        this.ended = true;
        this.emit('exit');
      }
    });

    this.shell = defaultShell;
    this.getTitle();
  }

  focus () {
    this.subscribed = true;
    this.getTitle();
  }

  blur () {
    this.subscribed = false;
    clearTimeout(this.titlePoll);
  }

  getTitle () {
    if ('win32' === process.platform) return;
    if (this.fetching) return;
    this.fetching = true;

    let tty = this.pty.stdout.ttyname;
    tty = tty.replace(/^\/dev\/tty/, '');

    // try to exclude grep from the results
    // by grepping for `[s]001` instead of `s001`
    tty = `[${tty[0]}]${tty.substr(1)}`;

    // TODO: limit the concurrency of how many processes we run?
    // TODO: only tested on mac
    exec(`ps uxac | grep ${tty} | head -n 1`, (err, out) => {
      this.fetching = false;
      if (this.ended) return;
      if (err) return;
      let title = out.split(' ').pop();
      if (title) {
        title = title.replace(/^\(/, '');
        title = title.replace(/\)?\n$/, '');
        if (title !== this.lastTitle) {
          this.emit('title', title);
          this.lastTitle = title;
        }
      }

      if (this.subscribed) {
        this.titlePoll = setTimeout(() => this.getTitle(), TITLE_POLL_INTERVAL);
      }
    });
  }

  exit () {
    this.destroy();
  }

  write (data) {
    this.pty.stdin.write(data);
  }

  resize ({ cols: columns, rows }) {
    try {
      this.pty.stdout.resize({ columns, rows });
    } catch (err) {
      console.error(err.stack);
    }
  }

  destroy () {
    try {
      this.pty.kill('SIGHUP');
    } catch (err) {
      console.error('exit error', err.stack);
    }
    this.emit('exit');
    this.ended = true;
    this.blur();
  }

};
<|endoftext|>


<commit_before>'use strict';

const path = require('path');
const fs = require('fs-extra');
const walk = require('fs-walk');
const rimraf = require('rimraf');
const tools = require('./tools');
const plist = require('simple-plist');
const plistBuild = require('plist').build;
const depSolver = require('./depsolver');
const EventEmitter = require('events').EventEmitter;
const isEncryptedSync = require('macho-is-encrypted');
const machoEntitlements = require('macho-entitlements');

function getResignedFilename (path) {
  if (!path) return null;
  const newPath = path.replace('.ipa', '-resigned.ipa');
  const pos = newPath.lastIndexOf('/');
  if (pos !== -1) return newPath.substring(pos + 1);
  return newPath;
}

function parentDirectory (root) {
  return path.normalize([root, '..'].join('/'));
}

function getExecutable (appdir, exename) {
  if (appdir) {
    const plistPath = [ appdir, 'Info.plist' ].join('/');
    const plistData = plist.readFileSync(plistPath);
    const cfBundleExecutable = plistData['CFBundleExecutable'];
    if (cfBundleExecutable) {
      return cfBundleExecutable;
    }
  }
  return exename;
}

function isMacho (buffer) {
  const magics = [
    [0xca, 0xfe, 0xba, 0xbe], // fat
    [0xce, 0xfa, 0xed, 0xfe], // 32bit
    [0xcf, 0xfa, 0xed, 0xfe], // 64bit
    [0xfe, 0xed, 0xfa, 0xce]  // big-endian
  ];
  for (let a of magics) {
    if (!buffer.compare(Buffer(a))) {
      return true;
    }
  }
  return false;
}

module.exports = class ApplesignSession {
  constructor (state, onEnd) {
    this.config = JSON.parse(JSON.stringify(state));
    this.events = new EventEmitter();
    this.events.config = this.config;
  }

  /* Event Wrapper API with cb support */
  emit (ev, msg, cb) {
    this.events.emit(ev, msg);
    if (typeof cb === 'function') {
      return cb(msg);
    }
  }

  on (ev, cb) {
    this.events.on(ev, cb);
    return this;
  }

  /* Public API */
  signIPA (cb) {
    if (typeof cb === 'function') {
      this.events.removeAllListeners('end');
      this.events.on('end', cb);
    }
    this.unzip(this.config.file, this.config.outdir, (error) => {
      if (error) { return this.emit('end', error); }
      this.signAppDirectory(this.config.outdir + '/Payload', (error, res) => {
        if (error) { return this.emit('end', error); }
        this.zip((error, res) => {
          if (error) { return this.emit('end', error); }
          this.cleanup((_) => {
            this.emit('end');
          });
        });
      });
    });
    return this;
  }

  signAppDirectory (path, next) {
    if (!path) {
      path = this.config.outdir + '/Payload';
    }
    function isDirectory () {
      try {
        return fs.statSync(path).isDirectory();
      } catch (e) {
        return false;
      }
    }
    if (!isDirectory(path)) {
      return this.cleanup(() => {
        next(new Error('Cannot find ' + path));
      });
    }
    this.emit('message', 'Payload found');
    const files = fs.readdirSync(path).filter((x) => {
      return x.indexOf('.app') === x.length - 4;
    });
    if (files.length !== 1) {
      return next(new Error('Invalid IPA'));
    }
    this.config.appdir = [ path, files[0] ].join('/');
    const binname = getExecutable(this.config.appdir, files[0].replace('.app', ''));
    this.config.appbin = [ this.config.appdir, binname ].join('/');
    if (fs.lstatSync(this.config.appbin).isFile()) {
      if (isEncryptedSync.path(this.config.appbin)) {
        if (!this.config.unfairPlay) {
          return next(new Error('This IPA is encrypted'));
        }
        this.emit('message', 'Main IPA executable is encrypted');
      } else {
        this.emit('message', 'Main IPA executable is not encrypted');
      }
      this.removeWatchApp(() => {
        const infoPlist = [ this.config.appdir, 'Info.plist' ].join('/');
        this.fixPlist(infoPlist, this.config.bundleid, (err) => {
          if (err) return this.events.emit('error', err, next);
          this.checkProvision(this.config.appdir, this.config.mobileprovision, (err) => {
            if (err) return this.emit('error', err, next);
            this.fixEntitlements(this.config.appbin, (err) => {
              if (err) return this.emit('error', err, next);
              this.signLibraries(this.config.appbin, this.config.appdir, (err) => {
                if (err) return this.emit('error', err, next);
                next(null, next);
              });
            });
          });
        });
      });
    } else {
      next(new Error('Invalid path'));
    }
  }

  removeWatchApp (cb) {
    if (!this.config.withoutWatchapp) {
      return cb();
    }
    const watchdir = [ this.config.appdir, 'Watch' ].join('/');
    this.emit('message', 'Stripping out the WatchApp at ' + watchdir);
    rimraf(watchdir, () => {
      const plugdir = [ this.config.appdir, 'PlugIns' ].join('/');
      this.emit('message', 'Stripping out the PlugIns at ' + plugdir);
      rimraf(plugdir, cb);
    });
  }
  /*
    TODO: verify is mobileprovision app-id glob string matches the bundleid
    read provision file in raw
    search for application-identifier and <string>...</string>
    check if prefix matches and last dot separated word is an asterisk
    const identifierInProvisioning = 'x'
    Read the one in Info.plist and compare with bundleid
  */
  checkProvision (appdir, file, next) {
    if (file && appdir) {
      this.emit('message', 'Embedding new mobileprovision');
      const mobileProvision = [ appdir, 'embedded.mobileprovision' ].join('/');
      return fs.copy(file, mobileProvision, next);
    }
    next();
  }

  adjustEntitlements (file, entMobProv, next) {
    /* TODO: check if this supports binary plist too */
    const ent = machoEntitlements.parseFile(file);
    const entMacho = plist.parse(ent.toString());
    let changed = false;
    ['application-identifier', 'com.apple.developer.team-identifier'].forEach((id) => {
      if (entMacho[id] !== entMobProv[id]) {
        changed = true;
        entMacho[id] = entMobProv[id];
      }
    });
    if (changed) {
      const newEntitlementsFile = file + '.entitlements';
      fs.writeFileSync(newEntitlementsFile, plistBuild(entMacho).toString());
      this.emit('message', 'Updated binary entitlements' + newEntitlementsFile);
      this.config.entitlement = newEntitlementsFile;
    }
    next();
  }

  fixEntitlements (file, next) {
    if (!this.config.mobileprovision) {
      return next();
    }
    this.emit('message', 'Grabbing entitlements from mobileprovision');
    tools.getEntitlementsFromMobileProvision(this.config.mobileprovision, (error, newEntitlements) => {
      if (error) {
        return next(error);
      }
      this.emit('message', JSON.stringify(newEntitlements));
      const pathToProvision = [ this.config.appdir, 'embedded.mobileprovision' ].join('/');
      /* replace ipa's mobileprovision with given the entitlements? */
      plist.writeFileSync(pathToProvision, newEntitlements);
      this.adjustEntitlements(file, newEntitlements, next);
    });
  }

  fixPlist (file, bundleid, next) {
    const appdir = this.config.appdir;
    if (!file || !appdir) {
      return next('Invalid parameters for fixPlist');
    }
    if (bundleid) {
      const data = plist.readFileSync(file);
      const oldBundleID = data['CFBundleIdentifier'];
      this.emit('message', 'Rebundle ' + file + ' : ' + oldBundleID + ' into ' + bundleid);
      data['CFBundleIdentifier'] = bundleid;
      plist.writeFileSync(file, data);
    }
    next();
  }

  signFile (file, next) {
    function codesignHasFailed (config, error, errmsg) {
      if (error && error.message.indexOf('Error:')) {
        return true;
      }
      return ((errmsg && errmsg.indexOf('no identity found') !== -1) || !config.ignoreCodesignErrors);
    }
    this.emit('message', 'Sign ' + file);
    tools.codesign(this.config.identity, this.config.entitlement, this.config.keychain, file, (error, stdout, stderr) => {
      if (error && codesignHasFailed(this.config, error, stderr)) {
        return this.emit('end', error, next);
      }
      if (this.config.verifyTwice) {
        this.emit('message', 'Verify ' + file);
        tools.verifyCodesign(file, this.config.keychain, (error, stdout, stderr) => {
          if (error) {
            if (this.config.ignoreVerificationErrors) {
              return this.emit('warning', error, next);
            }
            return this.emit('error', error, next);
          }
          next(undefined, error);
        });
      } else {
        next(undefined, error);
      }
    });
  }

  signLibraries (bpath, path, next) {
    this.emit('message', 'Signing libraries and frameworks');

    const libraries = [];
    const exe = '/' + getExecutable(this.config.appdir);

    let found = false;
    walk.walkSync(path, (basedir, filename, stat) => {
      const file = [ basedir, filename ].join('/');
      if (file.endsWith(exe)) {
        this.emit('message', 'Executable found at ' + file);
        libraries.push(file);
        found = true;
        return;
      }
      if (!fs.lstatSync(file).isFile()) {
        return;
      }
      try {
        const fd = fs.openSync(file, 'r');
        let buffer = new Buffer(4);
        fs.readSync(fd, buffer, 0, 4);
        if (isMacho(buffer)) {
          libraries.push(file);
        }
        fs.close(fd);
      } catch (e) {
        console.error(basedir, filename, e);
        next(e);
      }
    });
    if (!found) {
      next('Cannot find any MACH0 binary to sign');
    }
    depSolver(bpath, libraries, (err, libs) => {
      if (err) { return next(err); }
      if (libs.length === 0) {
        libs.push(bpath);
      }
      this.emit('message', 'Using tsort signing order method');
      let libsCopy = libs.slice(0);
      const peek = (cb) => {
        if (libsCopy.length === 0) {
          libsCopy = libs.slice(0);
          return cb();
        }
        const lib = libsCopy.pop();
        this.signFile(lib, () => {
          peek(cb);
        });
      };
      peek(() => {
        libsCopy = libs.slice(0);
        const verify = (cb) => {
          if (libsCopy.length === 0) {
            return cb();
          }
          const lib = libsCopy.pop();
          this.emit('message', 'Verifying ' + lib);
          tools.verifyCodesign(lib, null, () => {
            verify(cb);
          });
        };
        verify(next);
      });
    });
  }

  cleanup (cb) {
    const outdir = this.config.outdir;
    this.emit('message', 'Cleaning up ' + outdir);
    try {
      rimraf(outdir, cb);
    } catch (e) {
      this.emit('message', e);
    }
  }

  zip (next) {
    function getOutputPath (cwd, ofile) {
      if (ofile.startsWith('/')) {
        return ofile;
      }
      return [parentDirectory(cwd), ofile].join('/');
    }
    const ipaIn = this.config.file;
    const ipaOut = getOutputPath(this.config.outdir, this.config.outfile);
    try {
      fs.unlinkSync(ipaOut);
    } catch (e) {
      /* do nothing */
    }
    const continuation = () => {
      this.events.emit('message', 'Zipifying into ' + ipaOut + ' ...');
      tools.zip(this.config.outdir, ipaOut, 'Payload', (error) => {
        if (!error && this.config.replaceipa) {
          this.events.emit('message', 'mv into ' + ipaIn);
          return fs.rename(ipaOut, ipaIn, next);
        }
        next(error);
      });
    };
    if (this.config.withoutWatchapp) {
      const watchdir = [ this.config.appdir, 'Watch' ].join('/');
      this.emit('message', 'Stripping out the WatchApp: ' + watchdir);
      rimraf(watchdir, () => {
        const plugdir = [ this.config.appdir, 'PlugIns' ].join('/');
        rimraf(plugdir, continuation);
      });
    } else {
      continuation();
    }
  }

  setFile (name) {
    if (typeof name === 'string') {
      this.config.file = name;
      this.config.outdir = path.resolve(name + '.d');
      if (!this.config.outfile) {
        this.setOutputFile(getResignedFilename(name));
      }
    }
  }

  setOutputFile (name) {
    this.config.outfile = name;
  }

  /* TODO: move to tools.js */
  unzip (file, outdir, cb) {
    if (!file || !outdir) {
      return cb(new Error('No output specified'));
    }
    if (!outdir) {
      return cb(new Error('Invalid output directory'));
    }
    this.events.emit('message', ['rm -rf', outdir].join(' '));
    this.cleanup(() => {
      this.events.emit('message', 'Unzipping ' + file);
      tools.unzip(file, outdir, (error, stdout) => {
        if (error) {
          this.cleanup(() => { cb(error); });
        } else {
          cb(undefined, stdout);
        }
      });
    });
  }
};
<commit_msg>Use path.join and path.sep instead of hacky string assumptions<commit_after>'use strict';

const path = require('path');
const fs = require('fs-extra');
const walk = require('fs-walk');
const rimraf = require('rimraf');
const tools = require('./tools');
const plist = require('simple-plist');
const depSolver = require('./depsolver');
const plistBuild = require('plist').build;
const EventEmitter = require('events').EventEmitter;
const isEncryptedSync = require('macho-is-encrypted');
const machoEntitlements = require('macho-entitlements');

function getResignedFilename (path) {
  if (!path) return null;
  const newPath = path.replace('.ipa', '-resigned.ipa');
  const pos = newPath.lastIndexOf(path.sep);
  if (pos !== -1) return newPath.substring(pos + 1);
  return newPath;
}

function parentDirectory (root) {
  return path.normalize(path.join(root, '..'));
}

function getExecutable (appdir, exename) {
  if (appdir) {
    const plistPath = path.join(appdir, 'Info.plist');
    const plistData = plist.readFileSync(plistPath);
    const cfBundleExecutable = plistData['CFBundleExecutable'];
    if (cfBundleExecutable) {
      return cfBundleExecutable;
    }
  }
  return exename;
}

function isMacho (buffer) {
  const magics = [
    [0xca, 0xfe, 0xba, 0xbe], // fat
    [0xce, 0xfa, 0xed, 0xfe], // 32bit
    [0xcf, 0xfa, 0xed, 0xfe], // 64bit
    [0xfe, 0xed, 0xfa, 0xce]  // big-endian
  ];
  for (let a of magics) {
    if (!buffer.compare(Buffer(a))) {
      return true;
    }
  }
  return false;
}

module.exports = class ApplesignSession {
  constructor (state, onEnd) {
    this.config = JSON.parse(JSON.stringify(state));
    this.events = new EventEmitter();
    this.events.config = this.config;
  }

  /* Event Wrapper API with cb support */
  emit (ev, msg, cb) {
    this.events.emit(ev, msg);
    if (typeof cb === 'function') {
      return cb(msg);
    }
  }

  on (ev, cb) {
    this.events.on(ev, cb);
    return this;
  }

  /* Public API */
  signIPA (cb) {
    if (typeof cb === 'function') {
      this.events.removeAllListeners('end');
      this.events.on('end', cb);
    }
    this.unzip(this.config.file, this.config.outdir, (error) => {
      if (error) { return this.emit('end', error); }
      this.signAppDirectory(this.config.outdir + '/Payload', (error, res) => {
        if (error) { return this.emit('end', error); }
        this.zip((error, res) => {
          if (error) { return this.emit('end', error); }
          this.cleanup((_) => {
            this.emit('end');
          });
        });
      });
    });
    return this;
  }

  signAppDirectory (ipadir, next) {
    if (!ipadir) {
      ipadir = path.join(this.config.outdir, 'Payload');
    }
    function isDirectory () {
      try {
        return fs.statSync(ipadir).isDirectory();
      } catch (e) {
        return false;
      }
    }
    if (!isDirectory(ipadir)) {
      return this.cleanup(() => {
        next(new Error('Cannot find ' + ipadir));
      });
    }
    this.emit('message', 'Payload found');
    const files = fs.readdirSync(ipadir).filter((x) => {
      return x.indexOf('.app') === x.length - 4;
    });
    if (files.length !== 1) {
      return next(new Error('Invalid IPA'));
    }
    this.config.appdir = path.join(ipadir, files[0]);
    const binname = getExecutable(this.config.appdir, files[0].replace('.app', ''));
    this.config.appbin = path.join(this.config.appdir, binname);
    if (fs.lstatSync(this.config.appbin).isFile()) {
      if (isEncryptedSync.path(this.config.appbin)) {
        if (!this.config.unfairPlay) {
          return next(new Error('This IPA is encrypted'));
        }
        this.emit('message', 'Main IPA executable is encrypted');
      } else {
        this.emit('message', 'Main IPA executable is not encrypted');
      }
      this.removeWatchApp(() => {
        const infoPlist = path.join(this.config.appdir, 'Info.plist');
        this.fixPlist(infoPlist, this.config.bundleid, (err) => {
          if (err) return this.events.emit('error', err, next);
          this.checkProvision(this.config.appdir, this.config.mobileprovision, (err) => {
            if (err) return this.emit('error', err, next);
            this.fixEntitlements(this.config.appbin, (err) => {
              if (err) return this.emit('error', err, next);
              this.signLibraries(this.config.appbin, this.config.appdir, (err) => {
                if (err) return this.emit('error', err, next);
                next(null, next);
              });
            });
          });
        });
      });
    } else {
      next(new Error('Invalid path'));
    }
  }

  removeWatchApp (cb) {
    if (!this.config.withoutWatchapp) {
      return cb();
    }
    const watchdir = path.join(this.config.appdir, 'Watch');
    this.emit('message', 'Stripping out the WatchApp at ' + watchdir);
    rimraf(watchdir, () => {
      const plugdir = path.join(this.config.appdir, 'PlugIns');
      this.emit('message', 'Stripping out the PlugIns at ' + plugdir);
      rimraf(plugdir, cb);
    });
  }
  /*
    TODO: verify is mobileprovision app-id glob string matches the bundleid
    read provision file in raw
    search for application-identifier and <string>...</string>
    check if prefix matches and last dot separated word is an asterisk
    const identifierInProvisioning = 'x'
    Read the one in Info.plist and compare with bundleid
  */
  checkProvision (appdir, file, next) {
    if (file && appdir) {
      this.emit('message', 'Embedding new mobileprovision');
      const mobileProvision = path.join(appdir, 'embedded.mobileprovision');
      return fs.copy(file, mobileProvision, next);
    }
    next();
  }

  adjustEntitlements (file, entMobProv, next) {
    /* TODO: check if this supports binary plist too */
    const ent = machoEntitlements.parseFile(file);
    const entMacho = plist.parse(ent.toString());
    let changed = false;
    ['application-identifier', 'com.apple.developer.team-identifier'].forEach((id) => {
      if (entMacho[id] !== entMobProv[id]) {
        changed = true;
        entMacho[id] = entMobProv[id];
      }
    });
    if (changed) {
      const newEntitlementsFile = file + '.entitlements';
      fs.writeFileSync(newEntitlementsFile, plistBuild(entMacho).toString());
      this.emit('message', 'Updated binary entitlements' + newEntitlementsFile);
      this.config.entitlement = newEntitlementsFile;
    }
    next();
  }

  fixEntitlements (file, next) {
    if (!this.config.mobileprovision) {
      return next();
    }
    this.emit('message', 'Grabbing entitlements from mobileprovision');
    tools.getEntitlementsFromMobileProvision(this.config.mobileprovision, (error, newEntitlements) => {
      if (error) {
        return next(error);
      }
      this.emit('message', JSON.stringify(newEntitlements));
      const pathToProvision = path.join(this.config.appdir, 'embedded.mobileprovision');
      /* replace ipa's mobileprovision with given the entitlements? */
      plist.writeFileSync(pathToProvision, newEntitlements);
      this.adjustEntitlements(file, newEntitlements, next);
    });
  }

  fixPlist (file, bundleid, next) {
    const appdir = this.config.appdir;
    if (!file || !appdir) {
      return next('Invalid parameters for fixPlist');
    }
    if (bundleid) {
      const data = plist.readFileSync(file);
      const oldBundleID = data['CFBundleIdentifier'];
      this.emit('message', 'Rebundle ' + file + ' : ' + oldBundleID + ' into ' + bundleid);
      data['CFBundleIdentifier'] = bundleid;
      plist.writeFileSync(file, data);
    }
    next();
  }

  signFile (file, next) {
    function codesignHasFailed (config, error, errmsg) {
      if (error && error.message.indexOf('Error:')) {
        return true;
      }
      return ((errmsg && errmsg.indexOf('no identity found') !== -1) || !config.ignoreCodesignErrors);
    }
    this.emit('message', 'Sign ' + file);
    tools.codesign(this.config.identity, this.config.entitlement, this.config.keychain, file, (error, stdout, stderr) => {
      if (error && codesignHasFailed(this.config, error, stderr)) {
        return this.emit('end', error, next);
      }
      if (this.config.verifyTwice) {
        this.emit('message', 'Verify ' + file);
        tools.verifyCodesign(file, this.config.keychain, (error, stdout, stderr) => {
          if (error) {
            if (this.config.ignoreVerificationErrors) {
              return this.emit('warning', error, next);
            }
            return this.emit('error', error, next);
          }
          next(undefined, error);
        });
      } else {
        next(undefined, error);
      }
    });
  }

  signLibraries (bpath, appdir, next) {
    this.emit('message', 'Signing libraries and frameworks');

    const libraries = [];
    const exe = path.sep + getExecutable(this.config.appdir);

    let found = false;
    walk.walkSync(appdir, (basedir, filename, stat) => {
      const file = path.join(basedir, filename);
      if (file.endsWith(exe)) {
        this.emit('message', 'Executable found at ' + file);
        libraries.push(file);
        found = true;
        return;
      }
      if (!fs.lstatSync(file).isFile()) {
        return;
      }
      try {
        const fd = fs.openSync(file, 'r');
        let buffer = new Buffer(4);
        fs.readSync(fd, buffer, 0, 4);
        if (isMacho(buffer)) {
          libraries.push(file);
        }
        fs.close(fd);
      } catch (e) {
        console.error(basedir, filename, e);
        next(e);
      }
    });
    if (!found) {
      next('Cannot find any MACH0 binary to sign');
    }
    depSolver(bpath, libraries, (err, libs) => {
      if (err) { return next(err); }
      if (libs.length === 0) {
        libs.push(bpath);
      }
      this.emit('message', 'Using tsort signing order method');
      let libsCopy = libs.slice(0);
      const peek = (cb) => {
        if (libsCopy.length === 0) {
          libsCopy = libs.slice(0);
          return cb();
        }
        const lib = libsCopy.pop();
        this.signFile(lib, () => {
          peek(cb);
        });
      };
      peek(() => {
        libsCopy = libs.slice(0);
        const verify = (cb) => {
          if (libsCopy.length === 0) {
            return cb();
          }
          const lib = libsCopy.pop();
          this.emit('message', 'Verifying ' + lib);
          tools.verifyCodesign(lib, null, () => {
            verify(cb);
          });
        };
        verify(next);
      });
    });
  }

  cleanup (cb) {
    const outdir = this.config.outdir;
    this.emit('message', 'Cleaning up ' + outdir);
    try {
      rimraf(outdir, cb);
    } catch (e) {
      this.emit('message', e);
    }
  }

  zip (next) {
    function getOutputPath (cwd, ofile) {
      if (ofile.startsWith(path.sep)) {
        return ofile;
      }
      return path.join(parentDirectory(cwd), ofile);
    }
    const ipaIn = this.config.file;
    const ipaOut = getOutputPath(this.config.outdir, this.config.outfile);
    try {
      fs.unlinkSync(ipaOut);
    } catch (e) {
      /* do nothing */
    }
    const continuation = () => {
      this.events.emit('message', 'Zipifying into ' + ipaOut + ' ...');
      tools.zip(this.config.outdir, ipaOut, 'Payload', (error) => {
        if (!error && this.config.replaceipa) {
          this.events.emit('message', 'mv into ' + ipaIn);
          return fs.rename(ipaOut, ipaIn, next);
        }
        next(error);
      });
    };
    if (this.config.withoutWatchapp) {
      const watchdir = path.join(this.config.appdir, 'Watch');
      this.emit('message', 'Stripping out the WatchApp: ' + watchdir);
      rimraf(watchdir, () => {
        const plugdir = path.join(this.config.appdir, 'PlugIns');
        rimraf(plugdir, continuation);
      });
    } else {
      continuation();
    }
  }

  setFile (name) {
    if (typeof name === 'string') {
      this.config.file = name;
      this.config.outdir = path.resolve(name + '.d');
      if (!this.config.outfile) {
        this.setOutputFile(getResignedFilename(name));
      }
    }
  }

  setOutputFile (name) {
    this.config.outfile = name;
  }

  /* TODO: move to tools.js */
  unzip (file, outdir, cb) {
    if (!file || !outdir) {
      return cb(new Error('No output specified'));
    }
    if (!outdir) {
      return cb(new Error('Invalid output directory'));
    }
    this.events.emit('message', ['rm -rf', outdir].join(' '));
    this.cleanup(() => {
      this.events.emit('message', 'Unzipping ' + file);
      tools.unzip(file, outdir, (error, stdout) => {
        if (error) {
          this.cleanup(() => { cb(error); });
        } else {
          cb(undefined, stdout);
        }
      });
    });
  }
};
<|endoftext|>


<commit_before>/*
  Module Dependencies
 */
var when   =   require( 'when' );
var nodefn =   require( 'when/node' );
var request =  require( 'request' );
var mongojs =  require( 'mongojs' );
var twit =     require( 'twit' );
var objectId = mongojs.ObjectId;

/*

  Make DB connection

 */
var db = mongojs( process.env.MONGO_URL, [ 'socialDocs', 'skyttleDocs', 'geoDocs' ] );

var init = function () {
  var T = new twit({
      consumer_key:         process.env.TWITTER_CONSUMER_KEY
    , consumer_secret:      process.env.TWITTER_CONSUMER_SECRET
    , access_token:         process.env.TWITTER_ACCESS_TOKEN
    , access_token_secret:  process.env.TWITTER_ACCESS_TOKEN_SECRET
  });

  T.stream( 'statuses/filter', { locations: seattle } ).on( 'tweet', handleTweet );

  console.log( 'Twitter interface started' );
}

/*

  Setup tweet stream

 */

var seattle = [
  '-122.46',
  '47.5',
  '-122.22',
  '47.7'
];

var tweetCount = 0;

var handleTweet = function ( tweet ) {

  tweet._id = new objectId();

  insertIntoSocialDocs( tweet )
  .then( function () {
    return insertIntoSkyttleDocs( tweet );
  } )
  .then( function ( sentimentScores ) {
    return insertIntoGeoDocs( tweet, sentimentScores );
  } )
  .catch( handleError )
  .done( function ( tweet ) {
    console.log("Tweet Count: " + tweetCount++);
  } );
};

var handleError = function ( err ) {
  console.error( err );
};

/*

  Save this tweet

 */
var insertIntoSocialDocs = function ( tweet ) {
  return when.promise( function ( resolve, reject ) {
    db.socialDocs.insert( {
      _id: tweet._id,
      type: 'tweet',
      originId: tweet.id,
      text: tweet.text,
      createdAt: new Date( tweet.created_at ),
      user: tweet.user.screen_name
    }, function ( err ) {
      if ( err ) return reject( err );
      resolve();
    } );
  } );
};

/*

 If there is lat/lng and sentiment, save that separately for easy aggregations

 */
var insertIntoGeoDocs = function ( tweet, sentimentScores ) {

  if ( !tweet.coordinates || !sentimentScores ) {
    return when.resolve();
  }

  return when.promise( function ( resolve, reject ) {
    db.geoDocs.insert({
      socialId: tweet._id,
      type: 'tweet',
      analyzableText: getAnalyzableText( tweet.text ),
      createdAt: new Date( tweet.created_at ),
      location: tweet.coordinates,
      neg: sentimentScores.neg,
      neu: sentimentScores.neu,
      pos: sentimentScores.pos
    }, function ( err ) {
      if ( err ) return reject( err );
      resolve();
    } );
  } );
}

/*

  Basic scrubbing of tweet to remove non-sentiment

 */
var getAnalyzableText = function ( text ) {

  // find all links, then remove
  var urlMatcher = /(https?:\/\/?[\da-z\.-]+\.[a-z\.]{2,6})([\/\w\.-]*)*\/?/g;
  var urls = [];
  while  ( match = urlMatcher.exec( text ) ) {
    urls.push(match[0]);
  }
  for (var i = urls.length - 1; i >= 0; i--) {
    text = text.replace( urls[i], "" );
    text = text.trim();
  };

  //remove @ Place - should only match once, but for now let's just iterate
  var atMatcher = /(\(?@\s([\w\s,\.]+)\)?|@\.\.\.)$/;
  while  ( match = atMatcher.exec( text ) ) {
    text = text.replace( match[0], "" );
    text = text.trim();
  }

  // remove hashes from end of string sequentially
  var hashMatcher = /(#\w+)\s*?$/;
  while  ( match = hashMatcher.exec( text ) ) {
    text = text.replace( match[0], "" );
    text = text.trim();
  }

  return text;
}

/*

  Request params for making the call to skyttle

 */
var getRequestParams = function ( text ) {
  return {
    method: 'POST',
    url: 'https://sentinelprojects-skyttle20.p.mashape.com/',
    headers: {
      'X-Mashape-Key': process.env.MASHAPE_KEY,
      'Content-Type': 'application/x-www-form-urlencoded',
      'Accept': 'application/json'
    },
    form: {
      'annotate': 1,
      'keywords': 1,
      'lang': 'en',
      'sentiment' :1,
      'text': text
    }
  }
};

/*

  Get sentiment scores and save skyttle document

 */
var insertIntoSkyttleDocs = function ( tweet ) {

  if ( !tweet.coordinates ) {
    return when.resolve( null );
  }

  tweet.analyzableText = getAnalyzableText( tweet.text );

  return when.promise( function ( resolve, reject ) {

    request( getRequestParams( tweet.analyzableText ), function ( err, response, body ) {
      if ( err ) return reject( err );

      if ( response.statusCode !== 200 ) {
        return reject( new Error( body ) );
      }

      if ( typeof body != "object" ) {
        try {
          body = JSON.parse( body );
        } catch ( e ) {
          return reject( e );
        }
      }

      if ( !body.docs || !body.docs.length ) {
        return reject( new Error( 'Skyttle did not return a document' ) )
      }

      var doc = body.docs[0];

      delete doc.doc_id;

      doc.socialId = tweet._id;

      db.skyttleDocs.insert( doc, function ( err ) {
        if ( err ) return reject( err );
        resolve( doc.sentiment_scores );
      } );
    } );
  } );
};

/*

  Exports

 */

module.exports.init = init;
module.exports.insertIntoSkyttleDocs = insertIntoSkyttleDocs;
module.exports.insertIntoGeoDocs = insertIntoGeoDocs;
module.exports.insertIntoSocialDocs = insertIntoSocialDocs
module.exports.getAnalyzableText = getAnalyzableText;<commit_msg>Disabling twitter<commit_after>/*
  Module Dependencies
 */
var when   =   require( 'when' );
var nodefn =   require( 'when/node' );
var request =  require( 'request' );
var mongojs =  require( 'mongojs' );
var twit =     require( 'twit' );
var objectId = mongojs.ObjectId;

/*

  Make DB connection

 */
var db = mongojs( process.env.MONGO_URL, [ 'socialDocs', 'skyttleDocs', 'geoDocs' ] );

var init = function () {
  var T = new twit({
      consumer_key:         process.env.TWITTER_CONSUMER_KEY
    , consumer_secret:      process.env.TWITTER_CONSUMER_SECRET
    , access_token:         process.env.TWITTER_ACCESS_TOKEN
    , access_token_secret:  process.env.TWITTER_ACCESS_TOKEN_SECRET
  });

  T.stream( 'statuses/filter', { locations: seattle } ).on( 'tweet', handleTweet );

  console.log( 'Twitter interface started' );
}

/*

  Setup tweet stream

 */

var seattle = [
  '-122.46',
  '47.5',
  '-122.22',
  '47.7'
];

var tweetCount = 0;

var handleTweet = function ( tweet ) {

  tweet._id = new objectId();

  insertIntoSocialDocs( tweet )
  .then( function () {
    return insertIntoSkyttleDocs( tweet );
  } )
  .then( function ( sentimentScores ) {
    return insertIntoGeoDocs( tweet, sentimentScores );
  } )
  .catch( handleError )
  .done( function ( tweet ) {
    console.log("Tweet Count: " + tweetCount++);
  } );
};

var handleError = function ( err ) {
  console.error( err );
};

/*

  Save this tweet

 */
var insertIntoSocialDocs = function ( tweet ) {
  return when.promise( function ( resolve, reject ) {
    db.socialDocs.insert( {
      _id: tweet._id,
      type: 'tweet',
      originId: tweet.id,
      text: tweet.text,
      createdAt: new Date( tweet.created_at ),
      user: tweet.user.screen_name
    }, function ( err ) {
      if ( err ) return reject( err );
      resolve();
    } );
  } );
};

/*

 If there is lat/lng and sentiment, save that separately for easy aggregations

 */
var insertIntoGeoDocs = function ( tweet, sentimentScores ) {

  if ( !tweet.coordinates || !sentimentScores ) {
    return when.resolve();
  }

  return when.promise( function ( resolve, reject ) {
    db.geoDocs.insert({
      socialId: tweet._id,
      type: 'tweet',
      analyzableText: getAnalyzableText( tweet.text ),
      createdAt: new Date( tweet.created_at ),
      location: tweet.coordinates,
      neg: sentimentScores.neg,
      neu: sentimentScores.neu,
      pos: sentimentScores.pos
    }, function ( err ) {
      if ( err ) return reject( err );
      resolve();
    } );
  } );
}

/*

  Basic scrubbing of tweet to remove non-sentiment

 */
var getAnalyzableText = function ( text ) {

  // find all links, then remove
  var urlMatcher = /(https?:\/\/?[\da-z\.-]+\.[a-z\.]{2,6})([\/\w\.-]*)*\/?/g;
  var urls = [];
  while  ( match = urlMatcher.exec( text ) ) {
    urls.push(match[0]);
  }
  for (var i = urls.length - 1; i >= 0; i--) {
    text = text.replace( urls[i], "" );
    text = text.trim();
  };

  //remove @ Place - should only match once, but for now let's just iterate
  var atMatcher = /(\(?@\s([\w\s,\.]+)\)?|@\.\.\.)$/;
  while  ( match = atMatcher.exec( text ) ) {
    text = text.replace( match[0], "" );
    text = text.trim();
  }

  // remove hashes from end of string sequentially
  var hashMatcher = /(#\w+)\s*?$/;
  while  ( match = hashMatcher.exec( text ) ) {
    text = text.replace( match[0], "" );
    text = text.trim();
  }

  return text;
}

/*

  Request params for making the call to skyttle

 */
var getRequestParams = function ( text ) {
  return {
    method: 'POST',
    url: 'https://sentinelprojects-skyttle20.p.mashape.com/',
    headers: {
      'X-Mashape-Key': process.env.MASHAPE_KEY,
      'Content-Type': 'application/x-www-form-urlencoded',
      'Accept': 'application/json'
    },
    form: {
      'annotate': 1,
      'keywords': 1,
      'lang': 'en',
      'sentiment' :1,
      'text': text
    }
  }
};

/*

  Get sentiment scores and save skyttle document

 */
var insertIntoSkyttleDocs = function ( tweet ) {

  // Disabling Skyttle
  return when.resolve( null );

  if ( !tweet.coordinates ) {
    return when.resolve( null );
  }

  tweet.analyzableText = getAnalyzableText( tweet.text );

  return when.promise( function ( resolve, reject ) {

    request( getRequestParams( tweet.analyzableText ), function ( err, response, body ) {
      if ( err ) return reject( err );

      if ( response.statusCode !== 200 ) {
        return reject( new Error( body ) );
      }

      if ( typeof body != "object" ) {
        try {
          body = JSON.parse( body );
        } catch ( e ) {
          return reject( e );
        }
      }

      if ( !body.docs || !body.docs.length ) {
        return reject( new Error( 'Skyttle did not return a document' ) )
      }

      var doc = body.docs[0];

      delete doc.doc_id;

      doc.socialId = tweet._id;

      db.skyttleDocs.insert( doc, function ( err ) {
        if ( err ) return reject( err );
        resolve( doc.sentiment_scores );
      } );
    } );
  } );
};

/*

  Exports

 */

module.exports.init = init;
module.exports.insertIntoSkyttleDocs = insertIntoSkyttleDocs;
module.exports.insertIntoGeoDocs = insertIntoGeoDocs;
module.exports.insertIntoSocialDocs = insertIntoSocialDocs
module.exports.getAnalyzableText = getAnalyzableText;<|endoftext|>


<commit_before>/**
* shaaaaa.js
*
* Checks a domain for its certificate algorithm.
*
* Depends on openssl installed and accessible on the PATH.
*/

// used to call out to openssl
var exec = require("child_process").exec;
var fs = require('fs'); // loads root certs

// yorkie's fork, includes signatureAlgorithm
var x509 = require("x.509");

var Shaaa = {

  // root cert bundle, loaded when this file is required
  roots: null,

  // load root bundle, parse each cert
  loadRoots: function() {
    Shaaa.roots = [];

    // store a fingerprint of each one
    var certs = fs.readFileSync("./ca-bundle.crt", "utf-8").split("\n\n");
    for (var i=0; i<certs.length; i++)
      Shaaa.roots.push(x509.parseCert(certs[i]).fingerPrint);
  },

  // takes x509-parsed cert, compares fingerprint
  isRoot: function(cert) {
    return (Shaaa.roots.indexOf(cert.fingerPrint) > -1);
  },

  // fingerrints of SHA-1 intermediate certs with known SHA-2 certs
  fingerprints: null,
  loadFingerprintsJSON: function() {
    Shaaa.fingerprints = [];
    var fingerprintsFile = __dirname + '/fingerprints.json';
    var fingerprintsJSON = fs.readFileSync(fingerprintsFile, 'utf-8');
    Shaaa.fingerprints = JSON.parse(fingerprintsJSON);
  },

  algorithms: [
    // new gold standards
    "sha256", "sha224", "sha384", "sha512",

    "sha1", // common, but deprecated
    "md5", // old, broken
    "md2" // so old, so broken
  ],

  // given e.g. 'sha256WithRSAEncryption', return
  // {algorithm: 'sha256', raw: 'sha256WithRSAEncryption', good: true}
  algorithm: function(raw) {
    var raw_compare = raw.toLowerCase();

    var answer;
    for (var i=0; i<Shaaa.algorithms.length; i++) {
      var algorithm = Shaaa.algorithms[i];
      if (raw_compare.indexOf(algorithm) == 0) answer = algorithm;
      if (raw_compare == ("ecdsa-with-" + algorithm)) answer = algorithm;
    }
    if (!answer) answer = "unknown";

    var good = (
      (answer == "sha256") ||
      (answer == "sha224") ||
      (answer == "sha384") ||
      (answer == "sha512")
    );

    return {algorithm: answer, raw: raw, good: good};
  },

  certs: function(domain, callback, options) {
    if (!options) options = {};

    // This accounts for -servername/SNI, which cannot have a port
    var server_name = domain.replace(/[^\w\.\-]|[$:\d+]/g, '');

    // This accounts for -connect, which can have a port
    var server_connect = domain.replace(/[^\w\.\-:]/g, '');

    // If the address does not have a port defined, add default :443
    if (server_connect.match(/:\d+^/g) === null) {
      server_connect += ":443";
    }

    // adapted from http://askubuntu.com/a/201923/3096
    var command = "" +
      // piping into openssl tells it not to hold an open connection
      "echo -n" +
      // connect to given domain on port 443
      " | openssl s_client -connect " + server_connect +
      // specify hostname in case server uses SNI
      " -servername " + server_name +
      // ask for the full cert chain
      " -showcerts";

    if (options.verbose || options.debug) console.log(command + "\n");

    exec(command, function(error, stdout, stderr) {
      if (error) return callback(error);

      // stdout is a long block of openssl output - grab the certs
      var certs = [];
      // using multiline workaround: http://stackoverflow.com/a/1068308/16075
      var regex = /(\-+BEGIN CERTIFICATE\-+[\s\S]*?\-+END CERTIFICATE\-+)/g

      var match = regex.exec(stdout);
      while (match != null) {
        certs.push(match[1]);
        match = regex.exec(stdout);
      }

      callback(null, certs);
    });
  },

  sha2URL: function(fingerprint) {
    var url = null;
    var fingerprintsLength = Shaaa.fingerprints.certificates.length;
    for (var i=0; i<fingerprintsLength; i++) {
      if(Shaaa.fingerprints.certificates[i].fingerprints['sha-1'] == fingerprint) {
        url = Shaaa.fingerprints.certificates[i]['SHA-2 URL'];
        break;
      }
    }
    return url;
  },

  cert: function(text) {
    var cert = x509.parseCert(text);
    var answer = Shaaa.algorithm(cert.signatureAlgorithm);
    var root = Shaaa.isRoot(cert);
    var sha2url = Shaaa.sha2URL(cert.fingerPrint);

    return {
      algorithm: answer.algorithm,
      raw: answer.raw,
      good: (root || answer.good),
      root: root,
      sha2url: sha2url,

      expires: cert.notAfter,
      name: cert.subject.commonName
    };
  },

  /**
  * Desired output:
  * {
  *   domain: shaaaaaaaaaaaaa.com,
  *   cert: {
  *     algorithm: 'sha256',
  *     raw: 'sha256WithRSAEncryption',
  *     good: true,
  *     root: false,
  *     sha2url: null,
  *     expires: Tue Aug 18 2015 19:59:59 GMT-0400 (EDT),
  *     name: "www.konklone.com"
  *   },
  *   intermediates: [
  *     { ... } // same form as above
  *   ]
  * }
  */

  from: function(domain, callback, options) {
    if (!options) options = {};

    var data = {domain: domain};

    Shaaa.certs(domain, function(error, certs) {
      if (error) return callback(error);

      data.cert = Shaaa.cert(certs[0]);

      data.intermediates = [];
      certs.slice(1).forEach(function(cert) {
        data.intermediates.push(Shaaa.cert(cert));
      });

      var intergood = true;
      for (var i=0; i<data.intermediates.length; i++) {
        if (!data.intermediates[i].good) {
          if (i == 0) {
            intergood = false;
            break;
          }
        }
      }

      if (data.cert.good && intergood)
        data.diagnosis = "good";
      else if (data.cert.good && !intergood)
        data.diagnosis = "almost";
      else
        data.diagnosis = "bad";

      if (callback)
        callback(null, data);
      else
        console.log(data);
    }, options);
  }
}

// load roots on first require
Shaaa.loadRoots();
// load fingerprints
Shaaa.loadFingerprintsJSON();

module.exports = Shaaa;
<commit_msg>Add a simple condition to the running of the sha2URL function<commit_after>/**
* shaaaaa.js
*
* Checks a domain for its certificate algorithm.
*
* Depends on openssl installed and accessible on the PATH.
*/

// used to call out to openssl
var exec = require("child_process").exec;
var fs = require('fs'); // loads root certs

// yorkie's fork, includes signatureAlgorithm
var x509 = require("x.509");

var Shaaa = {

  // root cert bundle, loaded when this file is required
  roots: null,

  // load root bundle, parse each cert
  loadRoots: function() {
    Shaaa.roots = [];

    // store a fingerprint of each one
    var certs = fs.readFileSync("./ca-bundle.crt", "utf-8").split("\n\n");
    for (var i=0; i<certs.length; i++)
      Shaaa.roots.push(x509.parseCert(certs[i]).fingerPrint);
  },

  // takes x509-parsed cert, compares fingerprint
  isRoot: function(cert) {
    return (Shaaa.roots.indexOf(cert.fingerPrint) > -1);
  },

  // fingerrints of SHA-1 intermediate certs with known SHA-2 certs
  fingerprints: null,
  loadFingerprintsJSON: function() {
    Shaaa.fingerprints = [];
    var fingerprintsFile = __dirname + '/fingerprints.json';
    var fingerprintsJSON = fs.readFileSync(fingerprintsFile, 'utf-8');
    Shaaa.fingerprints = JSON.parse(fingerprintsJSON);
  },

  algorithms: [
    // new gold standards
    "sha256", "sha224", "sha384", "sha512",

    "sha1", // common, but deprecated
    "md5", // old, broken
    "md2" // so old, so broken
  ],

  // given e.g. 'sha256WithRSAEncryption', return
  // {algorithm: 'sha256', raw: 'sha256WithRSAEncryption', good: true}
  algorithm: function(raw) {
    var raw_compare = raw.toLowerCase();

    var answer;
    for (var i=0; i<Shaaa.algorithms.length; i++) {
      var algorithm = Shaaa.algorithms[i];
      if (raw_compare.indexOf(algorithm) == 0) answer = algorithm;
      if (raw_compare == ("ecdsa-with-" + algorithm)) answer = algorithm;
    }
    if (!answer) answer = "unknown";

    var good = (
      (answer == "sha256") ||
      (answer == "sha224") ||
      (answer == "sha384") ||
      (answer == "sha512")
    );

    return {algorithm: answer, raw: raw, good: good};
  },

  certs: function(domain, callback, options) {
    if (!options) options = {};

    // This accounts for -servername/SNI, which cannot have a port
    var server_name = domain.replace(/[^\w\.\-]|[$:\d+]/g, '');

    // This accounts for -connect, which can have a port
    var server_connect = domain.replace(/[^\w\.\-:]/g, '');

    // If the address does not have a port defined, add default :443
    if (server_connect.match(/:\d+^/g) === null) {
      server_connect += ":443";
    }

    // adapted from http://askubuntu.com/a/201923/3096
    var command = "" +
      // piping into openssl tells it not to hold an open connection
      "echo -n" +
      // connect to given domain on port 443
      " | openssl s_client -connect " + server_connect +
      // specify hostname in case server uses SNI
      " -servername " + server_name +
      // ask for the full cert chain
      " -showcerts";

    if (options.verbose || options.debug) console.log(command + "\n");

    exec(command, function(error, stdout, stderr) {
      if (error) return callback(error);

      // stdout is a long block of openssl output - grab the certs
      var certs = [];
      // using multiline workaround: http://stackoverflow.com/a/1068308/16075
      var regex = /(\-+BEGIN CERTIFICATE\-+[\s\S]*?\-+END CERTIFICATE\-+)/g

      var match = regex.exec(stdout);
      while (match != null) {
        certs.push(match[1]);
        match = regex.exec(stdout);
      }

      callback(null, certs);
    });
  },

  sha2URL: function(fingerprint) {
    var url = null;
    var fingerprintsLength = Shaaa.fingerprints.certificates.length;
    for (var i=0; i<fingerprintsLength; i++) {
      if(Shaaa.fingerprints.certificates[i].fingerprints['sha-1'] == fingerprint) {
        url = Shaaa.fingerprints.certificates[i]['SHA-2 URL'];
        break;
      }
    }
    return url;
  },

  cert: function(text) {
    var cert = x509.parseCert(text);
    var answer = Shaaa.algorithm(cert.signatureAlgorithm);
    var root = Shaaa.isRoot(cert);
    if(root == false) { // we only want to check this on non-root intermediate certificates
      var sha2url = Shaaa.sha2URL(cert.fingerPrint);
    } else {
      var sha2url = null;
    }

    return {
      algorithm: answer.algorithm,
      raw: answer.raw,
      good: (root || answer.good),
      root: root,
      sha2url: sha2url,

      expires: cert.notAfter,
      name: cert.subject.commonName
    };
  },

  /**
  * Desired output:
  * {
  *   domain: shaaaaaaaaaaaaa.com,
  *   cert: {
  *     algorithm: 'sha256',
  *     raw: 'sha256WithRSAEncryption',
  *     good: true,
  *     root: false,
  *     sha2url: null,
  *     expires: Tue Aug 18 2015 19:59:59 GMT-0400 (EDT),
  *     name: "www.konklone.com"
  *   },
  *   intermediates: [
  *     { ... } // same form as above
  *   ]
  * }
  */

  from: function(domain, callback, options) {
    if (!options) options = {};

    var data = {domain: domain};

    Shaaa.certs(domain, function(error, certs) {
      if (error) return callback(error);

      data.cert = Shaaa.cert(certs[0]);

      data.intermediates = [];
      certs.slice(1).forEach(function(cert) {
        data.intermediates.push(Shaaa.cert(cert));
      });

      var intergood = true;
      for (var i=0; i<data.intermediates.length; i++) {
        if (!data.intermediates[i].good) {
          if (i == 0) {
            intergood = false;
            break;
          }
        }
      }

      if (data.cert.good && intergood)
        data.diagnosis = "good";
      else if (data.cert.good && !intergood)
        data.diagnosis = "almost";
      else
        data.diagnosis = "bad";

      if (callback)
        callback(null, data);
      else
        console.log(data);
    }, options);
  }
}

// load roots on first require
Shaaa.loadRoots();
// load fingerprints
Shaaa.loadFingerprintsJSON();

module.exports = Shaaa;
<|endoftext|>