<commit_before>'use strict';
var npm = require('npm');
var matchdep = require('matchdep');
var NpmScrapper = require('npm-scrap');
var async = require('async');

/**
 * A thin wrapper of npm
 * @param {Object|undefined} installable - optional
 * @constructor
 */
function InstallableNpm (installable) {
    this.installable = installable;
}

/**
 * List installed (installable) modules
 * @return {Array} An array of installed modules (plugins)
 * @public
 */
InstallableNpm.prototype.installedModules = function () {
    var pkg = require(this.installable.pkgPath);
    var plugins = matchdep
        .filter(pkg.name + '-plugin-*', pkg)
        .map(function (pluginName) {
            return {
                name: pluginName,
                version: pkg.dependencies[pluginName]
            };
        });

    return plugins;
};

/**
 * Searches npmjs.org for installable packages
 * @param  {String} term - search term
 * @param  {Function} cb - callback
 * @public
 */
InstallableNpm.prototype.search = function (term, cb) {
    var terms = term;

    if (this.installable) {
        terms = terms + ' ' + this.installable.appName + '-plugin';
    }

    new NpmScrapper(terms).search(function (err, data) {
        if (err) {
            cb(err);
        }

        if (this.installable) {
            // filter results with pluginsRegExp cause
            // npmjs.org search ALWAYS returns results
            data = data.filter(function (moduleData) {
                return moduleData.name.match(this.installable.pluginsRegExp);
            }.bind(this));
        }

        cb(null, data);
    }.bind(this));
};

/**
 * Fetch module info from npm
 * @param  {String}   moduleName
 * @param  {Function} cb - callback
 * @public
 */
InstallableNpm.prototype.view = function (moduleName, cb) {
    async.waterfall([
        function (done) {
            npm.load(null, done);
        },
        function (data, done) {
            npm.view(moduleName, done);
        },
        function (data, a, b, done) {
            done(null, data);
        }
    ], function (err, result) {
        cb(err, result);
    });
};

/**
 * Checks availability of a module name in npm
 * @param  {String}   moduleName
 * @param  {Function} cb - callback
 * @public
 */
InstallableNpm.prototype.available = function (moduleName, cb) {
    async.waterfall([
        function (done) {
            npm.load(null, done);
        },
        function (data, done) {
            npm.view(moduleName, done);
        },
        function (data, done) {
            done(new Error('npm module ' + moduleName + ' already exists'));
        }
    ], function (err) {
        if (err.code === 'E404') {
            cb();
        } else {
            cb(err.message);
        }
    });
};

/**
 * npm install
 * @param  {String}   moduleName
 * @param  {Function} cb - callback
 * @public
 */
InstallableNpm.prototype.install = function (moduleName, cb) {
    this.command('install', moduleName, cb);
};

/**
 * npm uninstall
 * @param  {String}   moduleName
 * @param  {Function} cb - callback
 * @public
 */
InstallableNpm.prototype.uninstall = function (moduleName, cb) {
    this.command('uninstall', moduleName, cb);
};

/**
 * Validates a plugin name
 * @param  {String} moduleName
 * @return {String|undefined} String if validation error exists, otherwise undefined
 */
InstallableNpm.prototype.validatePluginName = function (moduleName) {
    if (!moduleName || moduleName.length === 0) {
        return 'No plugin name given.';
    }

    if (this.installable && !moduleName.match(this.installable.pluginsRegExp)) {
        return 'Requested module is not a plugin of this application.';
    }

    return undefined;
};

/**
 * Execute npm install or uninstall
 * @param  {String}   command - install or uninstall
 * @param  {String}   moduleName - the name of the module
 * @param  {Function} cb - callback
 * @private
 */
InstallableNpm.prototype.command = function (command, moduleName, cb) {
    var error = this.validatePluginName(moduleName);

    if (error) {
        var err = new Error(error);
        err.status = 400;
        return cb(err);
    }

    async.waterfall([
        function (done) {
            npm.load(null, done);
        },
        function (data, done) {
            npm.config.set('save', true);
            npm.commands[command]([moduleName], done);
        }
    ], function (err, result) {
        if (err && err.statusCode) {
            err.status = err.statusCode;
        }
        cb(err, result);
    });
};

module.exports = InstallableNpm;
<commit_msg>adds return statement <commit_after>'use strict';
var npm = require('npm');
var matchdep = require('matchdep');
var NpmScrapper = require('npm-scrap');
var async = require('async');

/**
 * A thin wrapper of npm
 * @param {Object|undefined} installable - optional
 * @constructor
 */
function InstallableNpm (installable) {
    this.installable = installable;
}

/**
 * List installed (installable) modules
 * @return {Array} An array of installed modules (plugins)
 * @public
 */
InstallableNpm.prototype.installedModules = function () {
    var pkg = require(this.installable.pkgPath);
    var plugins = matchdep
        .filter(pkg.name + '-plugin-*', pkg)
        .map(function (pluginName) {
            return {
                name: pluginName,
                version: pkg.dependencies[pluginName]
            };
        });

    return plugins;
};

/**
 * Searches npmjs.org for installable packages
 * @param  {String} term - search term
 * @param  {Function} cb - callback
 * @public
 */
InstallableNpm.prototype.search = function (term, cb) {
    var terms = term;

    if (this.installable) {
        terms = terms + ' ' + this.installable.appName + '-plugin';
    }

    new NpmScrapper(terms).search(function (err, data) {
        if (err) {
            return cb(err);
        }

        if (this.installable) {
            // filter results with pluginsRegExp cause
            // npmjs.org search ALWAYS returns results
            data = data.filter(function (moduleData) {
                return moduleData.name.match(this.installable.pluginsRegExp);
            }.bind(this));
        }

        cb(null, data);
    }.bind(this));
};

/**
 * Fetch module info from npm
 * @param  {String}   moduleName
 * @param  {Function} cb - callback
 * @public
 */
InstallableNpm.prototype.view = function (moduleName, cb) {
    async.waterfall([
        function (done) {
            npm.load(null, done);
        },
        function (data, done) {
            npm.view(moduleName, done);
        },
        function (data, a, b, done) {
            done(null, data);
        }
    ], function (err, result) {
        cb(err, result);
    });
};

/**
 * Checks availability of a module name in npm
 * @param  {String}   moduleName
 * @param  {Function} cb - callback
 * @public
 */
InstallableNpm.prototype.available = function (moduleName, cb) {
    async.waterfall([
        function (done) {
            npm.load(null, done);
        },
        function (data, done) {
            npm.view(moduleName, done);
        },
        function (data, done) {
            done(new Error('npm module ' + moduleName + ' already exists'));
        }
    ], function (err) {
        if (err.code === 'E404') {
            cb();
        } else {
            cb(err.message);
        }
    });
};

/**
 * npm install
 * @param  {String}   moduleName
 * @param  {Function} cb - callback
 * @public
 */
InstallableNpm.prototype.install = function (moduleName, cb) {
    this.command('install', moduleName, cb);
};

/**
 * npm uninstall
 * @param  {String}   moduleName
 * @param  {Function} cb - callback
 * @public
 */
InstallableNpm.prototype.uninstall = function (moduleName, cb) {
    this.command('uninstall', moduleName, cb);
};

/**
 * Validates a plugin name
 * @param  {String} moduleName
 * @return {String|undefined} String if validation error exists, otherwise undefined
 */
InstallableNpm.prototype.validatePluginName = function (moduleName) {
    if (!moduleName || moduleName.length === 0) {
        return 'No plugin name given.';
    }

    if (this.installable && !moduleName.match(this.installable.pluginsRegExp)) {
        return 'Requested module is not a plugin of this application.';
    }

    return undefined;
};

/**
 * Execute npm install or uninstall
 * @param  {String}   command - install or uninstall
 * @param  {String}   moduleName - the name of the module
 * @param  {Function} cb - callback
 * @private
 */
InstallableNpm.prototype.command = function (command, moduleName, cb) {
    var error = this.validatePluginName(moduleName);

    if (error) {
        var err = new Error(error);
        err.status = 400;
        return cb(err);
    }

    async.waterfall([
        function (done) {
            npm.load(null, done);
        },
        function (data, done) {
            npm.config.set('save', true);
            npm.commands[command]([moduleName], done);
        }
    ], function (err, result) {
        if (err && err.statusCode) {
            err.status = err.statusCode;
        }
        cb(err, result);
    });
};

module.exports = InstallableNpm;
<|endoftext|>


<commit_before>var buster = require("buster-core");
var busterSessionMiddlewareSession = require("./session");

module.exports = {
    respond: function (req, res) {
        if (req.method == "POST" && req.url == "/sessions") {
            createSessionFromRequest.call(this, req, res);
            return true;
        }

        for (var i = 0, ii = this.sessions.length; i < ii; i++) {
            var session =  this.sessions[i];

            if (req.url == session.rootPath && req.method == "DELETE") {
                this.logger.info("Destroying session via HTTP");
                unloadSession.call(this, i);
                res.writeHead(200);
                res.end();
                return true;
            }
        }
    },

    createSession: function (data) {
        var error = busterSessionMiddlewareSession.validate(data);
        if (error) throw new Error(error);
        this.logger.info("Creating session");
        this.logger.debug("Session data", data)
        return createSessionWithData.call(this, data);
    },

    destroySession: function (id) {
        for (var i = 0, ii = this.sessions.length; i < ii; i++) {
            var session = this.sessions[i];
            if (session.id == id) {
                this.logger.info("Destroying session");
                this.logger.debug("Session ID", id)
                unloadSession.call(this, i);
                break;
            }
        }
    },

    get sessions() {
        if (!this._sessions) this._sessions = [];
        return this._sessions;
    }
};

for (var k in buster.eventEmitter) module.exports[k] = buster.eventEmitter[k];

function unloadSession(i) {
    var session = this.sessions[i];
    session.destroy();
    this.sessions.splice(i, 1);

    // Did we unload the current session?
    if (i == 0) {
        this.logger.debug("Ending current session");
        this.emit("session:end");
        if (this.sessions.length > 0) {
            this.logger.debug("Starting queued session");
            loadSession.call(this);
        }
    }
}

function createSessionFromRequest(req, res) {
    var self = this;
    this.logger.debug("Creating session via HTTP request");

    var requestBody = "";
    req.on("data", function (chunk) { requestBody += chunk.toString("utf8") });
    req.on("end", function () {
        createSessionWithRequestBody.call(self, requestBody, req, res);
    });
}

function createSessionWithRequestBody(requestBody, req, res) {
    try {
        var data = JSON.parse(requestBody);
    } catch(e) {
        this.logger.error("Invalid JSON", requestBody);
        return failWithMessage(res, "Invalid JSON");
    }

    var error = busterSessionMiddlewareSession.validate(data);
    if (error) {
        this.logger.error("Invalid session", error);
        return failWithMessage(res, error);
    }

    try {
        var session = createSessionWithData.call(this, data);
    } catch (e) {
        this.logger.error("Failed to create session", e);
        res.writeHead(403);
        res.write(e.message);
        res.end();
        return;
    }

    var statusCode;
    if (this.sessions.length > 1) {
        statusCode = 202;
    } else {
        statusCode = 201;
    }

    res.writeHead(statusCode, {"Location": session.rootPath});
    res.write(JSON.stringify(session.toJSON()));
    res.end();
}

function createSessionWithData(data) {
    var session = busterSessionMiddlewareSession.create(data, this.resourceMiddleware, this.server);
    this.sessions.push(session);
    if (this.sessions.length == 1) {
        this.logger.debug("Starting newly created session immediately");
        loadSession.call(this);
    } else {
        this.logger.debug("Queuing newly created session");
    }
    return session;
}

function loadSession() {
    this.emit("session:start", this.sessions[0]);
}

function failWithMessage(res, message) {
    res.writeHead(500);
    res.write(message + "\n");
    res.end();
}<commit_msg>Cleaning up logging slightly, and removing session data logging, it's simply too much data.<commit_after>var buster = require("buster-core");
var busterSessionMiddlewareSession = require("./session");

module.exports = {
    respond: function (req, res) {
        if (req.method == "POST" && req.url == "/sessions") {
            this.logger.debug("Creating session via HTTP");
            createSessionFromRequest.call(this, req, res);
            return true;
        }

        for (var i = 0, ii = this.sessions.length; i < ii; i++) {
            var session =  this.sessions[i];

            if (req.url == session.rootPath && req.method == "DELETE") {
                this.logger.info("Destroying session via HTTP");
                unloadSession.call(this, i);
                res.writeHead(200);
                res.end();
                return true;
            }
        }
    },

    createSession: function (data) {
        var error = busterSessionMiddlewareSession.validate(data);
        if (error) throw new Error(error);
        this.logger.info("Creating session");
        return createSessionWithData.call(this, data);
    },

    destroySession: function (id) {
        for (var i = 0, ii = this.sessions.length; i < ii; i++) {
            var session = this.sessions[i];
            if (session.id == id) {
                this.logger.info("Destroying session");
                this.logger.debug("Session ID", id)
                unloadSession.call(this, i);
                break;
            }
        }
    },

    get sessions() {
        if (!this._sessions) this._sessions = [];
        return this._sessions;
    }
};

for (var k in buster.eventEmitter) module.exports[k] = buster.eventEmitter[k];

function unloadSession(i) {
    var session = this.sessions[i];
    session.destroy();
    this.sessions.splice(i, 1);

    // Did we unload the current session?
    if (i == 0) {
        this.logger.debug("Ending current session");
        this.emit("session:end");
        if (this.sessions.length > 0) {
            this.logger.debug("Starting queued session");
            loadSession.call(this);
        }
    }
}

function createSessionFromRequest(req, res) {
    var self = this;

    var requestBody = "";
    req.on("data", function (chunk) { requestBody += chunk.toString("utf8") });
    req.on("end", function () {
        createSessionWithRequestBody.call(self, requestBody, req, res);
    });
}

function createSessionWithRequestBody(requestBody, req, res) {
    try {
        var data = JSON.parse(requestBody);
    } catch(e) {
        this.logger.error("Invalid JSON", requestBody);
        return failWithMessage(res, "Invalid JSON");
    }

    var error = busterSessionMiddlewareSession.validate(data);
    if (error) {
        this.logger.error("Invalid session", error);
        return failWithMessage(res, error);
    }

    try {
        var session = createSessionWithData.call(this, data);
    } catch (e) {
        this.logger.error("Failed to create session", e);
        res.writeHead(403);
        res.write(e.message);
        res.end();
        return;
    }

    var statusCode;
    if (this.sessions.length > 1) {
        statusCode = 202;
    } else {
        statusCode = 201;
    }

    res.writeHead(statusCode, {"Location": session.rootPath});
    res.write(JSON.stringify(session.toJSON()));
    res.end();
}

function createSessionWithData(data) {
    var session = busterSessionMiddlewareSession.create(data, this.resourceMiddleware, this.server);
    this.sessions.push(session);
    if (this.sessions.length == 1) {
        this.logger.debug("Starting newly created session immediately");
        loadSession.call(this);
    } else {
        this.logger.debug("Queuing newly created session");
    }
    return session;
}

function loadSession() {
    this.emit("session:start", this.sessions[0]);
}

function failWithMessage(res, message) {
    res.writeHead(500);
    res.write(message + "\n");
    res.end();
}<|endoftext|>


<commit_before>/*
 * Copyright (c) 2017-2018 Digital Bazaar, Inc. All rights reserved.
 */
'use strict';

const constants = require('../constants');
const util = require('../util');
const Helper = require('../Helper');

// TODO: reorganize this class further and make it more obvious which
// methods need to be extended in proof plugins

// TODO: make signature and verification code (and potentially other code)
// more DRY, especially wrt. plugins having reimplement functionality

module.exports = class LinkedDataSignature {
  constructor(injector, algorithm) {
    this.injector = injector;
    this.algorithm = algorithm;
    this.helper = new Helper(injector);
  }

  async canonize(input, options) {
    const jsonld = this.injector.use('jsonld');
    const opts = {
      algorithm: 'URDNA2015',
      format: 'application/n-quads',
      expansionMap: options.expansionMap
    };
    if(options.documentLoader) {
      opts.documentLoader = options.documentLoader;
    }
    return jsonld.canonize(input, opts);
  }

  async createVerifyData(input, options) {
    // TODO: frame before getting signature, not just compact? considerations:
    // should the assumption be (for this library) that the signature is on
    // the top-level object and thus framing is unnecessary?

    const jsonld = this.injector.use('jsonld');
    const opts = {expansionMap: options.expansionMap};
    if(options.documentLoader) {
      opts.documentLoader = options.documentLoader;
    }
    const compacted = await jsonld.compact(
      input, constants.SECURITY_CONTEXT_URL, opts);

    // TODO: will need to preserve `proof` when chained signature
    // option is used and implemented in the future

    // delete the existing proofs(s) prior to canonicalization
    delete compacted.proof;

    // ensure signature values are removed from proof node
    const proof = await this.sanitizeProofNode(options.proof, options);

    // concatenate hash of c14n proof options and hash of c14n document
    const c14nProofOptions = await this.canonize(proof, options);
    const c14nDocument = await this.canonize(compacted, options);
    return {
      data: this._sha256(c14nProofOptions).getBytes() +
        this._sha256(c14nDocument).getBytes(),
      encoding: 'binary'
    };
  }

  async sanitizeProofNode(proof, options) {
    // `jws`,`signatureValue`,`proofValue` must not be included in the proof
    // options
    proof = util.deepClone(options.proof);
    delete proof.jws;
    delete proof.signatureValue;
    delete proof.proofValue;
    return proof;
  }

  async sign(input, options) {
    // copy options for setting defaults
    options = Object.assign({}, options || {});

    // validate common options
    if('creator' in options && typeof options.creator !== 'string') {
      throw new TypeError('"options.creator" must be a URL string.');
    }
    if('domain' in options && typeof options.domain !== 'string') {
      throw new TypeError('"options.domain" must be a string.');
    }
    if('nonce' in options && typeof options.nonce !== 'string') {
      throw new TypeError('"options.nonce" must be a string.');
    }

    // disallow dropping properties when expanding by default
    if(options.expansionMap !== false) {
      options.expansionMap = info => {
        if(info.unmappedProperty) {
          throw new Error('The property "' +
             info.unmappedProperty + '" in the input ' +
            'was not defined in the context.');
        }
      };
    }

    // build proof (aka signature options)
    let proof;
    if(options.proof) {
      // use proof JSON-LD document passed to API
      const jsonld = this.injector.use('jsonld');
      const opts = {expansionMap: options.expansionMap};
      if(options.documentLoader) {
        opts.documentLoader = options.documentLoader;
      }
      proof = await jsonld.compact(
        options.proof, constants.SECURITY_CONTEXT_URL, opts);
    } else {
      // create proof JSON-LD document
      proof = {'@context': constants.SECURITY_CONTEXT_URL};
    }

    // set default `now` date if not given in `proof` or `options`
    if(!('created' in proof) && !('date' in options)) {
      options.date = new Date();
    }

    // ensure date is in string format
    if('date' in options && typeof options.date !== 'string') {
      // TODO: parse non-string date and force to w3c format?
      options.date = util.w3cDate(options.date);
    }

    // ensure algorithm is set
    proof.type = options.algorithm;

    // add API overrides
    if('date' in options) {
      proof.created = options.date;
    }
    if('creator' in options) {
      proof.creator = options.creator;
    }
    if('domain' in options) {
      proof.domain = options.domain;
    }
    if('nonce' in options) {
      proof.nonce = options.nonce;
    }

    // produce data to sign
    options.proof = proof;
    const verifyData = await this.createVerifyData(input, options);

    // create proof node
    const proofNode = await this.createProofNode(verifyData, options);

    // attach proof node
    return this.attachProofNode(input, proofNode, options);
  }

  async createProofNode(verifyData, options) {
    const proof = options.proof;
    proof.jws = await this.createSignatureValue(verifyData, options);
    return proof;
  }

  async attachProofNode(input, proofNode, options) {
    // compact proof node to match input context
    const tmp = {
      'https://w3id.org/security#proof': {
        '@graph': proofNode
      }
    };
    const jsonld = this.injector.use('jsonld');
    const ctx = jsonld.getValues(input, '@context');
    const opts = {expansionMap: options.expansionMap};
    if(options.documentLoader) {
      opts.documentLoader = options.documentLoader;
    }
    const compactProofNode = await jsonld.compact(tmp, ctx, opts);

    // TODO: it is unclear how the signature would be easily added without
    // reshaping the input... so perhaps this library should just require
    // the caller to accept that the signature will be added to the top
    // level of the input

    // attach signature node to cloned input and return it
    const output = util.deepClone(input);
    delete compactProofNode['@context'];
    const proofKey = Object.keys(compactProofNode)[0];
    jsonld.addValue(output, proofKey, compactProofNode[proofKey]);
    return output;
  }

  async verify(framed, options) {
    options = Object.assign({}, options || {});

    const proof = framed.signature || framed.proof;
    proof['@context'] = framed['@context'];

    // destructure options
    let {
      maxTimestampDelta = (15 * 60),
      checkNonce = () => (
        proof.nonce === null || proof.nonce === undefined),
      checkDomain = () => (
        proof.domain === null || proof.domain === undefined),
      checkTimestamp = () => {
        const now = Date.now();
        const delta = maxTimestampDelta * 1000;
        const created = Date.parse(proof.created);
        if(created < (now - delta) || created > (now + delta)) {
          throw new Error('The digital signature timestamp is out of range.');
        }
        return true;
      },
      checkKey = this.helper.checkKey.bind(this.helper),
      publicKey: getPublicKey = this.helper.getPublicKey.bind(this.helper)
    } = options;

    // normalize function options
    if(checkNonce === false) {
      // not checking nonce, so return true
      checkNonce = () => true;
    }
    if(checkDomain === false) {
      // not checking domain, so return true
      checkDomain = () => true;
    }
    if(checkTimestamp === false) {
      // not checking timestamp, so return true
      checkTimestamp = () => true;
    }
    if(typeof getPublicKey !== 'function') {
      const key = getPublicKey;
      getPublicKey = keyId => {
        if(keyId !== key.id) {
          throw new Error('Public key not found.');
        }
        return key;
      };
    }
    checkNonce = util.normalizeAsyncFn(checkNonce, 2);
    checkDomain = util.normalizeAsyncFn(checkDomain, 2);
    checkTimestamp = util.normalizeAsyncFn(checkTimestamp, 2);
    checkKey = util.normalizeAsyncFn(checkKey, 2);
    getPublicKey = util.normalizeAsyncFn(getPublicKey, 2);

    // run nonce, domain, and timestamp checks in parallel
    const checks = await Promise.all([
      checkNonce(proof.nonce, options),
      checkDomain(proof.domain, options),
      checkTimestamp(proof.date, options)
    ]);

    if(!checks[0]) {
      throw new Error('The nonce is invalid.');
    }
    if(!checks[1]) {
      throw new Error('The domain is invalid.');
    }
    if(!checks[2]) {
      throw new Error('The timestamp is invalid.');
    }

    // get public key
    const publicKey = await getPublicKey(proof.creator, options);

    // TODO: should be able to override revocation check to ensure that
    // signatures made prior to the revocation check could potentially still
    // be verified

    // ensure key is not revoked
    if('revoked' in publicKey) {
      throw new Error(
        'The document was signed with a key that has been revoked.');
    }

    // ensure key is trusted before proceeding
    const isKeyTrusted = await checkKey(
      publicKey, Object.assign({}, options, {proof}));
    if(!isKeyTrusted) {
      throw new Error('The document was not signed with a trusted key.');
    }

    // verify input
    const verifyData = await this.createVerifyData(
      framed, Object.assign({}, options, {
        date: proof.created,
        nonce: proof.nonce,
        domain: proof.domain,
        proof
      }));

    return this.verifyProofNode(
      verifyData, proof,
      Object.assign({}, options, {publicKey: publicKey}));
  }

  async verifyProofNode(verifyData, proof, options) {
    throw new Error(
      '"verifyProofNode" must be implemented in a derived class.');
  }

  // TODO: use node `crypto` and Buffers in node environment
  // returns a forge buffer
  _sha256(str, encoding) {
    // browser or other environment
    const forge = this.injector.use('forge');
    const md = forge.md.sha256.create();
    md.update(str, encoding || 'utf8');
    return md.digest();
  }
};
<commit_msg>Pass `proof` instead of `options.proof`.<commit_after>/*
 * Copyright (c) 2017-2018 Digital Bazaar, Inc. All rights reserved.
 */
'use strict';

const constants = require('../constants');
const util = require('../util');
const Helper = require('../Helper');

// TODO: reorganize this class further and make it more obvious which
// methods need to be extended in proof plugins

// TODO: make signature and verification code (and potentially other code)
// more DRY, especially wrt. plugins having reimplement functionality

module.exports = class LinkedDataSignature {
  constructor(injector, algorithm) {
    this.injector = injector;
    this.algorithm = algorithm;
    this.helper = new Helper(injector);
  }

  async canonize(input, options) {
    const jsonld = this.injector.use('jsonld');
    const opts = {
      algorithm: 'URDNA2015',
      format: 'application/n-quads',
      expansionMap: options.expansionMap
    };
    if(options.documentLoader) {
      opts.documentLoader = options.documentLoader;
    }
    return jsonld.canonize(input, opts);
  }

  async createVerifyData(input, options) {
    // TODO: frame before getting signature, not just compact? considerations:
    // should the assumption be (for this library) that the signature is on
    // the top-level object and thus framing is unnecessary?

    const jsonld = this.injector.use('jsonld');
    const opts = {expansionMap: options.expansionMap};
    if(options.documentLoader) {
      opts.documentLoader = options.documentLoader;
    }
    const compacted = await jsonld.compact(
      input, constants.SECURITY_CONTEXT_URL, opts);

    // TODO: will need to preserve `proof` when chained signature
    // option is used and implemented in the future

    // delete the existing proofs(s) prior to canonicalization
    delete compacted.proof;

    // ensure signature values are removed from proof node
    const proof = await this.sanitizeProofNode(options.proof, options);

    // concatenate hash of c14n proof options and hash of c14n document
    const c14nProofOptions = await this.canonize(proof, options);
    const c14nDocument = await this.canonize(compacted, options);
    return {
      data: this._sha256(c14nProofOptions).getBytes() +
        this._sha256(c14nDocument).getBytes(),
      encoding: 'binary'
    };
  }

  async sanitizeProofNode(proof, options) {
    // `jws`,`signatureValue`,`proofValue` must not be included in the proof
    // options
    proof = util.deepClone(proof);
    delete proof.jws;
    delete proof.signatureValue;
    delete proof.proofValue;
    return proof;
  }

  async sign(input, options) {
    // copy options for setting defaults
    options = Object.assign({}, options || {});

    // validate common options
    if('creator' in options && typeof options.creator !== 'string') {
      throw new TypeError('"options.creator" must be a URL string.');
    }
    if('domain' in options && typeof options.domain !== 'string') {
      throw new TypeError('"options.domain" must be a string.');
    }
    if('nonce' in options && typeof options.nonce !== 'string') {
      throw new TypeError('"options.nonce" must be a string.');
    }

    // disallow dropping properties when expanding by default
    if(options.expansionMap !== false) {
      options.expansionMap = info => {
        if(info.unmappedProperty) {
          throw new Error('The property "' +
             info.unmappedProperty + '" in the input ' +
            'was not defined in the context.');
        }
      };
    }

    // build proof (aka signature options)
    let proof;
    if(options.proof) {
      // use proof JSON-LD document passed to API
      const jsonld = this.injector.use('jsonld');
      const opts = {expansionMap: options.expansionMap};
      if(options.documentLoader) {
        opts.documentLoader = options.documentLoader;
      }
      proof = await jsonld.compact(
        options.proof, constants.SECURITY_CONTEXT_URL, opts);
    } else {
      // create proof JSON-LD document
      proof = {'@context': constants.SECURITY_CONTEXT_URL};
    }

    // set default `now` date if not given in `proof` or `options`
    if(!('created' in proof) && !('date' in options)) {
      options.date = new Date();
    }

    // ensure date is in string format
    if('date' in options && typeof options.date !== 'string') {
      // TODO: parse non-string date and force to w3c format?
      options.date = util.w3cDate(options.date);
    }

    // ensure algorithm is set
    proof.type = options.algorithm;

    // add API overrides
    if('date' in options) {
      proof.created = options.date;
    }
    if('creator' in options) {
      proof.creator = options.creator;
    }
    if('domain' in options) {
      proof.domain = options.domain;
    }
    if('nonce' in options) {
      proof.nonce = options.nonce;
    }

    // produce data to sign
    options.proof = proof;
    const verifyData = await this.createVerifyData(input, options);

    // create proof node
    const proofNode = await this.createProofNode(verifyData, options);

    // attach proof node
    return this.attachProofNode(input, proofNode, options);
  }

  async createProofNode(verifyData, options) {
    const proof = options.proof;
    proof.jws = await this.createSignatureValue(verifyData, options);
    return proof;
  }

  async attachProofNode(input, proofNode, options) {
    // compact proof node to match input context
    const tmp = {
      'https://w3id.org/security#proof': {
        '@graph': proofNode
      }
    };
    const jsonld = this.injector.use('jsonld');
    const ctx = jsonld.getValues(input, '@context');
    const opts = {expansionMap: options.expansionMap};
    if(options.documentLoader) {
      opts.documentLoader = options.documentLoader;
    }
    const compactProofNode = await jsonld.compact(tmp, ctx, opts);

    // TODO: it is unclear how the signature would be easily added without
    // reshaping the input... so perhaps this library should just require
    // the caller to accept that the signature will be added to the top
    // level of the input

    // attach signature node to cloned input and return it
    const output = util.deepClone(input);
    delete compactProofNode['@context'];
    const proofKey = Object.keys(compactProofNode)[0];
    jsonld.addValue(output, proofKey, compactProofNode[proofKey]);
    return output;
  }

  async verify(framed, options) {
    options = Object.assign({}, options || {});

    const proof = framed.signature || framed.proof;
    proof['@context'] = framed['@context'];

    // destructure options
    let {
      maxTimestampDelta = (15 * 60),
      checkNonce = () => (
        proof.nonce === null || proof.nonce === undefined),
      checkDomain = () => (
        proof.domain === null || proof.domain === undefined),
      checkTimestamp = () => {
        const now = Date.now();
        const delta = maxTimestampDelta * 1000;
        const created = Date.parse(proof.created);
        if(created < (now - delta) || created > (now + delta)) {
          throw new Error('The digital signature timestamp is out of range.');
        }
        return true;
      },
      checkKey = this.helper.checkKey.bind(this.helper),
      publicKey: getPublicKey = this.helper.getPublicKey.bind(this.helper)
    } = options;

    // normalize function options
    if(checkNonce === false) {
      // not checking nonce, so return true
      checkNonce = () => true;
    }
    if(checkDomain === false) {
      // not checking domain, so return true
      checkDomain = () => true;
    }
    if(checkTimestamp === false) {
      // not checking timestamp, so return true
      checkTimestamp = () => true;
    }
    if(typeof getPublicKey !== 'function') {
      const key = getPublicKey;
      getPublicKey = keyId => {
        if(keyId !== key.id) {
          throw new Error('Public key not found.');
        }
        return key;
      };
    }
    checkNonce = util.normalizeAsyncFn(checkNonce, 2);
    checkDomain = util.normalizeAsyncFn(checkDomain, 2);
    checkTimestamp = util.normalizeAsyncFn(checkTimestamp, 2);
    checkKey = util.normalizeAsyncFn(checkKey, 2);
    getPublicKey = util.normalizeAsyncFn(getPublicKey, 2);

    // run nonce, domain, and timestamp checks in parallel
    const checks = await Promise.all([
      checkNonce(proof.nonce, options),
      checkDomain(proof.domain, options),
      checkTimestamp(proof.date, options)
    ]);

    if(!checks[0]) {
      throw new Error('The nonce is invalid.');
    }
    if(!checks[1]) {
      throw new Error('The domain is invalid.');
    }
    if(!checks[2]) {
      throw new Error('The timestamp is invalid.');
    }

    // get public key
    const publicKey = await getPublicKey(proof.creator, options);

    // TODO: should be able to override revocation check to ensure that
    // signatures made prior to the revocation check could potentially still
    // be verified

    // ensure key is not revoked
    if('revoked' in publicKey) {
      throw new Error(
        'The document was signed with a key that has been revoked.');
    }

    // ensure key is trusted before proceeding
    const isKeyTrusted = await checkKey(
      publicKey, Object.assign({}, options, {proof}));
    if(!isKeyTrusted) {
      throw new Error('The document was not signed with a trusted key.');
    }

    // verify input
    const verifyData = await this.createVerifyData(
      framed, Object.assign({}, options, {
        date: proof.created,
        nonce: proof.nonce,
        domain: proof.domain,
        proof
      }));

    return this.verifyProofNode(
      verifyData, proof,
      Object.assign({}, options, {publicKey: publicKey}));
  }

  async verifyProofNode(verifyData, proof, options) {
    throw new Error(
      '"verifyProofNode" must be implemented in a derived class.');
  }

  // TODO: use node `crypto` and Buffers in node environment
  // returns a forge buffer
  _sha256(str, encoding) {
    // browser or other environment
    const forge = this.injector.use('forge');
    const md = forge.md.sha256.create();
    md.update(str, encoding || 'utf8');
    return md.digest();
  }
};
<|endoftext|>


<commit_before>AT.prototype.atInputRendered = function(){
    var fieldId = this.data._id;

    var parentData = Template.currentData();
    var state = (parentData && parentData.state) || AccountsTemplates.getState();

    if (AccountsTemplates.options.focusFirstInput) {
      var firstVisibleInput = _.find(AccountsTemplates.getFields(), function(f){
        return _.contains(f.visible, state);
      });

      if (firstVisibleInput && firstVisibleInput._id === fieldId) {
        this.$("input#at-field-" + fieldId).focus();
      }
  }
};

AT.prototype.atInputHelpers = {
    disabled: function() {
        return AccountsTemplates.disabled();
    },
    showLabels: function() {
        return AccountsTemplates.options.showLabels;
    },
    displayName: function() {
        var parentData = Template.parentData();
        var state = (parentData && parentData.state) || AccountsTemplates.getState();
        var displayName = this.getDisplayName(state);
        return T9n.get(displayName, markIfMissing=false);
    },
    optionalText: function(){
        return "(" + T9n.get(AccountsTemplates.texts.optionalField, markIfMissing=false) + ")";
    },
    templateName: function() {
        if (this.template)
            return this.template;
        if (this.type === "checkbox")
            return "atCheckboxInput";
        if (this.type === "select")
            return "atSelectInput";
        if (this.type === "radio")
            return "atRadioInput";
        if (this.type === "hidden")
            return "atHiddenInput";
        return "atTextInput";
    },
    values: function(){
        var id = this._id;
        return _.map(this.select, function(select){
            var s = _.clone(select);
            s._id = id + "-" + select.value;
            s.id = id;
            return s;
        });
    },
    errorText: function() {
        var err = this.getStatus();
        return T9n.get(err, markIfMissing=false);
    },
    placeholder: function() {
        if (AccountsTemplates.options.showPlaceholders) {
            var parentData = Template.parentData();
            var state = (parentData && parentData.state) || AccountsTemplates.getState();
            var placeholder = this.getPlaceholder(state);
            return T9n.get(placeholder, markIfMissing=false);
        }
    },
};

AT.prototype.atInputEvents = {
    "focusin input": function(event, t){
        this.clearStatus();
    },
    "focusout input": function(event, t){
        var fieldId = this._id;
        var rawValue = this.getValue(t);
        var value = this.fixValue(rawValue);
        // Possibly updates the input value
        if (value !== rawValue) {
            this.setValue(t, value);
        }

        // Client-side only validation
        if (!this.validation)
            return;
        var parentData = Template.parentData();
        var state = (parentData && parentData.state) || AccountsTemplates.getState();
        // No validation during signIn
        if (state === "signIn")
            return;
        // Special case for password confirmation
        if (value && fieldId === "password_again"){
            if (value !== $("#at-field-password").val())
                return this.setError(AccountsTemplates.texts.errors.pwdMismatch);
        }
        this.validate(value);
    },
    "keyup input": function(event, t){
        // Client-side only continuous validation
        if (!this.continuousValidation)
            return;
        var parentData = Template.parentData();
        var state = (parentData && parentData.state) || AccountsTemplates.getState();
        // No validation during signIn
        if (state === "signIn")
            return;
        var fieldId = this._id;
        var rawValue = this.getValue(t);
        var value = this.fixValue(rawValue);
        // Possibly updates the input value
        if (value !== rawValue) {
            this.setValue(t, value);
        }
        // Special case for password confirmation
        if (value && fieldId === "password_again"){
            if (value !== $("#at-field-password").val())
                return this.setError(AccountsTemplates.texts.errors.pwdMismatch);
        }
        this.validate(value);
    },
};
<commit_msg>Syntax refactoring input helpers<commit_after>AT.prototype.atInputRendered = function() {
  var fieldId = this.data._id;
  var parentData = Template.currentData();
  var state = (parentData && parentData.state) || AccountsTemplates.getState();

  if (AccountsTemplates.options.focusFirstInput) {
    var firstVisibleInput = _.find(AccountsTemplates.getFields(), function(f){
      return _.contains(f.visible, state);
    });
  }

  if (firstVisibleInput && firstVisibleInput._id === fieldId) {
    this.$("input#at-field-" + fieldId).focus();
  }
};

AT.prototype.atInputHelpers = {
  disabled: function() {
    return AccountsTemplates.disabled();
  },

  showLabels: function() {
    return AccountsTemplates.options.showLabels;
  },

  displayName: function() {
    var parentData = Template.parentData();
    var state = (parentData && parentData.state) || AccountsTemplates.getState();
    var displayName = this.getDisplayName(state);

    return T9n.get(displayName, markIfMissing = false);
  },

  optionalText: function() {
    return "(" + T9n.get(AccountsTemplates.texts.optionalField, markIfMissing = false) + ")";
  },

  templateName: function() {
    if (this.template) {
      return this.template;
    }

    if (this.type === "checkbox") {
      return "atCheckboxInput";
    }

    if (this.type === "select") {
      return "atSelectInput";
    }

    if (this.type === "radio") {
      return "atRadioInput";
    }

    if (this.type === "hidden") {
      return "atHiddenInput";
    }

    return "atTextInput";
  },

  values: function() {
    var id = this._id;

    return _.map(this.select, function(select) {
      var _select = _.clone(select);

      _select._id = id + "-" + select.value;
      _select.id = id;

      return _select;
    });
  },

  errorText: function() {
    var err = this.getStatus();

    return T9n.get(err, markIfMissing = false);
  },

  placeholder: function() {
    if (AccountsTemplates.options.showPlaceholders) {
      var parentData = Template.parentData();
      var state = (parentData && parentData.state) || AccountsTemplates.getState();
      var placeholder = this.getPlaceholder(state);

      return T9n.get(placeholder, markIfMissing = false);
    }
  },
};

AT.prototype.atInputEvents = {
  "focusin input": function(event, t) {
    this.clearStatus();
  },

  "focusout input": function(event, t) {
    var fieldId = this._id;
    var rawValue = this.getValue(t);
    var value = this.fixValue(rawValue);
    var parentData = Template.parentData();
    var state = (parentData && parentData.state) || AccountsTemplates.getState();

    // Possibly updates the input value
    if (value !== rawValue) {
      this.setValue(t, value);
    }

    // Client-side only validation
    if (!this.validation) {
      return;
    }

    // No validation during signIn
    if (state === "signIn") {
      return;
    }

    // Special case for password confirmation
    if (value && fieldId === "password_again") {
      if (value !== $("#at-field-password").val()) {
        return this.setError(AccountsTemplates.texts.errors.pwdMismatch);
      }
    }

    this.validate(value);
  },

  "keyup input": function(event, t) {
    var parentData = Template.parentData();
    var state = (parentData && parentData.state) || AccountsTemplates.getState();
    var fieldId = this._id;
    var rawValue = this.getValue(t);
    var value = this.fixValue(rawValue);

    // Client-side only continuous validation
    if (!this.continuousValidation) {
      return;
    }

    // No validation during signIn
    if (state === "signIn") {
      return;
    }

    // Possibly updates the input value
    if (value !== rawValue) {
      this.setValue(t, value);
    }

    // Special case for password confirmation
    if (value && fieldId === "password_again") {
      if (value !== $("#at-field-password").val()) {
        return this.setError(AccountsTemplates.texts.errors.pwdMismatch);
      }
    }

    this.validate(value);
  },
};
<|endoftext|>


<commit_before>'use babel';

import {CompositeDisposable, Emitter} from 'atom';
const electron = require('electron')

const GENERIC_DEVICE = 'GenericDevice'

export default class ToolBarTheraView {

  constructor (options) {

    this.element = document.createElement('select')
    this.element.classList.add('form-control-tools')
    this.element.id = options.id

    option1 = document.createElement('option')
    option1.setAttribute("value", GENERIC_DEVICE)
    option1.text = 'Click to load simulator'
    option1.type = GENERIC_DEVICE

    this.element.add(option1);

    // init symbols-tree-view and weex-run
    atom.commands.dispatch(atom.views.getView(atom.workspace), 'atom-live-server:get-sims')
    atom.commands.dispatch(atom.views.getView(atom.workspace), 'symbols-tree-view:activate')

    this.subscriptions = new CompositeDisposable()

    this.subscriptions.add(
      atom.commands.add('atom-workspace', 'tool-bar:flash-devicelist', (allDevices) => {
        this.flashDeviceList(allDevices.detail)
      })
    );
  }

  flashDeviceList(options){

    this.element.innerHTML = '';

    let optionDefault = document.createElement('option')
    // optionDefault.setAttribute('value', GENERIC_DEVICE)
    // optionDefault.text = 'Generic Device'
    // this.element.add(optionDefault)

    for (var i=0;i<options.length;i++){
      optiontemp = document.createElement('option');
      optiontemp.setAttribute("value",options[i].udid);
      optiontemp.setAttribute("type", options[i].type);
      optiontemp.text = options[i].name;
      this.element.add(optiontemp);
    }

  }

  destroy () {
    this.subscriptions.dispose();
    if (this.element.parentNode) {
      this.element.parentNode.removeChild(this.element);
    }
    this.element = null;
  }
}
<commit_msg>change the simulator list to running list<commit_after>'use babel';

import {CompositeDisposable, Emitter} from 'atom';
const electron = require('electron')

const GENERIC_DEVICE = 'GenericDevice'

export default class ToolBarTheraView {

  constructor (options) {

    this.element = document.createElement('select')
    this.element.classList.add('form-control-tools')
    this.element.id = options.id

    option1 = document.createElement('option')
    option1.setAttribute("value", GENERIC_DEVICE)
    option1.text = 'Click to load simulator'
    option1.type = GENERIC_DEVICE

    this.element.add(option1);

    // init symbols-tree-view and weex-run
    atom.commands.dispatch(atom.views.getView(atom.workspace), 'atom-live-server:get-sims')
    atom.commands.dispatch(atom.views.getView(atom.workspace), 'symbols-tree-view:activate')

    this.subscriptions = new CompositeDisposable()

    this.subscriptions.add(
      atom.commands.add('atom-workspace', 'console.targets', (allDevices) => {
        this.flashDeviceList(allDevices.detail)
      })
    );
  }

  flashDeviceList(options){

    this.element.innerHTML = '';

    let optionDefault = document.createElement('option')
    // optionDefault.setAttribute('value', GENERIC_DEVICE)
    // optionDefault.text = 'Generic Device'
    // this.element.add(optionDefault)

    for (var i=0;i<options.length;i++){
      optiontemp = document.createElement('option');
      optiontemp.setAttribute("value",options[i].udid);
      optiontemp.setAttribute("type", options[i].type);
      optiontemp.text = options[i].name;
      this.element.add(optiontemp);
    }
  }

  destroy () {
    this.subscriptions.dispose();
    if (this.element.parentNode) {
      this.element.parentNode.removeChild(this.element);
    }
    this.element = null;
  }
}
<|endoftext|>


<commit_before>var request = require('request');
var jsonParser = require('../jsonparser.js');

var elasticsearch = function (parent, baseUrl) {
    if(baseUrl.substr(-1) === '/') {
        baseUrl = baseUrl.substr(0, baseUrl.length - 1);
    }
    this.baseUrl            = baseUrl;
    this.parent             = parent;
    this.lastScrollId       = null;
    this.totalSearchResults = 0;
    this.hasSkipped         = 0;
};

// accept callback
// return (error, arr) where arr is an array of objects
elasticsearch.prototype.get = function(limit, offset, callback){
    var self = this;
    var type = self.parent.options.type;
    if(type === 'data'){
        self.getData(limit, offset, callback);
    }else if(type === 'mapping'){
        self.getMapping(limit, offset, callback);
    }else{
        callback(new Error('unknown type option'), null);
    }
};

elasticsearch.prototype.getMapping = function (limit, offset, callback){
    var self = this;
    if(self.gotMapping === true){
        callback(null, []);
    }else{
        var url = self.baseUrl + '/_mapping';
        request.get(url, function(err, response){
            self.gotMapping = true;
            var payload = [];
            if(!err){ response = payload.push(response.body); }
            callback(err, payload);
        });
    }
};

elasticsearch.prototype.getData = function (limit, offset, callback){
    var searchRequest, self, uri;
    self = this;
    var searchBody = self.parent.options.searchBody;

    if (offset >= self.totalSearchResults && self.totalSearchResults !== 0) {
        callback(null, []);
        return;
    }

    if (self.lastScrollId !== null) {
        scrollResultSet(self, callback);
    } else {
        self.numberOfShards(self.baseUrl ,function(err, numberOfShards){
            var shardedLimit = Math.ceil(limit / numberOfShards);

            uri = self.baseUrl +
                  "/" +
                  "_search?search_type=scan&scroll=" +
                  self.parent.options.scrollTime +
                  "&size=" + shardedLimit;

            searchBody.size = shardedLimit;

            searchRequest = {
                "uri": uri,
                "method": "GET",
                "body": JSON.stringify(searchBody)
            };

            request.get(searchRequest, function requestResonse (err, response) {
                if(err){
                    callback(err, []);
                    return;
                }else if(response.statusCode !== 200){
                    err = new Error(response.body);
                    callback(err, []);
                    return;
                }

                var body = jsonParser.parse(response.body);
                self.lastScrollId = body._scroll_id;
                if(self.lastScrollId === undefined){
                    err = new Error("Unable to obtain scrollId; This tends to indicate an error with your index(es)");
                    callback(err, []);
                    return;
                }
                self.totalSearchResults = body.hits.total;

                scrollResultSet(self, callback);
            });
        });
    }
};

// to respect the --limit param, we need to set the scan/sroll limit = limit/#Shards
// http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/scan-scroll.html
elasticsearch.prototype.numberOfShards = function(baseUrl, callback){
    request.get(baseUrl + "/_settings", function(err, response){
        if(err){
            callback(err);
        }else{
            try{
                var body = jsonParser.parse(response.body);
                var indexParts = baseUrl.split('/');
                var index = indexParts[(indexParts.length - 1)];
                var numberOfShards = body[index].settings.index.number_of_shards;
                callback(err, numberOfShards);
            }catch(e){
                callback(err, 1);
            }
        }
    });
};

// accept arr, callback where arr is an array of objects
// return (error, writes)
elasticsearch.prototype.set = function(data, limit, offset, callback){
    var self = this;
    var type = self.parent.options.type;
    if(type === 'data'){
        self.setData(data, limit, offset, callback);
    }else if(type === 'mapping'){
        self.setMapping(data, limit, offset, callback);
    }else{
        callback(new Error('unknown type option'), null);
    }
};

elasticsearch.prototype.setMapping = function(data, limit, offset, callback){
    var self = this;
    if(self.haveSetMapping === true){
        callback(null, 0);
    }else{
    	try { data = jsonParser.parse(data[0]); }
        catch (e) { return callback(e); }
        var started = 0;
        var count   = 0;
        request.put(self.baseUrl, function(err, response){ // ensure the index exists
            for(var index in data){
                var mappings = data[index]['mappings'];
                for(var key in mappings){
                    var mapping  =  {};
                    mapping[key] = mappings[key];
                    var url = self.baseUrl + '/' + key + '/_mapping';
                    started++;
                    count++;
                    var payload = {
                        url: url,
                        body: JSON.stringify(mapping),
                        timeout: self.parent.options.timeout,
                    };
                    request.put(payload, function(err, response){ // upload the mapping
                        started--;
                        if(err){
                            var bodyError = jsonParser.parse(response.body).error;
                            if(bodyError){ err = bodyError; }
                        }
                        if(started === 0){
                            self.haveSetMapping = true;
                            callback(err, count);
                        }
                    });
                }
            }
        });
    }
};

elasticsearch.prototype.setData = function(data, limit, offset, callback){
    var self = this;
    var error = null;
    var extraFields = [ '_routing', '_parent', '_timestamp', '_ttl' ];
    var writes = 0;
    if (self.parent.options.bulk === true){
        var thisUrl = self.baseUrl + "/_bulk";
        var payload = {
            url:  thisUrl,
            body: "",
            timeout: self.parent.options.timeout,
        };
        var useOutputIndexName = self.parent.options['bulk-use-output-index-name'] === true;
        data.forEach(function(elem, index, theArray) {
            var val = "{ \"index\" : { ";
            if (!useOutputIndexName) { // Not setting _index should use the one defined in URL.
              val += "\"_index\" : \"" + elem._index + "\", ";
            }
            val += "\"_type\" : \""  + elem._type  + "\", ";
            val += "\"_id\" : \""    + elem._id    + "\"";
            if (elem.fields) {
              extraFields.forEach(function(field) {
                if (elem.fields[field]) {
                  val += ', "' + field + '" : "' + elem.fields[field] + '"';
                }
              });
            }
            val += " } }\n";
            val += JSON.stringify(elem._source) + "\n";
            theArray[index] = val;
        });
        payload.body = data.join("");
        request.put(payload, function(err, response){
            var writes = 0;
            try{
                var r = jsonParser.parse(response.body);
                if(r.items !== null){
                    if(r.ok === true ){
                        writes = data.length;
                    }else{
                        r.items.forEach(function(item){
                            if(item.index.status < 400){
                                writes++;
                            }
                        });
                    }
                }
            }catch(e){
                err = e;
            }
            self.reindex(function(err){
                callback(err, writes);
            });
        });
    }else{
        var started = 0;
        data.forEach(function(elem){
            started++;
            var thisUrl = self.baseUrl + "/";
            if(self.parent.options.all === true){ thisUrl += elem._index + "/"; }
            thisUrl += encodeURIComponent(elem._type) + "/" + encodeURIComponent(elem._id);
            if (elem.fields) {
              var and = "?";
              extraFields.forEach(function(field) {
                if (elem.fields[field]) {
                  thisUrl += and + field + "=" + encodeURIComponent(elem.fields[field]);
                  and = "&";
                }
              });
            }
            var payload = {
                url:  thisUrl,
                body: JSON.stringify(elem._source),
                timeout: self.parent.options.timeout,
            };
            self.parent.emit('debug', 'thisUrl: ' + thisUrl + ", elem._source: " + JSON.stringify(elem._source));

            request.put(payload, function(err, response){
                if(err){ error = err; }
                try{
                    var r = jsonParser.parse(response.body);
                    if(r.ok === true || r._version >= 1){ writes++; }
                }catch(e){ }
                started--;
                if(started === 0){
                    self.reindex(function(){
                        callback(error, writes);
                    });
                }
            });
        });

        if(data.length === 0){
            process.nextTick(function(){
                callback(error, writes);
            });
        }
    }
};

elasticsearch.prototype.del = function(elem, callback){
    var self = this;
    var thisUrl = self.baseUrl + "/" + encodeURIComponent(elem._type) + "/" + encodeURIComponent(elem._id);

    self.parent.emit('debug', 'deleteUrl: ' + thisUrl);
    request.del(thisUrl, function(err, response, body){
        if(typeof callback === 'function'){ callback(err, response, body); }
    });
};

elasticsearch.prototype.reindex = function(callback){
    var self = this;
    request.post(self.baseUrl + "/_refresh", function(err, response){
        callback(err, response);
    });
};

exports.elasticsearch = elasticsearch;

/////////////
// HELPERS //
/////////////

/**
 * Posts requests to the _search api to fetch the latest
 * scan result with scroll id
 * @param that
 * @param callback
 */
function scrollResultSet(that, callback) {
    var self = that;
    var baseUrl = self.baseUrl;
    var searchUrl;
    var body;

    if (baseUrl.split('/').length -1 <= 2) {
        searchUrl = baseUrl;
    } else {
        searchUrl = baseUrl.substr(0, baseUrl.lastIndexOf('/'));
        if(searchUrl.split('/').length > 3){ // indicates a request for a index/type search
            searchUrl = searchUrl.substr(0, searchUrl.lastIndexOf('/'));
        }
    }

    var scrollRequest = {
        "uri": searchUrl + "/" + "_search" + "/scroll?scroll=" + self.parent.options.scrollTime,
        "method": "POST",
        "body": self.lastScrollId
    };

    request.get(scrollRequest, function requestResonse (err, response) {
        if(err){
            callback(err, []);
            return;
        }

        if(err === null && response.statusCode != 200){
            err = new Error(response.body);
            callback(err, []);
            return;
        }

        try{
            body = jsonParser.parse(response.body);
        }catch(e){
            err = new Error("Cannot Parse: " + response.body);
            callback(err, []);
            return;
        }

        self.lastScrollId = body._scroll_id;
        var hits = body.hits.hits;

        if(self.parent.options.delete === true && hits.length > 0) {
            var started = 0;
            hits.forEach(function(elem){
                started++;
                self.del(elem, function(){
                    started--;
                    if(started === 0){
                        self.reindex(function(err){
                            if (hits.length === 0) {
                                self.lastScrollId = null;
                            }
                            callback(err, hits);
                        });
                    }
                });
            });
        } else {
            if (hits.length === 0) {
                self.lastScrollId = null;
            }

            // are we skipping and we have hits?
            if(self.parent.options.skip !== null && hits.length > 0 && self.hasSkipped < self.parent.options.skip) {
                // lets remove hits until we reach the skip number
                while(hits.length > 0 && self.hasSkipped < self.parent.options.skip) {
                    self.hasSkipped++;
                    hits.splice(0, 1);
                }

                if(hits.length > 0) {
                    // we have some hits after skipping, lets callback
                    callback(err, hits);
                } else {
                    // we skipped, but now we don't have any hits,
                    // scroll again for more data if possible
                    scrollResultSet(that, callback);
                }
            } else {
                // not skipping or done skipping
                callback(err, hits);
            }
        }
    });
}
<commit_msg>lib/transport/elasticsearch.js: Remove '_' in extraFields and try using both   underscored and not underscored versions of fields names<commit_after>var request = require('request');
var jsonParser = require('../jsonparser.js');

var elasticsearch = function (parent, baseUrl) {
    if(baseUrl.substr(-1) === '/') {
        baseUrl = baseUrl.substr(0, baseUrl.length - 1);
    }
    this.baseUrl            = baseUrl;
    this.parent             = parent;
    this.lastScrollId       = null;
    this.totalSearchResults = 0;
    this.hasSkipped         = 0;
};

// accept callback
// return (error, arr) where arr is an array of objects
elasticsearch.prototype.get = function(limit, offset, callback){
    var self = this;
    var type = self.parent.options.type;
    if(type === 'data'){
        self.getData(limit, offset, callback);
    }else if(type === 'mapping'){
        self.getMapping(limit, offset, callback);
    }else{
        callback(new Error('unknown type option'), null);
    }
};

elasticsearch.prototype.getMapping = function (limit, offset, callback){
    var self = this;
    if(self.gotMapping === true){
        callback(null, []);
    }else{
        var url = self.baseUrl + '/_mapping';
        request.get(url, function(err, response){
            self.gotMapping = true;
            var payload = [];
            if(!err){ response = payload.push(response.body); }
            callback(err, payload);
        });
    }
};

elasticsearch.prototype.getData = function (limit, offset, callback){
    var searchRequest, self, uri;
    self = this;
    var searchBody = self.parent.options.searchBody;

    if (offset >= self.totalSearchResults && self.totalSearchResults !== 0) {
        callback(null, []);
        return;
    }

    if (self.lastScrollId !== null) {
        scrollResultSet(self, callback);
    } else {
        self.numberOfShards(self.baseUrl ,function(err, numberOfShards){
            var shardedLimit = Math.ceil(limit / numberOfShards);

            uri = self.baseUrl +
                  "/" +
                  "_search?search_type=scan&scroll=" +
                  self.parent.options.scrollTime +
                  "&size=" + shardedLimit;

            searchBody.size = shardedLimit;

            searchRequest = {
                "uri": uri,
                "method": "GET",
                "body": JSON.stringify(searchBody)
            };

            request.get(searchRequest, function requestResonse (err, response) {
                if(err){
                    callback(err, []);
                    return;
                }else if(response.statusCode !== 200){
                    err = new Error(response.body);
                    callback(err, []);
                    return;
                }

                var body = jsonParser.parse(response.body);
                self.lastScrollId = body._scroll_id;
                if(self.lastScrollId === undefined){
                    err = new Error("Unable to obtain scrollId; This tends to indicate an error with your index(es)");
                    callback(err, []);
                    return;
                }
                self.totalSearchResults = body.hits.total;

                scrollResultSet(self, callback);
            });
        });
    }
};

// to respect the --limit param, we need to set the scan/sroll limit = limit/#Shards
// http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/scan-scroll.html
elasticsearch.prototype.numberOfShards = function(baseUrl, callback){
    request.get(baseUrl + "/_settings", function(err, response){
        if(err){
            callback(err);
        }else{
            try{
                var body = jsonParser.parse(response.body);
                var indexParts = baseUrl.split('/');
                var index = indexParts[(indexParts.length - 1)];
                var numberOfShards = body[index].settings.index.number_of_shards;
                callback(err, numberOfShards);
            }catch(e){
                callback(err, 1);
            }
        }
    });
};

// accept arr, callback where arr is an array of objects
// return (error, writes)
elasticsearch.prototype.set = function(data, limit, offset, callback){
    var self = this;
    var type = self.parent.options.type;
    if(type === 'data'){
        self.setData(data, limit, offset, callback);
    }else if(type === 'mapping'){
        self.setMapping(data, limit, offset, callback);
    }else{
        callback(new Error('unknown type option'), null);
    }
};

elasticsearch.prototype.setMapping = function(data, limit, offset, callback){
    var self = this;
    if(self.haveSetMapping === true){
        callback(null, 0);
    }else{
    	try { data = jsonParser.parse(data[0]); }
        catch (e) { return callback(e); }
        var started = 0;
        var count   = 0;
        request.put(self.baseUrl, function(err, response){ // ensure the index exists
            for(var index in data){
                var mappings = data[index]['mappings'];
                for(var key in mappings){
                    var mapping  =  {};
                    mapping[key] = mappings[key];
                    var url = self.baseUrl + '/' + key + '/_mapping';
                    started++;
                    count++;
                    var payload = {
                        url: url,
                        body: JSON.stringify(mapping),
                        timeout: self.parent.options.timeout,
                    };
                    request.put(payload, function(err, response){ // upload the mapping
                        started--;
                        if(err){
                            var bodyError = jsonParser.parse(response.body).error;
                            if(bodyError){ err = bodyError; }
                        }
                        if(started === 0){
                            self.haveSetMapping = true;
                            callback(err, count);
                        }
                    });
                }
            }
        });
    }
};

elasticsearch.prototype.setData = function(data, limit, offset, callback){
    var self = this;
    var error = null;
    var extraFields = [ 'routing', 'parent', 'timestamp', 'ttl' ];
    var writes = 0;
    if (self.parent.options.bulk === true){
        var thisUrl = self.baseUrl + "/_bulk";
        var payload = {
            url:  thisUrl,
            body: "",
            timeout: self.parent.options.timeout,
        };
        var useOutputIndexName = self.parent.options['bulk-use-output-index-name'] === true;
        data.forEach(function(elem, index, theArray) {
            var val = "{ \"index\" : { ";
            if (!useOutputIndexName) { // Not setting _index should use the one defined in URL.
              val += "\"_index\" : \"" + elem._index + "\", ";
            }
            val += "\"_type\" : \""  + elem._type  + "\", ";
            val += "\"_id\" : \""    + elem._id    + "\"";
            if (elem.fields) {
              extraFields.forEach(function(field) {
                if (elem.fields[field]) {
                  val += ', "' + field + '" : "' + elem.fields[field] + '"';
                }
                if (elem.fields['_'+field]) {
                  val += ', "' + field + '" : "' + elem.fields['_'+field] + '"';
                }
              });
            }
            val += " } }\n";
            val += JSON.stringify(elem._source) + "\n";
            theArray[index] = val;
        });
        payload.body = data.join("");
        request.put(payload, function(err, response){
            var writes = 0;
            try{
                var r = jsonParser.parse(response.body);
                if(r.items !== null){
                    if(r.ok === true ){
                        writes = data.length;
                    }else{
                        r.items.forEach(function(item){
                            if(item.index.status < 400){
                                writes++;
                            }
                        });
                    }
                }
            }catch(e){
                err = e;
            }
            self.reindex(function(err){
                callback(err, writes);
            });
        });
    }else{
        var started = 0;
        data.forEach(function(elem){
            started++;
            var thisUrl = self.baseUrl + "/";
            if(self.parent.options.all === true){ thisUrl += elem._index + "/"; }
            thisUrl += encodeURIComponent(elem._type) + "/" + encodeURIComponent(elem._id);
            if (elem.fields) {
              var and = "?";
              extraFields.forEach(function(field) {
                if (elem.fields[field]) {
                  thisUrl += and + field + "=" + encodeURIComponent(elem.fields[field]);
                  and = "&";
                }
                if (elem.fields['_'+field]) {
                  thisUrl += and + field + "=" + encodeURIComponent(elem.fields['_'+field]);
                  and = "&";
                }
              });
            }
            var payload = {
                url:  thisUrl,
                body: JSON.stringify(elem._source),
                timeout: self.parent.options.timeout,
            };
            self.parent.emit('debug', 'thisUrl: ' + thisUrl + ", elem._source: " + JSON.stringify(elem._source));

            request.put(payload, function(err, response){
                if(err){ error = err; }
                try{
                    var r = jsonParser.parse(response.body);
                    if(r.ok === true || r._version >= 1){ writes++; }
                }catch(e){ }
                started--;
                if(started === 0){
                    self.reindex(function(){
                        callback(error, writes);
                    });
                }
            });
        });

        if(data.length === 0){
            process.nextTick(function(){
                callback(error, writes);
            });
        }
    }
};

elasticsearch.prototype.del = function(elem, callback){
    var self = this;
    var thisUrl = self.baseUrl + "/" + encodeURIComponent(elem._type) + "/" + encodeURIComponent(elem._id);

    self.parent.emit('debug', 'deleteUrl: ' + thisUrl);
    request.del(thisUrl, function(err, response, body){
        if(typeof callback === 'function'){ callback(err, response, body); }
    });
};

elasticsearch.prototype.reindex = function(callback){
    var self = this;
    request.post(self.baseUrl + "/_refresh", function(err, response){
        callback(err, response);
    });
};

exports.elasticsearch = elasticsearch;

/////////////
// HELPERS //
/////////////

/**
 * Posts requests to the _search api to fetch the latest
 * scan result with scroll id
 * @param that
 * @param callback
 */
function scrollResultSet(that, callback) {
    var self = that;
    var baseUrl = self.baseUrl;
    var searchUrl;
    var body;

    if (baseUrl.split('/').length -1 <= 2) {
        searchUrl = baseUrl;
    } else {
        searchUrl = baseUrl.substr(0, baseUrl.lastIndexOf('/'));
        if(searchUrl.split('/').length > 3){ // indicates a request for a index/type search
            searchUrl = searchUrl.substr(0, searchUrl.lastIndexOf('/'));
        }
    }

    var scrollRequest = {
        "uri": searchUrl + "/" + "_search" + "/scroll?scroll=" + self.parent.options.scrollTime,
        "method": "POST",
        "body": self.lastScrollId
    };

    request.get(scrollRequest, function requestResonse (err, response) {
        if(err){
            callback(err, []);
            return;
        }

        if(err === null && response.statusCode != 200){
            err = new Error(response.body);
            callback(err, []);
            return;
        }

        try{
            body = jsonParser.parse(response.body);
        }catch(e){
            err = new Error("Cannot Parse: " + response.body);
            callback(err, []);
            return;
        }

        self.lastScrollId = body._scroll_id;
        var hits = body.hits.hits;

        if(self.parent.options.delete === true && hits.length > 0) {
            var started = 0;
            hits.forEach(function(elem){
                started++;
                self.del(elem, function(){
                    started--;
                    if(started === 0){
                        self.reindex(function(err){
                            if (hits.length === 0) {
                                self.lastScrollId = null;
                            }
                            callback(err, hits);
                        });
                    }
                });
            });
        } else {
            if (hits.length === 0) {
                self.lastScrollId = null;
            }

            // are we skipping and we have hits?
            if(self.parent.options.skip !== null && hits.length > 0 && self.hasSkipped < self.parent.options.skip) {
                // lets remove hits until we reach the skip number
                while(hits.length > 0 && self.hasSkipped < self.parent.options.skip) {
                    self.hasSkipped++;
                    hits.splice(0, 1);
                }

                if(hits.length > 0) {
                    // we have some hits after skipping, lets callback
                    callback(err, hits);
                } else {
                    // we skipped, but now we don't have any hits,
                    // scroll again for more data if possible
                    scrollResultSet(that, callback);
                }
            } else {
                // not skipping or done skipping
                callback(err, hits);
            }
        }
    });
}
<|endoftext|>