<commit_before>require('shelljs/global');
var deps = require('./dependencies.js');

// SpookyJS provides our bridge to CasperJS and PhantomJS
var Spooky = require('spooky');

var file = require('./file.js')
  , dl = require('./download.js')
  , url = require('./url.js')
  , dom = require('./dom.js')
  , Ticker = require('./ticker.js');

var scrape = module.exports;

// Bubble SpookyJS errors up to our interface,
// providing a clear context message and the SpookyJS message
// as detail.
//
// @param {String} err the SpookyJS error.
var handleInitError = function(err) {
  if (err) {
    var e = new Error('Failed to initialize SpookyJS');
    e.details = err;
    log.error(e);
    log.debug(e.stack);
  }
};

// generate SpookyJS settings
// @param {String} loglevel the loglevel
// @return {Object} the settings
var settings = function(loglevel) {
  env['PHANTOMJS_EXECUTABLE'] = deps.getbinpath('phantomjs');
  return {
    child: {
      command: deps.getbinpath('casperjs'),
      transport: 'http'
    },
    casper: {
      logLevel: loglevel,
      verbose: true,
      exitOnError: true,
      pageSettings: {
        loadImages: false,
        loadPlugins: false
      },
      resourceTimeout: 20000,
      onResourceTimeout: function(e) {
        console.log('page load timeout - has the connection dropped?');
        console.log(e.errorCode);   // it'll probably be 408
        console.log(e.errorString); // it'll probably be 'Network timeout on resource'
        console.log(e.url);         // the url whose request timed out
        casper.exit(1);
      }
    }
  };
}

// Scrape a URL using a ScraperJSON-defined scraper.
//
// @param {String} scrapeUrl the URL to scrape
// @param {Object} definition a dictionary defining the scraper
// @param {Function} cb callback to run once this scrape has finished
// @return {Object} a dictionary containing the scraping results
scrape.scrape = function(scrapeUrl, definition, cb, loglevel) {
  log.debug('function scrape: ' + scrapeUrl);

  // validate arguments
  url.checkUrl(scrapeUrl);

  // let's get our scrape on
  log.debug('creating spooky instance');
  var spooky = new Spooky(settings(loglevel), function() {
    log.debug('spooky initialising');
    spooky.start(scrapeUrl);

    spooky.then(function() {
      // in SpookyJS scope
      this.emit('pagedownload', this.evaluate(function() {
        // in rendered page scope
        return document.all[0].outerHTML;
      }));

    });

    spooky.run();
  });

  spooky.on('pagedownload', function(html) {
    log.debug('page downloaded and rendered');
    try {
      scrape.scrapeHtml(html, definition, scrapeUrl, cb);
    } catch(e) {
      log.error('problem scraping html:');
      log.error(e.message);
      log.error(e.stack);
    }
  });

  if (loglevel == 'debug') {
    spooky.on('console', function (line) {
      var parts = line.split(' ');
      log.debug(parts.slice(1, parts.length).join(' '));
    });

    spooky.on('log', function (log) {
      if (log.space === 'remote') {
        console.log(log.message.replace(/ \- .*/, ''));
      }
    });
  }

};

// Parse the rendered HTML using the scraper definition.
// Downloadable resources are saved to disk, and the successfully
// parsed elements are output as JSON.
//
// @param {String} html the HTML source of the rendered page
// @param {Object} definition the scraper definition
scrape.scrapeHtml = function(html, definition, scrapeUrl, cb) {
  var ticker = new Ticker(2, cb);

  // check rendered HTML
  if (!html) {
    log.error('error: SpookyJS failed to render the HTML');
  }

  // save the rendered html
  log.debug('saving rendered HTML');
  file.write('rendered.html', html, function(){ ticker.tick() });

  // load HTML into DOM
  log.debug('scraping rendered DOM');
  var doc = dom.render(html);

  // scrape the DOM using the ScraperJSON scraper
  var results = scrape.scrapeScraperJSON(definition,
                                         doc,
                                         scrapeUrl,
                                         cb,
                                         ticker);

  // save results JSON
  var json = JSON.stringify(results, undefined, 2);
  file.write('results.json', json, function(){ ticker.tick() });
}

scrape.scrapeScraperJSON =
  function(definition, doc, scrapeUrl, cb, ticker) {
  if (!ticker) {
    ticker = new Ticker(0, cb);
  }
  var results = [];

  // we don't know how many downloads will be spawned by each
  // element, so remove the Node.js cap of 10 listeners per
  // emitter (0 == infinty)
  process.setMaxListeners(0);

  for (var key in definition) {
    try {
      var element = definition[key];
      log.debug('scraping element:', key);
      // extract element
      var selector = element.selector;
      var attribute = element.attribute;
      var matches = dom.select(selector, doc);
      log.debug("found", matches.length, "matches");
      for (var i = 0; i < matches.length; i++) {
        var res = matches[i];
        if (res) {
          res = dom.getAttribute(res, attribute);
          log.data(key + ":", dom.cleanElement(res));

          // save the result
          var data = {};
          data[key] = res;
          results.push(data);

          // process downloads
          if (element.download) {
            // set download running
            var down = dl.downloadResource(res, scrapeUrl);
            // add it to the task ticker
            ticker.elongate();
            down.on('close', function() {
              log.debug('download done');
              ticker.tick();
            });
            down.on('error', function(err) {
              log.error('file download failed: ' + err);
              ticker.tick();
            });
          }
        } else {
          log.warn('could not capture element');
        }
      }
    } catch(err) {
      // errors deep in the stack should be presented, but
      // we want to continue with the rest of the execution
      log.error('problem applying scraperJSON scraper:');
      log.error(err.message);
      log.error(err.stack);
    }
  }
  if (results.length == 0) {
    log.warn('no elements were extracted from url:', scrapeUrl);
  }
  if (ticker.length > 2) {
    log.info('waiting for',
             ticker.length - 2,
             'downloads to complete in background');
  }
  return results;
}
<commit_msg>fetch entire rendered HTML using documentElement rather than all (fixes ContentMine/quickscrape#11)<commit_after>require('shelljs/global');
var deps = require('./dependencies.js');

// SpookyJS provides our bridge to CasperJS and PhantomJS
var Spooky = require('spooky');

var file = require('./file.js')
  , dl = require('./download.js')
  , url = require('./url.js')
  , dom = require('./dom.js')
  , Ticker = require('./ticker.js');

var scrape = module.exports;

// Bubble SpookyJS errors up to our interface,
// providing a clear context message and the SpookyJS message
// as detail.
//
// @param {String} err the SpookyJS error.
var handleInitError = function(err) {
  if (err) {
    var e = new Error('Failed to initialize SpookyJS');
    e.details = err;
    log.error(e);
    log.debug(e.stack);
  }
};

// generate SpookyJS settings
// @param {String} loglevel the loglevel
// @return {Object} the settings
var settings = function(loglevel) {
  env['PHANTOMJS_EXECUTABLE'] = deps.getbinpath('phantomjs');
  return {
    child: {
      command: deps.getbinpath('casperjs'),
      transport: 'http'
    },
    casper: {
      logLevel: loglevel,
      verbose: true,
      exitOnError: true,
      pageSettings: {
        loadImages: false,
        loadPlugins: false
      },
      resourceTimeout: 20000,
      onResourceTimeout: function(e) {
        console.log('page load timeout - has the connection dropped?');
        console.log(e.errorCode);   // it'll probably be 408
        console.log(e.errorString); // it'll probably be 'Network timeout on resource'
        console.log(e.url);         // the url whose request timed out
        casper.exit(1);
      }
    }
  };
}

// Scrape a URL using a ScraperJSON-defined scraper.
//
// @param {String} scrapeUrl the URL to scrape
// @param {Object} definition a dictionary defining the scraper
// @param {Function} cb callback to run once this scrape has finished
// @return {Object} a dictionary containing the scraping results
scrape.scrape = function(scrapeUrl, definition, cb, loglevel) {
  log.debug('function scrape: ' + scrapeUrl);

  // validate arguments
  url.checkUrl(scrapeUrl);

  // let's get our scrape on
  log.debug('creating spooky instance');
  var spooky = new Spooky(settings(loglevel), function() {
    log.debug('spooky initialising');
    spooky.start(scrapeUrl);

    spooky.then(function() {
      // in SpookyJS scope
      this.emit('pagedownload', this.evaluate(function() {
        // in rendered page scope
        return document.documentElement.outerHTML;
      }));

    });

    spooky.run();
  });

  spooky.on('pagedownload', function(html) {
    log.debug('page downloaded and rendered');
    try {
      scrape.scrapeHtml(html, definition, scrapeUrl, cb);
    } catch(e) {
      log.error('problem scraping html:');
      log.error(e.message);
      log.error(e.stack);
    }
  });

  if (loglevel == 'debug') {
    spooky.on('console', function (line) {
      var parts = line.split(' ');
      log.debug(parts.slice(1, parts.length).join(' '));
    });

    spooky.on('log', function (log) {
      if (log.space === 'remote') {
        console.log(log.message.replace(/ \- .*/, ''));
      }
    });
  }

};

// Parse the rendered HTML using the scraper definition.
// Downloadable resources are saved to disk, and the successfully
// parsed elements are output as JSON.
//
// @param {String} html the HTML source of the rendered page
// @param {Object} definition the scraper definition
scrape.scrapeHtml = function(html, definition, scrapeUrl, cb) {
  // check rendered HTML
  if (!html) {
    log.error('error: SpookyJS failed to render the HTML');
  }

  var ticker = new Ticker(2, cb);

  // save the rendered html
  log.debug('saving rendered HTML');
  file.write('rendered.html', html, function(){ ticker.tick() });

  // load HTML into DOM
  log.debug('scraping rendered DOM');
  var doc = dom.render(html);

  // scrape the DOM using the ScraperJSON scraper
  var results = scrape.scrapeScraperJSON(definition,
                                         doc,
                                         scrapeUrl,
                                         cb,
                                         ticker);

  // save results JSON
  var json = JSON.stringify(results, undefined, 2);
  file.write('results.json', json, function(){ ticker.tick() });
}

scrape.scrapeScraperJSON =
  function(definition, doc, scrapeUrl, cb, ticker) {
  if (!ticker) {
    ticker = new Ticker(0, cb);
  }
  var results = [];

  // we don't know how many downloads will be spawned by each
  // element, so remove the Node.js cap of 10 listeners per
  // emitter (0 == infinty)
  process.setMaxListeners(0);

  for (var key in definition) {
    try {
      var element = definition[key];
      log.debug('scraping element:', key);
      // extract element
      var selector = element.selector;
      var attribute = element.attribute;
      var matches = dom.select(selector, doc);
      log.debug("found", matches.length, "matches");
      for (var i = 0; i < matches.length; i++) {
        var res = matches[i];
        if (res) {
          res = dom.getAttribute(res, attribute);
          log.data(key + ":", dom.cleanElement(res));

          // save the result
          var data = {};
          data[key] = res;
          results.push(data);

          // process downloads
          if (element.download) {
            // set download running
            var down = dl.downloadResource(res, scrapeUrl);
            // add it to the task ticker
            ticker.elongate();
            down.on('close', function() {
              log.debug('download done');
              ticker.tick();
            });
            down.on('error', function(err) {
              log.error('file download failed: ' + err);
              ticker.tick();
            });
          }
        } else {
          log.warn('could not capture element');
        }
      }
    } catch(err) {
      // errors deep in the stack should be presented, but
      // we want to continue with the rest of the execution
      log.error('problem applying scraperJSON scraper:');
      log.error(err.message);
      log.error(err.stack);
    }
  }
  if (results.length == 0) {
    log.warn('no elements were extracted from url:', scrapeUrl);
  }
  if (ticker.length > 2) {
    log.info('waiting for',
             ticker.length - 2,
             'downloads to complete in background');
  }
  return results;
}
<|endoftext|>


<commit_before>var BufferReader = require('./bufferreader');
var BufferWriter = require('./bufferwriter');
var Opcode = require('./opcode');

var Script = function Script(buf) {
  if (!(this instanceof Script))
    return new Script(buf);

  this.chunks = [];

  if (Buffer.isBuffer(buf)) {
    this.fromBuffer(buf);
  }
  else if (typeof buf === 'string') {
    var str = buf;
    this.fromString(str);
  }
  else if (typeof buf !== 'undefined') {
    var obj = buf;
    this.set(obj);
  }
};

Script.prototype.set = function(obj) {
  this.chunks = obj.chunks || this.chunks;
  return this;
};

Script.prototype.fromJSON = function(json) {
  return this.fromString(json);
};

Script.prototype.toJSON = function() {
  return this.toString();
};

Script.prototype.fromBuffer = function(buf) {
  this.chunks = [];

  var br = new BufferReader(buf);
  while (!br.eof()) {
    var opcodenum = br.readUInt8();

    var len, buf;
    if (opcodenum > 0 && opcodenum < Opcode.map.OP_PUSHDATA1) {
      len = opcodenum;
      this.chunks.push({
        buf: br.read(len),
        len: len,
        opcodenum: opcodenum
      });
    } else if (opcodenum === Opcode.map.OP_PUSHDATA1) {
      len = br.readUInt8();
      var buf = br.read(len);
      this.chunks.push({
        buf: buf,
        len: len,
        opcodenum: opcodenum
      });
    } else if (opcodenum === Opcode.map.OP_PUSHDATA2) {
      len = br.readUInt16LE();
      buf = br.read(len);
      this.chunks.push({
        buf: buf,
        len: len,
        opcodenum: opcodenum
      });
    } else if (opcodenum === Opcode.map.OP_PUSHDATA4) {
      len = br.readUInt32LE();
      buf = br.read(len);
      this.chunks.push({
        buf: buf,
        len: len,
        opcodenum: opcodenum
      });
    } else {
      this.chunks.push(opcodenum);
    }
  }

  return this;
};

Script.prototype.toBuffer = function() {
  var bw = new BufferWriter();

  for (var i = 0; i < this.chunks.length; i++) {
    var chunk = this.chunks[i];
    if (typeof chunk === 'number') {
      var opcodenum = chunk;
      bw.writeUInt8(opcodenum);
    } else {
      var opcodenum = chunk.opcodenum;
      bw.writeUInt8(chunk.opcodenum);
      if (opcodenum < Opcode.map.OP_PUSHDATA1) {
        bw.write(chunk.buf);
      }
      else if (opcodenum === Opcode.map.OP_PUSHDATA1) {
        bw.writeUInt8(chunk.len);
        bw.write(chunk.buf);
      }
      else if (opcodenum === Opcode.map.OP_PUSHDATA2) {
        bw.writeUInt16LE(chunk.len);
        bw.write(chunk.buf);
      }
      else if (opcodenum === Opcode.map.OP_PUSHDATA4) {
        bw.writeUInt32LE(chunk.len);
        bw.write(chunk.buf);
      }
    }
  }

  return bw.concat();
};

Script.prototype.fromString = function(str) {
  this.chunks = [];

  var tokens = str.split(' ');
  var i = 0;
  while (i < tokens.length) {
    var token = tokens[i];
    var opcode = Opcode(token);
    var opcodenum = opcode.toNumber();

    if (typeof opcodenum === 'undefined') {
      opcodenum = parseInt(token);
      if (opcodenum > 0 && opcodenum < Opcode.map.OP_PUSHDATA1) {
        this.chunks.push({
          buf: new Buffer(tokens[i + 1].slice(2), 'hex'),
          len: opcodenum,
          opcodenum: opcodenum
        });
        i = i + 2;
      }
      else {
        throw new Error('Invalid script');
      }
    } else if (opcodenum === Opcode.map.OP_PUSHDATA1 || opcodenum === Opcode.map.OP_PUSHDATA2 || opcodenum === Opcode.map.OP_PUSHDATA4) {
      if (tokens[i + 2].slice(0, 2) != '0x')
        throw new Error('Pushdata data must start with 0x');
      this.chunks.push({
        buf: new Buffer(tokens[i + 2].slice(2), 'hex'),
        len: parseInt(tokens[i + 1]),
        opcodenum: opcodenum
      });
      i = i + 3;
    } else {
      this.chunks.push(opcodenum);
      i = i + 1;
    }
  }
  return this;
};

Script.prototype.toString = function() {
  var str = "";

  for (var i = 0; i < this.chunks.length; i++) {
    var chunk = this.chunks[i];
    if (typeof chunk === 'number') {
      var opcodenum = chunk;
      str = str + Opcode(opcodenum).toString() + " ";
    } else {
      var opcodenum = chunk.opcodenum;
      if (opcodenum === Opcode.map.OP_PUSHDATA1 || opcodenum === Opcode.map.OP_PUSHDATA2 || opcodenum === Opcode.map.OP_PUSHDATA4)
        str = str + Opcode(opcodenum).toString() + " " ;
      str = str + chunk.len + " " ;
      str = str + "0x" + chunk.buf.toString('hex') + " ";
    }
  }

  return str.substr(0, str.length - 1);
};

Script.prototype.isOpReturn = function() {
  if (this.chunks[0] === Opcode('OP_RETURN').toNumber()
    &&
    (this.chunks.length === 1
    ||
    (this.chunks.length === 2
    && this.chunks[1].buf
    && this.chunks[1].buf.length <= 40
    && this.chunks[1].length === this.chunks.len))) {
    return true;
  } else {
    return false;
  }
};

Script.prototype.isPubkeyhashOut = function() {
  if (this.chunks[0] === Opcode('OP_DUP').toNumber()
    && this.chunks[1] === Opcode('OP_HASH160').toNumber()
    && this.chunks[2].buf
    && this.chunks[3] === Opcode('OP_EQUALVERIFY').toNumber()
    && this.chunks[4] === Opcode('OP_CHECKSIG').toNumber()) {
    return true;
  } else {
    return false;
  }
};

Script.prototype.isPubkeyhashIn = function() {
  if (this.chunks.length === 2
    && this.chunks[0].buf
    && this.chunks[1].buf) {
    return true;
  } else {
    return false;
  }
};

Script.prototype.isScripthashOut = function() {
  if (this.chunks.length === 3
    && this.chunks[0] === Opcode('OP_HASH160').toNumber()
    && this.chunks[1].buf
    && this.chunks[1].buf.length === 20
    && this.chunks[2] === Opcode('OP_EQUAL').toNumber()) {
    return true;
  } else {
    return false;
  }
};

//note that these are frequently indistinguishable from pubkeyhashin
Script.prototype.isScripthashIn = function() {
  var allpush = this.chunks.every(function(chunk) {
    return Buffer.isBuffer(chunk.buf);
  });
  if (allpush) {
    return true;
  } else {
    return false;
  }
};

Script.prototype.write = function(obj) {
  if (typeof obj === 'string')
    this.writeOp(obj);
  else if (typeof obj === 'number')
    this.writeOp(obj);
  else if (Buffer.isBuffer(obj))
    this.writeBuffer(obj);
  else if (typeof obj === 'object')
    this.chunks.push(obj);
  else
    throw new Error('Invalid script chunk');
  return this;
};

Script.prototype.writeOp = function(str) {
  if (typeof str === 'number')
    this.chunks.push(str);
  else
    this.chunks.push(Opcode(str).toNumber());
  return this;
};

Script.prototype.writeBuffer = function(buf) {
  var opcodenum;
  var len = buf.length;
  if (buf.length > 0 && buf.length < Opcode.map.OP_PUSHDATA1) {
    opcodenum = buf.length;
  } else if (buf.length < Math.pow(2, 8)) {
    opcodenum = Opcode.map.OP_PUSHDATA1;
  } else if (buf.length < Math.pow(2, 16)) {
    opcodenum = Opcode.map.OP_PUSHDATA2;
  } else if (buf.length < Math.pow(2, 32)) {
    opcodenum = Opcode.map.OP_PUSHDATA4;
  }
  this.chunks.push({
    buf: buf,
    len: len,
    opcodenum: opcodenum
  });
  return this;
};

module.exports = Script;
<commit_msg>Add error for when pushdata is exceptionally large<commit_after>var BufferReader = require('./bufferreader');
var BufferWriter = require('./bufferwriter');
var Opcode = require('./opcode');

var Script = function Script(buf) {
  if (!(this instanceof Script))
    return new Script(buf);

  this.chunks = [];

  if (Buffer.isBuffer(buf)) {
    this.fromBuffer(buf);
  }
  else if (typeof buf === 'string') {
    var str = buf;
    this.fromString(str);
  }
  else if (typeof buf !== 'undefined') {
    var obj = buf;
    this.set(obj);
  }
};

Script.prototype.set = function(obj) {
  this.chunks = obj.chunks || this.chunks;
  return this;
};

Script.prototype.fromJSON = function(json) {
  return this.fromString(json);
};

Script.prototype.toJSON = function() {
  return this.toString();
};

Script.prototype.fromBuffer = function(buf) {
  this.chunks = [];

  var br = new BufferReader(buf);
  while (!br.eof()) {
    var opcodenum = br.readUInt8();

    var len, buf;
    if (opcodenum > 0 && opcodenum < Opcode.map.OP_PUSHDATA1) {
      len = opcodenum;
      this.chunks.push({
        buf: br.read(len),
        len: len,
        opcodenum: opcodenum
      });
    } else if (opcodenum === Opcode.map.OP_PUSHDATA1) {
      len = br.readUInt8();
      var buf = br.read(len);
      this.chunks.push({
        buf: buf,
        len: len,
        opcodenum: opcodenum
      });
    } else if (opcodenum === Opcode.map.OP_PUSHDATA2) {
      len = br.readUInt16LE();
      buf = br.read(len);
      this.chunks.push({
        buf: buf,
        len: len,
        opcodenum: opcodenum
      });
    } else if (opcodenum === Opcode.map.OP_PUSHDATA4) {
      len = br.readUInt32LE();
      buf = br.read(len);
      this.chunks.push({
        buf: buf,
        len: len,
        opcodenum: opcodenum
      });
    } else {
      this.chunks.push(opcodenum);
    }
  }

  return this;
};

Script.prototype.toBuffer = function() {
  var bw = new BufferWriter();

  for (var i = 0; i < this.chunks.length; i++) {
    var chunk = this.chunks[i];
    if (typeof chunk === 'number') {
      var opcodenum = chunk;
      bw.writeUInt8(opcodenum);
    } else {
      var opcodenum = chunk.opcodenum;
      bw.writeUInt8(chunk.opcodenum);
      if (opcodenum < Opcode.map.OP_PUSHDATA1) {
        bw.write(chunk.buf);
      }
      else if (opcodenum === Opcode.map.OP_PUSHDATA1) {
        bw.writeUInt8(chunk.len);
        bw.write(chunk.buf);
      }
      else if (opcodenum === Opcode.map.OP_PUSHDATA2) {
        bw.writeUInt16LE(chunk.len);
        bw.write(chunk.buf);
      }
      else if (opcodenum === Opcode.map.OP_PUSHDATA4) {
        bw.writeUInt32LE(chunk.len);
        bw.write(chunk.buf);
      }
    }
  }

  return bw.concat();
};

Script.prototype.fromString = function(str) {
  this.chunks = [];

  var tokens = str.split(' ');
  var i = 0;
  while (i < tokens.length) {
    var token = tokens[i];
    var opcode = Opcode(token);
    var opcodenum = opcode.toNumber();

    if (typeof opcodenum === 'undefined') {
      opcodenum = parseInt(token);
      if (opcodenum > 0 && opcodenum < Opcode.map.OP_PUSHDATA1) {
        this.chunks.push({
          buf: new Buffer(tokens[i + 1].slice(2), 'hex'),
          len: opcodenum,
          opcodenum: opcodenum
        });
        i = i + 2;
      }
      else {
        throw new Error('Invalid script');
      }
    } else if (opcodenum === Opcode.map.OP_PUSHDATA1 || opcodenum === Opcode.map.OP_PUSHDATA2 || opcodenum === Opcode.map.OP_PUSHDATA4) {
      if (tokens[i + 2].slice(0, 2) != '0x')
        throw new Error('Pushdata data must start with 0x');
      this.chunks.push({
        buf: new Buffer(tokens[i + 2].slice(2), 'hex'),
        len: parseInt(tokens[i + 1]),
        opcodenum: opcodenum
      });
      i = i + 3;
    } else {
      this.chunks.push(opcodenum);
      i = i + 1;
    }
  }
  return this;
};

Script.prototype.toString = function() {
  var str = "";

  for (var i = 0; i < this.chunks.length; i++) {
    var chunk = this.chunks[i];
    if (typeof chunk === 'number') {
      var opcodenum = chunk;
      str = str + Opcode(opcodenum).toString() + " ";
    } else {
      var opcodenum = chunk.opcodenum;
      if (opcodenum === Opcode.map.OP_PUSHDATA1 || opcodenum === Opcode.map.OP_PUSHDATA2 || opcodenum === Opcode.map.OP_PUSHDATA4)
        str = str + Opcode(opcodenum).toString() + " " ;
      str = str + chunk.len + " " ;
      str = str + "0x" + chunk.buf.toString('hex') + " ";
    }
  }

  return str.substr(0, str.length - 1);
};

Script.prototype.isOpReturn = function() {
  if (this.chunks[0] === Opcode('OP_RETURN').toNumber()
    &&
    (this.chunks.length === 1
    ||
    (this.chunks.length === 2
    && this.chunks[1].buf
    && this.chunks[1].buf.length <= 40
    && this.chunks[1].length === this.chunks.len))) {
    return true;
  } else {
    return false;
  }
};

Script.prototype.isPubkeyhashOut = function() {
  if (this.chunks[0] === Opcode('OP_DUP').toNumber()
    && this.chunks[1] === Opcode('OP_HASH160').toNumber()
    && this.chunks[2].buf
    && this.chunks[3] === Opcode('OP_EQUALVERIFY').toNumber()
    && this.chunks[4] === Opcode('OP_CHECKSIG').toNumber()) {
    return true;
  } else {
    return false;
  }
};

Script.prototype.isPubkeyhashIn = function() {
  if (this.chunks.length === 2
    && this.chunks[0].buf
    && this.chunks[1].buf) {
    return true;
  } else {
    return false;
  }
};

Script.prototype.isScripthashOut = function() {
  if (this.chunks.length === 3
    && this.chunks[0] === Opcode('OP_HASH160').toNumber()
    && this.chunks[1].buf
    && this.chunks[1].buf.length === 20
    && this.chunks[2] === Opcode('OP_EQUAL').toNumber()) {
    return true;
  } else {
    return false;
  }
};

//note that these are frequently indistinguishable from pubkeyhashin
Script.prototype.isScripthashIn = function() {
  var allpush = this.chunks.every(function(chunk) {
    return Buffer.isBuffer(chunk.buf);
  });
  if (allpush) {
    return true;
  } else {
    return false;
  }
};

Script.prototype.write = function(obj) {
  if (typeof obj === 'string')
    this.writeOp(obj);
  else if (typeof obj === 'number')
    this.writeOp(obj);
  else if (Buffer.isBuffer(obj))
    this.writeBuffer(obj);
  else if (typeof obj === 'object')
    this.chunks.push(obj);
  else
    throw new Error('Invalid script chunk');
  return this;
};

Script.prototype.writeOp = function(str) {
  if (typeof str === 'number')
    this.chunks.push(str);
  else
    this.chunks.push(Opcode(str).toNumber());
  return this;
};

Script.prototype.writeBuffer = function(buf) {
  var opcodenum;
  var len = buf.length;
  if (buf.length > 0 && buf.length < Opcode.map.OP_PUSHDATA1) {
    opcodenum = buf.length;
  } else if (buf.length < Math.pow(2, 8)) {
    opcodenum = Opcode.map.OP_PUSHDATA1;
  } else if (buf.length < Math.pow(2, 16)) {
    opcodenum = Opcode.map.OP_PUSHDATA2;
  } else if (buf.length < Math.pow(2, 32)) {
    opcodenum = Opcode.map.OP_PUSHDATA4;
  } else {
    throw new Error("You can't push that much data");
  }
  this.chunks.push({
    buf: buf,
    len: len,
    opcodenum: opcodenum
  });
  return this;
};

module.exports = Script;
<|endoftext|>


<commit_before>var logger = require('winston'),
    utils = require('./utils.js'),
    xmlParser = require('xml2js').parseString;

var parent;
var searchSession;

function mergeInto(o1, o2) {
    if (o1 === null || o2 === null)
        return o1;

    for (var key in o2)
        if (o2.hasOwnProperty(key))
            o1[key] = o2[key];

    return o1;
}

var parseCompactDecoded = function(resp, callback) {

    var columnsXml, dataXml, delimiter;
    xmlParser(resp, function(error, result) {

        if (!utils.replyCodeCheck(result, callback)) return;

        columnsXml = result.RETS.COLUMNS;

        if(!utils.xmlParseCheck(columnsXml, callback)) return;

        dataXml = result.RETS.DATA || [];

        if(!utils.xmlParseCheck(dataXml, callback)) return;
        delimiter = utils.hex2a(result.RETS.DELIMITER[0].$.value);

        if (delimiter === undefined)
        {
            if (callback)
                callback(new Error("No specified delimiter."));

            return;
        }

        var columns = columnsXml.toString().split(delimiter);
        var searchResults = {
            count: dataXml.length,
            total: result.RETS.COUNT[0].$.Records,
            data: []
        };
        for(var i = 0; i < dataXml.length; i++) {
            var data = dataXml[i].toString().split(delimiter);

            var model = {};
            for(var j = 1; j < columns.length-1; j++) {
                model[columns[j]] = data[j];
            }

            searchResults.data.push(model);
        }

        if (callback)
            callback(error, searchResults);
    });
};

//default query parameters
var queryOptions = {
    queryType:'DMQL2',
    format:'COMPACT-DECODED',
    count:1,
    standardNames:0,
    restrictedIndicator:'***',
    limit:"NONE",
    offset:1
};

/**
 * Invokes RETS search operation.
 *
 * @param _queryOptions Search query options.
 *        See RETS specification for query options.
 *
 *        Default values query params:
 *
 *           queryType:'DMQL2',
 *           format:'COMPACT-DECODED',
 *           count:1,
 *           standardNames:0,
 *           restrictedIndicator:'***',
 *           limit:"NONE",
 *           offset:0
 *
 * @param callback(error, data) (optional)
 */
var searchRets = function(_queryOptions, callback) {

    logger.debug("RETS method search with params _queryOptions=%j",_queryOptions);

    if (!_queryOptions) {
        if (callback)
            callback(new Error("_queryOptions is required."));

        return;
    }

    if (!_queryOptions.searchType) {
        if (callback)
            callback(new Error("_queryOptions.searchType field is required."));

        return;
    }

    if (!_queryOptions.class) {
        if (callback)
            callback(new Error("_queryOptions.class field is required."));

        return;
    }

    if (!_queryOptions.query) {
        if (callback)
            callback(new Error("_queryOptions.query field is required."));

        return;
    }

    if (!searchSession) {
        if (callback)
            callback(new Error("System data not set; invoke login first."));

        return;
    }

    mergeInto(queryOptions, _queryOptions);

    var searchOptions = {
        method: 'POST',
        formData: queryOptions
    };

    searchSession(searchOptions, function(error, response, body) {

        var isErr = false;

        if (error) {
            isErr = true;
        }

        if (response && response.statusCode != 200)
        {
            isErr = true;
            error = new Error("RETS method search returned unexpected status code: " + response.statusCode);
        }

        if (isErr) {
            logger.debug("Search Error:\n\n" + JSON.stringify(error));
            if (callback)
                callback(error);

            return;
        }

        if (callback) {
            callback(error, body);
        }
    });
};

/**
 *
 * Helper that performs a targeted RETS query and parses results.
 *
 * @param resourceType Rets resource type (ex: Property)
 * @param classType  Rets class type (ex: RESI)
 * @param queryString Rets query string. See RETS specification - (ex: MatrixModifiedDT=2014-01-01T00:00:00.000+)
 * @param options (optional) Other options, such as:
 *          _limit: limits the number of records returned.
 *          _select: specifies the fields to return.
 */
var query = function(resourceType, classType, queryString, callback, options) {
    if (!resourceType) {
        parent.emit("error", new Error("resourceType is required: (ex: Property)"));
        return;
    }

    if (!classType) {
        parent.emit("error", new Error("classType is required: (ex: RESI)"));
        return;
    }

    if (!queryString) {
        parent.emit("error", new Error("queryString is required: (ex: (MatrixModifiedDT=2014-01-01T00:00:00.000+))"));
        return;
    }

    options = options || {};

    var queryOpts = {
        searchType:resourceType,
        class:classType,
        query:queryString,
        limit:options._limit || "NONE",
        offset:options._offset || 1,
        count: options._count || 1,
        iteration: 0
    };
    if (options._select) queryOpts.select = options._select;

    var _fetch = function(queryOpts, fetchCallback) {
        queryOpts.iteration++;
        searchRets(queryOpts, function(error, data) {
            if (error) fetchCallback(error);
            else parseCompactDecoded(data, fetchCallback);
        });
    };
    var _checkResults = function(error, searchResults) {
        if (error) {
            parent.emit("error", error);
            return;
        }
        searchResults.iteration = queryOpts.iteration;
        parent.emit("data", searchResults);
        if (searchResults.count > 0) {
            if (queryOpts.offset < searchResults.total) {
                queryOpts.offset = Math.min(queryOpts.offset + searchResults.count, searchResults.total);
                //console.log("Fetched %s of %s total records", queryOpts.offset, searchResults.total);
                _fetch(queryOpts, _checkResults);
            }
        } else {
            parent.emit("end");
        }
    };
    _fetch(queryOpts, _checkResults);
};

module.exports = function(_parent, _searchSession) {

    parent = _parent;
    searchSession = _searchSession;

    return {
        searchRets:searchRets,
        query: query
    };
};

<commit_msg>Added iteration count<commit_after>var logger = require('winston'),
    utils = require('./utils.js'),
    xmlParser = require('xml2js').parseString;

var parent;
var searchSession;

function mergeInto(o1, o2) {
    if (o1 === null || o2 === null)
        return o1;

    for (var key in o2)
        if (o2.hasOwnProperty(key))
            o1[key] = o2[key];

    return o1;
}

var parseCompactDecoded = function(resp, callback) {

    var columnsXml, dataXml, delimiter;
    xmlParser(resp, function(error, result) {

        if (!utils.replyCodeCheck(result, callback)) return;

        columnsXml = result.RETS.COLUMNS;

        if(!utils.xmlParseCheck(columnsXml, callback)) return;

        dataXml = result.RETS.DATA || [];

        if(!utils.xmlParseCheck(dataXml, callback)) return;
        delimiter = utils.hex2a(result.RETS.DELIMITER[0].$.value);

        if (delimiter === undefined)
        {
            if (callback)
                callback(new Error("No specified delimiter."));

            return;
        }

        var columns = columnsXml.toString().split(delimiter);
        var searchResults = {
            count: dataXml.length,
            total: result.RETS.COUNT[0].$.Records,
            data: []
        };
        for(var i = 0; i < dataXml.length; i++) {
            var data = dataXml[i].toString().split(delimiter);

            var model = {};
            for(var j = 1; j < columns.length-1; j++) {
                model[columns[j]] = data[j];
            }

            searchResults.data.push(model);
        }

        if (callback)
            callback(error, searchResults);
    });
};

//default query parameters
var queryOptions = {
    queryType:'DMQL2',
    format:'COMPACT-DECODED',
    count:1,
    standardNames:0,
    restrictedIndicator:'***',
    limit:"NONE",
    offset:1
};

/**
 * Invokes RETS search operation.
 *
 * @param _queryOptions Search query options.
 *        See RETS specification for query options.
 *
 *        Default values query params:
 *
 *           queryType:'DMQL2',
 *           format:'COMPACT-DECODED',
 *           count:1,
 *           standardNames:0,
 *           restrictedIndicator:'***',
 *           limit:"NONE",
 *           offset:0
 *
 * @param callback(error, data) (optional)
 */
var searchRets = function(_queryOptions, callback) {

    logger.debug("RETS method search with params _queryOptions=%j",_queryOptions);

    if (!_queryOptions) {
        if (callback)
            callback(new Error("_queryOptions is required."));

        return;
    }

    if (!_queryOptions.searchType) {
        if (callback)
            callback(new Error("_queryOptions.searchType field is required."));

        return;
    }

    if (!_queryOptions.class) {
        if (callback)
            callback(new Error("_queryOptions.class field is required."));

        return;
    }

    if (!_queryOptions.query) {
        if (callback)
            callback(new Error("_queryOptions.query field is required."));

        return;
    }

    if (!searchSession) {
        if (callback)
            callback(new Error("System data not set; invoke login first."));

        return;
    }

    mergeInto(queryOptions, _queryOptions);

    var searchOptions = {
        method: 'POST',
        formData: queryOptions
    };

    searchSession(searchOptions, function(error, response, body) {

        var isErr = false;

        if (error) {
            isErr = true;
        }

        if (response && response.statusCode != 200)
        {
            isErr = true;
            error = new Error("RETS method search returned unexpected status code: " + response.statusCode);
        }

        if (isErr) {
            logger.debug("Search Error:\n\n" + JSON.stringify(error));
            if (callback)
                callback(error);

            return;
        }

        if (callback) {
            callback(error, body);
        }
    });
};

/**
 *
 * Helper that performs a targeted RETS query and parses results.
 *
 * @param resourceType Rets resource type (ex: Property)
 * @param classType  Rets class type (ex: RESI)
 * @param queryString Rets query string. See RETS specification - (ex: MatrixModifiedDT=2014-01-01T00:00:00.000+)
 * @param options (optional) Other options, such as:
 *          _limit: limits the number of records returned.
 *          _select: specifies the fields to return.
 */
var query = function(resourceType, classType, queryString, callback, options) {
    if (!resourceType) {
        parent.emit("error", new Error("resourceType is required: (ex: Property)"));
        return;
    }

    if (!classType) {
        parent.emit("error", new Error("classType is required: (ex: RESI)"));
        return;
    }

    if (!queryString) {
        parent.emit("error", new Error("queryString is required: (ex: (MatrixModifiedDT=2014-01-01T00:00:00.000+))"));
        return;
    }

    options = options || {};

    var queryOpts = {
        searchType:resourceType,
        class:classType,
        query:queryString,
        limit:options._limit || "NONE",
        offset:options._offset || 1,
        count: options._count || 1,
        iteration: 0
    };
    if (options._select) queryOpts.select = options._select;

    var _fetch = function(queryOpts, fetchCallback) {
        queryOpts.iteration++;
        searchRets(queryOpts, function(error, data) {
            if (error) fetchCallback(error);
            else parseCompactDecoded(data, fetchCallback);
        });
    };
    var _checkResults = function(error, searchResults) {
        if (error) {
            parent.emit("error", error);
            return;
        }
        searchResults.iteration = queryOpts.iteration;
        searchResults.offset = queryOpts.offset;
        parent.emit("data", searchResults);
        if (searchResults.count > 0) {
            if (queryOpts.offset < searchResults.total) {
                queryOpts.offset = Math.min(queryOpts.offset + searchResults.count, searchResults.total);
                //console.log("Fetched %s of %s total records", queryOpts.offset, searchResults.total);
                _fetch(queryOpts, _checkResults);
            }
        } else {
            parent.emit("end");
        }
    };
    _fetch(queryOpts, _checkResults);
};

module.exports = function(_parent, _searchSession) {

    parent = _parent;
    searchSession = _searchSession;

    return {
        searchRets:searchRets,
        query: query
    };
};

<|endoftext|>


<commit_before>var request = require('request-promise-native'),
    langs   = require('langs'),
    helpers = require('./helpers');

function search (show, season, episode, languages) {
    var headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
    };

    return request({
        uri:     helpers.addic7edURL + '/srch.php',
        qs:      {
            search: show.trim() + ' ' + helpers.formatShowNumber(season) + 'x' + helpers.formatShowNumber(episode)
        },
        headers: headers
    }).then(function (body) {
        if (/<b>\d+ results found<\/b>/.test(body)) {
            if (~body.indexOf('<b>0 results found<\/b>')) {
                // No result
                // =========
                console.log('[Search] Addic7ed.com error: No result.');
                return [];
            }

            // Multiple results
            // ================

            // Find result with proper season and episode in url
            // -------------------------------------------------
            var regexp   = new RegExp('href="(serie/[^/]+/' + parseInt(season) + '/' + parseInt(episode) + '/.+?)"'),
                urlMatch = body.match(regexp),
                url      = urlMatch && urlMatch[1];

            if (!url) {
                console.log('[Search] Addic7ed.com error: subtitles not found in a multiple result set.');
                return [];
            }

            return request({
                uri:     helpers.addic7edURL + '/' + url,
                headers: headers
            })
                .then(function (body) {
                    return findSubtitles(body, languages);
                })
                .catch(searchError);
        }

        return findSubtitles(body, languages);
    }).catch(searchError);
}

function findSubtitles (body, languages) {
    var subs          = [],
        refererMatch  = body.match(/\/show\/\d+/),
        referer       = refererMatch ? refererMatch[0] : '/show/1',
        versionRegExp = /Version (.+?),([^]+?)<\/table/g,
        versionMatch,
        version,
        hearingImpaired,
        subInfoRegExp = /class="language">([^]+?)<a[^]+?(% )?Completed[^]+?href="([^"]+?)"><strong>(?:most updated|Download)/g,
        subInfoMatch,
        lang,
        langId,
        notCompleted,
        link,
        distributionMatch,
        distribution,
        team;

    // Find subtitles HTML block parts
    // ===============================
    while ((versionMatch = versionRegExp.exec(body)) !== null) {
        version = versionMatch[1].toUpperCase();
        hearingImpaired = ~versionMatch[2].indexOf('Hearing Impaired');

        while ((subInfoMatch = subInfoRegExp.exec(versionMatch[2])) !== null) {
            notCompleted = subInfoMatch[2];
            if (notCompleted) {
                continue;
            }

            lang = subInfoMatch[1];
            // Find lang iso code 2B
            // ---------------------
            langId = langs.where('name', lang.replace(/\(.+\)/g, '').trim());
            langId = langId && langId['2B'] || lang.substr(0, 3).toLowerCase();

            link = subInfoMatch[3];

            if (languages && !~languages.indexOf(langId)) {
                continue;
            }

            distributionMatch = version.match(/HDTV|WEB(.DL|.?RIP)?|WR|BRRIP|BDRIP|BLURAY/i);

            distribution = distributionMatch
                ? distributionMatch[0].toUpperCase()
                .replace(/WEB(.DL|.?RIP)?|WR/, 'WEB-DL')
                .replace(/BRRIP|BDRIP|BLURAY/, 'BLURAY')
                : 'UNKNOWN';

            team = version.replace(/.?(REPACK|PROPER|[XH].?264|HDTV|480P|720P|1080P|2160P|AMZN|WEB(.DL|.?RIP)?|WR|BRRIP|BDRIP|BLURAY)+.?/g, '')
                    .trim().toUpperCase() || 'UNKNOWN';

            subs.push({
                lang:            lang,
                langId:          langId,
                distribution:    distribution,
                team:            team,
                version:         version,
                link:            link,
                referer:         referer,
                hearingImpaired: hearingImpaired
            });
        }
    }

    return subs;
}

function searchError (err) {
    return console.log('[Search] Addic7ed.com error', err.statusCode, err.options && err.options.qs.search);
}

module.exports = search;
<commit_msg>Ensures hearingImpaired is a boolean in search.js<commit_after>var request = require('request-promise-native'),
    langs   = require('langs'),
    helpers = require('./helpers');

function search (show, season, episode, languages) {
    var headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
    };

    return request({
        uri:     helpers.addic7edURL + '/srch.php',
        qs:      {
            search: show.trim() + ' ' + helpers.formatShowNumber(season) + 'x' + helpers.formatShowNumber(episode)
        },
        headers: headers
    }).then(function (body) {
        if (/<b>\d+ results found<\/b>/.test(body)) {
            if (~body.indexOf('<b>0 results found<\/b>')) {
                // No result
                // =========
                console.log('[Search] Addic7ed.com error: No result.');
                return [];
            }

            // Multiple results
            // ================

            // Find result with proper season and episode in url
            // -------------------------------------------------
            var regexp   = new RegExp('href="(serie/[^/]+/' + parseInt(season) + '/' + parseInt(episode) + '/.+?)"'),
                urlMatch = body.match(regexp),
                url      = urlMatch && urlMatch[1];

            if (!url) {
                console.log('[Search] Addic7ed.com error: subtitles not found in a multiple result set.');
                return [];
            }

            return request({
                uri:     helpers.addic7edURL + '/' + url,
                headers: headers
            })
                .then(function (body) {
                    return findSubtitles(body, languages);
                })
                .catch(searchError);
        }

        return findSubtitles(body, languages);
    }).catch(searchError);
}

function findSubtitles (body, languages) {
    var subs          = [],
        refererMatch  = body.match(/\/show\/\d+/),
        referer       = refererMatch ? refererMatch[0] : '/show/1',
        versionRegExp = /Version (.+?),([^]+?)<\/table/g,
        versionMatch,
        version,
        hearingImpaired,
        subInfoRegExp = /class="language">([^]+?)<a[^]+?(% )?Completed[^]+?href="([^"]+?)"><strong>(?:most updated|Download)/g,
        subInfoMatch,
        lang,
        langId,
        notCompleted,
        link,
        distributionMatch,
        distribution,
        team;

    // Find subtitles HTML block parts
    // ===============================
    while ((versionMatch = versionRegExp.exec(body)) !== null) {
        version = versionMatch[1].toUpperCase();
        hearingImpaired = versionMatch[2].indexOf('Hearing Impaired') !== -1;

        while ((subInfoMatch = subInfoRegExp.exec(versionMatch[2])) !== null) {
            notCompleted = subInfoMatch[2];
            if (notCompleted) {
                continue;
            }

            lang = subInfoMatch[1];
            // Find lang iso code 2B
            // ---------------------
            langId = langs.where('name', lang.replace(/\(.+\)/g, '').trim());
            langId = langId && langId['2B'] || lang.substr(0, 3).toLowerCase();

            link = subInfoMatch[3];

            if (languages && !~languages.indexOf(langId)) {
                continue;
            }

            distributionMatch = version.match(/HDTV|WEB(.DL|.?RIP)?|WR|BRRIP|BDRIP|BLURAY/i);

            distribution = distributionMatch
                ? distributionMatch[0].toUpperCase()
                .replace(/WEB(.DL|.?RIP)?|WR/, 'WEB-DL')
                .replace(/BRRIP|BDRIP|BLURAY/, 'BLURAY')
                : 'UNKNOWN';

            team = version.replace(/.?(REPACK|PROPER|[XH].?264|HDTV|480P|720P|1080P|2160P|AMZN|WEB(.DL|.?RIP)?|WR|BRRIP|BDRIP|BLURAY)+.?/g, '')
                    .trim().toUpperCase() || 'UNKNOWN';

            subs.push({
                lang:            lang,
                langId:          langId,
                distribution:    distribution,
                team:            team,
                version:         version,
                link:            link,
                referer:         referer,
                hearingImpaired: hearingImpaired
            });
        }
    }

    return subs;
}

function searchError (err) {
    return console.log('[Search] Addic7ed.com error', err.statusCode, err.options && err.options.qs.search);
}

module.exports = search;
<|endoftext|>


<commit_before>#!/usr/bin/env node

var debug = require("debug")("search");
var trie = require("trie-search");
var slug = require("slug");
var uniq = require("unq");
var dur = require("dur");

function search(opt, api){

	if (!(this instanceof search)) return new search(opt, api);
	var self = this;

	self.opt = opt||{};
	self.opt.reindex = (self.opt.hasOwnProperty("reindex") && self.opt.reindex) ? dur(self.opt.reindex) : dur("5m");

	self.api = api;
	self.trie = new trie(['name', 'aliases'], {min: 3, indexField: 'id'});

	self.rels = {};
	self.dtindex = {};
	self.ftindex = {};

	// reindexing
	self.index();
	self.reindex = setInterval(function(){
		self.index();
	}, self.opt.reindex);

	return this;
};

// entity autocomplete
search.prototype.autocomplete = function(str, fn){
	var self = this;
	fn(null, self.trie.get(str));
	return this;
};

// all data type sets
search.prototype.datatypes = function(fn){
	var self = this;
	fn(null, self.dtindex);
	return this;
};

// array of field types
search.prototype.fieldtypes = function(fn){
	var self = this;
	fn(null, self.ftindex);
	return this;
};

// reindex search data
search.prototype.index = function(fn){
	var self = this;
	if (typeof fn !== "function") var fn = function(){};

	// get relations for index
	self.api.rels(function(err, rels){
		if (err) debug("rel index error: %s", err);
		if (err) return fn(err);
		self.rels = rels;

		// get entities for index
		self.api.ents(function(err, result){
			if (err) debug("ent index error: %s", err);
			if (err) return fn(err);
			self.trie.reset();

			// temporary data type index
			var _dtindex = [];
			var _ftindex = {};

			// autocomplete index
			result.forEach(function(item){

				self.trie.add({
					id: item._id,
					name: item.name,
					aliases: item.aliases,
					relations: (self.rels.hasOwnProperty(item._id) ? self.rels[item._id].length : 0)
				});

				// data type index
				item.data.forEach(function(d){
					_dtindex.push(JSON.stringify({
						key: d.key,
						name: d.desc,
						format: d.format,
						mode: item.type
					}));
				});

				// field type index
				if (!_ftindex.hasOwnProperty(d.format)) _ftindex[d.format] = true

			});

			// FIXME: rel indexing

			// make unique data type index
			self.dtindex = uniq(_dtindex).map(function(d){ return JSON.parse(d); }).sort(function(a,b){
				return (a.key.localeCompare(b.key));
			});
			self.ftindex = Object.keys(_ftindex);

			debug("ent indexing done");
			fn(null);
		});
	});
	return this;
};

module.exports = search;
<commit_msg>fix field type indexing<commit_after>#!/usr/bin/env node

var debug = require("debug")("search");
var trie = require("trie-search");
var slug = require("slug");
var uniq = require("unq");
var dur = require("dur");

function search(opt, api){

	if (!(this instanceof search)) return new search(opt, api);
	var self = this;

	self.opt = opt||{};
	self.opt.reindex = (self.opt.hasOwnProperty("reindex") && self.opt.reindex) ? dur(self.opt.reindex) : dur("5m");

	self.api = api;
	self.trie = new trie(['name', 'aliases'], {min: 3, indexField: 'id'});

	self.rels = {};
	self.dtindex = {};
	self.ftindex = {};

	// reindexing
	self.index();
	self.reindex = setInterval(function(){
		self.index();
	}, self.opt.reindex);

	return this;
};

// entity autocomplete
search.prototype.autocomplete = function(str, fn){
	var self = this;
	fn(null, self.trie.get(str));
	return this;
};

// all data type sets
search.prototype.datatypes = function(fn){
	var self = this;
	fn(null, self.dtindex);
	return this;
};

// array of field types
search.prototype.fieldtypes = function(fn){
	var self = this;
	fn(null, self.ftindex);
	return this;
};

// reindex search data
search.prototype.index = function(fn){
	var self = this;
	if (typeof fn !== "function") var fn = function(){};

	// get relations for index
	self.api.rels(function(err, rels){
		if (err) debug("rel index error: %s", err);
		if (err) return fn(err);
		self.rels = rels;

		// get entities for index
		self.api.ents(function(err, result){
			if (err) debug("ent index error: %s", err);
			if (err) return fn(err);
			self.trie.reset();

			// temporary data type index
			var _dtindex = [];
			var _ftindex = {};

			// autocomplete index
			result.forEach(function(item){

				self.trie.add({
					id: item._id,
					name: item.name,
					aliases: item.aliases,
					relations: (self.rels.hasOwnProperty(item._id) ? self.rels[item._id].length : 0)
				});

				// data type index
				item.data.forEach(function(d){
					_dtindex.push(JSON.stringify({
						key: d.key,
						name: d.desc,
						format: d.format,
						mode: item.type
					}));

					// field type index
					if (!_ftindex.hasOwnProperty(d.format)) _ftindex[d.format] = true;

				});

			});

			// FIXME: rel indexing

			// make unique data type index
			self.dtindex = uniq(_dtindex).map(function(d){ return JSON.parse(d); }).sort(function(a,b){
				return (a.key.localeCompare(b.key));
			});
			self.ftindex = Object.keys(_ftindex);

			debug("ent indexing done");
			fn(null);
		});
	});
	return this;
};

module.exports = search;
<|endoftext|>


<commit_before>var es = require('elasticsearch');
var _  = require('lodash');

/**
 * # Search
 */

/**
 * Search the model based off of a elasticsearch Query string
 * @param  {String}   query   based of elasitcsarch query string
 * @param  {Object}   options Options for the es call
 * @param  {Function} callback  Optional callback
 * @return {Promise}       Returns a promise unless there is a callback
 */
exports.searchQuery = function (query, opt, cb) {
  var params = _.merge({
    index: this.index,
    type: this.type,
    q: query
  }, opt);

  return this
    .connection
    .search(params, cb);
}

/**
 * Search the model based of a elasticsearch body object
 * @param  {Object}   body object with more search options
 * @param  {Object}   options Options for the es call
 * @param  {Function} callback  Optional callback
 * @return {Promise}       Returns a promise unless there is a callback
 */
exports.searchBody = function (body, opt, cb) {
  var params = _.merge({
    index: this.index,
    type: this.type,
    body: body
  }, opt);

  return this
    .connection
    .search(params, cb);
}

/**
 * Get document by document Id
 * @param  {String|Number|ElasitcsearchId}   id  document id
 * @param  {Object}   options Options for the es call
 * @param  {Function} callback  Optional callback
 * @return {Promise}       Returns a promise unless there is a callback
 */
exports.get = function (id, opt, cb) {
  if (!opt) opt = {};

  return this
    .connection
    .get(_.merge(
      {
        index: this.index,
        type: this.type,
        id: id
      }, opt),
      cb
    );
}
<commit_msg>adding dynamic mapping options<commit_after>var es = require('elasticsearch');
var _  = require('lodash');

/**
 * # Search
 */

/**
 * Search the model based off of a elasticsearch Query string
 * @param  {String}   query   based of elasitcsarch query string
 * @param  {Object}   options Options for the es call
 * @param  {Function} callback  Optional callback
 * @return {Promise}       Returns a promise unless there is a callback
 */
exports.searchQuery = function (query, opt, cb) {
  var params = _.merge({
    index: this.index,
    type: this.type,
    q: query
  }, opt);

  return this
    .connection
    .search(params, cb);
}

/**
 * Search the model based of a elasticsearch body object
 * @param  {Object}   body object with more search options
 * @param  {Object}   options Options for the es call
 * @param  {Function} callback  Optional callback
 * @return {Promise}       Returns a promise unless there is a callback
 */
exports.searchBody = function (body, opt, cb) {
  var params = _.merge({
    index: this.index,
    type: this.type,
    body: body
  }, opt);

  return this
    .connection
    .search(params, cb);
}

/**
 * Scroll the model based of a elasticsearch body object
 * @param  {Object}   opt Object containing scrollId and scroll
 * @param  {Function} callback  Optional callback
 * @return {Promise}       Returns a promise unless there is a callback
 */
exports.scroll = function (opt, cb) {
  return this
      .connection
      .scroll(opt, cb);
}

/**
 * Get document by document Id
 * @param  {String|Number|ElasitcsearchId}   id  document id
 * @param  {Object}   options Options for the es call
 * @param  {Function} callback  Optional callback
 * @return {Promise}       Returns a promise unless there is a callback
 */
exports.get = function (id, opt, cb) {
  if (!opt) opt = {};

  return this
    .connection
    .get(_.merge(
      {
        index: this.index,
        type: this.type,
        id: id
      }, opt),
      cb
    );
}
<|endoftext|>


<commit_before>
seajs.config({
  base: './combo-map/'
})

seajs.on('fetch', function(data) {
  data.requestUri = data.uri.replace(/\/(a|b|c)\.js/, '/combo.js')
})

define(function(require) {

  var test = require('../../../test')
  var a = require('a')
  var b = require('b')
  var c = require('c')

  test.assert(a.name == 'a', a.name)
  test.assert(b.name == 'b', b.name)
  test.assert(c === null, 'return null when c is not found')

  test.next()

})
<commit_msg>Rename data argument to args<commit_after>
seajs.config({
  base: './combo-map/'
})

seajs.on('fetch', function(args) {
  args.requestUri = args.uri.replace(/\/(a|b|c)\.js/, '/combo.js')
})

define(function(require) {

  var test = require('../../../test')
  var a = require('a')
  var b = require('b')
  var c = require('c')

  test.assert(a.name == 'a', a.name)
  test.assert(b.name == 'b', b.name)
  test.assert(c === null, 'return null when c is not found')

  test.next()

})
<|endoftext|>


<commit_before>const inRange = (val, min, max) => (val >= min && val <= max)

export const anglePropType = (props, propName, component) => {
  const prop = props[propName], min = 0, max = 360
  if (!inRange(prop, min, max) || (prop !== "vertical" && prop !== "horizontal")) {
    return new Error(`Invalid angle in propType: ${propName}.
                      The angle must be horizontal, vertical or within ${min} and ${max}`)
  }
}

export const rangePropType = (props, propName, component) => {
  const prop = props[propName], min = 0, max = 100
  if (isNaN(prop) || !inRange(prop, min, max)) {
    return new Error(`Invalid number in propType: ${propName}.
                      The number must be within ${min} and ${max}`)
  }
}
<commit_msg>Fix bug in propTypes validation<commit_after>const inRange = (val, range) => (val >= range[0] && val <= range[1])

export const anglePropType = (props, propName, component) => {
  const prop = props[propName],
        range = [0, 360]
  if (!inRange(prop, range) && (prop !== "vertical" && prop !== "horizontal")) {
    return new Error(`Invalid angle in propType: ${propName}.
                      The angle must be horizontal, vertical or within ${range[0]} and ${range[1]}`)
  }
}

export const rangePropType = (props, propName, component) => {
  const prop = props[propName],
        range = [0, 100]
  if (isNaN(prop) || !inRange(prop, range)) {
    return new Error(`Invalid number in propType: ${propName}.
                      The number must be within ${range[0]} and ${range[1]}`)
  }
}
<|endoftext|>


<commit_before>(function(undefined) {
    'use strict';

    var instance;

    var app = module.exports = function(options) {
        if(instance) return instance;

        /**
         * Module dependencies.
         */
        var express = require('express')
          , http = require('http')
          , path = require('path');

        var app = instance = express();

        options = options || {};

        app.configure(function(){
          app.set('port', options.port || process.env.PORT || 3000);
          app.set('views', __dirname + '/views');
          app.set('view engine', options.engine || 'jade');
          app.use(express.favicon());
          app.use(express.logger('dev'));
          app.use(express.bodyParser());
          app.use(express.methodOverride());
          app.use(app.router);
          app.use(express.static(options.base && path.resolve(options.base) || process.cwd()));
        });

        app.configure('development', function(){
          app.use(express.errorHandler());
        });

        http.createServer(app).listen(app.get('port'), function(){
          console.log("Express server listening on port " + app.get('port'));
        });

        return instance;
    };

    var api = {

            start: function(options) {
                return app(options);
            }

        };

    module.exports = function(cmd, options) {
        cmd = api[cmd];
        return cmd && cmd(options) || this;
    };

})(undefined);<commit_msg>Don't use double exports...<commit_after>(function(undefined) {
    'use strict';

    var instance,
        app = function(options) {
            if(instance) return instance;

            /**
             * Module dependencies.
             */
            var express = require('express')
              , http = require('http')
              , path = require('path');

            var app = instance = express();

            options = options || {};

            app.configure(function(){
              app.set('port', options.port || process.env.PORT || 3000);
              app.set('views', __dirname + '/views');
              app.set('view engine', options.engine || 'jade');
              app.use(express.favicon());
              app.use(express.logger('dev'));
              app.use(express.bodyParser());
              app.use(express.methodOverride());
              app.use(app.router);
              app.use(express.static(options.base && path.resolve(options.base) || process.cwd()));
            });

            app.configure('development', function(){
              app.use(express.errorHandler());
            });

            http.createServer(app).listen(app.get('port'), function(){
              console.log("Express server listening on port " + app.get('port'));
            });

            return instance;
        };

    var api = {

            start: function(options) {
                return app(options);
            }

        };

    module.exports = function(cmd, options) {
        cmd = api[cmd];
        return cmd && cmd(options) || this;
    };

})(undefined);<|endoftext|>


<commit_before>var WS = require('ws').Server
var server = new WS({ port: 3000 })
var Device = require('ev3-js-device')
var devices = require('ev3-js-devices')

var motor = new Device(devices('c'))
var motor2 = new Device(devices('b'))
var ir = new Device(devices(1))
var sonic = new Device(devices(2))
var touch = new Device(devices(3))
var color = new Device(devices(4))

var running = false
var interval
var responses = {}

function readSensors () {
  responses.motor = motor.read('position')
  responses.motor2 = motor2.read('position')
  responses.ir = ir.read('value0')
  responses.sonic = sonic.read('value0')
  responses.touch = touch.read('value0')
  responses.color = color.read('value0')
}

server.on('connection', function (socket) {
  console.log('connected')
  setInterval(function () {
    socket.send(JSON.stringify(responses))
  }, 100)
  if (!running) {
    running = true
    interval = setInterval(readSensors, 100)
  }
})

server.on('close', function () {
  clearInterval(interval)
  running = false
})
<commit_msg>stringify response<commit_after>var WS = require('ws').Server
var server = new WS({ port: 3000 })
var Device = require('ev3-js-device')
var devices = require('ev3-js-devices')

var motor = new Device(devices('c'))
var motor2 = new Device(devices('b'))
var ir = new Device(devices(1))
var sonic = new Device(devices(2))
var touch = new Device(devices(3))
var color = new Device(devices(4))

var running = false
var readInterval
var sendInterval
var responses = {}

function readSensors () {
  responses.motor = motor.read('position')
  responses.motor2 = motor2.read('position')
  responses.ir = ir.read('value0')
  responses.sonic = sonic.read('value0')
  responses.touch = touch.read('value0')
  responses.color = color.read('value0')
}

server.on('connection', function (socket) {
  console.log('connected')
  sendInterval = setInterval(function () {
    socket.send(JSON.stringify(responses))
  }, 100)
  if (!running) {
    running = true
    interval = setInterval(readSensors, 100)
  }
})

server.on('close', function () {
  clearInterval(readInterval)
  clearInterval(sendInterval)
  running = false
})
<|endoftext|>


<commit_before>'use strict';

var express = require('express'),
    bodyParser = require('body-parser'),
    async = require('async'),
    glob = require('glob'),
    path = require('path'),
    fs = require('fs'),
    fsq = require('./fsq'),
    watchServer = require('./watch');

require('colors');

var root = process.cwd() + '/src';

var PORT = 3000,
    WATCH = false;

var app = express();

app.use(bodyParser.urlencoded({ extended: true, limit: '5mb' }));
app.use(bodyParser.json({ limit: '5mb' }));
app.use(express.static(root));

function id (x) { return x; }

function render (template, res, injector) {

    injector = injector || id;

    fsq.readFile(template)
        .then(function (html) {

            return injector(html, PORT);
        })
        .then(function (html) {

            res.writeHeader(200, {'Content-Type': 'text/html'});
            res.write(html);
            res.end();
        })
        .catch(function (err) { throw err; });
}

app.get('/', function (req, res) {

    render('index.html', res, WATCH && watchServer.inject);
});

app.get('/editor', function (req, res) {

    render('weltmeister.html', res);
});

app.get('/api/glob', function (req, res) {

    async.reduce(req.query.glob, [], function (memo, item, callback) {

        glob(item, {cwd: root}, function (e, matches) {

            for (var i in matches) { memo.push(matches[i]); }

            callback(null, memo);
        });
    },
    function (err, result) {

        res.send(result);
    });
});

app.post('/api/save', function (req, res) {

    var reqPath = req.body.path,
        data = req.body.data;

    reqPath && data ?

        /\.js$/.test(reqPath) ?

            fsq.outputFile(root + '/' + reqPath, data)
                .then(function() {

                    res.send({error: 0});
                })
                .catch(function() {

                    res.send({error: 2, msg: 'Couldn\'t write to file: ' + reqPath});
                })

            :

            res.send({error: 3, msg: 'File must have a .js suffix'})

        :

        res.send({error: 1, msg: 'No Data or Path specified'});
});

app.get('/api/browse', function (req, res) {

    var dir = req.query.dir || '',
        type = req.query.type,
        types = {scripts: ['.js'], images: ['.png', '.gif', '.jpg', '.jpeg']},
        result = {dirs: [], files: []},
        filter, stats, dirpath;

    filter = (type && types[type]) ? types[type] : false;

    result.parent = dir ? dir.substring(0, dir.lastIndexOf('/')) : false;

    dir[dir.length - 1] === '/' && (dir = dir.substring(0, dir.length - 1));

    dir += '/';

    dirpath = path.normalize(path.join(root, dir));

    fs.readdir(dirpath, function (err, files) {

        for (var i in files) {

            var resPath = (dir !== '/' ? dir : '') + files[i];

            stats = fs.statSync(dirpath + files[i]);

            stats.isDirectory() ?
                result.dirs.push(resPath) :
                stats.isFile() &&
                    filter ?
                        filter.indexOf(path.extname(files[i])) >= 0 && result.files.push(resPath) :
                        result.files.push(resPath);
        }

        res.send(result);
    });
});

module.exports = {

    run: function (port, watch) {

        PORT = Number(port[1]) || PORT;
        WATCH = watch[0];

        var server = app.listen(PORT);

        console.log('impact-node server is listening at '.green +
                    ('http://localhost:' + PORT).yellow +
                    ' Editor: '.green +
                    ('http://localhost:' + PORT + '/editor').yellow);

        if (WATCH) { watchServer.run(server); }
    }
};
<commit_msg>fix watch mode<commit_after>'use strict';

var express = require('express'),
    bodyParser = require('body-parser'),
    async = require('async'),
    glob = require('glob'),
    path = require('path'),
    fs = require('fs'),
    fsq = require('./fsq'),
    watchServer = require('./watch');

require('colors');

var root = process.cwd() + '/src';

var PORT = 3000,
    WATCH = false;

var app = express();

app.use(bodyParser.urlencoded({ extended: true, limit: '5mb' }));
app.use(bodyParser.json({ limit: '5mb' }));
app.use(express.static(root));

function id (x) { return x; }

function render (template, res, injector) {

    injector = injector || id;

    fsq.readFile(template, 'utf-8')
        .then(function (html) {

            return injector(html, PORT);
        })
        .then(function (html) {

            res.writeHeader(200, {'Content-Type': 'text/html'});
            res.write(html);
            res.end();
        })
        .catch(function (err) { throw err; });
}

app.get('/', function (req, res) {

    render('index.html', res, WATCH && watchServer.inject);
});

app.get('/editor', function (req, res) {

    render('weltmeister.html', res);
});

app.get('/api/glob', function (req, res) {

    async.reduce(req.query.glob, [], function (memo, item, callback) {

        glob(item, {cwd: root}, function (e, matches) {

            for (var i in matches) { memo.push(matches[i]); }

            callback(null, memo);
        });
    },
    function (err, result) {

        res.send(result);
    });
});

app.post('/api/save', function (req, res) {

    var reqPath = req.body.path,
        data = req.body.data;

    reqPath && data ?

        /\.js$/.test(reqPath) ?

            fsq.outputFile(root + '/' + reqPath, data)
                .then(function() {

                    res.send({error: 0});
                })
                .catch(function() {

                    res.send({error: 2, msg: 'Couldn\'t write to file: ' + reqPath});
                })

            :

            res.send({error: 3, msg: 'File must have a .js suffix'})

        :

        res.send({error: 1, msg: 'No Data or Path specified'});
});

app.get('/api/browse', function (req, res) {

    var dir = req.query.dir || '',
        type = req.query.type,
        types = {scripts: ['.js'], images: ['.png', '.gif', '.jpg', '.jpeg']},
        result = {dirs: [], files: []},
        filter, stats, dirpath;

    filter = (type && types[type]) ? types[type] : false;

    result.parent = dir ? dir.substring(0, dir.lastIndexOf('/')) : false;

    dir[dir.length - 1] === '/' && (dir = dir.substring(0, dir.length - 1));

    dir += '/';

    dirpath = path.normalize(path.join(root, dir));

    fs.readdir(dirpath, function (err, files) {

        for (var i in files) {

            var resPath = (dir !== '/' ? dir : '') + files[i];

            stats = fs.statSync(dirpath + files[i]);

            stats.isDirectory() ?
                result.dirs.push(resPath) :
                stats.isFile() &&
                    filter ?
                        filter.indexOf(path.extname(files[i])) >= 0 && result.files.push(resPath) :
                        result.files.push(resPath);
        }

        res.send(result);
    });
});

module.exports = {

    run: function (port, watch) {

        PORT = Number(port[1]) || PORT;
        WATCH = watch[0];

        var server = app.listen(PORT);

        console.log('impact-node server is listening at '.green +
                    ('http://localhost:' + PORT).yellow +
                    ' Editor: '.green +
                    ('http://localhost:' + PORT + '/editor').yellow);

        if (WATCH) { watchServer.run(server); }
    }
};
<|endoftext|>


<commit_before>var http = require('http');
var https = require('https');
var express = require('express');
var socket_io = require('socket.io');
var stitchit = require('stitchit');
var esSocketHandler = require('./socket_handler');

var ES_CLIENT_PATH= __dirname + '/../node_modules/es_client';
var LAYOUT_PATH = __dirname + '/layouts';

// Note: templates should be fixed to load using require in the client
// and not need the server to be aware of them
var TEMPLATE_PATH = ES_CLIENT_PATH + '/templates/';
var TEMPLATE_NAMESPACE = 'module.exports';

exports.createServer = function(config){

  /***********************************************
  * EtherSheet HTTP Server
  ***********************************************/
  var app = express();
  var server = http.createServer(app);

  // Server Settings
  app.set('views',LAYOUT_PATH);
  app.use(express.logger({ format: ':method :url' }));
  app.use('/es_client',express.static(ES_CLIENT_PATH));

  // listen after setup
  process.nextTick(function(){
    server.listen(config.port, config.host, function(){
      console.log('ethersheet is listening over https on port ' + config.port);
    });
  });

  /**********************************************
  * HTTP Routes
  *********************************************/
  //index
  app.get('/', function(req,res){
    res.render('index.ejs');
  });

  //load up the sheet
  app.get('/s/:sheetid', function(req,res){
    res.render('sheet.ejs',{
      sheet_id:req.params.sheetid
    });
  });

  // client-side templates concatenated together
  app.get('/es_client/templates.js', function(req,res){
    stitchit({path:TEMPLATE_PATH,namespace:TEMPLATE_NAMESPACE},function(err,templates){
      if(err) throw err;

      templates =
        "if (typeof define !== 'function') { var define = require('amdefine')(module) }\n"+
        "define( function(require,exports,module){\n\n"+
        "var _ = require('underscore');\n"+
        "var helpers = require('es_client/helpers');\n"+
        templates+
        "\n});\n";

      res.send(templates);
    });
  });

  /***********************************************
  * Socket.io
  ***********************************************/
  var io = socket_io.listen(server);

  io.sockets.on('connection', function(socket){
    esSocketHandler(socket,io,config);
  });

  return app;
}
<commit_msg>no longer generating templates for es_client<commit_after>var http = require('http');
var https = require('https');
var express = require('express');
var socket_io = require('socket.io');
var stitchit = require('stitchit');
var esSocketHandler = require('./socket_handler');

var ES_CLIENT_PATH= __dirname + '/../node_modules/es_client';
var LAYOUT_PATH = __dirname + '/layouts';

exports.createServer = function(config){

  /***********************************************
  * EtherSheet HTTP Server
  ***********************************************/
  var app = express();
  var server = http.createServer(app);

  // Server Settings
  app.set('views',LAYOUT_PATH);
  app.use(express.logger({ format: ':method :url' }));
  app.use('/es_client',express.static(ES_CLIENT_PATH));

  // listen after setup
  process.nextTick(function(){
    server.listen(config.port, config.host, function(){
      console.log('ethersheet is listening over https on port ' + config.port);
    });
  });

  /**********************************************
  * HTTP Routes
  *********************************************/
  //index
  app.get('/', function(req,res){
    res.render('index.ejs');
  });

  //load up the sheet
  app.get('/s/:sheetid', function(req,res){
    res.render('sheet.ejs',{
      sheet_id:req.params.sheetid
    });
  });

  /***********************************************
  * Socket.io
  ***********************************************/
  var io = socket_io.listen(server);

  io.sockets.on('connection', function(socket){
    esSocketHandler(socket,io,config);
  });

  return app;
}
<|endoftext|>