<commit_before>const getOptions = require('./getOptions')
const app = require('express')()
const worker = require('ddv-worker')
const http = require('http')
const fs = require('fs')
const path = require('path')
const {Nuxt} = require('nuxt')
const logger = require('./logger')
const nuxts = Object.create(null)
const templates = Object.create(null)
const hostToTemplateId = Object.create(null)
const vhostOptions = Object.create(null)

// 创建http服务
worker.server = http.createServer(app)
module.exports = worker
app.use(function (req, res, next) {
  return vhostRender.apply(this, arguments)
})
// 渲染中间件
function vhostRender (req, res, next) {
  var host = req.hostname || req.headers.host

  getTemplateIdByHost(host)
    .then(templateId => {
      getNuxtRender(templateId)
        .then(render => {
          render.apply(this, arguments)
        })
    })
    .catch(e => {
      echoErrorByRes(e, res)
    })
}
function echoErrorByRes (e, res) {
  res.writeHead(200, {'Content-Type': 'application/json; charset=utf8', 'Access-Control-Allow-Origin': '*', 'Accept-Language': 'zh-CN'})// 响应头
  res.write('错误-:' + e.message)
  res.write('错误stack:' + e.stack)
  res.end('错误body:' + e.body)
}
// 获取编译器的方法
function getNuxtRender (templateId) {
  if (nuxts[templateId]) {
    return Promise.resolve(nuxts[templateId])
  } else {
    return createNuxtRender(templateId)
      .then(nuxtRender => {
        nuxts[templateId] = nuxtRender
        return nuxts[templateId]
      })
  }
}
// function closeNuxtRender (templateId) {
//   return getNuxtRender(templateId)
//     .then(render => {
//       delete nuxts[templateId]
//       return render.close()
//     })
// }
// 创建编译器的方法
function createNuxtRender (templateId) {
  return getTemplateInfo(templateId)
    .then(info => {
      return getOptions(info.configFile)
        .then(options => {
          options.rootDir = options.rootDir || info.rootDir
          return options
        })
    })
    .then(options => {
      let distDir = path.resolve(
        options.rootDir,
        options.buildDir || '.nuxt',
        'dist'
      )
      return new Promise((resolve, reject) => {
        fs.stat(distDir, function (err, stats) {
          var isError = false
          if (err) {
            isError = true
          } else if (!stats.isDirectory) {
            isError = true
          }

          if (isError) {
            resolve(options)
          } else {
            reject(new Error(
              'No build files found, please run `nuxt build` before launching `nuxt start`'
            ))
          }
        })
      })
    })
    .then(options => {
      let nuxt = new Nuxt(options)
      return nuxt.render
    })
}
// 获取模板信息
function getTemplateInfo (templateId) {
  if (templates[templateId]) {
    return Promise.resolve(templates[templateId])
  }
  return loadTemplateByIdRun(templateId)
    .then(res => {
      templates[templateId] = res
      return templates[templateId]
    })
}

function getTemplateIdByHost (host) {
  if (hostToTemplateId[host]) {
    return Promise.resolve(hostToTemplateId[host])
  }
  return loadTemplateIdByHost(host)
    .then(templateId => {
      hostToTemplateId[host] = templateId
      return hostToTemplateId[host]
    })
}
function loadTemplateIdByHost (host) {
  if (vhostOptions.loadTemplateIdByHost && typeof vhostOptions.loadTemplateIdByHost === 'function') {
    return vhostOptions.loadTemplateIdByHost(host, vhostOptions)
  } else {
    return Promise.reject(new Error('vhostOptions not find loadTemplateIdByHost'))
  }
}
function loadTemplateByIdRun (templateId) {
  if (vhostOptions.loadTemplateById && typeof vhostOptions.loadTemplateById === 'function') {
    return vhostOptions.loadTemplateById(templateId, vhostOptions)
  } else {
    return Promise.reject(new Error('vhostOptions not find loadTemplateById'))
  }
}

worker.serverStart = function serverStart (config) {
  config.getConfig()
    .then(options => {
      Object.keys(vhostOptions).forEach(key => {
        if (options[key]) {
          vhostOptions[key] = options[key]
        } else {
          delete vhostOptions[key]
        }
      })
      Object.keys(options).forEach(key => {
        if (options[key]) {
          vhostOptions[key] = options[key]
        }
      })
      // 监听服务 - Listen the server
      worker.updateServerConf({
        defaultListen: options.defaultListen,
        listen: options.listen,
        cpuLen: options.cpuLen,
        notMasterListen: {
          port: config.argv.port || 3000,
          hostname: config.argv.hostname || void 0
        }
      }).then(() => {
        logger.log('监听配置参数 更新成功')
      }, e => {
        logger.error('监听配置参数 更新失败')
        logger.error(e)
      })
    })
}
<commit_msg>isError<commit_after>const getOptions = require('./getOptions')
const app = require('express')()
const worker = require('ddv-worker')
const http = require('http')
const fs = require('fs')
const path = require('path')
const {Nuxt} = require('nuxt')
const logger = require('./logger')
const nuxts = Object.create(null)
const templates = Object.create(null)
const hostToTemplateId = Object.create(null)
const vhostOptions = Object.create(null)

// 创建http服务
worker.server = http.createServer(app)
module.exports = worker
app.use(function (req, res, next) {
  return vhostRender.apply(this, arguments)
})
// 渲染中间件
function vhostRender (req, res, next) {
  var host = req.hostname || req.headers.host

  getTemplateIdByHost(host)
    .then(templateId => {
      getNuxtRender(templateId)
        .then(render => {
          render.apply(this, arguments)
        })
    })
    .catch(e => {
      echoErrorByRes(e, res)
    })
}
function echoErrorByRes (e, res) {
  res.writeHead(200, {'Content-Type': 'application/json; charset=utf8', 'Access-Control-Allow-Origin': '*', 'Accept-Language': 'zh-CN'})// 响应头
  res.write('错误-:' + e.message)
  res.write('错误stack:' + e.stack)
  res.end('错误body:' + e.body)
}
// 获取编译器的方法
function getNuxtRender (templateId) {
  if (nuxts[templateId]) {
    return Promise.resolve(nuxts[templateId])
  } else {
    return createNuxtRender(templateId)
      .then(nuxtRender => {
        nuxts[templateId] = nuxtRender
        return nuxts[templateId]
      })
  }
}
// function closeNuxtRender (templateId) {
//   return getNuxtRender(templateId)
//     .then(render => {
//       delete nuxts[templateId]
//       return render.close()
//     })
// }
// 创建编译器的方法
function createNuxtRender (templateId) {
  return getTemplateInfo(templateId)
    .then(info => {
      return getOptions(info.configFile)
        .then(options => {
          options.rootDir = options.rootDir || info.rootDir
          return options
        })
    })
    .then(options => {
      let distDir = path.resolve(
        options.rootDir,
        options.buildDir || '.nuxt',
        'dist'
      )
      return new Promise((resolve, reject) => {
        fs.stat(distDir, function (err, stats) {
          var isError = false
          if (err) {
            isError = true
          } else if (!stats.isDirectory) {
            isError = true
          }

          if (isError) {
            reject(new Error(
              'No build files found, please run `nuxt build` before launching `nuxt start`'
            ))
          } else {
            resolve(options)
          }
        })
      })
    })
    .then(options => {
      let nuxt = new Nuxt(options)
      return nuxt.render
    })
}
// 获取模板信息
function getTemplateInfo (templateId) {
  if (templates[templateId]) {
    return Promise.resolve(templates[templateId])
  }
  return loadTemplateByIdRun(templateId)
    .then(res => {
      templates[templateId] = res
      return templates[templateId]
    })
}

function getTemplateIdByHost (host) {
  if (hostToTemplateId[host]) {
    return Promise.resolve(hostToTemplateId[host])
  }
  return loadTemplateIdByHost(host)
    .then(templateId => {
      hostToTemplateId[host] = templateId
      return hostToTemplateId[host]
    })
}
function loadTemplateIdByHost (host) {
  if (vhostOptions.loadTemplateIdByHost && typeof vhostOptions.loadTemplateIdByHost === 'function') {
    return vhostOptions.loadTemplateIdByHost(host, vhostOptions)
  } else {
    return Promise.reject(new Error('vhostOptions not find loadTemplateIdByHost'))
  }
}
function loadTemplateByIdRun (templateId) {
  if (vhostOptions.loadTemplateById && typeof vhostOptions.loadTemplateById === 'function') {
    return vhostOptions.loadTemplateById(templateId, vhostOptions)
  } else {
    return Promise.reject(new Error('vhostOptions not find loadTemplateById'))
  }
}

worker.serverStart = function serverStart (config) {
  config.getConfig()
    .then(options => {
      Object.keys(vhostOptions).forEach(key => {
        if (options[key]) {
          vhostOptions[key] = options[key]
        } else {
          delete vhostOptions[key]
        }
      })
      Object.keys(options).forEach(key => {
        if (options[key]) {
          vhostOptions[key] = options[key]
        }
      })
      // 监听服务 - Listen the server
      worker.updateServerConf({
        defaultListen: options.defaultListen,
        listen: options.listen,
        cpuLen: options.cpuLen,
        notMasterListen: {
          port: config.argv.port || 3000,
          hostname: config.argv.hostname || void 0
        }
      }).then(() => {
        logger.log('监听配置参数 更新成功')
      }, e => {
        logger.error('监听配置参数 更新失败')
        logger.error(e)
      })
    })
}
<|endoftext|>


<commit_before>/*!
 * Module dependencies.
 */
var join = require('path').join
  , querystring = require('querystring')
  ;

function Queue(path, api) {
  this.path = path;
  this.api = api;
};

Queue.prototype.info = function(fn) {
  this.api.get(this.path, fn);
};

Queue.prototype.destroy = function(fn) {
  this.api.del(this.path, fn);
};

Queue.prototype.clear = function(fn) {
  this.api.post(join(this.path, 'clear'), null, fn);
};

Queue.prototype.post = function(messages, fn) {
  // convenience
  if (typeof messages === 'string') {
    messages = [{ body: messages }];
  } else if (!Array.isArray(messages)) {
    messages = [messages];
  }
  var body = { messages: messages };
  this.api.post(join(this.path, 'messages'), body, fn);
};

Queue.prototype.get = function(params, fn) {
  var msgPath = join(this.path, 'messages');
  var returnsSingleMessage = false;

  if (typeof params === 'function') {
    fn = params;
    params = {};
  } else {
    msgPath += '?' + querystring.stringify(params);
  }
  var self = this;
  this.api.get(msgPath, function(err, body) {
    if (err) return fn(err);
    var messages = body.messages.map(function(message) {
      // convenience
      message.del = function(fn) {
        self.del(this.id, fn);
      };
      return message;
    });

    if (typeof params.n === 'undefined'
      || params.n === 1) {
      return fn(null, messages[0]);
    }
    fn(null, messages);
  });
};

Queue.prototype.del = function(id, fn) {
  this.api.del(join(this.path, 'messages', id.toString()), fn);
};

/*!
 * Module exports.
 */
module.exports = Queue;
<commit_msg>removed uncessary variable in queue<commit_after>/*!
 * Module dependencies.
 */
var join = require('path').join
  , querystring = require('querystring')
  ;

function Queue(path, api) {
  this.path = path;
  this.api = api;
};

Queue.prototype.info = function(fn) {
  this.api.get(this.path, fn);
};

Queue.prototype.destroy = function(fn) {
  this.api.del(this.path, fn);
};

Queue.prototype.clear = function(fn) {
  this.api.post(join(this.path, 'clear'), null, fn);
};

Queue.prototype.post = function(messages, fn) {
  // convenience
  if (typeof messages === 'string') {
    messages = [{ body: messages }];
  } else if (!Array.isArray(messages)) {
    messages = [messages];
  }
  var body = { messages: messages };
  this.api.post(join(this.path, 'messages'), body, fn);
};

Queue.prototype.get = function(params, fn) {
  var msgPath = join(this.path, 'messages');

  if (typeof params === 'function') {
    fn = params;
    params = {};
  } else {
    msgPath += '?' + querystring.stringify(params);
  }
  var self = this;
  this.api.get(msgPath, function(err, body) {
    if (err) return fn(err);
    var messages = body.messages.map(function(message) {
      // convenience
      message.del = function(fn) {
        self.del(this.id, fn);
      };
      return message;
    });

    if (typeof params.n === 'undefined'
      || params.n === 1) {
      return fn(null, messages[0]);
    }
    fn(null, messages);
  });
};

Queue.prototype.del = function(id, fn) {
  this.api.del(join(this.path, 'messages', id.toString()), fn);
};

/*!
 * Module exports.
 */
module.exports = Queue;
<|endoftext|>


<commit_before>'use babel'

/* @flow */

declare module 'debugger' {

  declare type FunctionBreakpoint = { function: string }
  declare type LineBreakpoint     = { filePath: string, bufferRow: number }
  declare type BreakpointLocation = LineBreakpoint | FunctionBreakpoint

  declare interface Breakpoint {
    location:        BreakpointLocation;
    enabled:         boolean;
    condition:       ?string;
    activeBufferRow: ?number;

    constructor(location: BreakpointLocation, condition?: string): void;

    getLocation(): BreakpointLocation;

    isEnabled(): bool;

    equals(other: Breakpoint): bool;
  }

  declare type BreakpointEventType  = 'inserted' | 'removed' | 'enabled' |
                                      'disabled' | 'moved' |
                                      'condition-added' | 'condition-removed'

  declare interface BreakpointEvent {
    type:       BreakpointEventType;
    breakpoint: Breakpoint;
    bufferRow:  ?number;

    constructor(
      type: BreakpointEventType, breakpoint: Breakpoint, bufferRow?: number): void;
  }

  declare type SessionEventType         = 'launched' | 'will-terminate' |
                                          'terminated' | 'resumed' | 'suspended'
  declare type SessionTerminationReason = 'normally' | 'interrupt'
  declare type SessionSuspensionReason  = 'breakpoint' | 'step'

  declare type ExecutionLine = { filePath: string, bufferRow: number }

  declare class SessionEvent {
    type:          SessionEventType;
    reason:        SessionTerminationReason | SessionSuspensionReason;
    executionLine: ExecutionLine;

    constructor(
      type:           SessionEventType,
      reason?:        SessionTerminationReason | SessionSuspensionReason,
      executionLine?: ExecutionLine): void;
  }

  declare interface EventDefs {
    BreakpointEvent: BreakpointEvent;
    SessionEvent: SessionEvent;
  }

  declare interface DebuggerTarget  {
    filePath: string,
    args: ?string[]
  }

  declare interface DebuggerProxy {

    onSessionEvent(callback: ((event: SessionEvent) => void)): Disposable;

    onBreakpointEvent(callback: ((event: BreakpointEvent) => void)): Disposable;

    findBreakpoint(location: BreakpointLocation): ?Breakpoint;

    removeBreakpoint(breakpoint: Breakpoint): boolean;
  }

  declare interface DebuggerRegistry {

    getDebuggerProxy(): DebuggerProxy;
  }

  declare interface DebuggerController {

    debuggerRegistry: DebuggerRegistry;
  }
}
<commit_msg>Expose toHumanized function of Breakpoint<commit_after>'use babel'

/* @flow */

declare module 'debugger' {

  declare type FunctionBreakpoint = { function: string }
  declare type LineBreakpoint     = { filePath: string, bufferRow: number }
  declare type BreakpointLocation = LineBreakpoint | FunctionBreakpoint

  declare interface Breakpoint {
    location:        BreakpointLocation;
    enabled:         boolean;
    condition:       ?string;
    activeBufferRow: ?number;

    constructor(location: BreakpointLocation, condition?: string): void;

    getLocation(): BreakpointLocation;

    isEnabled(): bool;

    equals(other: Breakpoint): bool;

    toHumanized(): string;
  }

  declare type BreakpointEventType  = 'inserted' | 'removed' | 'enabled' |
                                      'disabled' | 'moved' |
                                      'condition-added' | 'condition-removed'

  declare interface BreakpointEvent {
    type:       BreakpointEventType;
    breakpoint: Breakpoint;
    bufferRow:  ?number;

    constructor(
      type: BreakpointEventType, breakpoint: Breakpoint, bufferRow?: number): void;
  }

  declare type SessionEventType         = 'launched' | 'will-terminate' |
                                          'terminated' | 'resumed' | 'suspended'
  declare type SessionTerminationReason = 'normally' | 'interrupt'
  declare type SessionSuspensionReason  = 'breakpoint' | 'step'

  declare type ExecutionLine = { filePath: string, bufferRow: number }

  declare class SessionEvent {
    type:          SessionEventType;
    reason:        SessionTerminationReason | SessionSuspensionReason;
    executionLine: ExecutionLine;

    constructor(
      type:           SessionEventType,
      reason?:        SessionTerminationReason | SessionSuspensionReason,
      executionLine?: ExecutionLine): void;
  }

  declare interface EventDefs {
    BreakpointEvent: BreakpointEvent;
    SessionEvent: SessionEvent;
  }

  declare interface DebuggerTarget  {
    filePath: string,
    args: ?string[]
  }

  declare interface DebuggerProxy {

    onSessionEvent(callback: ((event: SessionEvent) => void)): Disposable;

    onBreakpointEvent(callback: ((event: BreakpointEvent) => void)): Disposable;

    findBreakpoint(location: BreakpointLocation): ?Breakpoint;

    removeBreakpoint(breakpoint: Breakpoint): boolean;
  }

  declare interface DebuggerRegistry {

    getDebuggerProxy(): DebuggerProxy;
  }

  declare interface DebuggerController {

    debuggerRegistry: DebuggerRegistry;
  }
}
<|endoftext|>


<commit_before>var config = require(__dirname + '/../config.js');
var utils  = require(__dirname + '/../utils.js');
var market  = require(__dirname + '/markets/markets.js');

var request = function(url, callback) {
   var request = require("request");
   var options = {
      url: url,
      headers: {
         'User-Agent': 'request'
      },
      timeout: 6000
   };
   request(options, function(error, response, body){
      if (error) {
         console.log(error);
         callback(error);
      }
      else {
         var data = JSON.parse(body);
         callback(null, data);
      }
   });
};
exports.request = request;

// POST-Request
var prequest = function(options, callback) {
   var request = require("request");
   request(options, function(error, response, body){
      if (error) {
         console.log(error);
         callback(error);
      }
      else {
         var data = body;
         callback(null, data);
      }
   });
};
exports.prequest = prequest;

var addAid = function(rates, mysqlconnection, callback){
   var async = require('async');
   async.eachOfLimit(rates, 1, function(item, key, cb){
      var pair = market.splitPair(item.pair);
      market.getAssetId(pair, mysqlconnection, function(e, row){
         if (e) {
            callback(e);
         }
         else {
            rates[key].aid = row[0].aid;
         }
         cb();
      });
      // after all asset ids are added
   }, function(err){
      callback(null, rates, mysqlconnection);
   });
};
exports.addAid = addAid;

var toDb = function(rates, mysqlconnection, callback){
   var cols = [];
   let pair = new market.pair();
   for (let colName in pair) {
      cols.push(colName);
   }

   var insert = [];
   for (let i in rates) {
      let val = [];
      for (let s in rates[i]) {
         val.push("'" + rates[i][s] + "'");
      }
      insert.push('('+val.join(',')+')');
   }
   var query = 'INSERT INTO rates ('+cols.join(',')+') VALUES ' + insert.join(',');
   if (config.dev) console.log(query);
   mysqlconnection.query(query, function(e) {
      if(e) callback(e)
      else callback(null, rates);
   });
};
exports.toDb = toDb;

var all = function(mode, mysqlconnection, callback) {
   var async = require('async');
   /*var markets = {
      'poloniex': {
         file: 'poloniex.js'
      },
      'bitstamp': {
         file: 'bitstamp.js'
      },
      'bitgrail': {
         file: 'bitgrail.js'
      },
      '1broker': {
         file: '1broker.js'
      },
      'okex': {
         file: 'okex.js'
      }
   };
   var getter = [];
   for (let i in markets) {
      let app = require(__dirname + '/markets/' + markets[i].file);
      let f = function(callback) {
         app.ratesGet(mode, mysqlconnection, function(e, rates) {
            callback(null, rates);
         });
      }
      getter.push(f);
   }*/
   var polo = require(__dirname + '/markets/poloniex.js');
   var bitstamp = require(__dirname + '/markets/bitstamp.js');
   var bitgrail = require(__dirname + '/markets/bitgrail.js');
   var onebroker = require(__dirname + '/markets/1broker.js');
   var okex = require(__dirname + '/markets/okex.js');
   async.parallel({
      poloniex: function(callback) {
         polo.ratesGet(mode, mysqlconnection, function(e, rates) {
            callback(null, rates);
         });
      },
      bitstamp: function(callback) {
         bitstamp.ratesGet(mode, mysqlconnection, function(e, rates) {
            callback(null, rates);
         });
      },/*
      bitgrail: function(callback) {
         bitgrail.ratesGet(mode, mysqlconnection, function(e, rates) {
            callback(null, rates);
         });
      },*/
      onebroker: function(callback) {
         onebroker.ratesGet(mode, mysqlconnection, function(e, rates) {
            callback(null, rates);
         });
      },
      okex: function(callback) {
         okex.ratesGet(mode, mysqlconnection, function(e, rates) {
            callback(null, rates);
         });
      }
   }, function(e, rates) {
      // TODO: Error handler
      if (e) {
         console.log('error in rates.all():');
         console.log(e);
         callback(e);
      }
      let mergedRates = [];
      for (let i in rates) {
         for (let r in rates[i]) {
            mergedRates.push(rates[i][r]);
         }
      }
      callback(null, mergedRates);
   });
};
exports.all = all;<commit_msg>bugfix<commit_after>var config = require(__dirname + '/../config.js');
var utils  = require(__dirname + '/../utils.js');
var market  = require(__dirname + '/markets/markets.js');

var request = function(url, callback) {
   var request = require("request");
   var options = {
      url: url,
      headers: {
         'User-Agent': 'request'
      }
   };
   request(options, function(error, response, body){
      if (error) {
         console.log(error);
         callback(error);
      }
      else {
         var data = JSON.parse(body);
         callback(null, data);
      }
   });
};
exports.request = request;

// POST-Request
var prequest = function(options, callback) {
   var request = require("request");
   request(options, function(error, response, body){
      if (error) {
         console.log(error);
         callback(error);
      }
      else {
         var data = body;
         callback(null, data);
      }
   });
};
exports.prequest = prequest;

var addAid = function(rates, mysqlconnection, callback){
   var async = require('async');
   async.eachOfLimit(rates, 1, function(item, key, cb){
      var pair = market.splitPair(item.pair);
      market.getAssetId(pair, mysqlconnection, function(e, row){
         if (e) {
            callback(e);
         }
         else {
            rates[key].aid = row[0].aid;
         }
         cb();
      });
      // after all asset ids are added
   }, function(err){
      callback(null, rates, mysqlconnection);
   });
};
exports.addAid = addAid;

var toDb = function(rates, mysqlconnection, callback){
   var cols = [];
   let pair = new market.pair();
   for (let colName in pair) {
      cols.push(colName);
   }

   var insert = [];
   for (let i in rates) {
      let val = [];
      for (let s in rates[i]) {
         val.push("'" + rates[i][s] + "'");
      }
      insert.push('('+val.join(',')+')');
   }
   var query = 'INSERT INTO rates ('+cols.join(',')+') VALUES ' + insert.join(',');
   if (config.dev) console.log(query);
   mysqlconnection.query(query, function(e) {
      if(e) callback(e)
      else callback(null, rates);
   });
};
exports.toDb = toDb;

var all = function(mode, mysqlconnection, callback) {
   var async = require('async');
   /*var markets = {
      'poloniex': {
         file: 'poloniex.js'
      },
      'bitstamp': {
         file: 'bitstamp.js'
      },
      'bitgrail': {
         file: 'bitgrail.js'
      },
      '1broker': {
         file: '1broker.js'
      },
      'okex': {
         file: 'okex.js'
      }
   };
   var getter = [];
   for (let i in markets) {
      let app = require(__dirname + '/markets/' + markets[i].file);
      let f = function(callback) {
         app.ratesGet(mode, mysqlconnection, function(e, rates) {
            callback(null, rates);
         });
      }
      getter.push(f);
   }*/
   var polo = require(__dirname + '/markets/poloniex.js');
   var bitstamp = require(__dirname + '/markets/bitstamp.js');
   var bitgrail = require(__dirname + '/markets/bitgrail.js');
   var onebroker = require(__dirname + '/markets/1broker.js');
   var okex = require(__dirname + '/markets/okex.js');
   async.parallel({
      poloniex: function(callback) {
         polo.ratesGet(mode, mysqlconnection, function(e, rates) {
            callback(null, rates);
         });
      },
      bitstamp: function(callback) {
         bitstamp.ratesGet(mode, mysqlconnection, function(e, rates) {
            callback(null, rates);
         });
      },/*
      bitgrail: function(callback) {
         bitgrail.ratesGet(mode, mysqlconnection, function(e, rates) {
            callback(null, rates);
         });
      },*/
      onebroker: function(callback) {
         onebroker.ratesGet(mode, mysqlconnection, function(e, rates) {
            callback(null, rates);
         });
      },
      okex: function(callback) {
         okex.ratesGet(mode, mysqlconnection, function(e, rates) {
            callback(null, rates);
         });
      }
   }, function(e, rates) {
      // TODO: Error handler
      if (e) {
         console.log('error in rates.all():');
         console.log(e);
         callback(e);
      }
      let mergedRates = [];
      for (let i in rates) {
         for (let r in rates[i]) {
            mergedRates.push(rates[i][r]);
         }
      }
      callback(null, mergedRates);
   });
};
exports.all = all;<|endoftext|>


<commit_before>/*
 * DSS
 * https://github.com/darcyclarke/DSS
 *
 * Copyright (c) 2013 darcyclarke
 * Licensed under the MIT license.
 */

// Include dependancies
var mustache = require('mustache');
var fs = require('fs');
var path = require('path');
var wrench = require('wrench');

// Expose
module.exports = function(grunt) {

  // Register DSS
  grunt.registerTask('DSS', 'Parse DSS comment blocks', function() {

    // Setup async promise
    var promise = this.async();

    // Merge task-specific and/or target-specific options with defaults
    var options = this.options({
      location: process.cwd(),
      output: process.cwd() + '/docs/',
      template: process.cwd() + '/template/'
    });

    // DSS Object
    var dss = (function(){

      // Store reference
      var _dss = function(){};

      // Detect
      _dss.detect = function(){
        return true;
      };

      /*
       * Modify detector method
       *
       * @param (Function) The callback to be used to detect variables
       */
      _dss.detector = function(callback){
        _dss.detect = callback;
      };

      // Store parsers
      _dss.parsers = {};

      /*
       * Add a parser for a specific variable
       *
       * @param (String) The name of the variable
       * @param (Function) The callback to be executed at parse time
       */
      _dss.parser = function(name, callback){
        _dss.parsers[name] = callback;
      };

      /*
       * Trim whitespace from string
       *
       * @param (String) The string to be trimmed
       * @return (String) The trimmed string
       */
      _dss.trim = function(str, arr){
        var defaults = [ /^\s\s*/, /\s\s*$/ ];
        arr = (_dss.isArray(arr)) ? arr.concat(defaults) : defaults;
        arr.forEach(function(regEx){
          str = str.replace(regEx, '');
        });
        return str;
      };

      /*
       * Check if object is an array
       *
       * @param (Object) The object to check
       * @return (Boolean) The result of the test
       */
      _dss.isArray = function(obj){
        return toString.call(obj) == '[object Array]';
      };

      /*
       * Squeeze unnecessary extra characters/string
       *
       * @param (String) The string to be squeeze
       * @param (String) The string to be matched
       * @return (String) The modified string
       */
      _dss.squeeze = function(str, def){
        return str.replace(/\s{2,}/g, def);
      };

      /*
       * Takes and file path of a text file and extracts comments from it.
       *
       * @param (String) path to file
       * @param (Object) options
       */
      _dss.parse = function(options, callback){

        // Options
        options = (options) ? options : {};
        options.preserve_whitespace = !!(options.preserve_whitespace);

        // Setup
        var _this = this,
            current_block = '',
            inside_single_line_block = false,
            inside_multi_line_block = false,
            start = "{start}",
            end = "{/end}",
            _parsed = false,
            _blocks = [],
            parsed = '',
            blocks = [],
            temp = [],
            lines = grunt.file.read(options.file),
            lineNum = 0;

        /*
         * Parses line
         *
         * @param (Num) the line number
         * @param (Num) number of lines
         * @param (String) line to parse/check
         * @return (Boolean) result of parsing
         */
        var parser = function(lineNum, lines, line){
          var parts = line.replace(/.*@/, ''),
              i = parts.indexOf(' '),
              name = _dss.trim(parts.substr(0, i)),
              description = _dss.trim(parts.substr(i)),
              variable = _dss.parsers[name];
          line = {};
          line[name] = (variable) ? variable.apply(null, [lineNum, description, lines]) : '';
          return line;
        };

        /*
         * Comment block
         */
        var block = function(){
          this._raw = (comment_text) ? comment_text : '';
          this._filename = filename;
        };

        /*
         * Check for single-line comment
         *
         * @param (String) line to parse/check
         * @return (Boolean) result of check
         */
        var single_line_comment = function(line){
          return !!line.match(/^\s*\/\//);
        };

        /*
         * Checks for start of a multi-line comment
         *
         * @param (String) line to parse/check
         * @return (Boolean) result of check
         */
        var start_multi_line_comment = function(line){
          return !!line.match(/^\s*\/\*/);
        };

        /*
         * Check for end of a multi-line comment
         *
         * @parse (String) line to parse/check
         * @return (Boolean) result of check
         */
        var end_multi_line_comment = function(line){
          if(single_line_comment(line))
            return false;
          return !!line.match(/.*\*\//);
        };

        /*
         * Removes comment identifiers for single-line comments.
         *
         * @param (String) line to parse/check
         * @return (Boolean) result of check
         */
        var parse_single_line = function(line){
          return line.replace(/\s*\/\//, '');
        };

        /*
         * Remove comment identifiers for multi-line comments.
         *
         * @param (String) line to parse/check
         * @return (Boolean) result of check
         */
        var parse_multi_line = function(line){
          var cleaned = line.replace(/\s*\/\*/, '');
          return cleaned.replace(/\*\//, '');
        };

        lines = lines + '';
        lines.split(/\n/).forEach(function(line){

          lineNum = lineNum + 1;
          line = line + '';

          // Parse Single line comment
          if(single_line_comment(line)){
            parsed = _this.parse_single_line(line);
            if(inside_single_line_block){
              current_block += start + parsed + end;
            } else {
              current_block = parsed;
              inside_single_line_block = true;
            }
          }

          // Parse multi-line comments
          if(start_multi_line_comment(line)){
            current_block += start;
          }
          if(start_multi_line_comment(line) || inside_multi_line_block){
            parsed = parse_multi_line(line);
            if(inside_multi_line_block){
              current_block += parsed;
            } else {
              current_block += parsed;
              inside_multi_line_block = true;
            }
          }

          // End a multi-line block
          if(end_multi_line_comment(line)){
            inside_multi_line_block = false;
            current_block += end;
          }

          // Store current block if done
          if(!single_line_comment(line) || !inside_multi_line_block){
            if(current_block){
              _blocks.push(current_block);
            }
            inside_single_line_block = false;
            current_block = '';
          }
        });

        // Create new blocks with custom parsing
        var x = 0, length = _blocks.length;
        _parsed = true;
        _blocks.forEach(function(block){

          // Detect if block is DSS and add to blocks
          if(_dss.detector(block))
            blocks.push(parser(block));

          // Run callback if we're done with blocks
          x++;
          if(x >= length)
            callback({ file: options.file, blocks: blocks });

        });

      };

      /*
       * Build
       *
       * @param (String) location to file
       * @param (Object) options
       */
      _dss.build = function(location, template_dir, output_dir){

        // Find all CSS files
        var types = ['*.css', '*.sass', '*.scss', '*.less'],
            files = grunt.file.expand({ matchBase:true }, types),
            length = files.length,
            styleguide = [];

        // Parse files
        files.map(function(filename){

          // Report file
          grunt.log.writeln('• ' + path.relative(location, filename));

          // Parse
          _dss.parse({ file: filename, path: path.relative(location, filename) }, function(parsed){

            // Add comment block to styleguide
            styleguide.push(parsed);

            // Check if we're done
            if(length > 1){
              length = length - 1;
            } else {

              // Set output directories
              template_dir = template_dir || '../template';
              output_dir = output_dir || 'styleguide';

              // Set output template and file
              var template = template_dir + 'index.mustache',
                  output = output_dir + 'index.html';

              // Clone template directory structure
              wrench.copyDirSyncRecursive(template_dir, output_dir);

              // Read template
              var html = grunt.file.read(template);

              // Create HTML ouput
              html = mustache.render((html + ''), {project: grunt.file.readJSON('package.json'), files:styleguide});

              // Render file
              grunt.file.write(output, html);

              // Report build
              grunt.log.writeln('✓ Styleguide object generated!');
              grunt.log.writeln('✓ Documentation created at: ' + output_dir);

              // Return promise
              promise();

            }

          });

        });

      };

      // Return function
      return _dss;

    })();

    // Describe detection pattern
    dss.detector(function(line){
      if(typeof line !== 'string')
        return false;
      var reference = line.split("\n\n").pop();
      return !!reference.match(/.*@/);
    });

    // Describe parsing a name
    dss.parser('name', function(i, line, block){
      return line;
    });

    // Describe parsing a description
    dss.parser('description', function(i, line, block){
      return line;
    });

    // Describe parsing a state
    dss.parser('state', function(i, line, block){
      var state = line.split('-');
      return {
        name: (state[0]) ? _dss.trim(state[0].replace('.', ' ').replace(':', ' pseudo-class-')) : '',
        description: (state[1]) ? _dss.trim(state[1]) : ''
      };
    });

    // Describe parsing markup
    dss.parser('markup', function(i, line, block){
      var markup = block.splice(i, block.length).join('');
      return {
        example: markup,
        escaped: String(markup).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"')
      };
    });

    // Parse Utility
    // dss.parse({ file: filename, path: path.relative(location, filename) }, function(styleguide){
    //   console.log('parse utility: ', styleguide);
    // });

    // Build Documentation
    dss.build(options.location, options.template, options.output);

  });

};<commit_msg>Even more refactoring including fixes for file matches to negate template, generated and node modules directories<commit_after>/*
 * DSS
 * https://github.com/darcyclarke/DSS
 *
 * Copyright (c) 2013 darcyclarke
 * Licensed under the MIT license.
 */

// Include dependancies
var mustache = require('mustache');
var fs = require('fs');
var path = require('path');
var wrench = require('wrench');

// Expose
module.exports = function(grunt) {

  // Register DSS
  grunt.registerTask('DSS', 'Parse DSS comment blocks', function() {

    // Setup async promise
    var promise = this.async();

    // Merge task-specific and/or target-specific options with defaults
    var options = this.options({
      location: process.cwd(),
      output: process.cwd() + '/docs/',
      template: process.cwd() + '/template/'
    });

    // DSS Object
    var dss = (function(){

      // Store reference
      var _dss = function(){};

      // Detect
      _dss.detect = function(){
        return true;
      };

      /*
       * Modify detector method
       *
       * @param (Function) The callback to be used to detect variables
       */
      _dss.detector = function(callback){
        _dss.detect = callback;
      };

      // Store parsers
      _dss.parsers = {};

      /*
       * Add a parser for a specific variable
       *
       * @param (String) The name of the variable
       * @param (Function) The callback to be executed at parse time
       */
      _dss.parser = function(name, callback){
        _dss.parsers[name] = callback;
      };

      /*
       * Trim whitespace from string
       *
       * @param (String) The string to be trimmed
       * @return (String) The trimmed string
       */
      _dss.trim = function(str, arr){
        var defaults = [ /^\s\s*/, /\s\s*$/ ];
        arr = (_dss.isArray(arr)) ? arr.concat(defaults) : defaults;
        arr.forEach(function(regEx){
          str = str.replace(regEx, '');
        });
        return str;
      };

      /*
       * Check if object is an array
       *
       * @param (Object) The object to check
       * @return (Boolean) The result of the test
       */
      _dss.isArray = function(obj){
        return toString.call(obj) == '[object Array]';
      };

      /*
       * Squeeze unnecessary extra characters/string
       *
       * @param (String) The string to be squeeze
       * @param (String) The string to be matched
       * @return (String) The modified string
       */
      _dss.squeeze = function(str, def){
        return str.replace(/\s{2,}/g, def);
      };

      /*
       * Takes and file path of a text file and extracts comments from it.
       *
       * @param (String) path to file
       * @param (Object) options
       */
      _dss.parse = function(options, callback){

        // Options
        options = (options) ? options : {};
        options.preserve_whitespace = !!(options.preserve_whitespace);

        // Setup
        var _this = this,
            current_block = '',
            inside_single_line_block = false,
            inside_multi_line_block = false,
            start = "{start}",
            end = "{/end}",
            _parsed = false,
            _blocks = [],
            parsed = '',
            blocks = [],
            temp = [],
            lines = grunt.file.read(options.file),
            lineNum = 0;

        /*
         * Parses line
         *
         * @param (Num) the line number
         * @param (Num) number of lines
         * @param (String) line to parse/check
         * @return (Boolean) result of parsing
         */
        var parser = function(lineNum, lines, line){
          var parts = line.replace(/.*@/, ''),
              i = parts.indexOf(' '),
              name = _dss.trim(parts.substr(0, i)),
              description = _dss.trim(parts.substr(i)),
              variable = _dss.parsers[name];
          line = {};
          line[name] = (variable) ? variable.apply(null, [lineNum, description, lines]) : '';
          return line;
        };

        /*
         * Comment block
         */
        var block = function(){
          this._raw = (comment_text) ? comment_text : '';
          this._filename = filename;
        };

        /*
         * Check for single-line comment
         *
         * @param (String) line to parse/check
         * @return (Boolean) result of check
         */
        var single_line_comment = function(line){
          return !!line.match(/^\s*\/\//);
        };

        /*
         * Checks for start of a multi-line comment
         *
         * @param (String) line to parse/check
         * @return (Boolean) result of check
         */
        var start_multi_line_comment = function(line){
          return !!line.match(/^\s*\/\*/);
        };

        /*
         * Check for end of a multi-line comment
         *
         * @parse (String) line to parse/check
         * @return (Boolean) result of check
         */
        var end_multi_line_comment = function(line){
          if(single_line_comment(line))
            return false;
          return !!line.match(/.*\*\//);
        };

        /*
         * Removes comment identifiers for single-line comments.
         *
         * @param (String) line to parse/check
         * @return (Boolean) result of check
         */
        var parse_single_line = function(line){
          return line.replace(/\s*\/\//, '');
        };

        /*
         * Remove comment identifiers for multi-line comments.
         *
         * @param (String) line to parse/check
         * @return (Boolean) result of check
         */
        var parse_multi_line = function(line){
          var cleaned = line.replace(/\s*\/\*/, '');
          return cleaned.replace(/\*\//, '');
        };

        /*
         * Normalizes the comment block to ignore any consistent preceding
         * whitespace. Consistent means the same amount of whitespace on every line
         * of the comment block. Also strips any whitespace at the start and end of
         * the whole block.
         *
         * @param (String) Text block
         * @return (String) A cleaned up text block
         */
         var normalize = function(text_block){
          if(options.preserve_whitespace)
            return text_block;

          // Strip out any preceding [whitespace]* that occur on every line. Not
          // the smartest, but I wonder if I care.
          text_block = text_block.replace(/^(\s*\*+)/, '');

          // Strip consistent indenting by measuring first line's whitespace
          var indent_size = false;
          var unindented = (function(lines){
            return lines.map(function(line){
              var preceding_whitespace = line.match(/^\s*/)[0].length;
              if(!indent_size)
                indent_size = preceding_whitespace;
              if(line == ''){
                return '';
              } else if(indent_size <= preceding_whitespace && indent_size > 0){
                return line.slice(indent_size, (line.length - 1));
              } else {
                return line;
              }
            }).join("\n");
          })(text_block.split("\n"));

          return _dss.trim(text_block);

        };

        lines = lines + '';
        lines.split(/\n/).forEach(function(line){

          lineNum = lineNum + 1;
          line = line + '';

          // Parse Single line comment
          if(single_line_comment(line)){
            parsed = _this.parse_single_line(line);
            if(inside_single_line_block){
              current_block += start + parsed + end;
            } else {
              current_block = parsed;
              inside_single_line_block = true;
            }
          }

          // Parse multi-line comments
          if(start_multi_line_comment(line)){
            current_block += start;
          }
          if(start_multi_line_comment(line) || inside_multi_line_block){
            parsed = parse_multi_line(line);
            if(inside_multi_line_block){
              current_block += parsed;
            } else {
              current_block += parsed;
              inside_multi_line_block = true;
            }
          }

          // End a multi-line block
          if(end_multi_line_comment(line)){
            inside_multi_line_block = false;
            current_block += end;
          }

          // Store current block if done
          if(!single_line_comment(line) || !inside_multi_line_block){
            if(current_block){
              _blocks.push(current_block);
            }
            inside_single_line_block = false;
            current_block = '';
          }
        });

        // Create new blocks with custom parsing
        _parsed = true;
        _blocks.forEach(function(block, index){
          // Detect if block is DSS and add to blocks
          block = normalize(block);
          if(_dss.detect(block))
            blocks.push(parser(index, lines, block));
        });

        callback({ file: options.file, blocks: blocks });

      };

      /*
       * Build
       *
       * @param (String) location to file
       * @param (Object) options
       */
      _dss.build = function(location, template_dir, output_dir){

        // Find all CSS files
        var negate = function(dir){ return '!' + path.relative(process.cwd(), dir) + '/**'; },
            types = [
              '*.css',
              '*.sass',
              '*.scss',
              '*.less',
              negate(template_dir),
              negate(output_dir),
              '!node_modules/**'
            ],
            files = grunt.file.expand({ matchBase:true }, types),
            length = files.length,
            styleguide = [];

        // Parse files
        files.map(function(filename){

          // Report file
          grunt.log.writeln('• ' + path.relative(location, filename));

          // Parse
          _dss.parse({ file: filename, path: path.relative(location, filename) }, function(parsed){

            // Add comment block to styleguide
            styleguide.push(parsed);

            // Check if we're done
            if(length > 1){

              length = length - 1;

            } else {

              // Set output directories
              template_dir = template_dir || '../template';
              output_dir = output_dir || 'styleguide';

              // Set output template and file
              var template = template_dir + 'index.mustache',
                  output = output_dir + 'index.html';

              // Clone template directory structure
              wrench.copyDirSyncRecursive(template_dir, output_dir);

              // Read template
              var html = grunt.file.read(template);

              // Create HTML ouput
              html = mustache.render((html + ''), {project: grunt.file.readJSON('package.json'), files:styleguide});

              // Render file
              grunt.file.write(output, html);

              // Report build
              grunt.log.writeln('✓ Styleguide object generated!');
              grunt.log.writeln('✓ Documentation created at: ' + output_dir);

              // Return promise
              promise();

            }

          });

        });

      };

      // Return function
      return _dss;

    })();

    // Describe detection pattern
    dss.detector(function(line){
      if(typeof line !== 'string')
        return false;
      var reference = line.split("\n\n").pop();
      return !!reference.match(/.*@/);
    });

    // Describe parsing a name
    dss.parser('name', function(i, line, block){
      return line;
    });

    // Describe parsing a description
    dss.parser('description', function(i, line, block){
      return line;
    });

    // Describe parsing a state
    dss.parser('state', function(i, line, block){
      var state = line.split('-');
      return {
        name: (state[0]) ? dss.trim(state[0].replace('.', ' ').replace(':', ' pseudo-class-')) : '',
        description: (state[1]) ? dss.trim(state[1]) : ''
      };
    });

    // Describe parsing markup
    dss.parser('markup', function(i, line, block){
      var markup = block.splice(i, block.length).join('');
      return {
        example: markup,
        escaped: String(markup).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"')
      };
    });

    // Build Documentation
    dss.build(options.location, options.template, options.output);

  });

};<|endoftext|>


<commit_before>'use strict';

/**
 * @author      hustcer
 * @license     MIT
 * @copyright   TraceInvest.com
 * @create      06/24/2015
 */

let vm      = require('vm'),
    fs      = require('fs'),
    _       = require('lodash'),
    printf  = require('printf'),
    yaml    = require('js-yaml'),
    blessed = require('blessed'),
    Promise = require('bluebird'),
    cmd     = require('commander'),
    contrib = require('blessed-contrib'),
    pinyin  = require('pinyin');

let conf    = require('./conf.js').conf;
let Iconv   = require('iconv').Iconv;
let iconv   = new Iconv('GBK', 'UTF-8//TRANSLIT//IGNORE');
let request = Promise.promisifyAll(require('request'));

let table   = null;
let screen  = blessed.screen({fullUnicode:true});

const MSG = {
    INPUT_ERROR      : '输入错误，当前只支持通过证券代码看盘，请检查后重新输入！',
    TOO_MANY_SYMS    : '输入的股票太多，一次最多支持盯盘25只股票！',
    SYMBOL_NOT_EXIST : '该股票代码不存在:'
};

// A client error like 400 Bad Request happened
let ClientError = e => {
    return e.code >= 400 && e.code < 500 || e.code === 'ENOTFOUND';
};

const FMT = {
    three:{
        CODE    : '%7s',
        PRICE   : '%7.2f',
        INC     : '%7.2f',
        INC_PCT : '%7.2f %%'
    },
    four:{
        CODE    : '%6s',
        PRICE   : '%6.2f',
        INC     : '%6.2f',
        INC_PCT : '%6.2f %%'
    }
};

// Set current data source
let src    = (cmd.data || '').toUpperCase();
let source = (src === 'SINA' || src === 'TENCENT') ? src : conf.dataSource;
let ds     =  conf.provider[source];

/**
 * Check validity of input symbols, and return a valid symbols array if there is.
 * @param  {String} syms    Input symbols' string.
 * @return {Array}  Array of input symbols
 */
let getValidSymbols = function(syms){
    // When no symbols were provided, use symbols from watchList or holded symbols.
    if(syms === true){
        let Common   = require('./common.js').Common;
        let yamlConf = yaml.safeLoad(fs.readFileSync(Common.getSymbolFilePath(), 'utf8'));
        if(!cmd.hold && yamlConf.watchList && yamlConf.watchList.length > 0){
            syms = _.pluck(yamlConf.watchList, 'code').join(',');
        }else{
            syms = _.filter(yamlConf.symbols, s => s.hold);
            syms = _.pluck(syms, 'code').join(',');
        }
    }
    let symbols = syms.replace(/，/g, ',');
    let valid   = /^[0-9\,]+$/.test(symbols);
    if(!valid){
        console.error(MSG.INPUT_ERROR.error);
        return false;
    }
    symbols = _.trimRight(symbols, ',').split(',');
    if(symbols.length>conf.chunkSize){
        console.error(MSG.TOO_MANY_SYMS.error);
        return false;
    }
    return symbols;
};

/**
 * Init table layout and screen keyboard event
 */
let initUI = function(){
    table = contrib.table({
        keys          : true,
        fg            : 'white',
        selectedFg    : 'black',
        selectedBg    : 'cyan',
        interactive   : true,
        label         : 'watchlist',
        width         : '55%',
        height        : '80%',
        border        : {type: 'line', fg: 'cyan'},
        columnSpacing : 6,
        columnWidth   : [10, 7, 7, 7, 9]
    });

    screen.key(['escape', 'q', 'C-c'], function() {
        /* eslint no-process-exit:0 */
        return process.exit(0);
    });
};

/**
 * Update table data
 * @param  {Array} queryResults    New table data to be updated
 */
let updateData = function(queryResults){
    let queryData = [];
    let select    = 'four';

    _.each(queryResults, function(q){
        // Fix the padding of company name
        if(q.name.length === 4){
            select = 'four';
        }else if(q.name.length === 3){
            select = 'three';
            q.name = '  ' + q.name;
        }else{
            select = 'three';
        }
        q.code   = printf(FMT[select].CODE    , q.code);
        q.price  = printf(FMT[select].PRICE   , q.price);
        q.inc    = printf(FMT[select].INC     , q.inc);
        q.incPct = printf(FMT[select].INC_PCT , q.incPct);

        queryData.push([q.name, q.code, q.price, q.inc, q.incPct]);
    });

    table.setData({
        headers : ['name', 'code', 'curr', 'updn', 'per'],
        data    : queryData
    });
    table.focus();
    screen.append(table);
    screen.render();
};

/**
 * Query and refresh price data
 * @param  {Array} syms   Symbols' code array
 * @param  {String} query Query string
 */
let refreshData = function(syms, query){
    // symbol query results
    let queryResults = [];
    return request.getAsync(ds.url + query, {encoding:null}).spread(function(resp, body) {
        body = iconv.convert(body).toString();
        vm.runInThisContext(body);

        _.each(syms, function(s){
            let localVar = ds.flag + conf.market[s.substr(0,3)] + s;
            if(body.indexOf(localVar) < 0 || body.indexOf(localVar+'="";') >= 0){
                console.error(MSG.SYMBOL_NOT_EXIST.error, s.em);
                return false;
            }

            let res    = {};
            let splits = vm.runInThisContext(ds.flag + conf.market[s.substr(0,3)] + s).split(ds.sep);
            res.name   = pinyin(splits[ds.nameIdx],{
                style: pinyin.STYLE_INITIALS,
                heteronym: true
            });
            res.code   = s;
            // 对于停牌股票价格以上个交易日收盘价格为准
            res.close  = +splits[ds.closeIdx];
            res.price  = +splits[ds.priceIdx] || res.close;
            res.inc    = res.price - res.close;
            res.incPct = ((res.price - res.close)/res.close) * 100;
            queryResults.push(res);
        });

        updateData(queryResults);

    // FIXME: fix 'ENOTFOUND' error
    }).catch(ClientError, function() {
        process.exit(1);
    });
};

/**
 * Query symbols' price info
 * @param  {Array} syms  Symbols array to query
 * @return {Array}       Basic info array of queried symbols
 */
let querySymbols = function(syms){
    let query = _.map(syms, x => conf.market[x.substr(0,3)] + x).join(',');
    refreshData(syms, query);
    setInterval(refreshData.bind(this, syms, query), 3600);
};

/**
 * Do the query action.
 * @param  {String} syms    Input symbols' string.
 */
let doWatch = function(syms){
    initUI();
    Promise
        .resolve(syms)
        .then(getValidSymbols)
        .then(querySymbols);
};

// star -w 002625,600588,601179,600415,600816,600880,600703,000861,600410,000768,000712,600118,600893,002456,000902,000725,000002,000100
exports.Watch = {
    doWatch : doWatch
};
<commit_msg>remove pinyin<commit_after>'use strict';

/**
 * @author      hustcer
 * @license     MIT
 * @copyright   TraceInvest.com
 * @create      06/24/2015
 */

let vm      = require('vm'),
    fs      = require('fs'),
    _       = require('lodash'),
    printf  = require('printf'),
    yaml    = require('js-yaml'),
    blessed = require('blessed'),
    Promise = require('bluebird'),
    cmd     = require('commander'),
    contrib = require('blessed-contrib'),
    pinyin  = require('pinyin');

let conf    = require('./conf.js').conf;
let Iconv   = require('iconv').Iconv;
let iconv   = new Iconv('GBK', 'UTF-8//TRANSLIT//IGNORE');
let request = Promise.promisifyAll(require('request'));

let table   = null;
let screen  = blessed.screen({fullUnicode:true});

const MSG = {
    INPUT_ERROR      : '输入错误，当前只支持通过证券代码看盘，请检查后重新输入！',
    TOO_MANY_SYMS    : '输入的股票太多，一次最多支持盯盘25只股票！',
    SYMBOL_NOT_EXIST : '该股票代码不存在:'
};

// A client error like 400 Bad Request happened
let ClientError = e => {
    return e.code >= 400 && e.code < 500 || e.code === 'ENOTFOUND';
};

const FMT = {
    three:{
        CODE    : '%7s',
        PRICE   : '%7.2f',
        INC     : '%7.2f',
        INC_PCT : '%7.2f %%'
    },
    four:{
        CODE    : '%6s',
        PRICE   : '%6.2f',
        INC     : '%6.2f',
        INC_PCT : '%6.2f %%'
    }
};

// Set current data source
let src    = (cmd.data || '').toUpperCase();
let source = (src === 'SINA' || src === 'TENCENT') ? src : conf.dataSource;
let ds     =  conf.provider[source];

/**
 * Check validity of input symbols, and return a valid symbols array if there is.
 * @param  {String} syms    Input symbols' string.
 * @return {Array}  Array of input symbols
 */
let getValidSymbols = function(syms){
    // When no symbols were provided, use symbols from watchList or holded symbols.
    if(syms === true){
        let Common   = require('./common.js').Common;
        let yamlConf = yaml.safeLoad(fs.readFileSync(Common.getSymbolFilePath(), 'utf8'));
        if(!cmd.hold && yamlConf.watchList && yamlConf.watchList.length > 0){
            syms = _.pluck(yamlConf.watchList, 'code').join(',');
        }else{
            syms = _.filter(yamlConf.symbols, s => s.hold);
            syms = _.pluck(syms, 'code').join(',');
        }
    }
    let symbols = syms.replace(/，/g, ',');
    let valid   = /^[0-9\,]+$/.test(symbols);
    if(!valid){
        console.error(MSG.INPUT_ERROR.error);
        return false;
    }
    symbols = _.trimRight(symbols, ',').split(',');
    if(symbols.length>conf.chunkSize){
        console.error(MSG.TOO_MANY_SYMS.error);
        return false;
    }
    return symbols;
};

/**
 * Init table layout and screen keyboard event
 */
let initUI = function(){
    table = contrib.table({
        keys          : true,
        fg            : 'white',
        selectedFg    : 'black',
        selectedBg    : 'cyan',
        interactive   : true,
        label         : 'watchlist',
        width         : '55%',
        height        : '80%',
        border        : {type: 'line', fg: 'cyan'},
        columnSpacing : 6,
        columnWidth   : [10, 7, 7, 7, 9]
    });

    screen.key(['escape', 'q', 'C-c'], function() {
        /* eslint no-process-exit:0 */
        return process.exit(0);
    });
};

/**
 * Update table data
 * @param  {Array} queryResults    New table data to be updated
 */
let updateData = function(queryResults){
    let queryData = [];
    let select    = 'four';

    _.each(queryResults, function(q){
        // Fix the padding of company name
        if(q.name.length === 4){
            select = 'four';
        }else if(q.name.length === 3){
            select = 'three';
            q.name = '  ' + q.name;
        }else{
            select = 'three';
        }
        q.code   = printf(FMT[select].CODE    , q.code);
        q.price  = printf(FMT[select].PRICE   , q.price);
        q.inc    = printf(FMT[select].INC     , q.inc);
        q.incPct = printf(FMT[select].INC_PCT , q.incPct);

        queryData.push([q.name, q.code, q.price, q.inc, q.incPct]);
    });

    table.setData({
        headers : ['name', 'code', 'curr', 'updn', 'per'],
        data    : queryData
    });
    table.focus();
    screen.append(table);
    screen.render();
};

/**
 * Query and refresh price data
 * @param  {Array} syms   Symbols' code array
 * @param  {String} query Query string
 */
let refreshData = function(syms, query){
    // symbol query results
    let queryResults = [];
    return request.getAsync(ds.url + query, {encoding:null}).spread(function(resp, body) {
        body = iconv.convert(body).toString();
        vm.runInThisContext(body);

        _.each(syms, function(s){
            let localVar = ds.flag + conf.market[s.substr(0,3)] + s;
            if(body.indexOf(localVar) < 0 || body.indexOf(localVar+'="";') >= 0){
                console.error(MSG.SYMBOL_NOT_EXIST.error, s.em);
                return false;
            }

            let res    = {};
            let splits = vm.runInThisContext(ds.flag + conf.market[s.substr(0,3)] + s).split(ds.sep);
            res.name   = splits[ds.nameIdx];
            res.code   = s;
            // 对于停牌股票价格以上个交易日收盘价格为准
            res.close  = +splits[ds.closeIdx];
            res.price  = +splits[ds.priceIdx] || res.close;
            res.inc    = res.price - res.close;
            res.incPct = ((res.price - res.close)/res.close) * 100;
            queryResults.push(res);
        });

        updateData(queryResults);

    // FIXME: fix 'ENOTFOUND' error
    }).catch(ClientError, function() {
        process.exit(1);
    });
};

/**
 * Query symbols' price info
 * @param  {Array} syms  Symbols array to query
 * @return {Array}       Basic info array of queried symbols
 */
let querySymbols = function(syms){
    let query = _.map(syms, x => conf.market[x.substr(0,3)] + x).join(',');
    refreshData(syms, query);
    setInterval(refreshData.bind(this, syms, query), 3600);
};

/**
 * Do the query action.
 * @param  {String} syms    Input symbols' string.
 */
let doWatch = function(syms){
    initUI();
    Promise
        .resolve(syms)
        .then(getValidSymbols)
        .then(querySymbols);
};

// star -w 002625,600588,601179,600415,600816,600880,600703,000861,600410,000768,000712,600118,600893,002456,000902,000725,000002,000100
exports.Watch = {
    doWatch : doWatch
};
<|endoftext|>


<commit_before>/**
 * Provides Gulp tasks for working with the NPM CLI.
 *
 * The following tasks are defined:
 * `npm-install` - Runs `npm install` via NPM CLI.
 *
 * `npm-outdated` - Runs `npm outdated` via NPM CLI.
 *
 * `npm-uninstall` - Runs `npm uninstall <package>` via NPM CLI for all node modules installed.
 *
 * @param {Gulp}     gulp     - An instance of Gulp.
 * @param {object}   options  - Optional parameters
 */
module.exports = function(gulp, options)
{
   var rootPath = options.rootPath;

   /**
    * Runs `npm install` via NPM CLI.
    */
   gulp.task('npm-install', function(cb)
   {
      var exec = require('child_process').exec;
      exec('npm install', { cwd: rootPath }, function(err, stdout, stderr)
      {
         console.log(stdout);
         console.log(stderr);
         cb(err);
      });
   });

   /**
    * Runs `npm outdated` via NPM CLI.
    */
   gulp.task('npm-outdated', function(cb)
   {
      var exec = require('child_process').exec;
      exec('npm outdated', { cwd: rootPath }, function(err, stdout, stderr)
      {
         console.log(stdout);
         console.log(stderr);
         cb(err);
      });
   });

   /**
    * Runs `npm run test` via NPM CLI. This will run a script associated with `test`.
    */
   gulp.task('npm-run-test', function(cb)
   {
      var exec = require('child_process').exec;
      exec('npm run test', { cwd: rootPath }, function(err, stdout, stderr)
      {
         console.log(stdout);
         console.log(stderr);
         cb(err);
      });
   });

   /**
    * Runs `npm run test-coverage` via NPM CLI. This will run a script associated with `test-coverage`.
    */
   gulp.task('npm-run-test-coverage', function(cb)
   {
      var exec = require('child_process').exec;
      exec('npm run test-coverage', { cwd: rootPath }, function(err, stdout, stderr)
      {
         console.log(stdout);
         console.log(stderr);
         cb(err);
      });
   });

   /**
    * Runs `npm uninstall <package>` via NPM CLI for all node modules installed.
    */
   gulp.task('npm-uninstall', function(cb)
   {
      var exec = require('child_process').exec;
      exec('for package in `ls node_modules`; do npm uninstall $package; done;', { cwd: rootPath },
       function(err, stdout, stderr)
      {
         console.log(stdout);
         console.log(stderr);
         cb(err);
      });
   });
};
<commit_msg>0.4.0 release<commit_after>/**
 * Provides Gulp tasks for working with the NPM CLI.
 *
 * The following tasks are defined:
 * `npm-install` - Runs `npm install` via NPM CLI.
 *
 * `npm-outdated` - Runs `npm outdated` via NPM CLI.
 *
 * `npm-run-test` - Runs `npm run test` via NPM CLI.
 *
 * `npm-run-test-coverage` - Runs `npm run test-coverage` via NPM CLI.
 *
 * `npm-uninstall` - Runs `npm uninstall <package>` via NPM CLI for all node modules installed.
 *
 * @param {Gulp}     gulp     - An instance of Gulp.
 * @param {object}   options  - Optional parameters
 */
module.exports = function(gulp, options)
{
   var rootPath = options.rootPath;

   /**
    * Runs `npm install` via NPM CLI.
    */
   gulp.task('npm-install', function(cb)
   {
      var exec = require('child_process').exec;
      exec('npm install', { cwd: rootPath }, function(err, stdout, stderr)
      {
         console.log(stdout);
         console.log(stderr);
         cb(err);
      });
   });

   /**
    * Runs `npm outdated` via NPM CLI.
    */
   gulp.task('npm-outdated', function(cb)
   {
      var exec = require('child_process').exec;
      exec('npm outdated', { cwd: rootPath }, function(err, stdout, stderr)
      {
         console.log(stdout);
         console.log(stderr);
         cb(err);
      });
   });

   /**
    * Runs `npm run test` via NPM CLI. This will run a script associated with `test`.
    */
   gulp.task('npm-run-test', function(cb)
   {
      var exec = require('child_process').exec;
      exec('npm run test', { cwd: rootPath }, function(err, stdout, stderr)
      {
         console.log(stdout);
         console.log(stderr);
         cb(err);
      });
   });

   /**
    * Runs `npm run test-coverage` via NPM CLI. This will run a script associated with `test-coverage`.
    */
   gulp.task('npm-run-test-coverage', function(cb)
   {
      var exec = require('child_process').exec;
      exec('npm run test-coverage', { cwd: rootPath }, function(err, stdout, stderr)
      {
         console.log(stdout);
         console.log(stderr);
         cb(err);
      });
   });

   /**
    * Runs `npm uninstall <package>` via NPM CLI for all node modules installed.
    */
   gulp.task('npm-uninstall', function(cb)
   {
      var exec = require('child_process').exec;
      exec('for package in `ls node_modules`; do npm uninstall $package; done;', { cwd: rootPath },
       function(err, stdout, stderr)
      {
         console.log(stdout);
         console.log(stderr);
         cb(err);
      });
   });
};
<|endoftext|>


<commit_before>'use strict';

var fs = require('fs');
var _ = require('lodash');
var path = require('path');
var bar = require('./bar');
var commands = require('./commands');
var config = require('./config');
var files = require('./files');
var listFiles = require('./list-files');
var Logger = require('./logger');
var md5hex = require('./md5hex');
var minimatches = require('./minimatches');
var Promise = require('./promise');
var promisify = require('./promisify');
var mkdirp = promisify(require('mkdirp'));
var lstat = promisify(fs.lstat);
var readFile = promisify(fs.readFile);
var writeFile = promisify(fs.writeFile);
var log = Logger.create('wendy');

function timeSince(start) {
	var ms = Date.now() - start;
	return ms < 100 ? ms + 'ms' : (ms / 1000).toFixed(1) + 's';
}

function bytes(nr) {
	return (nr / 1024).toFixed(1) + 'K';
}

function percentage(a, b) {
	return (b ? (a * 100 / b).toFixed(1) : '∞') + '%';
}

function processLegacy(inputPaths, processor, pConfig, pLog) {
	var inputFiles = files.getList(inputPaths);
	var outputFilesPromise = processor(pConfig, pLog, inputFiles);
	return Promise.resolve(outputFilesPromise).then(function(outputFiles) {
		_.difference(outputFiles, inputFiles).forEach(function(file) {
			files.add(file);
		});
		_.difference(inputFiles, outputFiles).forEach(function(file) {
			files.remove(file);
		});
	});
}

function processBatch(processor, inputs, outputs) {
	var inputSize = 0;
	inputs.forEach(function(input) {
		switch (input.type) {
			// TODO: Re-enable in memory caching of text and json parsed buffer? (used to be in File)
			case 'buffer': input.data = input.buffer; break;
			case 'text': input.data = input.buffer.toString(); break;
			case 'json': input.data = JSON.parse(input.buffer.toString()); break;
			default: throw new Error('Invalid type: ' + input.type);
		}
		if (input.trackRatio) {
			inputSize += input.buffer.length;
		}
		delete input.buffer;
	});

	outputs.forEach(function(output) {
		var resolve = output.resolve;
		output.resolve = function(data) {
			var buffer;
			switch (output.type) {
				case 'buffer': buffer = data; break;
				case 'text': buffer = new Buffer(data); break;
				case 'json': buffer = new Buffer(JSON.stringify(data, null, '\t')); break;
				default: throw new Error('Invalid type: ' + output.type);
			}
			resolve(buffer);
			var outputSize = buffer.length;
			if (output.trackRatio) {
				processor.log.ok('%s (%s %s)', output.path, bytes(outputSize), percentage(outputSize, inputSize));
			} else {
				processor.log.ok('%s', output.path);
			}
		};
	});

	// TODO: Distribute process calls over this and other threads to improve multi-core performance?
	return Promise.resolve(processor.process(inputs, outputs));
}

module.exports = function(execCommands) {
	var wendyStart = Date.now();

	files.init(minimatches(listFiles('.'), config.project.files));
	log.spam('Files:\n  %s', files.getPaths().join('\n  '));

	// Keep track of all processor promises to know when all async processing is resolved
	var processorPromises = [];

	// Processors selected through commands are turned into files task functions
	// These should return a Promise indicating they are done manipulating files
	var filesTasks = _.filter(commands.select(execCommands), 'processor').map(function(command) {
		var processorName = command.processor;
		var commandName = commands.getName(command);
		var shortName = commands.getName(command, true);
		var pConfig = _.extend({}, config[commandName], { project: config.project });
		var pLog = Logger.create(shortName);

		// TODO: Instantiate Processor here instead of in processor module so they do not depend on nbob
		var processor = require('./processor/' + processorName);

		// Legacy processor
		// TODO: Create LegacyProcessor that wraps this flow into a regular processor.getBatches() call
		if (processor instanceof Function) {
			return function() {
				var inputPaths = pConfig.files ? minimatches(files.getPaths(), pConfig.files) : files.getPaths();
				var processorStart = Date.now();
				pLog.debug('Starting');
				bar.add(shortName);
				return processLegacy(inputPaths, processor, pConfig, pLog).then(function() {
					pLog.info('Finished after: %s', timeSince(processorStart));
					bar.finish(shortName);
				});
			};
		}

		// TODO: Clean and split up this code to make it more readable/manageable
		// TODO: Improve caching approach to reduce number of files and amount of data read
		// E.g: Cache MD5 values of files in <cacheDir>.json and make file reading lazy
		return function() {
			if (!processor.initialized) {
				processor.init(pConfig, pLog);
			}

			var processorStart;
			var inputPaths = pConfig.files ? minimatches(files.getPaths(), pConfig.files) : files.getPaths();
			var batchesPromise = Promise.resolve(processor.getBatches(inputPaths, files.getList(inputPaths)));

			return batchesPromise.then(function(batches) {
				var nrBatches = batches.length;
				if (nrBatches === 0) {
					return;
				}
				pLog.debug('Starting %d batches', nrBatches);

				var batchPromises = batches.map(function(batch, batchIndex) {
					var inputs = batch.inputs;
					var outputs = batch.outputs;

					// Prepare input promises
					var inputPromises = inputs.map(function(input) {
						var file = files.get(input.path);
						if (!file) {
							throw new Error('File not found: ' + input.path + ', for command: ' + commandName);
						}
						return Promise.apply([ file.getBuffer(), file.getMD5() ], function(buffer, md5) {
							input.buffer = buffer;
							input.md5 = md5;
							return input;
						});
					});

					// Use existing or add new files and prepare output buffer promises and callbacks
					var bufferPromises = outputs.map(function(output) {
						var file = files.get(output.path) || files.add(output.path);
						return file.setBuffer(new Promise(function(resolve, reject) {
							output.resolve = function(buffer) {
								resolve(output.write ? writeFile(output.path, buffer) : buffer);
							};
							output.reject = reject;
						}));
					});

					// Remove files
					var inputPaths = _.pluck(_.filter(inputs, { isReadOnly: false }), 'path');
					var outputPaths = _.pluck(outputs, 'path');
					_.difference(inputPaths, outputPaths).forEach(function(removedPath) {
						files.remove(removedPath);
					});

					// Wait for input buffers to become available and start processing
					return Promise.all(inputPromises).then(function(inputs) {
						if (!processorStart) {
							processorStart = Date.now();
							bar.add(shortName, nrBatches);
						}
						var batchStart = Date.now();
						pLog.spam('Starting batch #%d: %s => %s', batchIndex,
							_.pluck(inputs, 'path').join(', '), outputPaths.join(', '));

						// Combination of file paths and md5s should provide a suitable md5 for these inputs
						// It also reduces overhead of md5 calculation by enabling in memory caching of input md5s
						var inputsDigestData = inputs.map(function(input) {
							return input.path + input.md5;
						});
						var processInputDigest = md5hex([ JSON.stringify(pConfig) ].concat(inputsDigestData), 'utf8');
						var cacheDir = path.join(config.project.buildDir, processorName, processInputDigest);
						var wasCached = true;

						return lstat(cacheDir).then(function() {
							return Promise.all(outputs.map(function(output) {
								var outputPath = path.join(cacheDir, output.path);
								return readFile(outputPath).then(output.resolve, function(error) {
									pLog.warn('Invalid cache: ' + error.message);
									throw error;
								});
							}));
						}).catch(function() {
							wasCached = false;
							return processBatch(processor, inputs, outputs).then(function() {
								return Promise.all(bufferPromises).then(function(buffers) {
									// Cache batch if not disabled and everything resolves
									// Note: create cache dir even if empty to enable caching of read-only batches
									// Note: do not block or reject on writing away cache files
									if (batch.doNotCache) {
										return;
									}
									mkdirp(cacheDir);
									buffers.forEach(function(buffer, i) {
										var outputPath = path.join(cacheDir, outputs[i].path);
										mkdirp(path.dirname(outputPath)).then(function() {
											writeFile(outputPath, buffer);
											log.spam('Cached: %s', outputPath);
										});
									});
								});
							});
						}).then(function() {
							pLog.spam('Finished batch #%d after: %s%s',
								batchIndex, timeSince(batchStart), wasCached ? ' (cached)' : '');
							bar.finish(shortName);
						});
					});
				});

				var processorPromise = Promise.all(batchPromises);
				processorPromise.then(function() {
					pLog.info('Finished %d batches after: %s', nrBatches, timeSince(processorStart));
				});
				processorPromises.push(processorPromise);

				// All files changes have been made already (add, remove, path change, setBuffer promise)
				// So we implicitly resolve this task now, allowing following tasks to start
			});
		};
	});

	// Call files tasks sequentially
	var wendyPromise = filesTasks.reduce(function(prevPromise, task) {
		return prevPromise.then(task);
	}, Promise.resolve()).then(function() {
		// Wait for all (including async) processing to finish
		return Promise.all(processorPromises).then(function() {
			log.info('All finished after: %s', timeSince(wendyStart));
		});
	});

	wendyPromise.catch(function() {
		bar.reset();
		log.info('Rejected after: %s', timeSince(wendyStart));
	});

	return wendyPromise;
};
<commit_msg>Fixed cache when output.write is true and also improved platform independence of caching<commit_after>'use strict';

var fs = require('fs');
var _ = require('lodash');
var path = require('path');
var bar = require('./bar');
var commands = require('./commands');
var config = require('./config');
var files = require('./files');
var listFiles = require('./list-files');
var Logger = require('./logger');
var md5hex = require('./md5hex');
var minimatches = require('./minimatches');
var Promise = require('./promise');
var promisify = require('./promisify');
var mkdirp = promisify(require('mkdirp'));
var lstat = promisify(fs.lstat);
var readFile = promisify(fs.readFile);
var writeFile = promisify(fs.writeFile);
var log = Logger.create('wendy');

function timeSince(start) {
	var ms = Date.now() - start;
	return ms < 100 ? ms + 'ms' : (ms / 1000).toFixed(1) + 's';
}

function bytes(nr) {
	return (nr / 1024).toFixed(1) + 'K';
}

function percentage(a, b) {
	return (b ? (a * 100 / b).toFixed(1) : '∞') + '%';
}

function unixPathJoin() {
	return _.toArray(arguments).join('/');
}

function processLegacy(inputPaths, processor, pConfig, pLog) {
	var inputFiles = files.getList(inputPaths);
	var outputFilesPromise = processor(pConfig, pLog, inputFiles);
	return Promise.resolve(outputFilesPromise).then(function(outputFiles) {
		_.difference(outputFiles, inputFiles).forEach(function(file) {
			files.add(file);
		});
		_.difference(inputFiles, outputFiles).forEach(function(file) {
			files.remove(file);
		});
	});
}

function processBatch(processor, inputs, outputs) {
	var inputSize = 0;
	inputs.forEach(function(input) {
		switch (input.type) {
			// TODO: Re-enable in memory caching of text and json parsed buffer? (used to be in File)
			case 'buffer': input.data = input.buffer; break;
			case 'text': input.data = input.buffer.toString(); break;
			case 'json': input.data = JSON.parse(input.buffer.toString()); break;
			default: throw new Error('Invalid type: ' + input.type);
		}
		if (input.trackRatio) {
			inputSize += input.buffer.length;
		}
		delete input.buffer;
	});

	outputs.forEach(function(output) {
		var resolve = output.resolve;
		output.resolve = function(data) {
			var buffer;
			switch (output.type) {
				case 'buffer': buffer = data; break;
				case 'text': buffer = new Buffer(data); break;
				case 'json': buffer = new Buffer(JSON.stringify(data, null, '\t')); break;
				default: throw new Error('Invalid type: ' + output.type);
			}
			resolve(buffer);
			var outputSize = buffer.length;
			if (output.trackRatio) {
				processor.log.ok('%s (%s %s)', output.path, bytes(outputSize), percentage(outputSize, inputSize));
			} else {
				processor.log.ok('%s', output.path);
			}
		};
	});

	// TODO: Distribute process calls over this and other threads to improve multi-core performance?
	return Promise.resolve(processor.process(inputs, outputs));
}

module.exports = function(execCommands) {
	var wendyStart = Date.now();

	files.init(minimatches(listFiles('.'), config.project.files));
	log.spam('Files:\n  %s', files.getPaths().join('\n  '));

	// Keep track of all processor promises to know when all async processing is resolved
	var processorPromises = [];

	// Processors selected through commands are turned into files task functions
	// These should return a Promise indicating they are done manipulating files
	var filesTasks = _.filter(commands.select(execCommands), 'processor').map(function(command) {
		var processorName = command.processor;
		var commandName = commands.getName(command);
		var shortName = commands.getName(command, true);
		var pConfig = _.extend({}, config[commandName], { project: config.project });
		var pLog = Logger.create(shortName);

		// TODO: Instantiate Processor here instead of in processor module so they do not depend on nbob
		var processor = require('./processor/' + processorName);

		// Legacy processor
		// TODO: Create LegacyProcessor that wraps this flow into a regular processor.getBatches() call
		if (processor instanceof Function) {
			return function() {
				var inputPaths = pConfig.files ? minimatches(files.getPaths(), pConfig.files) : files.getPaths();
				var processorStart = Date.now();
				pLog.debug('Starting');
				bar.add(shortName);
				return processLegacy(inputPaths, processor, pConfig, pLog).then(function() {
					pLog.info('Finished after: %s', timeSince(processorStart));
					bar.finish(shortName);
				});
			};
		}

		// TODO: Clean and split up this code to make it more readable/manageable
		// TODO: Improve caching approach to reduce number of files and amount of data read
		// E.g: Cache MD5 values of files in <cacheDir>.json and make file reading lazy
		return function() {
			if (!processor.initialized) {
				processor.init(pConfig, pLog);
			}

			var processorStart;
			var inputPaths = pConfig.files ? minimatches(files.getPaths(), pConfig.files) : files.getPaths();
			var batchesPromise = Promise.resolve(processor.getBatches(inputPaths, files.getList(inputPaths)));

			return batchesPromise.then(function(batches) {
				var nrBatches = batches.length;
				if (nrBatches === 0) {
					return;
				}
				pLog.debug('Starting %d batches', nrBatches);

				var batchPromises = batches.map(function(batch, batchIndex) {
					var inputs = batch.inputs;
					var outputs = batch.outputs;

					// Prepare input promises
					var inputPromises = inputs.map(function(input) {
						var file = files.get(input.path);
						if (!file) {
							throw new Error('File not found: ' + input.path + ', for command: ' + commandName);
						}
						return Promise.apply([ file.getBuffer(), file.getMD5() ], function(buffer, md5) {
							input.buffer = buffer;
							input.md5 = md5;
							return input;
						});
					});

					// Use existing or add new files and prepare output buffer promises and callbacks
					var bufferPromises = outputs.map(function(output) {
						var file = files.get(output.path) || files.add(output.path);
						return file.setBuffer(new Promise(function(resolve, reject) {
							output.resolve = function(buffer) {
								resolve(!output.write ? buffer : writeFile(output.path, buffer).then(function() {
									return buffer;
								}));
							};
							output.reject = reject;
						}));
					});

					// Remove files
					var inputPaths = _.pluck(_.filter(inputs, { isReadOnly: false }), 'path');
					var outputPaths = _.pluck(outputs, 'path');
					_.difference(inputPaths, outputPaths).forEach(function(removedPath) {
						files.remove(removedPath);
					});

					// Wait for input buffers to become available and start processing
					return Promise.all(inputPromises).then(function(inputs) {
						if (!processorStart) {
							processorStart = Date.now();
							bar.add(shortName, nrBatches);
						}
						var batchStart = Date.now();
						pLog.spam('Starting batch #%d: %s => %s', batchIndex,
							_.pluck(inputs, 'path').join(', '), outputPaths.join(', '));

						// Combination of file paths and md5s should provide a suitable md5 for these inputs
						// It also reduces overhead of md5 calculation by enabling in memory caching of input md5s
						var inputsDigestData = inputs.map(function(input) {
							return input.path + input.md5;
						});
						var processInputDigest = md5hex([ JSON.stringify(pConfig) ].concat(inputsDigestData), 'utf8');
						var cacheDir = unixPathJoin(config.project.buildDir, processorName, processInputDigest);
						var wasCached = true;

						return lstat(cacheDir).then(function() {
							return Promise.all(outputs.map(function(output) {
								var outputPath = unixPathJoin(cacheDir, output.path);
								return readFile(outputPath).then(output.resolve, function(error) {
									pLog.warn('Invalid cache: ' + error.message);
									throw error;
								});
							}));
						}).catch(function() {
							wasCached = false;
							return processBatch(processor, inputs, outputs).then(function() {
								return Promise.all(bufferPromises).then(function(buffers) {
									// Cache batch if not disabled and everything resolves
									// Note: create cache dir even if empty to enable caching of read-only batches
									// Note: do not block or reject on writing away cache files
									if (batch.doNotCache) {
										return;
									}
									mkdirp(cacheDir);
									buffers.forEach(function(buffer, i) {
										var outputPath = unixPathJoin(cacheDir, outputs[i].path);
										mkdirp(path.dirname(outputPath)).then(function() {
											log.spam('Caching: %s', outputPath);
											return writeFile(outputPath, buffer);
										}).catch(function(e) {
											log.warn('Caching %s failed: %s', outputPath, e.message);
										});
									});
								});
							});
						}).then(function() {
							pLog.spam('Finished batch #%d after: %s%s',
								batchIndex, timeSince(batchStart), wasCached ? ' (cached)' : '');
							bar.finish(shortName);
						});
					});
				});

				var processorPromise = Promise.all(batchPromises);
				processorPromise.then(function() {
					pLog.info('Finished %d batches after: %s', nrBatches, timeSince(processorStart));
				});
				processorPromises.push(processorPromise);

				// All files changes have been made already (add, remove, path change, setBuffer promise)
				// So we implicitly resolve this task now, allowing following tasks to start
			});
		};
	});

	// Call files tasks sequentially
	var wendyPromise = filesTasks.reduce(function(prevPromise, task) {
		return prevPromise.then(task);
	}, Promise.resolve()).then(function() {
		// Wait for all (including async) processing to finish
		return Promise.all(processorPromises).then(function() {
			log.info('All finished after: %s', timeSince(wendyStart));
		});
	});

	wendyPromise.catch(function() {
		bar.reset();
		log.info('Rejected after: %s', timeSince(wendyStart));
	});

	return wendyPromise;
};
<|endoftext|>