<commit_before>'use strict';

var EventEmitter = require('events').EventEmitter
  , util = require('util')
  , crypto = require('crypto')
  , net = require('net')
  , fs = require('fs')
  , C2SStream = require('./stream')

/**
 * params:
 *   options : port on which to listen to C2S connections
 *   options.port : xmpp tcp socket port
 *   options.domain : domain of xmpp server
 *   options.autostart : if we start listening at given port
 *   options.requestCert : expect a client certificate (see tls.createSecurePair for more)
 *   options.rejectUnauthorized : reject when client cert missmatches (see tls.createSecurePair for more)
 *   options.tls : wrapper object for tlc config
 *   options.tls.key : private key string
 *   options.tls.cert : certificate string
 *   options.tls.keyPath : path to key
 *   options.tls.certPath : path to certificate
 */
function C2SServer(options) {
    this.options = {}
    if (options) this.options = options
    this.availableSaslMechanisms = []
    this.c2sPort = null

    // don't allow anybody by default when using client cert auth
    if ((this.options.requestCert) &&
        (this.options.rejectUnauthorized !== false)) {
        this.options.rejectUnauthorized = true
    }

    /* And now start listening to connections on the
     * port provided as an option.
     */
    this._setupAutoStart()

    // Load TLS key material
    this._setupTls()
}

util.inherits(C2SServer, EventEmitter)

C2SServer.prototype.C2S_PORT = 5222

C2SServer.prototype._setupTls = function() {
    if (!this.options.tls) return
    var details = this.options.tls
    details.key = details.key || fs.readFileSync(details.keyPath, 'ascii')
    details.cert = details.cert || fs.readFileSync(details.certPath, 'ascii')
    this.credentials = crypto.createCredentials(details)
}

C2SServer.prototype._setupAutoStart = function() {
    if (this.options.autostart === false) return
    var self = this
    var port = this.options.port || this.C2S_PORT
    var bindAddress = this.options.bindAddress || '::'
    this.c2sPort = net.createServer(function(inStream) {
        self.acceptConnection(inStream)
    }).listen(port, bindAddress)
}
/**
 * Called upon TCP connection from client. */
C2SServer.prototype.acceptConnection = function(socket) {
    var stream = new C2SStream({
        rejectUnauthorized: this.options.rejectUnauthorized,
        requestCert: this.options.requestCert,
        socket: socket,
        server: this
    })
    this.emit('connect', stream)
    socket.addListener('error', function() {})
    stream.addListener('error', function() {})
}

C2SServer.prototype.registerSaslMechanism = function() {
    var args = arguments.length > 0 ? Array.prototype.slice.call(arguments) : []
    this.availableSaslMechanisms = this.availableSaslMechanisms.concat(args)
}

C2SServer.prototype.shutdown = function() {

    // we have to shutdown all connections
    this.emit('shutdown')

    // shutdown server
    this.c2sPort.close()
}

module.exports = C2SServer<commit_msg>api to set modified C2SStreams<commit_after>'use strict';

var EventEmitter = require('events').EventEmitter
  , util = require('util')
  , crypto = require('crypto')
  , net = require('net')
  , fs = require('fs')
  , C2SStream = require('./stream')

/**
 * params:
 *   options : port on which to listen to C2S connections
 *   options.port : xmpp tcp socket port
 *   options.domain : domain of xmpp server
 *   options.autostart : if we start listening at given port
 *   options.requestCert : expect a client certificate (see tls.createSecurePair for more)
 *   options.rejectUnauthorized : reject when client cert missmatches (see tls.createSecurePair for more)
 *   options.tls : wrapper object for tlc config
 *   options.tls.key : private key string
 *   options.tls.cert : certificate string
 *   options.tls.keyPath : path to key
 *   options.tls.certPath : path to certificate
 */
function C2SServer(options) {
    this.options = {}
    if (options) this.options = options
    this.availableSaslMechanisms = []
    this.C2SStream = C2SStream
    this.c2sPort = null

    // don't allow anybody by default when using client cert auth
    if ((this.options.requestCert) &&
        (this.options.rejectUnauthorized !== false)) {
        this.options.rejectUnauthorized = true
    }

    /* And now start listening to connections on the
     * port provided as an option.
     */
    this._setupAutoStart()

    // Load TLS key material
    this._setupTls()
}

util.inherits(C2SServer, EventEmitter)

C2SServer.prototype.C2S_PORT = 5222

C2SServer.prototype._setupTls = function() {
    if (!this.options.tls) return
    var details = this.options.tls
    details.key = details.key || fs.readFileSync(details.keyPath, 'ascii')
    details.cert = details.cert || fs.readFileSync(details.certPath, 'ascii')
    this.credentials = crypto.createCredentials(details)
}

C2SServer.prototype._setupAutoStart = function() {
    if (this.options.autostart === false) return
    var self = this
    var port = this.options.port || this.C2S_PORT
    var bindAddress = this.options.bindAddress || '::'
    this.c2sPort = net.createServer(function(inStream) {
        self.acceptConnection(inStream)
    }).listen(port, bindAddress)
}
/**
 * Called upon TCP connection from client.
 * If you want to use your own C2SStream class just overwrite server.C2SStream*/
C2SServer.prototype.acceptConnection = function(socket) {
    var stream = new this.C2SStream({
        rejectUnauthorized: this.options.rejectUnauthorized,
        requestCert: this.options.requestCert,
        socket: socket,
        server: this
    })
    this.emit('connect', stream)
    socket.addListener('error', function() {})
    stream.addListener('error', function() {})
}

C2SServer.prototype.registerSaslMechanism = function() {
    var args = arguments.length > 0 ? Array.prototype.slice.call(arguments) : []
    this.availableSaslMechanisms = this.availableSaslMechanisms.concat(args)
}

C2SServer.prototype.shutdown = function() {

    // we have to shutdown all connections
    this.emit('shutdown')

    // shutdown server
    this.c2sPort.close()
}

module.exports = C2SServer<|endoftext|>


<commit_before>// Copyright 2016 Luca-SAS, licensed under the Apache License 2.0

'use strict';

var fs = require('fs');
var util = require('util');
var url = require('url');
var zlib = require('zlib');
var uuid = require('node-uuid');
var mkdirp = require('mkdirp');

var SkaleClient = require('./client.js');
var dataset = require('./dataset.js');
var Task = require('./task.js');

module.exports = SkaleContext;

var start = process.hrtime();

util.inherits(SkaleContext, SkaleClient);

function SkaleContext(arg) {
  if (!(this instanceof SkaleContext))
    return new SkaleContext(arg);
  var self = this;

  arg = arg || {};
  arg.data = arg.data || {};
  arg.data.type = 'master';
  SkaleClient.call(this, arg);
  var maxWorker = process.env.SKALE_MAX_WORKER;
  var tmp = arg.tmp || process.env.SKALE_TMP || '/tmp';

  this.started = this.ended = false;
  this.jobId = 0;
  this.worker = [];
  this.log = log;
  this.contextId = uuid.v4(); // context id which will be used as scratch directory name

  this.basedir = tmp + '/skale/' + this.contextId + '/';
  mkdirp.sync(this.basedir + 'tmp');
  mkdirp.sync(this.basedir + 'stream');

  this.once('connect', function(data) {
    var i;
    process.title = 'skale-master_' + data.devices[0].wsid + ' ' + __filename;
    if (!maxWorker || maxWorker > data.devices.length)
      maxWorker = data.devices.length;
    for (i = 0; i < maxWorker; i++) {
      self.worker.push(new Worker(data.devices[i]));
    }
    self.started = true;
  });

  this.on('workerError', function workerError(msg) {
    console.error('Error from worker id %d:', msg.from);
    console.error(msg.args);
  });

  this.on('remoteClose', function getWorkerClose() {
    throw 'Fatal error: unexpected worker exit';
  });

  this.getWorkers = function (callback) {
    if (self.started) return callback();
    this.once('connect', callback);
  };

  function Worker(w) {
    this.uuid = w.uuid;
    this.id = w.id;
    this.ip = w.ip;
    this.ntask = 0;
  }

  Worker.prototype.rpc = function (cmd, args, done) {
    self.request({uuid: this.uuid, id: this.id}, {cmd: cmd, args: args}, done);
  };

  Worker.prototype.send = function (cmd, args) {
    self.send(this.uuid, {cmd: cmd, args: args});
  };

  this.on('request', function (msg) { // Protocol to handle stream flow control: reply when data is consumed
    if (msg.data.cmd == 'stream') {
      self.emit(msg.data.stream, msg.data.data, function() {
        try {self.reply(msg);} catch(err) { console.log(err); }
      });
    }
  });

  this.on('sendFile', function (msg) {
    fs.createReadStream(msg.path, msg.opt).pipe(self.createStreamTo(msg));
  });

  this.end = function () {
    if (self.ended) return;
    self.ended = true;
    if (this.started) self.set({complete: 1});
    self._end();
  };

  this.datasetIdCounter = 0;  // global dataset id counter

  this.parallelize = function (localArray, nPartitions) { return dataset.parallelize(this, localArray, nPartitions);};
  this.range = function (start, end, step, nPartitions) { return dataset.range(this, start, end, step, nPartitions);};
  this.lineStream = function (stream, config) {return new dataset.Stream(this, stream, 'line', config);};
  this.objectStream = function (stream, config) {return new dataset.Stream(this, stream, 'object', config);};

  this.textFile = function (file, nPartitions) {
    var u = url.parse(file);

    if (u.protocol === 's3:') {
      if (file.slice(-1) === '/')
        return new dataset.TextS3Dir(this, file.slice(5, -1));
      return new dataset.TextS3File(this, file.slice(5));
    }
    if (file.slice(-1) === '/')
      return new dataset.TextDir(this, file);
    if (file.slice(-3) === '.gz')
      return new dataset.GzipFile(this, file);
    return new dataset.TextFile(this, file, nPartitions);
  };

  this.getReadStream = function (fileObj, opt) {
    try {
      return fs.createReadStream(fileObj.path, opt);
    } catch (err) {
      return this.createStreamFrom(fileObj.host, {cmd: 'sendFile', path: fileObj.path, opt: opt});
    }
  };

  this.runTask = function(task, callback) {
    function getLeastBusyWorkerId(/* preferredLocation */) {
      var wid, ntask;
      for (var i = 0; i < self.worker.length; i++) {
        if ((ntask == undefined) || (ntask > self.worker[i].ntask)) {
          ntask = self.worker[i].ntask;
          wid = i;
        }
      }
      return wid;
    }

    function serialize(obj) {
      return str = JSON.stringify(obj,function(key, value) {
        if (key == 'sc') return undefined;
        if (key == 'dependencies') {
          var dep = [];
          for (var i = 0; i < value.length; i++) dep[i] = value[i].id;
          return dep;
        }
        return (typeof value === 'function' ) ? value.toString() : value;
      });
    }

    var wid = getLeastBusyWorkerId(task.nodes[task.datasetId].getPreferedLocation(task.pid));
    this.worker[wid].ntask++;
    var str = serialize(task);
    log('task size for worker ' + wid + ':', str.length);
    if (str.length > 1000000) {
      zlib.gzip(str, {chunkSize: 65536}, function (err, res) {
        var filename = task.basedir + 'task-' + uuid.v4() + '.gz';
        fs.writeFile(filename, res, function (err) {
          if (err) throw new Error(err);
          rpc('runztask', wid, filename, function (err, res) {
            self.worker[wid].ntask--;
            callback(err, res);
          });
        });
      });
    } else {
      rpc('runTask', wid, str, function(err, res) {
        self.worker[wid].ntask--;
        callback(err, res);
      });
    }
  };

  this.runJob = function(opt, root, action, callback) {
    var jobId = this.jobId++;
    var stream = new this.createReadStream(jobId, opt); // user readable stream instance

    this.getWorkers(function () {
      findShuffleStages(function(shuffleStages) {
        if (shuffleStages.length == 0) runResultStage();
        else {
          var cnt = 0;
          runShuffleStage(shuffleStages[cnt], shuffleDone);
        }
        function shuffleDone() {
          if (++cnt < shuffleStages.length) runShuffleStage(shuffleStages[cnt], shuffleDone);
          else runResultStage();
        }
      });
    });

    function runShuffleStage(stage, done) {
      findNodes(stage, function(nodes) {
        var pid = 0, shuffleTasks = [], cnt = 0, i, j;
        stage.shufflePartitions = [];

        for (i = 0; i < stage.dependencies.length; i++) {
          for (j = 0; j < stage.dependencies[i].partitions.length; j++)
            stage.shufflePartitions[pid++] = new dataset.Partition(stage.id, pid, stage.dependencies[i].id, stage.dependencies[i].partitions[j].partitionIndex);
        }

        for (i = 0; i < stage.shufflePartitions.length; i++)
          shuffleTasks.push(new Task(self.basedir, jobId, nodes, stage.id, i));

        for (i = 0; i < shuffleTasks.length; i++) {
          self.runTask(shuffleTasks[i], function (err, res) {
            stage.shufflePartitions[res.pid].files = res.files;
            if (++cnt < shuffleTasks.length) return;
            stage.executed = true;
            done();
          });
        }
      });
    }

    function runResultStage() {
      findNodes(root, function(nodes) {
        var tasks = [];
        for (var i = 0; i < root.partitions.length; i++)
          tasks.push(new Task(self.basedir, jobId, nodes, root.id, i, action));
        callback({id: jobId, stream: stream}, tasks);
      });
    }

    function findNodes(node, done) {
      var nodes = {};
      interruptibleTreewalk(node, function cin(n, done) {
        done(n.shuffling && n.executed);
      }, function cout(n, done) {
        n.getPartitions(function() {
          if (nodes[n.id] == undefined) nodes[n.id] = n;
          done();
        });
      }, function() {done(nodes);});
    }

    function findShuffleStages(callback) {
      var stages = [];
      interruptibleTreewalk(root, function cin(n, done) {
        if (n.shuffling && !n.executed) stages.unshift(n);
        done(n.shuffling && n.executed);  // stage boundary are shuffle nodes
      }, function cout(n, done) {done();}, function() {callback(stages);});
    }

    function interruptibleTreewalk(n, cin, cout, done) {
      cin(n, function(uturn) { // if uturn equals true the subtree under node won't be treewalked
        if (!uturn) {
          var nDependencies = 0;
          for (var i = 0; i < n.dependencies.length; i++)
            interruptibleTreewalk(n.dependencies[i], cin, cout, function() {
              if (++nDependencies == n.dependencies.length) cout(n, done);
            });
          if (n.dependencies.length == 0) cout(n, done);
        } else cout(n, done);
      });
    }

    return stream;
  };

  function rpc(cmd, workerNum, args, callback) {
    self.request(self.worker[workerNum], {cmd: cmd, args: args, master_uuid: self.uuid, worker: self.worker}, callback);
  }
}

if (process.env.SKALE_DEBUG) {
  var log =  function() {
    var args = Array.prototype.slice.call(arguments);
    var elapsed = process.hrtime(start);
    args.unshift('[master ' + (elapsed[0] + elapsed[1] / 1e9).toFixed(3) + 's]');
    console.error.apply(null, args);
  };
} else {
  log = function () {};
}
<commit_msg>distributed mode: limit number of tasks in parallel to the number of workers<commit_after>// Copyright 2016 Luca-SAS, licensed under the Apache License 2.0

'use strict';

var fs = require('fs');
var util = require('util');
var url = require('url');
var zlib = require('zlib');
var uuid = require('node-uuid');
var mkdirp = require('mkdirp');

var SkaleClient = require('./client.js');
var dataset = require('./dataset.js');
var Task = require('./task.js');

module.exports = SkaleContext;

var start = process.hrtime();

util.inherits(SkaleContext, SkaleClient);

function SkaleContext(arg) {
  if (!(this instanceof SkaleContext))
    return new SkaleContext(arg);
  var self = this;

  arg = arg || {};
  arg.data = arg.data || {};
  arg.data.type = 'master';
  SkaleClient.call(this, arg);
  var maxWorker = process.env.SKALE_MAX_WORKER;
  var tmp = arg.tmp || process.env.SKALE_TMP || '/tmp';

  this.started = this.ended = false;
  this.jobId = 0;
  this.worker = [];
  this.log = log;
  this.contextId = uuid.v4(); // context id which will be used as scratch directory name

  this.basedir = tmp + '/skale/' + this.contextId + '/';
  mkdirp.sync(this.basedir + 'tmp');
  mkdirp.sync(this.basedir + 'stream');

  this.once('connect', function(data) {
    var i;
    process.title = 'skale-master_' + data.devices[0].wsid + ' ' + __filename;
    if (!maxWorker || maxWorker > data.devices.length)
      maxWorker = data.devices.length;
    for (i = 0; i < maxWorker; i++) {
      self.worker.push(new Worker(data.devices[i]));
    }
    self.started = true;
  });

  this.on('workerError', function workerError(msg) {
    console.error('Error from worker id %d:', msg.from);
    console.error(msg.args);
  });

  this.on('remoteClose', function getWorkerClose() {
    throw 'Fatal error: unexpected worker exit';
  });

  this.getWorkers = function (callback) {
    if (self.started) return callback();
    this.once('connect', callback);
  };

  function Worker(w) {
    this.uuid = w.uuid;
    this.id = w.id;
    this.ip = w.ip;
    this.ntask = 0;
  }

  Worker.prototype.rpc = function (cmd, args, done) {
    self.request({uuid: this.uuid, id: this.id}, {cmd: cmd, args: args}, done);
  };

  Worker.prototype.send = function (cmd, args) {
    self.send(this.uuid, {cmd: cmd, args: args});
  };

  this.on('request', function (msg) { // Protocol to handle stream flow control: reply when data is consumed
    if (msg.data.cmd == 'stream') {
      self.emit(msg.data.stream, msg.data.data, function() {
        try {self.reply(msg);} catch(err) { console.log(err); }
      });
    }
  });

  this.on('sendFile', function (msg) {
    fs.createReadStream(msg.path, msg.opt).pipe(self.createStreamTo(msg));
  });

  this.end = function () {
    if (self.ended) return;
    self.ended = true;
    if (this.started) self.set({complete: 1});
    self._end();
  };

  this.datasetIdCounter = 0;  // global dataset id counter

  this.parallelize = function (localArray, nPartitions) { return dataset.parallelize(this, localArray, nPartitions);};
  this.range = function (start, end, step, nPartitions) { return dataset.range(this, start, end, step, nPartitions);};
  this.lineStream = function (stream, config) {return new dataset.Stream(this, stream, 'line', config);};
  this.objectStream = function (stream, config) {return new dataset.Stream(this, stream, 'object', config);};

  this.textFile = function (file, nPartitions) {
    var u = url.parse(file);

    if (u.protocol === 's3:') {
      if (file.slice(-1) === '/')
        return new dataset.TextS3Dir(this, file.slice(5, -1));
      return new dataset.TextS3File(this, file.slice(5));
    }
    if (file.slice(-1) === '/')
      return new dataset.TextDir(this, file);
    if (file.slice(-3) === '.gz')
      return new dataset.GzipFile(this, file);
    return new dataset.TextFile(this, file, nPartitions);
  };

  this.getReadStream = function (fileObj, opt) {
    try {
      return fs.createReadStream(fileObj.path, opt);
    } catch (err) {
      return this.createStreamFrom(fileObj.host, {cmd: 'sendFile', path: fileObj.path, opt: opt});
    }
  };

  this.runTask = function(task, callback) {
    function getLeastBusyWorkerId(/* preferredLocation */) {
      var wid, ntask;
      for (var i = 0; i < self.worker.length; i++) {
        if ((ntask == undefined) || (ntask > self.worker[i].ntask)) {
          ntask = self.worker[i].ntask;
          wid = i;
        }
      }
      return wid;
    }

    function serialize(obj) {
      return str = JSON.stringify(obj,function(key, value) {
        if (key == 'sc') return undefined;
        if (key == 'dependencies') {
          var dep = [];
          for (var i = 0; i < value.length; i++) dep[i] = value[i].id;
          return dep;
        }
        return (typeof value === 'function' ) ? value.toString() : value;
      });
    }

    var wid = getLeastBusyWorkerId(task.nodes[task.datasetId].getPreferedLocation(task.pid));
    this.worker[wid].ntask++;
    var str = serialize(task);
    log('task size for worker ' + wid + ':', str.length);
    if (str.length > 1000000) {
      zlib.gzip(str, {chunkSize: 65536}, function (err, res) {
        var filename = task.basedir + 'task-' + uuid.v4() + '.gz';
        fs.writeFile(filename, res, function (err) {
          if (err) throw new Error(err);
          rpc('runztask', wid, filename, function (err, res) {
            self.worker[wid].ntask--;
            callback(err, res);
          });
        });
      });
    } else {
      rpc('runTask', wid, str, function(err, res) {
        self.worker[wid].ntask--;
        callback(err, res);
      });
    }
  };

  this.runJob = function(opt, root, action, callback) {
    var jobId = this.jobId++;
    var stream = new this.createReadStream(jobId, opt); // user readable stream instance

    this.getWorkers(function () {
      findShuffleStages(function(shuffleStages) {
        if (shuffleStages.length == 0) runResultStage();
        else {
          var cnt = 0;
          runShuffleStage(shuffleStages[cnt], shuffleDone);
        }
        function shuffleDone() {
          if (++cnt < shuffleStages.length) runShuffleStage(shuffleStages[cnt], shuffleDone);
          else runResultStage();
        }
      });
    });

    function runShuffleStage(stage, done) {
      findNodes(stage, function(nodes) {
        var pid = 0, shuffleTasks = [], i, j;
        stage.shufflePartitions = [];

        for (i = 0; i < stage.dependencies.length; i++) {
          for (j = 0; j < stage.dependencies[i].partitions.length; j++)
            stage.shufflePartitions[pid++] = new dataset.Partition(stage.id, pid, stage.dependencies[i].id, stage.dependencies[i].partitions[j].partitionIndex);
        }

        for (i = 0; i < stage.shufflePartitions.length; i++)
          shuffleTasks.push(new Task(self.basedir, jobId, nodes, stage.id, i));

        function runBatch(ntasks, index, done) {
          var cnt = 0, tmax = index + ntasks - 1;
          log('start shuffle tasks', index + '-' + tmax);
          for (var i = 0; i < ntasks; i ++) {
            self.runTask(shuffleTasks[index + i], function (err, res) {
              stage.shufflePartitions[res.pid].files = res.files;
              if (++cnt < ntasks) return;
              index += ntasks;
              ntasks = shuffleTasks.length - index;
              log('finished shuffle tasks:', index + '/' + shuffleTasks.length);
              if (ntasks > maxWorker) ntasks = maxWorker;
              if (ntasks > 0) return runBatch(ntasks, index, done);
              stage.executed = true;
              done();
            });
          }
        }

        var n = shuffleTasks.length < maxWorker ? shuffleTasks.length : maxWorker;
        log('start shuffle stage, partitions:', stage.shufflePartitions.length);
        runBatch(n, 0, done);
      });
    }

    function runResultStage() {
      findNodes(root, function(nodes) {
        var tasks = [];
        for (var i = 0; i < root.partitions.length; i++)
          tasks.push(new Task(self.basedir, jobId, nodes, root.id, i, action));
        callback({id: jobId, stream: stream}, tasks);
      });
    }

    function findNodes(node, done) {
      var nodes = {};
      interruptibleTreewalk(node, function cin(n, done) {
        done(n.shuffling && n.executed);
      }, function cout(n, done) {
        n.getPartitions(function() {
          if (nodes[n.id] == undefined) nodes[n.id] = n;
          done();
        });
      }, function() {done(nodes);});
    }

    function findShuffleStages(callback) {
      var stages = [];
      interruptibleTreewalk(root, function cin(n, done) {
        if (n.shuffling && !n.executed) stages.unshift(n);
        done(n.shuffling && n.executed);  // stage boundary are shuffle nodes
      }, function cout(n, done) {done();}, function() {callback(stages);});
    }

    function interruptibleTreewalk(n, cin, cout, done) {
      cin(n, function(uturn) { // if uturn equals true the subtree under node won't be treewalked
        if (!uturn) {
          var nDependencies = 0;
          for (var i = 0; i < n.dependencies.length; i++)
            interruptibleTreewalk(n.dependencies[i], cin, cout, function() {
              if (++nDependencies == n.dependencies.length) cout(n, done);
            });
          if (n.dependencies.length == 0) cout(n, done);
        } else cout(n, done);
      });
    }

    return stream;
  };

  function rpc(cmd, workerNum, args, callback) {
    self.request(self.worker[workerNum], {cmd: cmd, args: args, master_uuid: self.uuid, worker: self.worker}, callback);
  }
}

if (process.env.SKALE_DEBUG) {
  var log =  function() {
    var args = Array.prototype.slice.call(arguments);
    var elapsed = process.hrtime(start);
    args.unshift('[master ' + (elapsed[0] + elapsed[1] / 1e9).toFixed(3) + 's]');
    console.error.apply(null, args);
  };
} else {
  log = function () {};
}
<|endoftext|>


<commit_before>/**
 * @author Gilles Coomans <gilles.coomans@gmail.com>
 * an observable data holder.
 * it's the heart of yamvish data-binding.
 */

var utils = require('./utils'),
	env = require('./env'),
	AsyncManager = require('./async');
//_______________________________________________________ DATA BIND CONTEXT

function Context(data, parent, path, env) {
	AsyncManager.call(this);
	this.__yContext__ = true;
	this.data = (data !== undefined) ? data : {};
	this.map = {};
	if (parent) {
		this.parent = parent;
		this.env = env ? new Context(env) : parent.env;
		if (path) {
			var self = this;
			this.path = path;
			this.binds = [];
			this.parent.subscribe(path, function(value) {
				self.reset(value);
			}, false, this.binds);
		}
	} else
		this.env = env ? (env.__yContext__ ? env : new Context(env)) : Context.env;
}

function unsub(context, path, fn, upstream) {
	return function() {
		if (!context.destroyed)
			context.unsubscribe(path, fn, upstream);
	};
}

Context.prototype = {
	destroy: function() {
		if (this.binds)
			this.binds.forEach(function(unbind) {
				unbind();
			});
		this.destroyed = true;
		this.binds = null;
		this.parent = null;
		this.data = null;
		this.map = null;
	},
	get: function(path) {
		if (!path.forEach)
			path = path.split('.');
		switch (path[0]) {
			case '$this':
				if (path.length === 1)
					return this.data;
				break;
			case '$parent':
				if (!this.parent)
					throw new Error('yamvish.Context : there is no parent in current context. could not find : ' + path.join('.'));
				return this.parent.get(path.slice(1));
				break;
			case '$env':
				return this.env.get(path.slice(1));
		}
		var r = utils.getProp(this.data, path);
		if (r === undefined)
			return '';
		return r;
	},
	dependent: function(path, dependencies, func) {
		var argsOutput = [],
			willFire,
			self = this,
			count = 0;
		this.binds = this.binds ||  [];
		dependencies.forEach(function(dependency) {
			argsOutput.push(this.get(dependency));
			// subscribe to arguments[i]
			var index = count++; // localise var in scope for local func closure below
			this.subscribe(dependency, function(value, type, p, key) {
				argsOutput[index] = value;
				if (!willFire)
					willFire = self.delay(function() {
						if (willFire) {
							willFire = null;
							self.set(path, func.apply(self, argsOutput));
						}
					}, 0);
			}, false, this.binds);
		}, this);
		this.set(path, func.apply(this, argsOutput));
		return this;
	},
	reset: function(data) {
		if (data === this.data || (this.data instanceof Date && data instanceof Date && this.data.valueOf() === data.valueOf()))
			return this;
		this.data = data || {};
		this.notifyAll('reset', null, this.map, data, '*');
		return this;
	},
	toggle: function(path, value) {
		this.set(path, !this.get(path));
		return this;
	},
	toggleInArray: function(path, value) {
		var arr = this.get(path);
		for (var i = 0, len = arr.length; i < len; ++i)
			if (arr[i] === value) {
				this.del(path + '.' + i)
				return this;
			}
		return this.push(path, value);
	},
	set: function(path, value) {
		if (arguments.length === 1) {
			for (var i in path)
				this.set(i, path[i]);
			return this;
		}
		if (!path.forEach)
			path = path.split('.');
		switch (path[0]) {
			case '$this':
				if (path.length === 1)
					return this.reset(value);
				else
					path.shift();
				break;
			case '$parent':
				if (this.parent)
					return this.parent.set(path.slice(1), value) && this;
				throw new Error('yamvish.Context : there is no parent in current context. could not find : ' + path.join('.'));
			case '$env':
				return this.env.set(path.slice(1), value) && this;
		}
		var old = utils.setProp(this.data, path, value);
		if (old !== value) {
			if (old instanceof Date && value instanceof Date && old == value)
				return this;
			this.notify('set', path, value, path[path.length - 1]);
		}
		return this;
	},
	push: function(path, value) {
		if (!path.forEach)
			path = path.split('.');
		switch (path[0]) {
			case '$parent':
				if (this.parent)
					return this.parent.push(path.slice(1), value) && this;
				throw new Error('yamvish.Context : there is no parent in current context. could not find : ' + path.join('.'));
			case '$env':
				return this.env.push(path.slice(1), value) && this;
		}
		var arr;
		if (path[0] === '$this')
			arr = this.data;
		else
			arr = utils.getProp(this.data, path);
		if (!arr)
			throw new Error("yamvish.Context : Missing array at " + path.join(".") + " : couldn't push object.");
		if (!arr.forEach) {
			console.error(path, 'is not array at context.push : ', arr);
			throw new Error("yamvish.Context : Object is not array at " + path.join(".") + " : couldn't push object.");
		}
		arr.push(value);
		this.notify('push', path, value, arr.length - 1);
		return this;
	},
	del: function(path) {
		if (!path.forEach)
			path = path.split('.');
		else
			path = path.slice();
		switch (path[0]) {
			case '$parent':
				if (this.parent)
					return this.parent.del(path.slice(1)) && this;
				throw new Error('yamvish.Context : there is no parent in current context. could not find : ' + path.join('.'));
			case '$env':
				return this.env.del(path.slice(1)) && this;
		}
		var path2 = path.slice();
		var key = path2.pop(),
			parent = path2.length ? utils.getProp(this.data, path2) : this.data;
		if (parent)
			if (parent.forEach) {
				var index = parseInt(key, 10);
				if (index < parent.length)
					this.notify('removeAt', path2, parent.splice(index, 1), index);
			} else {
				var val = parent[key];
				delete parent[key];
				this.notify('delete', path, val, key);
			}
		return this;
	},
	subscribe: function(path, fn, upstream, binds) {
		// console.log('context subscribe : ', path, fn, upstream);
		if (!path.forEach)
			path = path.split('.');
		var space;
		switch (path[0]) {
			case '$this':
				space = this.map;
				break;
			case '$env':
				return this.env.subscribe(path.slice(1), fn, upstream, binds);
			case '$parent':
				if (!this.parent)
					throw new Error('yamvish.Context : there is no parent in current context. could not find : ' + path.join('.'));
				return this.parent.subscribe(path.slice(1), fn, upstream, binds);
			default:
				space = utils.getProp(this.map, path);
		}
		if (upstream) {
			if (!space)
				utils.setProp(this.map, path, {
					_upstreams: [fn]
				});
			else
				(space._upstreams = space._upstreams || []).push(fn);
		} else if (!space)
			utils.setProp(this.map, path, {
				_listeners: [fn]
			});
		else
			(space._listeners = space._listeners || []).push(fn);
		if (binds)
			binds.push(unsub(this, path, fn, upstream));
		return this;
	},
	unsubscribe: function(path, fn, upstream) {
		if (this.distroyed)
			return this;
		if (!path.forEach)
			path = path.split('.');
		var space;
		switch (path[0]) {
			case '$this':
				space = this.map;
				break;
			case '$parent':
				return this.env.unsubscribe(path.slice(1), fn, upstream) && this;
			case '$parent':
				if (this.parent)
					return this.parent.unsubscribe(path.slice(1), fn, upstream) && this;
				throw new Error('yamvish.Context : there is no parent in current context. could not find : ' + path.join('.'));
			default:
				space = utils.getProp(this.map, path);
		}
		if (!space)
			return this;
		var arr = upstream ? space._upstreams : space._listeners;
		for (var i = 0, len = arr.length; i < len; ++i)
			if (arr[i] === fn) {
				arr.splice(i, 1);
				break;
			}
		return this;
	},
	notifyAll: function(type, path, space, value, index) {
		space = space ||  this.map;
		value = (arguments.length < 2) ? this.data : value;
		if (space._listeners)
			for (var i = 0, len = space._listeners.length; i < len; ++i) {
				var listener = space._listeners[i];
				if (!listener) {
					// maybe it's because listeners length has change so update it
					len = space._listeners.length;
					continue;
				}
				var r = listener.call(this, value, type, path, index);
				if (r && r.then)
					this.waiting(r);
			}
		if (type !== 'push' && type !== 'removeAt')
			for (var j in space) {
				if (j === '_listeners' || j === '_upstreams')
					continue;
				this.notifyAll(type, path, space[j], value ? value[j] : undefined, index);
			}
		return this;
	},
	notify: function(type, path, value, index) {
		if (!path.forEach)
			path = path.split('.');
		if (path[0] === '$this')
			path = path.slice(1);
		else if (path[0] === '$parent') {
			if (!this.parent)
				throw new Error('could not notify parent : no parent found. for : ' + path.join('.'));
			this.parent.notify(type, path.slice(1), value, index);
			return this;
		}
		var space = this.map,
			i = 0,
			star;
		for (var len = path.length; i < len; ++i) {
			star = space['*'];
			if (star && star._upstreams)
				notifyUpstreams.call(this, star, type, path, value, index);
			if (!(space = space[path[i]]))
				break;
			if (space._upstreams)
				notifyUpstreams.call(this, space, type, path, value, index);
		}
		if (star)
			this.notifyAll(type, path, star, value, index);
		if (space)
			this.notifyAll(type, path, space, value, index);
		return this;
	},
	setAsync: function(path, promise) {
		var self = this;
		return this.waiting(promise.then(function(s) {
			self.set(path, s);
			return s;
		}, function(e) {
			console.error('error while Context.setAsync : ', e);
			throw e;
		}));
	},
	pushAsync: function(path, promise) {
		var self = this;
		return this.waiting(promise.then(function(s) {
			self.push(path, s);
			return s;
		}, function(e) {
			console.error('error while Context.pushAsync : ', e);
			throw e;
		}));
	},
	//______________________ AGORA MANAGEMENT
	onAgora: function(messageName, handler) {
		var agora = this.env.data.agora,
			self = this;
		agora.on(messageName, handler, [this]);
		(this.binds = this.binds ||  []).push(function() {
			agora.off(messageName, handler);
		});
		return this;
	},
	toAgora: function(name) {
		var args = [name].concat([].slice.call(arguments, 1))
		this.env.data.agora.emit.apply(this.env.data.agora, args);
		return this;
	},
	output: function() {
		var output = {};
		for (var i in this.data)
			if (!outputFiler.test(i))
				output[i] = this.data[i];
		return output;
	}
};

var outputFiler = /^\$/;
// general env
Context.env = new Context(env);
delete Context.env.env;

utils.shallowMerge(AsyncManager.prototype, Context.prototype);

function notifyUpstreams(space, type, path, value, index) {
	for (var i = 0, len = space._upstreams.length; i < len; ++i) {
		var upstream = space._upstreams[i];
		if (!upstream) {
			// maybe it's because upstreams length has change so update it
			len = space._upstreams.length;
			continue;
		}
		var r = upstream.call(this, value, type, path, index);
		if (r && r.then)
			this.waiting(r);
	}
}

module.exports = Context;
<commit_msg>add auto creation of array on push when not defined<commit_after>/**
 * @author Gilles Coomans <gilles.coomans@gmail.com>
 * an observable data holder.
 * it's the heart of yamvish data-binding.
 */

var utils = require('./utils'),
	env = require('./env'),
	AsyncManager = require('./async');
//_______________________________________________________ DATA BIND CONTEXT

function Context(data, parent, path, env) {
	AsyncManager.call(this);
	this.__yContext__ = true;
	this.data = (data !== undefined) ? data : {};
	this.map = {};
	if (parent) {
		this.parent = parent;
		this.env = env ? new Context(env) : parent.env;
		if (path) {
			var self = this;
			this.path = path;
			this.binds = [];
			this.parent.subscribe(path, function(value) {
				self.reset(value);
			}, false, this.binds);
		}
	} else
		this.env = env ? (env.__yContext__ ? env : new Context(env)) : Context.env;
}

function unsub(context, path, fn, upstream) {
	return function() {
		if (!context.destroyed)
			context.unsubscribe(path, fn, upstream);
	};
}

Context.prototype = {
	destroy: function() {
		if (this.binds)
			this.binds.forEach(function(unbind) {
				unbind();
			});
		this.destroyed = true;
		this.binds = null;
		this.parent = null;
		this.data = null;
		this.map = null;
	},
	get: function(path) {
		if (!path.forEach)
			path = path.split('.');
		switch (path[0]) {
			case '$this':
				if (path.length === 1)
					return this.data;
				break;
			case '$parent':
				if (!this.parent)
					throw new Error('yamvish.Context : there is no parent in current context. could not find : ' + path.join('.'));
				return this.parent.get(path.slice(1));
				break;
			case '$env':
				return this.env.get(path.slice(1));
		}
		var r = utils.getProp(this.data, path);
		if (r === undefined)
			return '';
		return r;
	},
	dependent: function(path, dependencies, func) {
		var argsOutput = [],
			willFire,
			self = this,
			count = 0;
		this.binds = this.binds ||  [];
		dependencies.forEach(function(dependency) {
			argsOutput.push(this.get(dependency));
			// subscribe to arguments[i]
			var index = count++; // localise var in scope for local func closure below
			this.subscribe(dependency, function(value, type, p, key) {
				argsOutput[index] = value;
				if (!willFire)
					willFire = self.delay(function() {
						if (willFire) {
							willFire = null;
							self.set(path, func.apply(self, argsOutput));
						}
					}, 0);
			}, false, this.binds);
		}, this);
		this.set(path, func.apply(this, argsOutput));
		return this;
	},
	reset: function(data) {
		if (data === this.data || (this.data instanceof Date && data instanceof Date && this.data.valueOf() === data.valueOf()))
			return this;
		this.data = data || {};
		this.notifyAll('reset', null, this.map, data, '*');
		return this;
	},
	toggle: function(path, value) {
		this.set(path, !this.get(path));
		return this;
	},
	toggleInArray: function(path, value) {
		var arr = this.get(path);
		for (var i = 0, len = arr.length; i < len; ++i)
			if (arr[i] === value) {
				this.del(path + '.' + i)
				return this;
			}
		return this.push(path, value);
	},
	set: function(path, value) {
		if (arguments.length === 1) {
			for (var i in path)
				this.set(i, path[i]);
			return this;
		}
		if (!path.forEach)
			path = path.split('.');
		switch (path[0]) {
			case '$this':
				if (path.length === 1)
					return this.reset(value);
				else
					path.shift();
				break;
			case '$parent':
				if (this.parent)
					return this.parent.set(path.slice(1), value) && this;
				throw new Error('yamvish.Context : there is no parent in current context. could not find : ' + path.join('.'));
			case '$env':
				return this.env.set(path.slice(1), value) && this;
		}
		var old = utils.setProp(this.data, path, value);
		if (old !== value) {
			if (old instanceof Date && value instanceof Date && old == value)
				return this;
			this.notify('set', path, value, path[path.length - 1]);
		}
		return this;
	},
	push: function(path, value) {
		if (!path.forEach)
			path = path.split('.');
		switch (path[0]) {
			case '$parent':
				if (this.parent)
					return this.parent.push(path.slice(1), value) && this;
				throw new Error('yamvish.Context : there is no parent in current context. could not find : ' + path.join('.'));
			case '$env':
				return this.env.push(path.slice(1), value) && this;
		}
		var arr;
		if (path[0] === '$this')
			arr = this.data;
		else
			arr = utils.getProp(this.data, path);
		if (!arr) {
			arr = [];
			utils.setProp(this.data, path, arr);
			// throw new Error("yamvish.Context : Missing array at " + path.join(".") + " : couldn't push object.");
		}
		if (!arr.forEach) {
			console.error(path, 'is not array at context.push : ', arr);
			throw new Error("yamvish.Context : Object is not array at " + path.join(".") + " : couldn't push object.");
		}
		arr.push(value);
		this.notify('push', path, value, arr.length - 1);
		return this;
	},
	del: function(path) {
		if (!path.forEach)
			path = path.split('.');
		else
			path = path.slice();
		switch (path[0]) {
			case '$parent':
				if (this.parent)
					return this.parent.del(path.slice(1)) && this;
				throw new Error('yamvish.Context : there is no parent in current context. could not find : ' + path.join('.'));
			case '$env':
				return this.env.del(path.slice(1)) && this;
		}
		var path2 = path.slice();
		var key = path2.pop(),
			parent = path2.length ? utils.getProp(this.data, path2) : this.data;
		if (parent)
			if (parent.forEach) {
				var index = parseInt(key, 10);
				if (index < parent.length)
					this.notify('removeAt', path2, parent.splice(index, 1), index);
			} else {
				var val = parent[key];
				delete parent[key];
				this.notify('delete', path, val, key);
			}
		return this;
	},
	subscribe: function(path, fn, upstream, binds) {
		// console.log('context subscribe : ', path, fn, upstream);
		if (!path.forEach)
			path = path.split('.');
		var space;
		switch (path[0]) {
			case '$this':
				space = this.map;
				break;
			case '$env':
				return this.env.subscribe(path.slice(1), fn, upstream, binds);
			case '$parent':
				if (!this.parent)
					throw new Error('yamvish.Context : there is no parent in current context. could not find : ' + path.join('.'));
				return this.parent.subscribe(path.slice(1), fn, upstream, binds);
			default:
				space = utils.getProp(this.map, path);
		}
		if (upstream) {
			if (!space)
				utils.setProp(this.map, path, {
					_upstreams: [fn]
				});
			else
				(space._upstreams = space._upstreams || []).push(fn);
		} else if (!space)
			utils.setProp(this.map, path, {
				_listeners: [fn]
			});
		else
			(space._listeners = space._listeners || []).push(fn);
		if (binds)
			binds.push(unsub(this, path, fn, upstream));
		return this;
	},
	unsubscribe: function(path, fn, upstream) {
		if (this.distroyed)
			return this;
		if (!path.forEach)
			path = path.split('.');
		var space;
		switch (path[0]) {
			case '$this':
				space = this.map;
				break;
			case '$parent':
				return this.env.unsubscribe(path.slice(1), fn, upstream) && this;
			case '$parent':
				if (this.parent)
					return this.parent.unsubscribe(path.slice(1), fn, upstream) && this;
				throw new Error('yamvish.Context : there is no parent in current context. could not find : ' + path.join('.'));
			default:
				space = utils.getProp(this.map, path);
		}
		if (!space)
			return this;
		var arr = upstream ? space._upstreams : space._listeners;
		for (var i = 0, len = arr.length; i < len; ++i)
			if (arr[i] === fn) {
				arr.splice(i, 1);
				break;
			}
		return this;
	},
	notifyAll: function(type, path, space, value, index) {
		space = space ||  this.map;
		value = (arguments.length < 2) ? this.data : value;
		if (space._listeners)
			for (var i = 0, len = space._listeners.length; i < len; ++i) {
				var listener = space._listeners[i];
				if (!listener) {
					// maybe it's because listeners length has change so update it
					len = space._listeners.length;
					continue;
				}
				var r = listener.call(this, value, type, path, index);
				if (r && r.then)
					this.waiting(r);
			}
		if (type !== 'push' && type !== 'removeAt')
			for (var j in space) {
				if (j === '_listeners' || j === '_upstreams')
					continue;
				this.notifyAll(type, path, space[j], value ? value[j] : undefined, index);
			}
		return this;
	},
	notify: function(type, path, value, index) {
		if (!path.forEach)
			path = path.split('.');
		if (path[0] === '$this')
			path = path.slice(1);
		else if (path[0] === '$parent') {
			if (!this.parent)
				throw new Error('could not notify parent : no parent found. for : ' + path.join('.'));
			this.parent.notify(type, path.slice(1), value, index);
			return this;
		}
		var space = this.map,
			i = 0,
			star;
		for (var len = path.length; i < len; ++i) {
			star = space['*'];
			if (star && star._upstreams)
				notifyUpstreams.call(this, star, type, path, value, index);
			if (!(space = space[path[i]]))
				break;
			if (space._upstreams)
				notifyUpstreams.call(this, space, type, path, value, index);
		}
		if (star)
			this.notifyAll(type, path, star, value, index);
		if (space)
			this.notifyAll(type, path, space, value, index);
		return this;
	},
	setAsync: function(path, promise) {
		var self = this;
		return this.waiting(promise.then(function(s) {
			self.set(path, s);
			return s;
		}, function(e) {
			console.error('error while Context.setAsync : ', e);
			throw e;
		}));
	},
	pushAsync: function(path, promise) {
		var self = this;
		return this.waiting(promise.then(function(s) {
			self.push(path, s);
			return s;
		}, function(e) {
			console.error('error while Context.pushAsync : ', e);
			throw e;
		}));
	},
	//______________________ AGORA MANAGEMENT
	onAgora: function(messageName, handler) {
		var agora = this.env.data.agora,
			self = this;
		agora.on(messageName, handler, [this]);
		(this.binds = this.binds ||  []).push(function() {
			agora.off(messageName, handler);
		});
		return this;
	},
	toAgora: function(name) {
		var args = [name].concat([].slice.call(arguments, 1))
		this.env.data.agora.emit.apply(this.env.data.agora, args);
		return this;
	},
	output: function() {
		var output = {};
		for (var i in this.data)
			if (!outputFiler.test(i))
				output[i] = this.data[i];
		return output;
	}
};

var outputFiler = /^\$/;
// general env
Context.env = new Context(env);
delete Context.env.env;

utils.shallowMerge(AsyncManager.prototype, Context.prototype);

function notifyUpstreams(space, type, path, value, index) {
	for (var i = 0, len = space._upstreams.length; i < len; ++i) {
		var upstream = space._upstreams[i];
		if (!upstream) {
			// maybe it's because upstreams length has change so update it
			len = space._upstreams.length;
			continue;
		}
		var r = upstream.call(this, value, type, path, index);
		if (r && r.then)
			this.waiting(r);
	}
}

module.exports = Context;
<|endoftext|>


<commit_before>// Copyright 2016 Luca-SAS, licensed under the Apache License 2.0

'use strict';

var util = require('util');
var uuid = require('node-uuid');
var fs = require('fs');

var SkaleClient = require('./client.js');
var dataset = require('./dataset.js');
var mkdir = require('./mkdir.js');

module.exports = SkaleContext;

util.inherits(SkaleContext, SkaleClient);

function SkaleContext(arg) {
	if (!(this instanceof SkaleContext))
		return new SkaleContext(arg);
	var self = this;

	arg = arg || {};
	arg.data = arg.data || {};
	arg.data.type = 'master';
	SkaleClient.call(this, arg);
	var maxWorker = process.env.SKALE_MAX_WORKER;

	this.started = this.ended = false;
	this.jobId = 0;
	this.firstData = {};
	this.worker = [];
	this.contextId = uuid.v4();	// context id which will be used as scratch directory name

	this.basedir = '/tmp/skale/' + this.contextId + '/';
	mkdir(this.basedir + 'tmp');
	mkdir(this.basedir + 'stream');

	this.once('connect', function(data) {
		var i;
		if (!maxWorker || maxWorker > data.devices.length)
			maxWorker = data.devices.length;
		for (i = 0; i < maxWorker; i++) {
			self.worker.push(new Worker(data.devices[i]));
			self.firstData[data.devices[i].uuid] = data.devices;
		}
		self.started = true;
	});

	this.on('workerError', function workerError(msg) {
		console.error('Error from worker id %d:', msg.from);
		console.error(msg.args);
	});

	this.on('remoteClose', function getWorkerClose() {
		throw 'Fatal error: unexpected worker exit';
	});

	this.getWorkers = function (callback) {
		if (self.started) return callback();
		this.once('connect', function() {callback();});
	};

	function Worker(w) {
		this.uuid = w.uuid;
		this.id = w.id;
		this.ip = w.ip;
		this.ntask = 0;
	}

	Worker.prototype.rpc = function (cmd, args, done) {
		self.request({uuid: this.uuid, id: this.id}, {cmd: cmd, args: args}, done);
	};

	Worker.prototype.send = function (cmd, args) {
		self.send(this.uuid, {cmd: cmd, args: args});
	};

	this.on('request', function (msg) { // Protocol to handle stream flow control: reply when data is consumed
		if (msg.data.cmd == 'stream') {
			self.emit(msg.data.stream, msg.data.data, function() {
				try {self.reply(msg);} catch(err) { console.log(err); }
			});
		}
	});

	this.on('sendFile', function (msg) {
		fs.createReadStream(msg.path, msg.opt).pipe(self.createStreamTo(msg));
	});

	this.end = function () {
		if (self.ended) return;
		self.ended = true;
		if (this.started) self.set({complete: 1});
		self._end();
	};

	this.datasetIdCounter = 0;	// global dataset id counter

	this.parallelize = function (localArray, nPartitions) { return dataset.parallelize(this, localArray, nPartitions);};
	this.range = function (start, end, step, nPartitions) { return dataset.range(this, start, end, step, nPartitions);};
	this.textFile = function (file, nPartitions) {return new dataset.TextFile(this, file, nPartitions);};
	this.lineStream = function (stream, config) {return new dataset.Stream(this, stream, 'line', config);};
	this.objectStream = function (stream, config) {return new dataset.Stream(this, stream, 'object', config);};

	this.runTask = function(task, callback) {
		function getLeastBusyWorkerId(/* preferredLocation */) {
			var wid, ntask;
			for (var i = 0; i < self.worker.length; i++) {
				if ((ntask == undefined) || (ntask > self.worker[i].ntask)) {
					ntask = self.worker[i].ntask;
					wid = i;
				}
			}
			return wid;
		}

		function serialize(obj) {
			return JSON.stringify(obj,function(key, value) {
				if (key == 'sc') return undefined;
				return (typeof value === 'function' ) ? value.toString() : value;
			});
		}

		var wid = getLeastBusyWorkerId(task.nodes[task.datasetId].getPreferedLocation(task.pid));
		this.worker[wid].ntask++;
		rpc('runTask', wid, serialize(task), function(err, res) {
			self.worker[wid].ntask--;
			callback(err, res);
		});
	};

	this.runJob = function(opt, root, action, callback) {
		var jobId = this.jobId++;
		var stream = opt._stream || new this.createReadStream(jobId, opt);	// user redable stream instance

		this.getWorkers(function () {
			findShuffleStages(function(shuffleStages) {
				if (shuffleStages.length == 0) runResultStage();
				else {
					var cnt = 0;
					runShuffleStage(shuffleStages[cnt], shuffleDone);
				}
				function shuffleDone() {
					if (++cnt < shuffleStages.length) runShuffleStage(shuffleStages[cnt], shuffleDone);
					else runResultStage();
				}
			});
		});

		function runShuffleStage(stage, done) {
			findNodes(stage, function(nodes) {
				var pid = 0, shuffleTasks = [], cnt = 0, i, j;
				stage.shufflePartitions = [];

				for (i = 0; i < stage.dependencies.length; i++) {
					for (j = 0; j < stage.dependencies[i].partitions.length; j++)
						stage.shufflePartitions[pid++] = new dataset.Partition(stage.id, pid, stage.dependencies[i].id, stage.dependencies[i].partitions[j].partitionIndex);
				}

				for (i = 0; i < stage.shufflePartitions.length; i++)
					shuffleTasks.push(new Task(self.contextId, jobId, nodes, stage.id, i));

				for (i = 0; i < shuffleTasks.length; i++) {
					self.runTask(shuffleTasks[i], function (err, res) {
						stage.shufflePartitions[res.pid].files = res.files;
						if (++cnt < shuffleTasks.length) return;
						stage.executed = true;
						done();
					});
				}
			});
		}

		function runResultStage() {
			findNodes(root, function(nodes) {
				var tasks = [];
				for (var i = 0; i < root.partitions.length; i++)
					tasks.push(new Task(self.contextId, jobId, nodes, root.id, i, action));
				callback({id: jobId, stream: stream}, tasks);
			});
		}

		function findNodes(node, done) {
			var nodes = {};
			interruptibleTreewalk(node, function cin(n, done) {
				done(n.shuffling && n.executed);
			}, function cout(n, done) {
				n.getPartitions(function() {
					if (nodes[n.id] == undefined) nodes[n.id] = n;
					done();
				});
			}, function() {done(nodes);});
		}

		function findShuffleStages(callback) {
			var stages = [];
			interruptibleTreewalk(root, function cin(n, done) {
				if (n.shuffling && !n.executed) stages.unshift(n);
				done(n.shuffling && n.executed); 	// stage boundary are shuffle nodes
			}, function cout(n, done) {done();}, function() {callback(stages);});
		}

		function interruptibleTreewalk(n, cin, cout, done) {
			cin(n, function(uturn) { // if uturn equals true the subtree under node won't be treewalked
				if (!uturn) {
					var nDependencies = 0;
					for (var i = 0; i < n.dependencies.length; i++)
						interruptibleTreewalk(n.dependencies[i], cin, cout, function() {
							if (++nDependencies == n.dependencies.length) cout(n, done);
						});
					if (n.dependencies.length == 0) cout(n, done);
				} else cout(n, done);
			});
		}

		return stream;
	};

	function rpc(cmd, workerNum, args, callback) {
		self.request(self.worker[workerNum], {cmd: cmd, args: args, master_uuid: self.uuid, worker: self.worker}, callback);
	}
}

function Task(contextId, jobId, nodes, datasetId, pid, action) {
	this.id = jobId;					// cette variable est surement inutile
	this.contextId = contextId;
	this.datasetId = datasetId;
	this.pid = pid;
	this.nodes = nodes;
	this.action = action;
	this.files = {};			// object in which we store shuffle file informations to be sent back to master
	this.lib;					// handler to libraries required on worker side (which cannot be serialized)
	this.mm;					// handler to worker side memory manager instance
	this.grid;					// handler to socket object instance

	this.run = function(done) {
		var pipeline = [], self = this, mm = this.mm, action = this.action;
		var p = this.pid;
		var tmpPart = action ? this.nodes[this.datasetId].partitions[p] : this.nodes[this.datasetId].shufflePartitions[p];
		var tmpDataset = this.nodes[tmpPart.datasetId];
		var blocksToRegister = [];

		this.basedir = '/tmp/skale/' + this.contextId + '/';
		this.lib.mkdir(this.basedir + 'shuffle');

		if (action)
			pipeline.push({transform: function aggregate(context, data) {
				for (var i = 0; i < data.length; i++)
					action.init = action.src(action.init, data[i]);
			}});

		for (;;) {
			var tmpPartAvailable = mm.isAvailable(tmpPart);							// is partition available in memory
			if (!tmpPartAvailable && tmpDataset.persistent)	{							// if data must be stored in memory
				if ((action != undefined) || (tmpDataset.id != this.datasetId)) { 			// no persist if no action and shuffleRDD
					blocksToRegister.push(tmpPart);									// register block inside memory manager
					pipeline.unshift(tmpPart);										// add it to pipeline
				}
			}
			if (tmpPartAvailable || (tmpPart.parentDatasetId == undefined)) break;		// source partition found
			pipeline.unshift(tmpDataset);												// else add current dataset transform to pipeline
			tmpPart = this.nodes[tmpPart.parentDatasetId].partitions[tmpPart.parentPartitionIndex];
			tmpPart.mm = this.mm;
			tmpDataset = this.nodes[tmpPart.datasetId];
		}

		if (tmpPartAvailable) mm.partitions[tmpPart.datasetId + '.' + tmpPart.partitionIndex].iterate(this, tmpPart.partitionIndex, pipeline, iterateDone);
		else this.nodes[tmpPart.datasetId].iterate(this, tmpPart.partitionIndex, pipeline, iterateDone);

		function iterateDone() {
			blocksToRegister.map(function(block) {mm.register(block);});
			if (action) done({data: action.init});
			else self.nodes[self.datasetId].spillToDisk(self, function() {
				done({pid: self.pid, files: self.files});
			});
		}
	};

	this.getReadStream = function (fileObj, opt) {
		var fs = this.lib.fs;
		try {
			fs.accessSync(fileObj.path);
			return fs.createReadStream(fileObj.path, opt);
		} catch (err) {
			if (!fileObj.host) fileObj.host = this.grid.muuid;
			return this.grid.createStreamFrom(fileObj.host, {cmd: 'sendFile', path: fileObj.path, opt: opt});
		}
	};
}
<commit_msg>worker: fix init of Partition.mm<commit_after>// Copyright 2016 Luca-SAS, licensed under the Apache License 2.0

'use strict';

var util = require('util');
var uuid = require('node-uuid');
var fs = require('fs');

var SkaleClient = require('./client.js');
var dataset = require('./dataset.js');
var mkdir = require('./mkdir.js');

module.exports = SkaleContext;

util.inherits(SkaleContext, SkaleClient);

function SkaleContext(arg) {
	if (!(this instanceof SkaleContext))
		return new SkaleContext(arg);
	var self = this;

	arg = arg || {};
	arg.data = arg.data || {};
	arg.data.type = 'master';
	SkaleClient.call(this, arg);
	var maxWorker = process.env.SKALE_MAX_WORKER;

	this.started = this.ended = false;
	this.jobId = 0;
	this.firstData = {};
	this.worker = [];
	this.contextId = uuid.v4();	// context id which will be used as scratch directory name

	this.basedir = '/tmp/skale/' + this.contextId + '/';
	mkdir(this.basedir + 'tmp');
	mkdir(this.basedir + 'stream');

	this.once('connect', function(data) {
		var i;
		if (!maxWorker || maxWorker > data.devices.length)
			maxWorker = data.devices.length;
		for (i = 0; i < maxWorker; i++) {
			self.worker.push(new Worker(data.devices[i]));
			self.firstData[data.devices[i].uuid] = data.devices;
		}
		self.started = true;
	});

	this.on('workerError', function workerError(msg) {
		console.error('Error from worker id %d:', msg.from);
		console.error(msg.args);
	});

	this.on('remoteClose', function getWorkerClose() {
		throw 'Fatal error: unexpected worker exit';
	});

	this.getWorkers = function (callback) {
		if (self.started) return callback();
		this.once('connect', function() {callback();});
	};

	function Worker(w) {
		this.uuid = w.uuid;
		this.id = w.id;
		this.ip = w.ip;
		this.ntask = 0;
	}

	Worker.prototype.rpc = function (cmd, args, done) {
		self.request({uuid: this.uuid, id: this.id}, {cmd: cmd, args: args}, done);
	};

	Worker.prototype.send = function (cmd, args) {
		self.send(this.uuid, {cmd: cmd, args: args});
	};

	this.on('request', function (msg) { // Protocol to handle stream flow control: reply when data is consumed
		if (msg.data.cmd == 'stream') {
			self.emit(msg.data.stream, msg.data.data, function() {
				try {self.reply(msg);} catch(err) { console.log(err); }
			});
		}
	});

	this.on('sendFile', function (msg) {
		fs.createReadStream(msg.path, msg.opt).pipe(self.createStreamTo(msg));
	});

	this.end = function () {
		if (self.ended) return;
		self.ended = true;
		if (this.started) self.set({complete: 1});
		self._end();
	};

	this.datasetIdCounter = 0;	// global dataset id counter

	this.parallelize = function (localArray, nPartitions) { return dataset.parallelize(this, localArray, nPartitions);};
	this.range = function (start, end, step, nPartitions) { return dataset.range(this, start, end, step, nPartitions);};
	this.textFile = function (file, nPartitions) {return new dataset.TextFile(this, file, nPartitions);};
	this.lineStream = function (stream, config) {return new dataset.Stream(this, stream, 'line', config);};
	this.objectStream = function (stream, config) {return new dataset.Stream(this, stream, 'object', config);};

	this.runTask = function(task, callback) {
		function getLeastBusyWorkerId(/* preferredLocation */) {
			var wid, ntask;
			for (var i = 0; i < self.worker.length; i++) {
				if ((ntask == undefined) || (ntask > self.worker[i].ntask)) {
					ntask = self.worker[i].ntask;
					wid = i;
				}
			}
			return wid;
		}

		function serialize(obj) {
			return JSON.stringify(obj,function(key, value) {
				if (key == 'sc') return undefined;
				return (typeof value === 'function' ) ? value.toString() : value;
			});
		}

		var wid = getLeastBusyWorkerId(task.nodes[task.datasetId].getPreferedLocation(task.pid));
		this.worker[wid].ntask++;
		rpc('runTask', wid, serialize(task), function(err, res) {
			self.worker[wid].ntask--;
			callback(err, res);
		});
	};

	this.runJob = function(opt, root, action, callback) {
		var jobId = this.jobId++;
		var stream = opt._stream || new this.createReadStream(jobId, opt);	// user redable stream instance

		this.getWorkers(function () {
			findShuffleStages(function(shuffleStages) {
				if (shuffleStages.length == 0) runResultStage();
				else {
					var cnt = 0;
					runShuffleStage(shuffleStages[cnt], shuffleDone);
				}
				function shuffleDone() {
					if (++cnt < shuffleStages.length) runShuffleStage(shuffleStages[cnt], shuffleDone);
					else runResultStage();
				}
			});
		});

		function runShuffleStage(stage, done) {
			findNodes(stage, function(nodes) {
				var pid = 0, shuffleTasks = [], cnt = 0, i, j;
				stage.shufflePartitions = [];

				for (i = 0; i < stage.dependencies.length; i++) {
					for (j = 0; j < stage.dependencies[i].partitions.length; j++)
						stage.shufflePartitions[pid++] = new dataset.Partition(stage.id, pid, stage.dependencies[i].id, stage.dependencies[i].partitions[j].partitionIndex);
				}

				for (i = 0; i < stage.shufflePartitions.length; i++)
					shuffleTasks.push(new Task(self.contextId, jobId, nodes, stage.id, i));

				for (i = 0; i < shuffleTasks.length; i++) {
					self.runTask(shuffleTasks[i], function (err, res) {
						stage.shufflePartitions[res.pid].files = res.files;
						if (++cnt < shuffleTasks.length) return;
						stage.executed = true;
						done();
					});
				}
			});
		}

		function runResultStage() {
			findNodes(root, function(nodes) {
				var tasks = [];
				for (var i = 0; i < root.partitions.length; i++)
					tasks.push(new Task(self.contextId, jobId, nodes, root.id, i, action));
				callback({id: jobId, stream: stream}, tasks);
			});
		}

		function findNodes(node, done) {
			var nodes = {};
			interruptibleTreewalk(node, function cin(n, done) {
				done(n.shuffling && n.executed);
			}, function cout(n, done) {
				n.getPartitions(function() {
					if (nodes[n.id] == undefined) nodes[n.id] = n;
					done();
				});
			}, function() {done(nodes);});
		}

		function findShuffleStages(callback) {
			var stages = [];
			interruptibleTreewalk(root, function cin(n, done) {
				if (n.shuffling && !n.executed) stages.unshift(n);
				done(n.shuffling && n.executed); 	// stage boundary are shuffle nodes
			}, function cout(n, done) {done();}, function() {callback(stages);});
		}

		function interruptibleTreewalk(n, cin, cout, done) {
			cin(n, function(uturn) { // if uturn equals true the subtree under node won't be treewalked
				if (!uturn) {
					var nDependencies = 0;
					for (var i = 0; i < n.dependencies.length; i++)
						interruptibleTreewalk(n.dependencies[i], cin, cout, function() {
							if (++nDependencies == n.dependencies.length) cout(n, done);
						});
					if (n.dependencies.length == 0) cout(n, done);
				} else cout(n, done);
			});
		}

		return stream;
	};

	function rpc(cmd, workerNum, args, callback) {
		self.request(self.worker[workerNum], {cmd: cmd, args: args, master_uuid: self.uuid, worker: self.worker}, callback);
	}
}

function Task(contextId, jobId, nodes, datasetId, pid, action) {
	this.id = jobId;					// cette variable est surement inutile
	this.contextId = contextId;
	this.datasetId = datasetId;
	this.pid = pid;
	this.nodes = nodes;
	this.action = action;
	this.files = {};			// object in which we store shuffle file informations to be sent back to master
	this.lib;					// handler to libraries required on worker side (which cannot be serialized)
	this.mm;					// handler to worker side memory manager instance
	this.grid;					// handler to socket object instance

	this.run = function(done) {
		var pipeline = [], self = this, mm = this.mm, action = this.action;
		var p = this.pid;
		var tmpPart = action ? this.nodes[this.datasetId].partitions[p] : this.nodes[this.datasetId].shufflePartitions[p];
		var tmpDataset = this.nodes[tmpPart.datasetId];
		var blocksToRegister = [];

		this.basedir = '/tmp/skale/' + this.contextId + '/';
		this.lib.mkdir(this.basedir + 'shuffle');

		if (action)
			pipeline.push({transform: function aggregate(context, data) {
				for (var i = 0; i < data.length; i++)
					action.init = action.src(action.init, data[i]);
			}});

		for (;;) {
			var tmpPartAvailable = mm.isAvailable(tmpPart);							// is partition available in memory
			if (!tmpPartAvailable && tmpDataset.persistent)	{							// if data must be stored in memory
				if ((action != undefined) || (tmpDataset.id != this.datasetId)) { 			// no persist if no action and shuffleRDD
					blocksToRegister.push(tmpPart);									// register block inside memory manager
					pipeline.unshift(tmpPart);										// add it to pipeline
					tmpPart.mm = this.mm;
				}
			}
			if (tmpPartAvailable || (tmpPart.parentDatasetId == undefined)) break;		// source partition found
			pipeline.unshift(tmpDataset);												// else add current dataset transform to pipeline
			tmpPart = this.nodes[tmpPart.parentDatasetId].partitions[tmpPart.parentPartitionIndex];
			tmpDataset = this.nodes[tmpPart.datasetId];
		}

		if (tmpPartAvailable) mm.partitions[tmpPart.datasetId + '.' + tmpPart.partitionIndex].iterate(this, tmpPart.partitionIndex, pipeline, iterateDone);
		else this.nodes[tmpPart.datasetId].iterate(this, tmpPart.partitionIndex, pipeline, iterateDone);

		function iterateDone() {
			blocksToRegister.map(function(block) {mm.register(block);});
			if (action) done({data: action.init});
			else self.nodes[self.datasetId].spillToDisk(self, function() {
				done({pid: self.pid, files: self.files});
			});
		}
	};

	this.getReadStream = function (fileObj, opt) {
		var fs = this.lib.fs;
		try {
			fs.accessSync(fileObj.path);
			return fs.createReadStream(fileObj.path, opt);
		} catch (err) {
			if (!fileObj.host) fileObj.host = this.grid.muuid;
			return this.grid.createStreamFrom(fileObj.host, {cmd: 'sendFile', path: fileObj.path, opt: opt});
		}
	};
}
<|endoftext|>


<commit_before>'use strict';

var $ = require('jquery'),
	PulsarFormComponent = require('../../js/PulsarFormComponent');

describe('Pulsar Form Component', function() {

	beforeEach(function() {
		this.$html = $('');
		this.$body = $('').appendTo(this.$html);
		this.$markup = $('\
<form class="form">\
	<select class="js-select2">\
		<option>foo</option>\
		<option>bar</option>\
		<option>baz</option>\
	</select>\
</form>\
').appendTo(this.$html);

		this.$form = this.$html.find('.form');
		this.$select2 = this.$form.find('.js-select2');

		this.pulsarForm = new PulsarFormComponent(this.$html);

	});

	describe('Select2 elements', function() {

		beforeEach(function() {
			sinon.spy($.fn, 'select2');
			this.pulsarForm.init();
		});

		it('should have called the select2 plugin', function() {
			expect($.fn.select2).to.have.been.called;
		});

	});

});
<commit_msg>[WIP] Remove redundant variables from PulsarFormComponentTest<commit_after>'use strict';

var $ = require('jquery'),
	PulsarFormComponent = require('../../js/PulsarFormComponent');

describe('Pulsar Form Component', function() {

	beforeEach(function() {
		this.$html = $('');
		this.$body = $('').appendTo(this.$html);
		this.$markup = $('\
<form class="form">\
	<select class="js-select2" data-html="true">\
		<option>foo</option>\
		<option>bar</option>\
		<option>baz</option>\
	</select>\
</form>\
').appendTo(this.$html);

		this.pulsarForm = new PulsarFormComponent(this.$html);

	});

	describe('Select2 elements within a form', function() {

		beforeEach(function() {
			sinon.spy($.fn, 'select2');
			this.pulsarForm.init();
		});

		it('should call the select2 plugin', function() {
			expect($.fn.select2).to.have.been.called;
		});

	});

});
<|endoftext|>


<commit_before>/*global Buffer: false */

/**
 * Module dependencies
 */

var url = require('url'),
    http = require('http'),
    https = require('https'),
    logger = require('./logger'),
    querystring = require('querystring');

var STATUS_MSGS = {
    400: '400: Bad Request',
    401: '401: Unauthorized',
    402: '402: Payment Required',
    403: '403: Forbidden',
    404: '404: Not Found',
    405: '405: Method Not Allowed',
    406: '406: Not Acceptable',
    407: '407: Proxy Authentication Required',
    408: '408: Request Timeout',
    409: '409: Conflict',
    410: '410: Gone',
    411: '411: Length Required',
    412: '412: Precondition Failed',
    413: '413: Request Entity Too Large',
    414: '414: Request-URI Too Long',
    415: '415: Unsupported Media Type',
    416: '416: Requested Range Not Satisfiable',
    417: '417: Expectation Failed',
    418: '418: I\'m a teapot',
    422: '422: Unprocessable Entity',
    423: '423: Locked',
    424: '424: Failed Dependency',
    425: '425: Unordered Collection',
    444: '444: No Response',
    426: '426: Upgrade Required',
    449: '449: Retry With',
    450: '450: Blocked by Windows Parental Controls',
    499: '499: Client Closed Request',
    500: '500: Internal Server Error',
    501: '501: Not Implemented',
    502: '502: Bad Gateway',
    503: '503: Service Unavailable',
    504: '504: Gateway Timeout',
    505: '505: HTTP Version Not Supported',
    506: '506: Variant Also Negotiates',
    507: '507: Insufficient Storage',
    509: '509: Bandwidth Limit Exceeded',
    510: '510: Not Extended'
};

var CouchDB;

/**
 * Convenience method for creating a CouchDB object instance.
 *
 * @param {String} db_url
 * @api public
 */

var exports = module.exports = function (db_url) {
    return new CouchDB(db_url);
};

/**
 * The CouchDB object constructor.
 *
 * @class CouchDB
 * @constructor
 * @param {String} db_url
 * @api public
 */

CouchDB = exports.CouchDB = function (db_url) {
    var ins = this.instance = url.parse(db_url);
    if (!ins.port) {
        if (ins.protocol === 'https:') {
            ins.port = 443;
        }
        else {
            ins.port = 80;
        }
    }
};

/**
 * Tests if a database exists, creates it if not.
 *
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.ensureDB = function (callback) {
    var that = this;
    this.exists('', function (err, exists) {
        if (err || exists) {
            return callback(err, that);
        }
        that.createDB(callback);
    });
};

/**
 * Creates a database.
 *
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.createDB = function (callback) {
    this.client('PUT', '', null, callback);
};

/**
 * Convenience HTTP client for querying a CouchDB instance. Buffers and parses
 * JSON responses before passing to callback. JSON.stringify's data before
 * sending.
 *
 * @param {String} method
 * @param {String} path
 * @param data
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.client = function (method, path, data, callback) {
    var pathname = this.instance.pathname;
    path = (pathname ? pathname + '/' + path: '/' + path);
    path = path.replace(/^\/\/+/, '/');

    var headers = {
        'Host': this.instance.hostname,
        'Accept': 'application/json'
    };
    if (method === 'POST' || method === 'PUT') {
        if (typeof data !== 'string') {
            try {
                data = JSON.stringify(data);
            }
            catch (e) {
                return callback(e);
            }
        }
        headers['Content-Type'] = 'application/json';
    }
    else if (data) {
        path = url.parse(path).pathname + '?' + querystring.stringify(data);
    }

    if (this.instance.auth) {
        var auth = decodeURIComponent(this.instance.auth);
        var enc = new Buffer(auth).toString('base64');
        headers.Authorization = "Basic " + enc;
    }

    var proto = (this.instance.protocol === 'https:') ? https: http;

    var request = proto.request({
        host: this.instance.hostname,
        port: this.instance.port,
        method: method,
        path: path,
        headers: headers
    });

    logger.debug('request', method + ' ' + path);
    logger.debug('request headers', headers);
    logger.debug('request data', data);

    request.on('response', function (response) {
        logger.debug('response:', {
            headers: response.headers,
            url: response.url,
            method: response.method,
            statusCode: response.statusCode
        });
        var buffer = [];
        response.on('data', function (chunk) {
            buffer.push(chunk.toString());
        });
        response.on('end', function () {
            try {
                var data = buffer.length ? JSON.parse(buffer.join('')): null;
            }
            catch (e) {
                logger.info('Unexpected response', buffer.join(''));
            }
            logger.debug('data:', data);
            if (response.statusCode >= 300) {
                if (data && data.error) {
                    var err = new Error(
                        data.error + (data.reason ? '\n' + data.reason: '')
                    );
                    err.error = data.error;
                    err.reason = data.reason;
                    err.response = response;
                    callback(err, data, response);
                }
                else {
                    var err2 = exports.statusCodeError(response.statusCode);
                    err2.response = response;
                    callback(err2, data, response);
                }
            }
            else {
                process.nextTick(function () {
                    callback(null, data, response);
                });
            }
        });
    });

    if (data && (method === 'POST' || method === 'PUT')) {
        request.write(data, 'utf8');
    }
    request.end();

    //logger.debug('request:', request.output[0]);
};

/**
 * Creates an error object with a message depending on the HTTP status code
 * of a response.
 */

exports.statusCodeError = function (code) {
    if (code in STATUS_MSGS) {
        return new Error(STATUS_MSGS[code]);
    }
    return new Error('Status code: ' + code);
};

/**
 * Encodes a document id or view, list or show name.
 *
 * @name encode(str)
 * @param {String} str
 * @returns {String}
 * @api public
 */

exports.encode = function (str) {
    return encodeURIComponent(str).replace(/^_design%2F/, '_design/');
};

/**
 * Test if a doc exists in the db by doing a HEAD request - doesn't fetch
 * the whole document.
 *
 * @param {String} id
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.exists = function (id, callback) {
    id = exports.encode(id || '');
    this.client('HEAD', id, null, function (err, data, res) {
        res = res || {};
        if (res.statusCode !== 404 && err) {
            return callback(err);
        }
        var exists = (res.statusCode === 200);
        var etag = res.headers.etag;
        var _rev = etag ? etag.substr(1, etag.length - 2): null;
        callback(null, exists, _rev);
    });
};

/**
 * Retrieve a document from a CouchDB instance.
 *
 * @param {String} id
 * @param {Object} data
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.get = function (id, /*optional*/data, callback) {
    if (arguments.length < 3) {
        callback = data;
        data = null;
    }
    id = exports.encode(id || '');
    this.client('GET', id, data, callback);
};

/**
 * Saves a document to a CouchDB instance.
 *
 * Options:
 *      {Boolean} force - write document regardless of conflicts!
 *
 * @param {String} id
 * @param {Object} doc
 * @param {Object} options
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.save = function (id, doc, /*optional*/options, callback) {
    var that = this;

    if (!callback) {
        callback = options;
        options = {};
    }
    var method = id ? 'PUT': 'POST';
    var path = exports.encode(id || '');

    if (options.force && id) {
        // WARNING! this is a brute-force document update
        // updates revision number to latest revision before saving
        this.exists(id, function (err, exists, rev) {
            if (err) {
                return callback(err);
            }
            if (exists) {
                doc._rev = rev;
            }
            that.client(method, path, doc, function (err, d) {
                if (err) {
                    return callback(err);
                }
                doc._id = d.id;
                doc._rev = d.rev;
                callback(null, doc);
            });
        });
    }
    else {
        this.client(method, path, doc, callback);
    }
};

/**
 * Deletes a document from a CouchDB instance.
 *
 * Options:
 *      {Boolean} force - delete document regardless of conflicts!
 *
 * @param {String} id
 * @param {Object} rev
 * @param {Object} options
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.delete = function (id, rev, /*optional*/options, callback) {
    var that = this;

    if (!callback) {
        callback = options;
        options = {};
    }
    var args = {};
    if (rev) {
        args.rev = rev;
    }
    var path = exports.encode(id || '');

    if (options.force) {
        // WARNING! this is a brute-force document delete
        // updates revision number to latest revision before deleting
        this.exists(id, function (err, exists, rev) {
            if (err) {
                return callback(err);
            }
            if (exists) {
                args.rev = rev;
            }
            that.client('DELETE', path, args, callback);
        });
    }
    else {
        this.client('DELETE', path, args, callback);
    }
};

CouchDB.prototype.uuids = function (count, callback) {
    if (!callback) {
        callback = count;
        count = undefined;
    }
    count = count || 1;
    this.client('GET', '_uuids', {count: count}, function (err, data) {
        callback(err, data.uuids);
    });
};
<commit_msg>add content length header for couchdb running behind nginx, and show appropriate error messages when response sends connection close header, closes #226<commit_after>/*global Buffer: false */

/**
 * Module dependencies
 */

var url = require('url'),
    http = require('http'),
    https = require('https'),
    logger = require('./logger'),
    querystring = require('querystring');

var STATUS_MSGS = {
    400: '400: Bad Request',
    401: '401: Unauthorized',
    402: '402: Payment Required',
    403: '403: Forbidden',
    404: '404: Not Found',
    405: '405: Method Not Allowed',
    406: '406: Not Acceptable',
    407: '407: Proxy Authentication Required',
    408: '408: Request Timeout',
    409: '409: Conflict',
    410: '410: Gone',
    411: '411: Length Required',
    412: '412: Precondition Failed',
    413: '413: Request Entity Too Large',
    414: '414: Request-URI Too Long',
    415: '415: Unsupported Media Type',
    416: '416: Requested Range Not Satisfiable',
    417: '417: Expectation Failed',
    418: '418: I\'m a teapot',
    422: '422: Unprocessable Entity',
    423: '423: Locked',
    424: '424: Failed Dependency',
    425: '425: Unordered Collection',
    444: '444: No Response',
    426: '426: Upgrade Required',
    449: '449: Retry With',
    450: '450: Blocked by Windows Parental Controls',
    499: '499: Client Closed Request',
    500: '500: Internal Server Error',
    501: '501: Not Implemented',
    502: '502: Bad Gateway',
    503: '503: Service Unavailable',
    504: '504: Gateway Timeout',
    505: '505: HTTP Version Not Supported',
    506: '506: Variant Also Negotiates',
    507: '507: Insufficient Storage',
    509: '509: Bandwidth Limit Exceeded',
    510: '510: Not Extended'
};

var CouchDB;

/**
 * Convenience method for creating a CouchDB object instance.
 *
 * @param {String} db_url
 * @api public
 */

var exports = module.exports = function (db_url) {
    return new CouchDB(db_url);
};

/**
 * The CouchDB object constructor.
 *
 * @class CouchDB
 * @constructor
 * @param {String} db_url
 * @api public
 */

CouchDB = exports.CouchDB = function (db_url) {
    var ins = this.instance = url.parse(db_url);
    if (!ins.port) {
        if (ins.protocol === 'https:') {
            ins.port = 443;
        }
        else {
            ins.port = 80;
        }
    }
};

/**
 * Tests if a database exists, creates it if not.
 *
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.ensureDB = function (callback) {
    var that = this;
    this.exists('', function (err, exists) {
        if (err || exists) {
            return callback(err, that);
        }
        that.createDB(callback);
    });
};

/**
 * Creates a database.
 *
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.createDB = function (callback) {
    this.client('PUT', '', null, callback);
};

/**
 * Convenience HTTP client for querying a CouchDB instance. Buffers and parses
 * JSON responses before passing to callback. JSON.stringify's data before
 * sending.
 *
 * @param {String} method
 * @param {String} path
 * @param data
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.client = function (method, path, data, callback) {
    var pathname = this.instance.pathname;
    path = (pathname ? pathname + '/' + path: '/' + path);
    path = path.replace(/^\/\/+/, '/');

    var headers = {
        'Host': this.instance.hostname,
        'Accept': 'application/json'
    };
    if (method === 'POST' || method === 'PUT') {
        if (typeof data !== 'string' && !Buffer.isBuffer(data)) {
            try {
                data = JSON.stringify(data);
            }
            catch (e) {
                return callback(e);
            }
        }
        if (!Buffer.isBuffer(data)) {
            data = new Buffer(data);
        }
        headers['Content-Type'] = 'application/json';
        headers['Content-Length'] = data.length;
    }
    else if (data) {
        path = url.parse(path).pathname + '?' + querystring.stringify(data);
    }

    if (this.instance.auth) {
        var auth = decodeURIComponent(this.instance.auth);
        var enc = new Buffer(auth).toString('base64');
        headers.Authorization = "Basic " + enc;
    }

    var proto = (this.instance.protocol === 'https:') ? https: http;

    var request = proto.request({
        host: this.instance.hostname,
        port: this.instance.port,
        method: method,
        path: path,
        headers: headers
    });

    logger.debug('request', method + ' ' + path);
    logger.debug('request headers', headers);
    logger.debug('request data', data);

    request.on('response', function (response) {
        logger.debug('response:', {
            headers: response.headers,
            url: response.url,
            method: response.method,
            statusCode: response.statusCode
        });
        if (response.headers.connection === 'close' &&
            response.statusCode >= 300) {
            var err3 = exports.statusCodeError(response.statusCode);
            err3.response = response;
            callback(err3, data, response);
        }
        else {
            var buffer = [];
            response.on('data', function (chunk) {
                buffer.push(chunk.toString());
            });
            response.on('end', function () {
                try {
                    var data = buffer.length ? JSON.parse(buffer.join('')): null;
                }
                catch (e) {
                    logger.info('Unexpected response', buffer.join(''));
                }
                logger.debug('data:', data);
                if (response.statusCode >= 300) {
                    if (data && data.error) {
                        var err = new Error(
                            data.error + (data.reason ? '\n' + data.reason: '')
                        );
                        err.error = data.error;
                        err.reason = data.reason;
                        err.response = response;
                        callback(err, data, response);
                    }
                    else {
                        var err2 = exports.statusCodeError(response.statusCode);
                        err2.response = response;
                        callback(err2, data, response);
                    }
                }
                else {
                    process.nextTick(function () {
                        callback(null, data, response);
                    });
                }
            });
        }
    });

    if (data && (method === 'POST' || method === 'PUT')) {
        request.write(data, 'utf8');
    }
    request.end();

    //logger.debug('request:', request.output[0]);
};

/**
 * Creates an error object with a message depending on the HTTP status code
 * of a response.
 */

exports.statusCodeError = function (code) {
    if (code in STATUS_MSGS) {
        return new Error(STATUS_MSGS[code]);
    }
    return new Error('Status code: ' + code);
};

/**
 * Encodes a document id or view, list or show name.
 *
 * @name encode(str)
 * @param {String} str
 * @returns {String}
 * @api public
 */

exports.encode = function (str) {
    return encodeURIComponent(str).replace(/^_design%2F/, '_design/');
};

/**
 * Test if a doc exists in the db by doing a HEAD request - doesn't fetch
 * the whole document.
 *
 * @param {String} id
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.exists = function (id, callback) {
    id = exports.encode(id || '');
    this.client('HEAD', id, null, function (err, data, res) {
        res = res || {};
        if (res.statusCode !== 404 && err) {
            return callback(err);
        }
        var exists = (res.statusCode === 200);
        var etag = res.headers.etag;
        var _rev = etag ? etag.substr(1, etag.length - 2): null;
        callback(null, exists, _rev);
    });
};

/**
 * Retrieve a document from a CouchDB instance.
 *
 * @param {String} id
 * @param {Object} data
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.get = function (id, /*optional*/data, callback) {
    if (arguments.length < 3) {
        callback = data;
        data = null;
    }
    id = exports.encode(id || '');
    this.client('GET', id, data, callback);
};

/**
 * Saves a document to a CouchDB instance.
 *
 * Options:
 *      {Boolean} force - write document regardless of conflicts!
 *
 * @param {String} id
 * @param {Object} doc
 * @param {Object} options
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.save = function (id, doc, /*optional*/options, callback) {
    var that = this;

    if (!callback) {
        callback = options;
        options = {};
    }
    var method = id ? 'PUT': 'POST';
    var path = exports.encode(id || '');

    if (options.force && id) {
        // WARNING! this is a brute-force document update
        // updates revision number to latest revision before saving
        this.exists(id, function (err, exists, rev) {
            if (err) {
                return callback(err);
            }
            if (exists) {
                doc._rev = rev;
            }
            that.client(method, path, doc, function (err, d) {
                if (err) {
                    return callback(err);
                }
                doc._id = d.id;
                doc._rev = d.rev;
                callback(null, doc);
            });
        });
    }
    else {
        this.client(method, path, doc, callback);
    }
};

/**
 * Deletes a document from a CouchDB instance.
 *
 * Options:
 *      {Boolean} force - delete document regardless of conflicts!
 *
 * @param {String} id
 * @param {Object} rev
 * @param {Object} options
 * @param {Function} callback
 * @api public
 */

CouchDB.prototype.delete = function (id, rev, /*optional*/options, callback) {
    var that = this;

    if (!callback) {
        callback = options;
        options = {};
    }
    var args = {};
    if (rev) {
        args.rev = rev;
    }
    var path = exports.encode(id || '');

    if (options.force) {
        // WARNING! this is a brute-force document delete
        // updates revision number to latest revision before deleting
        this.exists(id, function (err, exists, rev) {
            if (err) {
                return callback(err);
            }
            if (exists) {
                args.rev = rev;
            }
            that.client('DELETE', path, args, callback);
        });
    }
    else {
        this.client('DELETE', path, args, callback);
    }
};

CouchDB.prototype.uuids = function (count, callback) {
    if (!callback) {
        callback = count;
        count = undefined;
    }
    count = count || 1;
    this.client('GET', '_uuids', {count: count}, function (err, data) {
        callback(err, data.uuids);
    });
};
<|endoftext|>


<commit_before>import React, { PropTypes, Component } from 'react';
import Card, { CardContent } from 'components/Card/Card';
import SocialButton from 'components/SocialButton/SocialButton';
import StarRating from 'react-star-rating';
import { Link } from 'react-router';
import { FormattedPlural } from 'react-intl';
import ImagesPreview from 'components/ImagesPreview/ImagesPreview';
import classNames from 'classnames';
import tooltip from 'react-toolbox/lib/tooltip';
import styles from './styles.scss';

const TooltipLink = tooltip(Link);

export default class Cook extends Component {
  static propTypes = {
    cook: PropTypes.object.isRequired,
    lunch: PropTypes.object.isRequired,
    className: PropTypes.string
  };

  render() {
    const {cook, cook: { user: { facebook, vkontakte, instagram } }, lunch, className} = this.props;
    return (
        <Card className={classNames(styles.root, className)}>
          <CardContent>

              <ImagesPreview image={cook.main_photo} currentImageId={0}/>

              {cook.first_name} {cook.last_name}

              <StarRating name="cook-rating" totalStars={5} editing={false} rating={cook.rating} size={18}/>
              <Link className={styles.feedbackLink} to={`/lunches/${lunch.id}/reviews`}>
                {cook.reviews_count}

                <FormattedPlural value={cook.reviews_count} one="отзыв" few="отзыва" many="отзывов" other="отзывов"/>
              </Link>

            { cook.sanitary_book_photos.length > 0 &&
            <ImagesPreview images={cook.sanitary_book_photos} currentImageId={0} template={(onClick) =>

                              Санитарная книжка
                              <TooltipLink label="Санитарная книжка" to="#" className={styles.sanitaryHelp}
                                           tooltip="Санитарная книжка еще действует">

                              </TooltipLink>

}/>
            }

Фотографии кухни

                {cook.other_photos.map((photo, index) =>

                      <ImagesPreview images={cook.other_photos} currentImageId={index}/>

                )}

            {(vkontakte || facebook || instagram) &&

В соцсетях

                {vkontakte && <SocialButton target="_blank" className={styles.socialButton} name="vk"
                                            href={'http://vk.com/id' + vkontakte}/>}
                {facebook && <SocialButton target="_blank" className={styles.socialButton} name="fb"
                                           href={'http://facebook.com/profile.php?id=' + facebook}/>}
                {instagram && <SocialButton target="_blank" className={styles.socialButton} name="instagram"
                                            href={'http://instagram.com/' + instagram}/>}

}
          </CardContent>
        </Card>
    );
  }
}
<commit_msg>cook.sanitary_book_photos hot fix<commit_after>import React, { PropTypes, Component } from 'react';
import Card, { CardContent } from 'components/Card/Card';
import SocialButton from 'components/SocialButton/SocialButton';
import StarRating from 'react-star-rating';
import { Link } from 'react-router';
import { FormattedPlural } from 'react-intl';
import ImagesPreview from 'components/ImagesPreview/ImagesPreview';
import classNames from 'classnames';
import tooltip from 'react-toolbox/lib/tooltip';
import styles from './styles.scss';

const TooltipLink = tooltip(Link);

export default class Cook extends Component {
  static propTypes = {
    cook: PropTypes.object.isRequired,
    lunch: PropTypes.object.isRequired,
    className: PropTypes.string
  };

  render() {
    const {cook, cook: { user: { facebook, vkontakte, instagram } }, lunch, className} = this.props;
    return (
        <Card className={classNames(styles.root, className)}>
          <CardContent>

              <ImagesPreview image={cook.main_photo} currentImageId={0}/>

              {cook.first_name} {cook.last_name}

              <StarRating name="cook-rating" totalStars={5} editing={false} rating={cook.rating} size={18}/>
              <Link className={styles.feedbackLink} to={`/lunches/${lunch.id}/reviews`}>
                {cook.reviews_count}

                <FormattedPlural value={cook.reviews_count} one="отзыв" few="отзыва" many="отзывов" other="отзывов"/>
              </Link>

            { cook.sanitary_book_photos && cook.sanitary_book_photos.length > 0 &&
            <ImagesPreview images={cook.sanitary_book_photos} currentImageId={0} template={(onClick) =>

                              Санитарная книжка
                              <TooltipLink label="Санитарная книжка" to="#" className={styles.sanitaryHelp}
                                           tooltip="Санитарная книжка еще действует">

                              </TooltipLink>

}/>
            }

Фотографии кухни

                {cook.other_photos.map((photo, index) =>

                      <ImagesPreview images={cook.other_photos} currentImageId={index}/>

                )}

            {(vkontakte || facebook || instagram) &&

В соцсетях

                {vkontakte && <SocialButton target="_blank" className={styles.socialButton} name="vk"
                                            href={'http://vk.com/id' + vkontakte}/>}
                {facebook && <SocialButton target="_blank" className={styles.socialButton} name="fb"
                                           href={'http://facebook.com/profile.php?id=' + facebook}/>}
                {instagram && <SocialButton target="_blank" className={styles.socialButton} name="instagram"
                                            href={'http://instagram.com/' + instagram}/>}

}
          </CardContent>
        </Card>
    );
  }
}
<|endoftext|>


<commit_before>'use strict';

/**
 * This pipeline stage loads song info from external services.
 */

define((require) => {
	const ChromeStorage = require('storage/chrome-storage');
	const ScrobbleService = require('service/scrobble-service');

	const options = ChromeStorage.getStorage(ChromeStorage.OPTIONS);

	const infoToCopy = [
		'duration', 'artist', 'track'
	];
	const metadataToCopy = [
		'artistThumbUrl', 'artistUrl', 'trackUrl', 'albumUrl'
	];

	/**
	 * Load song info using ScrobblerService object.
	 * @param  {Object} song Song instance
	 * @return {Promise} Promise resolved when task has complete
	 */
	function process(song) {
		if (song.isEmpty()) {
			return Promise.resolve(false);
		}

		return ScrobbleService.getSongInfo(song).then((songInfoArr) => {
			let songInfo = getInfo(songInfoArr);
			let isSongValid = songInfo !== null;
			if (isSongValid) {
				for (let field of infoToCopy) {
					song.processed[field] = songInfo[field];
				}
				for (let field of metadataToCopy) {
					song.metadata[field] = songInfo[field];
				}
				if (!song.getAlbum()) {
					song.processed.album = songInfo.album;
				}
			}

			return options.get().then((data) => {
				song.flags.isValid = isSongValid || data.forceRecognize;
			});
		});
	}

	/**
	 * Get song info from array contains the highest keys count.
	 * @param  {Array} songInfoArr Array of song info objects
	 * @return {Object} Song info object
	 */
	function getInfo(songInfoArr) {
		return songInfoArr.reduce((prev, current) => {
			if (!current) {
				return prev;
			}
			if (!prev) {
				return current;
			}
			if (getNonEmptyKeyCount(current) > getNonEmptyKeyCount(prev)) {
				return current;
			}

			return prev;
		}, null);
	}

	/**
	 * Return number of non-empty object keys.
	 * @param  {Object} obj Object instance
	 * @return {Number} Number of non-empty object keys
	 */
	function getNonEmptyKeyCount(obj) {
		let keyCount = 0;
		for (let key in obj) {
			if (obj[key]) {
				++keyCount;
			}
		}

		return keyCount;
	}

	return { process };
});
<commit_msg>Update track metadata correctly<commit_after>'use strict';

/**
 * This pipeline stage loads song info from external services.
 */

define((require) => {
	const ChromeStorage = require('storage/chrome-storage');
	const ScrobbleService = require('service/scrobble-service');

	const options = ChromeStorage.getStorage(ChromeStorage.OPTIONS);

	const infoToCopy = [
		'duration', 'artist', 'track'
	];
	const metadataToCopy = [
		'artistThumbUrl', 'artistUrl', 'trackUrl', 'albumUrl'
	];

	/**
	 * Load song info using ScrobblerService object.
	 * @param  {Object} song Song instance
	 * @return {Promise} Promise resolved when task has complete
	 */
	function process(song) {
		if (song.isEmpty()) {
			return Promise.resolve(false);
		}

		return ScrobbleService.getSongInfo(song).then((songInfoArr) => {
			for (let field of metadataToCopy) {
				delete song.metadata[field];
			}

			let songInfo = getInfo(songInfoArr);
			let isSongValid = songInfo !== null;
			if (isSongValid) {
				for (let field of infoToCopy) {
					song.processed[field] = songInfo[field];
				}
				for (let field of metadataToCopy) {
					song.metadata[field] = songInfo[field];
				}
				if (!song.getAlbum()) {
					song.processed.album = songInfo.album;
				}
			}

			return options.get().then((data) => {
				song.flags.isValid = isSongValid || data.forceRecognize;
			});
		});
	}

	/**
	 * Get song info from array contains the highest keys count.
	 * @param  {Array} songInfoArr Array of song info objects
	 * @return {Object} Song info object
	 */
	function getInfo(songInfoArr) {
		return songInfoArr.reduce((prev, current) => {
			if (!current) {
				return prev;
			}
			if (!prev) {
				return current;
			}
			if (getNonEmptyKeyCount(current) > getNonEmptyKeyCount(prev)) {
				return current;
			}

			return prev;
		}, null);
	}

	/**
	 * Return number of non-empty object keys.
	 * @param  {Object} obj Object instance
	 * @return {Number} Number of non-empty object keys
	 */
	function getNonEmptyKeyCount(obj) {
		let keyCount = 0;
		for (let key in obj) {
			if (obj[key]) {
				++keyCount;
			}
		}

		return keyCount;
	}

	return { process };
});
<|endoftext|>