<commit_before>var cp = require('child_process');
var path = require('path');
var fs = require('fs');
var zipper = require('zip-local');
var _ = require('underscore');
var router = require('./router.js');

/*
 * packs the app into a zip file
 * @param job {Object}: the job specs
 * @return {Object}: a packed app descriptor
 */
function pack(job) {
    var app = path.parse(job.app);
    var packed_buff = zipper.sync.zip(app.dir).compress().memory();

    var descriptor = {
        entry: app.base,
        data: packed_buff.toString('base64')
    }

    return descriptor;
}

/*
 * unpacks a packed app from remote root
 * @param app {Object}: the app descriptor recived from remote root
 */
 function unpack(app) {
     return new Promise(function(resolve, reject) {
         // check if the unpaking directory exists
         fs.stat(__dirname + '/../.unpacks', function(err, stats) {
             if(err) {
                 fs.mkdirSync(__dirname + '/../.unpacks');
             }

             var data = new Buffer(app.data, 'base64');
             var appdir = __dirname + '/../.unpacks/app_' + Date.now();
             fs.mkdirSync(appdir);

             zipper.unzip(data, function(unzipped) {
                 unzipped.save(appdir, function() {
                     resolve(appdir + "/" + app.entry);
                 });
             });
         });
     });
 }

 /*
  * divides the job processes over the specified hosts using round-robin divison
  * @param job {Object}: the job specs
  * @return {Object}: a job distribution plan
 */
 function divide(job) {

     // strip out the password from job.hosts
     var hosts = _.object(
         _.map(_.keys(job.hosts), function(host) {
             if(host !== "local")
                return host.match(/([0-9\.]*@[0-9]*):/)[1];
             return "local";
         }),
         _.map(_.values(job.hosts), function(max) {
             return {count: 0, max: max};
         })
     );

     // round-robin over the hosts and allocated processes numbers
     var hosts_list = _.keys(hosts);
     var allocated = 0;
     var next = 0;
     var counter = 0;
     var overallocate = false;  // if true: hosts will be overallocated with processes to meet job.size
     while(allocated < job.size) {
         var host = hosts[hosts_list[next++ % hosts_list.length]];
         if(host.count != host.max || overallocate) {
             host.count++;
             allocated++;
             if(!overallocate) {
                 // if not in overallocate mode already, reset the counter to indicate allocation
                 counter = 0;
             }
         }
         else {
             counter++;
             // set the overallocate to true if gone through the list without allocation
             overallocate = (counter === hosts_list.length);
         }
     }

     var start_index = 0;
     // register the 0 index process to the local host if exists
     if(_.has(hosts, "local")) {
         hosts.local.start = 0;
         start_index += hosts.local.count;
     }
     // assign the rest of indecies to the rest of hosts
     for(var host in hosts) {
         if(host !== "local") {
             hosts[host].start = start_index;
            start_index += hosts[host].count;
        }
        delete hosts[host]["max"];
     }

     return hosts;
 }

/*
 * runs the given subjob on the local machine
 * @param subjob {Object}: subjob description
 */
function runLocally(subjob) {

    for(var i = subjob.startId; i < subjob.size; i++) {

        var jobInstance = cp.spawn('node', [subjob.app, subjob.size, i], {
            stdio: [null, null, null, 'ipc']
        });

        // register a handler for instance's stdout data
        // reports back to the parent klyng process
        jobInstance.stdout.on('data', function(log) {
            log = log.toString().replace('\n', '');
            router.routeToParent({type: "process:stdout", data: {line: log}});
        });

        // register a handler for instance's stderr data
        // reports back to the parent klyng process
        jobInstance.stderr.on('data', function(elog) {
            elog = elog.toString().replace('\n', '');
            router.routeToParent({type: "process:stderr", data: {line: elog}});
        })

        // register a handler for instance exit event
        // reports back to the parent klyng process
        jobInstance.on('exit', function(code) {
            router.routeToParent({type: "process:exit", data: {code: code}});
        });

        // register a handler for instance's messages
        jobInstance.on('message', function(msg) {
            if(msg.type === 'klyng:msg') {
                router.routeTo(msg.header.to, msg);
            }
        });

        jobInstance.unref();

        router.setLocalChannel(i, jobInstance);
    }
};

module.exports = {
    pack: pack,
    unpack: unpack,
    divide,
    runLocally: runLocally
};
<commit_msg>updated divide function to reflect changes in 435c644<commit_after>var cp = require('child_process');
var path = require('path');
var fs = require('fs');
var zipper = require('zip-local');
var _ = require('underscore');
var router = require('./router.js');

/*
 * packs the app into a zip file
 * @param job {Object}: the job specs
 * @return {Object}: a packed app descriptor
 */
function pack(job) {
    var app = path.parse(job.app);
    var packed_buff = zipper.sync.zip(app.dir).compress().memory();

    var descriptor = {
        entry: app.base,
        data: packed_buff.toString('base64')
    }

    return descriptor;
}

/*
 * unpacks a packed app from remote root
 * @param app {Object}: the app descriptor recived from remote root
 */
 function unpack(app) {
     return new Promise(function(resolve, reject) {
         // check if the unpaking directory exists
         fs.stat(__dirname + '/../.unpacks', function(err, stats) {
             if(err) {
                 fs.mkdirSync(__dirname + '/../.unpacks');
             }

             var data = new Buffer(app.data, 'base64');
             var appdir = __dirname + '/../.unpacks/app_' + Date.now();
             fs.mkdirSync(appdir);

             zipper.unzip(data, function(unzipped) {
                 unzipped.save(appdir, function() {
                     resolve(appdir + "/" + app.entry);
                 });
             });
         });
     });
 }

 /*
  * divides the job processes over the specified hosts using round-robin divison
  * @param job {Object}: the job specs
  * @return {Object}: a job distribution plan
 */
 function divide(job) {

     var hosts = {};

     for(var host in job.hosts) {
         hosts[host] = {count: 0, max: job.hosts[host].max_procs}
     }

     // round-robin over the hosts and allocated processes numbers
     var hosts_list = _.keys(hosts);
     var allocated = 0;
     var next = 0;
     var counter = 0;
     var overallocate = false;  // if true: hosts will be overallocated with processes to meet job.size
     while(allocated < job.size) {
         var host = hosts[hosts_list[next++ % hosts_list.length]];
         if(host.count != host.max || overallocate) {
             host.count++;
             allocated++;
             if(!overallocate) {
                 // if not in overallocate mode already, reset the counter to indicate allocation
                 counter = 0;
             }
         }
         else {
             counter++;
             // set the overallocate to true if gone through the list without allocation
             overallocate = (counter === hosts_list.length);
         }
     }

     var start_index = 0;
     // register the 0 index process to the local host if exists
     if(_.has(hosts, "local")) {
         hosts.local.start = 0;
         start_index += hosts.local.count;
     }
     // assign the rest of indecies to the rest of hosts
     for(var host in hosts) {
         if(host !== "local") {
             hosts[host].start = start_index;
            start_index += hosts[host].count;
        }
        delete hosts[host]["max"];
     }

     return hosts;
 }

/*
 * runs the given subjob on the local machine
 * @param subjob {Object}: subjob description
 */
function runLocally(subjob) {

    for(var i = subjob.startId; i < subjob.size; i++) {

        var jobInstance = cp.spawn('node', [subjob.app, subjob.size, i], {
            stdio: [null, null, null, 'ipc']
        });

        // register a handler for instance's stdout data
        // reports back to the parent klyng process
        jobInstance.stdout.on('data', function(log) {
            log = log.toString().replace('\n', '');
            router.routeToParent({type: "process:stdout", data: {line: log}});
        });

        // register a handler for instance's stderr data
        // reports back to the parent klyng process
        jobInstance.stderr.on('data', function(elog) {
            elog = elog.toString().replace('\n', '');
            router.routeToParent({type: "process:stderr", data: {line: elog}});
        })

        // register a handler for instance exit event
        // reports back to the parent klyng process
        jobInstance.on('exit', function(code) {
            router.routeToParent({type: "process:exit", data: {code: code}});
        });

        // register a handler for instance's messages
        jobInstance.on('message', function(msg) {
            if(msg.type === 'klyng:msg') {
                router.routeTo(msg.header.to, msg);
            }
        });

        jobInstance.unref();

        router.setLocalChannel(i, jobInstance);
    }
};

module.exports = {
    pack: pack,
    unpack: unpack,
    divide,
    runLocally: runLocally
};
<|endoftext|>


<commit_before>var cp = require('child_process');
var path = require('path');
var fs = require('fs');
var readline = require('readline');
var zipper = require('zip-local');
var browserify = require('browserify');
var router = require('./router.js');

/*
 * packs the app into a zip file
 * @param job {Object}: the job specs
 * @return {Promise}: of the app descriptor {pckg, id}
 */
function pack(job) {
    return new Promise(function(resolve, reject) {

        browserify(job.app, {
            builtins: false,
            browserField: false,
            insertGlobalVars: {
                'process': undefined
            }
        })
        .external('fibers')
        .bundle(function (err, buffer) {
            if(err) {
                return reject(err);
            }

            var app_id = Date.now().toString();

            zipper.zip(buffer, "app_" + app_id + ".js", function(err, zipped) {
                if(err) {
                    return reject(err);
                }

                var packed_buff = zipped.compress().memory();
                resolve({
                    id: app_id,
                    pckg: packed_buff.toString('base64')
                });
            });
        });
    });
}

/*
 * unpacks a packed app from remote root
 * @param app {Object}: the app descriptor recived from remote root
 */
 function unpack(app) {
     return new Promise(function(resolve, reject) {
         // check if the unpaking directory exists
         var unpacks_dir = __dirname + '/../.unpacks/';
         fs.stat(unpacks_dir, function(err, stats) {
             if(err) {
                 try {
                    fs.mkdirSync(unpacks_dir);
                 }
                 catch(err) { return reject(err); }
             }

             var data = new Buffer(app.pckg, 'base64');

             zipper.unzip(data, function(err, unzipped) {
                 if(err) {
                     return reject(err);
                 }

                 unzipped.save(unpacks_dir, function(err) {
                     if(err) {
                         return reject(err);
                     }

                     resolve(unpacks_dir + "app_" + app.id + ".js");
                 });
             });
         });
     });
 }

 /*
  * divides the job processes over the specified hosts using round-robin divison
  * @param job {Object}: the job specs
  * @return {Object}: a job distribution plan
 */
 function divide(job) {

     var hosts = {};

     for(var host in job.hosts) {
         hosts[host] = {count: 0, max: job.hosts[host].max_procs}
     }

     // round-robin over the hosts and allocated processes numbers
     var hosts_list = Object.keys(hosts);
     var allocated = 0;
     var next = 0;
     var counter = 0;
     var overallocate = false;  // if true: hosts will be overallocated with processes to meet job.size
     while(allocated < job.size) {
         var host = hosts[hosts_list[next++ % hosts_list.length]];
         if(host.count != host.max || overallocate) {
             host.count++;
             allocated++;
             if(!overallocate) {
                 // if not in overallocate mode already, reset the counter to indicate allocation
                 counter = 0;
             }
         }
         else {
             counter++;
             // set the overallocate to true if gone through the list without allocation
             overallocate = (counter === hosts_list.length);
         }
     }

     var start_index = 0;
     // register the 0 index process to the local host if exists
     if(!!hosts["local"]) {
         hosts.local.start = 0;
         start_index += hosts.local.count;
     }
     // assign the rest of indecies to the rest of hosts
     for(var host in hosts) {
         if(host !== "local") {
             hosts[host].start = start_index;
            start_index += hosts[host].count;
        }
        delete hosts[host]["max"];
     }

     return hosts;
 }

/*
 * runs the given subjob on the local machine
 * @param subjob {Object}: subjob description
 */
function runLocally(subjob) {

    for(var i = 0; i < subjob.subsize; i++) {

        var rank = subjob.start + i;
        var jobInstance = cp.spawn('node', [subjob.app, subjob.size, rank], {
            stdio: [null, null, null, 'ipc']
        });

        var stdoutRlInterface = readline.createInterface({input: jobInstance.stdout});
        var stderrRlInterface = readline.createInterface({input: jobInstance.stderr});

        // register a handler for instance's stdout data
        // reports back to the parent klyng process
        stdoutRlInterface.on('line', function(log) {
            router.routeToParent({type: "process:stdout", data: {line: log}});
        });

        // register a handler for instance's stderr data
        // reports back to the parent klyng process
        stderrRlInterface.on('line', function(elog) {
            router.routeToParent({type: "process:stderr", data: {line: elog}});
        })

        // register a handler for instance exit event
        // reports back to the parent klyng process
        jobInstance.on('exit', function(code) {
            router.routeToParent({type: "process:exit", data: {code: code}});
        });

        // register a handler for instance's messages
        jobInstance.on('message', function(msg) {
            if(msg.type === 'klyng:msg') {
                router.routeTo(msg.header.to, msg);
            }
        });

        jobInstance.unref();

        router.setLocalChannel(rank, jobInstance);
    }
};

module.exports = {
    pack: pack,
    unpack: unpack,
    divide,
    runLocally: runLocally
};
<commit_msg>added abort logic<commit_after>var cp = require('child_process');
var path = require('path');
var fs = require('fs');
var readline = require('readline');
var zipper = require('zip-local');
var browserify = require('browserify');
var router = require('./router.js');
var tcp = require('./tcp.js');

/*
 * packs the app into a zip file
 * @param job {Object}: the job specs
 * @return {Promise}: of the app descriptor {pckg, id}
 */
function pack(job) {
    return new Promise(function(resolve, reject) {

        browserify(job.app, {
            builtins: false,
            browserField: false,
            insertGlobalVars: {
                'process': undefined
            }
        })
        .external('fibers')
        .bundle(function (err, buffer) {
            if(err) {
                return reject(err);
            }

            var app_id = Date.now().toString();

            zipper.zip(buffer, "app_" + app_id + ".js", function(err, zipped) {
                if(err) {
                    return reject(err);
                }

                var packed_buff = zipped.compress().memory();
                resolve({
                    id: app_id,
                    pckg: packed_buff.toString('base64')
                });
            });
        });
    });
}

/*
 * unpacks a packed app from remote root
 * @param app {Object}: the app descriptor recived from remote root
 */
 function unpack(app) {
     return new Promise(function(resolve, reject) {
         // check if the unpaking directory exists
         var unpacks_dir = __dirname + '/../.unpacks/';
         fs.stat(unpacks_dir, function(err, stats) {
             if(err) {
                 try {
                    fs.mkdirSync(unpacks_dir);
                 }
                 catch(err) { return reject(err); }
             }

             var data = new Buffer(app.pckg, 'base64');

             zipper.unzip(data, function(err, unzipped) {
                 if(err) {
                     return reject(err);
                 }

                 unzipped.save(unpacks_dir, function(err) {
                     if(err) {
                         return reject(err);
                     }

                     resolve(unpacks_dir + "app_" + app.id + ".js");
                 });
             });
         });
     });
 }

 /*
  * divides the job processes over the specified hosts using round-robin divison
  * @param job {Object}: the job specs
  * @return {Object}: a job distribution plan
 */
 function divide(job) {

     var hosts = {};

     for(var host in job.hosts) {
         hosts[host] = {count: 0, max: job.hosts[host].max_procs}
     }

     // round-robin over the hosts and allocated processes numbers
     var hosts_list = Object.keys(hosts);
     var allocated = 0;
     var next = 0;
     var counter = 0;
     var overallocate = false;  // if true: hosts will be overallocated with processes to meet job.size
     while(allocated < job.size) {
         var host = hosts[hosts_list[next++ % hosts_list.length]];
         if(host.count != host.max || overallocate) {
             host.count++;
             allocated++;
             if(!overallocate) {
                 // if not in overallocate mode already, reset the counter to indicate allocation
                 counter = 0;
             }
         }
         else {
             counter++;
             // set the overallocate to true if gone through the list without allocation
             overallocate = (counter === hosts_list.length);
         }
     }

     var start_index = 0;
     // register the 0 index process to the local host if exists
     if(!!hosts["local"]) {
         hosts.local.start = 0;
         start_index += hosts.local.count;
     }
     // assign the rest of indecies to the rest of hosts
     for(var host in hosts) {
         if(host !== "local") {
             hosts[host].start = start_index;
            start_index += hosts[host].count;
        }
        delete hosts[host]["max"];
     }

     return hosts;
 }

/*
 * runs the given subjob on the local machine
 * @param subjob {Object}: subjob description
 */
function runLocally(subjob) {

    for(var i = 0; i < subjob.subsize; i++) {

        var rank = subjob.start + i;
        var jobInstance = cp.spawn('node', [subjob.app, subjob.size, rank], {
            stdio: [null, null, null, 'ipc']
        });

        var stdoutRlInterface = readline.createInterface({input: jobInstance.stdout});
        var stderrRlInterface = readline.createInterface({input: jobInstance.stderr});

        // register a handler for instance's stdout data
        // reports back to the parent klyng process
        stdoutRlInterface.on('line', function(log) {
            router.routeToParent({type: "process:stdout", data: {line: log}});
        });

        // register a handler for instance's stderr data
        // reports back to the parent klyng process
        stderrRlInterface.on('line', function(elog) {
            router.routeToParent({type: "process:stderr", data: {line: elog}});
        })

        // register a handler for instance exit event
        // reports back to the parent klyng process
        jobInstance.on('exit', function(code) {
            router.routeToParent({type: "process:exit", data: {code: code}});
        });

        // register a handler for instance's messages
        jobInstance.on('message', function(msg) {
            if(msg.type === 'klyng:msg') {
                router.routeTo(msg.header.to, msg);
            }
        });

        jobInstance.unref();

        router.setLocalChannel(rank, jobInstance);
    }
};

/*
 * aborts the job running on the current host
 * @param reason {String}: the reason for abortion
 */
function abort(reason) {

     router.routeToParent({type: 'job:aborted', reason: reason});

     // kill all local processes
     var locals = router.getLocalChannels();
     for(var rank in locals) {
         var proc = locals[rank];
         proc.kill();
     }

     // wait for a second to ensure that job:aborted is on the network if possible
     setTimeout(() => {
         // disconnect from all remote beacon
         var remotes = router.getRemoteChannels();
         for(var socketId in remotes) {
             var connection = remotes[socketId];
             tcp.disconnectFrom(connection.path, connection.port);
         }
         router.clear();
     }, 1000);
 }

module.exports = {
    pack: pack,
    unpack: unpack,
    divide,
    runLocally: runLocally,
    abort: abort
};
<|endoftext|>


<commit_before>var util = require("util"),
    vm = require("vm");

/*
 JSON Command class
*/

JSON.Command = function(args) {
  this.args = null;

  this.debugOn = false;
  this.fileNames = [];
  this.files = null;
  this.keys = [];
  this.transformedKeys = [];
  this.uglyOutput = false;
  this.inspectOutput = false;
  this.headerPassthrough = false;
  this.columnOutput = false;
  this.useObject = null;
  this.inputIsArray = false;

  this.conditionals = [];
  this.executables = [];

  this.stdin = null;
  this.buffer = "";

  if (args) { this.processArgs(args); }
};

JSON.Command.prototype.printhelp = function() {
  console.log("usage: stdout_generator | json [options] [fields]");
  console.log("");
  console.log("json processes standard input and parses json objects. json currently handles a");
  console.log("few different standard input formats and provides a number of options tailored");
  console.log("toward inspecting and transforming the parsed json objects.");
  console.log("");
  console.log("options:\n");
  console.log("  -h                    print this help info and exit\n");
  console.log("  -v (-V | --version)   print version number and exit\n");
  console.log("  -u                    print ugly json output, each object on a single line\n");
  console.log("  -d                    print debugging output including exception messages\n");
  console.log("  -o object.path        specify the path to an array to be iterated on\n");
  console.log("  new.key=old_key       move old_key to new.key in output object\n");
  console.log("  -a                    input object is an array, process each element separately\n");
  console.log("  -c \"js conditional\"   js conditional to be run in the context of each object");
  console.log("                         that determines whether an object is printed\n");
  console.log("  -C                    print the output fields as tab delimited columns in");
  console.log("                         the order specified by fields\n");
  console.log("  -e \"js expression\"    execute arbitrary js in the context of each object.\n");
  console.log("  -i                    use node's util.inspect instead of JSON.stringify\n");
  console.log("  -H                    print headers, if they are supplied.");
  console.log("                        Useful for output from curl -i.\n");

  console.log("examples:\n");
  console.log("  curl http://search.twitter.com/search.json?q=node.js 2> /dev/null |");
  console.log("   json -o results\n");
  console.log("  curl http://search.twitter.com/search.json?q=node.js 2> /dev/null |");
  console.log("   json -o results new_id=id\n");
  console.log("  curl http://search.twitter.com/search.json?q=node.js 2> /dev/null |");
  console.log("   json -o results -C from_user from_user_id text\n");
  console.log("more help:\n");
  console.log("  use \"man json\" or visit http://github.com/zpoley/json-command\n");
  process.exit();
};

JSON.Command.prototype.printversion = function() {
  var npm = require("npm");
  npm.load([], function(er) {
    console.log("json command line toolkit\n version: ");
    npm.commands.view([ "json", "version" ], function(er, data) {
      process.exit();
    });
  });
};

JSON.Command.prototype.stringify = function(obj) {
  return( this.inspectOutput ? util.inspect(obj, false, Infinity, true)
        : this.uglyOutput ? JSON.stringify(obj)
        : JSON.stringify(obj, null, 2) );
};

JSON.Command.prototype.debug = function(msg) {
  if (this.debugOn) { console.log(msg); }
};

JSON.Command.prototype.printex = function(ex) {
  this.debug("ex: " + JSON.stringify(ex, null, 2));
};

/*
  Process Command line arguments to JSON Command
*/

JSON.Command.prototype.processArgs = function processArgs(args) {

  // copy argv to chop it up
  var a = args.slice(0);
  // turn -iH into -i -H
  // nb: don't cache length.  it may change.
  for (var i = 0; i < a.length; i ++) {
    if (a[i].charAt(0) === "-" && a[i].length > 2) {
      var arg = a[i].replace(/^-+/, "").split("").map(function (a) {
        return "-" + a;
      });
      a.splice.apply(a, [i, 1].concat(arg));
    }
  }

  while (a.length > 0) {
    var arg = a.shift();
    switch(arg) {
      case "-h": // display help and exit
        this.printhelp();
        break;
      case "-v": // display version and exit
      case "-V":
      case "--version":
        this.printversion();
        break;
      case "-f": // file
        this.fileNames.push(a.shift());
        break;
      case "-d": // debug
        this.debugOn = true;
        break;
      case "-u": // pretty printing (turn off)
        this.uglyOutput = true;
        break;
      case "-c": // conditional
        this.conditionals.push(a.shift());
        break;
      case "-C": // column output
        this.columnOutput = true;
        break;
      case "-e": // executable (transform data)
        this.executables.push(a.shift());
        break;
      case "-o": // use object
        this.useObject = a.shift();
        break;
      case "-a": // array
        this.inputIsArray = true;
        break;
      case "-i": // use util.inspect
        this.inspectOutput = true;
        break;
      case "-H": // header passthrough
        this.headerPassthrough = true;
        break;
      default: // json object keys
        if (arg.match("=")) {
          var kk = arg.split("=");
          this.keys.push(kk[0]);
          this.transformedKeys.push({
            newKey : kk[0],
            oldKey : kk[1]
          });
        }
        else {
          this.keys.push(arg);
        }
        break;
    }
  }
};

/*
  Create any reuested keys that don't already exist. Init values with null.
   The default value could be an option.
*/

JSON.Command.prototype.createRequestedKeys = function(parsedObject) {
  // instantiate any requested keys
  for(var j = 0; (j < this.keys.length); j++) {
    if (typeof(parsedObject[this.keys[j]]) === 'undefined') {
      parsedObject[this.keys[j]] = null;
    }
  }
};

/*
  Check conditionals against object.
*/

JSON.Command.prototype.checkConditionals = function(parsedObject) {
  if (this.conditionals.length) {
    try {
      var conditionsFailed = false;
      for(var i = 0; (i < this.conditionals.length); i++) {
        if (!vm.runInNewContext(this.conditionals[i], parsedObject)) {
          conditionsFailed = true;
        }
      }
      // if any conditions failed return false
      if (conditionsFailed) { return false; }
    }
    catch(ex) {
      // if any conditional fails, return false,
      //  the conditional may access something not present, etc..
      this.printex(ex);
      return false;
    }
  }
  // all conditionals passed
  return true;
};

/*
  Process key transforms against object.
*/

JSON.Command.prototype.processKeyTransforms = function(parsedObject) {
  if (this.transformedKeys.length) {
    for(var i = 0; (i < this.transformedKeys.length); i++) {
      try {
        vm.runInNewContext(this.transformedKeys[i].newKey +
          " = " + this.transformedKeys[i].oldKey, parsedObject)
      }
      catch (ex) {
        this.printex(ex);
      }
    }
  }
};

/*
  Process executables against object.
*/

JSON.Command.prototype.processExecutables = function(parsedObject) {
  if (this.executables.length) {
    for(var i = 0; (i < this.executables.length); i++) {
      try {
        vm.runInNewContext(this.executables[i], parsedObject);
      }
      catch (ex) {
        // stop catstrophic failure if any executable fails.
        //  TODO: this may not be the desired behavior.
        this.printex(ex);
      }
    }
  }
};

/*
  Process requested keys against parsedObject.
   This is one of the most complicated parts of this code, and it may
   very well not need to be. If you have a better idea of how to do this
   please let me know: zpoley@gmail.com.

   What's happening here is:
    1. Create a new object to replace the old one since we don't want all
     the keys from the old object.
    2. Create each object necessary in the chain in order for the resulting
     object to retain the same structure of the parsedObject.
     (arrays and dictionaries require separate handling)
    3. Assign each requested key value from the parsedObject into the new
     object. (arrays and dictionaries require separate handling)

*/

JSON.Command.prototype.processKeys = function(parsedObject) {
  if (this.keys.length) {
    var hsh = {}, cols = [], evalStr = null, instStr = null, simpleKey = null;
    function instantiateArrayAndPush(key) {
      simpleKey = key.split("[").shift();
      // instantiate array in new object if not exists
      instStr = "if (!hsh." + simpleKey + ") { hsh." + simpleKey + " = []; }";
      eval(instStr);
      // push new value into array (command line order matters)
      evalStr = "hsh." + simpleKey + ".push(" + "parsedObject." + key + ")";
      eval(evalStr);
      cols.push(eval("parsedObject." + key));
    }
    for (var i = 0; (i < this.keys.length); i++) {
      try {
        if ((this.keys[i].indexOf(".") > -1) || (this.keys[i].indexOf("[") > -1)) {
          // create any keys that don't exist in the object chain
          if (this.keys[i].indexOf(".") > -1) {
            var s = this.keys[i].split(".");
            for (var j = 1; (j < s.length); j++) {
              // create necessary keys
              evalStr = "hsh." + s.slice(0,j).join(".");
              if (!eval(evalStr)) {
                eval("hsh." + s.slice(0,j).join(".") + " = {};");
              }
            }
            if (this.keys[i].indexOf("[") > -1) {
              instantiateArrayAndPush(this.keys[i]);
            }
            else {
              evalStr = "hsh." + s.join(".") + " = " + "parsedObject." + s.join(".");
              eval(evalStr);
              cols.push(eval("parsedObject." + s.join(".")));
            }
          }
          else if (this.keys[i].indexOf("[") > -1) {
            instantiateArrayAndPush(this.keys[i]);
          }
        }
        else {
          // no expansion
          hsh[this.keys[i]] = parsedObject[this.keys[i]];
          cols.push(parsedObject[this.keys[i]]);
        }
      }
      catch(ex) {
        this.debug("Failed to read property " + this.keys[i] + " from object: " + JSON.stringify(parsedObject));
        ex.message = "Failed to read property";
        throw ex;
      }
    }
    return this.columnOutput ? cols : hsh;
  }
  else {
    return parsedObject;
  }
};

/*
  Process input objects.
*/

JSON.Command.prototype.processObjects = function(objects) {

  var rawObject = null, parsedObject = null;

  try {
    if (this.useObject && objects && objects.length > 0 && typeof(objects[0]) == 'string' && objects[0].trim().length > 0) {
      objects = vm.runInNewContext(this.useObject, JSON.parse(objects[0]));
    }
  }
  catch(ex) {
    this.printex(ex);
  }

  if (this.inputIsArray && objects && objects.length > 0 && typeof(objects[0]) == 'string') {
    var trimmed = objects[0].trim();
    if (trimmed.length > 0 && trimmed[0] == '[') {
      objects = JSON.parse(objects[0]);
    }
  }

  try {
    for (var i = 0; (i < (objects.length)); i++) {
      // if there's no object, there's nothing to do
      //  (null object is not the same as string null)
      if ((objects[i] == null) || (objects[i] == undefined)) { continue; }

      try {
        if (typeof(objects[i]) == "string") {
          rawObject = objects[i];
          if (rawObject.trim().length == 0) continue;
          parsedObject = JSON.parse(rawObject);
        }
        else {
          rawObject = JSON.stringify(objects[i]);
          parsedObject = objects[i];
        }
      } catch(ex) {
        // discard bad records
        this.printex(ex);
        continue;
      }

      // create any requested keys that are missing
      this.createRequestedKeys(parsedObject);

      // process key transformations  on this parsedObject
      this.processKeyTransforms(parsedObject);

      // process executables on this parsedObject
      this.processExecutables(parsedObject);

      // continue if any conditionals fail on this parsedObject
      if (!this.checkConditionals(parsedObject)) {
        continue;
      }

      try {
        // process requested keys on the parsed object
        outputObject = this.processKeys(parsedObject);
      }
      catch(ex) { continue; }

      // finally, print output
      if (this.columnOutput) {
        process.stdout.write(outputObject.join("\t") + "\n");
      }
      else {
        process.stdout.write(this.stringify(outputObject) + "\n");
      }
    }
  }
  catch(ex) {
    this.printex(ex);
  }
};

/*
  Process input.
*/

JSON.Command.prototype.processInput = function() {
  if (this.files) {
    // TODO: implement file support?
  }
  else {
    this.stdin = process.openStdin();
    this.stdin.setEncoding("utf8");
    this.stdin.jsonC = this; // context for closure

    this.stdin.on("data", function(chunk) {
      this.jsonC.buffer += chunk;
      if (this.jsonC.inputIsArray) return;

      var objects = null;
      if (this.jsonC.buffer.match(/\n/g) ||
          this.jsonC.buffer.match(/\r\n/g) ||
          this.jsonC.buffer.match(/\0/g) ||
          this.jsonC.buffer.match("}{")) {
        if (this.jsonC.buffer.match(/\n/g)) {
          objects = this.jsonC.buffer.split("\n");
        }
        if (this.jsonC.buffer.match(/\r\n/g)) {
          objects = this.jsonC.buffer.split("\r\n");
        }
        if (this.jsonC.buffer.match(/\0/g)) {
          objects = this.jsonC.buffer.split("\0");
        }
        if (this.jsonC.buffer.match("}{")) {
          objects = this.jsonC.buffer.split("}{").join("}\n{").split("\n");
        }

        this.jsonC.buffer = objects.pop();

        if (this.jsonC.headerPassthrough) {
          for (var i = 0, l = objects.length; i < l; i ++) {
            process.stdout.write(objects[i]+"\r\n");
            if (objects[i] === "") {
              this.jsonC.headerPassthrough = false;
              break;
            }
          }
          objects.splice(0, i);
        }

        if (objects.length) this.jsonC.processObjects(objects);
      }
    });

    var handleEPIPE = function(e) {
      if (e.code !== "EPIPE") {
        process.emit("error", e);
      }
    };
    process.stdout.on("error", handleEPIPE);

    this.stdin.on("end", function() {
      this.jsonC.processObjects([this.jsonC.buffer, null]);
      process.stdout.removeListener("error", handleEPIPE);
    });
  }
};

<commit_msg>Added -, for leading-comma / diff-friendly / Node-style output.<commit_after>var util = require("util"),
    vm = require("vm");

/*
 JSON Command class
*/

JSON.Command = function(args) {
  this.args = null;

  this.debugOn = false;
  this.fileNames = [];
  this.files = null;
  this.keys = [];
  this.transformedKeys = [];
  this.uglyOutput = false;
  this.leadingComma = false;
  this.inspectOutput = false;
  this.headerPassthrough = false;
  this.columnOutput = false;
  this.useObject = null;
  this.inputIsArray = false;

  this.conditionals = [];
  this.executables = [];

  this.stdin = null;
  this.buffer = "";

  if (args) { this.processArgs(args); }
};

JSON.Command.prototype.printhelp = function() {
  console.log("usage: stdout_generator | json [options] [fields]");
  console.log("");
  console.log("json processes standard input and parses json objects. json currently handles a");
  console.log("few different standard input formats and provides a number of options tailored");
  console.log("toward inspecting and transforming the parsed json objects.");
  console.log("");
  console.log("options:\n");
  console.log("  -h                    print this help info and exit\n");
  console.log("  -v (-V | --version)   print version number and exit\n");
  console.log("  -u                    print ugly json output, each object on a single line\n");
  console.log("  -,                    print leading-comma diff-friendly output, one line per property\n");
  console.log("  -d                    print debugging output including exception messages\n");
  console.log("  -o object.path        specify the path to an array to be iterated on\n");
  console.log("  new.key=old_key       move old_key to new.key in output object\n");
  console.log("  -a                    input object is an array, process each element separately\n");
  console.log("  -c \"js conditional\"   js conditional to be run in the context of each object");
  console.log("                         that determines whether an object is printed\n");
  console.log("  -C                    print the output fields as tab delimited columns in");
  console.log("                         the order specified by fields\n");
  console.log("  -e \"js expression\"    execute arbitrary js in the context of each object.\n");
  console.log("  -i                    use node's util.inspect instead of JSON.stringify\n");
  console.log("  -H                    print headers, if they are supplied.");
  console.log("                        Useful for output from curl -i.\n");

  console.log("examples:\n");
  console.log("  curl http://search.twitter.com/search.json?q=node.js 2> /dev/null |");
  console.log("   json -o results\n");
  console.log("  curl http://search.twitter.com/search.json?q=node.js 2> /dev/null |");
  console.log("   json -o results new_id=id\n");
  console.log("  curl http://search.twitter.com/search.json?q=node.js 2> /dev/null |");
  console.log("   json -o results -C from_user from_user_id text\n");
  console.log("more help:\n");
  console.log("  use \"man json\" or visit http://github.com/zpoley/json-command\n");
  process.exit();
};

JSON.Command.prototype.printversion = function() {
  var npm = require("npm");
  npm.load([], function(er) {
    console.log("json command line toolkit\n version: ");
    npm.commands.view([ "json", "version" ], function(er, data) {
      process.exit();
    });
  });
};

JSON.Command.prototype.stringify = function(obj) {
  return( this.inspectOutput ? util.inspect(obj, false, Infinity, true)
        : this.leadingComma ? this.diffFriendly(obj)
        : this.uglyOutput ? JSON.stringify(obj)
        : JSON.stringify(obj, null, 2) );
};

JSON.Command.prototype.diffFriendly = function(obj) {
  return JSON.stringify(obj, null, 2)
    .replace(/ ?([[{,])\n ( *)(?: )/gm, '\n$2$1 ')
    .replace(/(^|, ?)\n *{/gm, '$1{');
};

JSON.Command.prototype.debug = function(msg) {
  if (this.debugOn) { console.log(msg); }
};

JSON.Command.prototype.printex = function(ex) {
  this.debug("ex: " + JSON.stringify(ex, null, 2));
};

/*
  Process Command line arguments to JSON Command
*/

JSON.Command.prototype.processArgs = function processArgs(args) {

  // copy argv to chop it up
  var a = args.slice(0);
  // turn -iH into -i -H
  // nb: don't cache length.  it may change.
  for (var i = 0; i < a.length; i ++) {
    if (a[i].charAt(0) === "-" && a[i].length > 2) {
      var arg = a[i].replace(/^-+/, "").split("").map(function (a) {
        return "-" + a;
      });
      a.splice.apply(a, [i, 1].concat(arg));
    }
  }

  while (a.length > 0) {
    var arg = a.shift();
    switch(arg) {
      case "-h": // display help and exit
        this.printhelp();
        break;
      case "-v": // display version and exit
      case "-V":
      case "--version":
        this.printversion();
        break;
      case "-f": // file
        this.fileNames.push(a.shift());
        break;
      case "-d": // debug
        this.debugOn = true;
        break;
      case "-u": // pretty printing (turn off)
        this.uglyOutput = true;
        break;
      case "-,": // diff-friendlier output
        this.leadingComma = true;
        break;
      case "-c": // conditional
        this.conditionals.push(a.shift());
        break;
      case "-C": // column output
        this.columnOutput = true;
        break;
      case "-e": // executable (transform data)
        this.executables.push(a.shift());
        break;
      case "-o": // use object
        this.useObject = a.shift();
        break;
      case "-a": // array
        this.inputIsArray = true;
        break;
      case "-i": // use util.inspect
        this.inspectOutput = true;
        break;
      case "-H": // header passthrough
        this.headerPassthrough = true;
        break;
      default: // json object keys
        if (arg.match("=")) {
          var kk = arg.split("=");
          this.keys.push(kk[0]);
          this.transformedKeys.push({
            newKey : kk[0],
            oldKey : kk[1]
          });
        }
        else {
          this.keys.push(arg);
        }
        break;
    }
  }
};

/*
  Create any reuested keys that don't already exist. Init values with null.
   The default value could be an option.
*/

JSON.Command.prototype.createRequestedKeys = function(parsedObject) {
  // instantiate any requested keys
  for(var j = 0; (j < this.keys.length); j++) {
    if (typeof(parsedObject[this.keys[j]]) === 'undefined') {
      parsedObject[this.keys[j]] = null;
    }
  }
};

/*
  Check conditionals against object.
*/

JSON.Command.prototype.checkConditionals = function(parsedObject) {
  if (this.conditionals.length) {
    try {
      var conditionsFailed = false;
      for(var i = 0; (i < this.conditionals.length); i++) {
        if (!vm.runInNewContext(this.conditionals[i], parsedObject)) {
          conditionsFailed = true;
        }
      }
      // if any conditions failed return false
      if (conditionsFailed) { return false; }
    }
    catch(ex) {
      // if any conditional fails, return false,
      //  the conditional may access something not present, etc..
      this.printex(ex);
      return false;
    }
  }
  // all conditionals passed
  return true;
};

/*
  Process key transforms against object.
*/

JSON.Command.prototype.processKeyTransforms = function(parsedObject) {
  if (this.transformedKeys.length) {
    for(var i = 0; (i < this.transformedKeys.length); i++) {
      try {
        vm.runInNewContext(this.transformedKeys[i].newKey +
          " = " + this.transformedKeys[i].oldKey, parsedObject)
      }
      catch (ex) {
        this.printex(ex);
      }
    }
  }
};

/*
  Process executables against object.
*/

JSON.Command.prototype.processExecutables = function(parsedObject) {
  if (this.executables.length) {
    for(var i = 0; (i < this.executables.length); i++) {
      try {
        vm.runInNewContext(this.executables[i], parsedObject);
      }
      catch (ex) {
        // stop catstrophic failure if any executable fails.
        //  TODO: this may not be the desired behavior.
        this.printex(ex);
      }
    }
  }
};

/*
  Process requested keys against parsedObject.
   This is one of the most complicated parts of this code, and it may
   very well not need to be. If you have a better idea of how to do this
   please let me know: zpoley@gmail.com.

   What's happening here is:
    1. Create a new object to replace the old one since we don't want all
     the keys from the old object.
    2. Create each object necessary in the chain in order for the resulting
     object to retain the same structure of the parsedObject.
     (arrays and dictionaries require separate handling)
    3. Assign each requested key value from the parsedObject into the new
     object. (arrays and dictionaries require separate handling)

*/

JSON.Command.prototype.processKeys = function(parsedObject) {
  if (this.keys.length) {
    var hsh = {}, cols = [], evalStr = null, instStr = null, simpleKey = null;
    function instantiateArrayAndPush(key) {
      simpleKey = key.split("[").shift();
      // instantiate array in new object if not exists
      instStr = "if (!hsh." + simpleKey + ") { hsh." + simpleKey + " = []; }";
      eval(instStr);
      // push new value into array (command line order matters)
      evalStr = "hsh." + simpleKey + ".push(" + "parsedObject." + key + ")";
      eval(evalStr);
      cols.push(eval("parsedObject." + key));
    }
    for (var i = 0; (i < this.keys.length); i++) {
      try {
        if ((this.keys[i].indexOf(".") > -1) || (this.keys[i].indexOf("[") > -1)) {
          // create any keys that don't exist in the object chain
          if (this.keys[i].indexOf(".") > -1) {
            var s = this.keys[i].split(".");
            for (var j = 1; (j < s.length); j++) {
              // create necessary keys
              evalStr = "hsh." + s.slice(0,j).join(".");
              if (!eval(evalStr)) {
                eval("hsh." + s.slice(0,j).join(".") + " = {};");
              }
            }
            if (this.keys[i].indexOf("[") > -1) {
              instantiateArrayAndPush(this.keys[i]);
            }
            else {
              evalStr = "hsh." + s.join(".") + " = " + "parsedObject." + s.join(".");
              eval(evalStr);
              cols.push(eval("parsedObject." + s.join(".")));
            }
          }
          else if (this.keys[i].indexOf("[") > -1) {
            instantiateArrayAndPush(this.keys[i]);
          }
        }
        else {
          // no expansion
          hsh[this.keys[i]] = parsedObject[this.keys[i]];
          cols.push(parsedObject[this.keys[i]]);
        }
      }
      catch(ex) {
        this.debug("Failed to read property " + this.keys[i] + " from object: " + JSON.stringify(parsedObject));
        ex.message = "Failed to read property";
        throw ex;
      }
    }
    return this.columnOutput ? cols : hsh;
  }
  else {
    return parsedObject;
  }
};

/*
  Process input objects.
*/

JSON.Command.prototype.processObjects = function(objects) {

  var rawObject = null, parsedObject = null;

  try {
    if (this.useObject && objects && objects.length > 0 && typeof(objects[0]) == 'string' && objects[0].trim().length > 0) {
      objects = vm.runInNewContext(this.useObject, JSON.parse(objects[0]));
    }
  }
  catch(ex) {
    this.printex(ex);
  }

  if (this.inputIsArray && objects && objects.length > 0 && typeof(objects[0]) == 'string') {
    var trimmed = objects[0].trim();
    if (trimmed.length > 0 && trimmed[0] == '[') {
      objects = JSON.parse(objects[0]);
    }
  }

  try {
    for (var i = 0; (i < (objects.length)); i++) {
      // if there's no object, there's nothing to do
      //  (null object is not the same as string null)
      if ((objects[i] == null) || (objects[i] == undefined)) { continue; }

      try {
        if (typeof(objects[i]) == "string") {
          rawObject = objects[i];
          if (rawObject.trim().length == 0) continue;
          parsedObject = JSON.parse(rawObject);
        }
        else {
          rawObject = JSON.stringify(objects[i]);
          parsedObject = objects[i];
        }
      } catch(ex) {
        // discard bad records
        this.printex(ex);
        continue;
      }

      // create any requested keys that are missing
      this.createRequestedKeys(parsedObject);

      // process key transformations  on this parsedObject
      this.processKeyTransforms(parsedObject);

      // process executables on this parsedObject
      this.processExecutables(parsedObject);

      // continue if any conditionals fail on this parsedObject
      if (!this.checkConditionals(parsedObject)) {
        continue;
      }

      try {
        // process requested keys on the parsed object
        outputObject = this.processKeys(parsedObject);
      }
      catch(ex) { continue; }

      // finally, print output
      if (this.columnOutput) {
        process.stdout.write(outputObject.join("\t") + "\n");
      }
      else {
        process.stdout.write(this.stringify(outputObject) + "\n");
      }
    }
  }
  catch(ex) {
    this.printex(ex);
  }
};

/*
  Process input.
*/

JSON.Command.prototype.processInput = function() {
  if (this.files) {
    // TODO: implement file support?
  }
  else {
    this.stdin = process.openStdin();
    this.stdin.setEncoding("utf8");
    this.stdin.jsonC = this; // context for closure

    this.stdin.on("data", function(chunk) {
      this.jsonC.buffer += chunk;
      if (this.jsonC.inputIsArray) return;

      var objects = null;
      if (this.jsonC.buffer.match(/\n/g) ||
          this.jsonC.buffer.match(/\r\n/g) ||
          this.jsonC.buffer.match(/\0/g) ||
          this.jsonC.buffer.match("}{")) {
        if (this.jsonC.buffer.match(/\n/g)) {
          objects = this.jsonC.buffer.split("\n");
        }
        if (this.jsonC.buffer.match(/\r\n/g)) {
          objects = this.jsonC.buffer.split("\r\n");
        }
        if (this.jsonC.buffer.match(/\0/g)) {
          objects = this.jsonC.buffer.split("\0");
        }
        if (this.jsonC.buffer.match("}{")) {
          objects = this.jsonC.buffer.split("}{").join("}\n{").split("\n");
        }

        this.jsonC.buffer = objects.pop();

        if (this.jsonC.headerPassthrough) {
          for (var i = 0, l = objects.length; i < l; i ++) {
            process.stdout.write(objects[i]+"\r\n");
            if (objects[i] === "") {
              this.jsonC.headerPassthrough = false;
              break;
            }
          }
          objects.splice(0, i);
        }

        if (objects.length) this.jsonC.processObjects(objects);
      }
    });

    var handleEPIPE = function(e) {
      if (e.code !== "EPIPE") {
        process.emit("error", e);
      }
    };
    process.stdout.on("error", handleEPIPE);

    this.stdin.on("end", function() {
      this.jsonC.processObjects([this.jsonC.buffer, null]);
      process.stdout.removeListener("error", handleEPIPE);
    });
  }
};

<|endoftext|>


<commit_before>var sys = require("sys"),
    Script  = process.binding("evals").Script;

/*
 JSON Command class
*/

JSON.Command = function(args) {
  this.args = null;

  this.debugOn = false;
  this.fileNames = [];
  this.files = null;
  this.keys = [];
  this.transformedKeys = [];
  this.uglyOutput = false;
  this.columnOutput = false;
  this.useObject = null;

  this.conditionals = [];
  this.executables = [];

  this.stdin = null;
  this.buffer = "";
  this.scriptObject;

  if (args) { this.processArgs(args); }
};

JSON.Command.prototype.printhelp = function() {
  sys.puts("usage: stdout_generator | json [options] [fields]");
  sys.puts("");
  sys.puts("json processes standard input and parses json objects. json currently handles a");
  sys.puts("few different standard input formats and provides a number of options tailored");
  sys.puts("toward inspecting and transforming the parsed json objects.");
  sys.puts("");
  sys.puts("options:\n");
  sys.puts("  -h                    print this help info and exit\n");
  sys.puts("  -u                    print ugly json output, each object on a single line\n");
  sys.puts("  -d                    print debugging output including exception messages\n");
  sys.puts("  -o object.path        specify the path to an array to be iterated on\n");
  sys.puts("  new.key=old_key       move old_key to new.key in output object\n");
  sys.puts("  -c \"js conditional\"   js conditional to be run in the context of each object");
  sys.puts("                         that determines whether an object is printed\n");
  sys.puts("  -C                    print the output fields as tab delimited columns in");
  sys.puts("                         the order specified by fields\n");
  sys.puts("  -e \"js expression\"    execute arbitrary js in the context of each object.\n");
  sys.puts("examples:\n");
  sys.puts("  curl http://search.twitter.com/search.json?q=node.js 2> /dev/null |");
  sys.puts("   json -o results\n");
  sys.puts("  curl http://search.twitter.com/search.json?q=node.js 2> /dev/null |");
  sys.puts("   json -o results new_id=id\n");
  sys.puts("  curl http://search.twitter.com/search.json?q=node.js 2> /dev/null |");
  sys.puts("   json -o results -C from_user from_user_id text\n");
  sys.puts("more help:\n");
  sys.puts("  use \"man json-command\" or visit http://github.com/zpoley/json-command\n");
  process.exit();
};

JSON.Command.prototype.stringify = function(obj) {
  return(this.uglyOutput ? JSON.stringify(obj) : JSON.stringify(obj, null, 2));
};

JSON.Command.prototype.debug = function(msg) {
  if (this.debugOn) { sys.puts(msg); }
};

JSON.Command.prototype.printex = function(ex) {
  this.debug("ex: " + JSON.stringify(ex, null, 2));
};

/*
  Process Command line arguments to JSON Command
*/

JSON.Command.prototype.processArgs = function processArgs(args) {

  // copy argv to chop it up
  var a = args.slice(0);

  while (a.length > 0) {
    var arg = a.shift();
    switch(arg) {
      case "-h": // display help and exit
        this.printhelp();
        break;
      case "-f": // file
        this.fileNames.push(a.shift());
        break;
      case "-d": // debug
        this.debugOn = true;
        break;
      case "-u": // pretty printing (turn off)
        this.uglyOutput = true;
        break;
      case "-c": // conditional
        this.conditionals.push(a.shift());
        break;
      case "-C": // column output
        this.columnOutput = true;
        break;
      case "-e": // executable (transform data)
        this.executables.push(a.shift());
        break;
      case "-o": // use object
        this.useObject = a.shift();
        break;
      default: // json object keys
        if (arg.match("=")) {
          var kk = arg.split("=");
          this.keys.push(kk[0]);
          this.transformedKeys.push({
            newKey : kk[0],
            oldKey : kk[1]
          });
        }
        else {
          this.keys.push(arg);
        }
        break;
    }
  }
};

/*
  Create any reuested keys that don't already exist. Init values with null.
   The default value could be an option.
*/

JSON.Command.prototype.createRequestedKeys = function(parsedObject) {
  // instantiate any requested keys
  for(var j = 0; (j < this.keys.length); j++) {
    if (!parsedObject[this.keys[j]]) {
      parsedObject[this.keys[j]] = null;
    }
  }
};

/*
  Check conditionals against object.
*/

JSON.Command.prototype.checkConditionals = function(parsedObject) {
  if (this.conditionals.length) {
    try {
      var conditionsFailed = false;
      for(var i = 0; (i < this.conditionals.length); i++) {
        this.scriptObject = new Script("(" + this.conditionals[i] + ")");
        if (!this.scriptObject.runInNewContext(parsedObject)) {
          conditionsFailed = true;
        }
      }
      // if any conditions failed return false
      if (conditionsFailed) { return false; }
    }
    catch(ex) {
      // if any conditional fails, return false,
      //  the conditional may access something not present, etc..
      this.printex(ex);
      return false;
    }
  }
  // all conditionals passed
  return true;
};

/*
  Process key transforms against object.
*/

JSON.Command.prototype.processKeyTransforms = function(parsedObject) {
  if (this.transformedKeys.length) {
    for(var i = 0; (i < this.transformedKeys.length); i++) {
      try {
        this.scriptObject = new Script(this.transformedKeys[i].newKey +
          " = " + this.transformedKeys[i].oldKey );
        this.scriptObject.runInNewContext(parsedObject);
      }
      catch (ex) {
        this.printex(ex);
      }
    }
  }
};

/*
  Process executables against object.
*/

JSON.Command.prototype.processExecutables = function(parsedObject) {
  if (this.executables.length) {
    for(var i = 0; (i < this.executables.length); i++) {
      try {
        // create a new script object for the executable
        this.scriptObject = new Script(this.executables[i]);
        // run the new script in the context of the parsed object
        this.scriptObject.runInNewContext(parsedObject);
      }
      catch (ex) {
        // stop catstrophic failure if any executable fails.
        //  TODO: this may not be the desired behavior.
        this.printex(ex);
      }
    }
  }
};

/*
  Process requested keys against parsedObject.
   This may be one of the most complicated parts of this code, and it may
   very well not need to be. If you have a better idea of how to do this
   please let me know: zpoley@gmail.com.

   What's happening here is:
    1. Create a new object to replace the old one since we don't want all
     the keys from the old object.
    2. Create each object necessary in the chain in order for the resulting
     object to retain the same structure of the parsedObject.
    3. Assign each requested key value from the parsedObject into the new
     object.

*/

JSON.Command.prototype.processKeys = function(parsedObject) {
  if (this.keys.length) {
    var hsh = {}, cols = [];
    for (var i = 0; (i < this.keys.length); i++) {
      try {
        if (this.keys[i].match(".")) {
          // create any keys that don't exist in the object chain
          var s = this.keys[i].split(".");
          for (var j = 1; (j < s.length); j++) {
            // create necessary keys
            var evalStr = "hsh." + s.slice(0,j).join(".");
            if (!eval(evalStr)) {
              eval("hsh." + s.slice(0,j).join(".") + " = {};");
            }
          }
          var evalStr = "hsh." + s.join(".") + " = " + "parsedObject." + s.join(".");
          eval(evalStr);
          cols.push(eval("parsedObject." + s.join(".")));
        }
        else {
          hsh[keys[i]] = parsedObject[keys[i]];
          cols.push(parsedObject[keys[i]]);
        }
      }
      catch(ex) {
        this.debug("Failed to read property " + this.keys[i] + " from object: " + JSON.stringify(parsedObject));
        ex.message = "Failed to read property";
        throw ex;
      }
    }
    return this.columnOutput ? cols : hsh;
  }
  else {
    return parsedObject;
  }
};

/*
  Process input objects.
*/

JSON.Command.prototype.processObjects = function(objects) {

  var rawObject = null, parsedObject = null;

  try {
    if (this.useObject && objects && objects.length > 0) {
      this.scriptObject = new Script(this.useObject);
      objects = this.scriptObject.runInNewContext(JSON.parse(objects[0]));
    }
  }
  catch(ex) {
    this.printex(ex);
  }

  try {
    for (var i = 0; (i < (objects.length)); i++) {
      // if there's no object, there's nothing to do
      //  (null object is not the same as string null)
      if ((objects[i] == null) || (objects[i] == undefined)) { continue; }

      try {
        if (typeof(objects[i]) == "string") {
          rawObject = objects[i];
          parsedObject = JSON.parse(rawObject);
        }
        else {
          rawObject = JSON.stringify(objects[i]);
          parsedObject = objects[i];
        }
      } catch(ex) {
        // discard bad records
        this.printex(ex);
        continue;
      }

      // create any requested keys that are missing
      this.createRequestedKeys(parsedObject);

      // process key transformations  on this parsedObject
      this.processKeyTransforms(parsedObject);

      // process executables on this parsedObject
      this.processExecutables(parsedObject);

      // continue if any conditionals fail on this parsedObject
      if (!this.checkConditionals(parsedObject)) {
        continue;
      }

      try {
        // process requested keys on the parsed object
        outputObject = this.processKeys(parsedObject);
      }
      catch(ex) { continue; }

      // finally, print output
      if (this.columnOutput) {
        process.stdout.write(outputObject.join("\t") + "\n");
      }
      else {
        process.stdout.write(this.stringify(outputObject) + "\n");
      }
    }
  }
  catch(ex) {
    this.printex(ex);
  }
};

/*
  Process input.
*/

JSON.Command.prototype.processInput = function() {
  if (this.files) {
    // TODO: implement file support?
  }
  else {
    this.stdin = process.openStdin();
    this.stdin.setEncoding("utf8");
    this.stdin.jsonC = this; // context for closure

    this.stdin.on("data", function(chunk) {
      this.jsonC.buffer += chunk;
      var objects = null;
      if (this.jsonC.buffer.match(/\n/g) ||
          this.jsonC.buffer.match(/\r\n/g) ||
          this.jsonC.buffer.match(/\0/g) ||
          this.jsonC.buffer.match("}{")) {
        if (this.jsonC.buffer.match(/\n/g)) {
          objects = this.jsonC.buffer.split("\n");
        }
        if (this.jsonC.buffer.match(/\r\n/g)) {
          objects = this.jsonC.buffer.split("\r\n");
        }
        if (this.jsonC.buffer.match(/\0/g)) {
          objects = this.jsonC.buffer.split("\0");
        }
        if (this.jsonC.buffer.match("}{")) {
          objects = this.jsonC.buffer.split("}{").join("}\n{").split("\n");
        }
        this.jsonC.buffer = objects.pop();
        this.jsonC.processObjects(objects);
      }
    });

    this.stdin.on("end", function() {
      this.jsonC.processObjects([this.jsonC.buffer, null]);
    });
  }
};

<commit_msg>Add -i flag to do sys.inspect output.<commit_after>var sys = require("sys"),
    Script  = process.binding("evals").Script;

/*
 JSON Command class
*/

JSON.Command = function(args) {
  this.args = null;

  this.debugOn = false;
  this.fileNames = [];
  this.files = null;
  this.keys = [];
  this.transformedKeys = [];
  this.uglyOutput = false;
  this.inspectOutput = false;
  this.columnOutput = false;
  this.useObject = null;

  this.conditionals = [];
  this.executables = [];

  this.stdin = null;
  this.buffer = "";
  this.scriptObject;

  if (args) { this.processArgs(args); }
};

JSON.Command.prototype.printhelp = function() {
  sys.puts("usage: stdout_generator | json [options] [fields]");
  sys.puts("");
  sys.puts("json processes standard input and parses json objects. json currently handles a");
  sys.puts("few different standard input formats and provides a number of options tailored");
  sys.puts("toward inspecting and transforming the parsed json objects.");
  sys.puts("");
  sys.puts("options:\n");
  sys.puts("  -h                    print this help info and exit\n");
  sys.puts("  -u                    print ugly json output, each object on a single line\n");
  sys.puts("  -d                    print debugging output including exception messages\n");
  sys.puts("  -o object.path        specify the path to an array to be iterated on\n");
  sys.puts("  new.key=old_key       move old_key to new.key in output object\n");
  sys.puts("  -c \"js conditional\"   js conditional to be run in the context of each object");
  sys.puts("                         that determines whether an object is printed\n");
  sys.puts("  -C                    print the output fields as tab delimited columns in");
  sys.puts("                         the order specified by fields\n");
  sys.puts("  -e \"js expression\"    execute arbitrary js in the context of each object.\n");
  sys.puts("  -i                    use node's sys.inspect instead of JSON.stringify\n");
  sys.puts("examples:\n");
  sys.puts("  curl http://search.twitter.com/search.json?q=node.js 2> /dev/null |");
  sys.puts("   json -o results\n");
  sys.puts("  curl http://search.twitter.com/search.json?q=node.js 2> /dev/null |");
  sys.puts("   json -o results new_id=id\n");
  sys.puts("  curl http://search.twitter.com/search.json?q=node.js 2> /dev/null |");
  sys.puts("   json -o results -C from_user from_user_id text\n");
  sys.puts("more help:\n");
  sys.puts("  use \"man json-command\" or visit http://github.com/zpoley/json-command\n");
  process.exit();
};

JSON.Command.prototype.stringify = function(obj) {
  return( this.inspectOutput ? sys.inspect(obj, false, Infinity, true)
        : this.uglyOutput ? JSON.stringify(obj)
        : JSON.stringify(obj, null, 2) );
};

JSON.Command.prototype.debug = function(msg) {
  if (this.debugOn) { sys.puts(msg); }
};

JSON.Command.prototype.printex = function(ex) {
  this.debug("ex: " + JSON.stringify(ex, null, 2));
};

/*
  Process Command line arguments to JSON Command
*/

JSON.Command.prototype.processArgs = function processArgs(args) {

  // copy argv to chop it up
  var a = args.slice(0);

  while (a.length > 0) {
    var arg = a.shift();
    switch(arg) {
      case "-h": // display help and exit
        this.printhelp();
        break;
      case "-f": // file
        this.fileNames.push(a.shift());
        break;
      case "-d": // debug
        this.debugOn = true;
        break;
      case "-u": // pretty printing (turn off)
        this.uglyOutput = true;
        break;
      case "-c": // conditional
        this.conditionals.push(a.shift());
        break;
      case "-C": // column output
        this.columnOutput = true;
        break;
      case "-e": // executable (transform data)
        this.executables.push(a.shift());
        break;
      case "-o": // use object
        this.useObject = a.shift();
        break;
      case "-i": // use sys.inspect
        this.inspectOutput = true;
        break;
      default: // json object keys
        if (arg.match("=")) {
          var kk = arg.split("=");
          this.keys.push(kk[0]);
          this.transformedKeys.push({
            newKey : kk[0],
            oldKey : kk[1]
          });
        }
        else {
          this.keys.push(arg);
        }
        break;
    }
  }
};

/*
  Create any reuested keys that don't already exist. Init values with null.
   The default value could be an option.
*/

JSON.Command.prototype.createRequestedKeys = function(parsedObject) {
  // instantiate any requested keys
  for(var j = 0; (j < this.keys.length); j++) {
    if (!parsedObject[this.keys[j]]) {
      parsedObject[this.keys[j]] = null;
    }
  }
};

/*
  Check conditionals against object.
*/

JSON.Command.prototype.checkConditionals = function(parsedObject) {
  if (this.conditionals.length) {
    try {
      var conditionsFailed = false;
      for(var i = 0; (i < this.conditionals.length); i++) {
        this.scriptObject = new Script("(" + this.conditionals[i] + ")");
        if (!this.scriptObject.runInNewContext(parsedObject)) {
          conditionsFailed = true;
        }
      }
      // if any conditions failed return false
      if (conditionsFailed) { return false; }
    }
    catch(ex) {
      // if any conditional fails, return false,
      //  the conditional may access something not present, etc..
      this.printex(ex);
      return false;
    }
  }
  // all conditionals passed
  return true;
};

/*
  Process key transforms against object.
*/

JSON.Command.prototype.processKeyTransforms = function(parsedObject) {
  if (this.transformedKeys.length) {
    for(var i = 0; (i < this.transformedKeys.length); i++) {
      try {
        this.scriptObject = new Script(this.transformedKeys[i].newKey +
          " = " + this.transformedKeys[i].oldKey );
        this.scriptObject.runInNewContext(parsedObject);
      }
      catch (ex) {
        this.printex(ex);
      }
    }
  }
};

/*
  Process executables against object.
*/

JSON.Command.prototype.processExecutables = function(parsedObject) {
  if (this.executables.length) {
    for(var i = 0; (i < this.executables.length); i++) {
      try {
        // create a new script object for the executable
        this.scriptObject = new Script(this.executables[i]);
        // run the new script in the context of the parsed object
        this.scriptObject.runInNewContext(parsedObject);
      }
      catch (ex) {
        // stop catstrophic failure if any executable fails.
        //  TODO: this may not be the desired behavior.
        this.printex(ex);
      }
    }
  }
};

/*
  Process requested keys against parsedObject.
   This may be one of the most complicated parts of this code, and it may
   very well not need to be. If you have a better idea of how to do this
   please let me know: zpoley@gmail.com.

   What's happening here is:
    1. Create a new object to replace the old one since we don't want all
     the keys from the old object.
    2. Create each object necessary in the chain in order for the resulting
     object to retain the same structure of the parsedObject.
    3. Assign each requested key value from the parsedObject into the new
     object.

*/

JSON.Command.prototype.processKeys = function(parsedObject) {
  if (this.keys.length) {
    var hsh = {}, cols = [];
    for (var i = 0; (i < this.keys.length); i++) {
      try {
        if (this.keys[i].match(".")) {
          // create any keys that don't exist in the object chain
          var s = this.keys[i].split(".");
          for (var j = 1; (j < s.length); j++) {
            // create necessary keys
            var evalStr = "hsh." + s.slice(0,j).join(".");
            if (!eval(evalStr)) {
              eval("hsh." + s.slice(0,j).join(".") + " = {};");
            }
          }
          var evalStr = "hsh." + s.join(".") + " = " + "parsedObject." + s.join(".");
          eval(evalStr);
          cols.push(eval("parsedObject." + s.join(".")));
        }
        else {
          hsh[keys[i]] = parsedObject[keys[i]];
          cols.push(parsedObject[keys[i]]);
        }
      }
      catch(ex) {
        this.debug("Failed to read property " + this.keys[i] + " from object: " + JSON.stringify(parsedObject));
        ex.message = "Failed to read property";
        throw ex;
      }
    }
    return this.columnOutput ? cols : hsh;
  }
  else {
    return parsedObject;
  }
};

/*
  Process input objects.
*/

JSON.Command.prototype.processObjects = function(objects) {

  var rawObject = null, parsedObject = null;

  try {
    if (this.useObject && objects && objects.length > 0) {
      this.scriptObject = new Script(this.useObject);
      objects = this.scriptObject.runInNewContext(JSON.parse(objects[0]));
    }
  }
  catch(ex) {
    this.printex(ex);
  }

  try {
    for (var i = 0; (i < (objects.length)); i++) {
      // if there's no object, there's nothing to do
      //  (null object is not the same as string null)
      if ((objects[i] == null) || (objects[i] == undefined)) { continue; }

      try {
        if (typeof(objects[i]) == "string") {
          rawObject = objects[i];
          parsedObject = JSON.parse(rawObject);
        }
        else {
          rawObject = JSON.stringify(objects[i]);
          parsedObject = objects[i];
        }
      } catch(ex) {
        // discard bad records
        this.printex(ex);
        continue;
      }

      // create any requested keys that are missing
      this.createRequestedKeys(parsedObject);

      // process key transformations  on this parsedObject
      this.processKeyTransforms(parsedObject);

      // process executables on this parsedObject
      this.processExecutables(parsedObject);

      // continue if any conditionals fail on this parsedObject
      if (!this.checkConditionals(parsedObject)) {
        continue;
      }

      try {
        // process requested keys on the parsed object
        outputObject = this.processKeys(parsedObject);
      }
      catch(ex) { continue; }

      // finally, print output
      if (this.columnOutput) {
        process.stdout.write(outputObject.join("\t") + "\n");
      }
      else {
        process.stdout.write(this.stringify(outputObject) + "\n");
      }
    }
  }
  catch(ex) {
    this.printex(ex);
  }
};

/*
  Process input.
*/

JSON.Command.prototype.processInput = function() {
  if (this.files) {
    // TODO: implement file support?
  }
  else {
    this.stdin = process.openStdin();
    this.stdin.setEncoding("utf8");
    this.stdin.jsonC = this; // context for closure

    this.stdin.on("data", function(chunk) {
      this.jsonC.buffer += chunk;
      var objects = null;
      if (this.jsonC.buffer.match(/\n/g) ||
          this.jsonC.buffer.match(/\r\n/g) ||
          this.jsonC.buffer.match(/\0/g) ||
          this.jsonC.buffer.match("}{")) {
        if (this.jsonC.buffer.match(/\n/g)) {
          objects = this.jsonC.buffer.split("\n");
        }
        if (this.jsonC.buffer.match(/\r\n/g)) {
          objects = this.jsonC.buffer.split("\r\n");
        }
        if (this.jsonC.buffer.match(/\0/g)) {
          objects = this.jsonC.buffer.split("\0");
        }
        if (this.jsonC.buffer.match("}{")) {
          objects = this.jsonC.buffer.split("}{").join("}\n{").split("\n");
        }
        this.jsonC.buffer = objects.pop();
        this.jsonC.processObjects(objects);
      }
    });

    this.stdin.on("end", function() {
      this.jsonC.processObjects([this.jsonC.buffer, null]);
    });
  }
};

<|endoftext|>


<commit_before>if (typeof define !== 'function') {
  var define = require('amdefine')(module);
}
define(function (require, exports, module) {
  'use strict';

  var EventEmitter = require('./events').EventEmitter;

  var nestedProps = ['style', 'dataset'];

  function Parser() {

    /* Public */

    this.parse = parse;
    this.parseString = parseComplexString;
    this.addEventListener = addEventListener;
    this.removeEventListener = removeEventListener;

    var MAX_PLACEABLES = 100;

    var _emitter = new EventEmitter();

    var patterns = {
      comment: /^\s*#|^\s*$/,
      entity: /^([^=\s]*)\s*=\s*(.+)$/,
      multiline: /[^\\]\\$/,
      plural: /\{\[\s*plural\(([^\)]+)\)\s*\]\}/,
      unicode: /\\u([0-9a-fA-F]{1,4})/g
    };

    function parse(source) {
      var ast = {};

      var entries = source.split(/[\r\n]+/);
      for (var i = 0; i < entries.length; i++) {
        var line = entries[i];

        if (patterns['comment'].test(line)) {
          continue;
        }

        while (patterns['multiline'].test(line) && i < entries.length) {
          line = line.slice(0, line.length - 1) + entries[++i].trim();
        }

        var entity_match = line.match(patterns['entity']);
        if (entity_match && entity_match.length == 3) {

          parseEntity(entity_match, ast);
        }
      }
      return {
        type: 'WebL10n',
        body: ast
      };
    }

    function setEntityValue(id, attr, key, value, ast) {
      var entry = ast[id];
      var item;

      if (!entry) {
        entry = {};
        ast[id] = entry;
      }

      if (attr) {
        if (!entry.attrs) {
          entry.attrs = {};
        }
        if (!entry.attrs[attr]) {
          entry.attrs[attr] = {};
        }
        item = entry.attrs[attr];
      } else {
        item = entry;
      }

      if (!key) {
        item.value = value;
      } else {
        if (item.value.type !== 'Hash') {
          var match = item.value.content.match(patterns['plural']);
          if (!match) {
            throw new ParserError('Expected macro call');
          }
          item.index = [
          {
            type: 'CallExpression',
              callee: {
                type: 'Identifier',
                name: 'plural'
              },
              arguments: [{
                type: 'Identifier',
                name: match[1].trim()
              }]
          }
          ];
          item.value = {
            type: 'Hash',
            content: []
          };
        }
        item.value.content.push({
          type: 'HashItem',
          key: key,
          value: value
        });
        if (key === 'other') {
          item.value.content[item.value.content.length - 1].default = true;
        }
      }
    }

    function parseEntity(match, ast) {
      var name, key, attr, pos;
      var value = parseString(match[2]);

      pos = match[1].indexOf('[');
      if (pos !== -1) {
        name = match[1].substr(0, pos);
        key = match[1].substring(pos + 1, match[1].length - 1);
      } else {
        name = match[1];
        key = null;
      }

      var nameElements = name.split('.');

      if (nameElements.length > 1) {
        var attrElements = [];
        attrElements.push(nameElements.pop());
        if (nameElements.length > 1) {
          // special quirk to comply with webl10n's behavior
          if (nestedProps.indexOf(
                nameElements[nameElements.length - 1]) !== -1) {
            attrElements.push(nameElements.pop());
          }
        }
        name = nameElements.join('.');
        attr = attrElements.reverse().join('.');
      } else {
        attr = null;
      }

      setEntityValue(name, attr, key, value, ast);
    }

    function unescapeUnicode(str) {
      if (str.lastIndexOf('\\') !== -1) {
        str = str.replace(/\\\\/g, '\\')
                 .replace(/\\n/g, '\n')
                 .replace(/\\r/g, '\r')
                 .replace(/\\t/g, '\t')
                 .replace(/\\b/g, '\b')
                 .replace(/\\f/g, '\f')
                 .replace(/\\{/g, '{')
                 .replace(/\\}/g, '}')
                 .replace(/\\"/g, '"')
                 .replace(/\\'/g, "'");
      }
      return str.replace(patterns.unicode, function(match, token) {
        return unescape('%u' + '0000'.slice(token.length) + token);
      });
    }

    function parseString(str) {
      if (str.indexOf('{{') === -1) {
        return {
          content: unescapeUnicode(str)
        };
      } else {
        return {
          content: str,
          maybeComplex: true
        };
      }
    }

    function parseComplexString(str) {

      var placeables = 0;

      if (str.indexOf('{{') === -1) {
        return {
          content: unescapeUnicode(str)
        };
      }
      var chunks = str.split('{{');
      var body = [];

      chunks.forEach(function (chunk, num) {
        if (num === 0) {
          if (chunk.length > 0) {
            body.push({
              content: unescapeUnicode(chunk)
            });
          }
          return;
        }
        var parts = chunk.split('}}');
        if (parts.length < 2) {
          throw new ParserError('Expected "}}"');
        }
        body.push({
          type: 'Identifier',
          name: parts[0].trim()
        });
        placeables++;
        if (placeables > MAX_PLACEABLES) {
          throw new ParserError('Too many placeables, maximum allowed is ' +
            MAX_PLACEABLES);
        }
        if (parts[1].length > 0) {
          body.push({
            content: unescapeUnicode(parts[1])
          });
        }
      });
      return {
        type: 'ComplexString',
        content: body
      };
    }

    function addEventListener(type, listener) {
      if (!_emitter) {
        throw Error('Emitter not available');
      }
      return _emitter.addEventListener(type, listener);
    }

    function removeEventListener(type, listener) {
      if (!_emitter) {
        throw Error('Emitter not available');
      }
      return _emitter.removeEventListener(type, listener);
    }
  }

  /* ParserError class */

  Parser.Error = ParserError;

  function ParserError(message) {
    this.name = 'ParserError';
    this.message = message;
  }
  ParserError.prototype = Object.create(Error.prototype);
  ParserError.prototype.constructor = ParserError;

  exports.Parser = Parser;

});
<commit_msg>require entity to have an id<commit_after>if (typeof define !== 'function') {
  var define = require('amdefine')(module);
}
define(function (require, exports, module) {
  'use strict';

  var EventEmitter = require('./events').EventEmitter;

  var nestedProps = ['style', 'dataset'];

  function Parser() {

    /* Public */

    this.parse = parse;
    this.parseString = parseComplexString;
    this.addEventListener = addEventListener;
    this.removeEventListener = removeEventListener;

    var MAX_PLACEABLES = 100;

    var _emitter = new EventEmitter();

    var patterns = {
      comment: /^\s*#|^\s*$/,
      entity: /^([^=\s]+)\s*=\s*(.+)$/,
      multiline: /[^\\]\\$/,
      plural: /\{\[\s*plural\(([^\)]+)\)\s*\]\}/,
      unicode: /\\u([0-9a-fA-F]{1,4})/g
    };

    function parse(source) {
      var ast = {};

      var entries = source.split(/[\r\n]+/);
      for (var i = 0; i < entries.length; i++) {
        var line = entries[i];

        if (patterns['comment'].test(line)) {
          continue;
        }

        while (patterns['multiline'].test(line) && i < entries.length) {
          line = line.slice(0, line.length - 1) + entries[++i].trim();
        }

        var entity_match = line.match(patterns['entity']);
        if (entity_match && entity_match.length == 3) {

          parseEntity(entity_match, ast);
        }
      }
      return {
        type: 'WebL10n',
        body: ast
      };
    }

    function setEntityValue(id, attr, key, value, ast) {
      var entry = ast[id];
      var item;

      if (!entry) {
        entry = {};
        ast[id] = entry;
      }

      if (attr) {
        if (!entry.attrs) {
          entry.attrs = {};
        }
        if (!entry.attrs[attr]) {
          entry.attrs[attr] = {};
        }
        item = entry.attrs[attr];
      } else {
        item = entry;
      }

      if (!key) {
        item.value = value;
      } else {
        if (item.value.type !== 'Hash') {
          var match = item.value.content.match(patterns['plural']);
          if (!match) {
            throw new ParserError('Expected macro call');
          }
          item.index = [
          {
            type: 'CallExpression',
              callee: {
                type: 'Identifier',
                name: 'plural'
              },
              arguments: [{
                type: 'Identifier',
                name: match[1].trim()
              }]
          }
          ];
          item.value = {
            type: 'Hash',
            content: []
          };
        }
        item.value.content.push({
          type: 'HashItem',
          key: key,
          value: value
        });
        if (key === 'other') {
          item.value.content[item.value.content.length - 1].default = true;
        }
      }
    }

    function parseEntity(match, ast) {
      var name, key, attr, pos;
      var value = parseString(match[2]);

      pos = match[1].indexOf('[');
      if (pos !== -1) {
        name = match[1].substr(0, pos);
        key = match[1].substring(pos + 1, match[1].length - 1);
      } else {
        name = match[1];
        key = null;
      }

      var nameElements = name.split('.');

      if (nameElements.length > 1) {
        var attrElements = [];
        attrElements.push(nameElements.pop());
        if (nameElements.length > 1) {
          // special quirk to comply with webl10n's behavior
          if (nestedProps.indexOf(
                nameElements[nameElements.length - 1]) !== -1) {
            attrElements.push(nameElements.pop());
          }
        }
        name = nameElements.join('.');
        attr = attrElements.reverse().join('.');
      } else {
        attr = null;
      }

      setEntityValue(name, attr, key, value, ast);
    }

    function unescapeUnicode(str) {
      if (str.lastIndexOf('\\') !== -1) {
        str = str.replace(/\\\\/g, '\\')
                 .replace(/\\n/g, '\n')
                 .replace(/\\r/g, '\r')
                 .replace(/\\t/g, '\t')
                 .replace(/\\b/g, '\b')
                 .replace(/\\f/g, '\f')
                 .replace(/\\{/g, '{')
                 .replace(/\\}/g, '}')
                 .replace(/\\"/g, '"')
                 .replace(/\\'/g, "'");
      }
      return str.replace(patterns.unicode, function(match, token) {
        return unescape('%u' + '0000'.slice(token.length) + token);
      });
    }

    function parseString(str) {
      if (str.indexOf('{{') === -1) {
        return {
          content: unescapeUnicode(str)
        };
      } else {
        return {
          content: str,
          maybeComplex: true
        };
      }
    }

    function parseComplexString(str) {

      var placeables = 0;

      if (str.indexOf('{{') === -1) {
        return {
          content: unescapeUnicode(str)
        };
      }
      var chunks = str.split('{{');
      var body = [];

      chunks.forEach(function (chunk, num) {
        if (num === 0) {
          if (chunk.length > 0) {
            body.push({
              content: unescapeUnicode(chunk)
            });
          }
          return;
        }
        var parts = chunk.split('}}');
        if (parts.length < 2) {
          throw new ParserError('Expected "}}"');
        }
        body.push({
          type: 'Identifier',
          name: parts[0].trim()
        });
        placeables++;
        if (placeables > MAX_PLACEABLES) {
          throw new ParserError('Too many placeables, maximum allowed is ' +
            MAX_PLACEABLES);
        }
        if (parts[1].length > 0) {
          body.push({
            content: unescapeUnicode(parts[1])
          });
        }
      });
      return {
        type: 'ComplexString',
        content: body
      };
    }

    function addEventListener(type, listener) {
      if (!_emitter) {
        throw Error('Emitter not available');
      }
      return _emitter.addEventListener(type, listener);
    }

    function removeEventListener(type, listener) {
      if (!_emitter) {
        throw Error('Emitter not available');
      }
      return _emitter.removeEventListener(type, listener);
    }
  }

  /* ParserError class */

  Parser.Error = ParserError;

  function ParserError(message) {
    this.name = 'ParserError';
    this.message = message;
  }
  ParserError.prototype = Object.create(Error.prototype);
  ParserError.prototype.constructor = ParserError;

  exports.Parser = Parser;

});
<|endoftext|>