<commit_before>#!/usr/bin/env node

'use strict';

// if you want to use this command line tool you must 'npm install' to get 'optimist'
var argv = require('optimist').argv,
    path = require('path'),
    fs = require('fs'),
    Ginx = require(path.join(__dirname, '/../lib/ginx')),
    parser, input, output, format, usage, stats, storage,
    isFile, isDir, io_notice, verbose, type, writer, csvRow;

function usage(notice) {
    if (notice) console.log('\n' + notice);
    console.log(
        '\nThis tool is still under develpment(issues #1, #4, #8 and probably more) -'
        + '\nUsage: ginx --input [path] --output [path] -t [type] -j -g -v -c -n -f [format] -s [path]'
        + '\n\nthis tool will parse nginx log files and save it/them in either CSV or JSON format. I don\'t know how much that is useful,'
        + '\nAll of the CSV values are surrounded with double quotes, any inner commas are also surrounded by double quotes and, each original inner " is replaced by "" this was done following the CSV parsing conventions'
        + '\nPlus, there is a HUGE performance hit by writing another JSON file(s) at the same time, Don\'t output to JSON please ! '
        + '\nThe other problem with this (issue #1) is that, if you use JSON, is closing the JSON brackets at the end of each file, to complete the structure,'
        + '\nso if you parse a log file and it completes. Then sometime later you parse the same file again if there is more logs appended to it,'
        + '\nthe total resulted JSON structure will not be correct, syntax wise. However, if a crash or a kill occurs, '
        + '\nand the JSON is not closed yet, the resulted syntax will be fine once you resume the parsing'
        + '\nI will be implementing a tail -f like feature soon, it may make this tool more useful, stay tuned. Meanwhile, limit usage to 50 files max '
        + '\n[BROTIP] if I were you I would just use the module in a node.js program, get what you need out of each row, persist to a DB or something -- i don\'t even know why I wrote this tool'
        + '\nif you have a large log file to parse, you will end up with an even larger JSON file. \n'
        + '\n-i | --input          : input file OR directory to parse'
        + '\n-o | --output         : destination file OR directory of where you would like to program the save the JSON file(s)'
        + '\n-f | --format         : [OPTIONAL] the format of your nginx access_log, if different then default [IMPORTANT] you must escape double quotes, the format MUST be exactly the same, single quotes in nginx format are not supported yet. (issue #5)'
        + '\n-n | --nonpersistent  : [OPTIONAL] a boolean value, defaults to true if not passed, if passed the program will not keep track of the file\'s positions, aka cursors, so it can\'t pick it where it left off in case of a crash or kill'
        + '\n-j | --fields2objects : [OPTIONAL] a boolean value, defaults to false if not passed, if passed the program will try to parse each field to a corresponding object, such as Date, Integer and NULL if \'-\' - may hit performance bad, about 50% more time if large or too many files'
        + '\n-s | --storage        : [OPTIONAL] a custome storage file for the file\'s positions cursors, it uses ./tmp/storage.cursors by default, but it won\'t save anything if --nonpersistent is passed'
        + '\n-g | --original       : [OPTIONAL] a boolean value, defaults to false, if passed, it will augment each row JSON object with its original text, if you need it, use it, if you don\' leave it out, it will make your output a lot larger'
        + '\n-h | --help           : [OPTIONAL] displays this message'
        + '\n-v | --verbose        : [OPTIONAL] verbose, but this could get ugly, printing each row on the screen, huge performance hit, don\'t use it'
        + '\n-c | --clear          : [OPTIONAL] will clear the cursors storage first, use it with -s if you have a custom storage file, otherwise it clears the default' + '\n\n');
        + '\n-t | --type           : [OPTIONAL] followed by "json" or "csv", defaults to csv if not specified, specifies the structure of the output files'
};

// print error and throw it
function error(err) {
    return console.log("[GINX-ERROR]:" + err.message);
    throw err;
}

// check if parser has a cached reference for this file in the loaded cursors from storage file, that latter happens in the Ginx constructor
function isNewFile(rfile) {
    return parser.__mem.cursors[rfile] ? false : true;
}

// if -h or --help
if (argv.h || argv.help) {
    return usage();
}
input = argv.i || argv.input;
output = argv.o || argv.output;
io_notice = 'You must pass both input and output, if input is a file, output will be a file, same for directories (limit to <50 files please)'
            + 'if input is a directory, output will be a directory as well, and you can\'t use the same directory nor same file for both arguments';
if (!input || !output) {
    return usage(io_notice);
} else {
    output = path.resolve(output);
    input = path.resolve(input);
    // synchronous, but necessary before it can continue
    stats = fs.statSync(input);
    if ((!stats.isDirectory() && !stats.isFile()) || (output === input)) {
        return usage(io_notice);
    }
}

//defaults
format = argv.f || argv.format || null;
verbose = argv.v || argv.verbose || false;
storage = argv.s || argv.storage || path.join(__dirname + '/../tmp/stored.cursors');

type = argv.t || argv.type || 'csv';
if (type !== 'csv' && type !== 'json'){
    console.log("[GINX-WARN] type : " + type + " is not supported or unknown, falling back to csv");
    type = 'csv';
}

// MAKING SURE the storage file is there before we continue, bunch of Sync calls but needed
// this will need to be updated when fs.writeFile be able to create directories recursivly
if(!fs.existsSync(storage)){
    var storageDir = path.normalize(storage.substring(0, storage.lastIndexOf(path.sep)));
    if(!fs.existsSync(storageDir)){
        fs.mkdirSync(storageDir);
    }
    fs.writeFileSync(storage, "{}");
} else if (argv.c || argv.clear) {
    //clearing storage if requested before constructing Ginx.
    fs.writeFileSync(storage, "{}");
    console.log("[GINX-INFO] Emptied " + storage + " storage file");
}

//construct the Ginx parser
parser = new Ginx(format, {
    'persistent': argv.n || argv.nonpersistent ? false : true,
    'fieldsToObjects': argv.j || argv.fields2objects ? true : false,
    'storageFile': argv.s || argv.storage || null,
    'originalText': argv.g || argv.original ? true : false
});

if(type === 'csv'){
  csvRow = generateCsvRow(this, parser.attrs);
}

// creating a writer to handle the data buffering from the parser's readstreams
writer = {
    wstreams: {},
    append: function(data, wfile, rfile){
        if(data
            && this.wstreams[wfile]
            && this.wstreams[wfile].write(data) == false) {
                parser.pause(rfile);
        }
    },
    addStream: function(wfile, rfile, flag, start, callback){
        var wstream = fs.createWriteStream(wfile, {'flags': flag ? flag : 'r+', 'start': start ? start : 0, 'mode': '0666'});
        this.wstreams[wfile] = wstream;
        this.wstreams[wfile].on('drain', function(){
            parser.resume(rfile);
        });
        callback();
    }
}
// let the parsing begin !!!

// process directory parsing
if (stats.isDirectory()) {
    fs.mkdir(output, function (err) {
        if (err){console.log(output + " already exists")}
        fs.readdir(input, function (err, files) {
            if (err) error(err);
            files.forEach(function (file) {
                var wfile = path.join(output, file),
                    rfile = path.join(input, file),
                    rnew = isNewFile(input),
                    wnew = false;
                    fs.stat(wfile, function(err, stats){
                        if(err){
                            wnew = true;
                        }
                        if (wnew || rnew) {
                            writer.addStream(wfile, rfile, 'w+', 0, function(){
                                if (type === 'csv'){
                                    writer.append(parser.attrs.join(',')+'\r\n', wfile, rfile);
                                } else if (type === 'json'){
                                    writer.append("{[", wfile, rfile);
                                }
                            });
                        } else {
                            writer.addStream(wfile, rfile,'r+', stats.size, function(){});
                        }
                    });

            });
            processDirectory(input, output);
        });
    });
} else if (stats.isFile()) {
    var rnew = isNewFile(input),
        wnew = false;
    fs.stat(output, function(err, stats){
        if(err){
            wnew = true;
        }
        if (wnew || rnew) {
            writer.addStream(output, input, 'w+', 0, function(){
                if (type === 'csv'){
                    writer.append(parser.attrs.join(',')+'\r\n', output, input);
                } else if (type === 'json'){
                    writer.append("{[", output, input);
                }
                processFile(input, output);
            });
        } else {
            writer.addStream(output, input,'r+', stats.size, function(){
                processFile(input, output);
            });
        }
    });
}
// process file parsing to JSON output
function processFile(input, ouput) {
    parser.parseFile(input,

    function (err, row) {
        if (err) error(err);
        writer.append(formatRow(row), output, row.__file);
    },

    function (err, rfile) {
        if (err) error(err);
        if(type === 'json') { writer.append("]}", output, rfile); }
    });
}

// process directory parsing to JSON outputs
function processDirectory(input, output) {
    parser.parseDir(input,

    function (err, row) {
        if (err) error(err);
        writer.append(formatRow(row), path.join(output, row.__fname), row.__file);
    },

    function (err, rfile) {
        if (err) error(err);
        var wfile = path.join(output, rfile.substring(rfile.lastIndexOf(path.sep) + 1));
        if(type === 'json') writer.append("]}", wfile, rfile);
    },

    function (err, filesCount) {
        if (err) error(err);
        if (parser.__mem.child && parser.__mem.child.connected) {
            parser.__mem.child.kill('SIGKILL');
        }
        //process.exit(0)
    });
}
//another ugly function
function generateCsvRow(ctx, attrs) {
    var args = 'row', i,
        funcode = 'var csvRow="";\n';
     for (i = 0; i < attrs.length; i++) {
         if (attrs[i] !== '__originalText'){
             funcode += 'csvRow += \'\"\' + row[\''+attrs[i]+'\'] + \'\"\';\n';
         } else {
             funcode += 'csvRow += \'\"\' + row[\''+attrs[i]+'\'].replace(/\\"/g, \'""\') + \'\"\';\n';
         }
         funcode +=  i !== attrs.length -1 ? 'csvRow += ",";\n' : '';
     }
     funcode += 'return csvRow+"\\r\\n";'
     //console.log(funcode);
     return Function.apply(ctx, [args, funcode]);
}

function formatRow(row){
    if(type === 'csv') return csvRow(row);
    if(type === 'json') return jsonRow(row);
    return row;
}
function jsonRow(row){
    return !row.__lastrow ? JSON.stringify(row) + "," : JSON.stringify(row);
}<commit_msg>usage text fix again<commit_after>#!/usr/bin/env node

'use strict';

// if you want to use this command line tool you must 'npm install' to get 'optimist'
var argv = require('optimist').argv,
    path = require('path'),
    fs = require('fs'),
    Ginx = require(path.join(__dirname, '/../lib/ginx')),
    parser, input, output, format, usage, stats, storage,
    isFile, isDir, io_notice, verbose, type, writer, csvRow;

function usage(notice) {
    if (notice) console.log('\n' + notice);
    console.log(
        '\nThis tool is still under develpment(issues #1, #4, #8 and probably more) -'
        + '\nUsage: ginx --input [path] --output [path] -t [type] -j -g -v -c -n -f [format] -s [path]'
        + '\n\nthis tool will parse nginx log files and save it/them in either CSV or JSON format. I don\'t know how much that is useful,'
        + '\nAll of the CSV values are surrounded with double quotes, any inner commas are also surrounded by double quotes and, each original inner " is replaced by "" this was done following the CSV parsing conventions'
        + '\nPlus, there is a HUGE performance hit by writing another JSON file(s) at the same time, Don\'t output to JSON please ! '
        + '\nThe other problem with this (issue #1) is that, if you use JSON, is closing the JSON brackets at the end of each file, to complete the structure,'
        + '\nso if you parse a log file and it completes. Then sometime later you parse the same file again if there is more logs appended to it,'
        + '\nthe total resulted JSON structure will not be correct, syntax wise. However, if a crash or a kill occurs, '
        + '\nand the JSON is not closed yet, the resulted syntax will be fine once you resume the parsing'
        + '\nI will be implementing a tail -f like feature soon, it may make this tool more useful, stay tuned. Meanwhile, limit usage to 50 files max '
        + '\n[BROTIP] if I were you I would just use the module in a node.js program, get what you need out of each row, persist to a DB or something -- i don\'t even know why I wrote this tool'
        + '\nif you have a large log file to parse, you will end up with an even larger file if JSON for example.\n'
        + '\n-i | --input          : input file OR directory to parse'
        + '\n-o | --output         : destination file OR directory of where you would like to program the save the JSON file(s)'
        + '\n-f | --format         : [OPTIONAL] the format of your nginx access_log, if different then default [IMPORTANT] you must escape double quotes, the format MUST be exactly the same, single quotes in nginx format are not supported yet. (issue #5)'
        + '\n-n | --nonpersistent  : [OPTIONAL] a boolean value, defaults to true if not passed, if passed the program will not keep track of the file\'s positions, aka cursors, so it can\'t pick it where it left off in case of a crash or kill'
        + '\n-j | --fields2objects : [OPTIONAL] a boolean value, defaults to false if not passed, if passed the program will try to parse each field to a corresponding object, such as Date, Integer and NULL if \'-\' - may hit performance bad, about 50% more time if large or too many files'
        + '\n-s | --storage        : [OPTIONAL] a custome storage file for the file\'s positions cursors, it uses ./tmp/storage.cursors by default, but it won\'t save anything if --nonpersistent is passed'
        + '\n-g | --original       : [OPTIONAL] a boolean value, defaults to false, if passed, it will augment each row JSON object with its original text, if you need it, use it, if you don\' leave it out, it will make your output a lot larger'
        + '\n-h | --help           : [OPTIONAL] displays this message'
        + '\n-v | --verbose        : [OPTIONAL] verbose, but this could get ugly, printing each row on the screen, huge performance hit, don\'t use it'
        + '\n-c | --clear          : [OPTIONAL] will clear the cursors storage first, use it with -s if you have a custom storage file, otherwise it clears the default' + '\n\n');
        + '\n-t | --type           : [OPTIONAL] followed by "json" or "csv", defaults to csv if not specified, specifies the structure of the output files'
};

// print error and throw it
function error(err) {
    return console.log("[GINX-ERROR]:" + err.message);
    throw err;
}

// check if parser has a cached reference for this file in the loaded cursors from storage file, that latter happens in the Ginx constructor
function isNewFile(rfile) {
    return parser.__mem.cursors[rfile] ? false : true;
}

// if -h or --help
if (argv.h || argv.help) {
    return usage();
}
input = argv.i || argv.input;
output = argv.o || argv.output;
io_notice = 'You must pass both input and output, if input is a file, output will be a file, same for directories (limit to <50 files please)'
            + 'if input is a directory, output will be a directory as well, and you can\'t use the same directory nor same file for both arguments';
if (!input || !output) {
    return usage(io_notice);
} else {
    output = path.resolve(output);
    input = path.resolve(input);
    // synchronous, but necessary before it can continue
    stats = fs.statSync(input);
    if ((!stats.isDirectory() && !stats.isFile()) || (output === input)) {
        return usage(io_notice);
    }
}

//defaults
format = argv.f || argv.format || null;
verbose = argv.v || argv.verbose || false;
storage = argv.s || argv.storage || path.join(__dirname + '/../tmp/stored.cursors');

type = argv.t || argv.type || 'csv';
if (type !== 'csv' && type !== 'json'){
    console.log("[GINX-WARN] type : " + type + " is not supported or unknown, falling back to csv");
    type = 'csv';
}

// MAKING SURE the storage file is there before we continue, bunch of Sync calls but needed
// this will need to be updated when fs.writeFile be able to create directories recursivly
if(!fs.existsSync(storage)){
    var storageDir = path.normalize(storage.substring(0, storage.lastIndexOf(path.sep)));
    if(!fs.existsSync(storageDir)){
        fs.mkdirSync(storageDir);
    }
    fs.writeFileSync(storage, "{}");
} else if (argv.c || argv.clear) {
    //clearing storage if requested before constructing Ginx.
    fs.writeFileSync(storage, "{}");
    console.log("[GINX-INFO] Emptied " + storage + " storage file");
}

//construct the Ginx parser
parser = new Ginx(format, {
    'persistent': argv.n || argv.nonpersistent ? false : true,
    'fieldsToObjects': argv.j || argv.fields2objects ? true : false,
    'storageFile': argv.s || argv.storage || null,
    'originalText': argv.g || argv.original ? true : false
});

if(type === 'csv'){
  csvRow = generateCsvRow(this, parser.attrs);
}

// creating a writer to handle the data buffering from the parser's readstreams
writer = {
    wstreams: {},
    append: function(data, wfile, rfile){
        if(data
            && this.wstreams[wfile]
            && this.wstreams[wfile].write(data) == false) {
                parser.pause(rfile);
        }
    },
    addStream: function(wfile, rfile, flag, start, callback){
        var wstream = fs.createWriteStream(wfile, {'flags': flag ? flag : 'r+', 'start': start ? start : 0, 'mode': '0666'});
        this.wstreams[wfile] = wstream;
        this.wstreams[wfile].on('drain', function(){
            parser.resume(rfile);
        });
        callback();
    }
}
// let the parsing begin !!!

// process directory parsing
if (stats.isDirectory()) {
    fs.mkdir(output, function (err) {
        if (err){console.log(output + " already exists")}
        fs.readdir(input, function (err, files) {
            if (err) error(err);
            files.forEach(function (file) {
                var wfile = path.join(output, file),
                    rfile = path.join(input, file),
                    rnew = isNewFile(input),
                    wnew = false;
                    fs.stat(wfile, function(err, stats){
                        if(err){
                            wnew = true;
                        }
                        if (wnew || rnew) {
                            writer.addStream(wfile, rfile, 'w+', 0, function(){
                                if (type === 'csv'){
                                    writer.append(parser.attrs.join(',')+'\r\n', wfile, rfile);
                                } else if (type === 'json'){
                                    writer.append("{[", wfile, rfile);
                                }
                            });
                        } else {
                            writer.addStream(wfile, rfile,'r+', stats.size, function(){});
                        }
                    });

            });
            processDirectory(input, output);
        });
    });
} else if (stats.isFile()) {
    var rnew = isNewFile(input),
        wnew = false;
    fs.stat(output, function(err, stats){
        if(err){
            wnew = true;
        }
        if (wnew || rnew) {
            writer.addStream(output, input, 'w+', 0, function(){
                if (type === 'csv'){
                    writer.append(parser.attrs.join(',')+'\r\n', output, input);
                } else if (type === 'json'){
                    writer.append("{[", output, input);
                }
                processFile(input, output);
            });
        } else {
            writer.addStream(output, input,'r+', stats.size, function(){
                processFile(input, output);
            });
        }
    });
}
// process file parsing to JSON output
function processFile(input, ouput) {
    parser.parseFile(input,

    function (err, row) {
        if (err) error(err);
        writer.append(formatRow(row), output, row.__file);
    },

    function (err, rfile) {
        if (err) error(err);
        if(type === 'json') { writer.append("]}", output, rfile); }
    });
}

// process directory parsing to JSON outputs
function processDirectory(input, output) {
    parser.parseDir(input,

    function (err, row) {
        if (err) error(err);
        writer.append(formatRow(row), path.join(output, row.__fname), row.__file);
    },

    function (err, rfile) {
        if (err) error(err);
        var wfile = path.join(output, rfile.substring(rfile.lastIndexOf(path.sep) + 1));
        if(type === 'json') writer.append("]}", wfile, rfile);
    },

    function (err, filesCount) {
        if (err) error(err);
        if (parser.__mem.child && parser.__mem.child.connected) {
            parser.__mem.child.kill('SIGKILL');
        }
        //process.exit(0)
    });
}
//another ugly function
function generateCsvRow(ctx, attrs) {
    var args = 'row', i,
        funcode = 'var csvRow="";\n';
     for (i = 0; i < attrs.length; i++) {
         if (attrs[i] !== '__originalText'){
             funcode += 'csvRow += \'\"\' + row[\''+attrs[i]+'\'] + \'\"\';\n';
         } else {
             funcode += 'csvRow += \'\"\' + row[\''+attrs[i]+'\'].replace(/\\"/g, \'""\') + \'\"\';\n';
         }
         funcode +=  i !== attrs.length -1 ? 'csvRow += ",";\n' : '';
     }
     funcode += 'return csvRow+"\\r\\n";'
     //console.log(funcode);
     return Function.apply(ctx, [args, funcode]);
}

function formatRow(row){
    if(type === 'csv') return csvRow(row);
    if(type === 'json') return jsonRow(row);
    return row;
}
function jsonRow(row){
    return !row.__lastrow ? JSON.stringify(row) + "," : JSON.stringify(row);
}<|endoftext|>


<commit_before>#!/usr/bin/env node

"use strict";

var mott = require('../'),
    fs = require('fs-extra'),
    argv = require('optimist').argv;

if(argv._[0] === 'new'){
    var path = require('path'),
        async = require('async'),
        child_process = require('child_process'),
        recipe = mott(),
        name = argv._[1],
        _uses = ((Array.isArray(argv.use) ? argv.use : (argv.use) ? [argv.use] : []));

    recipe.task('new', function(ctx, done){
        ctx.projectName = name;
        ctx.packageJson = {
            'name': name,
            'version': '0.0.0',
            'dependencies': {},
            'devDependencies': {
                'mott': 'git://github.com/imlucas/mott.git'
                // @todo (lucas) include anything from uses
            },
            'mott': {
                'recipe': {
                    'build': ["js", "less", "pages"],
                    'run': ["build", "dev server", "watch"],
                    'metadata': {
                        "js": {
                            "./index.js": "index.js"
                        },
                        "less": {
                            "./index.less": "index.css"
                        },
                        "includes": {
                            "./index.html": "index.html"
                        },
                        'deploy': 'github',
                        'export config': ['url']
                    }
                },
                'development': {
                    'url': 'http://localhost:8080'
                },
                'production': {}
            }
        };
        async.parallel([
            function (callback){
                fs.mkdirs('./' + name, callback);
                ctx.baseDir = path.resolve('./' + name);
            },
            function (callback){
                fs.writeFile('./' + name + '/package.json',
                    JSON.stringify(ctx.packageJson, null, 4), callback);
            },
            function (callback){
                fs.copy(__dirname + '/../assets/Makefile.tpl', ctx.path('Makefile'), callback);
            },
            function (callback){
                fs.copy(__dirname + '/../assets/index.html.tpl', ctx.path('index.html'), callback);
            },
            function (callback){
                fs.copy(__dirname + '/../assets/index.js.tpl', ctx.path('index.js'), callback);
            },
            function (callback){
                fs.copy(__dirname + '/../assets/index.less.tpl', ctx.path('index.less'), callback);
            }
        ], function(){
            child_process.exec('npm link mott', {'cwd': ctx.baseDir}, done);
        });
    });
    _uses.map(function(_use){
        // @todo (lucas) npm install
        recipe.use(mott.resolve(_use));
    });

    recipe.cook().exec('new', {}, function(err){
        console.log('done');
    });
    return;
}

// @todo (lucas) If a local mott exists, shell to that bin

fs.readJson('./package.json', function(err, data){
    if(err){
        console.log(err);
        return;
    }
    var environments = {},
        metadata,
        key,
        recipe;

    for(key in data.mott){
        if(key !== "recipe"){
            environments[key] = data.mott[key];
        }
    }

    recipe = mott();
    for(key in data.mott.recipe){
        if(key === "metadata"){
            metadata = data.mott.recipe.metadata;
        }
        else if(key === "use"){
            recipe.use(mott.resolve(data.mott.recipe[key]));
        }
        else {
            recipe.task(key, mott.resolve(data.mott.recipe[key]));
        }
    }
    recipe.cli(metadata, environments);
});
<commit_msg>Dont make new dir if no name<commit_after>#!/usr/bin/env node

"use strict";

var mott = require('../'),
    fs = require('fs-extra'),
    argv = require('optimist').argv;

if(argv._[0] === 'new'){
    var path = require('path'),
        async = require('async'),
        child_process = require('child_process'),
        recipe = mott(),
        name = argv._[1],
        _uses = ((Array.isArray(argv.use) ? argv.use : (argv.use) ? [argv.use] : []));

    recipe.task('new', function(ctx, done){
        ctx.projectName = name;
        ctx.packageJson = {
            'name': name,
            'version': '0.0.0',
            'dependencies': {},
            'devDependencies': {
                'mott': 'git://github.com/imlucas/mott.git'
                // @todo (lucas) include anything from uses
            },
            'mott': {
                'recipe': {
                    'build': ["js", "less", "pages"],
                    'run': ["build", "dev server", "watch"],
                    'metadata': {
                        "js": {
                            "./index.js": "index.js"
                        },
                        "less": {
                            "./index.less": "index.css"
                        },
                        "includes": {
                            "./index.html": "index.html"
                        },
                        'deploy': 'github',
                        'export config': ['url']
                    }
                },
                'development': {
                    'url': 'http://localhost:8080'
                },
                'production': {}
            }
        };
        async.parallel([
            function (callback){
                if(!name){
                    return callback();
                }
                fs.mkdirs('./' + name, callback);
                ctx.baseDir = path.resolve('./' + name);
            },
            function (callback){
                fs.writeFile(ctx.path('package.json'),
                    JSON.stringify(ctx.packageJson, null, 4), callback);
            },
            function (callback){
                fs.copy(__dirname + '/../assets/Makefile.tpl', ctx.path('Makefile'), callback);
            },
            function (callback){
                fs.copy(__dirname + '/../assets/index.html.tpl', ctx.path('index.html'), callback);
            },
            function (callback){
                fs.copy(__dirname + '/../assets/index.js.tpl', ctx.path('index.js'), callback);
            },
            function (callback){
                fs.copy(__dirname + '/../assets/index.less.tpl', ctx.path('index.less'), callback);
            }
        ], function(){
            child_process.exec('npm link mott', {'cwd': ctx.baseDir}, done);
        });
    });
    _uses.map(function(_use){
        // @todo (lucas) npm install
        recipe.use(mott.resolve(_use));
    });

    recipe.cook().exec('new', {}, function(err){
        console.log('done');
    });
    return;
}

// @todo (lucas) If a local mott exists, shell to that bin

fs.readJson('./package.json', function(err, data){
    if(err){
        console.log(err);
        return;
    }
    var environments = {},
        metadata,
        key,
        recipe;

    for(key in data.mott){
        if(key !== "recipe"){
            environments[key] = data.mott[key];
        }
    }

    recipe = mott();
    for(key in data.mott.recipe){
        if(key === "metadata"){
            metadata = data.mott.recipe.metadata;
        }
        else if(key === "use"){
            recipe.use(mott.resolve(data.mott.recipe[key]));
        }
        else {
            recipe.task(key, mott.resolve(data.mott.recipe[key]));
        }
    }
    recipe.cli(metadata, environments);
});
<|endoftext|>


<commit_before>export default [
    {
        path: 'actions.js',
        text: `
// copied from https://github.com/reactjs/redux/blob/d5d1572cba55942d571b4b52cd12e5045142b9ff/examples/todomvc/src/actions/index.js
import * as types from '../constants/ActionTypes'

export const addTodo = (text) => ({
  type: types.ADD_TODO,
  text
})
export const deleteTodo = id => ({
  type: types.DELETE_TODO,
  id
})
export const editTodo = (id, text) => ({
  type: types.EDIT_TODO,
  id,
  text
})
export const completeTodo = id => ({
  type: types.COMPLETE_TODO,
  id
})
export const completeAll = () => ({
  type: types.COMPLETE_ALL
})
export const clearCompleted = () => ({
  type: types.CLEAR_COMPLETED
})
`
    },

    {
        path: 'actions.spec.js',
        text: `
// copied from https://github.com/reactjs/redux/blob/d5d1572cba55942d571b4b52cd12e5045142b9ff/examples/todomvc/src/actions/index.spec.js
import * as types from '../constants/ActionTypes'
import * as actions from './index'

describe('todo actions', () => {
  it('addTodo should create ADD_TODO action', () => {
    expect(actions.addTodo('Use Redux')).toEqual({
      type: types.ADD_TODO,
      text: 'Use Redux'
    })
  })

  it('deleteTodo should create DELETE_TODO action', () => {
    expect(actions.deleteTodo(1)).toEqual({
      type: types.DELETE_TODO,
      id: 1
    })
  })

  it('editTodo should create EDIT_TODO action', () => {
    expect(actions.editTodo(1, 'Use Redux everywhere')).toEqual({
      type: types.EDIT_TODO,
      id: 1,
      text: 'Use Redux everywhere'
    })
  })

  it('completeTodo should create COMPLETE_TODO action', () => {
    expect(actions.completeTodo(1)).toEqual({
      type: types.COMPLETE_TODO,
      id: 1
    })
  })

  it('completeAll should create COMPLETE_ALL action', () => {
    expect(actions.completeAll()).toEqual({
      type: types.COMPLETE_ALL
    })
  })

  it('clearCompleted should create CLEAR_COMPLETED action', () => {
    expect(actions.clearCompleted()).toEqual({
      type: types.CLEAR_COMPLETED
    })
  })
})
`
    },

    {
        path: 'Header.js',
        text: `
// copied from https://github.com/reactjs/redux/blob/d5d1572cba55942d571b4b52cd12e5045142b9ff/examples/todomvc/src/components/Header.js
// modified to use arrow functions instead of class syntax
import React, { PropTypes, Component } from 'react'
import TodoTextInput from './TodoTextInput'

export default ({addTodo}) => {
  let handleSave = (text) => {
    if (text.length !== 0) {
      addTodo(text)
    }
  }

  return (

todos

      <TodoTextInput newTodo
                     onSave={handleSave}
                     placeholder="What needs to be done?" />

  )
};
`
    },

    {
        path: 'Header.spec.js',
        text: `
// copied from https://github.com/reactjs/redux/blob/d5d1572cba55942d571b4b52cd12e5045142b9ff/examples/todomvc/src/components/Header.spec.js
import React from 'react'
import TestUtils from 'react-addons-test-utils'
import Header from './Header'
import TodoTextInput from './TodoTextInput'

const setup = () => {
  const props = {
    addTodo: jest.fn()
  }

  const renderer = TestUtils.createRenderer()
  renderer.render()
  const output = renderer.getRenderOutput()

  return {
    props: props,
    output: output,
    renderer: renderer
  }
}

describe('components', () => {
  describe('Header', () => {
    it('should render correctly', () => {
      const { output } = setup()

      expect(output.type).toBe('header')
      expect(output.props.className).toBe('header')

      const [ h1, input ] = output.props.children

      expect(h1.type).toBe('h1')
      expect(h1.props.children).toBe('todos')

      expect(input.type).toBe(TodoTextInput)
      expect(input.props.newTodo).toBe(true)
      expect(input.props.placeholder).toBe('What needs to be done?')
    })

    it('should call addTodo if length of text is greater than 0', () => {
      const { output, props } = setup()
      const input = output.props.children[1]
      input.props.onSave('')
      expect(props.addTodo).not.toBeCalled()
      input.props.onSave('Use Redux')
      expect(props.addTodo).toBeCalled()
    })
  })
})
`
    }

]
<commit_msg> #1 remove first new line of demo files<commit_after>export default [
    {
        path: 'actions.js',
        text: `// copied from https://github.com/reactjs/redux/blob/d5d1572cba55942d571b4b52cd12e5045142b9ff/examples/todomvc/src/actions/index.js
import * as types from '../constants/ActionTypes'

export const addTodo = (text) => ({
  type: types.ADD_TODO,
  text
})
export const deleteTodo = id => ({
  type: types.DELETE_TODO,
  id
})
export const editTodo = (id, text) => ({
  type: types.EDIT_TODO,
  id,
  text
})
export const completeTodo = id => ({
  type: types.COMPLETE_TODO,
  id
})
export const completeAll = () => ({
  type: types.COMPLETE_ALL
})
export const clearCompleted = () => ({
  type: types.CLEAR_COMPLETED
})
`
    },

    {
        path: 'actions.spec.js',
        text: `// copied from https://github.com/reactjs/redux/blob/d5d1572cba55942d571b4b52cd12e5045142b9ff/examples/todomvc/src/actions/index.spec.js
import * as types from '../constants/ActionTypes'
import * as actions from './index'

describe('todo actions', () => {
  it('addTodo should create ADD_TODO action', () => {
    expect(actions.addTodo('Use Redux')).toEqual({
      type: types.ADD_TODO,
      text: 'Use Redux'
    })
  })

  it('deleteTodo should create DELETE_TODO action', () => {
    expect(actions.deleteTodo(1)).toEqual({
      type: types.DELETE_TODO,
      id: 1
    })
  })

  it('editTodo should create EDIT_TODO action', () => {
    expect(actions.editTodo(1, 'Use Redux everywhere')).toEqual({
      type: types.EDIT_TODO,
      id: 1,
      text: 'Use Redux everywhere'
    })
  })

  it('completeTodo should create COMPLETE_TODO action', () => {
    expect(actions.completeTodo(1)).toEqual({
      type: types.COMPLETE_TODO,
      id: 1
    })
  })

  it('completeAll should create COMPLETE_ALL action', () => {
    expect(actions.completeAll()).toEqual({
      type: types.COMPLETE_ALL
    })
  })

  it('clearCompleted should create CLEAR_COMPLETED action', () => {
    expect(actions.clearCompleted()).toEqual({
      type: types.CLEAR_COMPLETED
    })
  })
})
`
    },

    {
        path: 'Header.js',
        text: `// copied from https://github.com/reactjs/redux/blob/d5d1572cba55942d571b4b52cd12e5045142b9ff/examples/todomvc/src/components/Header.js
// modified to use arrow functions instead of class syntax
import React, { PropTypes, Component } from 'react'
import TodoTextInput from './TodoTextInput'

export default ({addTodo}) => {
  let handleSave = (text) => {
    if (text.length !== 0) {
      addTodo(text)
    }
  }

  return (

todos

      <TodoTextInput newTodo
                     onSave={handleSave}
                     placeholder="What needs to be done?" />

  )
};
`
    },

    {
        path: 'Header.spec.js',
        text: `// copied from https://github.com/reactjs/redux/blob/d5d1572cba55942d571b4b52cd12e5045142b9ff/examples/todomvc/src/components/Header.spec.js
import React from 'react'
import TestUtils from 'react-addons-test-utils'
import Header from './Header'
import TodoTextInput from './TodoTextInput'

const setup = () => {
  const props = {
    addTodo: jest.fn()
  }

  const renderer = TestUtils.createRenderer()
  renderer.render()
  const output = renderer.getRenderOutput()

  return {
    props: props,
    output: output,
    renderer: renderer
  }
}

describe('components', () => {
  describe('Header', () => {
    it('should render correctly', () => {
      const { output } = setup()

      expect(output.type).toBe('header')
      expect(output.props.className).toBe('header')

      const [ h1, input ] = output.props.children

      expect(h1.type).toBe('h1')
      expect(h1.props.children).toBe('todos')

      expect(input.type).toBe(TodoTextInput)
      expect(input.props.newTodo).toBe(true)
      expect(input.props.placeholder).toBe('What needs to be done?')
    })

    it('should call addTodo if length of text is greater than 0', () => {
      const { output, props } = setup()
      const input = output.props.children[1]
      input.props.onSave('')
      expect(props.addTodo).not.toBeCalled()
      input.props.onSave('Use Redux')
      expect(props.addTodo).toBeCalled()
    })
  })
})
`
    }

]
<|endoftext|>


<commit_before>'use strict';

let path = require('path'),
  messageMaker = require('../../support/messageMaker'),
  aggregator = require('./aggregator'),
  pagexrayAggregator = require('./pagexrayAggregator'),
  pagexray = require('pagexray'),
  filterRegistry = require('../../support/filterRegistry');

const make = messageMaker('coach').make;

const DEFAULT_METRICS_SUMMARY = ['score.*','performance.score.*', 'bestpractice.score.*', 'accessibility.score.*'];
const DEFAULT_METRICS_PAGESUMMARY = ['advice.score','advice.performance.score','advice.bestpractice.score', 'advice.accessibility.score'];
const DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS = ['contentTypes','transferSize','contentSize','requests','firstParty', 'thirdParty','responseCodes','expireStats', 'domains', 'lastModifiedStats'];
const DEFAULT_PAGEXRAY_SUMMARY_METRICS = ['contentTypes','transferSize','contentSize','requests','firstParty', 'thirdParty','responseCodes','expireStats', 'domains', 'lastModifiedStats'];

module.exports = {
  name() {
    return path.basename(__dirname);
  },
  open(context, options) {
    this.options = options;
    filterRegistry.registerFilterForType(DEFAULT_METRICS_SUMMARY, 'coach.summary');
    filterRegistry.registerFilterForType(DEFAULT_METRICS_PAGESUMMARY, 'coach.pageSummary');
    filterRegistry.registerFilterForType(DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS, 'pagexray.pageSummary');
    filterRegistry.registerFilterForType(DEFAULT_PAGEXRAY_SUMMARY_METRICS, 'pagexray.summary');
  },
  processMessage(message, queue) {
    switch (message.type) {
      case 'coach.run':
      {
        if (message.runIndex === 0) {
          // For now, choose the first run to represent the whole page.
          // Later we might want to change the median run (based on some metric) similar to the WPT approach.
          const url = message.url;
          const group = message.group;
          queue.postMessage(make('coach.pageSummary', message.data, {url, group}));
        }

        aggregator.addToAggregate(message.data, message.group);
        break;
      }

      case 'browsertime.har':
      {
        const url = message.url;
        const group = message.group;
        let config = {
          includeAssets: true,
          firstParty: this.options.firstParty ? this.options.firstParty : undefined
        };
        const pageSummary = pagexray.convert(message.data, config);

        pagexrayAggregator.addToAggregate(pageSummary, group);

        queue.postMessage(make('pagexray.pageSummary', pageSummary[0], {url, group}));

        pageSummary.forEach((run, runIndex) => {
          queue.postMessage(make('pagexray.run', run, {url, group, runIndex}));
        });
        break;
      }

      case 'summarize':
      {
        let summary = aggregator.summarize();
        if (summary && Object.keys(summary).length > 0) {
          for (let group of Object.keys(summary.groups)) {
            queue.postMessage(make('coach.summary', summary.groups[group], {group}));
          }
        }

        let pagexraySummary = pagexrayAggregator.summarize();
        if (pagexraySummary && Object.keys(pagexraySummary.groups).length > 0) {
          for (let group of Object.keys(pagexraySummary.groups)) {
            queue.postMessage(make('pagexray.summary', pagexraySummary.groups[group], {group}));
          }
        }
        break;
      }
    }
  }
};
<commit_msg>send only number of domains to Graphite closes #1176<commit_after>'use strict';

let path = require('path'),
  messageMaker = require('../../support/messageMaker'),
  aggregator = require('./aggregator'),
  pagexrayAggregator = require('./pagexrayAggregator'),
  pagexray = require('pagexray'),
  filterRegistry = require('../../support/filterRegistry');

const make = messageMaker('coach').make;

const DEFAULT_METRICS_SUMMARY = ['score.*','performance.score.*', 'bestpractice.score.*', 'accessibility.score.*'];
const DEFAULT_METRICS_PAGESUMMARY = ['advice.score','advice.performance.score','advice.bestpractice.score', 'advice.accessibility.score'];
const DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS = ['contentTypes','transferSize','contentSize','requests','firstParty', 'thirdParty','responseCodes','expireStats', 'totalDomains', 'lastModifiedStats'];
const DEFAULT_PAGEXRAY_SUMMARY_METRICS = ['contentTypes','transferSize','contentSize','requests','firstParty', 'thirdParty','responseCodes','expireStats', 'domains', 'lastModifiedStats'];

module.exports = {
  name() {
    return path.basename(__dirname);
  },
  open(context, options) {
    this.options = options;
    filterRegistry.registerFilterForType(DEFAULT_METRICS_SUMMARY, 'coach.summary');
    filterRegistry.registerFilterForType(DEFAULT_METRICS_PAGESUMMARY, 'coach.pageSummary');
    filterRegistry.registerFilterForType(DEFAULT_PAGEXRAY_PAGESUMMARY_METRICS, 'pagexray.pageSummary');
    filterRegistry.registerFilterForType(DEFAULT_PAGEXRAY_SUMMARY_METRICS, 'pagexray.summary');
  },
  processMessage(message, queue) {
    switch (message.type) {
      case 'coach.run':
      {
        if (message.runIndex === 0) {
          // For now, choose the first run to represent the whole page.
          // Later we might want to change the median run (based on some metric) similar to the WPT approach.
          const url = message.url;
          const group = message.group;
          queue.postMessage(make('coach.pageSummary', message.data, {url, group}));
        }

        aggregator.addToAggregate(message.data, message.group);
        break;
      }

      case 'browsertime.har':
      {
        const url = message.url;
        const group = message.group;
        let config = {
          includeAssets: true,
          firstParty: this.options.firstParty ? this.options.firstParty : undefined
        };
        const pageSummary = pagexray.convert(message.data, config);

        pagexrayAggregator.addToAggregate(pageSummary, group);

        queue.postMessage(make('pagexray.pageSummary', pageSummary[0], {url, group}));

        pageSummary.forEach((run, runIndex) => {
          queue.postMessage(make('pagexray.run', run, {url, group, runIndex}));
        });
        break;
      }

      case 'summarize':
      {
        let summary = aggregator.summarize();
        if (summary && Object.keys(summary).length > 0) {
          for (let group of Object.keys(summary.groups)) {
            queue.postMessage(make('coach.summary', summary.groups[group], {group}));
          }
        }

        let pagexraySummary = pagexrayAggregator.summarize();
        if (pagexraySummary && Object.keys(pagexraySummary.groups).length > 0) {
          for (let group of Object.keys(pagexraySummary.groups)) {
            queue.postMessage(make('pagexray.summary', pagexraySummary.groups[group], {group}));
          }
        }
        break;
      }
    }
  }
};
<|endoftext|>


<commit_before>"use strict";

require("setimmediate"); // polyfill for env != IE10+

/* global requestIdleCallback */
if (global.window) { // eslint-disable-line no-undef
	require("requestidlecallback"); // polyfill for env != Chrome46+
} else {
	global.requestIdleCallback = fn => {
		return setTimeout(() => {
			let deadline = Date.now() + 50;
			fn({
				timeRemaining: () => Math.max(0, deadline - Date.now()),
				didTimeout: false
			});
		}, 0);
	};
	global.cancelIdleCallback = handle => clearTimeout(handle);
}

let jsonop = require("jsonop"),
	OrderedArray = require("./OrderedArray"),
	RangeArray = require("./RangeArray"),
	keySlice  = require("./keyslice"),
	keyToSlice = keySlice.keyToSlice,
	sliceToKey = keySlice.sliceToKey;

module.exports = class Cache {
	constructor(options) {
		options = jsonop({
			id: (entity) => { return entity.id; },
			is: (entity, type) => { return entity.type === type; },
			entityOp: {},
			appOp: {}
		}, options || {});

		this.entities = {};
		this.indexes = {};
		this.queries = {};
		this.knowledge = {};
		this.app = {};
		this.keyToSlice = keyToSlice;
		this.sliceToKey = sliceToKey;

		/*
			map of types to index keys to index slices for fast lookup
			{
				items: {
					"items:createTime(tag:hidden)": { type: items, order: createTime, filters: { tag: hidden } }
				}
			}
		*/
		this._types = {};
		this._changeListeners = [];
		this._queryCallbacks = {};
		this._recentQueries = { entities: {}};
		this._querySeqNum = 0;
		this._id = options.id;
		this._is = options.is;
		this._options = options;
	}

	/*
		Get index keys and corresponding values that must include this entity,
		taking into account knowledge for the same key.
		@param entity
		@returns { key: value }
	*/
	getKeyValues(entity) {
		let keyValues = {};

		for (let type in this._types) {
			if (!this._is(entity, type)) { continue; }

			keyloop:
			for (let key in this._types[type].base) {
				let slice = this._types[type].base[key],
					value = entity[slice.order];

				if (typeof value === "undefined") { continue; }

				for (let prop in slice.filters) {
					if (entity[prop] !== slice.filters[prop]) {
						continue keyloop;
					}
				}
				if (this.knowledge[key].contains(value)) {
					keyValues[key] = { value, item: (slice.link || slice.join) ?
						{ [type]: entity } : entity };
				}
			}

			keyloop:
			for (let key in this._types[type].join) {
				let slice = this._types[type].join[key],
					base = this.entities[entity[slice.join[type]]],
					value = base[slice.order];

				if (typeof value === "undefined") { continue; }

				for (let prop in slice.filters) {
					if (entity[prop] !== slice.filters[prop]) {
						continue keyloop;
					}
				}
				if (this.knowledge[key].contains(value)) {
					keyValues[key] = { value, item: { [type]: entity } };
				}
			}

			// TODO: Implement link types. This is not currently
			// required in HeyNeighbor, so not targeted for first
			// release.
		}

		return keyValues;
	}

	/*
		Adds or updates an entity and updates indexes if necessary.
	*/
	putEntity(entity, excludeKeys) {
		let id = this._id(entity),
			oldKeyValues,
			newKeyValues;

		if (this.entities[id]) {
			oldKeyValues = this.getKeyValues(this.entities[id]);
			entity = jsonop(this.entities[id], entity, this.options.entityOp);
		}
		this.entities[id] = entity;
		newKeyValues = this.getKeyValues(entity);

		if (oldKeyValues) {
			for (let key in oldKeyValues) {
				if (oldKeyValues[key].value !== newKeyValues[key].value) {
					this.indexes[key].delete(oldKeyValues[key].value);
				} else {
					delete newKeyValues[key];
				}
			}
		}

		if (excludeKeys) { for (let key of excludeKeys) {
			delete newKeyValues[key];
		}}

		for (let key in newKeyValues) {
			this.indexes[key].add(newKeyValues[key].item);
		}

		return entity;
	}

	deleteEntity(id) {
		let keyValues = this.getKeyValues(this.entities[id]);
		for (let key in keyValues) {
			if (
				keyValues[key].item === this.entities[id] ||
				keyValues[key].item[keyToSlice(key).type] === this.entities[id]
			) {
				this.indexes[key].delete(keyValues[key].value);
			} else {
				for (let type in keyValues[key].item) {
					if (keyValues[key].item[type] === this.entities[id]) {
						this.indexes[key].put({ [type]: null });
					}
				}
			}
		}
		delete this.entities[id];
	}

	putIndex(key, knowledge, items) {
		let slice = keyToSlice(key),
			addType = (comp, type) => {
				this._types[type] = this._types[type] || {
					base: {}, link: {}, join: {}
				};
				this._types[type][comp][key] = slice;
			};
		if (!this.knowledge[key]) {
			addType("base", slice.type);
			if (slice.link) { for (let type in slice.link) {
				addType("link", type);
			}}
			if (slice.join) { for (let type in slice.join) {
				addType("join", type);
			}}

			this.knowledge[key] = knowledge;
			this.indexes[key] = items;
			this.indexEntities(key);
		} else {
			for (let range of knowledge) {
				let rangeItems = items.slice(...range);
				this.indexes[key].splice(...range, rangeItems);
				this.knowledge[key].add(range);
			}
		}
		// insert all the items into entities, updating all indexes
		// except this one.
		if (slice.join || slice.link) {
			for (let item of items) {
				for (let name in item) {
					this.putEntity(item[name], [ key ]);
				}
			}
		} else {
			for (let item of items) {
				this.putEntity(item, [ key ]);
			}
		}

		if (this._queryCallbacks[key]) {
			for (let i in this._queryCallbacks[key]) {
				let { ranges, callback } = this._queryCallbacks[key][i];
				if (ranges.subtract(this.knowledge[key]).length === 0) {
					// No gaps in knowledge; query is satisfied.
					this.query(key, ranges[0], callback);
				}
			}
		}
	}

	/*
		Checks every entity for inclusion in a particular index. Called when a
		new index is created.
	*/
	indexEntities(key) {
		let slice = keyToSlice(key);
		for (let id in this.entities) {
			let entity = this.entities[id],
				value = entity[slice.order];
			if (typeof value !== "undefined") {
				this.indexes[key].add(entity);
			}
		}
		/* TODO: Rewrite for perf: use push() then sort() */
	}

	pruneEntities(/* deadline */) {
		/* TODO: Implement in a nonblocking fashion */
	}

	/*
		Make changes to this cache (setState).
	*/
	put(changes) {
		/* handle the addition of ranges of results */
		if (changes.knowledge) { for (let key in changes.knowledge) {
			let items = changes.indexes && changes.indexes[key] || [];

			if (!(changes.knowledge[key] instanceof RangeArray)) {
				changes.knowledge[key] =
					new RangeArray(changes.knowledge[key]);
			}
			if (!items) {
				items = new OrderedArray(keyToSlice(key).order, []);
			} else if (!(items instanceof OrderedArray)) {
				changes.indexes[key] = items =
					new OrderedArray(keyToSlice(key).order, changes.indexes[key]);
			}

			this.putIndex(key, changes.knowledge[key], items);
		}}

		/* handle the addition of individual entities */
		if (changes.entities) { for (let id in changes.entities) {
			if (changes.entities[id] === null) this.deleteEntity(id);
			else this.putEntity(changes.entities[id]);
		}}

		if (changes.app) {
			this.app = jsonop(this.app, changes.app, this.options.appOp);
		}

		/* handle the addition of a query */
		if (changes.queries) { for (let key in changes.queries) {
			if (!this.queries[key]) {
				this.queries[key] = changes.queries[key];
			} else {
				for (let range in changes.queries[key]) {
					this.queries[key].add(range);
				}
			}
		}}

		/* if there are too many entities, clean up */
		if (Object.keys(this.entities).length > this._options.maxEntities) {
			requestIdleCallback(this.pruneEntities.bind(this));
		}
		this._changeListeners.forEach(fn => fn(changes));
	}

	query(key, range, callback) {
		let items, gaps, ranges = new RangeArray([ range ]);
		range.seqNum = this._querySeqNum++;

		/* Extract items and calculate gaps in knowledge */
		if (!this.knowledge[key]) {
			let { order } = keyToSlice(key);
			items = new OrderedArray(order, []);
			gaps = ranges;
		} else {
			items = this.indexes[key].slice(...range);
			gaps = (range.length === 3 ? items.getRange() : ranges)
				.difference(this.knowledge[key]);

			// If it is a before/after range, gaps will not show missing data at
			// start and end; we need to detect and correct this.

			if (range.length === 3 && items.length < range[1] + range[2]) {
				let pos = items.indexOf(range[0]);
				if (
					pos < range[1] &&
					!this.knowledge[key].hasStart()
				) { gaps.add([
					items.get(0)[items.order], range[1] - pos, 0
				]); }
				if (
					items.length - pos < range[2] &&
					!this.knowledge[key].hasEnd()
				) { gaps.add([
					items.get(items.length - 1)[items.order], 0,
					range[2] + pos - items.length
				]); }
			}
		}

		/* Insert “loading” placeholders corresponding to the gaps. */
		for (let gap of gaps) {
			if (gap.length === 2) {
				items.put({
					type: "loading", start: gap[0], end: gap[1],
					[items.order]: gap[1] === Infinity ? gap[1] : gap[0]
				});
			} else {
				items.put({
					type: "loading", start: gap[0], before: gap[1], after: gap[2],
					[items.order]: gap[0]
				}, !!items.after);
			}
		}

		/* Make queries corresponding to the gaps. */
		if (gaps.length) {
			this.put({ queries: { [key]: gaps }});
			if (callback) {
				this._queryCallbacks[key] = this._queryCallbacks[key] || [];
				this._queryCallbacks[key].push({ ranges, callback });
			}
		} else {
			if (callback) {
				setImmediate(() => callback(null, items));
			}
		}

		/* Update the most recently accessed map */
		if (!this._recentQueries[key]) {
			this._recentQueries[key] = items.getRange();
		} else {
			this._recentQueries[key].put(items.getRange());
		}

		return items;
	}

	onChange(fn) {
		this._changeListeners.push(fn);
	}

	get() {
		let args = Array.prototype.slice.call(arguments),
			pointer = this; // eslint-disable-line consistent-this
		while (typeof pointer === "object" && pointer !== null && args.length) {
			pointer = pointer[args.shift()];
		}

		return pointer;
	}

	getEntity(id, callback) {
		var result;
		this._recentQueries.entities[id] = this._querySeqNum++;
		if (this.entities && this.entities[id]) result = this.entities[id];

		if (result) {
			if (callback) callback(null, result);
			return result;
		} else {
			this.put({
				queries: {
					entities: {
						id: true
					}
				}
			});

			if (callback) {
				this._queryCallback.entities = this._queryCallback.entities || {};
				(this._queryCallback.entities = this._queryCallback.entities || []).push(callback);
			}

			return {type: "loading"};
		}
	}

	with(changes) {
		var cache = new Cache();
		cache.put({ app: jsonop(this.app, changes.app) });
		return cache;
	}
};
<commit_msg>Adding watch apis.<commit_after>"use strict";

require("setimmediate"); // polyfill for env != IE10+

/* global requestIdleCallback */
if (global.window) { // eslint-disable-line no-undef
	require("requestidlecallback"); // polyfill for env != Chrome46+
} else {
	global.requestIdleCallback = fn => {
		return setTimeout(() => {
			let deadline = Date.now() + 50;
			fn({
				timeRemaining: () => Math.max(0, deadline - Date.now()),
				didTimeout: false
			});
		}, 0);
	};
	global.cancelIdleCallback = handle => clearTimeout(handle);
}

let jsonop = require("jsonop"),
	OrderedArray = require("./OrderedArray"),
	RangeArray = require("./RangeArray"),
	keySlice  = require("./keyslice"),
	keyToSlice = keySlice.keyToSlice,
	sliceToKey = keySlice.sliceToKey;

module.exports = class Cache {
	constructor(options) {
		options = jsonop({
			id: (entity) => { return entity.id; },
			is: (entity, type) => { return entity.type === type; },
			entityOp: {},
			appOp: {}
		}, options || {});

		this.entities = {};
		this.indexes = {};
		this.queries = {};
		this.knowledge = {};
		this.app = {};
		this.keyToSlice = keyToSlice;
		this.sliceToKey = sliceToKey;

		/*
			map of types to index keys to index slices for fast lookup
			{
				items: {
					"items:createTime(tag:hidden)": { type: items, order: createTime, filters: { tag: hidden } }
				}
			}
		*/
		this._types = {};
		this._changeListeners = [];
		this._queryCallbacks = {};
		this._recentQueries = { entities: {}};
		this._querySeqNum = 0;
		this._id = options.id;
		this._is = options.is;
		this._options = options;
	}

	/*
		Get index keys and corresponding values that must include this entity,
		taking into account knowledge for the same key.
		@param entity
		@returns { key: value }
	*/
	getKeyValues(entity) {
		let keyValues = {};

		for (let type in this._types) {
			if (!this._is(entity, type)) { continue; }

			keyloop:
			for (let key in this._types[type].base) {
				let slice = this._types[type].base[key],
					value = entity[slice.order];

				if (typeof value === "undefined") { continue; }

				for (let prop in slice.filters) {
					if (entity[prop] !== slice.filters[prop]) {
						continue keyloop;
					}
				}
				if (this.knowledge[key].contains(value)) {
					keyValues[key] = { value, item: (slice.link || slice.join) ?
						{ [type]: entity } : entity };
				}
			}

			keyloop:
			for (let key in this._types[type].join) {
				let slice = this._types[type].join[key],
					base = this.entities[entity[slice.join[type]]],
					value = base[slice.order];

				if (typeof value === "undefined") { continue; }

				for (let prop in slice.filters) {
					if (entity[prop] !== slice.filters[prop]) {
						continue keyloop;
					}
				}
				if (this.knowledge[key].contains(value)) {
					keyValues[key] = { value, item: { [type]: entity } };
				}
			}

			// TODO: Implement link types. This is not currently
			// required in HeyNeighbor, so not targeted for first
			// release.
		}

		return keyValues;
	}

	/*
		Adds or updates an entity and updates indexes if necessary.
	*/
	putEntity(entity, excludeKeys) {
		let id = this._id(entity),
			changes = { entities: {}, knowledge: {} },
			oldKeyValues,
			newKeyValues;

		if (this.entities[id]) {
			oldKeyValues = this.getKeyValues(this.entities[id]);
			entity = jsonop(this.entities[id], entity, this.options.entityOp);
		}
		this.entities[id] = changes.entities[id] = entity;
		newKeyValues = this.getKeyValues(entity);

		if (oldKeyValues) {
			for (let key in oldKeyValues) {
				if (oldKeyValues[key].value !== newKeyValues[key].value) {
					this.indexes[key].delete(oldKeyValues[key].value);
					changes.knowledge[key] = changes.knowledge[key] || new RangeArray([]);
					changes.knowledge[key].add([oldKeyValues[key].value, oldKeyValues[key].value]);
				} else {
					delete newKeyValues[key];
				}
			}
		}

		if (excludeKeys) { for (let key of excludeKeys) {
			delete newKeyValues[key];
		}}

		for (let key in newKeyValues) {
			this.indexes[key].add(newKeyValues[key].item);
			changes.knowledge[key] = changes.knowledge[key] || new RangeArray([]);
			changes.knowledge[key].add([newKeyValues[key].value, newKeyValues[key].value]);
		}

		return entity;
	}

	deleteEntity(id) {
		let keyValues = this.getKeyValues(this.entities[id]),
			changes = { entities: {}, knowledge: {} };
		for (let key in keyValues) {
			if (
				keyValues[key].item === this.entities[id] ||
				keyValues[key].item[keyToSlice(key).type] === this.entities[id]
			) {
				this.indexes[key].delete(keyValues[key].value);
				changes.knowledge[key] = changes.knowledge[key] || new RangeArray([]);
				changes.knowledge[key].add([keyValues[key].value, keyValues[key].value]);
			} else {
				for (let type in keyValues[key].item) {
					if (keyValues[key].item[type] === this.entities[id]) {
						// TODO: Decide what to do with compound index entries
						// when one of the referred entities is deleted.

//						this.indexes[key].put({ [type]: null });
					}
				}
			}
		}
		delete this.entities[id];
	}

	putIndex(key, knowledge, items) {
		let slice = keyToSlice(key),
			addType = (comp, type) => {
				this._types[type] = this._types[type] || {
					base: {}, link: {}, join: {}
				};
				this._types[type][comp][key] = slice;
			};
		if (!this.knowledge[key]) {
			addType("base", slice.type);
			if (slice.link) { for (let type in slice.link) {
				addType("link", type);
			}}
			if (slice.join) { for (let type in slice.join) {
				addType("join", type);
			}}

			this.knowledge[key] = knowledge;
			this.indexes[key] = items;
			this.indexEntities(key);
		} else {
			for (let range of knowledge) {
				let rangeItems = items.slice(...range);
				this.indexes[key].splice(...range, rangeItems);
				this.knowledge[key].add(range);
			}
		}
		// insert all the items into entities, updating all indexes
		// except this one.
		if (slice.join || slice.link) {
			for (let item of items) {
				for (let name in item) {
					this.putEntity(item[name], [ key ]);
				}
			}
		} else {
			for (let item of items) {
				this.putEntity(item, [ key ]);
			}
		}

		if (this._queryCallbacks[key]) {
			for (let i in this._queryCallbacks[key]) {
				let { range, callback, watch } = this._queryCallbacks[key][i];
				if (!watch && ranges.subtract(this.knowledge[key]).length === 0) {
					// No gaps in knowledge; query is satisfied.
					this.query(key, range, callback);
					this.removeQueryCallback(key, i);
				}
			}
		}
	}

	/*
		Checks every entity for inclusion in a particular index. Called when a
		new index is created.
	*/
	indexEntities(key) {
		let slice = keyToSlice(key);
		for (let id in this.entities) {
			let entity = this.entities[id],
				value = entity[slice.order];
			if (typeof value !== "undefined") {
				this.indexes[key].add(entity);
			}
		}
		/* TODO: Rewrite for perf: use push() then sort() */
	}

	pruneEntities(/* deadline */) {
		/* TODO: Implement in a nonblocking fashion */
	}

	/*
		Make changes to this cache (setState).
	*/
	put(changes) {
		let allChanges = {};

		function mergeChanges(ch) {
			jsonop(allChanges.entities, ch.entities);
			allChanges.knowledge.merge(ch.knowledge);
		}

		mergeChanges(allChanges, changes);

		/* handle the addition of ranges of results */
		if (changes.knowledge) { for (let key in changes.knowledge) {
			let items = changes.indexes && changes.indexes[key] || [];

			if (!(changes.knowledge[key] instanceof RangeArray)) {
				changes.knowledge[key] =
					new RangeArray(changes.knowledge[key]);
			}
			if (!items) {
				// Todo: generalize for compound indexes:
				// the order (first argument to constructor)
				// should be an array of path segments (["room", "statusTime"])
				// rather than a string.

				// (slice.link || slice.join) ? [slice.type, slice.order] : [slice.order]

				items = new OrderedArray(keyToSlice(key).order, []);
			} else if (!(items instanceof OrderedArray)) {
				changes.indexes[key] = items =
					new OrderedArray(keyToSlice(key).order, changes.indexes[key]);
			}

			jsonop(allChanges, this.putIndex(key, changes.knowledge[key], items));
		}}

		/* handle the addition of individual entities */
		if (changes.entities) { for (let id in changes.entities) {
			if (changes.entities[id] === null) { jsonop(allChanges, this.deleteEntity(id)); }
			else { jsonop(allChanges, this.putEntity(changes.entities[id])); }
		}}

		if (changes.app) {
			this.app = jsonop(this.app, changes.app, this.options.appOp);
		}

		/* handle the addition of a query */
		if (changes.queries) { for (let key in changes.queries) {
			if (!this.queries[key]) {
				this.queries[key] = changes.queries[key];
			} else {
				for (let range in changes.queries[key]) {
					this.queries[key].add(range);
				}
			}
		}}

		/* Trigger query callbacks */
		if (allChanges.knowledge) { for (const key in allChanges.knowledge) {
			if (this._queryCallbacks[key]) {
				const { range, callback, watch } = this._queryCallbacks[key];
				if (watch) {
					// fire callback
				}
			}
		}}

		if (allChanges.entities) { for (const id in allChanges.entities) {
			if (this._queryCallbacks.entities[id]) {
				// TODO: fire, delete if not watching.
			}
		}}

		/* if there are too many entities, clean up */
		if (Object.keys(this.entities).length > this._options.maxEntities) {
			requestIdleCallback(this.pruneEntities.bind(this));
		}
		this._changeListeners.forEach(fn => fn(allChanges));
	}

	getItemsGaps(key, range, callback, watch) {
		let items, gaps, ranges = new RangeArray([ range ]);
		range.seqNum = this._querySeqNum++;

		/* Extract items and calculate gaps in knowledge */
		if (!this.knowledge[key]) {
			let { order } = keyToSlice(key);
			items = new OrderedArray(order, []);
			gaps = ranges;
		} else {
			items = this.indexes[key].slice(...range);
			gaps = (range.length === 3 ? items.getRange() : ranges)
				.difference(this.knowledge[key]);

			// If it is a before/after range, gaps will not show missing data at
			// start and end; we need to detect and correct this.

			if (range.length === 3 && items.length < range[1] + range[2]) {
				let pos = items.indexOf(range[0]);
				if (
					pos < range[1] &&
					!this.knowledge[key].hasStart()
				) { gaps.add([
					items.get(0)[items.order], range[1] - pos, 0
				]); }
				if (
					items.length - pos < range[2] &&
					!this.knowledge[key].hasEnd()
				) { gaps.add([
					items.get(items.length - 1)[items.order], 0,
					range[2] + pos - items.length
				]); }
			}
		}

		/* Insert “loading” placeholders corresponding to the gaps. */
		for (let gap of gaps) {
			if (gap.length === 2) {
				items.put({
					type: "loading", start: gap[0], end: gap[1],
					[items.order]: gap[1] === Infinity ? gap[1] : gap[0]
				});
			} else {
				items.put({
					type: "loading", start: gap[0], before: gap[1], after: gap[2],
					[items.order]: gap[0]
				}, !!items.after);
			}
		}

		/* Update the most recently accessed map */
		if (!this._recentQueries[key]) {
			this._recentQueries[key] = items.getRange();
		} else {
			this._recentQueries[key].put(items.getRange());
		}

		return { items, gaps };
	}

	query (key, range, callback) {
		let { items, gaps } = this.getItemsGaps(key, range);

		/* Make queries corresponding to the gaps. */
		if (gaps.length) {
			this.put({ queries: { [key]: gaps }});
			if (callback) {
				this._queryCallbacks[key] = this._queryCallbacks[key] || [];
				this._queryCallbacks[key].push({ range, callback, watch: false });
			}
		} else {
			if (callback) {
				setImmediate(() => callback(null, items));
			}
		}

		return items;
	}

	watch (key, range, callback) {
		if (!callback) { throw Error("ERR_WATCH_NO_CALLBACK"); }

		let { items, gaps } = getItemsGaps (key, range, callback, true);
		setImmediate(() => callback(null, items));

		this._queryCallbacks[key] = this._queryCallbacks[key] || [];
		let index = this._queryCallbacks[key].push({ range, callback, watch: true }) - 1;

		return () => this.removeQueryCallback(key, index)
	}

	removeQueryCallback(key, index) {
		this._queryCallbacks[key].splice(index, 1);
		if (this._queryCallbacks[key].length === 0) {
			delete this._queryCallbacks[key];
		}
	}

	onChange(fn) {
		this._changeListeners.push(fn);
	}

	get() {
		let args = Array.prototype.slice.call(arguments),
			pointer = this; // eslint-disable-line consistent-this
		while (typeof pointer === "object" && pointer !== null && args.length) {
			pointer = pointer[args.shift()];
		}

		return pointer;
	}

	getEntity(id, callback) {
		var result;
		this._recentQueries.entities[id] = this._querySeqNum++;
		if (this.entities && this.entities[id]) result = this.entities[id];

		if (result) {
			if (callback) callback(null, result);
			return result;
		} else {
			this.put({
				queries: {
					entities: {
						id: true
					}
				}
			});

			if (callback) {
				this._queryCallback.entities = this._queryCallback.entities || {};
				(this._queryCallback.entities = this._queryCallback.entities || []).push(callback);
			}

			return {type: "loading"};
		}
	}

	with(changes) {
		var cache = new Cache();
		cache.put({ app: jsonop(this.app, changes.app) });
		return cache;
	}
};
<|endoftext|>


<commit_before>var _ = require('lodash');
var di = require('di');
var Package = require('./Package');
var sortByDependency = require('./util/dependency-sort');
var validate = require('validate.js');
var Q = require('q');

/**
 * Create an instance of the Dgeni documentation generator, loading any packages passed in as a
 * parameter.
 * @param {Package[]} [packages] A collection of packages to load
 */
function Dgeni(packages) {
  this.packages = {};

  packages = packages || [];
  if ( !Array.isArray(packages) ) { throw new Error('packages must be an array'); }
  _.map(packages, this.package, this);
}

/**
 * @type {Package}
 */
Dgeni.Package = Package;

/**
 * Load a package into dgeni
 * @param  {Package|string} package              The package to load or the name of a new package to
 *                                               create.
 * @param  {Array.<Package|string>} dependencies A collection of dependencies for this package
 * @return {Package}                             The package that was loaded, to allow chaining
 */
Dgeni.prototype.package = function(package, dependencies) {

  if ( typeof package === 'string' ) { package = new Package(package, dependencies); }
  if ( !(package instanceof Package) ) { throw new Error('package must be an instance of Package'); }
  if ( this.packages[package.name] ) {
    throw new Error('The "' + package.name + '" package has already been loaded');
  }
  this.packages[package.name] = package;

  // Extract all inline packages and load them into dgeni;
  package.namedDependencies = package.dependencies.map(function(dependency) {
    if ( dependency instanceof Package ) {
      // Only load dependent package if not already loaded
      if ( !this.packages[dependency.name] ) { this.package(dependency); }
      return dependency.name;
    }
    return dependency;
  }, this);

  // Return the package to allow chaining
  return package;
};

/**
 * Configure the injector using the loaded packages.
 * It assigns the injector to a property on `this`.
 */
Dgeni.prototype.configureInjector = function() {

  if ( !this.injector ) {

    // Sort the packages by their dependency - ensures that services and configs are loaded in the
    // correct order
    var packages = sortByDependency(this.packages, 'namedDependencies');

    // Create a module containing basic shared services
    var dgeniConfig = {
      stopOnValidationError: true,
      stopOnProcessingError: true
    };
    var dgeniModule = new di.Module()
      .value('dgeni', dgeniConfig)
      .factory('log', require('./util/log'))
      .factory('getInjectables', require('./util/getInjectables'));

    // Create the dependency injection container, from all the packages' modules
    var modules = packages.map(function(package) { return package.module; });
    modules.unshift(dgeniModule);

    // Create the injector and
    var injector = this.injector = new di.Injector(modules);

    // Apply the config blocks
    packages.forEach(function(package) {
      package.configFns.forEach(function(configFn) {
        injector.invoke(configFn);
      });
    });

    // Get the collection of processors
    // We use a Map here so that we only get one of each processor name
    var processors = this.processors = {};
    packages.forEach(function(package) {
      package.processors.forEach(function(processorName) {
        var processor = injector.get(processorName);
        processor.name = processorName;
        processor.$package = package.name;

        // Ignore disabled processors
        if ( processor.$enabled !== false ) {
          processors[processorName] = processor;
        }
      });
    });
  }

  return this.injector;
};

/**
 * Generate the documentation using the loaded packages
 * @return {Promise} A promise to the generated documents
 */
Dgeni.prototype.generate = function() {

  this.configureInjector();

  var dgeniConfig = this.injector.get('dgeni');

  // Once we have configured everything sort the processors.
  // This allows the config blocks to modify the $runBefore and $runAfter
  // properties of processors.
  // (Crazy idea, I know, but useful for things like debugDumpProcessor)
  processors = sortByDependency(this.processors, '$runAfter', '$runBefore');

  var log = this.injector.get('log');
  var processingPromise = Q();

  // Apply the validations on each processor
  _.forEach(processors, function(processor) {
    processingPromise = processingPromise.then(function() {
      return validate.async(processor, processor.$validate);
    }).catch(function(errors) {
      log.error('Invalid Processor Configuration: ' + processor.name + ' (in "' + processor.$package + '" package)');
      log.error(errors);
      if ( dgeniConfig.stopOnValidationError ) { return Q.reject(errors); }
    });
  });

  // Process the docs
  var currentDocs = [];
  processingPromise = processingPromise.then(function() {
    return currentDocs;
  });

  _.forEach(processors, function(processor) {

    if( processor.$process ) {

      processingPromise = processingPromise.then(function(docs) {
        currentDocs = docs;
        log.info('running processor:', processor.name);
        return processor.$process(docs) || docs;

      }).catch(function(error) {

        error.message = 'Error running processor "' + processor.name + '":\n' + error.message;
        log.error(error.stack);
        if ( dgeniConfig.stopOnProcessingError ) { throw error; }
        return currentDocs;

      });

    }
  });

  processingPromise.catch(function(error) {
    log.error('Error processing docs:\n' + error.stack);
  });

  return processingPromise;
};

/**
 * @module Dgeni
 */
module.exports = Dgeni;
<commit_msg>docs(Dgeni): add documentation for `configureInjector()`<commit_after>var _ = require('lodash');
var di = require('di');
var Package = require('./Package');
var sortByDependency = require('./util/dependency-sort');
var validate = require('validate.js');
var Q = require('q');

/**
 * Create an instance of the Dgeni documentation generator, loading any packages passed in as a
 * parameter.
 * @param {Package[]} [packages] A collection of packages to load
 */
function Dgeni(packages) {
  this.packages = {};

  packages = packages || [];
  if ( !Array.isArray(packages) ) { throw new Error('packages must be an array'); }
  _.map(packages, this.package, this);
}

/**
 * @type {Package}
 */
Dgeni.Package = Package;

/**
 * Load a package into dgeni
 * @param  {Package|string} package              The package to load or the name of a new package to
 *                                               create.
 * @param  {Array.<Package|string>} dependencies A collection of dependencies for this package
 * @return {Package}                             The package that was loaded, to allow chaining
 */
Dgeni.prototype.package = function(package, dependencies) {

  if ( typeof package === 'string' ) { package = new Package(package, dependencies); }
  if ( !(package instanceof Package) ) { throw new Error('package must be an instance of Package'); }
  if ( this.packages[package.name] ) {
    throw new Error('The "' + package.name + '" package has already been loaded');
  }
  this.packages[package.name] = package;

  // Extract all inline packages and load them into dgeni;
  package.namedDependencies = package.dependencies.map(function(dependency) {
    if ( dependency instanceof Package ) {
      // Only load dependent package if not already loaded
      if ( !this.packages[dependency.name] ) { this.package(dependency); }
      return dependency.name;
    }
    return dependency;
  }, this);

  // Return the package to allow chaining
  return package;
};

/**
 * Configure the injector using the loaded packages.
 *
 * The injector is assigned to the `injector` property on `this`, which is used by the
 * `generate()` method. Subsequent calls to this method will just return the same injector.
 *
 * This method is useful in unit testing services and processors as it gives an easy way to
 * get hold of an instance of a ready instantiated component without having to load in all
 * the potential dependencies manually:
 *
 * ```
 * var Dgeni = require('dgeni');
 *
 * function getInjector() {
 *   var dgeni = new Dgeni();
 *   dgeni.package('testPackage', [require('dgeni-packages/base')])
 *     .factory('templateEngine', function dummyTemplateEngine() {});
 *   return dgeni.configureInjector();
 * };
 *
 * describe('someService', function() {
 *   var someService;
 *   beforeEach(function() {
 *     var injector = getInjector();
 *     someService = injector.get('someService');
 *   });
 *
 *   it("should do something", function() {
 *     someService.doSomething();
 *     ...
 *   });
 * });
 * ```
 */
Dgeni.prototype.configureInjector = function() {

  if ( !this.injector ) {

    // Sort the packages by their dependency - ensures that services and configs are loaded in the
    // correct order
    var packages = sortByDependency(this.packages, 'namedDependencies');

    // Create a module containing basic shared services
    var dgeniConfig = {
      stopOnValidationError: true,
      stopOnProcessingError: true
    };
    var dgeniModule = new di.Module()
      .value('dgeni', dgeniConfig)
      .factory('log', require('./util/log'))
      .factory('getInjectables', require('./util/getInjectables'));

    // Create the dependency injection container, from all the packages' modules
    var modules = packages.map(function(package) { return package.module; });
    modules.unshift(dgeniModule);

    // Create the injector and
    var injector = this.injector = new di.Injector(modules);

    // Apply the config blocks
    packages.forEach(function(package) {
      package.configFns.forEach(function(configFn) {
        injector.invoke(configFn);
      });
    });

    // Get the collection of processors
    // We use a Map here so that we only get one of each processor name
    var processors = this.processors = {};
    packages.forEach(function(package) {
      package.processors.forEach(function(processorName) {
        var processor = injector.get(processorName);
        processor.name = processorName;
        processor.$package = package.name;

        // Ignore disabled processors
        if ( processor.$enabled !== false ) {
          processors[processorName] = processor;
        }
      });
    });
  }

  return this.injector;
};

/**
 * Generate the documentation using the loaded packages
 * @return {Promise} A promise to the generated documents
 */
Dgeni.prototype.generate = function() {

  this.configureInjector();

  var dgeniConfig = this.injector.get('dgeni');

  // Once we have configured everything sort the processors.
  // This allows the config blocks to modify the $runBefore and $runAfter
  // properties of processors.
  // (Crazy idea, I know, but useful for things like debugDumpProcessor)
  processors = sortByDependency(this.processors, '$runAfter', '$runBefore');

  var log = this.injector.get('log');
  var processingPromise = Q();

  // Apply the validations on each processor
  _.forEach(processors, function(processor) {
    processingPromise = processingPromise.then(function() {
      return validate.async(processor, processor.$validate);
    }).catch(function(errors) {
      log.error('Invalid Processor Configuration: ' + processor.name + ' (in "' + processor.$package + '" package)');
      log.error(errors);
      if ( dgeniConfig.stopOnValidationError ) { return Q.reject(errors); }
    });
  });

  // Process the docs
  var currentDocs = [];
  processingPromise = processingPromise.then(function() {
    return currentDocs;
  });

  _.forEach(processors, function(processor) {

    if( processor.$process ) {

      processingPromise = processingPromise.then(function(docs) {
        currentDocs = docs;
        log.info('running processor:', processor.name);
        return processor.$process(docs) || docs;

      }).catch(function(error) {

        error.message = 'Error running processor "' + processor.name + '":\n' + error.message;
        log.error(error.stack);
        if ( dgeniConfig.stopOnProcessingError ) { throw error; }
        return currentDocs;

      });

    }
  });

  processingPromise.catch(function(error) {
    log.error('Error processing docs:\n' + error.stack);
  });

  return processingPromise;
};

/**
 * @module Dgeni
 */
module.exports = Dgeni;
<|endoftext|>


<commit_before>var stream   = require('stream')
  , util     = require('util')
  , request  = require('superagent')
  , _        = require('underscore')
  , uuid     = require('node-uuid')
  , oauth1a  = require('../auth/oauth1a')
  , PubSubConnection  = require('../auth/pubsub')
  , Message = require('../message')

  , Transform = stream.Transform

  , IG_API_URL = 'https://api.instagram.com/v1/'

  , IG_OBJECT_USER = 'user'

  , IG_OBJECT_TAG = 'tag'

  , IG_OBJECT_LOCATION = 'location'

  , IG_OBJECT_GEOGRAPHY = 'geography'

  , IG_DEFAULT_SUBSRIPTION_TYPE = IG_OBJECT_USER

;

function Instagram(opts){

  Transform.call(this, {
    objectMode: true
  });

  this.clientId     = opts.appId;
  this.clientSecret = opts.appSecret;
  this.subscriptionCallbackURL = opts.subscriptionCallbackURL;

  this.pubsub         = opts.pubsub || new PubSubConnection( this.clientSecret, this);
  this.uuid           = uuid.v1();

  this._tags = []
};
util.inherits(Instagram, Transform);

Instagram.prototype.stream = function(request, auth, socializr){
  if( !this._piped){
    this._piped = true;
    this.pubsub
    .pipe( this)
    .pipe( socializr);
    this.on('warning', socializr._onError.bind(socializr));
  }
  var data;
  this._tags = this._tags.concat( request.tags);
  for( var i in request.tags){
    data = {
      type: IG_OBJECT_TAG,
      tag: request.tags[i].toString().replace('#','')
    };
    this.subscribe( data, function(err,res){
      // console.log('++++++++ instagram',arguments);
    });
  }
  return this;
};

Instagram.prototype.auth = function(auth){
  auth._provider = this;
  return auth;
};

Instagram.prototype.subscribe = function(sub, cb){

  // var conn = new PubSubConnection( this.appSecret)
    // , verify_token = conn.verifyToken()
  // ;
  var verify_token = this.pubsub.verifyToken();
  // this.pubsubConnections[ verify_token] = conn;

  var data = {
        client_id: this.clientId,
        client_secret: this.clientSecret,
        object: sub.type || IG_DEFAULT_SUBSRIPTION_TYPE,
        aspect: 'media',
        verify_token: verify_token,
        callback_url: this.subscriptionCallbackURL
      }
  ;

  switch( sub.type){
    case IG_OBJECT_TAG:
      data = _.extend(data,{
        object_id: sub.tag
      });
      break;

    case IG_OBJECT_LOCATION:
      data = _.extend(data,{
        object_id: sub.location
      });
      break;

    case IG_OBJECT_GEOGRAPHY:
      data = _.extend(data,{
        lat: sub.lat,
        lng: sub.lng,
        radius: sub.radius
      });
      break;
  };

  this._apiRequest( 'subscriptions', 'POST', data, function(err,res){
    cb&&cb(err,res);
  });
};

Instagram.prototype.unsubscribe = function(type){
  this._apiRequest( 'subscriptions', 'DELETE', {
    client_id:     this.clientId,
    client_secret: this.clientSecret,
    object: type
  }, function(){

  });
};

Instagram.prototype._transform = function(chunk, encoding, done){

  // console.log("Instagram transform", chunk)

  var expected = 0
    , stream = this
    , obj = null
    , complete = _.after(chunk.length, function(){
      done();
    })
  ;
  for( var i in chunk){

    obj = chunk[i];
    switch( obj.object){
      case IG_OBJECT_TAG:
        this.getMediaByTag(chunk[i].object_id, function(err,res){
          console.log('media response',res.body);
          // var data = res.body.data;
          for( var j in data){

            stream.push(
              new Message(stream,data[j])
            );
            // console.log( data[j]);
          }
          complete();
        });

        break;
      default:
        complete();
        break;
    }
  }

};

Instagram.prototype.getMediaByTag = function(tag, cb){
  this._apiRequest( 'tags/' + tag + '/media/recent', 'GET', {
    client_id: this.clientId
  }, function(err,res){
    cb&&cb(err, res);
  });
};

Instagram.prototype.getFeed = function(token, cb){
  this._apiRequest( 'users/self/feed', 'GET', {
    access_token: token
  }, function(err,res){
    cb&&cb(err, res);
  });
};

Instagram.prototype.findMediaAtLocation = function(id){
  this._apiRequest( 'locations/' + id + '/media/recent', 'GET', {
  }, function(err,res){
    cb&&cb(err, res);
  });
};

Instagram.prototype.getLocations = function(lat, lng){
  this._apiRequest( 'locations/search', 'GET', {
    lat: '',
    lng: ''
  }, function(err,res){
    cb&&cb(err, res);
  });
};

Instagram.prototype.getMediaByLocation = function(id){
  this._apiRequest( 'locations/' + id + '/media/recent', 'GET', {
  }, function(err,res){
    cb&&cb(err, res);
  });
};

Instagram.prototype.getMediaByGeo = function(id){
  this._apiRequest( 'geographies/' + id + '/media/recent', 'GET', {
    client_id: this.clientId
  }, function(err,res){
    cb&&cb(err, res);
  });
};

Instagram.prototype._apiRequest = function(endpoint, method, data, cb){

  var url = IG_API_URL + endpoint
  ;

  switch( method.toUpperCase()){

    case 'GET':

      request.get(url)
        .query(data)
        .end( function(err,res){
          cb&&cb(err,res);
        });
      break;

    case 'POST':

      request.post(url)
        .type('form')
        .send(data)
        .end( function(err,res){
          cb&&cb(err,res);
        });

      break;

    case 'DELETE':

      request.del(url)
        .send(data)
        .end( function(err,res){
          cb&&cb(err,res);
        });
      break;
  }

};

module.exports = Instagram;
<commit_msg>Fixed bug in comment out the wrong line during cleanup<commit_after>var stream   = require('stream')
  , util     = require('util')
  , request  = require('superagent')
  , _        = require('underscore')
  , uuid     = require('node-uuid')
  , oauth1a  = require('../auth/oauth1a')
  , PubSubConnection  = require('../auth/pubsub')
  , Message = require('../message')

  , Transform = stream.Transform

  , IG_API_URL = 'https://api.instagram.com/v1/'

  , IG_OBJECT_USER = 'user'

  , IG_OBJECT_TAG = 'tag'

  , IG_OBJECT_LOCATION = 'location'

  , IG_OBJECT_GEOGRAPHY = 'geography'

  , IG_DEFAULT_SUBSRIPTION_TYPE = IG_OBJECT_USER

;

function Instagram(opts){

  Transform.call(this, {
    objectMode: true
  });

  this.clientId     = opts.appId;
  this.clientSecret = opts.appSecret;
  this.subscriptionCallbackURL = opts.subscriptionCallbackURL;

  this.pubsub         = opts.pubsub || new PubSubConnection( this.clientSecret, this);
  this.uuid           = uuid.v1();

  this._tags = []
};
util.inherits(Instagram, Transform);

Instagram.prototype.stream = function(request, auth, socializr){
  if( !this._piped){
    this._piped = true;
    this.pubsub
    .pipe( this)
    .pipe( socializr);
    this.on('warning', socializr._onError.bind(socializr));
  }
  var data;
  this._tags = this._tags.concat( request.tags);
  for( var i in request.tags){
    data = {
      type: IG_OBJECT_TAG,
      tag: request.tags[i].toString().replace('#','')
    };
    this.subscribe( data, function(err,res){
      // console.log('++++++++ instagram',arguments);
    });
  }
  return this;
};

Instagram.prototype.auth = function(auth){
  auth._provider = this;
  return auth;
};

Instagram.prototype.subscribe = function(sub, cb){

  // var conn = new PubSubConnection( this.appSecret)
    // , verify_token = conn.verifyToken()
  // ;
  var verify_token = this.pubsub.verifyToken();
  // this.pubsubConnections[ verify_token] = conn;

  var data = {
        client_id: this.clientId,
        client_secret: this.clientSecret,
        object: sub.type || IG_DEFAULT_SUBSRIPTION_TYPE,
        aspect: 'media',
        verify_token: verify_token,
        callback_url: this.subscriptionCallbackURL
      }
  ;

  switch( sub.type){
    case IG_OBJECT_TAG:
      data = _.extend(data,{
        object_id: sub.tag
      });
      break;

    case IG_OBJECT_LOCATION:
      data = _.extend(data,{
        object_id: sub.location
      });
      break;

    case IG_OBJECT_GEOGRAPHY:
      data = _.extend(data,{
        lat: sub.lat,
        lng: sub.lng,
        radius: sub.radius
      });
      break;
  };

  this._apiRequest( 'subscriptions', 'POST', data, function(err,res){
    cb&&cb(err,res);
  });
};

Instagram.prototype.unsubscribe = function(type){
  this._apiRequest( 'subscriptions', 'DELETE', {
    client_id:     this.clientId,
    client_secret: this.clientSecret,
    object: type
  }, function(){

  });
};

Instagram.prototype._transform = function(chunk, encoding, done){

  // console.log("Instagram transform", chunk)

  var expected = 0
    , stream = this
    , obj = null
    , complete = _.after(chunk.length, function(){
      done();
    })
  ;
  for( var i in chunk){

    obj = chunk[i];
    switch( obj.object){
      case IG_OBJECT_TAG:
        this.getMediaByTag(chunk[i].object_id, function(err,res){
          //console.log('media response',res.body);
          var data = res.body.data;
          for( var j in data){

            stream.push(
              new Message(stream,data[j])
            );
            // console.log( data[j]);
          }
          complete();
        });

        break;
      default:
        complete();
        break;
    }
  }

};

Instagram.prototype.getMediaByTag = function(tag, cb){
  this._apiRequest( 'tags/' + tag + '/media/recent', 'GET', {
    client_id: this.clientId
  }, function(err,res){
    cb&&cb(err, res);
  });
};

Instagram.prototype.getFeed = function(token, cb){
  this._apiRequest( 'users/self/feed', 'GET', {
    access_token: token
  }, function(err,res){
    cb&&cb(err, res);
  });
};

Instagram.prototype.findMediaAtLocation = function(id){
  this._apiRequest( 'locations/' + id + '/media/recent', 'GET', {
  }, function(err,res){
    cb&&cb(err, res);
  });
};

Instagram.prototype.getLocations = function(lat, lng){
  this._apiRequest( 'locations/search', 'GET', {
    lat: '',
    lng: ''
  }, function(err,res){
    cb&&cb(err, res);
  });
};

Instagram.prototype.getMediaByLocation = function(id){
  this._apiRequest( 'locations/' + id + '/media/recent', 'GET', {
  }, function(err,res){
    cb&&cb(err, res);
  });
};

Instagram.prototype.getMediaByGeo = function(id){
  this._apiRequest( 'geographies/' + id + '/media/recent', 'GET', {
    client_id: this.clientId
  }, function(err,res){
    cb&&cb(err, res);
  });
};

Instagram.prototype._apiRequest = function(endpoint, method, data, cb){

  var url = IG_API_URL + endpoint
  ;

  switch( method.toUpperCase()){

    case 'GET':

      request.get(url)
        .query(data)
        .end( function(err,res){
          cb&&cb(err,res);
        });
      break;

    case 'POST':

      request.post(url)
        .type('form')
        .send(data)
        .end( function(err,res){
          cb&&cb(err,res);
        });

      break;

    case 'DELETE':

      request.del(url)
        .send(data)
        .end( function(err,res){
          cb&&cb(err,res);
        });
      break;
  }

};

module.exports = Instagram;
<|endoftext|>


<commit_before>var _ = require('lodash');

var Const = require('./accessors/Const');
var InvalidFieldValue = require('./errors/InvalidFieldValue');

function Field(resource, name, value, opts) {
	opts = _.defaults({}, opts, {
		readable: true,
		writable: true,
		nullable: false,
	});
	this._resource = resource;
	this._name = name;
	this._accessor = toAccessor(value);
	this._readable = opts.readable;
	this._writable = opts.writable;
	this._nullable = opts.nullable;
}

Object.defineProperty(Field.prototype, 'resource', {
	get: function() { return this._resource; },
});

Object.defineProperty(Field.prototype, 'name', {
	get: function() { return this._name; },
});

Object.defineProperty(Field.prototype, 'readable', {
	get: function() { return this._readable; },
});

Object.defineProperty(Field.prototype, 'writable', {
	get: function() { return this._writable; },
});

Object.defineProperty(Field.prototype, 'nullable', {
	get: function() { return this._nullable; },
});

Field.prototype.adjustQuery = function(query, action) {
	var accessor = this._accessor;
	return accessor.adjustQuery.apply(accessor, arguments);
};

Field.prototype.serialize = function(transaction, object, callback) {
	var self = this;
	if (!this._readable) return callback(null);
	this._accessor.serialize(this, transaction, object, function(err, value) {
		if (err) return callback(err);
		if (_.isUndefined(value) && !this._nullable)
			return callback(new InvalidFieldValue(self, undefined));
		return callback(null, value);
	});
};

Field.prototype.deserialize = function(transaction, resdata, object, callback) {
	if (!this._writable) return callback(null, object);
	if (_.isUndefined(resdata) && !this._nullable)
		return callback(new InvalidFieldValue(this, undefined));
	this._accessor.deserialize(this, transaction, resdata, object, callback);
};

function toAccessor(object) {
	return isFieldAccessor(object) ? object : new Const(object);
}

function isFieldAccessor(object) {
	return object && object.serialize && object.deserialize;
}

module.exports = Field;
<commit_msg>Fixed typo<commit_after>var _ = require('lodash');

var Const = require('./accessors/Const');
var InvalidFieldValue = require('./errors/InvalidFieldValue');

function Field(resource, name, value, opts) {
	opts = _.defaults({}, opts, {
		readable: true,
		writable: true,
		nullable: false,
	});
	this._resource = resource;
	this._name = name;
	this._accessor = toAccessor(value);
	this._readable = opts.readable;
	this._writable = opts.writable;
	this._nullable = opts.nullable;
}

Object.defineProperty(Field.prototype, 'resource', {
	get: function() { return this._resource; },
});

Object.defineProperty(Field.prototype, 'name', {
	get: function() { return this._name; },
});

Object.defineProperty(Field.prototype, 'readable', {
	get: function() { return this._readable; },
});

Object.defineProperty(Field.prototype, 'writable', {
	get: function() { return this._writable; },
});

Object.defineProperty(Field.prototype, 'nullable', {
	get: function() { return this._nullable; },
});

Field.prototype.adjustQuery = function(query, action) {
	var accessor = this._accessor;
	return accessor.adjustQuery.apply(accessor, arguments);
};

Field.prototype.serialize = function(transaction, object, callback) {
	var self = this;
	if (!this._readable) return callback(null);
	this._accessor.serialize(this, transaction, object, function(err, value) {
		if (err) return callback(err);
		if (_.isUndefined(value) && !self._nullable)
			return callback(new InvalidFieldValue(self, undefined));
		return callback(null, value);
	});
};

Field.prototype.deserialize = function(transaction, resdata, object, callback) {
	if (!this._writable) return callback(null, object);
	if (_.isUndefined(resdata) && !this._nullable)
		return callback(new InvalidFieldValue(this, undefined));
	this._accessor.deserialize(this, transaction, resdata, object, callback);
};

function toAccessor(object) {
	return isFieldAccessor(object) ? object : new Const(object);
}

function isFieldAccessor(object) {
	return object && object.serialize && object.deserialize;
}

module.exports = Field;
<|endoftext|>