<commit_before>#!/usr/bin/env node

// increase the libuv threadpool size to 1.5x the number of logical CPUs.
process.env.UV_THREADPOOL_SIZE = Math.ceil(Math.max(4, require('os').cpus().length * 1.5));

process.title = 'tm2';

if (process.platform === 'win32') {
    // HOME is undefined on windows
    process.env.HOME = process.env.USERPROFILE;
    // Add custom library paths to the PATH
    process.env.PATH = path.join(__dirname,"node_modules/mapnik/lib/binding/");
}

var _ = require('underscore');
var qs = require('querystring');
var tm = require('./lib/tm');
var fs = require('fs');
var url = require('url');
var path = require('path');
var source = require('./lib/source');
var style = require('./lib/style');
var middleware = require('./lib/middleware');
var express = require('express');
var cors = require('cors');
var config = require('optimist')
    .config('config')
    .options('db', {
        describe: 'path to tm2 db',
        default: path.join(process.env.HOME, '.tilemill', 'v2', 'app.db')
    })
    .options('mapboxauth', {
        describe: 'URL to mapbox auth API',
        default: 'https://api.mapbox.com'
    })
    .options('port', {
        describe: 'Port to run tm2 on',
        default: '3000'
    })
    .argv;
tm.config(config);
var request = require('request');
var crypto = require('crypto');

var app = express();
app.use(express.bodyParser());
app.use(require('./lib/oauth'));
app.use(app.router);
app.use('/app', express.static(__dirname + '/app', { maxAge:3600e3 }));
app.use('/ext', express.static(__dirname + '/ext', { maxAge:3600e3 }));

// Check for an active export. If present, redirect to the export page
// effectively locking the application from use until export is complete.
function exporting(req, res, next) {
    tm.copytask(null, null, function(err, job) {
        if (err) {
            next(err);
        } else if (job && (req.path !== '/mbtiles' || req.query.id !== job.id)) {
            res.redirect('/mbtiles?id=' + job.id);
        } else {
            next();
        }
    });
};

app.param('style', middleware.auth, exporting, middleware.basemap, middleware.loadStyle);

app.param('source', middleware.auth, exporting, middleware.loadSource);

app.param('history', middleware.history);

app.put('/style.json', middleware.writeStyle, function(req, res, next) {
    res.send({
        _recache: false,
        mtime: req.style.data.mtime,
        background: req.style.data.background
    });
});

app.get('/:style(style.json)', function(req, res, next) {
    res.send(req.style.data);
});

app.get('/:style(style):history()', function(req, res, next) {
    res.set({'content-type':'text/html'});

    // identify user's OS for styling docs shortcuts
    var agent = function() {
        var agent = req.headers['user-agent'];
        if (agent.indexOf('Win') != -1) return 'windows';
        if (agent.indexOf('Mac') != -1) return 'mac';
        if (agent.indexOf('X11') != -1 || agent.indexOf('Linux') != -1) return 'linux';
        return 'mac'; // default to Mac.
    };

    try {
        var page = tm.templates.style({
            cwd: process.env.HOME,
            fontsRef: require('mapnik').fonts(),
            cartoRef: require('carto').tree.Reference.data,
            sources: [req.style._backend._source.data],
            style: req.style.data,
            history: req.history,
            basemap: req.basemap,
            user: tm.db._docs.user,
            test: 'test' in req.query,
            agent: agent()
        });
    } catch(err) {
        return next(new Error('style template: ' + err.message));
    }
    return res.send(page);
});

app.get('/:style(style|source)/:z(\\d+)/:x(\\d+)/:y(\\d+).grid.json', function(req, res, next) {
    var z = req.params.z | 0;
    var x = req.params.x | 0;
    var y = req.params.y | 0;
    req.style.getGrid(z,x,y, function(err, data, headers) {
        if (err && err.message === 'Tilesource not loaded') {
            return res.redirect(req.path);
        } else if (err) {
            return next(err);
        }
        headers['cache-control'] = 'max-age=3600';
        res.set(headers);
        return res.json(data);
    });
});

app.get('/:style(style|source)/:z(\\d+)/:x(\\d+)/:y(\\d+).:format([\\w\\.]+)', cors(), function(req, res, next) {
    var z = req.params.z | 0;
    var x = req.params.x | 0;
    var y = req.params.y | 0;
    var source = req.params.format === 'vector.pbf'
        ? req.style._backend._source
        : req.style;
    var done = function(err, data, headers) {
        if (err && err.message === 'Tilesource not loaded') {
            return res.redirect(req.path);
        } else if (err) {
            // Set errors cookie for this style.
            style.error(req.style.data.id, err);
            res.cookie('errors', _(style.error(req.style.data.id)).join('|'));
            return next(err);
        }

        // Set drawtime cookie for a given style.
        style.stats(req.style.data.id, 'drawtime', z, data._drawtime);
        res.cookie('drawtime', _(style.stats(req.style.data.id, 'drawtime'))
            .reduce(function(memo, stat, z) {
                memo.push([z,stat.min,stat.avg|0,stat.max].join('-'));
                return memo;
            }, []).join('.'));

        // Set srcbytes cookie for a given style.
        style.stats(req.style.data.id, 'srcbytes', z, data._srcbytes);
        res.cookie('srcbytes', _(style.stats(req.style.data.id, 'srcbytes')).
            reduce(function(memo, stat, z) {
                memo.push([z,stat.min,stat.avg|0,stat.max].join('-'));
                return memo;
            }, []).join('.'));

        // Clear out tile errors.
        res.cookie('errors', '');

        // If debug flag is set, reduce json data.
        if (done.format === 'json' && 'debug' in req.query) {
            data = _(data).reduce(function(memo, layer) {
                memo[layer.name] = {
                    features: layer.features.length,
                    jsonsize: JSON.stringify(layer).length
                };
                return memo;
            }, {});
        }

        headers['cache-control'] = 'max-age=3600';
        if (req.params.format === 'vector.pbf') {
            headers['content-encoding'] = 'deflate';
        }
        res.set(headers);
        return res.send(data);
    };
    if (req.params.format !== 'png') done.format = req.params.format;
    source.getTile(z,x,y, done);
});

app.get('/:style(style).xml', function(req, res, next) {
    res.set({'content-type':'text/xml'});
    return res.send(req.style._xml);
});

app.get('/:style(style).tm2z', function(req, res, next) {
    style.toPackage(req.style.data.id, res, function(err) {
        if (err) next(err);
        res.end();
    });
});

app.get('/upload', middleware.auth, function(req, res, next) {
    if (style.tmpid(req.query.styleid))
        return next(new Error('Style must be saved first'));
    if (typeof tm.db._docs.user.plan.tm2z == 'undefined' || !tm.db._docs.user.plan.tm2z)
        return next(new Error('You are not allowed access to tm2z uploads yet.'));

    style.upload({
        id: req.query.styleid,
        oauth: tm.db._docs.oauth,
        cache: tm._config.cache
    }, function(err){
        if (err) next(err);
        res.end();
    });
});

app.get('/:source(source).xml', function(req, res, next) {
    res.set({'content-type':'text/xml'});
    return res.send(req.source._xml);
});

app.get('/:source(source).mbtiles', function(req, res, next) {
    res.set({'content-type':'text/xml'});
    source.toMBTiles(req.source.data.id, res, function(err) {
        if (err) next(err);
        res.end();
    });
});

app.all('/mbtiles', function(req, res, next) {
    source.info(req.query.id, function(err, info) {
        if (err) return next(err);
        source.mbtiles(req.query.id, false, function(err, job) {
            if (err) return next(err);
            if (/application\/json/.test(req.headers.accept||'')) {
                res.send(job);
            } else {
                res.set({'content-type':'text/html'});
                res.send(tm.templates.export({
                    tm: tm,
                    job: job,
                    source: info
                }));
            }
        });
    });
});

app.all('/mbtiles.json', function(req, res, next) {
    if (req.method === 'DELETE') return tm.cleartask(function(err) {
        if (err) return next(err);
        res.send({});
    });
    source.info(req.query.id, function(err, info) {
        if (err) return next(err);
        source.mbtiles(req.query.id, req.method === 'PUT', function(err, job) {
            if (err) return next(err);
            res.send(job);
        });
    });
});

app.get('/:source(source):history()', function(req, res, next) {

    // identify user's OS for styling docs shortcuts
    var agent = function() {
        var agent = req.headers['user-agent'];
        if (agent.indexOf('Win') != -1) return 'windows';
        if (agent.indexOf('Mac') != -1) return 'mac';
        if (agent.indexOf('X11') != -1 || agent.indexOf('Linux') != -1) return 'linux';
        return 'mac'; // default to Mac.
    };

    res.set({'content-type':'text/html'});
    try {
        var page = tm.templates.source({
            tm: tm,
            cwd: process.env.HOME,
            remote: url.parse(req.query.id).protocol !== 'tmsource:',
            source: req.source.data,
            history: req.history,
            basemap: req.basemap,
            user: tm.db._docs.user,
            agent: agent()
        });
    } catch(err) {
        return next(new Error('source template: ' + err.message));
    }
    return res.send(page);
});

app.put('/source.json', middleware.writeSource, function(req, res, next) {
    res.send({
        mtime:req.source.data.mtime,
        vector_layers:req.source.data.vector_layers,
        _template:req.source.data._template
    });
});

app.get('/:source(source.json)', function(req, res, next) {
    res.send(req.source.data);
});

app.get('/browse*', function(req, res, next) {
    tm.dirfiles('/' + req.params[0], function(err, dirfiles) {
        if (err) return next(err);
        res.send(dirfiles);
    });
});

app.get('/thumb.png', function(req, res, next) {
    if (!req.query.id) return next(new Error('No id specified'));
    style.thumb(req.query.id, function(err, thumb) {
        if (err && err.message === 'Tile does not exist') {
            return res.send(err.toString(), 404);
        } else if (err) {
            return next(err);
        }
        var headers = {};
        headers['cache-control'] = 'max-age=3600';
        headers['content-type'] = 'image/png';
        res.set(headers);
        res.send(thumb);
    });
});

app.get('/font.png', function(req, res, next) {
    if (!req.query.id) return next(new Error('No id specified'));
    tm.font(req.query.id, function(err, buffer) {
        if (err) return next(err);
        var headers = {};
        headers['cache-control'] = 'max-age=3600';
        headers['content-type'] = 'image/png';
        res.set(headers);
        res.send(buffer);
    });
});

app.get('/app/lib.js', function(req, res, next) {
    res.set({
        'content-type':'application/javascript',
        'cache-control':'max-age=3600'
    });
    res.send(tm.templates.libjs({
        cartoRef: require('carto').tree.Reference.data
    }));
});

app.get('/new/style', exporting, middleware.writeStyle, function(req, res) {
    res.redirect('/style?id=' + req.style.data.id);
});

app.get('/new/source', exporting, middleware.writeSource, function(req, res, next) {
    res.redirect('/source?id=' + req.source.data.id + '#addlayer');
});

app.get('/', function(req, res, next) {
    res.redirect('/new/style');
});

app.del('/history/:type(style|source)', function(req, res, next) {
    if (!req.query.id) return next(new Error('No id specified'));
    tm.history(req.params.type,req.query.id, true);
    res.send(200);
});

app.use(function(err, req, res, next) {
    // Error on loading a tile, send 404.
    if (err && 'z' in req.params) return res.send(err.toString(), 404);

    console.error(err.stack);

    // Otherwise 500 for now.
    if (/application\/json/.test(req.headers.accept)) {
        res.set({'content-type':'application/javascript'});
        res.send({message:err.toString()}, 500);
    } else if (/text\/html/.test(req.headers.accept)) {
        res.send(tm.templates.error({ error:err }), 500);
    } else {
        res.send(err.toString(), 500);
    }
});

app.get('/geocode', middleware.auth, function(req, res, next) {
    var query = 'http://api.tiles.mapbox.com/v3/'+req.basemap.id+'/geocode/{query}.json';
    res.redirect(query.replace('{query}', req.query.search));
});

app.listen(config.port);
console.log('TM2 @ http://localhost:'+config.port+'/');
<commit_msg>geocoder needs basemap access<commit_after>#!/usr/bin/env node

// increase the libuv threadpool size to 1.5x the number of logical CPUs.
process.env.UV_THREADPOOL_SIZE = Math.ceil(Math.max(4, require('os').cpus().length * 1.5));

process.title = 'tm2';

if (process.platform === 'win32') {
    // HOME is undefined on windows
    process.env.HOME = process.env.USERPROFILE;
    // Add custom library paths to the PATH
    process.env.PATH = path.join(__dirname,"node_modules/mapnik/lib/binding/");
}

var _ = require('underscore');
var qs = require('querystring');
var tm = require('./lib/tm');
var fs = require('fs');
var url = require('url');
var path = require('path');
var source = require('./lib/source');
var style = require('./lib/style');
var middleware = require('./lib/middleware');
var express = require('express');
var cors = require('cors');
var config = require('optimist')
    .config('config')
    .options('db', {
        describe: 'path to tm2 db',
        default: path.join(process.env.HOME, '.tilemill', 'v2', 'app.db')
    })
    .options('mapboxauth', {
        describe: 'URL to mapbox auth API',
        default: 'https://api.mapbox.com'
    })
    .options('port', {
        describe: 'Port to run tm2 on',
        default: '3000'
    })
    .argv;
tm.config(config);
var request = require('request');
var crypto = require('crypto');

var app = express();
app.use(express.bodyParser());
app.use(require('./lib/oauth'));
app.use(app.router);
app.use('/app', express.static(__dirname + '/app', { maxAge:3600e3 }));
app.use('/ext', express.static(__dirname + '/ext', { maxAge:3600e3 }));

// Check for an active export. If present, redirect to the export page
// effectively locking the application from use until export is complete.
function exporting(req, res, next) {
    tm.copytask(null, null, function(err, job) {
        if (err) {
            next(err);
        } else if (job && (req.path !== '/mbtiles' || req.query.id !== job.id)) {
            res.redirect('/mbtiles?id=' + job.id);
        } else {
            next();
        }
    });
};

app.param('style', middleware.auth, exporting, middleware.basemap, middleware.loadStyle);

app.param('source', middleware.auth, exporting, middleware.loadSource);

app.param('history', middleware.history);

app.put('/style.json', middleware.writeStyle, function(req, res, next) {
    res.send({
        _recache: false,
        mtime: req.style.data.mtime,
        background: req.style.data.background
    });
});

app.get('/:style(style.json)', function(req, res, next) {
    res.send(req.style.data);
});

app.get('/:style(style):history()', function(req, res, next) {
    res.set({'content-type':'text/html'});

    // identify user's OS for styling docs shortcuts
    var agent = function() {
        var agent = req.headers['user-agent'];
        if (agent.indexOf('Win') != -1) return 'windows';
        if (agent.indexOf('Mac') != -1) return 'mac';
        if (agent.indexOf('X11') != -1 || agent.indexOf('Linux') != -1) return 'linux';
        return 'mac'; // default to Mac.
    };

    try {
        var page = tm.templates.style({
            cwd: process.env.HOME,
            fontsRef: require('mapnik').fonts(),
            cartoRef: require('carto').tree.Reference.data,
            sources: [req.style._backend._source.data],
            style: req.style.data,
            history: req.history,
            basemap: req.basemap,
            user: tm.db._docs.user,
            test: 'test' in req.query,
            agent: agent()
        });
    } catch(err) {
        return next(new Error('style template: ' + err.message));
    }
    return res.send(page);
});

app.get('/:style(style|source)/:z(\\d+)/:x(\\d+)/:y(\\d+).grid.json', function(req, res, next) {
    var z = req.params.z | 0;
    var x = req.params.x | 0;
    var y = req.params.y | 0;
    req.style.getGrid(z,x,y, function(err, data, headers) {
        if (err && err.message === 'Tilesource not loaded') {
            return res.redirect(req.path);
        } else if (err) {
            return next(err);
        }
        headers['cache-control'] = 'max-age=3600';
        res.set(headers);
        return res.json(data);
    });
});

app.get('/:style(style|source)/:z(\\d+)/:x(\\d+)/:y(\\d+).:format([\\w\\.]+)', cors(), function(req, res, next) {
    var z = req.params.z | 0;
    var x = req.params.x | 0;
    var y = req.params.y | 0;
    var source = req.params.format === 'vector.pbf'
        ? req.style._backend._source
        : req.style;
    var done = function(err, data, headers) {
        if (err && err.message === 'Tilesource not loaded') {
            return res.redirect(req.path);
        } else if (err) {
            // Set errors cookie for this style.
            style.error(req.style.data.id, err);
            res.cookie('errors', _(style.error(req.style.data.id)).join('|'));
            return next(err);
        }

        // Set drawtime cookie for a given style.
        style.stats(req.style.data.id, 'drawtime', z, data._drawtime);
        res.cookie('drawtime', _(style.stats(req.style.data.id, 'drawtime'))
            .reduce(function(memo, stat, z) {
                memo.push([z,stat.min,stat.avg|0,stat.max].join('-'));
                return memo;
            }, []).join('.'));

        // Set srcbytes cookie for a given style.
        style.stats(req.style.data.id, 'srcbytes', z, data._srcbytes);
        res.cookie('srcbytes', _(style.stats(req.style.data.id, 'srcbytes')).
            reduce(function(memo, stat, z) {
                memo.push([z,stat.min,stat.avg|0,stat.max].join('-'));
                return memo;
            }, []).join('.'));

        // Clear out tile errors.
        res.cookie('errors', '');

        // If debug flag is set, reduce json data.
        if (done.format === 'json' && 'debug' in req.query) {
            data = _(data).reduce(function(memo, layer) {
                memo[layer.name] = {
                    features: layer.features.length,
                    jsonsize: JSON.stringify(layer).length
                };
                return memo;
            }, {});
        }

        headers['cache-control'] = 'max-age=3600';
        if (req.params.format === 'vector.pbf') {
            headers['content-encoding'] = 'deflate';
        }
        res.set(headers);
        return res.send(data);
    };
    if (req.params.format !== 'png') done.format = req.params.format;
    source.getTile(z,x,y, done);
});

app.get('/:style(style).xml', function(req, res, next) {
    res.set({'content-type':'text/xml'});
    return res.send(req.style._xml);
});

app.get('/:style(style).tm2z', function(req, res, next) {
    style.toPackage(req.style.data.id, res, function(err) {
        if (err) next(err);
        res.end();
    });
});

app.get('/upload', middleware.auth, function(req, res, next) {
    if (style.tmpid(req.query.styleid))
        return next(new Error('Style must be saved first'));
    if (typeof tm.db._docs.user.plan.tm2z == 'undefined' || !tm.db._docs.user.plan.tm2z)
        return next(new Error('You are not allowed access to tm2z uploads yet.'));

    style.upload({
        id: req.query.styleid,
        oauth: tm.db._docs.oauth,
        cache: tm._config.cache
    }, function(err){
        if (err) next(err);
        res.end();
    });
});

app.get('/:source(source).xml', function(req, res, next) {
    res.set({'content-type':'text/xml'});
    return res.send(req.source._xml);
});

app.get('/:source(source).mbtiles', function(req, res, next) {
    res.set({'content-type':'text/xml'});
    source.toMBTiles(req.source.data.id, res, function(err) {
        if (err) next(err);
        res.end();
    });
});

app.all('/mbtiles', function(req, res, next) {
    source.info(req.query.id, function(err, info) {
        if (err) return next(err);
        source.mbtiles(req.query.id, false, function(err, job) {
            if (err) return next(err);
            if (/application\/json/.test(req.headers.accept||'')) {
                res.send(job);
            } else {
                res.set({'content-type':'text/html'});
                res.send(tm.templates.export({
                    tm: tm,
                    job: job,
                    source: info
                }));
            }
        });
    });
});

app.all('/mbtiles.json', function(req, res, next) {
    if (req.method === 'DELETE') return tm.cleartask(function(err) {
        if (err) return next(err);
        res.send({});
    });
    source.info(req.query.id, function(err, info) {
        if (err) return next(err);
        source.mbtiles(req.query.id, req.method === 'PUT', function(err, job) {
            if (err) return next(err);
            res.send(job);
        });
    });
});

app.get('/:source(source):history()', function(req, res, next) {

    // identify user's OS for styling docs shortcuts
    var agent = function() {
        var agent = req.headers['user-agent'];
        if (agent.indexOf('Win') != -1) return 'windows';
        if (agent.indexOf('Mac') != -1) return 'mac';
        if (agent.indexOf('X11') != -1 || agent.indexOf('Linux') != -1) return 'linux';
        return 'mac'; // default to Mac.
    };

    res.set({'content-type':'text/html'});
    try {
        var page = tm.templates.source({
            tm: tm,
            cwd: process.env.HOME,
            remote: url.parse(req.query.id).protocol !== 'tmsource:',
            source: req.source.data,
            history: req.history,
            basemap: req.basemap,
            user: tm.db._docs.user,
            agent: agent()
        });
    } catch(err) {
        return next(new Error('source template: ' + err.message));
    }
    return res.send(page);
});

app.put('/source.json', middleware.writeSource, function(req, res, next) {
    res.send({
        mtime:req.source.data.mtime,
        vector_layers:req.source.data.vector_layers,
        _template:req.source.data._template
    });
});

app.get('/:source(source.json)', function(req, res, next) {
    res.send(req.source.data);
});

app.get('/browse*', function(req, res, next) {
    tm.dirfiles('/' + req.params[0], function(err, dirfiles) {
        if (err) return next(err);
        res.send(dirfiles);
    });
});

app.get('/thumb.png', function(req, res, next) {
    if (!req.query.id) return next(new Error('No id specified'));
    style.thumb(req.query.id, function(err, thumb) {
        if (err && err.message === 'Tile does not exist') {
            return res.send(err.toString(), 404);
        } else if (err) {
            return next(err);
        }
        var headers = {};
        headers['cache-control'] = 'max-age=3600';
        headers['content-type'] = 'image/png';
        res.set(headers);
        res.send(thumb);
    });
});

app.get('/font.png', function(req, res, next) {
    if (!req.query.id) return next(new Error('No id specified'));
    tm.font(req.query.id, function(err, buffer) {
        if (err) return next(err);
        var headers = {};
        headers['cache-control'] = 'max-age=3600';
        headers['content-type'] = 'image/png';
        res.set(headers);
        res.send(buffer);
    });
});

app.get('/app/lib.js', function(req, res, next) {
    res.set({
        'content-type':'application/javascript',
        'cache-control':'max-age=3600'
    });
    res.send(tm.templates.libjs({
        cartoRef: require('carto').tree.Reference.data
    }));
});

app.get('/new/style', exporting, middleware.writeStyle, function(req, res) {
    res.redirect('/style?id=' + req.style.data.id);
});

app.get('/new/source', exporting, middleware.writeSource, function(req, res, next) {
    res.redirect('/source?id=' + req.source.data.id + '#addlayer');
});

app.get('/', function(req, res, next) {
    res.redirect('/new/style');
});

app.del('/history/:type(style|source)', function(req, res, next) {
    if (!req.query.id) return next(new Error('No id specified'));
    tm.history(req.params.type,req.query.id, true);
    res.send(200);
});

app.use(function(err, req, res, next) {
    // Error on loading a tile, send 404.
    if (err && 'z' in req.params) return res.send(err.toString(), 404);

    console.error(err.stack);

    // Otherwise 500 for now.
    if (/application\/json/.test(req.headers.accept)) {
        res.set({'content-type':'application/javascript'});
        res.send({message:err.toString()}, 500);
    } else if (/text\/html/.test(req.headers.accept)) {
        res.send(tm.templates.error({ error:err }), 500);
    } else {
        res.send(err.toString(), 500);
    }
});

app.get('/geocode', middleware.auth, middleware.basemap, function(req, res, next) {
    var query = 'http://api.tiles.mapbox.com/v3/'+req.basemap.id+'/geocode/{query}.json';
    res.redirect(query.replace('{query}', req.query.search));
});

app.listen(config.port);
console.log('TM2 @ http://localhost:'+config.port+'/');
<|endoftext|>


<commit_before>'use strict';

var _ = require('lodash'),
    cache = require('./data/cache'),
    LRU = require('lru-cache'),
    Updater = require('./lib/updater');

var updateInterval = 7 * 24 * 60 * 60 * 1000; // 7 days

function finalizeResult(result) {
  if(result.uaInfoUrl !== 'unknown') {
    result.uaInfoUrl = 'http://user-agent-string.info' + result.uaInfoUrl;
  }

  if(result.uaIcon !== 'unknown') {
    result.uaIcon = 'http://user-agent-string.info/pub/img/ua/' + result.uaIcon;
  }

  if(result.osIcon !== 'unknown') {
    result.osIcon = 'http://user-agent-string.info/pub/img/os/' + result.osIcon;
  }

  if(result.deviceIcon !== 'unknown') {
    result.deviceIcon = 'http://user-agent-string.info/pub/img/device/' + result.deviceIcon;
  }

  if(result.deviceInfoUrl !== 'unknown') {
    result.deviceInfoUrl = 'http://user-agent-string.info' + result.deviceInfoUrl;
  }

  return result;
}

exports.parse = function(userAgent) {
  var result = {
    type: 'unknown',
    uaFamily: 'unknown',
    uaName: 'unknown',
    uaUrl: 'unknown',
    uaCompany: 'unknown',
    uaCompanyUrl: 'unknown',
    uaIcon: 'unknown.png',
    uaInfoUrl: 'unknown',
    osFamily: 'unknown',
    osName: 'unknown',
    osUrl: 'unknown',
    osCompany: 'unknown',
    osCompanyUrl: 'unknown',
    osIcon: 'unknown.png',
    deviceType: 'unknown',
    deviceIcon: 'unknown.png',
    deviceInfoUrl: 'unknown',
  };

  for(var i = 0; i < cache.robots.order.length; i++) {
    var robotId = cache.robots.order[i];
    var robot = cache.robots[robotId];

    if(robot.userAgent === userAgent) {
      result.type = 'Robot';
      result = _.extend(result, robot.metadata);
      _.extend(result, cache.device['1']);

      return finalizeResult(result);
    }
  }

  var osId;
  for(i = 0; i < cache.browserReg.order.length; i++) {
    var browserRegId = cache.browserReg.order[i];
    var browserReg = cache.browserReg[browserRegId];

    var matches = userAgent.match(browserReg.regexp);
    if(matches) {
      var browser = cache.browser[browserReg.browserId];
      if(browser) {
        result = _.extend(result, browser.metadata);

        var browserType = cache.browserType[browser.typeId];
        if(browserType) {
          result.type = browserType;
        }

        result.uaName = browser.metadata.uaFamily;
        if(matches[1]) {
          result.uaName += ' ' + matches[1];
        }
      }

      osId = cache.browserOs[browserReg.browserId];

      break;
    }
  }

  if(!osId) {
    for(i = 0; i < cache.osReg.order.length; i++) {
      var osRegId = cache.osReg.order[i];
      var osReg = cache.osReg[osRegId];

      if(osReg.regexp.test(userAgent)) {
        osId = osReg.osId;
        break;
      }
    }
  }

  if(osId) {
    var os = cache.os[osId];
    if(os) {
      result = _.extend(result, os);
    }
  }

  var device;
  if(result.type === 'Robot') {
    device = cache.device['1'];
  } else {
    for(i = 0; i < cache.deviceReg.order.length; i++) {
      var deviceRegId = cache.deviceReg.order[i];
      var deviceReg = cache.deviceReg[deviceRegId];

      if(deviceReg.regexp.test(userAgent)) {
        device = cache.device[deviceReg.deviceId];
        break;
      }
    }
  }

  if(!device) {
    if(['Other', 'Library', 'Validator', 'Useragent Anonymizer'].indexOf(result.type) !==  -1) {
      device = cache.device['1'];
    } else if(['Mobile Browser', 'Wap Browser'].indexOf(result.type) !==  -1) {
      device = cache.device['3'];
    } else {
      device = cache.device['2'];
    }
  }

  if(device) {
    result = _.extend(result, device);
  }

  return finalizeResult(result);
};

var lookupCache = LRU({
  max: 5000,
  maxAge: updateInterval,
});

exports.lookup = function(userAgent) {
  var cached = lookupCache.get(userAgent);
  if(!cached) {
    cached = exports.parse(userAgent);
    lookupCache.set(userAgent, cached);
  }

  return cached;
};

function updateData(callback) {
  new Updater(updateInterval, function(error, newCache) {
    if(!error) {
      cache = newCache;
    }

    if(callback) {
      callback(error);
    }
  });
}

updateData();
var interval = setInterval(updateData, updateInterval);
interval.unref();
<commit_msg>Fix REDOS<commit_after>'use strict';

var _ = require('lodash'),
    cache = require('./data/cache'),
    LRU = require('lru-cache'),
    Updater = require('./lib/updater');

var updateInterval = 7 * 24 * 60 * 60 * 1000; // 7 days

function finalizeResult(result) {
  if(result.uaInfoUrl !== 'unknown') {
    result.uaInfoUrl = 'http://user-agent-string.info' + result.uaInfoUrl;
  }

  if(result.uaIcon !== 'unknown') {
    result.uaIcon = 'http://user-agent-string.info/pub/img/ua/' + result.uaIcon;
  }

  if(result.osIcon !== 'unknown') {
    result.osIcon = 'http://user-agent-string.info/pub/img/os/' + result.osIcon;
  }

  if(result.deviceIcon !== 'unknown') {
    result.deviceIcon = 'http://user-agent-string.info/pub/img/device/' + result.deviceIcon;
  }

  if(result.deviceInfoUrl !== 'unknown') {
    result.deviceInfoUrl = 'http://user-agent-string.info' + result.deviceInfoUrl;
  }

  return result;
}

exports.parse = function(userAgent) {
  var result = {
    type: 'unknown',
    uaFamily: 'unknown',
    uaName: 'unknown',
    uaUrl: 'unknown',
    uaCompany: 'unknown',
    uaCompanyUrl: 'unknown',
    uaIcon: 'unknown.png',
    uaInfoUrl: 'unknown',
    osFamily: 'unknown',
    osName: 'unknown',
    osUrl: 'unknown',
    osCompany: 'unknown',
    osCompanyUrl: 'unknown',
    osIcon: 'unknown.png',
    deviceType: 'unknown',
    deviceIcon: 'unknown.png',
    deviceInfoUrl: 'unknown',
  };

  /* Many patterns are vulnerable to REDOS. */
  var MAX_REASONABLE_LENGTH = 1000;
  if (MAX_REASONABLE_LENGTH < userAgent.length) {
    return finalizeResult(result);
  }

  for(var i = 0; i < cache.robots.order.length; i++) {
    var robotId = cache.robots.order[i];
    var robot = cache.robots[robotId];

    if(robot.userAgent === userAgent) {
      result.type = 'Robot';
      result = _.extend(result, robot.metadata);
      _.extend(result, cache.device['1']);

      return finalizeResult(result);
    }
  }

  var osId;
  for(i = 0; i < cache.browserReg.order.length; i++) {
    var browserRegId = cache.browserReg.order[i];
    var browserReg = cache.browserReg[browserRegId];

    var matches = userAgent.match(browserReg.regexp);
    if(matches) {
      var browser = cache.browser[browserReg.browserId];
      if(browser) {
        result = _.extend(result, browser.metadata);

        var browserType = cache.browserType[browser.typeId];
        if(browserType) {
          result.type = browserType;
        }

        result.uaName = browser.metadata.uaFamily;
        if(matches[1]) {
          result.uaName += ' ' + matches[1];
        }
      }

      osId = cache.browserOs[browserReg.browserId];

      break;
    }
  }

  if(!osId) {
    for(i = 0; i < cache.osReg.order.length; i++) {
      var osRegId = cache.osReg.order[i];
      var osReg = cache.osReg[osRegId];

      if(osReg.regexp.test(userAgent)) {
        osId = osReg.osId;
        break;
      }
    }
  }

  if(osId) {
    var os = cache.os[osId];
    if(os) {
      result = _.extend(result, os);
    }
  }

  var device;
  if(result.type === 'Robot') {
    device = cache.device['1'];
  } else {
    for(i = 0; i < cache.deviceReg.order.length; i++) {
      var deviceRegId = cache.deviceReg.order[i];
      var deviceReg = cache.deviceReg[deviceRegId];

      if(deviceReg.regexp.test(userAgent)) {
        device = cache.device[deviceReg.deviceId];
        break;
      }
    }
  }

  if(!device) {
    if(['Other', 'Library', 'Validator', 'Useragent Anonymizer'].indexOf(result.type) !==  -1) {
      device = cache.device['1'];
    } else if(['Mobile Browser', 'Wap Browser'].indexOf(result.type) !==  -1) {
      device = cache.device['3'];
    } else {
      device = cache.device['2'];
    }
  }

  if(device) {
    result = _.extend(result, device);
  }

  return finalizeResult(result);
};

var lookupCache = LRU({
  max: 5000,
  maxAge: updateInterval,
});

exports.lookup = function(userAgent) {
  var cached = lookupCache.get(userAgent);
  if(!cached) {
    cached = exports.parse(userAgent);
    lookupCache.set(userAgent, cached);
  }

  return cached;
};

function updateData(callback) {
  new Updater(updateInterval, function(error, newCache) {
    if(!error) {
      cache = newCache;
    }

    if(callback) {
      callback(error);
    }
  });
}

updateData();
var interval = setInterval(updateData, updateInterval);
interval.unref();
<|endoftext|>


<commit_before>var drjoint = (function(){
'use strict';

  var destinationDB;
  var sourceDB;

  var createDBSchema = function(data){

    var dbSchema = $('

');
    dbSchema.addClass('db-container');
    var fieldElement;

    // render table names
    for (var table in data) {
      dbSchema.append('
' + table + '
');
      dbSchema.attr('data-table-acc', table);
      fieldElement = $('

');
      fieldElement.appendTo(dbSchema);
      $.each(data[table], function(index, field){
        var fieldLabel = $('

').attr({
          'data-table': table,
          'data-field': field
        }).addClass('db-field').on('click', function(){
          var that = $(this);
          if (that.hasClass('added')) {
            return;
          }
          that.closest('.db-container').find('.selected').removeClass('selected');
          that.toggleClass('selected');
          checkSelected();
        });

        fieldLabel.append($('

').addClass('remove-button').on('click', function(){
          var that = $(this);
          var field = that.closest('.db-field');

          field.removeClass('added', 'selected');
          field.find('.field-label').text(field.attr('data-field'));
          var linkedField = $('[data-linked-to="' + field.attr('data-table') + '.' + field.attr('data-field') + '"]');
          linkedField.removeClass('added', 'selected');
          linkedField.find('.field-label').text(linkedField.attr('data-field'));

        }));
        fieldLabel.append($('

').addClass('field-label').text(field));

        fieldElement.append(
          fieldLabel
        );
      });
    }

    dbSchema.accordion({
      active: false,
      collapsible: true,
      heightStyle: 'content'
    });
    return dbSchema;
  };

  $.get('sql/prestashop.json', function (data) {
    destinationDB = createDBSchema(data);
    destinationDB.css({
      float: 'right'
    });
    destinationDB.appendTo($('#content'));
    $('#destination-search').on('keyup', searchAction(destinationDB));

  });

  $.get('sql/opencart.json', function (data) {
    sourceDB = createDBSchema(data);
    sourceDB.appendTo($('#content'));
    $('#source-search').on('keyup', searchAction(sourceDB));
  });

  // nanoajax.ajax('sql/prestashop.json', function (code, responseText) {
  //   var data = JSON.parse(responseText);
  //   document.body.innerHTML += '
prestashop tables: ' + Object.keys(data).length;
  // });

  var checkSelected = function(){
    var selected = $('.selected');
    if (selected.length === 2) {
      selected.addClass('added');

      var first = $(selected[0]);
      var second = $(selected[1]);
      var firstText = first.find('.field-label').text();
      var secondText = second.find('.field-label').text();
      var firstLinkedTo = second.attr('data-table')+'.' + second.attr('data-field');
      var secondLinkedTo = first.attr('data-table')+'.' + first.attr('data-field');
      first.find('.field-label').html(firstText + ' linked to ' + firstLinkedTo + '');
      first.attr('data-linked-to', firstLinkedTo);
      second.attr('data-linked-to', secondLinkedTo);
      second.find('.field-label').html(secondText + ' linked to ' + secondLinkedTo + '');
      selected.removeClass('selected');
    }
  };

  // search elements
  var searchAction = function(db) {
    return function() {
      var searchterm = $(this).val();
      db.find('.hidden').removeClass('hidden');
      var tables = db.children();
      $.each(tables, function(ind, el){
        if ($(el).text().indexOf(searchterm) === -1) {
          $(el).addClass('hidden');
        }
      });
    };
  };
})();
<commit_msg>:curry: download result json<commit_after>var drjoint = (function(){
'use strict';

  var destinationDB;
  var sourceDB;

  var createDBSchema = function(data){

    var dbSchema = $('

');
    dbSchema.addClass('db-container');
    var fieldElement;

    // render table names
    for (var table in data) {
      dbSchema.append('
' + table + '
');
      dbSchema.attr('data-table-acc', table);
      fieldElement = $('

');
      fieldElement.appendTo(dbSchema);
      $.each(data[table], function(index, field){
        var fieldLabel = $('

').attr({
          'data-table': table,
          'data-field': field
        }).addClass('db-field').on('click', function(){
          var that = $(this);
          if (that.hasClass('added')) {
            return;
          }
          that.closest('.db-container').find('.selected').removeClass('selected');
          that.toggleClass('selected');
          checkSelected();
        });

        fieldLabel.append($('

').addClass('remove-button').on('click', function(){
          var that = $(this);
          var field = that.closest('.db-field');

          field.removeClass('added', 'selected');
          field.find('.field-label').text(field.attr('data-field'));
          var linkedField = $('[data-linked-to="' + field.attr('data-table') + '.' + field.attr('data-field') + '"]');
          linkedField.removeClass('added', 'selected');
          linkedField.find('.field-label').text(linkedField.attr('data-field'));

        }));
        fieldLabel.append($('

').addClass('field-label').text(field));

        fieldElement.append(
          fieldLabel
        );
      });
    }

    dbSchema.accordion({
      active: false,
      collapsible: true,
      heightStyle: 'content'
    });
    return dbSchema;
  };

  $.get('sql/prestashop.json', function (data) {
    destinationDB = createDBSchema(data);
    destinationDB.css({
      float: 'right'
    });
    destinationDB.appendTo($('#content'));
    $('#destination-search').on('keyup', searchAction(destinationDB));

  });

  $.get('sql/opencart.json', function (data) {
    sourceDB = createDBSchema(data);
    sourceDB.appendTo($('#content'));
    $('#source-search').on('keyup', searchAction(sourceDB));
  });

  var checkSelected = function(){
    var selected = $('.selected');
    if (selected.length === 2) {
      selected.addClass('added');

      var first = $(selected[0]);
      var second = $(selected[1]);
      var firstText = first.find('.field-label').text();
      var secondText = second.find('.field-label').text();
      var firstLinkedTo = second.attr('data-table')+'.' + second.attr('data-field');
      var secondLinkedTo = first.attr('data-table')+'.' + first.attr('data-field');
      first.find('.field-label').html(firstText + ' linked to ' + firstLinkedTo + '');
      first.attr('data-linked-to', firstLinkedTo);
      second.attr('data-linked-to', secondLinkedTo);
      second.find('.field-label').html(secondText + ' linked to ' + secondLinkedTo + '');
      selected.removeClass('selected');
    }
  };

  // search elements
  var searchAction = function(db) {
    return function() {
      var searchterm = $(this).val();
      db.find('.hidden').removeClass('hidden');
      var tables = db.children();
      $.each(tables, function(ind, el){
        if ($(el).text().indexOf(searchterm) === -1) {
          $(el).addClass('hidden');
        }
      });
    };
  };

  var generate = function(){
    var output = {};
    var nodes = sourceDB.find('.added');
    $.each(nodes, function(index, node){
      node = $(node);
      output['`' + node.attr('data-table')+'`.`' + node.attr('data-field') + '`'] = node.attr('data-linked-to');
    });

    return output;
  };

  $('#generate-button').on('click', function(){
    var output = generate();
      var element = $('')
        .attr('href', 'data:text/plain;charset=utf-8,' + encodeURIComponent(JSON.stringify(output)))
        .attr('download', 'output.txt')
        .css('display', 'none')
        .appendTo(document.body);

        console.log(element);
      element[0].click();
      //element.remove();
  });
})();
<|endoftext|>


<commit_before>module.exports = Swarm

var EventEmitter = require('events').EventEmitter
var inherits = require('inherits')
var net = require('net') // or chrome-net
var portfinder = require('portfinder') // or chrome-portfinder
var once = require('once')
var Wire = require('bittorrent-protocol')

// Use random port above 1024
portfinder.basePort = Math.floor(Math.random() * 60000) + 1025

var MAX_SIZE = 100
var HANDSHAKE_TIMEOUT = 5000
var RECONNECT_WAIT = [1000, 5000, 15000, 30000, 60000, 120000, 300000, 600000]

/**
 * Peer
 * ====
 * A peer in the swarm. Comprised of a `net.Socket` and a `Wire`.
 *
 * @param {string} addr
 */
function Peer (addr) {
  this.addr = addr

  this.conn = null
  this.wire = null

  this.timeout = null
  this.retries = 0
}

/**
 * Called once the peer's `conn` has connected (i.e. fired 'connect')
 * @param {Socket} conn
 */
Peer.prototype.onconnect = function (conn) {
  this.conn = conn

  var wire = this.wire = new Wire()
  wire.remoteAddress = this.addr
  var destroy = once(function () {
    this.conn.destroy()
    this.conn = null
  }.bind(this))

  // Close the wire when the connection is destroyed
  conn.once('end', function () { destroy() })
  conn.once('error', function () { destroy() })
  conn.once('close', function () { wire.end() })

  wire.once('end', function () {
    this.wire = null
  }.bind(this))

  // Duplex streaming magic!
  conn.pipe(wire).pipe(conn)
}

/**
 * Pool
 * ====
 * A "pool" is a bunch of swarms all listening on the same TCP port for
 * incoming connections from peers who are interested in one of our swarms.
 * There is one Pool for every port that a swarm is listening on, and they are
 * all stored in the `Pool.pools` object. When a connection comes in, the pool
 * does the wire protocol handshake with the peer to determine which swarm they
 * are interested in, and routes the connection to the right swarm.
 *
 * @param {number} port
 */
function Pool (port) {
  this.port = port
  this.swarms = {} // infoHash -> Swarm
  this.listening = false

  // Keep track of incoming connections so we can destroy them if we need to
  // close the server later.
  this.conns = []

  this.server = net.createServer(this._onconn.bind(this))
  this.server.listen(this.port, this._onlistening.bind(this))
  this.server.on('error', this._onerror.bind(this))

  this._retries = 0
}

/**
 * In-use Pools (port -> Pool)
 */
Pool.pools = {}

/**
 * STATIC METHOD: Add a swarm to a pool, creating a new pool if necessary.
 * @param {Swarm} swarm
 */
Pool.add = function (swarm) {
  var port = swarm.port
  var pool = Pool.pools[port]

  if (!pool)
    pool = Pool.pools[port] = new Pool(port)

  pool.addSwarm(swarm)
}

/**
 * STATIC METHOD: Remove a swarm from its pool.
 * @param  {Swarm} swarm
 */
Pool.remove = function (swarm) {
  var port = swarm.port
  var pool = Pool.pools[port]
  if (!pool) return

  pool.removeSwarm(swarm)

  if (Object.keys(pool.swarms).length === 0)
    delete Pool.pools[port]
}

Pool.prototype._onlistening = function () {
  this.listening = true
  for (var infoHash in this.swarms) {
    var swarm = this.swarms[infoHash]
    swarm.emit('listening', this.port)
  }
}

Pool.prototype._onconn = function (conn) {
  // Track all conns in this pool
  this.conns.push(conn)
  conn.on('close', function () {
    this.conns.splice(this.conns.indexOf(conn))
  }.bind(this))

  var addr = conn.remoteAddress + ':' + conn.remotePort

  // On incoming connections, we expect the remote peer to send a handshake
  // first. Based on the infoHash in that handshake, route the peer to the
  // right swarm.
  var peer = new Peer(addr)
  peer.onconnect(conn)

  // Peer must send handshake in timely manner - they connected to us after all
  var timeout = setTimeout(function () {
    conn.destroy()
  }, HANDSHAKE_TIMEOUT)

  peer.wire.on('handshake', function (infoHash, peerId, extensions) {
    clearTimeout(timeout)
    var swarm = this.swarms[infoHash.toString('hex')]

    // Destroy connections from peers that handshake for an infoHash not in
    // this pool.
    if (!swarm)
      return conn.destroy()

    swarm._onincoming(peer)
  }.bind(this))
}

Pool.prototype._onerror = function (err) {
  if (err.code === 'EADDRINUSE' && this._retries < 5) {
    console.error('Address in use, retrying...')
    setTimeout(function () {
      this._retries += 1
      this.server.close()
      this.server.listen(this.port)
    }.bind(this), 1000)
  } else {
    this.listening = false
    this.swarms.forEach(function (swarm) {
      swarm.emit('error', 'Swarm listen error: ' + err.message)
    })
  }
}

/**
 * Destroy this pool.
 * @param  {function} cb
 */
Pool.prototype.destroy = function (cb) {
  // Destroy all open connections & wire objects so the server can gracefully
  // close without waiting for timeout or the remote peer to disconnect.
  this.conns.forEach(function (conn) {
    conn.destroy()
  })
  this.listening = false
  this.server.close(cb)
}

/**
 * Add a swarm to this pool.
 * @param {Swarm} swarm
 */
Pool.prototype.addSwarm = function (swarm) {
  var infoHash = swarm.infoHash.toString('hex')

  if (this.listening) {
    process.nextTick(function () {
      swarm.emit('listening')
    })
  }

  if (this.swarms[infoHash]) {
    process.nextTick(function () {
      swarm.emit('error', new Error('Swarm listen error: There is already a ' +
        'swarm with infoHash ' + swarm.infoHash.toString('hex') +
        ' listening on port ' + swarm.port))
    })
    return
  }

  this.swarms[infoHash] = swarm
}

/**
 * Remove a swarm from this pool.
 * @param  {Swarm} swarm
 */
Pool.prototype.removeSwarm = function (swarm) {
  var infoHash = swarm.infoHash.toString('hex')
  delete this.swarms[infoHash]

  if (Object.keys(this.swarms).length === 0)
    this.destroy()
}

inherits(Swarm, EventEmitter)

/**
 * Swarm
 * =====
 * Abstraction of a BitTorrent "swarm", which is handy for managing all peer
 * connections for a given torrent download. This handles connecting to peers,
 * listening for incoming connections, and doing the initial peer wire protocol
 * handshake with peers. It also tracks total data uploaded/downloaded to/from
 * the swarm.
 *
 * Events: wire, download, upload, error, close
 *
 * @param {Buffer|string} infoHash
 * @param {Buffer|string} peerId
 * @param {Object} extensions
 */
function Swarm (infoHash, peerId, extensions) {
  if (!(this instanceof Swarm)) return new Swarm(infoHash, peerId)

  EventEmitter.call(this)

  this.infoHash = typeof infoHash === 'string'
    ? new Buffer(infoHash, 'hex')
    : infoHash

  this.peerId = typeof peerId === 'string'
    ? new Buffer(peerId, 'utf8')
    : peerId

  this.extensions = extensions

  this.port = 0
  this.downloaded = 0
  this.uploaded = 0

  this.wires = [] // open wires (added *after* handshake)

  this._queue = [] // queue of peers to connect to
  this._peers = {} // connected peers (addr -> Peer)

  this._paused = false
  this._destroyed = false
}

Object.defineProperty(Swarm.prototype, 'ratio', {
  get: function () {
    if (this.downloaded === 0)
      return 0
    else
      return this.uploaded / this.downloaded
  }
})

Object.defineProperty(Swarm.prototype, 'numQueued', {
  get: function () {
    return this._queue.length
  }
})

Object.defineProperty(Swarm.prototype, 'numConns', {
  get: function () {
    return Object.keys(this._peers)
      .map(function (addr) {
        return this._peers[addr].conn ? 1 : 0
      }.bind(this))
      .reduce(function (prev, current) {
        return prev + current
      }, 0)
  }
})

/**
 * Add a peer to the swarm.
 * @param {string} addr  ip address and port (ex: 12.34.56.78:12345)
 */
Swarm.prototype.add = function (addr) {
  if (this._destroyed || this._peers[addr]) return
  if (!validAddr(addr)) return

  var peer = new Peer(addr)
  this._peers[addr] = peer
  this._queue.push(peer)

  this._drain()
}

/**
 * Temporarily stop connecting to new peers. Note that this does not pause new
 * incoming connections, nor does it pause the streams of existing connections
 * or their wires.
 */
Swarm.prototype.pause = function () {
  this._paused = true
}

/**
 * Resume connecting to new peers.
 */
Swarm.prototype.resume = function () {
  this._paused = false
  this._drain()
}

/**
 * Remove a peer from the swarm.
 * @param  {string} addr  ip address and port (ex: 12.34.56.78:12345)
 */
Swarm.prototype.remove = function (addr) {
  this._remove(addr)
  this._drain()
}

/**
 * Private method to remove a peer from the swarm without calling _drain().
 * @param  {string} addr  ip address and port (ex: 12.34.56.78:12345)
 */
Swarm.prototype._remove = function (addr) {
  var peer = this._peers[addr]
  if (!peer) return
  delete this._peers[addr]
  if (peer.node)
    this._queue.splice(this._queue.indexOf(peer), 1)
  if (peer.timeout)
    clearTimeout(peer.timeout)
  if (peer.wire)
    peer.wire.destroy()
}

/**
 * Listen on the given port for peer connections.
 * @param  {number=} port
 * @param  {function} onlistening
 */
Swarm.prototype.listen = function (port, onlistening) {
  if (typeof port === 'function') {
    onlistening = port
    port = undefined
  }

  if (onlistening)
    this.once('listening', onlistening)

  var onPort = function (err, port) {
    if (err)
      return this.emit('error', err)
    this.port = port
    Pool.add(this)
  }.bind(this)

  if (port)
    onPort(null, port)
  else
    portfinder.getPort(onPort)
}

/**
 * Destroy the swarm, close all open peer connections, and do cleanup.
 */
Swarm.prototype.destroy = function () {
  this._destroyed = true

  for (var addr in this._peers) {
    this._remove(addr)
  }

  Pool.remove(this)

  process.nextTick(function () {
    this.emit('close')
  }.bind(this))
}

//
// HELPER METHODS
//

/**
 * Pop a peer off the FIFO queue and connect to it. When _drain() gets called,
 * the queue will usually have only one peer in it, except when there are too
 * many peers (over `this.maxSize`) in which case they will just sit in the queue
 * until another connection closes.
 */
Swarm.prototype._drain = function () {
  if (this.numConns >= MAX_SIZE || this._paused) return

  var peer = this._queue.shift()
  if (!peer) return

  if (peer.timeout) {
    clearTimeout(peer.timeout)
    peer.timeout = null
  }

  var parts = peer.addr.split(':')
  var conn = net.connect(parts[1], parts[0])

  console.log('Connecting to ' + peer.addr)

  // Peer must respond to handshake in timely manner
  var timeout = setTimeout(function () {
    conn.destroy()
  }, HANDSHAKE_TIMEOUT)

  var onhandshake = function (infoHash) {
    clearTimeout(timeout)
    if (infoHash.toString('hex') !== this.infoHash.toString('hex'))
      return peer.conn.destroy()
    this._onwire(peer)
  }.bind(this)

  var onconnect = function () {
    peer.onconnect(conn)
    this._onconn(peer)

    var wire = peer.wire
    wire.on('handshake', onhandshake)

    // When wire dies, repeatedly attempt to reconnect to the peer, after a
    // timeout, with exponential backoff.
    wire.on('end', function () {
      if (this._destroyed
          || wire.destroyed
          || peer.retries >= RECONNECT_WAIT.length)
        return this._remove(peer.addr)

      var readd = function () {
        this._queue.push(peer)
        this._drain()
      }.bind(this)

      peer.timeout = setTimeout(readd, RECONNECT_WAIT[peer.retries++])
    }.bind(this))

    wire.handshake(this.infoHash, this.peerId, this.extensions)
  }.bind(this)

  conn.on('connect', onconnect)
  conn.on('error', function (err) {
    console.log('Failed to connect to ' + peer.addr)
  })
}

/**
 * Called whenever a new peer wants to connects to this swarm. Called with a
 * peer that has already sent us a handshake.
 * @param  {Peer} peer
 */
Swarm.prototype._onincoming = function (peer) {
  this._peers[peer.wire.remoteAddress] = peer
  peer.wire.handshake(this.infoHash, this.peerId, this.extensions)

  this._onconn(peer)
  this._onwire(peer)
}

//
// CONNECTION AND WIRE HANDLERS
//

/**
 * Called whenever a new connection is connected.
 * @param  {Socket} conn
 */
Swarm.prototype._onconn = function (peer) {
  peer.conn.once('close', function () {
    this._drain() // allow another connection to be opened
  }.bind(this))
}

/**
 * Called whenever we've handshaken with a new wire.
 * @param  {Peer} peer
 */
Swarm.prototype._onwire = function (peer) {
  var conn = peer.conn
  var wire = peer.wire

  peer.retries = 0

  // Track total bytes downloaded by the swarm
  wire.on('download', function (downloaded) {
    this.downloaded += downloaded
    this.emit('download', downloaded)
  }.bind(this))

  // Track total bytes uploaded by the swarm
  wire.on('upload', function (uploaded) {
    this.uploaded += uploaded
    this.emit('upload', uploaded)
  }.bind(this))

  var cleanup = once(function () {
    this.wires.splice(this.wires.indexOf(wire), 1)
    conn.destroy()
  }.bind(this))

  wire.on('end', cleanup)
  wire.on('close', cleanup)
  wire.on('error', cleanup)
  wire.on('finish', cleanup)

  this.wires.push(wire)
  this.emit('wire', wire)
}

/**
 * Is the address valid?
 * @param  {string} addr
 * @return {boolean}
 */
function validAddr (addr) {
  var port = Number(addr.split(':')[1]);
  return port > 0 && port < 65535;
}<commit_msg>add swarm.numPeers property<commit_after>module.exports = Swarm

var EventEmitter = require('events').EventEmitter
var inherits = require('inherits')
var net = require('net') // or chrome-net
var portfinder = require('portfinder') // or chrome-portfinder
var once = require('once')
var Wire = require('bittorrent-protocol')

// Use random port above 1024
portfinder.basePort = Math.floor(Math.random() * 60000) + 1025

var MAX_SIZE = 100
var HANDSHAKE_TIMEOUT = 5000
var RECONNECT_WAIT = [1000, 5000, 15000, 30000, 60000, 120000, 300000, 600000]

/**
 * Peer
 * ====
 * A peer in the swarm. Comprised of a `net.Socket` and a `Wire`.
 *
 * @param {string} addr
 */
function Peer (addr) {
  this.addr = addr

  this.conn = null
  this.wire = null

  this.timeout = null
  this.retries = 0
}

/**
 * Called once the peer's `conn` has connected (i.e. fired 'connect')
 * @param {Socket} conn
 */
Peer.prototype.onconnect = function (conn) {
  this.conn = conn

  var wire = this.wire = new Wire()
  wire.remoteAddress = this.addr
  var destroy = once(function () {
    this.conn.destroy()
    this.conn = null
  }.bind(this))

  // Close the wire when the connection is destroyed
  conn.once('end', function () { destroy() })
  conn.once('error', function () { destroy() })
  conn.once('close', function () { wire.end() })

  wire.once('end', function () {
    this.wire = null
  }.bind(this))

  // Duplex streaming magic!
  conn.pipe(wire).pipe(conn)
}

/**
 * Pool
 * ====
 * A "pool" is a bunch of swarms all listening on the same TCP port for
 * incoming connections from peers who are interested in one of our swarms.
 * There is one Pool for every port that a swarm is listening on, and they are
 * all stored in the `Pool.pools` object. When a connection comes in, the pool
 * does the wire protocol handshake with the peer to determine which swarm they
 * are interested in, and routes the connection to the right swarm.
 *
 * @param {number} port
 */
function Pool (port) {
  this.port = port
  this.swarms = {} // infoHash -> Swarm
  this.listening = false

  // Keep track of incoming connections so we can destroy them if we need to
  // close the server later.
  this.conns = []

  this.server = net.createServer(this._onconn.bind(this))
  this.server.listen(this.port, this._onlistening.bind(this))
  this.server.on('error', this._onerror.bind(this))

  this._retries = 0
}

/**
 * In-use Pools (port -> Pool)
 */
Pool.pools = {}

/**
 * STATIC METHOD: Add a swarm to a pool, creating a new pool if necessary.
 * @param {Swarm} swarm
 */
Pool.add = function (swarm) {
  var port = swarm.port
  var pool = Pool.pools[port]

  if (!pool)
    pool = Pool.pools[port] = new Pool(port)

  pool.addSwarm(swarm)
}

/**
 * STATIC METHOD: Remove a swarm from its pool.
 * @param  {Swarm} swarm
 */
Pool.remove = function (swarm) {
  var port = swarm.port
  var pool = Pool.pools[port]
  if (!pool) return

  pool.removeSwarm(swarm)

  if (Object.keys(pool.swarms).length === 0)
    delete Pool.pools[port]
}

Pool.prototype._onlistening = function () {
  this.listening = true
  for (var infoHash in this.swarms) {
    var swarm = this.swarms[infoHash]
    swarm.emit('listening', this.port)
  }
}

Pool.prototype._onconn = function (conn) {
  // Track all conns in this pool
  this.conns.push(conn)
  conn.on('close', function () {
    this.conns.splice(this.conns.indexOf(conn))
  }.bind(this))

  var addr = conn.remoteAddress + ':' + conn.remotePort

  // On incoming connections, we expect the remote peer to send a handshake
  // first. Based on the infoHash in that handshake, route the peer to the
  // right swarm.
  var peer = new Peer(addr)
  peer.onconnect(conn)

  // Peer must send handshake in timely manner - they connected to us after all
  var timeout = setTimeout(function () {
    conn.destroy()
  }, HANDSHAKE_TIMEOUT)

  peer.wire.on('handshake', function (infoHash, peerId, extensions) {
    clearTimeout(timeout)
    var swarm = this.swarms[infoHash.toString('hex')]

    // Destroy connections from peers that handshake for an infoHash not in
    // this pool.
    if (!swarm)
      return conn.destroy()

    swarm._onincoming(peer)
  }.bind(this))
}

Pool.prototype._onerror = function (err) {
  if (err.code === 'EADDRINUSE' && this._retries < 5) {
    console.error('Address in use, retrying...')
    setTimeout(function () {
      this._retries += 1
      this.server.close()
      this.server.listen(this.port)
    }.bind(this), 1000)
  } else {
    this.listening = false
    this.swarms.forEach(function (swarm) {
      swarm.emit('error', 'Swarm listen error: ' + err.message)
    })
  }
}

/**
 * Destroy this pool.
 * @param  {function} cb
 */
Pool.prototype.destroy = function (cb) {
  // Destroy all open connections & wire objects so the server can gracefully
  // close without waiting for timeout or the remote peer to disconnect.
  this.conns.forEach(function (conn) {
    conn.destroy()
  })
  this.listening = false
  this.server.close(cb)
}

/**
 * Add a swarm to this pool.
 * @param {Swarm} swarm
 */
Pool.prototype.addSwarm = function (swarm) {
  var infoHash = swarm.infoHash.toString('hex')

  if (this.listening) {
    process.nextTick(function () {
      swarm.emit('listening')
    })
  }

  if (this.swarms[infoHash]) {
    process.nextTick(function () {
      swarm.emit('error', new Error('Swarm listen error: There is already a ' +
        'swarm with infoHash ' + swarm.infoHash.toString('hex') +
        ' listening on port ' + swarm.port))
    })
    return
  }

  this.swarms[infoHash] = swarm
}

/**
 * Remove a swarm from this pool.
 * @param  {Swarm} swarm
 */
Pool.prototype.removeSwarm = function (swarm) {
  var infoHash = swarm.infoHash.toString('hex')
  delete this.swarms[infoHash]

  if (Object.keys(this.swarms).length === 0)
    this.destroy()
}

inherits(Swarm, EventEmitter)

/**
 * Swarm
 * =====
 * Abstraction of a BitTorrent "swarm", which is handy for managing all peer
 * connections for a given torrent download. This handles connecting to peers,
 * listening for incoming connections, and doing the initial peer wire protocol
 * handshake with peers. It also tracks total data uploaded/downloaded to/from
 * the swarm.
 *
 * Events: wire, download, upload, error, close
 *
 * @param {Buffer|string} infoHash
 * @param {Buffer|string} peerId
 * @param {Object} extensions
 */
function Swarm (infoHash, peerId, extensions) {
  if (!(this instanceof Swarm)) return new Swarm(infoHash, peerId)

  EventEmitter.call(this)

  this.infoHash = typeof infoHash === 'string'
    ? new Buffer(infoHash, 'hex')
    : infoHash

  this.peerId = typeof peerId === 'string'
    ? new Buffer(peerId, 'utf8')
    : peerId

  this.extensions = extensions

  this.port = 0
  this.downloaded = 0
  this.uploaded = 0

  this.wires = [] // open wires (added *after* handshake)

  this._queue = [] // queue of peers to connect to
  this._peers = {} // connected peers (addr -> Peer)

  this._paused = false
  this._destroyed = false
}

Object.defineProperty(Swarm.prototype, 'ratio', {
  get: function () {
    if (this.downloaded === 0)
      return 0
    else
      return this.uploaded / this.downloaded
  }
})

Object.defineProperty(Swarm.prototype, 'numQueued', {
  get: function () {
    return this._queue.length
  }
})

Object.defineProperty(Swarm.prototype, 'numConns', {
  get: function () {
    return Object.keys(this._peers)
      .map(function (addr) {
        return this._peers[addr].conn ? 1 : 0
      }.bind(this))
      .reduce(function (prev, current) {
        return prev + current
      }, 0)
  }
})

Object.defineProperty(Swarm.prototype, 'numPeers', {
  get: function () {
    return this.wires.length
  }
})

/**
 * Add a peer to the swarm.
 * @param {string} addr  ip address and port (ex: 12.34.56.78:12345)
 */
Swarm.prototype.add = function (addr) {
  if (this._destroyed || this._peers[addr]) return
  if (!validAddr(addr)) return

  var peer = new Peer(addr)
  this._peers[addr] = peer
  this._queue.push(peer)

  this._drain()
}

/**
 * Temporarily stop connecting to new peers. Note that this does not pause new
 * incoming connections, nor does it pause the streams of existing connections
 * or their wires.
 */
Swarm.prototype.pause = function () {
  this._paused = true
}

/**
 * Resume connecting to new peers.
 */
Swarm.prototype.resume = function () {
  this._paused = false
  this._drain()
}

/**
 * Remove a peer from the swarm.
 * @param  {string} addr  ip address and port (ex: 12.34.56.78:12345)
 */
Swarm.prototype.remove = function (addr) {
  this._remove(addr)
  this._drain()
}

/**
 * Private method to remove a peer from the swarm without calling _drain().
 * @param  {string} addr  ip address and port (ex: 12.34.56.78:12345)
 */
Swarm.prototype._remove = function (addr) {
  var peer = this._peers[addr]
  if (!peer) return
  delete this._peers[addr]
  if (peer.node)
    this._queue.splice(this._queue.indexOf(peer), 1)
  if (peer.timeout)
    clearTimeout(peer.timeout)
  if (peer.wire)
    peer.wire.destroy()
}

/**
 * Listen on the given port for peer connections.
 * @param  {number=} port
 * @param  {function} onlistening
 */
Swarm.prototype.listen = function (port, onlistening) {
  if (typeof port === 'function') {
    onlistening = port
    port = undefined
  }

  if (onlistening)
    this.once('listening', onlistening)

  var onPort = function (err, port) {
    if (err)
      return this.emit('error', err)
    this.port = port
    Pool.add(this)
  }.bind(this)

  if (port)
    onPort(null, port)
  else
    portfinder.getPort(onPort)
}

/**
 * Destroy the swarm, close all open peer connections, and do cleanup.
 */
Swarm.prototype.destroy = function () {
  this._destroyed = true

  for (var addr in this._peers) {
    this._remove(addr)
  }

  Pool.remove(this)

  process.nextTick(function () {
    this.emit('close')
  }.bind(this))
}

//
// HELPER METHODS
//

/**
 * Pop a peer off the FIFO queue and connect to it. When _drain() gets called,
 * the queue will usually have only one peer in it, except when there are too
 * many peers (over `this.maxSize`) in which case they will just sit in the queue
 * until another connection closes.
 */
Swarm.prototype._drain = function () {
  if (this.numConns >= MAX_SIZE || this._paused) return

  var peer = this._queue.shift()
  if (!peer) return

  if (peer.timeout) {
    clearTimeout(peer.timeout)
    peer.timeout = null
  }

  var parts = peer.addr.split(':')
  var conn = net.connect(parts[1], parts[0])

  console.log('Connecting to ' + peer.addr)

  // Peer must respond to handshake in timely manner
  var timeout = setTimeout(function () {
    conn.destroy()
  }, HANDSHAKE_TIMEOUT)

  var onhandshake = function (infoHash) {
    clearTimeout(timeout)
    if (infoHash.toString('hex') !== this.infoHash.toString('hex'))
      return peer.conn.destroy()
    this._onwire(peer)
  }.bind(this)

  var onconnect = function () {
    peer.onconnect(conn)
    this._onconn(peer)

    var wire = peer.wire
    wire.on('handshake', onhandshake)

    // When wire dies, repeatedly attempt to reconnect to the peer, after a
    // timeout, with exponential backoff.
    wire.on('end', function () {
      if (this._destroyed
          || wire.destroyed
          || peer.retries >= RECONNECT_WAIT.length)
        return this._remove(peer.addr)

      var readd = function () {
        this._queue.push(peer)
        this._drain()
      }.bind(this)

      peer.timeout = setTimeout(readd, RECONNECT_WAIT[peer.retries++])
    }.bind(this))

    wire.handshake(this.infoHash, this.peerId, this.extensions)
  }.bind(this)

  conn.on('connect', onconnect)
  conn.on('error', function (err) {
    console.log('Failed to connect to ' + peer.addr)
  })
}

/**
 * Called whenever a new peer wants to connects to this swarm. Called with a
 * peer that has already sent us a handshake.
 * @param  {Peer} peer
 */
Swarm.prototype._onincoming = function (peer) {
  this._peers[peer.wire.remoteAddress] = peer
  peer.wire.handshake(this.infoHash, this.peerId, this.extensions)

  this._onconn(peer)
  this._onwire(peer)
}

//
// CONNECTION AND WIRE HANDLERS
//

/**
 * Called whenever a new connection is connected.
 * @param  {Socket} conn
 */
Swarm.prototype._onconn = function (peer) {
  peer.conn.once('close', function () {
    this._drain() // allow another connection to be opened
  }.bind(this))
}

/**
 * Called whenever we've handshaken with a new wire.
 * @param  {Peer} peer
 */
Swarm.prototype._onwire = function (peer) {
  var conn = peer.conn
  var wire = peer.wire

  peer.retries = 0

  // Track total bytes downloaded by the swarm
  wire.on('download', function (downloaded) {
    this.downloaded += downloaded
    this.emit('download', downloaded)
  }.bind(this))

  // Track total bytes uploaded by the swarm
  wire.on('upload', function (uploaded) {
    this.uploaded += uploaded
    this.emit('upload', uploaded)
  }.bind(this))

  var cleanup = once(function () {
    this.wires.splice(this.wires.indexOf(wire), 1)
    conn.destroy()
  }.bind(this))

  wire.on('end', cleanup)
  wire.on('close', cleanup)
  wire.on('error', cleanup)
  wire.on('finish', cleanup)

  this.wires.push(wire)
  this.emit('wire', wire)
}

/**
 * Is the address valid?
 * @param  {string} addr
 * @return {boolean}
 */
function validAddr (addr) {
  var port = Number(addr.split(':')[1]);
  return port > 0 && port < 65535;
}<|endoftext|>


<commit_before>'use strict';

var util = require('./lib/util');

exports = module.exports = Mase;

/**
### README

The module exports a constructor.

```js
var Mase = require('mase');
var db = new Mase(string name, array data)
```

Memory dbs with the same `name` share memory.
It can take an array of `data` as a second or first argument.
If so, `data` is upserted into the db by only using each
document's `_id` as a selector.

### spec
```js
function Mase([string name, array data])
```

_arguments_
- `name` type string, name for the collection
- `data` type array to insert in memory db

_defaults_
- `name` to a random 24 length string using `[A-Za-z0-9]` characters
- `data`
  - to the previous collection `name` array if there was one
  - to empty array if collection `name` it wasn't

_returns_ a new instance

_instance properties_
- `store` array of data of the instance

**/

// dbs namespace
var mase = {};

function Mase(name, data){
  if(!(this instanceof Mase)){
    return new Mase(name, data);
  }

  data = data || name;
  this.name = util.type(name).string || util.randomID();
  if(mase[this.name]){ return mase[this.name]; }

  this.store = [];
  mase[this.name] = this;

  if(util.type(data).array){
    data.forEach(function(item){
      this.update({_id: item._id}, item, {$upsert: true});
    }, this);
  }
}

/**
## insert

Insert a new document. When `document` is not a plain object,
i.e. `'one', 1, etc.`, it will be inserted using its value.

```js
db.insert(1); // => {value: 1, _id: 'wZy7rd33Q8478pvi'}
```

### spec
```js
function insert(object|any document)
```

_arguments_
 - `document` type object or any, element to insert.

_defaults_
 - `document._id` to a 24 length random string

_returns_ a clone of the inserted `document`

**/
Mase.prototype.insert = function(_doc){
  var doc = util.clone(util.type(_doc).plainObject || {value: _doc}, true);
  if(doc._id === void 0){ doc._id = util.randomID(); }
  this.changed('insert', doc, null);
  this.store.push(doc);
  return doc;
};

/**
## find

Find documents. By default, documents are tested
for value equality with [`lodash.isEqual`][p-lodash.isEqual].
This can be changed using a `$test` function.

### spec
```js
function find(object fields[, object options|function $test])
```

_arguments_
 - `fields` type object, document fields for lookup
 - `$test` type function that will test `fields` for each document
 - `options` type object, helper object on how to do the lookup

_options_ properties
 - `$acc` type boolean, return value of the `$test` function
 - `$test` type function that tests `fields` against each document
 - `$break` type boolean, wether to break the search after match
 - `$count` type boolean, wether to have `result` start as `0` or `[]`
 - `$result` type array or number, result returned. It will start as:
  - empty array when `$count` is falsy
  - `0` when `$count` is truthy

`$test(fields, doc, key, options)` _arguments_
 - `fields` object fields given as argument
 - `doc` document object found
 - `key` property of `fields` to be tested
 - `options` the options object passed in to `find`

_defaults_
- `options.$test` an equality test using [`lodash.isEqual`][p-lodash.isEqual]

_returns_ `o.$result`

[p-lodash.isEqual]: https://lodash.com/docs#isEqual
**/
Mase.prototype.find = function(fields, o){
  if(fields && !util.type(fields).match(/function|plainObject/)){
    throw new TypeError('find([fields, object options|function $test]) '
      + '`fields` should be function or plainObject'
    );
  }

  var spec = Object.keys(fields || {});
  var length = this.store.length;
  o = util.type(o).plainObject || {$test: o || fields};

  if(!length || (!spec.length && typeof o.$test !== 'function')){
    return o.$count ? length : util.clone(this.store, true);
  }

  if(typeof o.$test !== 'function'){
    o.$test = util.test.$equal;
  }

  if(typeof o.$match !== 'function'){
    o.$match = (o.$count && util.match.$count) || util.match.$equal;
  }

  --length;
  o.$result = o.$count ? 0 : [];
  if(!spec.length){ spec = ''; }
  var index = -1, store = this.store;
  // ^ better than index < len-1 in `whilst`

  (function whilst(){
    o.$acc = true;
    var doc = store[++index];
    var match = (spec || Object.keys(doc)).filter(function(key){
      return (o.$acc = o.$test(fields, doc, key, o));
    }).length;

    if(match){
      o.$match(doc, o);
      if(o.$break){ return ; }
    }
    if(index < length){ whilst(); }
  })();

  if(o.$ref){ return o.$result; }
  return util.clone(o.$result, true);
};

/**
## findOne

This method is just sugar on top of `find` enforcing
it to always return an object or null if nothing was found.

```js
var foundOne = find(fields, {
  $test: [Function] // if given
  $break: true,
  $count: false
})[0];
```

### spec
```js
function findOne(fields[, options|function $test])
```

Same as [`find`'s spec](./find.md#spec), differences below

_options_ properties
 - `$count` is always set to `false`
 - `$break` is always set to `true`

_returns_
 - the object document if found
 - `null` otherwise

**/

Mase.prototype.findOne = function(fields, o){
  o = util.type(o).plainObject || {$test: o};

  o.$break = true;
  o.$count = false;

  return this.find(fields, o)[0] || null;
};

/**
## update

Update/Upsert documents in the memory db. They are found first using
 `find` and then updated. Documents updates can be customized
 by using an `$update` function.

### spec
```js
function update(
  object fields,
  [object update|function $update],
  [object options|function $update]
)
```

_arguments_
 - `fields` type object, document(s) fields to lookup
 - `update` type object, which fields to update on document(s) found
 - `$update` type function, customizes how to update fields
 - `options` type object, modifies how to do the update

_options_. Same they are for [`find`](./find.md) plus
 - `$upsert` type boolean, wether to `insert` if no documents found
 - `$update` type function, customizer for the update

`$update(doc, update)` _arguments_
 - `doc` document found using `fields`
 - `update` same object given as argument

_defaults_
- `options.$update` to merge properties of `doc` with `update`

_returns_ `this`

**/
Mase.prototype.update = function(fields, update, o){
  if(
    !util.type(fields).plainObject &&
    !util.type(update).match(/function|plainObject/)
  ){
    throw new TypeError('update(fields, update[, options])\n'
      + ' `fields` should be plainObject\n'
      + ' `update` should be function or plainObject'
    );
  }

  if(update.$update){ o = update; }
  o = util.type(o).plainObject || {$update: o || update};

  if(typeof o.$update !== 'function'){
    o.$update = util.merge;
  }

  o.$ref = true; o.$count = false;
  var found = this.find(fields, o).map(function(doc){
    this.changed('update', update, doc);
    o.$update(doc, update);
    return doc;
  }, this);

  if(!found.length && o.$upsert){
    this.insert(o.$update(fields, update));
  }

  return this;
};

/**
## remove

```js
function remove(any id)
```
Remove only one document at a time.

_arguments_
 - `id` type any, id of the document to delete

_returns_
 - `true` if there was a document removed
 - `false` if there was no document removed

**/
Mase.prototype.remove = function(_id){
  var index = 0, length = this.store.length;

  if(!_id && _id !== 0){
    this.changed('remove', null, this.store);
    this.store = [];
    return Boolean(length);
  }

  var self = this;
  (function whilst(){
    if(util.isEqual(_id, self.store[index]._id)){
      self.changed('remove', null, self.store[index]);
      self.store.splice(index, 1);
      index += length + 1;
    }
    if(++index < length){ whilst(); }
  })();

  return index > length;
};

/**
## changed

Function called before any `insert`, `update` and `remove` .
Should be set manually since its only stored on the `prototype`
and or your instance.

```js
var db = new Mase('name', [1,2,3]);

db.changed = function(method, newDoc, oldDoc){
  switch(method){
    case 'insert':
      // call your websockets, etc.
    break;
    // etc
    default:
      return ;
    break;
  }
}
```

The `oldDoc` and `newDoc` are passed just before the change is done
 so beware to not mutate them. If you pass them to another function on the
 same process I would highly-super-obnoxiously tell you to clone them.
 If you are using this as is plotted above (for websocket action, etc.)
 it would be fine.

### spec

```js
function changed(string method, object newDoc, object oldDoc)
```

_arguments_
 - `method` type string, method that maked the change
 - `newDoc` type object, change performed on document
 - `oldDoc` type object, document to be changed

**/
Mase.prototype.changed = function onChange(){ /* empty */ };
<commit_msg>docs + enhance: update docs, make $break be usable even if there is no match<commit_after>'use strict';

var util = require('./lib/util');

exports = module.exports = Mase;

/**
### README

The module exports a constructor.

```js
var Mase = require('mase');
var db = new Mase(string name, array data)
```

Memory dbs with the same `name` share memory.
It can take an array of `data` as a second or first argument.
If so, `data` is upserted into the db by only using each
document's `_id` as a selector.

### spec
```js
function Mase([string name, array data])
```

_arguments_
- `name` type string, name for the collection
- `data` type array to insert in memory db

_defaults_
- `name` to a random 24 length string using `[A-Za-z0-9]` characters
- `data`
  - to the previous collection `name` array if there was one
  - to empty array if collection `name` it wasn't

_returns_ a new instance

_instance properties_
- `store` array of data of the instance

**/

// dbs namespace
var mase = {};

function Mase(name, data){
  if(!(this instanceof Mase)){
    return new Mase(name, data);
  }

  data = data || name;
  this.name = util.type(name).string || util.randomID();
  if(mase[this.name]){ return mase[this.name]; }

  this.store = [];
  mase[this.name] = this;

  if(util.type(data).array){
    data.forEach(function(item){
      this.update({_id: item._id}, item, {$upsert: true});
    }, this);
  }
}

/**
## insert

Insert a new document. When `document` is not a plain object,
i.e. `'one', 1, etc.`, it will be inserted using its value.

```js
db.insert(1); // => {value: 1, _id: 'wZy7rd33Q8478pvi'}
```

### spec
```js
function insert(object|any document)
```

_arguments_
 - `document` type object or any, element to insert.

_defaults_
 - `document._id` to a 24 length random string

_returns_ a clone of the inserted `document`

**/
Mase.prototype.insert = function(_doc){
  var doc = util.clone(util.type(_doc).plainObject || {value: _doc}, true);
  if(doc._id === void 0){ doc._id = util.randomID(); }
  this.changed('insert', doc, null);
  this.store.push(doc);
  return doc;
};

/**
## find

Find documents. By default, documents are tested
for value equality with [`lodash.isEqual`][p-lodash.isEqual].
This can be changed using a `$test` function.

### spec
```js
function find([object fields, object options|function $test])
```

_arguments_
 - `fields` type object, document fields for lookup
 - `$test` type function that will test `fields` for each document
 - `options` type object, helper object on how to do the lookup

_options_ properties
 - `$acc` type boolean, return value of the `$test` function
 - `$test` type function that tests `fields` against each document
 - `$break` type boolean, wether to break stop the search
 - `$count` type boolean, wether to have `result` start as `0` or `[]`
 - `$result` type array or number, result returned. It will start as:
  - empty array when `$count` is falsy
  - `0` when `$count` is truthy

`$test(fields, doc, key, options)` _arguments_
 - `fields` object fields given as argument
 - `doc` document object found
 - `key` property of `fields` to be tested
 - `options` the options object passed in to `find`

_defaults_
- `options.$test` an equality test using [`lodash.isEqual`][p-lodash.isEqual]

_returns_ `o.$result`

[p-lodash.isEqual]: https://lodash.com/docs#isEqual
**/
Mase.prototype.find = function(fields, o){
  if(fields && !util.type(fields).match(/function|plainObject/)){
    throw new TypeError('find([fields, object options|function $test]) '
      + '`fields` should be function or plainObject'
    );
  }

  var spec = Object.keys(fields || {});
  var length = this.store.length;
  o = util.type(o).plainObject || {$test: o || fields};

  if(!length || (!spec.length && typeof o.$test !== 'function')){
    return o.$count ? length : util.clone(this.store, true);
  }

  if(typeof o.$test !== 'function'){
    o.$test = util.test.$equal;
  }

  if(typeof o.$match !== 'function'){
    o.$match = (o.$count && util.match.$count) || util.match.$equal;
  }

  --length;
  o.$result = o.$count ? 0 : [];
  if(!spec.length){ spec = ''; }
  var index = -1, store = this.store;
  // ^ better than index < len-1 in `whilst`

  (function whilst(){
    o.$acc = true;
    var doc = store[++index];
    var match = (spec || Object.keys(doc)).filter(function(key){
      return (o.$acc = o.$test(fields, doc, key, o));
    }).length;

    if(match){ o.$match(doc, o); }
    if(o.$break){ return ; }
    if(index < length){ whilst(); }
  })();

  if(o.$ref){ return o.$result; }
  return util.clone(o.$result, true);
};

/**
## findOne

This method is just sugar on top of `find` enforcing
it to always return an object or null if nothing was found.

```js
var foundOne = find(fields, {
  $test: [Function] // if given
  $break: true,
  $count: false
})[0];
```

### spec
```js
function findOne(fields[, options|function $test])
```

Same as [`find`'s spec](./find.md#spec), differences below

_options_ properties
 - `$count` is always set to `false`
 - `$break` is always set to `true`

_returns_
 - the object document if found
 - `null` otherwise

**/

Mase.prototype.findOne = function(fields, o){
  o = util.type(o).plainObject || {$test: o};

  o.$break = true;
  o.$count = false;

  return this.find(fields, o)[0] || null;
};

/**
## update

Update/Upsert documents in the memory db. They are found first using
 `find` and then updated. Documents updates can be customized
 by using an `$update` function.

### spec
```js
function update(
  object fields,
  [object update|function $update],
  [object options|function $update]
)
```

_arguments_
 - `fields` type object, document(s) fields to lookup
 - `update` type object, which fields to update on document(s) found
 - `$update` type function, customizes how to update fields
 - `options` type object, modifies how to do the update

_options_. Same they are for [`find`](./find.md) plus
 - `$upsert` type boolean, wether to `insert` if no documents found
 - `$update` type function, customizer for the update

`$update(doc, update)` _arguments_
 - `doc` document found using `fields`
 - `update` same object given as argument

_defaults_
- `options.$update` to merge properties of `doc` with `update`

_returns_ `this`

**/
Mase.prototype.update = function(fields, update, o){
  if(
    !util.type(fields).plainObject &&
    !util.type(update).match(/function|plainObject/)
  ){
    throw new TypeError('update(fields, update[, options])\n'
      + ' `fields` should be plainObject\n'
      + ' `update` should be function or plainObject'
    );
  }

  if(update.$update){ o = update; }
  o = util.type(o).plainObject || {$update: o || update};

  if(typeof o.$update !== 'function'){
    o.$update = util.merge;
  }

  o.$ref = true; o.$count = false;
  var found = this.find(fields, o).map(function(doc){
    this.changed('update', update, doc);
    o.$update(doc, update);
    return doc;
  }, this);

  if(!found.length && o.$upsert){
    this.insert(o.$update(fields, update));
  }

  return this;
};

/**
## remove

```js
function remove(any id)
```
Remove only one document at a time.

_arguments_
 - `id` type any, id of the document to delete

_returns_
 - `true` if there was a document removed
 - `false` if there was no document removed

**/
Mase.prototype.remove = function(_id){
  var index = 0, length = this.store.length;

  if(!_id && _id !== 0){
    this.changed('remove', null, this.store);
    this.store = [];
    return Boolean(length);
  }

  var self = this;
  (function whilst(){
    if(util.isEqual(_id, self.store[index]._id)){
      self.changed('remove', null, self.store[index]);
      self.store.splice(index, 1);
      index += length + 1;
    }
    if(++index < length){ whilst(); }
  })();

  return index > length;
};

/**
## changed

Function called before any `insert`, `update` and `remove` .
Should be set manually since its only stored on the `prototype`
and or your instance.

```js
var db = new Mase('name', [1,2,3]);

db.changed = function(method, newDoc, oldDoc){
  switch(method){
    case 'insert':
      // call your websockets, etc.
    break;
    // etc
    default:
      return ;
    break;
  }
}
```

The `oldDoc` and `newDoc` are passed just before the change is done
 so beware to not mutate them. If you pass them to another function on the
 same process I would highly-super-obnoxiously tell you to clone them.
 If you are using this as is plotted above (for websocket action, etc.)
 it would be fine.

### spec

```js
function changed(string method, object newDoc, object oldDoc)
```

_arguments_
 - `method` type string, method that maked the change
 - `newDoc` type object, change performed on document
 - `oldDoc` type object, document to be changed

**/
Mase.prototype.changed = function onChange(){ /* empty */ };
<|endoftext|>


<commit_before>'use strict'

const path = require('path')

const cloneDeep = require('lodash.clonedeep')

const collector = require('./lib/collector')
const currentEnv = require('./lib/currentEnv')
const ResolvedConfig = require('./lib/ResolvedConfig')
const Verifier = require('./lib/Verifier')

function createConfig (options) {
  if (!options || !options.options || !options.source) {
    throw new TypeError("Expected 'options' and 'source' options")
  }
  if (typeof options.options !== 'object' || Array.isArray(options.options)) {
    throw new TypeError("'options' must be an actual object")
  }

  const source = options.source
  const dir = options.dir || path.dirname(source)
  const hash = options.hash || null
  const json5 = options.json5 !== false
  const babelOptions = cloneDeep(options.options)

  return new collector.Config(dir, null, hash, json5, babelOptions, source)
}
exports.createConfig = createConfig

exports.currentEnv = currentEnv

function fromConfig (baseConfig, options) {
  options = options || {}
  return collector.fromConfig(baseConfig, options.cache)
    .then(chains => chains && new ResolvedConfig(chains, options.cache))
}
exports.fromConfig = fromConfig

function fromDirectory (dir, options) {
  options = options || {}
  return collector.fromDirectory(dir, options.cache)
    .then(chains => chains && new ResolvedConfig(chains, options.cache))
}
exports.fromDirectory = fromDirectory

function prepareCache () {
  return {
    dependencyHashes: new Map(),
    fileExistence: new Map(),
    files: new Map(),
    pluginsAndPresets: new Map(),
    sourceHashes: new Map()
  }
}
exports.prepareCache = prepareCache

function restoreVerifier (buffer) {
  return Verifier.fromBuffer(buffer)
}
exports.restoreVerifier = restoreVerifier
<commit_msg>Clean up creating ResolvedConfig in fromConfig<commit_after>'use strict'

const path = require('path')

const cloneDeep = require('lodash.clonedeep')

const collector = require('./lib/collector')
const currentEnv = require('./lib/currentEnv')
const ResolvedConfig = require('./lib/ResolvedConfig')
const Verifier = require('./lib/Verifier')

function createConfig (options) {
  if (!options || !options.options || !options.source) {
    throw new TypeError("Expected 'options' and 'source' options")
  }
  if (typeof options.options !== 'object' || Array.isArray(options.options)) {
    throw new TypeError("'options' must be an actual object")
  }

  const source = options.source
  const dir = options.dir || path.dirname(source)
  const hash = options.hash || null
  const json5 = options.json5 !== false
  const babelOptions = cloneDeep(options.options)

  return new collector.Config(dir, null, hash, json5, babelOptions, source)
}
exports.createConfig = createConfig

exports.currentEnv = currentEnv

function fromConfig (baseConfig, options) {
  options = options || {}
  return collector.fromConfig(baseConfig, options.cache)
    .then(chains => new ResolvedConfig(chains, options.cache))
}
exports.fromConfig = fromConfig

function fromDirectory (dir, options) {
  options = options || {}
  return collector.fromDirectory(dir, options.cache)
    .then(chains => chains && new ResolvedConfig(chains, options.cache))
}
exports.fromDirectory = fromDirectory

function prepareCache () {
  return {
    dependencyHashes: new Map(),
    fileExistence: new Map(),
    files: new Map(),
    pluginsAndPresets: new Map(),
    sourceHashes: new Map()
  }
}
exports.prepareCache = prepareCache

function restoreVerifier (buffer) {
  return Verifier.fromBuffer(buffer)
}
exports.restoreVerifier = restoreVerifier
<|endoftext|>


<commit_before>var express = require("express");
var app = express();
var path =require('path');
var bodyparser = require('body-parser');
app.use( bodyparser.json() );
app.use(bodyparser.urlencoded({extended: false}));

var request = require('./requestData');

var port = Number(process.argv[2]);
var apikey = process.argv[3];

var fakeMessage = {
  "text" : "##~~text, this field may accept markdown~~",
  // "attachments" : [
  //   {
  //     "title" : "Ask:",
  //     "text" : "Who am I?",
  //     "color" : "#666666",
  //     "images" : [
  //       {
  //         "url" : "https://lh4.googleusercontent.com/-hdQ2HH5-hog/AAAAAAAAAAI/AAAAAAAAAAA/GW2OyU89jXE/W96-H96/photo.jpg"
  //       }
  //     ]
  //   }
  // ]
};

// TODO: save log
app.post('/chengyu', function(req, res) {
  const { token, text, trigger_word, user_name } = req.body;
  if (token !== '8d37093bb3548d9be7d876147d288337') {
    res.status(404)        // HTTP status 404: NotFound
      .end('Not found');
  }
  var word = text.split(' ').slice(1).join('');
  console.log(word);
  request(apikey, word, function(data) {
    const {
      total, result, error_code, reason
    } = data;
    if (error_code !== 0) {
      fakeMessage.text = reason;
    } else {
      fakeMessage.text = result.reduce(funtion(string, v, i) {
        return string += v.name;
      }, '');
    }
    res.end(JSON.stringify(fakeMessage));
  });
})
app.listen(port);
<commit_msg>fix typo<commit_after>var express = require("express");
var app = express();
var path =require('path');
var bodyparser = require('body-parser');
app.use( bodyparser.json() );
app.use(bodyparser.urlencoded({extended: false}));

var request = require('./requestData');

var port = Number(process.argv[2]);
var apikey = process.argv[3];

var fakeMessage = {
  "text" : "##~~text, this field may accept markdown~~",
  // "attachments" : [
  //   {
  //     "title" : "Ask:",
  //     "text" : "Who am I?",
  //     "color" : "#666666",
  //     "images" : [
  //       {
  //         "url" : "https://lh4.googleusercontent.com/-hdQ2HH5-hog/AAAAAAAAAAI/AAAAAAAAAAA/GW2OyU89jXE/W96-H96/photo.jpg"
  //       }
  //     ]
  //   }
  // ]
};

// TODO: save log
app.post('/chengyu', function(req, res) {
  const { token, text, trigger_word, user_name } = req.body;
  if (token !== '8d37093bb3548d9be7d876147d288337') {
    res.status(404)        // HTTP status 404: NotFound
      .end('Not found');
  }
  var word = text.split(' ').slice(1).join('');
  console.log(word);
  request(apikey, word, function(data) {
    const {
      total, result, error_code, reason
    } = data;
    if (error_code !== 0) {
      fakeMessage.text = reason;
    } else {
      fakeMessage.text = result.reduce(function(string, v, i) {
        return string += v.name;
      }, '');
    }
    res.end(JSON.stringify(fakeMessage));
  });
})
app.listen(port);
<|endoftext|>


<commit_before>var url = require('url');
var http = require('http');
var https = require('https');
var through = require('through');
var duplexer = require('duplexer');
var Stream = require('stream');

module.exports = hyperquest;

function hyperquest (uri, opts, cb, extra) {
    if (typeof uri === 'object') {
        cb = opts;
        opts = uri;
        uri = undefined;
    }
    if (!opts) opts = {};
    if (uri !== undefined) opts.uri = uri;
    if (extra) opts.method = extra.method;

    var req = new Req(opts);
    var ws = req.duplex && through();
    if (ws) ws.pause();
    var rs = through();

    var dup = req.duplex ? duplexer(ws, rs) : rs;
    if (!req.duplex) {
        rs.writable = false;
    }
    dup.request = req;
    dup.setHeader = req.setHeader.bind(req);
    dup.setLocation = req.setLocation.bind(req);

    var closed = false;
    dup.on('close', function () { closed = true });

    process.nextTick(function () {
        if (closed) return;
        dup.on('close', function () { r.destroy() });

        var r = req._send();
        r.on('error', dup.emit.bind(dup, 'error'));

        r.on('response', function (res) {
            dup.response = res;
            dup.emit('response', res);
            if (req.duplex) res.pipe(rs)
            else {
                res.on('data', function (buf) { rs.queue(buf) });
                res.on('end', function () { rs.queue(null) });
            }
        });

        if (req.duplex) {
            ws.pipe(r);
            ws.resume();
        }
        else r.end();
    });

    if (cb) {
        dup.on('error', cb);
        dup.on('response', cb.bind(dup, null));
    }
    return dup;
}

hyperquest.get = hyperquest;

hyperquest.post = function (uri, opts, cb) {
    return hyperquest(uri, opts, cb, { method: 'POST' });
};

hyperquest.put = function (uri, opts, cb) {
    return hyperquest(uri, opts, cb, { method: 'PUT' });
};

hyperquest['delete'] = function (uri, opts, cb) {
    return hyperquest(uri, opts, cb, { method: 'DELETE' });
};

function Req (opts) {
    this.headers = opts.headers || {};

    var method = (opts.method || 'GET').toUpperCase();
    this.method = method;
    this.duplex = !(method === 'GET' || method === 'DELETE');

    if (opts.uri) this.setLocation(opts.uri);
}

Req.prototype._send = function () {
    this._sent = true;

    var headers = this.headers || {};
    var u = url.parse(this.uri);
    if (u.auth) {
        headers.authorization = 'Basic ' + Buffer(u.auth).toString('base64');
    }

    var interface = (u.protocol === 'https:') ? https : http;
    var req = interface.request({
        method: this.method,
        host: u.hostname,
        port: Number(u.port),
        path: u.path,
        agent: false,
        headers: headers
    });

    if (req.setTimeout) req.setTimeout(Math.pow(2, 32) * 1000);
    return req;
};

Req.prototype.setHeader = function (key, value) {
    if (this._sent) throw new Error('request already sent');
    this.headers[key] = value;
    return this;
};

Req.prototype.setLocation = function (uri) {
    this.uri = uri;
    return this;
};
<commit_msg>bind helper for compatibility<commit_after>var url = require('url');
var http = require('http');
var https = require('https');
var through = require('through');
var duplexer = require('duplexer');
var Stream = require('stream');

module.exports = hyperquest;

function bind (obj, fn) {
  var args = Array.prototype.slice.call(arguments, 2);
  return function () {
    args = args.concat(Array.prototype.slice.call(arguments));
    return fn.apply(obj, args);
  }
}

function hyperquest (uri, opts, cb, extra) {
    if (typeof uri === 'object') {
        cb = opts;
        opts = uri;
        uri = undefined;
    }
    if (!opts) opts = {};
    if (uri !== undefined) opts.uri = uri;
    if (extra) opts.method = extra.method;

    var req = new Req(opts);
    var ws = req.duplex && through();
    if (ws) ws.pause();
    var rs = through();

    var dup = req.duplex ? duplexer(ws, rs) : rs;
    if (!req.duplex) {
        rs.writable = false;
    }
    dup.request = req;
    dup.setHeader = bind(req, req.setHeader);
    dup.setLocation = bind(req, req.setLocation);

    var closed = false;
    dup.on('close', function () { closed = true });

    process.nextTick(function () {
        if (closed) return;
        dup.on('close', function () { r.destroy() });

        var r = req._send();
        r.on('error', bind(dup, dup.emit, 'error'));

        r.on('response', function (res) {
            dup.response = res;
            dup.emit('response', res);
            if (req.duplex) res.pipe(rs)
            else {
                res.on('data', function (buf) { rs.queue(buf) });
                res.on('end', function () { rs.queue(null) });
            }
        });

        if (req.duplex) {
            ws.pipe(r);
            ws.resume();
        }
        else r.end();
    });

    if (cb) {
        dup.on('error', cb);
        dup.on('response', bind(dup, cb, null));
    }
    return dup;
}

hyperquest.get = hyperquest;

hyperquest.post = function (uri, opts, cb) {
    return hyperquest(uri, opts, cb, { method: 'POST' });
};

hyperquest.put = function (uri, opts, cb) {
    return hyperquest(uri, opts, cb, { method: 'PUT' });
};

hyperquest['delete'] = function (uri, opts, cb) {
    return hyperquest(uri, opts, cb, { method: 'DELETE' });
};

function Req (opts) {
    this.headers = opts.headers || {};

    var method = (opts.method || 'GET').toUpperCase();
    this.method = method;
    this.duplex = !(method === 'GET' || method === 'DELETE');

    if (opts.uri) this.setLocation(opts.uri);
}

Req.prototype._send = function () {
    this._sent = true;

    var headers = this.headers || {};
    var u = url.parse(this.uri);
    if (u.auth) {
        headers.authorization = 'Basic ' + Buffer(u.auth).toString('base64');
    }

    var interface = (u.protocol === 'https:') ? https : http;
    var req = interface.request({
        method: this.method,
        host: u.hostname,
        port: Number(u.port),
        path: u.path,
        agent: false,
        headers: headers
    });

    if (req.setTimeout) req.setTimeout(Math.pow(2, 32) * 1000);
    return req;
};

Req.prototype.setHeader = function (key, value) {
    if (this._sent) throw new Error('request already sent');
    this.headers[key] = value;
    return this;
};

Req.prototype.setLocation = function (uri) {
    this.uri = uri;
    return this;
};
<|endoftext|>