<commit_before>/**
 * Created by Luigi Ilie Aron on 16.01.15.
 * email: luigi@kreditech.com
 */
'use strict';
module.exports = function(connectionObject){
    var cb = connectionObject.db.cb,
        es = connectionObject.db.es,
        cbtools = require('./cbtools/cbtools')(connectionObject.config.cb),
        async = require('async'),
        _ = require('lodash');

    /**
     * @param modelType
     * @param query
     * @returns {{index: *, body: {query: {filtered: {query: {bool: {must: {term: {_type: {value: *}}}[]}}, filter: {}}}}}}
     */
    function buildQuery(modelType, query){
        if (typeof query !== 'object'){
            throw new Error('Query is not an object!');
        }

        if (typeof modelType !== 'string'){
            throw new Error('modelType is not an string');
        }

        var _q = {
            query: {
                filtered: {
                    query: {
                        bool: {
                            must: [{
                                term: {
                                    _type: {
                                        value: modelType
                                    }
                                }
                            }]
                        }
                    },
                    filter: {}
                }
            }
        };

        if (query.hasOwnProperty('where')) {
            _q.query.filtered.filter = query.where;
        }

        if (query.hasOwnProperty('limit')) {
            _q.size = query.limit;
        }

        if (query.hasOwnProperty('skip')) {
            _q.from = query.skip;
        }

        if (query.hasOwnProperty('sort')) {
            if (Object.keys(query.sort).length > 0) {
                var sort = [];

                for (var key in query.sort) {
                    var _tmp = {};
                    _tmp[key] = {
                        'order': query.sort[key] > 0 ? 'asc' : 'desc'
                    };

                    sort.push(_tmp);
                }

                _q.sort = sort;
            }
        }

        if (query.hasOwnProperty('aggs')) {
            _q['aggs'] = _.cloneDeep(query['aggs']);
            delete query['aggs'];
        }

        return {
            index: connectionObject.config.es.index,
            type: modelType,
            body : _q
        }
    }

    /**
     * @param entity
     * @param data
     * @param callback
     */
    function updateEntity(entity, data, callback){
        if (data.hasOwnProperty('id')){
            delete data.id;
        }

        if (data.hasOwnProperty('_TYPE')){
            delete data._TYPE;
        }

        var updatedEntity = _.extend(entity, data),
            key = updatedEntity._TYPE + ':' + updatedEntity.id;

        cb.replace(key, updatedEntity, function(err, res){
            if (err) {
                return callback(err, false)
            }

            var esData = {
                index: connectionObject.config.es.index,
                type : updatedEntity._TYPE,
                id   : key,
                body : {
                    doc: updatedEntity
                }
            };

            es.update(esData, function(err, res){
                if (err) {
                    return callback(err, false);
                }

                callback(null, updatedEntity);
            });
        });
    }

    /**
     * @param data
     * @param current
     * @param total
     * @param callback
     */
    function deleteRecond(data, current, total, callback){
        es.delete(data, function(err, _res){
            if (err) {
                return callback(err);
            }

            cb.remove(data.id, function(err, _res){
                if (err) {
                    return callback(err);
                }

                if (current === total) {
                    callback(null);
                }
            });
        });
    }

    return {
        /**
         * @param collection
         * @param data
         * @param callback
         */
        create: function(collection, data, _ttl, callback){
            cb.counter(collection + ":count" , 1, {initial: 0, offset: 1}, function(err, res){
                if(err){
                    return callback(err);
                }

                var key = collection + ":" + res.value,
                    meta = {},
                    ttlInterval = connectionObject.config.es.ttlInterval;

                data.id = res.value;
                data._TYPE = collection;

                if (_ttl) {
                    meta.expiry = Math.floor((new Date().getTime() + _ttl + ttlInterval) / 1000);
                }

                cb.insert(key, data, meta, function(err, res){
                    if (err) {
                        return callback(err, false);
                    }

                    var esData = {
                        index: connectionObject.config.es.index,
                        type : data._TYPE,
                        id   : key,
                        body : data
                    };

                    if (_ttl) {
                        esData.body._ttl = _ttl;
                    }

                    es.create(esData, function(err, res){
                        if (err) {
                            return callback(err, false);
                        }

                        callback(null, data);
                    });
                });
            });
        },

        /**
         * @param collection
         * @param filterQuery
         * @param callback
         */
        find: function(collection, filterQuery, callback){
            if (filterQuery.where === null &&
                !filterQuery.hasOwnProperty('limit') &&
                !filterQuery.hasOwnProperty('skip') &&
                !filterQuery.hasOwnProperty('sort')
            ) {
                return this.getCollection(collection, function(err, res){
                    if (err) {
                        return callback(err, false);
                    }

                    var data = res.map(function(obj){
                        return obj.doc;
                    });

                    callback(null, data);
                });
            }

            var query = buildQuery(collection, filterQuery),
                data = [];

            es.search(query, function(err, res){
                if (err) {
                    return callback(err, false)
                }

                var esData = res.hits.hits;
                for (var key in esData) {
                    data.push(esData[key]._source);
                }

                callback(null, data);
            });
        },

        /**
         * @param collection
         * @param filterQuery
         * @param callback
         */
        destroy: function(collection, filterQuery, callback){
            // force limit update
            filterQuery.limit = 999999999;

            this.find(collection, filterQuery, function(err, res){
                if (err) {
                    return callback(err, false);
                }

                if (res.length < 1) {
                    return callback(null, []);
                }

                for (var key in res) {
                    var entity = res[key],
                        data = {
                            index: connectionObject.config.es.index,
                            type: entity._TYPE,
                            id: entity._TYPE + ':' + entity.id
                        };

                    deleteRecond(data, (parseInt(key) + 1), res.length, function(err){
                        if (err) {
                            return callback(err, false);
                        }

                        callback(null, res);
                    })
                }
            });
        },

        /**
         * @param collection
         * @param filterQuery
         * @param callback
         */
        count: function(collection, filterQuery, callback){
            var query = buildQuery(collection, filterQuery);
            es.search(query, function(err, res){
                if (err) {
                    return callback(err, false)
                }

                callback(null, res.hits.total);
            });
        },

        /**
         * @param collection
         * @param filterQuery
         * @param callback
         */
        aggregate: function(collection, filterQuery, callback){
            var query = buildQuery(collection, filterQuery);
            es.search(query, function(err, res){
                if (err) {
                    return callback(err, false)
                }
                if (res.hasOwnProperty('aggregations')) {
                    callback(null, res.aggregations);
                } else {
                    callback(null, null);
                }
            });
        },

        /**
         * @param collection
         * @param filterQuery
         * @param data
         * @param callback
         */
        update: function(collection, filterQuery, data, callback){
            // force limit update
            filterQuery.limit = 999999999;

            this.find(collection, filterQuery, function(err, res){
                if (err) {
                    return callback(err, false);
                }

                if (res.length < 1) {
                    return callback(null, []);
                }

                var i = 0,
                    _data = [];

                for (var key in res){
                    updateEntity(res[key], data, function(err, _res){
                        if (err) {
                            return callback(err, false);
                        }

                        i++;
                        _data.push(_res);

                        if (i == res.length) {
                            callback(null, _data);
                        }
                    });
                }
            });
        },

        /**
         * @param collection
         * @param callback
         */
        getCollection: function(collection, callback){
            var ViewQuery =  require('couchbase').ViewQuery,
                query = ViewQuery.from(connectionObject.config.cb.bucket.name, collection);

            cb.query(query, {}, function(err, res){
                if (err) {
                    return callback(err, false);
                }

                if (res.length < 1) {
                    return callback(null, []);
                }

                var data = [];
                for (var key in res) {
                    data.push(res[key].value);
                }

                callback(null, data);
            });
        },

        reindex: function(collection, mapping, _ttl, callback){
            var self = this,
                index = connectionObject.config.es.index,
                deleteParam = {
                    index: index,
                    type: collection
                },
                fullMapping = {
                    index: index,
                    type: collection,
                    body : {
                        properties: mapping
                    }
                },
                esEntry = function(data){
                    return {
                        index: index,
                        type: collection,
                        id: collection + ':' + data.id,
                        body: data
                    }
                };

            if (_ttl) {
                fullMapping.body._ttl = _ttl;
            }

            es.indices.deleteMapping(deleteParam, function(err){
                if (err) {
                    var notFound = (err.message.search('TypeMissingException') > -1);

                    if (!notFound) {
                        return callback(err);
                    }
                }

                es.indices.putMapping(fullMapping, function(err){
                    if (err) {
                        return callback(err);
                    }

                    self.getCollection(collection, function(err, res){
                        if (err) {
                            return callback(err);
                        }

                        function iterator(obj, next) {
                            var data = obj.doc;

                            if (_ttl) {
                                data._ttl = Math.round((obj.meta.expiration * 1000) - (new Date().getTime()));

                                if (data._ttl <= 0) {
                                    next(new Error('ttl negative'), false);
                                }
                            }

                            es.create(esEntry(data), next);
                        }

                        var docs = Object.keys(res).map(function(key) { return res[key];});

                        async.forEachLimit(docs, 5000, iterator, callback);
                    });
                });
            });
        },

        backup: cbtools.cbBackup,

        restore: cbtools.cbRestore
    }
};<commit_msg>SEC-231 - added script based sorting<commit_after>/**
 * Created by Luigi Ilie Aron on 16.01.15.
 * email: luigi@kreditech.com
 */
'use strict';
module.exports = function(connectionObject){
    var cb = connectionObject.db.cb,
        es = connectionObject.db.es,
        cbtools = require('./cbtools/cbtools')(connectionObject.config.cb),
        async = require('async'),
        _ = require('lodash');

    /**
     * @param modelType
     * @param query
     * @returns {{index: *, body: {query: {filtered: {query: {bool: {must: {term: {_type: {value: *}}}[]}}, filter: {}}}}}}
     */
    function buildQuery(modelType, query){
        if (typeof query !== 'object'){
            throw new Error('Query is not an object!');
        }

        if (typeof modelType !== 'string'){
            throw new Error('modelType is not an string');
        }

        var _q = {
            query: {
                filtered: {
                    query: {
                        bool: {
                            must: [{
                                term: {
                                    _type: {
                                        value: modelType
                                    }
                                }
                            }]
                        }
                    },
                    filter: {}
                }
            }
        };

        if (query.hasOwnProperty('where')) {
            _q.query.filtered.filter = query.where;
        }

        if (query.hasOwnProperty('limit')) {
            _q.size = query.limit;
        }

        if (query.hasOwnProperty('skip')) {
            _q.from = query.skip;
        }

        if (query.hasOwnProperty('sort')) {
            if (Object.keys(query.sort).length > 0) {
                var sort = [];

                for (var key in query.sort) {
                    var _tmp = {};
                    if (key == "_script") {
                        _tmp = {
                            "_script": query.sort[key]
                        };
                    } else {
                        _tmp[key] = {
                            'order': query.sort[key] > 0 ? 'asc' : 'desc'
                        };
                    }

                    sort.push(_tmp);
                }

                _q.sort = sort;
            }
        }

        if (query.hasOwnProperty('aggs')) {
            _q['aggs'] = _.cloneDeep(query['aggs']);
            delete query['aggs'];
        }

        return {
            index: connectionObject.config.es.index,
            type: modelType,
            body : _q
        };
    }

    /**
     * @param entity
     * @param data
     * @param callback
     */
    function updateEntity(entity, data, callback){
        if (data.hasOwnProperty('id')){
            delete data.id;
        }

        if (data.hasOwnProperty('_TYPE')){
            delete data._TYPE;
        }

        var updatedEntity = _.extend(entity, data),
            key = updatedEntity._TYPE + ':' + updatedEntity.id;

        cb.replace(key, updatedEntity, function(err, res){
            if (err) {
                return callback(err, false)
            }

            var esData = {
                index: connectionObject.config.es.index,
                type : updatedEntity._TYPE,
                id   : key,
                body : {
                    doc: updatedEntity
                }
            };

            es.update(esData, function(err, res){
                if (err) {
                    return callback(err, false);
                }

                callback(null, updatedEntity);
            });
        });
    }

    /**
     * @param data
     * @param current
     * @param total
     * @param callback
     */
    function deleteRecond(data, current, total, callback){
        es.delete(data, function(err, _res){
            if (err) {
                return callback(err);
            }

            cb.remove(data.id, function(err, _res){
                if (err) {
                    return callback(err);
                }

                if (current === total) {
                    callback(null);
                }
            });
        });
    }

    return {
        /**
         * @param collection
         * @param data
         * @param callback
         */
        create: function(collection, data, _ttl, callback){
            cb.counter(collection + ":count" , 1, {initial: 0, offset: 1}, function(err, res){
                if(err){
                    return callback(err);
                }

                var key = collection + ":" + res.value,
                    meta = {},
                    ttlInterval = connectionObject.config.es.ttlInterval;

                data.id = res.value;
                data._TYPE = collection;

                if (_ttl) {
                    meta.expiry = Math.floor((new Date().getTime() + _ttl + ttlInterval) / 1000);
                }

                cb.insert(key, data, meta, function(err, res){
                    if (err) {
                        return callback(err, false);
                    }

                    var esData = {
                        index: connectionObject.config.es.index,
                        type : data._TYPE,
                        id   : key,
                        body : data
                    };

                    if (_ttl) {
                        esData.body._ttl = _ttl;
                    }

                    es.create(esData, function(err, res){
                        if (err) {
                            return callback(err, false);
                        }

                        callback(null, data);
                    });
                });
            });
        },

        /**
         * @param collection
         * @param filterQuery
         * @param callback
         */
        find: function(collection, filterQuery, callback){
            if (filterQuery.where === null &&
                !filterQuery.hasOwnProperty('limit') &&
                !filterQuery.hasOwnProperty('skip') &&
                !filterQuery.hasOwnProperty('sort')
            ) {
                return this.getCollection(collection, function(err, res){
                    if (err) {
                        return callback(err, false);
                    }

                    var data = res.map(function(obj){
                        return obj.doc;
                    });

                    callback(null, data);
                });
            }

            var query = buildQuery(collection, filterQuery),
                data = [];

            es.search(query, function(err, res){
                if (err) {
                    return callback(err, false)
                }

                var esData = res.hits.hits;
                for (var key in esData) {
                    data.push(esData[key]._source);
                }

                callback(null, data);
            });
        },

        /**
         * @param collection
         * @param filterQuery
         * @param callback
         */
        destroy: function(collection, filterQuery, callback){
            // force limit update
            filterQuery.limit = 999999999;

            this.find(collection, filterQuery, function(err, res){
                if (err) {
                    return callback(err, false);
                }

                if (res.length < 1) {
                    return callback(null, []);
                }

                for (var key in res) {
                    var entity = res[key],
                        data = {
                            index: connectionObject.config.es.index,
                            type: entity._TYPE,
                            id: entity._TYPE + ':' + entity.id
                        };

                    deleteRecond(data, (parseInt(key) + 1), res.length, function(err){
                        if (err) {
                            return callback(err, false);
                        }

                        callback(null, res);
                    })
                }
            });
        },

        /**
         * @param collection
         * @param filterQuery
         * @param callback
         */
        count: function(collection, filterQuery, callback){
            var query = buildQuery(collection, filterQuery);
            es.search(query, function(err, res){
                if (err) {
                    return callback(err, false)
                }

                callback(null, res.hits.total);
            });
        },

        /**
         * @param collection
         * @param filterQuery
         * @param callback
         */
        aggregate: function(collection, filterQuery, callback){
            var query = buildQuery(collection, filterQuery);
            es.search(query, function(err, res){
                if (err) {
                    return callback(err, false)
                }
                if (res.hasOwnProperty('aggregations')) {
                    callback(null, res.aggregations);
                } else {
                    callback(null, null);
                }
            });
        },

        /**
         * @param collection
         * @param filterQuery
         * @param data
         * @param callback
         */
        update: function(collection, filterQuery, data, callback){
            // force limit update
            filterQuery.limit = 999999999;

            this.find(collection, filterQuery, function(err, res){
                if (err) {
                    return callback(err, false);
                }

                if (res.length < 1) {
                    return callback(null, []);
                }

                var i = 0,
                    _data = [];

                for (var key in res){
                    updateEntity(res[key], data, function(err, _res){
                        if (err) {
                            return callback(err, false);
                        }

                        i++;
                        _data.push(_res);

                        if (i == res.length) {
                            callback(null, _data);
                        }
                    });
                }
            });
        },

        /**
         * @param collection
         * @param callback
         */
        getCollection: function(collection, callback){
            var ViewQuery =  require('couchbase').ViewQuery,
                query = ViewQuery.from(connectionObject.config.cb.bucket.name, collection);

            cb.query(query, {}, function(err, res){
                if (err) {
                    return callback(err, false);
                }

                if (res.length < 1) {
                    return callback(null, []);
                }

                var data = [];
                for (var key in res) {
                    data.push(res[key].value);
                }

                callback(null, data);
            });
        },

        reindex: function(collection, mapping, _ttl, callback){
            var self = this,
                index = connectionObject.config.es.index,
                deleteParam = {
                    index: index,
                    type: collection
                },
                fullMapping = {
                    index: index,
                    type: collection,
                    body : {
                        properties: mapping
                    }
                },
                esEntry = function(data){
                    return {
                        index: index,
                        type: collection,
                        id: collection + ':' + data.id,
                        body: data
                    }
                };

            if (_ttl) {
                fullMapping.body._ttl = _ttl;
            }

            es.indices.deleteMapping(deleteParam, function(err){
                if (err) {
                    var notFound = (err.message.search('TypeMissingException') > -1);

                    if (!notFound) {
                        return callback(err);
                    }
                }

                es.indices.putMapping(fullMapping, function(err){
                    if (err) {
                        return callback(err);
                    }

                    self.getCollection(collection, function(err, res){
                        if (err) {
                            return callback(err);
                        }

                        function iterator(obj, next) {
                            var data = obj.doc;

                            if (_ttl) {
                                data._ttl = Math.round((obj.meta.expiration * 1000) - (new Date().getTime()));

                                if (data._ttl <= 0) {
                                    next(new Error('ttl negative'), false);
                                }
                            }

                            es.create(esEntry(data), next);
                        }

                        var docs = Object.keys(res).map(function(key) { return res[key];});

                        async.forEachLimit(docs, 5000, iterator, callback);
                    });
                });
            });
        },

        backup: cbtools.cbBackup,

        restore: cbtools.cbRestore
    }
};
<|endoftext|>


<commit_before>
var mmm = require("multi-master-merge");
var level = require("level"); // LevelUP + LevelDOWN
var path = require("path");
var url = require("url");
var mkdirp = require("mkdirp");
var sublevel = require("level-sublevel");
var Map = require("es6-map");
var _ = require("lodash");
var net = require("net");
var DHT = require("bittorrent-dht");
var SSDP = require("node-ssdp");
var ip = require("ip");

// hole punch for our replication interface
var entry = require("entry");
var natPmp = require("nat-pmp");

var log = require("./log");

var dht = new DHT();
var ssdp = new SSDP.Client();

var dbPath = path.join(module.parent.dbPath);
mkdirp.sync(dbPath);
var db = mmm(level(dbPath), { encoding: "json", gc: true, postupdate: updateIndexes });
var AVLTree = require("binary-search-tree").AVLTree;

/* Indexes
 */
db.indexes = { meta: new AVLTree(), seeders: new Map() };
function updateIndexes(entry, callback)
{
	var torrent = entry.value;
	var maxSeeders = Math.max.apply(Math, _.values(torrent.popularity).map(function(x) { return x[0] }));
	// index by seeders  ; consider availability
	// index by metadata
	//console.log(maxSeeders)
	torrent.files.forEach(function(x) {
		var hash = [x.imdb_id, x.season, x.episode && x.episode[0]].filter(function(x) { return x }).join(" ");
		db.indexes.meta.insert(hash, torrent.infoHash); // TODO: find a way to sort by maxSeeders in this idx
		db.indexes.seeders.set(torrent.infoHash, maxSeeders);
	});
	callback && callback();
}
db.createValueStream().on("data", function(val) {  updateIndexes({ value: val.pop() }) });

/* Replication - server & swarm
 */
var server;
db.listenReplications = function(id) {
	server = net.createServer(function(c) {
		log.important("DB replication connection established from "+c.remoteAddress+":"+c.remotePort);
		c.pipe(db.sync()).pipe(c);
		c.on("error", function(err) { console.log(err) });
	});
	server.listen(function() {
		var port = server.address().port;

		log.important("DB replication server listening at "+port);
		dht.announce(id, port);

		// Hole punch so this server is accessible behind router firewalls
		// We can play around with public / private - maybe we want a consistent public port?
		try {
			// Both throw err in their async code which we cannot catch
			entry.map({ external: port, internal: port, name: require("../package").name }, function(e) { e && console.error(e) });
			//natPmp.connect("10.0.1.1").portMapping({ public: port, private: port, ttl: 1000, description: require("../package").name }, function(e) { e && console.error(e) });
		} catch(e) { console.error(e) }
	});

	// Announce as an SSDP server
	var ssdpServer = new SSDP.Server({ location: ip.address() +":" + server.address().port });
	ssdpServer.addUSN("upnp:rootdevice");
	ssdpServer.addUSN("urn:schemas-upnp-org:service:MultiPassTorrent:"+id);
	ssdpServer.start("0.0.0.0");
};

db.findReplications = function(id) {
	log.important(id+": finding other instances to replicate with");

	// WARNING: what if it emits beforehand?
	dht.on("ready", function() {
		dht.lookup(id);
	});

	ssdp.search("urn:schemas-upnp-org:service:MultiPassTorrent:"+id);
};

var peers = { };
function onpeer(addr)
{
	if (ip.address()+":"+server.address().port == addr) return; // Do not connect to ourselves

	if (peers[addr]) return;
	peers[addr] = true;

	var spl = addr.split(":");
	var c = net.connect(spl[1], spl[0]);
	c.on("connect", function() {
		log.message("connected to peer "+addr); // TODO: handle errs
		c.pipe(db.sync()).pipe(c);
	}).on("error", function(e) {
		c.destroy()
	}).on("end", function() {
		c.destroy();
		delete peers[addr];
	}).on("close", function() {
		// cleanup sync pipe?
	});
};
dht.on("peer", onpeer);
ssdp.on("response", function(meta, status, matchine) { onpeer(meta.LOCATION) });

module.exports = db;<commit_msg>Implemented db.find function, although maybe it should be called db.lookup<commit_after>
var mmm = require("multi-master-merge");
var level = require("level"); // LevelUP + LevelDOWN
var path = require("path");
var url = require("url");
var mkdirp = require("mkdirp");
var sublevel = require("level-sublevel");
var Map = require("es6-map");
var _ = require("lodash");
var net = require("net");
var DHT = require("bittorrent-dht");
var SSDP = require("node-ssdp");
var ip = require("ip");

// hole punch for our replication interface
var entry = require("entry");
var natPmp = require("nat-pmp");

var log = require("./log");

var dht = new DHT();
var ssdp = new SSDP.Client();

var dbPath = path.join(module.parent.dbPath);
mkdirp.sync(dbPath);
var db = mmm(level(dbPath), { encoding: "json", gc: true, postupdate: updateIndexes });
var AVLTree = require("binary-search-tree").AVLTree;

/* Indexes
 */
db.indexes = { meta: new AVLTree(), seeders: new Map() };
function getHash(x) {
	return [x.imdb_id, x.season, (x.episode && x.episode[0]) || x.episode ].filter(function(x) { return x }).join(" ")
}
function updateIndexes(entry, callback)
{
	var torrent = entry.value;
	var maxSeeders = Math.max.apply(Math, _.values(torrent.popularity).map(function(x) { return x[0] }));
	// index by seeders  ; consider availability
	// index by metadata
	//console.log(maxSeeders)
	torrent.files.forEach(function(x) {
		var hash = getHash(x);
		db.indexes.meta.insert(hash, torrent.infoHash); // TODO: find a way to sort by maxSeeders in this idx
		db.indexes.seeders.set(torrent.infoHash, maxSeeders);
	});
	callback && callback();
}
db.createValueStream().on("data", function(val) {  updateIndexes({ value: val.pop() }) });

/* Querying
 */
db.find = function(query, items, callback)
{
	var ids = db.indexes.meta.search(getHash(query))
		.map(function(id) { return { id: id, seeders: db.indexes.seeders.get(id) || 0 } })
		.sort(function(a,b) { return b.seeders - a.seeders })
		.slice(0, items);
};
/*
setTimeout(function() {
	db.find({ imdb_id: "tt1392190" },5)
	db.find({ imdb_id: "tt2243973", season: 3, episode: 11 },5)
}, 5000);
*/

/* Replication - server & swarm
 */
var server;
db.listenReplications = function(id) {
	server = net.createServer(function(c) {
		log.important("DB replication connection established from "+c.remoteAddress+":"+c.remotePort);
		c.pipe(db.sync()).pipe(c);
		c.on("error", function(err) { console.log(err) });
	});
	server.listen(function() {
		var port = server.address().port;

		log.important("DB replication server listening at "+port);
		dht.announce(id, port);

		// Hole punch so this server is accessible behind router firewalls
		// We can play around with public / private - maybe we want a consistent public port?
		try {
			// Both throw err in their async code which we cannot catch
			entry.map({ external: port, internal: port, name: require("../package").name }, function(e) { e && console.error(e) });
			//natPmp.connect("10.0.1.1").portMapping({ public: port, private: port, ttl: 1000, description: require("../package").name }, function(e) { e && console.error(e) });
		} catch(e) { console.error(e) }
	});

	// Announce as an SSDP server
	var ssdpServer = new SSDP.Server({ location: ip.address() +":" + server.address().port });
	ssdpServer.addUSN("upnp:rootdevice");
	ssdpServer.addUSN("urn:schemas-upnp-org:service:MultiPassTorrent:"+id);
	ssdpServer.start("0.0.0.0");
};

db.findReplications = function(id) {
	log.important(id+": finding other instances to replicate with");

	// WARNING: what if it emits beforehand?
	dht.on("ready", function() {
		dht.lookup(id);
	});

	ssdp.search("urn:schemas-upnp-org:service:MultiPassTorrent:"+id);
};

/* Swarm
 */
var peers = { };
function onpeer(addr)
{
	if (ip.address()+":"+server.address().port == addr) return; // Do not connect to ourselves

	if (peers[addr]) return;
	peers[addr] = true;

	var spl = addr.split(":");
	var c = net.connect(spl[1], spl[0]);
	c.on("connect", function() {
		log.message("connected to peer "+addr); // TODO: handle errs
		c.pipe(db.sync()).pipe(c);
	}).on("error", function(e) {
		c.destroy()
	}).on("end", function() {
		c.destroy();
		delete peers[addr];
	}).on("close", function() {
		// cleanup sync pipe?
	});
};
dht.on("peer", onpeer);
ssdp.on("response", function(meta, status, matchine) { onpeer(meta.LOCATION) });

module.exports = db;<|endoftext|>


<commit_before>/**
 * lib/db.js - Module for database connection handling.
 *
 * Authors: Ian McGaunn; Dave Zimmelman
 * Modified: 13 Apr 15
 */

var mongoose = require('mongoose');

// connect to database and define a basic error handler
mongoose.connect('mongodb://localhost/podplay_db');
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));

db.once('open', function (callback) {
    console.log('connected to database');
});

module.exports = {
    mongoose : mongoose,

    /**
     * Handle unexpected database errors.
     * @param {Error} err - The error returned by database operation.
     * @param {failCallback} cb
     */
    dbErrorHandler : function (err, cb) {
        var error = new Error('Internal Server Error');
        error.err = err;
        error.status = 500;
        cb(error, "An unexpected database error occurred.");
    },

    /**
     * Save document to database.
     * @param {Object} doc - Document to save
     * @param {failCallback} fail
     * @param {successCallback} pass
     */
    dbSaveDoc : function (doc, fail, pass) {
        doc.save(function (err, doc) {
            if (err) {
                // Unexpected database error.
                dbErr(err, fail);
            }
            else {
                // Save succeeded.
                pass(doc);
            }
        });
    }
};<commit_msg>[Fix] Bad identifier<commit_after>/**
 * lib/db.js - Module for database connection handling.
 *
 * Authors: Ian McGaunn; Dave Zimmelman
 * Modified: 13 Apr 15
 */

var mongoose = require('mongoose');

// connect to database and define a basic error handler
mongoose.connect('mongodb://localhost/podplay_db');
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));

db.once('open', function (callback) {
    console.log('connected to database');
});

module.exports = {
    mongoose : mongoose,

    /**
     * Handle unexpected database errors.
     * @param {Error} err - The error returned by database operation.
     * @param {failCallback} cb
     */
    dbErrorHandler : function (err, cb) {
        var error = new Error('Internal Server Error');
        error.err = err;
        error.status = 500;
        cb(error, "An unexpected database error occurred.");
    },

    /**
     * Save document to database.
     * @param {Object} doc - Document to save
     * @param {failCallback} fail
     * @param {successCallback} pass
     */
    dbSaveDoc : function (doc, fail, pass) {
        doc.save(function (err, doc) {
            if (err) {
                // Unexpected database error.
                dbErrorHandler(err, fail);
            }
            else {
                // Save succeeded.
                pass(doc);
            }
        });
    }
};<|endoftext|>


<commit_before>/* fs.js: file system primitives
 * Copyright © 2011 Jan Keromnes, Thaddee Tyl. All rights reserved.
 * The following code is covered by the GPLv2 license. */

var camp = require ('../camp/camp');

// All data currently available in memory will be stored here.
//

var nodefs = require ('fs'),
    nodepath = require ('path');

// Here, we assume that the code runs in the lib/ directory.
var rootdir = '../root';  // directory that contains the root of the fs.

// Type system.
//

var types = [];        // contains lists of parent types (all being integers).
var typenamefromtype = [];
var Type = (function () {
  var mime = {};       // contains link between mime type and integer.
  var nbtypes = 0;     // number of different types.

  // `newtype` is a string of the mime type.
  // `parents` is a list of integers containing the parent types.
  // You get the number corresponding to the type.
  return function (newtype, parents) {
    if (mime[newtype] !== undefined) {
      return mime[newtype];
    } else {
      types.push (parents || []);
      mime[newtype] = nbtypes;
      typenamefromtype.push (newtype);
      return nbtypes++;
    }
  };
})();

// Defining current types (and their compatibility table).
Type ('notfound');
Type ('dir');            // contains a JSON string of files.
Type ('binary');
Type ('text/plain');
Type ('text/html', [Type('text/plain')]);
Type ('text/xml', [Type('text/plain')]);
Type ('text/css', [Type('text/plain')]);
Type ('text/dtd', [Type('text/plain')]);

Type ('text/javascript', [Type('text/plain')]);
Type ('application/json', [Type('text/plain')]);
Type ('text/csv', [Type('text/plain')]);
Type ('text/x-csrc', [Type('text/plain')]);
Type ('text/x-c++src', [Type('text/plain')]);
Type ('text/x-java', [Type('text/plain')]);
Type ('text/x-groovy', [Type('text/plain')]);
Type ('text/x-clojure', [Type('text/plain')]);
Type ('text/x-coffeescript', [Type('text/plain')]);
Type ('text/x-diff', [Type('text/plain')]);
Type ('text/x-haskell', [Type('text/plain')]);
Type ('text/x-lua', [Type('text/plain')]);
Type ('text/x-markdown', [Type('text/plain')]);
Type ('text/n-triples', [Type('text/plain')]);
Type ('text/x-pascal', [Type('text/plain')]);
Type ('text/x-perl', [Type('text/plain')]);
Type ('text/x-php', [Type('text/plain')]);
Type ('text/x-plsql', [Type('text/plain')]);
Type ('text/x-plsql', [Type('text/plain')]);
Type ('text/x-python', [Type('text/plain')]);
Type ('text/x-rsrc', [Type('text/plain')]);
Type ('text/x-rst', [Type('text/plain')]);
Type ('text/x-ruby', [Type('text/plain')]);
Type ('text/x-rustsrc', [Type('text/plain')]);
Type ('text/x-scheme', [Type('text/plain')]);
Type ('text/x-scheme', [Type('text/plain')]);
Type ('text/x-scheme', [Type('text/plain')]);
Type ('text/x-stsrc', [Type('text/plain')]);
Type ('text/x-sparql-query', [Type('text/plain')]);
Type ('text/x-stex', [Type('text/plain')]);
Type ('text/x-stex', [Type('text/plain')]);
Type ('text/velocity', [Type('text/plain')]);
Type ('text/x-yaml', [Type('text/plain')]);

Type ('application/ps', [Type('binary')]);
Type ('application/pdf', [Type('binary'), Type('application/ps')]);

Type ('image/jpeg', [Type('binary')]);
Type ('image/tiff', [Type('binary')]);
Type ('image/gif', [Type('binary')]);
Type ('image/vnd.microsoft.icon', [Type('binary')]);
Type ('image/png', [Type('binary')]);
Type ('image/svg+xml', [Type('text/plain')]);

// Here is the main interface to this whole type system.

// Checks whether a `file` is of type `typename` (or falls back into it).
//
// `file`: a File.
// `typename`: a string, case-perfect match of a defined Type.
//
// Warning: this is not *cycle-safe* just yet. This is only an issue if you
// have mistakenly created a loop. It can really screw everything up though.
function isoftype (file, typename) {
  if (file === undefined) {
    console.log('FS: isoftype caught undefined file.');
    return;
  }
  return iscompatibletype(file.type, Type(typename));
};
function iscompatibletype (typenum, ancestor) {
  if (typenum === ancestor)
    return true;
  for (var i = 0;  i < types[typenum].length;  i++)
    if (iscompatibletype(types[typenum][i], ancestor))
      return true;
  return false;
};

// Gather information.
//
// Numbers are to computer science what a telescope is to an astrophysicist.

var profile = [
  {doc:'FS running', data:true}
];

// Directory primitives.

/* `typename` is a string taken from the `types` enumeration.
 * `name` is a string.
 * `getcontent: function ( whengot: function(err, content) )`
 * is a function returning further information.
 *
 * - the content as a string for text files,
 * - an object with a property for each file, that links to the corresponding
 *   <File> object, for directories.
 *
 * This is the constructor for the File object.
 *
 *     var f = new File ( 'text/plain', 'myfile', function(whengot) {
 *       …
 *       var data = whatever ();
 *       whengot ( err, data );
 *     });
 *
 * - File::type is the type of the file.
 * - File::name is the name of the file (supposedly doesn't contain `/`).
 * - File::content is a function which you feed `function(err, content) {…}`.
 * - File::open will count one more user editing that file.
 * - File::close will count one less user. (That doesn't really matter
 * for folders).
 */
function File (typename, name, getcontent) {
  this.type = Type(typename) || Type('text/plain');
  this.name = name;
  this._gotcontent = false;
  this._content;
  this.content = function memoizer(dowithcontent) {
    console.log('FS: file.content has just been called.');
    if (!this._gotcontent) {
      var that = this;
      getcontent(function(err, content) {
        ///console.log('FS: We just got file.content:', content);
        // Give them the data.
        dowithcontent(err, content);
        if (!err) {
          that._content = content;
          that._gotcontent = true;
        } else console.error('FS: getcontent had an error!');
      });
    } else {
      ///console.log('FS: We just got file.content [cached]:', this._content);
      dowithcontent(undefined, this._content);
    }
  };
  this.usercount = 0;
};

File.prototype.open = function () {
  this.usercount++;
};

File.prototype.close = function () {
  if (this.usercount < 0) {
	console.log('FS:FILE: usercount is negative (%s). Keeping it at 0.',
		this.usercount);
  } else {
	this.usercount--;
  }
};

// "Alias" of the raw isoftype.
File.prototype.isoftype = function (nametype) {
  return isoftype(this, nametype);
};

// `Subfiles` gives all the leafs of the tree recursively, as relative paths.
//
// The callback `cb` is of the form `function(err, leafs) { ... }`.
File.prototype.subfiles = function (cb, depth) {
  depth = depth !== undefined? depth: Infinity;

  // This function is meant to return [] if this is not a directory.
  // It mustn't go deeper if depth has reached the lowest level.
  if (!this.isoftype('dir')) { cb(undefined, []); return; }

  // The basic idea is that `processedfiles` will be incremented until it
  // reaches `totalfiles`, at which point we will be done, `paths` will be
  // complete.
  var paths = [], processedfiles = 0, totalfiles;

  var whengotsubfiles = function(filename) {
    return function(err, subfiles) {
      // Add all subfiles.
      for (var i = 0; i < subfiles.length; i++) {
        paths.push(nodepath.join(filename, subfiles[i]));
      }
      processedfiles++;
      if (processedfiles === totalfiles) {
        cb(undefined, paths);
      }
    };
  };

  this.content(function(err, content) {
    if (err) { cb(err); return; }

    totalfiles = Object.keys(content).length;
    for (var filename in content) {
      var file = content[filename];
      // Special case: what if `file` is not a leaf?
      if (file.isoftype('dir') && depth > 1) {
        // We need to recursively go deeper! ;)
        file.subfiles(whengotsubfiles(filename), depth - 1);
      } else {
        paths.push(filename);
        processedfiles++;
      }
    }
    // Some directories contain only leafs.
    if (processedfiles === totalfiles) {
      cb(undefined, paths);
    }
  });
};

// Read files from the hard drive.
//

// Return a fake path from the fake root (with no leading `/`).
function sanitizepath (path) {
  return nodepath.relative ('.', path).replace (/^(\.\.\/?)+/, '');
};

/* Convert a virtual path from the /root/ folder, to the filesystem path.
 * `path`: virtual path (in the form of a String).
 * It needs to be sanitized with `sanitizepath`.
 */
function torealpath (path) {
  return nodepath.join (process.cwd(), nodepath.join (rootdir, path));
};

/* This hashmap registers all files ever asked for.
 * The keys are the *fake* paths of those files.
 */
var fsfiles = {};

/* Read the path, construct the file according to the filename.
 * `path`: String of "fake" path starting with a `/`.
 * `callback`: function (err, <new File()>).
 */
function getfile (path, callback) {
  var oldpath = path;
  path = sanitizepath(path);
  console.log('FS:GETFILE: %s was sanitized to "%s"', oldpath, path);
  var realpath = torealpath (path);
  // If this file is already in memory, return that.
  if (fsfiles[path] !== undefined) {
    callback(undefined, fsfiles[path]);
    return;
  }

  nodefs.stat (realpath, function (err, stats) {
    if (!err) {

      //console.log('stats says that', realpath, (stats.isDirectory()? 'is a directory': 'is a file'));

      if (stats.isDirectory()) {
        // When creating a directory to memory, the getcontent
        // component is the following function.
        // It reads the files in the entry.
        var getcontent = function (whengotfile) {
          nodefs.readdir (realpath, function (err, files) {
            if (!err) {
			  console.log('FS: directory files are', files);
			  if (files.length === 0) {
				whengotfile (undefined, {});
			  }
              var content = {};
              for (var i = 0; i < files.length; i++) {
                // We want to call back only when we've got all files.
                // We use a decrement to trace the progress.
                var fileslefttoprocess = files.length;
                getfile (
                  nodepath.join (path, files[i]),
                  function (err, file) {
                    fileslefttoprocess--;
                    if (err) { whengotfile (err); return; }
                    // We put each file in the content.
                    content[file.name] = file;
                    if (fileslefttoprocess === 0) {
                      whengotfile (undefined, content);
                    }
                  }
                );
              }
            } else {
              whengotfile (err);
            }
          });
        };
        var newfile = new File (
            'dir',
            nodepath.basename (realpath),
            getcontent
        );
        callback (undefined, newfile);
        fsfiles[path] = newfile;

      } else if (stats.isFile()) {
        // When creating a file, the getcontent
        // seeks the content of the file.
        var getcontent = function (whengotfile) {
          // FIXME: right now, we assume utf8 text.
          nodefs.readFile (realpath, 'utf8', function (err, data) {
            if (err) { whengotfile (err, undefined); return; }
            whengotfile (undefined, data);
          });
        };
        var newfile = new File (
            typefromextention(realpath),
            nodepath.basename (realpath),
            getcontent
        );
        callback (undefined, newfile);
        fsfiles[path] = newfile;
      }

    } else {
      callback (err);
    }
  });
};

// Get the type associated to the extention of the file.
//
function typefromextention (filename) {
  //console.log('FS:TYPEFROMEXTENTION:', filename, '-',
      //camp.Server.mime[nodepath.extname(filename).slice(1)]);
  return camp.Server.mime[nodepath.extname(filename).slice(1)] || 'text/plain';
};

// Create new files as requested.
//

/* `file` is a File (easily constructed with `new File()`).
 * `directory` is a File of type `dir`.
 */
function addfiletodir (file, directory) {
  // First, we need to know what the directory contains.
  //var files = directory.getcontent ();

  // TODO nodejs apis to write to disk.
  // nodefs.writeFile(filename, data, encoding='utf8',[callback])
  // nodefs.mkdir(path, mode, [callback]) // unix permissions mode (eg. 0777)
};

// Remove a file.
// TODO
function rmfilefromdir (file, directory) {
};

// Initialization of the File System.
//
//   Yes, right here, right now. Impressive huh?
//

// Choice to make here. Do we read the whole thing on startup?
//   + Faster boot time.
//   - Too much info to deal with; could take all memory.
//
// `getroot` yields a file of type `dir`, which is here initialized to the
// contents of the `/root/` folder (named `/` in here).
function getroot (gotroot) {
  getfile('/', function(err, root) { gotroot(err, root); });
};

// EXPORTS ARE HERE YOU BITCHES
//

exports.typenamefromtype = typenamefromtype;
exports.isoftype = isoftype;
exports.File = File;
exports.fsfiles = fsfiles;
exports.sanitizepath = sanitizepath;
exports.getfile = getfile;
exports.getroot = getroot;

exports.profile = profile;

<commit_msg>[Fuzzy] Bugfix about duplication of leaves<commit_after>/* fs.js: file system primitives
 * Copyright © 2011 Jan Keromnes, Thaddee Tyl. All rights reserved.
 * The following code is covered by the GPLv2 license. */

var camp = require ('../camp/camp');

// All data currently available in memory will be stored here.
//

var nodefs = require ('fs'),
    nodepath = require ('path');

// Here, we assume that the code runs in the lib/ directory.
var rootdir = '../root';  // directory that contains the root of the fs.

// Type system.
//

var types = [];        // contains lists of parent types (all being integers).
var typenamefromtype = [];
var Type = (function () {
  var mime = {};       // contains link between mime type and integer.
  var nbtypes = 0;     // number of different types.

  // `newtype` is a string of the mime type.
  // `parents` is a list of integers containing the parent types.
  // You get the number corresponding to the type.
  return function (newtype, parents) {
    if (mime[newtype] !== undefined) {
      return mime[newtype];
    } else {
      types.push (parents || []);
      mime[newtype] = nbtypes;
      typenamefromtype.push (newtype);
      return nbtypes++;
    }
  };
})();

// Defining current types (and their compatibility table).
Type ('notfound');
Type ('dir');            // contains a JSON string of files.
Type ('binary');
Type ('text/plain');
Type ('text/html', [Type('text/plain')]);
Type ('text/xml', [Type('text/plain')]);
Type ('text/css', [Type('text/plain')]);
Type ('text/dtd', [Type('text/plain')]);

Type ('text/javascript', [Type('text/plain')]);
Type ('application/json', [Type('text/plain')]);
Type ('text/csv', [Type('text/plain')]);
Type ('text/x-csrc', [Type('text/plain')]);
Type ('text/x-c++src', [Type('text/plain')]);
Type ('text/x-java', [Type('text/plain')]);
Type ('text/x-groovy', [Type('text/plain')]);
Type ('text/x-clojure', [Type('text/plain')]);
Type ('text/x-coffeescript', [Type('text/plain')]);
Type ('text/x-diff', [Type('text/plain')]);
Type ('text/x-haskell', [Type('text/plain')]);
Type ('text/x-lua', [Type('text/plain')]);
Type ('text/x-markdown', [Type('text/plain')]);
Type ('text/n-triples', [Type('text/plain')]);
Type ('text/x-pascal', [Type('text/plain')]);
Type ('text/x-perl', [Type('text/plain')]);
Type ('text/x-php', [Type('text/plain')]);
Type ('text/x-plsql', [Type('text/plain')]);
Type ('text/x-plsql', [Type('text/plain')]);
Type ('text/x-python', [Type('text/plain')]);
Type ('text/x-rsrc', [Type('text/plain')]);
Type ('text/x-rst', [Type('text/plain')]);
Type ('text/x-ruby', [Type('text/plain')]);
Type ('text/x-rustsrc', [Type('text/plain')]);
Type ('text/x-scheme', [Type('text/plain')]);
Type ('text/x-scheme', [Type('text/plain')]);
Type ('text/x-scheme', [Type('text/plain')]);
Type ('text/x-stsrc', [Type('text/plain')]);
Type ('text/x-sparql-query', [Type('text/plain')]);
Type ('text/x-stex', [Type('text/plain')]);
Type ('text/x-stex', [Type('text/plain')]);
Type ('text/velocity', [Type('text/plain')]);
Type ('text/x-yaml', [Type('text/plain')]);

Type ('application/ps', [Type('binary')]);
Type ('application/pdf', [Type('binary'), Type('application/ps')]);

Type ('image/jpeg', [Type('binary')]);
Type ('image/tiff', [Type('binary')]);
Type ('image/gif', [Type('binary')]);
Type ('image/vnd.microsoft.icon', [Type('binary')]);
Type ('image/png', [Type('binary')]);
Type ('image/svg+xml', [Type('text/plain')]);

// Here is the main interface to this whole type system.

// Checks whether a `file` is of type `typename` (or falls back into it).
//
// `file`: a File.
// `typename`: a string, case-perfect match of a defined Type.
//
// Warning: this is not *cycle-safe* just yet. This is only an issue if you
// have mistakenly created a loop. It can really screw everything up though.
function isoftype (file, typename) {
  if (file === undefined) {
    console.log('FS: isoftype caught undefined file.');
    return;
  }
  return iscompatibletype(file.type, Type(typename));
};
function iscompatibletype (typenum, ancestor) {
  if (typenum === ancestor)
    return true;
  for (var i = 0;  i < types[typenum].length;  i++)
    if (iscompatibletype(types[typenum][i], ancestor))
      return true;
  return false;
};

// Gather information.
//
// Numbers are to computer science what a telescope is to an astrophysicist.

var profile = [
  {doc:'FS running', data:true}
];

// Directory primitives.

/* `typename` is a string taken from the `types` enumeration.
 * `name` is a string.
 * `getcontent: function ( whengot: function(err, content) )`
 * is a function returning further information.
 *
 * - the content as a string for text files,
 * - an object with a property for each file, that links to the corresponding
 *   <File> object, for directories.
 *
 * This is the constructor for the File object.
 *
 *     var f = new File ( 'text/plain', 'myfile', function(whengot) {
 *       …
 *       var data = whatever ();
 *       whengot ( err, data );
 *     });
 *
 * - File::type is the type of the file.
 * - File::name is the name of the file (supposedly doesn't contain `/`).
 * - File::content is a function which you feed `function(err, content) {…}`.
 * - File::open will count one more user editing that file.
 * - File::close will count one less user. (That doesn't really matter
 * for folders).
 */
function File (typename, name, getcontent) {
  this.type = Type(typename) || Type('text/plain');
  this.name = name;
  this._gotcontent = false;
  this._content;
  this.content = function memoizer(dowithcontent) {
    console.log('FS: file.content has just been called.');
    if (!this._gotcontent) {
      var that = this;
      getcontent(function(err, content) {
        ///console.log('FS: We just got file.content:', content);
        // Give them the data.
        dowithcontent(err, content);
        if (!err) {
          that._content = content;
          that._gotcontent = true;
        } else console.error('FS: getcontent had an error!');
      });
    } else {
      ///console.log('FS: We just got file.content [cached]:', this._content);
      dowithcontent(undefined, this._content);
    }
  };
  this.usercount = 0;
};

File.prototype.open = function () {
  this.usercount++;
};

File.prototype.close = function () {
  if (this.usercount < 0) {
	console.log('FS:FILE: usercount is negative (%s). Keeping it at 0.',
		this.usercount);
  } else {
	this.usercount--;
  }
};

// "Alias" of the raw isoftype.
File.prototype.isoftype = function (nametype) {
  return isoftype(this, nametype);
};

// `Subfiles` gives all the leafs of the tree recursively, as relative paths.
//
// The callback `cb` is of the form `function(err, leafs) { ... }`.
File.prototype.subfiles = function (cb, depth) {
  depth = depth !== undefined? depth: Infinity;

  // This function is meant to return [] if this is not a directory.
  if (!this.isoftype('dir')) { cb(undefined, []); return; }

  // The basic idea is that `processedfiles` will be incremented until it
  // reaches `totalfiles`, at which point we will be done, `paths` will be
  // complete.
  var paths = [], processedfiles = 0, totalfiles;

  var whengotsubfiles = function(filename) {
    //console.log('FS:SUBFILES: whengotsubfiles called with filename', filename);
    return function(err, subfiles) {
      // Add all subfiles.
      for (var i = 0; i < subfiles.length; i++) {
        //console.log('FS:SUBFILES: %s + %s -> %s', filename, subfiles[i], nodepath.join(filename, subfiles[i]));
        //if (paths.indexOf(nodepath.join(filename, subfiles[i])) !== -1) {
          //console.log('FS:SUBFILES: adding duplicate %s/%s to %s', filename, subfiles[i], paths);
        //}
        paths.push(nodepath.join(filename, subfiles[i]));
      }
      processedfiles++;
      //console.log('FS:SUBFILES: got subfiles %s from filename %s (%s/%s)', subfiles, filename, processedfiles, totalfiles);
      if (processedfiles === totalfiles) {
        //var util = require('util');
        //console.log('FS:SUBFILES: filename %s gives paths %s', filename, util.inspect(paths));
        cb(undefined, paths);
      }
    };
  };

  this.content(function(err, content) {
    if (err) { cb(err); return; }

    totalfiles = Object.keys(content).length;
    for (var filename in content) {
      var file = content[filename];
      // Special case: what if `file` is not a leaf?
      if (file.isoftype('dir') && depth > 0) {
        // We need to recursively go deeper! ;)
        file.subfiles(whengotsubfiles(filename), depth - 1);
      } else {
        whengotsubfiles(this)(undefined, [filename]);
      }
    }
  });
};

// Read files from the hard drive.
//

// Return a fake path from the fake root (with no leading `/`).
function sanitizepath (path) {
  return nodepath.relative ('.', path).replace (/^(\.\.\/?)+/, '');
};

/* Convert a virtual path from the /root/ folder, to the filesystem path.
 * `path`: virtual path (in the form of a String).
 * It needs to be sanitized with `sanitizepath`.
 */
function torealpath (path) {
  return nodepath.join (process.cwd(), nodepath.join (rootdir, path));
};

/* This hashmap registers all files ever asked for.
 * The keys are the *fake* paths of those files.
 */
var fsfiles = {};

/* Read the path, construct the file according to the filename.
 * `path`: String of "fake" path starting with a `/`.
 * `callback`: function (err, <new File()>).
 */
function getfile (path, callback) {
  var oldpath = path;
  path = sanitizepath(path);
  console.log('FS:GETFILE: %s was sanitized to "%s"', oldpath, path);
  var realpath = torealpath (path);
  // If this file is already in memory, return that.
  if (fsfiles[path] !== undefined) {
    callback(undefined, fsfiles[path]);
    return;
  }

  nodefs.stat (realpath, function (err, stats) {
    if (!err) {

      //console.log('stats says that', realpath, (stats.isDirectory()? 'is a directory': 'is a file'));

      if (stats.isDirectory()) {
        // When creating a directory to memory, the getcontent
        // component is the following function.
        // It reads the files in the entry.
        var getcontent = function (whengotfile) {
          nodefs.readdir (realpath, function (err, files) {
            if (!err) {
			  console.log('FS: directory files are', files);
			  if (files.length === 0) {
				whengotfile (undefined, {});
			  }
              var content = {};
              for (var i = 0; i < files.length; i++) {
                // We want to call back only when we've got all files.
                // We use a decrement to trace the progress.
                var fileslefttoprocess = files.length;
                getfile (
                  nodepath.join (path, files[i]),
                  function (err, file) {
                    fileslefttoprocess--;
                    if (err) { whengotfile (err); return; }
                    // We put each file in the content.
                    content[file.name] = file;
                    if (fileslefttoprocess === 0) {
                      whengotfile (undefined, content);
                    }
                  }
                );
              }
            } else {
              whengotfile (err);
            }
          });
        };
        var newfile = new File (
            'dir',
            nodepath.basename (realpath),
            getcontent
        );
        callback (undefined, newfile);
        fsfiles[path] = newfile;

      } else if (stats.isFile()) {
        // When creating a file, the getcontent
        // seeks the content of the file.
        var getcontent = function (whengotfile) {
          // FIXME: right now, we assume utf8 text.
          nodefs.readFile (realpath, 'utf8', function (err, data) {
            if (err) { whengotfile (err, undefined); return; }
            whengotfile (undefined, data);
          });
        };
        var newfile = new File (
            typefromextention(realpath),
            nodepath.basename (realpath),
            getcontent
        );
        callback (undefined, newfile);
        fsfiles[path] = newfile;
      }

    } else {
      callback (err);
    }
  });
};

// Get the type associated to the extention of the file.
//
function typefromextention (filename) {
  //console.log('FS:TYPEFROMEXTENTION:', filename, '-',
      //camp.Server.mime[nodepath.extname(filename).slice(1)]);
  return camp.Server.mime[nodepath.extname(filename).slice(1)] || 'text/plain';
};

// Create new files as requested.
//

/* `file` is a File (easily constructed with `new File()`).
 * `directory` is a File of type `dir`.
 */
function addfiletodir (file, directory) {
  // First, we need to know what the directory contains.
  //var files = directory.getcontent ();

  // TODO nodejs apis to write to disk.
  // nodefs.writeFile(filename, data, encoding='utf8',[callback])
  // nodefs.mkdir(path, mode, [callback]) // unix permissions mode (eg. 0777)
};

// Remove a file.
// TODO
function rmfilefromdir (file, directory) {
};

// Initialization of the File System.
//
//   Yes, right here, right now. Impressive huh?
//

// Choice to make here. Do we read the whole thing on startup?
//   + Faster boot time.
//   - Too much info to deal with; could take all memory.
//
// `getroot` yields a file of type `dir`, which is here initialized to the
// contents of the `/root/` folder (named `/` in here).
function getroot (gotroot) {
  getfile('/', function(err, root) { gotroot(err, root); });
};

// EXPORTS ARE HERE YOU BITCHES
//

exports.typenamefromtype = typenamefromtype;
exports.isoftype = isoftype;
exports.File = File;
exports.fsfiles = fsfiles;
exports.sanitizepath = sanitizepath;
exports.getfile = getfile;
exports.getroot = getroot;

exports.profile = profile;

<|endoftext|>


<commit_before>var git = require('nodegit');
var exec = require('child_process').exec;
var chalk = require('chalk');

// Output styling.
var success = chalk.bold.green;
var notice = chalk.bold.yellow;
var whoops = chalk.bold.red; // Can't name error as exec uses error in callback.
var underline = chalk.underline;

exports.getCurrentRepo = function() {
	return git.Repository.open("./");
};

exports.init = function() {
	exec('git init', function(error, stdout, stderr) {
		if (stdout.substring(0, 13) == 'Reinitialized') {
			console.log(notice('[~] Reinitialized existing Git repository.'));
		} else {
			console.log(success('[✓] Initialized existing Git repository.'));
		}
	});
};

exports.addAll = function() {
	exec('git add -A');

	console.log(success('[✓] Added everything!'));
};

exports.commit = function(message) {
	// No commit message.
	if (message === '') {
		console.log(notice('[~] You must enter a commit message.'));
	} else {
		this.addAll();
		exec('git commit -m "' + message + '"');

		console.log(success('[✓] Commited!'));
		console.log('\nMessage: "' + underline(message) + '"');
	}
};

exports.push = function() {
	exec('git push', function(error, stdout, stderr) {
		console.log(stdout);
		// if (stderr.substring(0, 10) == 'Everything') {
		// 	console.log(notice('[~] Everything already pushed.'));
		// } else if (stdout.substring(0, 2) == 'To') {
		// 	console.log(success('[✓] Pushed!'));
		// } else {
		// 	console.log(whoops('[!] Could not push commits.'));
		// }
	});
};
<commit_msg>Fixed push shortcut v5.<commit_after>var git = require('nodegit');
var exec = require('child_process').exec;
var chalk = require('chalk');

// Output styling.
var success = chalk.bold.green;
var notice = chalk.bold.yellow;
var whoops = chalk.bold.red; // Can't name error as exec uses error in callback.
var underline = chalk.underline;

exports.getCurrentRepo = function() {
	return git.Repository.open("./");
};

exports.init = function() {
	exec('git init', function(error, stdout, stderr) {
		if (stdout.substring(0, 13) == 'Reinitialized') {
			console.log(notice('[~] Reinitialized existing Git repository.'));
		} else {
			console.log(success('[✓] Initialized existing Git repository.'));
		}
	});
};

exports.addAll = function() {
	exec('git add -A');

	console.log(success('[✓] Added everything!'));
};

exports.commit = function(message) {
	// No commit message.
	if (message === '') {
		console.log(notice('[~] You must enter a commit message.'));
	} else {
		this.addAll();
		exec('git commit -m "' + message + '"');

		console.log(success('[✓] Commited!'));
		console.log('\nMessage: "' + underline(message) + '"');
	}
};

exports.push = function() {
	exec('git push', function(error, stdout, stderr) {
		console.log(stderr);
		// if (stderr.substring(0, 10) == 'Everything') {
		// 	console.log(notice('[~] Everything already pushed.'));
		// } else if (stdout.substring(0, 2) == 'To') {
		// 	console.log(success('[✓] Pushed!'));
		// } else {
		// 	console.log(whoops('[!] Could not push commits.'));
		// }
	});
};
<|endoftext|>


<commit_before>(function (window) {
    'use strict';

    var storageTypes = {
        localStorage: window.localStorage,
        sessionStorage: window.sessionStorage
    };

    /**
     * main constructor
     *
     * @param {Object} settings
     * @return {this} for chaining
     * @public
     */

    var LS = function LS (settings) {
        var currentDate = new Date().getTime();

        // merge default settings with custom settings
        this.settings = Object.assign({}, LS.defaultSettings, settings);

        if (this.settings.endpoint) {

            // if local storage exists configure instance settings and properties accordingly
            if (this.check()) {
                this.storageItem = this.get().storageItem;

                // remove expired storage
                if (this.settings.expires !== undefined) {
                    if (currentDate > (this.storageItem.dateCreated + this.settings.expires)) {
                        this.remove();
                    }
                }

                return this;
            } else {
                return this;
            }
        } else {
            throw new Error('This constructor function requires a settings object');
        }
    };

    /**
     * LS default settings
     */

    LS.defaultSettings = {
        storageType: storageTypes.localStorage,
        endpoint: undefined,
        expires: undefined
    };

    /**
     * set local storage and instance data
     *
     * @param {Object} data
     * @return {this} for chaining
     * @public
     */

    LS.prototype.set = function (data) {

        // configure storage object
        var storageType = getStorageType.call(this);

        if (data) {
            this.storageItem = {};
            this.storageItem.endpoint = this.settings.endpoint;
            this.storageItem.data = data;
            this.storageItem.dateCreated = this.storageItem.dateCreated || new Date().getTime();
            storageType[this.settings.endpoint] = JSON.stringify(this.storageItem);

            return this;
        } else {
            throw new Error('This function requires an object or array argument');
        }
    };

    /**
     * get local storage data and provide object to client with optional callback
     *
     * @param {Function} fn
     * @return {this} if no callback for chaining
     * @public
     */

    LS.prototype.get = LS.prototype.find = function (fn) {

        // configure storage object
        var storageType = getStorageType.call(this);

        if (this.check()) {
            this.storageItem = JSON.parse(storageType[this.settings.endpoint]);

            if (fn && typeof fn === 'function') {
                fn(this);
            } else {
                return this;
            }
        } else {
            throw new Error('Endpoint does not exist.');
        }
    };

    /**
     * check if local storage data exists
     *
     * @return {Boolean}
     * @public
     */

    LS.prototype.check = function () {

        // configure storage object
        var storageType = getStorageType.call(this);

        return storageType[this.settings.endpoint] !== undefined;
    };

    /**
     * explicitly clear local storage and instance data
     *
     * @return {this} for chaining
     * @public
     */

    LS.prototype.remove = LS.prototype.delete = function () {

        // configure storage object
        var storageType = getStorageType.call(this);

        if (this.check()) {
            storageType.removeItem(this.settings.endpoint);

            this.storageItem = {};

            return this;
        } else {
            throw new Error('No LS instances exist.');
        }
    };

    /**
     * explicitly update local storage data
     *
     * @return {this} for chaining
     * @public
     */

    LS.prototype.update = LS.prototype.save = function (bool) {

        // configure storage object
        var storageType = getStorageType.call(this);

        if (this.check() && this.storageItem.endpoint) {

            // if user provides true for update dateCreated
            if (bool) {
                this.storageItem.dateCreated = new Date().getTime();
            }

            storageType[this.settings.endpoint] = JSON.stringify(this.storageItem);

            return this;
        } else {
            throw new Error('No LS instances exist.');
        }
    };

    function getStorageType () {
        return storageTypes[this.settings.storageType] || storageTypes.localStorage;
    }

    // Object.assign shim
    if (typeof Object.assign != 'function') {
        (function () {
            Object.assign = function (target) {
                if (target === undefined || target === null) {
                    throw new TypeError('Cannot convert undefined or null to object');
                }

                var output = Object(target);
                for (var index = 1; index < arguments.length; index++) {
                    var source = arguments[index];
                    if (source !== undefined && source !== null) {
                        for (var nextKey in source) {
                            if (source.hasOwnProperty(nextKey)) {
                                output[nextKey] = source[nextKey];
                            }
                        }
                    }
                }
                return output;
            };
        })();
    }

    // globalize LS object/constructor
    window.LS = LS;

})(window);
<commit_msg>update comments<commit_after>(function (window) {
    'use strict';

    var storageTypes = {
        localStorage: window.localStorage,
        sessionStorage: window.sessionStorage
    };

    /**
     * main constructor
     *
     * @param {Object} settings
     * @return {this} for chaining
     * @public
     */

    var LS = function LS (settings) {
        var currentDate = new Date().getTime();

        // merge default settings with custom settings
        this.settings = Object.assign({}, LS.defaultSettings, settings);

        if (this.settings.endpoint) {

            // if local storage exists configure instance settings and properties accordingly
            if (this.check()) {
                this.storageItem = this.get().storageItem;

                // remove expired storage
                if (this.settings.expires !== undefined) {
                    if (currentDate > (this.storageItem.dateCreated + this.settings.expires)) {
                        this.remove();
                    }
                }

                return this;
            } else {
                return this;
            }
        } else {
            throw new Error('This constructor function requires a settings object');
        }
    };

    /**
     * LS default settings
     */

    LS.defaultSettings = {
        storageType: storageTypes.localStorage,
        endpoint: undefined,
        expires: undefined
    };

    /**
     * set local storage and instance data
     *
     * @param {Object} data
     * @return {this} for chaining
     * @public
     */

    LS.prototype.set = function (data) {

        // configure storage object
        var storageType = getStorageType.call(this);

        if (data) {
            this.storageItem = {};
            this.storageItem.endpoint = this.settings.endpoint;
            this.storageItem.data = data;
            this.storageItem.dateCreated = this.storageItem.dateCreated || new Date().getTime();
            storageType[this.settings.endpoint] = JSON.stringify(this.storageItem);

            return this;
        } else {
            throw new Error('This function requires an object or array argument');
        }
    };

    /**
     * get local storage data and provide object to client with optional callback
     *
     * @param {Function} fn
     * @return {this} if no callback for chaining
     * @public
     */

    LS.prototype.get = LS.prototype.find = function (fn) {

        // configure storage object
        var storageType = getStorageType.call(this);

        if (this.check()) {
            this.storageItem = JSON.parse(storageType[this.settings.endpoint]);

            if (fn && typeof fn === 'function') {
                fn(this);
            } else {
                return this;
            }
        } else {
            throw new Error('Endpoint does not exist.');
        }
    };

    /**
     * check if local storage data exists
     *
     * @return {Boolean}
     * @public
     */

    LS.prototype.check = function () {

        // configure storage object
        var storageType = getStorageType.call(this);

        return storageType[this.settings.endpoint] !== undefined;
    };

    /**
     * explicitly clear local storage and instance data
     *
     * @return {this} for chaining
     * @public
     */

    LS.prototype.remove = LS.prototype.delete = function () {

        // configure storage object
        var storageType = getStorageType.call(this);

        if (this.check()) {
            storageType.removeItem(this.settings.endpoint);

            this.storageItem = {};

            return this;
        } else {
            throw new Error('No LS instances exist.');
        }
    };

    /**
     * explicitly update local storage data
     *
     * @param {Boolean} bool
     * @return {this} for chaining
     * @public
     */

    LS.prototype.update = LS.prototype.save = function (bool) {

        // configure storage object
        var storageType = getStorageType.call(this);

        if (this.check() && this.storageItem.endpoint) {

            // if user provides true for update dateCreated
            if (bool) {
                this.storageItem.dateCreated = new Date().getTime();
            }

            storageType[this.settings.endpoint] = JSON.stringify(this.storageItem);

            return this;
        } else {
            throw new Error('No LS instances exist.');
        }
    };

    function getStorageType () {
        return storageTypes[this.settings.storageType] || storageTypes.localStorage;
    }

    // Object.assign shim
    if (typeof Object.assign != 'function') {
        (function () {
            Object.assign = function (target) {
                if (target === undefined || target === null) {
                    throw new TypeError('Cannot convert undefined or null to object');
                }

                var output = Object(target);
                for (var index = 1; index < arguments.length; index++) {
                    var source = arguments[index];
                    if (source !== undefined && source !== null) {
                        for (var nextKey in source) {
                            if (source.hasOwnProperty(nextKey)) {
                                output[nextKey] = source[nextKey];
                            }
                        }
                    }
                }
                return output;
            };
        })();
    }

    // globalize LS object/constructor
    window.LS = LS;

})(window);
<|endoftext|>


<commit_before>#!/usr/bin/env node

var express = require('express');
var http = require('http')
var path = require('path');
var fs = require('fs');
var async = require('async');
var _ = require('underscore');
var crypto = require('crypto');
var MobileDetect = require('mobile-detect');
var config = require('./config');
var utils = require('./utils');
var socketio = require('socket.io');
var pngparse = require("pngparse");
var image = require('./image');

var ROOTDIR = path.join(__dirname, config.mosaic.folders.root);

var app = express();

/*
 *
 */
var lastCommittedImage = [];

app.configure(function(){
	app.set('port', process.env.PORT || 3003);
	app.set('views', __dirname + '/views');
	app.set('view engine', 'jade');
	app.use(express.favicon());
	app.use(express.bodyParser());
	app.use(express.methodOverride());
	app.use(express.cookieParser('MediaGoo'));
	app.use(express.session());
	app.use(app.router);
	app.use(require('stylus').middleware(__dirname + '/public'));
	app.use(express.static(path.join(__dirname, 'public')));
});

app.configure('development', function(){
	app.use(express.errorHandler());
});

var server = http.createServer(app).listen(app.get('port'), function(){
	console.log("Express server listening on port " + app.get('port'));
});

// Socket IO
var io = socketio.listen(server);
io.set('log level', 0);

app.get('/', function(req, res){
	res.render('client', { title: 'MediaGoo' });
});

app.get('/viz', function(req, res){
	res.render('visualizer', { title: 'MediaGoo', startImage:lastCommittedImage });
});

app.post('/xhrupload', function (req, res){
	if(!req.xhr) return utils.sendError(new Error('got no xhr request'), res);

	var size = req.header('x-file-size');
	var type = req.header('x-file-type');
	var name = path.basename(req.header('x-file-name'));

	//name = crypto.createHash('md5').update( ''+(Date.now()) ).digest('hex') + '_' + name;
	name = (Date.now()) + '_' + (utils.removeFileExt(name)).substr(0,6) + path.extname(name) ;
	var uploadedFile = path.join(ROOTDIR, config.mosaic.folders.full, name);

	var ws = fs.createWriteStream( uploadedFile );

	req.on('data', function (data) {
		ws.write(data);
	});

	req.on('end', function () {
		if(config.showDebugInfo) console.log("XHR Upload done");
		ws.end();

		res.send(200);
		resizeImage(name);
	});
});

function resizeImage(imgName){
	var uploadedFile = path.join(ROOTDIR, config.mosaic.folders.full, imgName);
	var targetFile = path.join(ROOTDIR, config.mosaic.folders.resizes, imgName) + '.png';

	image.crop(uploadedFile, 200, 200, targetFile, function (err, imgres) {
		if(err) return console.log(err);

		pngparse.parseFile(targetFile, function (err, data) {
			if(err) return console.log(err);

		    console.log(data);
		    lastCommittedImage = data.data;
		    io.sockets.emit( 'new', data.data );
		})
	});
}

io.sockets.on('connection', function (socket) {
	socket.emit( 'new', lastCommittedImage);
});

function loadStartImage(){
	var startFile = path.join(ROOTDIR, config.mosaic.folders.startImage);
	pngparse.parseFile(startFile, function (err, data) {
		if(err) return console.log(err);

		console.log(data);
		lastCommittedImage = data.data;
		// io.sockets.emit( 'new', data.data );
	})
}

loadStartImage();

<commit_msg>poort 3000<commit_after>#!/usr/bin/env node

var express = require('express');
var http = require('http')
var path = require('path');
var fs = require('fs');
var async = require('async');
var _ = require('underscore');
var crypto = require('crypto');
var MobileDetect = require('mobile-detect');
var config = require('./config');
var utils = require('./utils');
var socketio = require('socket.io');
var pngparse = require("pngparse");
var image = require('./image');

var ROOTDIR = path.join(__dirname, config.mosaic.folders.root);

var app = express();

/*
 *
 */
var lastCommittedImage = [];

app.configure(function(){
	app.set('port', process.env.PORT || 3000);
	app.set('views', __dirname + '/views');
	app.set('view engine', 'jade');
	app.use(express.favicon());
	app.use(express.bodyParser());
	app.use(express.methodOverride());
	app.use(express.cookieParser('MediaGoo'));
	app.use(express.session());
	app.use(app.router);
	app.use(require('stylus').middleware(__dirname + '/public'));
	app.use(express.static(path.join(__dirname, 'public')));
});

app.configure('development', function(){
	app.use(express.errorHandler());
});

var server = http.createServer(app).listen(app.get('port'), function(){
	console.log("Express server listening on port " + app.get('port'));
});

// Socket IO
var io = socketio.listen(server);
io.set('log level', 0);

app.get('/', function(req, res){
	res.render('client', { title: 'MediaGoo' });
});

app.get('/viz', function(req, res){
	res.render('visualizer', { title: 'MediaGoo', startImage:lastCommittedImage });
});

app.post('/xhrupload', function (req, res){
	if(!req.xhr) return utils.sendError(new Error('got no xhr request'), res);

	var size = req.header('x-file-size');
	var type = req.header('x-file-type');
	var name = path.basename(req.header('x-file-name'));

	//name = crypto.createHash('md5').update( ''+(Date.now()) ).digest('hex') + '_' + name;
	name = (Date.now()) + '_' + (utils.removeFileExt(name)).substr(0,6) + path.extname(name) ;
	var uploadedFile = path.join(ROOTDIR, config.mosaic.folders.full, name);

	var ws = fs.createWriteStream( uploadedFile );

	req.on('data', function (data) {
		ws.write(data);
	});

	req.on('end', function () {
		if(config.showDebugInfo) console.log("XHR Upload done");
		ws.end();

		res.send(200);
		resizeImage(name);
	});
});

function resizeImage(imgName){
	var uploadedFile = path.join(ROOTDIR, config.mosaic.folders.full, imgName);
	var targetFile = path.join(ROOTDIR, config.mosaic.folders.resizes, imgName) + '.png';

	image.crop(uploadedFile, 200, 200, targetFile, function (err, imgres) {
		if(err) return console.log(err);

		pngparse.parseFile(targetFile, function (err, data) {
			if(err) return console.log(err);

		    console.log(data);
		    lastCommittedImage = data.data;
		    io.sockets.emit( 'new', data.data );
		})
	});
}

io.sockets.on('connection', function (socket) {
	socket.emit( 'new', lastCommittedImage);
});

function loadStartImage(){
	var startFile = path.join(ROOTDIR, config.mosaic.folders.startImage);
	pngparse.parseFile(startFile, function (err, data) {
		if(err) return console.log(err);

		console.log(data);
		lastCommittedImage = data.data;
		// io.sockets.emit( 'new', data.data );
	})
}

loadStartImage();

<|endoftext|>