<commit_before>/*
 * Library contain windows specific functions
 * like getting information about volumes
 */

(function(){
    "use strict";

    if(!global.cloudcmd)
        return console.log(
             '# win.js'                                         + '\n'  +
             '# -----------'                                    + '\n'  +
             '# Module is part of Cloud Commander,'             + '\n'  +
             '# used for work with windows specific'            + '\n'  +
             '# functions like work with drives(etc c).'        + '\n'  +
             '# http://coderaiser.github.io/cloudcmd'           + '\n');

    var main            = global.cloudcmd.main,
        Charset         ={
            UNICODE : 65001,
            WIN32   : 866
        },

        exec            = main.child_process.exec,
        Util            = main.util;

    exports.getVolumes  = function(pCallBack){
        var lCHCP       = 'chcp ' + Charset.UNICODE,
            lGetVolumes = 'wmic logicaldisk get name'

        exec(lCHCP + ' &&  ' + lGetVolumes, retProcessOuput(pCallBack));
    };

    function retProcessOuput(pCallBack){
        return function(pError, pStdout, pStderr){
            var lRemoveStr  = [
                    '\r', '\n',
                    'Name',
                    'Active code page: 65001  '
                ],
                lVolumes    = [],
                lError      = pError || pStderr;
            exec('chcp ' + Charset.WIN32);

            if(!lError) {
                lVolumes = Util.removeStr(pStdout, lRemoveStr)
                    .split('    ')

                lVolumes.pop();
            }

            Util.exec(pCallBack, lError || lVolumes);
        };
    }
})();
<commit_msg>chore(win) add ";"<commit_after>/*
 * Library contain windows specific functions
 * like getting information about volumes
 */

(function(){
    "use strict";

    if(!global.cloudcmd)
        return console.log(
             '# win.js'                                         + '\n'  +
             '# -----------'                                    + '\n'  +
             '# Module is part of Cloud Commander,'             + '\n'  +
             '# used for work with windows specific'            + '\n'  +
             '# functions like work with drives(etc c).'        + '\n'  +
             '# http://coderaiser.github.io/cloudcmd'           + '\n');

    var main            = global.cloudcmd.main,
        Charset         ={
            UNICODE : 65001,
            WIN32   : 866
        },

        exec            = main.child_process.exec,
        Util            = main.util;

    exports.getVolumes  = function(pCallBack){
        var lCHCP       = 'chcp ' + Charset.UNICODE,
            lGetVolumes = 'wmic logicaldisk get name';

        exec(lCHCP + ' &&  ' + lGetVolumes, retProcessOuput(pCallBack));
    };

    function retProcessOuput(pCallBack){
        return function(pError, pStdout, pStderr){
            var lRemoveStr  = [
                    '\r', '\n',
                    'Name',
                    'Active code page: 65001  '
                ],
                lVolumes    = [],
                lError      = pError || pStderr;
            exec('chcp ' + Charset.WIN32);

            if(!lError) {
                lVolumes = Util.removeStr(pStdout, lRemoveStr)
                    .split('    ');

                lVolumes.pop();
            }

            Util.exec(pCallBack, lError || lVolumes);
        };
    }
})();
<|endoftext|>


<commit_before>// emit JSON describing versions of all packages currently installed (for later
// use with shrinkwrap install)

module.exports = exports = shrinkwrap

var path = require('path')
var log = require('npmlog')
var writeFileAtomic = require('write-file-atomic')
var iferr = require('iferr')
var readPackageTree = require('read-package-tree')
var validate = require('aproba')
var npm = require('./npm.js')
var recalculateMetadata = require('./install/deps.js').recalculateMetadata
var validatePeerDeps = require('./install/deps.js').validatePeerDeps
var isExtraneous = require('./install/is-extraneous.js')
var isOnlyDev = require('./install/is-dev.js').isOnlyDev

shrinkwrap.usage = 'npm shrinkwrap'

function shrinkwrap (args, silent, cb) {
  if (typeof cb !== 'function') {
    cb = silent
    silent = false
  }

  if (args.length) {
    log.warn('shrinkwrap', "doesn't take positional args")
  }

  var dir = path.resolve(npm.dir, '..')
  npm.config.set('production', true)
  readPackageTree(dir, andRecalculateMetadata(iferr(cb, function (tree) {
    var pkginfo = treeToShrinkwrap(tree, !!npm.config.get('dev'))
    shrinkwrap_(pkginfo, silent, cb)
  })))
}

function andRecalculateMetadata (next) {
  validate('F', arguments)
  return function (er, tree) {
    validate('EO', arguments)
    if (er) return next(er)
    recalculateMetadata(tree, log, next)
  }
}

function treeToShrinkwrap (tree, dev) {
  validate('OB', arguments)
  var pkginfo = {}
  if (tree.package.name) pkginfo.name = tree.package.name
  if (tree.package.version) pkginfo.version = tree.package.version
  var problems = []
  if (tree.children.length) {
    shrinkwrapDeps(dev, problems, pkginfo.dependencies = {}, tree)
  }
  if (problems.length) pkginfo.problems = problems
  return pkginfo
}

function shrinkwrapDeps (dev, problems, deps, tree) {
  validate('BAOO', arguments)
  Object.keys(tree.missingDeps).forEach(function (name) {
    var invalid = tree.children.filter(function (dep) { return dep.package.name === name })[0]
    if (invalid) {
      problems.push('invalid: ' + invalid.package._id + invalid.path)
    } else {
      var topname = tree.package._id ||
                    (tree.package.name && tree.package.name + tree.package.version ? '@' + tree.package.version : '')
      problems.push('missing: ' + name + '@' + tree.package.dependencies[name] +
        (topname ? ', required by ' + topname : ''))
    }
  })
  tree.children.sort(function (aa, bb) { return aa.package.name.localeCompare(bb.package.name) }).forEach(function (child) {
    if (!dev && isOnlyDev(child)) {
      log.warn('shrinkwrap', 'Excluding devDependency: %s', child.package.name, child.parent.package.dependencies)
      return
    }
    var pkginfo = deps[child.package.name] = {}
    pkginfo.version = child.package.version
    pkginfo.from = child.package._from
    pkginfo.resolved = child.package._resolved
    if (isExtraneous(child)) {
      problems.push('extraneous: ' + child.package._id + ' ' + child.path)
    }
    validatePeerDeps(child, function (tree, pkgname, version) {
      problems.push('peer invalid: ' + pkgname + '@' + version +
        ', required by ' + child.package._id)
    })
    if (child.children.length) {
      shrinkwrapDeps(dev, problems, pkginfo.dependencies = {}, child)
    }
  })
}

function shrinkwrap_ (pkginfo, silent, cb) {
  if (pkginfo.problems) {
    return cb(new Error('Problems were encountered\n' +
                        'Please correct and try again.\n' +
                        pkginfo.problems.join('\n')))
  }

  save(pkginfo, silent, cb)
}

function save (pkginfo, silent, cb) {
  // copy the keys over in a well defined order
  // because javascript objects serialize arbitrarily
  var swdata
  try {
    swdata = JSON.stringify(pkginfo, null, 2) + '\n'
  } catch (er) {
    log.error('shrinkwrap', 'Error converting package info to json')
    return cb(er)
  }

  var file = path.resolve(npm.prefix, 'npm-shrinkwrap.json')

  writeFileAtomic(file, swdata, function (er) {
    if (er) return cb(er)
    if (silent) return cb(null, pkginfo)
    log.clearProgress()
    console.log('wrote npm-shrinkwrap.json')
    log.showProgress()
    cb(null, pkginfo)
  })
}
<commit_msg>shrinkwrap: Improve error messages for invalid deps when shrinkwraping<commit_after>// emit JSON describing versions of all packages currently installed (for later
// use with shrinkwrap install)

module.exports = exports = shrinkwrap

var path = require('path')
var log = require('npmlog')
var writeFileAtomic = require('write-file-atomic')
var iferr = require('iferr')
var readPackageTree = require('read-package-tree')
var validate = require('aproba')
var npm = require('./npm.js')
var recalculateMetadata = require('./install/deps.js').recalculateMetadata
var validatePeerDeps = require('./install/deps.js').validatePeerDeps
var isExtraneous = require('./install/is-extraneous.js')
var isOnlyDev = require('./install/is-dev.js').isOnlyDev

shrinkwrap.usage = 'npm shrinkwrap'

function shrinkwrap (args, silent, cb) {
  if (typeof cb !== 'function') {
    cb = silent
    silent = false
  }

  if (args.length) {
    log.warn('shrinkwrap', "doesn't take positional args")
  }

  var dir = path.resolve(npm.dir, '..')
  npm.config.set('production', true)
  readPackageTree(dir, andRecalculateMetadata(iferr(cb, function (tree) {
    var pkginfo = treeToShrinkwrap(tree, !!npm.config.get('dev'))
    shrinkwrap_(pkginfo, silent, cb)
  })))
}

function andRecalculateMetadata (next) {
  validate('F', arguments)
  return function (er, tree) {
    validate('EO', arguments)
    if (er) return next(er)
    recalculateMetadata(tree, log, next)
  }
}

function treeToShrinkwrap (tree, dev) {
  validate('OB', arguments)
  var pkginfo = {}
  if (tree.package.name) pkginfo.name = tree.package.name
  if (tree.package.version) pkginfo.version = tree.package.version
  var problems = []
  if (tree.children.length) {
    shrinkwrapDeps(dev, problems, pkginfo.dependencies = {}, tree)
  }
  if (problems.length) pkginfo.problems = problems
  return pkginfo
}

function shrinkwrapDeps (dev, problems, deps, tree) {
  validate('BAOO', arguments)
  Object.keys(tree.missingDeps).forEach(function (name) {
    var invalid = tree.children.filter(function (dep) { return dep.package.name === name })[0]
    if (invalid) {
      problems.push('invalid: have ' + invalid.package._id + ' (expected: ' + tree.missingDeps[name] + ') ' + invalid.path)
    } else {
      var topname = tree.package._id ||
                    (tree.package.name && tree.package.name + tree.package.version ? '@' + tree.package.version : '')
      problems.push('missing: ' + name + '@' + tree.package.dependencies[name] +
        (topname ? ', required by ' + topname : ''))
    }
  })
  tree.children.sort(function (aa, bb) { return aa.package.name.localeCompare(bb.package.name) }).forEach(function (child) {
    if (!dev && isOnlyDev(child)) {
      log.warn('shrinkwrap', 'Excluding devDependency: %s', child.package.name, child.parent.package.dependencies)
      return
    }
    var pkginfo = deps[child.package.name] = {}
    pkginfo.version = child.package.version
    pkginfo.from = child.package._from
    pkginfo.resolved = child.package._resolved
    if (isExtraneous(child)) {
      problems.push('extraneous: ' + child.package._id + ' ' + child.path)
    }
    validatePeerDeps(child, function (tree, pkgname, version) {
      problems.push('peer invalid: ' + pkgname + '@' + version +
        ', required by ' + child.package._id)
    })
    if (child.children.length) {
      shrinkwrapDeps(dev, problems, pkginfo.dependencies = {}, child)
    }
  })
}

function shrinkwrap_ (pkginfo, silent, cb) {
  if (pkginfo.problems) {
    return cb(new Error('Problems were encountered\n' +
                        'Please correct and try again.\n' +
                        pkginfo.problems.join('\n')))
  }

  save(pkginfo, silent, cb)
}

function save (pkginfo, silent, cb) {
  // copy the keys over in a well defined order
  // because javascript objects serialize arbitrarily
  var swdata
  try {
    swdata = JSON.stringify(pkginfo, null, 2) + '\n'
  } catch (er) {
    log.error('shrinkwrap', 'Error converting package info to json')
    return cb(er)
  }

  var file = path.resolve(npm.prefix, 'npm-shrinkwrap.json')

  writeFileAtomic(file, swdata, function (er) {
    if (er) return cb(er)
    if (silent) return cb(null, pkginfo)
    log.clearProgress()
    console.log('wrote npm-shrinkwrap.json')
    log.showProgress()
    cb(null, pkginfo)
  })
}
<|endoftext|>


<commit_before>
var itemId = 0;

function ValueDimension() {
    this.values = [];
}

ValueDimension.prototype.getValue = function (value) {
    var index = this.values.indexOf(value);
    if (index >= 0)
        return this.values[index];
    this.values.push(value);
    return value;
};

function Item(data, tags) {
    var id = ++itemId;

    this.getId = function () { return id; }

    this.asObject = function () {
        return { id: id, data: data, tags: tags.slice() };
    };

    this.hasTags = function (taglist) {
        for (var n in taglist) {
            var tagitem = taglist[n];

            if (typeof tagitem === 'object') {
                var key = Object.keys(tagitem)[0];
                var value = tagitem[key];
                var hastag = false;

                for (var m in tags) {
                    var tag = tags[m];
                    if (tag[key] === value) {
                        hastag = true;
                        break;
                    }
                }

                if (!hastag)
                    return false;
            }
            else if (tags.indexOf(tagitem) < 0)
                return false;
        }

        return true;
    }
}

function tagIsValid(tag) {
    if (!tag)
        return false;

    if (typeof tag === 'string')
        return true;

    if (typeof tag !== 'object')
        return false;

    var keys = Object.keys(tag);

    if (!keys || keys.length != 1)
        return false;

    if (!tag[keys[0]])
        return false;

    return true;
}

function objectAsTags(obj) {
    var tags = [];
    var tag;

    for (var n in obj) {
        tag = { };
        tag[n] = obj[n];
        tags.push(tag);
    }

    return tags;
}

function Engine() {
    var items = [];
    var strings = new ValueDimension();

    this.createItem = function (data, tags) {
        if (!Array.isArray(tags) && typeof tags === 'object')
            tags = objectAsTags(tags);

        var ntags = [];

        for (var n in tags) {
            var tag = tags[n];
            if (!tagIsValid(tag))
                throw "invalid tag";
            if (typeof tag === 'string')
                ntags.push(strings.getValue(tag));
            else
                ntags.push(tag);
        }

        var item = new Item(data, ntags);
        var id = item.getId();
        items[id] = item;
        return id;
    }

    this.getItemById = function (id) {
        var item = items[id];

        if (item)
            return item.asObject();

        return null;
    }

    this.getItems = function (tags) {
        var result = [];

        items.forEach(function (item) {
            if (item.hasTags(tags))
                result.push(item.asObject());
        });

        return result;
    }
}

function createEngine() {
    return new Engine();
}

module.exports = {
    createEngine: createEngine
};

<commit_msg>Refactor: isValidTag removed, toInternalTag added<commit_after>
var itemId = 0;

function ValueDimension() {
    this.values = [];
}

ValueDimension.prototype.getValue = function (value) {
    var index = this.values.indexOf(value);
    if (index >= 0)
        return this.values[index];
    this.values.push(value);
    return value;
};

function Item(data, tags) {
    var id = ++itemId;

    this.getId = function () { return id; }

    this.asObject = function () {
        return { id: id, data: data, tags: tags.slice() };
    };

    this.hasTags = function (taglist) {
        for (var n in taglist) {
            var tagitem = taglist[n];

            if (typeof tagitem === 'object') {
                var key = Object.keys(tagitem)[0];
                var value = tagitem[key];
                var hastag = false;

                for (var m in tags) {
                    var tag = tags[m];
                    if (tag[key] === value) {
                        hastag = true;
                        break;
                    }
                }

                if (!hastag)
                    return false;
            }
            else if (tags.indexOf(tagitem) < 0)
                return false;
        }

        return true;
    }
}

function objectAsTags(obj) {
    var tags = [];
    var tag;

    for (var n in obj) {
        tag = { };
        tag[n] = obj[n];
        tags.push(tag);
    }

    return tags;
}

function Engine() {
    var items = [];
    var strings = new ValueDimension();

    function toInternalTag(tag) {
        if (!tag)
            return null;

        if (typeof tag === 'string')
            return strings.getValue(tag);

        if (typeof tag !== 'object')
            return null;

        var keys = Object.keys(tag);

        if (!keys || keys.length != 1)
            return null;

        if (!tag[keys[0]])
            return null;

        return tag;
    }

    this.createItem = function (data, tags) {
        if (!Array.isArray(tags) && typeof tags === 'object')
            tags = objectAsTags(tags);

        var internaltags = [];

        for (var n in tags) {
            var internaltag = toInternalTag(tags[n]);
            if (!internaltag)
                throw "invalid tag";
            internaltags.push(internaltag);
        }

        var item = new Item(data, internaltags);
        var id = item.getId();
        items[id] = item;
        return id;
    }

    this.getItemById = function (id) {
        var item = items[id];

        if (item)
            return item.asObject();

        return null;
    }

    this.getItems = function (tags) {
        var result = [];

        items.forEach(function (item) {
            if (item.hasTags(tags))
                result.push(item.asObject());
        });

        return result;
    }
}

function createEngine() {
    return new Engine();
}

module.exports = {
    createEngine: createEngine
};

<|endoftext|>


<commit_before>// diskdatabank.js
//
// On-disk implementation of Databank interface
//
// Copyright 2011,2012 StatusNet Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

var databank = require('../databank'),
    rimraf = require('rimraf'),
    Step = require('step'),
    mkdirp = require('mkdirp'),
    Queue = require('jankyqueue'),
    Databank = databank.Databank,
    DatabankError = databank.DatabankError,
    AlreadyExistsError = databank.AlreadyExistsError,
    NoSuchThingError = databank.NoSuchThingError,
    NotConnectedError = databank.NotConnectedError,
    AlreadyConnectedError = databank.AlreadyConnectedError,
    crypto = require('crypto'),
    fs = require('fs-ext'),
    path = require('path'),
    os = require('os');

var DiskDatabank = function(params) {
    this.mktmp = params.mktmp || false;
    this.dir = params.dir || '/var/lib/diskdatabank/';
    this.mode = params.mode || 0660;
    this.hashDepth = params.hashDepth || 3;
    this.schema = params.schema || {};
    this.noLock = params.noLock || false; // Optionally lock
    this.queueSize = params.queueSize || 32;
    this.q = new Queue(this.queueSize);
    this.connected = false;
};

DiskDatabank.prototype = new Databank();

DiskDatabank.prototype.toFilename = function(type, id) {
    var hash = this.toHash(id);

    return path.join(this.toDirname(type, id), hash + '.json');
};

DiskDatabank.prototype.toHash = function(id) {

    var hash = crypto.createHash('md5'),
        str, data;

    data = "" + id;

    hash.update(data);
    str = hash.digest('base64');

    // Make it a little more FS-safe

    str = str.replace(/\+/g, '-');
    str = str.replace(/\//g, '_');
    str = str.replace(/=/g, '');

    return str;
};

DiskDatabank.prototype.toDirname = function(type, id) {
    var n;
    var dirname = path.join(this.dir, type);
    var hash = this.toHash(id);

    for (n = 0; n < Math.min(this.hashDepth, hash.length); n++) {
        dirname = path.join(dirname, hash.substr(0, n + 1));
    }

    return dirname;
};

DiskDatabank.prototype.connect = function(params, onCompletion) {
    var bank = this,
        checkTree = function() {
            bank.ensureTree(function(err) {
                if (err) {
                    onCompletion(err);
                } else {
                    bank.connected = true;
                    onCompletion(null);
                }
            });
        };

    if (bank.connected) {
        onCompletion(new AlreadyConnectedError());
        return;
    }

    if (bank.mktmp) {
        bank.makeTempDir(function(err) {
            if (err) {
                onCompletion(err);
            } else {
                checkTree();
            }
        });
    } else {
        checkTree();
    }
};

DiskDatabank.prototype.makeTempDir = function(callback) {
    var bank = this,
        tmp = os.tmpDir(),
        tryAgain = function() {
            randomString(16, function(err, rs) {
                var dirname;
                if (err) {
                    callback(err);
                } else {
                    dirname = path.join(tmp, rs);
                    bank.q.enqueue(fs.stat, [dirname], function(err, stats) {
		        if (err && err.code == 'ENOENT') {
                            bank.dir = dirname;
                            callback(null);
                        } else if (err) {
                            callback(err);
                        } else {
                            // XXX: bounded retries; 10?
                            tryAgain();
                        }
                    });
                }
            });
        },
        randomString = function(bytes, cb) {
            crypto.randomBytes(bytes, function(err, buf) {
                var str;
                if (err) {
                    cb(err, null);
                } else {
                    str = buf.toString("base64");

                    str = str.replace(/\+/g, "-");
                    str = str.replace(/\//g, "_");
                    str = str.replace(/=/g, "");

                    cb(null, str);
                }
            });
        };

    tryAgain();
};

DiskDatabank.prototype.disconnect = function(onCompletion) {
    var bank = this;

    if (!bank.connected) {
        onCompletion(new NotConnectedError());
        return;
    }

    if (bank.mktmp) {
        bank.q.enqueue(rimraf, [bank.dir], function(err) {
            bank.connected = false;
            onCompletion(err);
        });
    } else {
        bank.connected = false;
        onCompletion(null);
    }
};

DiskDatabank.prototype.create = function(type, id, value, callback) {

    var bank = this,
        dirname = this.toDirname(type, id),
        filename = this.toFilename(type, id),
        fd;

    if (!bank.connected) {
        callback(new NotConnectedError());
        return;
    }

    Step(
        function() {
            bank.ensureDir(dirname, this);
        },
        function(err) {
            if (err) throw err;
            bank.q.enqueue(fs.stat, [filename], this);
        },
        function(err, stats) {
            if (err && (!err.code || err.code != 'ENOENT')) throw err;
            if (!err) throw new AlreadyExistsError(type, id);
            bank.q.enqueue(fs.open, [filename, 'wx', bank.mode], this);
        },
        function(err, result) {
            if (err) throw err; // Check for O_EXCL err and make into AlreadyExistsError?
            fd = result;
            // XXX: skip locking if bank.noLock
            bank.q.enqueue(fs.flock, [fd, 'ex'], this);
        },
        function(err) {
            var b;
            if (err) throw err;
            b = new Buffer(JSON.stringify(value), 'utf8');
            bank.q.enqueue(fs.write, [fd, b, 0, b.length, null], this);
        },
        function(err, written, buffer) {
            if (err) throw err;
            bank.q.enqueue(fs.close, [fd], this);
        },
        function(err) {
            if (err) {
                callback(err, null);
            } else {
                callback(null, value);
            }
        }
    );
};

DiskDatabank.prototype.update = function(type, id, value, callback) {
    var bank = this,
        filename = this.toFilename(type, id),
        fd;

    if (!this.connected) {
        callback(new NotConnectedError());
        return;
    }

    Step(
        function() {
            bank.q.enqueue(fs.stat, [filename], this);
        },
        function(err, stats) {
            if (err) {
                if (err.code == 'ENOENT') {
                    throw new NoSuchThingError(type, id);
                } else {
                    throw err;
                }
            }
            // open in r+ so we don't truncate existing file before locking
            bank.q.enqueue(fs.open, [filename, 'r+', bank.mode], this);
        },
        function(err, result) {
            if (err) throw err;
            fd = result;
            // XXX: skip locking if bank.noLock
            bank.q.enqueue(fs.flock, [fd, 'ex'], this);
        },
        function(err) {
            if (err) throw err;
            // Locked; now we truncate
            bank.q.enqueue(fs.truncate, [fd, 0], this);
        },
        function(err) {
            var b;
            if (err) throw err;
            b = new Buffer(JSON.stringify(value), 'utf8');
            bank.q.enqueue(fs.write, [fd, b, 0, b.length, null], this);
        },
        function(err, written, buffer) {
            if (err) throw err;
            bank.q.enqueue(fs.close, [fd], this);
        },
        function(err) {
            if (err) {
                callback(err, null);
            } else {
                callback(null, value);
            }
        }
    );
};

DiskDatabank.prototype.read = function(type, id, callback) {
    var bank = this,
        filename = this.toFilename(type, id),
        fsize,
        buf,
        fd;

    if (!this.connected) {
        callback(new NotConnectedError());
        return;
    }

    Step(
        function() {
            bank.q.enqueue(fs.open, [filename, 'r', bank.mode], this);
        },
        function(err, result) {
            if (err) {
                if (err.code == 'ENOENT') {
                    throw new NoSuchThingError(type, id);
                } else {
                    throw err;
                }
            }
            fd = result;
            bank.q.enqueue(fs.flock, [fd, 'sh'], this);
        },
        function(err) {
            if (err) throw err;
            bank.q.enqueue(fs.seek, [fd, 0, 2], this);
        },
        function(err, pos) {
            if (err) throw err;
            fsize = pos;
            buf = new Buffer(fsize);
            bank.q.enqueue(fs.read, [fd, buf, 0, fsize, 0], this);
        },
        function(err, count, b) {
            var parsed;
            if (err) throw err;
            bank.q.enqueue(fs.close, [fd], this);
        },
        function(err) {
            var parsed;
            if (err) {
                callback(err);
            } else {
                try {
                    parsed = JSON.parse(buf.toString('utf8'));
                    callback(null, parsed);
                } catch (e) {
                    callback(e, null);
                }
            }
        }
    );
};

DiskDatabank.prototype.del = function(type, id, callback) {
    var bank = this,
        filename = this.toFilename(type, id);

    if (!this.connected) {
        callback(new NotConnectedError());
        return;
    }

    Step(
        function() {
            bank.q.enqueue(fs.unlink, [filename], this);
        },
        function(err) {
            if (err) {
                if (err.code == 'ENOENT') {
                    callback(new NoSuchThingError(type, id), null);
                } else {
                    callback(err, null);
                }
            } else {
                callback(null);
            }
        }
    );
};

DiskDatabank.prototype.ensureDir = function(dir, callback) {

    var bank = this;

    Step(
        function() {
	    bank.q.enqueue(fs.stat, [dir], this);
        },
        function(err, stat) {
            if (err) {
                if (err.code == 'ENOENT') {
                    bank.q.enqueue(mkdirp, [dir, (bank.mode | 0111)], this);
                } else {
                    throw err;
                }
            } else {
                // no error
                if (stat.isDirectory()) {
                    callback(null);
                } else {
                    callback(new Error(dir + " exists and is not a directory."));
                }
            }
        },
        function(err, made) {
            if (err) {
                callback(err);
            } else {
                callback(null);
            }
        }
    );
};

DiskDatabank.prototype.search = function(type, criteria, onResult, onCompletion) {
    var bank = this,
	counter = 0,
        checkOrWalk = function(path, callback) {
            Step(
                function() {
		    bank.q.enqueue(fs.stat, [path], this);
                },
                function(err, stat) {
                    if (err) {
                        if (err.code == 'ENOENT') {
                            // silently ignore missing directory entries
                            callback(null);
                        } else {
                            throw err;
                        }
                    } else if (stat.isDirectory()) {
			walk(path, this);
                    } else {
                        check(path, this);
                    }
                },
                callback
            );
        },
	walk = function(dirname, callback) { // originally from https://gist.github.com/514983
            Step(
                function() {
	            bank.q.enqueue(fs.readdir, [dirname], this);
                },
                function(err, relnames) {
                    var group = this.group(), i;
		    if (err) throw err;
                    for (i = 0; i < relnames.length; i++) {
                        checkOrWalk(path.join(dirname, relnames[i]), group());
                    }
		},
                callback
            );
        },
        check = function(path, callback) {
            var fd, buf, fsize;
            Step(
                function() {
                    bank.q.enqueue(fs.open, [path, 'r', bank.mode], this);
                },
                function(err, result) {
                    if (err) throw err;
                    fd = result;
                    bank.q.enqueue(fs.flock, [fd, 'sh'], this);
                },
                function(err) {
                    if (err) throw err;
                    bank.q.enqueue(fs.seek, [fd, 0, 2], this);
                },
                function(err, pos) {
                    if (err) throw err;
                    fsize = pos;
                    buf = new Buffer(fsize);
                    bank.q.enqueue(fs.read, [fd, buf, 0, fsize, 0], this);
                },
                function(err, count, b) {
                    if (err) throw err;
                    bank.q.enqueue(fs.close, [fd], this);
                },
                function(err) {
                    var parsed;
                    if (err) {
                        callback(err);
                    } else {
                        try {
                            parsed = JSON.parse(buf.toString('utf8'));
		            if (bank.matchesCriteria(parsed, criteria)) {
			        onResult(parsed);
		            }
                            callback(null);
                        } catch (e) {
                            callback(e);
                        }
                    }
                }
	    );
	};

    if (!this.connected) {
        onCompletion(new NotConnectedError());
        return;
    }

    walk(path.join(bank.dir, type), onCompletion);
};

DiskDatabank.prototype.ensureTree = function(callback) {
    var bank = this;

    Step(
        function() {
            bank.ensureDir(bank.dir, this);
        },
        function(err) {
            var type, group = this.group();
            if (err) throw err;
            for (type in bank.schema) {
                if (bank.schema.hasOwnProperty(type)) {
                    bank.ensureDir(path.join(bank.dir, type), group());
                }
            }
        },
        function(err) {
            callback(err);
        }
    );
};

module.exports = DiskDatabank;
<commit_msg>Locking version of readAndModify() for DiskDatabank<commit_after>// diskdatabank.js
//
// On-disk implementation of Databank interface
//
// Copyright 2011,2012 StatusNet Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

var databank = require('../databank'),
    rimraf = require('rimraf'),
    Step = require('step'),
    mkdirp = require('mkdirp'),
    Queue = require('jankyqueue'),
    Databank = databank.Databank,
    DatabankError = databank.DatabankError,
    AlreadyExistsError = databank.AlreadyExistsError,
    NoSuchThingError = databank.NoSuchThingError,
    NotConnectedError = databank.NotConnectedError,
    AlreadyConnectedError = databank.AlreadyConnectedError,
    crypto = require('crypto'),
    fs = require('fs-ext'),
    path = require('path'),
    os = require('os');

var DiskDatabank = function(params) {
    this.mktmp = params.mktmp || false;
    this.dir = params.dir || '/var/lib/diskdatabank/';
    this.mode = params.mode || 0660;
    this.hashDepth = params.hashDepth || 3;
    this.schema = params.schema || {};
    this.noLock = params.noLock || false; // Optionally lock
    this.queueSize = params.queueSize || 32;
    this.q = new Queue(this.queueSize);
    this.connected = false;
};

DiskDatabank.prototype = new Databank();

DiskDatabank.prototype.toFilename = function(type, id) {
    var hash = this.toHash(id);

    return path.join(this.toDirname(type, id), hash + '.json');
};

DiskDatabank.prototype.toHash = function(id) {

    var hash = crypto.createHash('md5'),
        str, data;

    data = "" + id;

    hash.update(data);
    str = hash.digest('base64');

    // Make it a little more FS-safe

    str = str.replace(/\+/g, '-');
    str = str.replace(/\//g, '_');
    str = str.replace(/=/g, '');

    return str;
};

DiskDatabank.prototype.toDirname = function(type, id) {
    var n;
    var dirname = path.join(this.dir, type);
    var hash = this.toHash(id);

    for (n = 0; n < Math.min(this.hashDepth, hash.length); n++) {
        dirname = path.join(dirname, hash.substr(0, n + 1));
    }

    return dirname;
};

DiskDatabank.prototype.connect = function(params, onCompletion) {
    var bank = this,
        checkTree = function() {
            bank.ensureTree(function(err) {
                if (err) {
                    onCompletion(err);
                } else {
                    bank.connected = true;
                    onCompletion(null);
                }
            });
        };

    if (bank.connected) {
        onCompletion(new AlreadyConnectedError());
        return;
    }

    if (bank.mktmp) {
        bank.makeTempDir(function(err) {
            if (err) {
                onCompletion(err);
            } else {
                checkTree();
            }
        });
    } else {
        checkTree();
    }
};

DiskDatabank.prototype.makeTempDir = function(callback) {
    var bank = this,
        tmp = os.tmpDir(),
        tryAgain = function() {
            randomString(16, function(err, rs) {
                var dirname;
                if (err) {
                    callback(err);
                } else {
                    dirname = path.join(tmp, rs);
                    bank.q.enqueue(fs.stat, [dirname], function(err, stats) {
		        if (err && err.code == 'ENOENT') {
                            bank.dir = dirname;
                            callback(null);
                        } else if (err) {
                            callback(err);
                        } else {
                            // XXX: bounded retries; 10?
                            tryAgain();
                        }
                    });
                }
            });
        },
        randomString = function(bytes, cb) {
            crypto.randomBytes(bytes, function(err, buf) {
                var str;
                if (err) {
                    cb(err, null);
                } else {
                    str = buf.toString("base64");

                    str = str.replace(/\+/g, "-");
                    str = str.replace(/\//g, "_");
                    str = str.replace(/=/g, "");

                    cb(null, str);
                }
            });
        };

    tryAgain();
};

DiskDatabank.prototype.disconnect = function(onCompletion) {
    var bank = this;

    if (!bank.connected) {
        onCompletion(new NotConnectedError());
        return;
    }

    if (bank.mktmp) {
        bank.q.enqueue(rimraf, [bank.dir], function(err) {
            bank.connected = false;
            onCompletion(err);
        });
    } else {
        bank.connected = false;
        onCompletion(null);
    }
};

DiskDatabank.prototype.create = function(type, id, value, callback) {

    var bank = this,
        dirname = this.toDirname(type, id),
        filename = this.toFilename(type, id),
        fd;

    if (!bank.connected) {
        callback(new NotConnectedError());
        return;
    }

    Step(
        function() {
            bank.ensureDir(dirname, this);
        },
        function(err) {
            if (err) throw err;
            bank.q.enqueue(fs.stat, [filename], this);
        },
        function(err, stats) {
            if (err && (!err.code || err.code != 'ENOENT')) throw err;
            if (!err) throw new AlreadyExistsError(type, id);
            bank.q.enqueue(fs.open, [filename, 'wx', bank.mode], this);
        },
        function(err, result) {
            if (err) throw err; // Check for O_EXCL err and make into AlreadyExistsError?
            fd = result;
            // XXX: skip locking if bank.noLock
            bank.q.enqueue(fs.flock, [fd, 'ex'], this);
        },
        function(err) {
            var b;
            if (err) throw err;
            b = new Buffer(JSON.stringify(value), 'utf8');
            bank.q.enqueue(fs.write, [fd, b, 0, b.length, null], this);
        },
        function(err, written, buffer) {
            if (err) throw err;
            bank.q.enqueue(fs.close, [fd], this);
        },
        function(err) {
            if (err) {
                callback(err, null);
            } else {
                callback(null, value);
            }
        }
    );
};

DiskDatabank.prototype.update = function(type, id, value, callback) {
    var bank = this,
        filename = this.toFilename(type, id),
        fd;

    if (!this.connected) {
        callback(new NotConnectedError());
        return;
    }

    Step(
        function() {
            bank.q.enqueue(fs.stat, [filename], this);
        },
        function(err, stats) {
            if (err) {
                if (err.code == 'ENOENT') {
                    throw new NoSuchThingError(type, id);
                } else {
                    throw err;
                }
            }
            // open in r+ so we don't truncate existing file before locking
            bank.q.enqueue(fs.open, [filename, 'r+', bank.mode], this);
        },
        function(err, result) {
            if (err) throw err;
            fd = result;
            // XXX: skip locking if bank.noLock
            bank.q.enqueue(fs.flock, [fd, 'ex'], this);
        },
        function(err) {
            if (err) throw err;
            // Locked; now we truncate
            bank.q.enqueue(fs.truncate, [fd, 0], this);
        },
        function(err) {
            var b;
            if (err) throw err;
            b = new Buffer(JSON.stringify(value), 'utf8');
            bank.q.enqueue(fs.write, [fd, b, 0, b.length, null], this);
        },
        function(err, written, buffer) {
            if (err) throw err;
            bank.q.enqueue(fs.close, [fd], this);
        },
        function(err) {
            if (err) {
                callback(err, null);
            } else {
                callback(null, value);
            }
        }
    );
};

DiskDatabank.prototype.read = function(type, id, callback) {
    var bank = this,
        filename = this.toFilename(type, id),
        fsize,
        buf,
        fd;

    if (!this.connected) {
        callback(new NotConnectedError());
        return;
    }

    Step(
        function() {
            bank.q.enqueue(fs.open, [filename, 'r', bank.mode], this);
        },
        function(err, result) {
            if (err) {
                if (err.code == 'ENOENT') {
                    throw new NoSuchThingError(type, id);
                } else {
                    throw err;
                }
            }
            fd = result;
            bank.q.enqueue(fs.flock, [fd, 'sh'], this);
        },
        function(err) {
            if (err) throw err;
            bank.q.enqueue(fs.seek, [fd, 0, 2], this);
        },
        function(err, pos) {
            if (err) throw err;
            fsize = pos;
            buf = new Buffer(fsize);
            bank.q.enqueue(fs.read, [fd, buf, 0, fsize, 0], this);
        },
        function(err, count, b) {
            var parsed;
            if (err) throw err;
            bank.q.enqueue(fs.close, [fd], this);
        },
        function(err) {
            var parsed;
            if (err) {
                callback(err);
            } else {
                try {
                    parsed = JSON.parse(buf.toString('utf8'));
                    callback(null, parsed);
                } catch (e) {
                    callback(e, null);
                }
            }
        }
    );
};

DiskDatabank.prototype.del = function(type, id, callback) {
    var bank = this,
        filename = this.toFilename(type, id);

    if (!this.connected) {
        callback(new NotConnectedError());
        return;
    }

    Step(
        function() {
            bank.q.enqueue(fs.unlink, [filename], this);
        },
        function(err) {
            if (err) {
                if (err.code == 'ENOENT') {
                    callback(new NoSuchThingError(type, id), null);
                } else {
                    callback(err, null);
                }
            } else {
                callback(null);
            }
        }
    );
};

DiskDatabank.prototype.ensureDir = function(dir, callback) {

    var bank = this;

    Step(
        function() {
	    bank.q.enqueue(fs.stat, [dir], this);
        },
        function(err, stat) {
            if (err) {
                if (err.code == 'ENOENT') {
                    bank.q.enqueue(mkdirp, [dir, (bank.mode | 0111)], this);
                } else {
                    throw err;
                }
            } else {
                // no error
                if (stat.isDirectory()) {
                    callback(null);
                } else {
                    callback(new Error(dir + " exists and is not a directory."));
                }
            }
        },
        function(err, made) {
            if (err) {
                callback(err);
            } else {
                callback(null);
            }
        }
    );
};

DiskDatabank.prototype.search = function(type, criteria, onResult, onCompletion) {
    var bank = this,
	counter = 0,
        checkOrWalk = function(path, callback) {
            Step(
                function() {
		    bank.q.enqueue(fs.stat, [path], this);
                },
                function(err, stat) {
                    if (err) {
                        if (err.code == 'ENOENT') {
                            // silently ignore missing directory entries
                            callback(null);
                        } else {
                            throw err;
                        }
                    } else if (stat.isDirectory()) {
			walk(path, this);
                    } else {
                        check(path, this);
                    }
                },
                callback
            );
        },
	walk = function(dirname, callback) { // originally from https://gist.github.com/514983
            Step(
                function() {
	            bank.q.enqueue(fs.readdir, [dirname], this);
                },
                function(err, relnames) {
                    var group = this.group(), i;
		    if (err) throw err;
                    for (i = 0; i < relnames.length; i++) {
                        checkOrWalk(path.join(dirname, relnames[i]), group());
                    }
		},
                callback
            );
        },
        check = function(path, callback) {
            var fd, buf, fsize;
            Step(
                function() {
                    bank.q.enqueue(fs.open, [path, 'r', bank.mode], this);
                },
                function(err, result) {
                    if (err) throw err;
                    fd = result;
                    bank.q.enqueue(fs.flock, [fd, 'sh'], this);
                },
                function(err) {
                    if (err) throw err;
                    bank.q.enqueue(fs.seek, [fd, 0, 2], this);
                },
                function(err, pos) {
                    if (err) throw err;
                    fsize = pos;
                    buf = new Buffer(fsize);
                    bank.q.enqueue(fs.read, [fd, buf, 0, fsize, 0], this);
                },
                function(err, count, b) {
                    if (err) throw err;
                    bank.q.enqueue(fs.close, [fd], this);
                },
                function(err) {
                    var parsed;
                    if (err) {
                        callback(err);
                    } else {
                        try {
                            parsed = JSON.parse(buf.toString('utf8'));
		            if (bank.matchesCriteria(parsed, criteria)) {
			        onResult(parsed);
		            }
                            callback(null);
                        } catch (e) {
                            callback(e);
                        }
                    }
                }
	    );
	};

    if (!this.connected) {
        onCompletion(new NotConnectedError());
        return;
    }

    walk(path.join(bank.dir, type), onCompletion);
};

DiskDatabank.prototype.ensureTree = function(callback) {
    var bank = this;

    Step(
        function() {
            bank.ensureDir(bank.dir, this);
        },
        function(err) {
            var type, group = this.group();
            if (err) throw err;
            for (type in bank.schema) {
                if (bank.schema.hasOwnProperty(type)) {
                    bank.ensureDir(path.join(bank.dir, type), group());
                }
            }
        },
        function(err) {
            callback(err);
        }
    );
};

DiskDatabank.prototype.readAndModify = function(type, id, def, modify, callback) {
    var bank = this,
        filename = this.toFilename(type, id),
        fd,
        fsize,
        buf,
        newValue;

    if (!this.connected) {
        callback(new NotConnectedError());
        return;
    }

    Step(
        function() {
            bank.q.enqueue(fs.open, [filename, 'r+', bank.mode], this);
        },
        function(err, result) {
            if (err) {
                if (err.code == 'ENOENT') {
                    bank.create(type, id, def, callback);
                    return;
                } else {
                    throw err;
                }
            }
            fd = result;
            bank.q.enqueue(fs.flock, [fd, 'ex'], this);
        },
        function(err) {
            if (err) throw err;
            bank.q.enqueue(fs.seek, [fd, 0, 2], this);
        },
        function(err, pos) {
            if (err) throw err;
            fsize = pos;
            buf = new Buffer(fsize);
            bank.q.enqueue(fs.read, [fd, buf, 0, fsize, 0], this);
        },
        function(err, count, b) {
            var oldValue;
            if (err) throw err;
            // May throw an error; let it!
            console.log(buf);
            oldValue = JSON.parse(buf.toString('utf8'));
            newValue = modify(oldValue);
            bank.q.enqueue(fs.truncate, [fd, 0], this);
        },
        function(err) {
            var newBuf;
            if (err) throw err;
            newBuf = new Buffer(JSON.stringify(newValue), 'utf8');
            bank.q.enqueue(fs.write, [fd, newBuf, 0, newBuf.length, 0], this);
        },
        function(err) {
            if (err) throw err;
            bank.q.enqueue(fs.close, [fd], this);
        },
        function(err) {
            if (err) {
                callback(err, null);
            } else {
                callback(null, newValue);
            }
        }
    );
};

module.exports = DiskDatabank;
<|endoftext|>


<commit_before>/**
 * @depend ../sinon.js
 * @depend stub.js
 * @depend mock.js
 * @depend sandbox.js
 */
/*jslint eqeqeq: false, onevar: false, forin: true, plusplus: false*/
/*global module, require, sinon*/
/**
 * Test function, sandboxes fakes
 *
 * @author Christian Johansen (christian@cjohansen.no)
 * @license BSD
 *
 * Copyright (c) 2010-2011 Christian Johansen
 */
"use strict";

(function (sinon) {
    var commonJSModule = typeof module == "object" && typeof require == "function";

    if (!sinon && commonJSModule) {
        sinon = require("../sinon");
    }

    if (!sinon) {
        return;
    }

    function test(callback) {
        var type = typeof callback;

        if (type != "function") {
            throw new TypeError("sinon.test needs to wrap a test function, got " + type);
        }

        return function () {
            var config = sinon.getConfig(sinon.config);
            config.injectInto = config.injectIntoThis && this || config.injectInto;
            var sandbox = sinon.sandbox.create(config);
            var exception, result;
            var args = Array.prototype.slice.call(arguments).concat(sandbox.args);

            try {
                result = callback.apply(this, args);
            } catch (e) {
                exception = e;
            }

            sandbox.verifyAndRestore();

            if (exception) {
                throw exception;
            }

            return result;
        };
    }

    test.config = {
        injectIntoThis: true,
        injectInto: null,
        properties: ["spy", "stub", "mock", "clock", "server", "requests"],
        useFakeTimers: true,
        useFakeServer: true
    };

    if (commonJSModule) {
        module.exports = test;
    } else {
        sinon.test = test;
    }
}(typeof sinon == "object" && sinon || null));
<commit_msg>sinon.test: use try/finally instead of catch and throw.<commit_after>/**
 * @depend ../sinon.js
 * @depend stub.js
 * @depend mock.js
 * @depend sandbox.js
 */
/*jslint eqeqeq: false, onevar: false, forin: true, plusplus: false*/
/*global module, require, sinon*/
/**
 * Test function, sandboxes fakes
 *
 * @author Christian Johansen (christian@cjohansen.no)
 * @license BSD
 *
 * Copyright (c) 2010-2011 Christian Johansen
 */
"use strict";

(function (sinon) {
    var commonJSModule = typeof module == "object" && typeof require == "function";

    if (!sinon && commonJSModule) {
        sinon = require("../sinon");
    }

    if (!sinon) {
        return;
    }

    function test(callback) {
        var type = typeof callback;

        if (type != "function") {
            throw new TypeError("sinon.test needs to wrap a test function, got " + type);
        }

        return function () {
            var config = sinon.getConfig(sinon.config);
            config.injectInto = config.injectIntoThis && this || config.injectInto;
            var sandbox = sinon.sandbox.create(config);
            var exception, result;
            var args = Array.prototype.slice.call(arguments).concat(sandbox.args);

            try {
                result = callback.apply(this, args);
            } finally {
                sandbox.verifyAndRestore();
            }

            return result;
        };
    }

    test.config = {
        injectIntoThis: true,
        injectInto: null,
        properties: ["spy", "stub", "mock", "clock", "server", "requests"],
        useFakeTimers: true,
        useFakeServer: true
    };

    if (commonJSModule) {
        module.exports = test;
    } else {
        sinon.test = test;
    }
}(typeof sinon == "object" && sinon || null));
<|endoftext|>


<commit_before>// Filter function for
//
// Copyright 2011, 2012 StatusNet Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

var databank = require('databank'),
    Databank = databank.Databank,
    NoSuchThingError = databank.NoSuchThingError,
    Step = require('step'),
    crypto = require('crypto'),
    dateFormat = require('dateformat'),
    Tokenizer = require('./tokenizer').Tokenizer;

var RELEVANCE_CUTOFF = 20;
var MINIMUM_OCCURENCES = 3;
var MINPROB = 0.0001;
var MAXPROB = 0.9999;
var DEFAULT_PROB = 0.4; // default probability for unseen values
var SPAM_PROB = 0.90; // cutoff for saying is or isn't
var UP = +1;
var DOWN = -1;
var DIGRAMS = true;
var PREFIXES = true;

var SpamFilter = {

    db: null,

    opposite: function(cat) {
        if (cat === 'ham') {
            return 'spam';
        } else if (cat === 'spam') {
            return 'ham';
        } else {
            throw new Error('Unknown category: ' + cat);
        }
    },

    updateTokenCounts: function(cat, tokens, dir, callback) {

        var filter = this,
            opp = this.opposite(cat),
            catTotalKey = cat + 'total',
            oppTotalKey = opp + 'total',
            catTotal, oppTotal;

        Step(
            function() {
                if (dir === UP) {
                    filter.db.incr(catTotalKey, 1, this);
                } else {
                    filter.db.decr(catTotalKey, 1, this);
                }
            },
            function(err, result) {
                if (err) throw err;
                catTotal = result;
                filter.db.read(oppTotalKey, 1, this);
            },
            function(err, result) {
                var i, group = this.group();
                if (err) {
                    if (err instanceof NoSuchThingError) {
                        oppTotal = 0;
                    } else {
                        throw err;
                    }
                } else {
                    oppTotal = result;
                }
                for (i in tokens) { // Not sure I love this
                    filter.updateTokenCount(cat, tokens[i], dir, catTotal, oppTotal, group());
                }
            },
            function(err, results) {
                if (err) {
                    callback(err, null);
                } else {
                    callback(null, results);
                }
            }
        );
    },

    swap: function(cat, tokens, callback) {

        var filter = this,
            opp = this.opposite(cat),
            catTotalKey = cat + 'total',
            oppTotalKey = opp + 'total',
            catTotal, oppTotal;

        Step(
            function() {
                filter.db.incr(catTotalKey, 1, this);
            },
            function(err, result) {
                if (err) throw err;
                catTotal = result;
                filter.db.decr(oppTotalKey, 1, this);
            },
            function(err, result) {
                var i, group = this.group();
                if (err) {
                    if (err instanceof NoSuchThingError) {
                        oppTotal = 1;
                    } else {
                        throw err;
                    }
                } else {
                    oppTotal = result;
                }
                for (i in tokens) { // Not sure I love this
                    filter.swapTokenCount(cat, tokens[i], catTotal, oppTotal, group());
                }
            },
            function(err, results) {
                if (err) {
                    callback(err, null);
                } else {
                    callback(null, results);
                }
            }
        );
    },

    learn: function(cat, tokens, callback) { // forget that tokens mean cat
        this.updateTokenCounts(cat, tokens, UP, callback);
    },

    forget: function(cat, tokens, callback) { // forget that tokens mean cat
        this.updateTokenCounts(cat, tokens, DOWN, callback);
    },

    updateTokenCount: function(cat, token, dir, catTotal, oppTotal, callback) {
        var opp = this.opposite(cat),
            filter = this,
            catCount, oppCount;

        Step(
            function() {
                if (dir === UP) {
                    filter.db.incr(cat, token, this);
                } else {
                    filter.db.decr(cat, token, this);
                }
            },
            function(err, result) {
                if (err) throw err;
                catCount = result;
                filter.db.read(opp, token, this);
            },
            function(err, result) {
                if (err instanceof NoSuchThingError) {
                    oppCount = 0;
                } else if (err) {
                    callback(err, null);
                } else {
                    oppCount = result;
                }

                if (cat === 'spam') {
                    filter.updateTokenProb(token, catCount, oppCount, catTotal, oppTotal, callback);
                } else {
                    filter.updateTokenProb(token, oppCount, catCount, oppTotal, catTotal, callback);
                }
            }
        );
    },

    swapTokenCount: function(cat, token, catTotal, oppTotal, callback) {
        var opp = this.opposite(cat),
            filter = this,
            catCount, oppCount;

        Step(
            function() {
                filter.db.incr(cat, token, this);
            },
            function(err, result) {
                if (err) throw err;
                catCount = result;
                filter.db.decr(opp, token, this);
            },
            function(err, result) {
                if (err instanceof NoSuchThingError) {
                    oppCount = 0;
                } else if (err) {
                    callback(err, null);
                } else {
                    oppCount = result;
                }

                if (cat === 'spam') {
                    filter.updateTokenProb(token, catCount, oppCount, catTotal, oppTotal, callback);
                } else {
                    filter.updateTokenProb(token, oppCount, catCount, oppTotal, catTotal, callback);
                }
            }
        );
    },

    updateTokenProb: function(token, spamCount, hamCount, spamTotal, hamTotal, callback) {
        var g = 2 * hamCount,
            b = spamCount;

        if (g + b > 5 && spamTotal !== 0 && hamTotal !== 0) {
            var p = Math.max(MINPROB,
                             Math.min(MAXPROB,
                                      Math.min(1, b/spamTotal)/
                                      (Math.min(1, g/hamTotal) + Math.min(1, b/spamTotal))));
            this.db.save('prob', token, p, callback);
        } else {
            callback(null, null);
        }
    },

    getProbabilities: function(tokens, callback) {
        this.db.readAll('prob', tokens, function(err, probs) {
            if (err) {
                callback(err, null);
                return;
            }
            var i, probabilities = [], token;
            for (i = 0; i < tokens.length; i++) {
                token = tokens[i];
                // There's probably a nicer data structure for this
                if (probs[token] === null) {
                    probabilities.push([token, DEFAULT_PROB]);
                } else {
                    probabilities.push([token, probs[token]]);
                }
            }
            callback(null, probabilities);
        });
    },

    bestProbabilities: function(probs) {
        probs.sort(function (a, b) {
            var adist = Math.abs(a[1] - 0.5);
            var bdist = Math.abs(b[1] - 0.5);

            if (adist > bdist) {
                return -1;
            } else if (bdist > adist) {
                return 1;
            } else {
                return 0;
            }
        });

        // Get the most relevant
        return probs.slice(0, Math.min(probs.length, RELEVANCE_CUTOFF));
    },

    combineProbabilities: function(probs) {
        var prod = probs.reduce(function(coll, cur, index, array) {
            return coll * cur[1];
        }, 1.0);

        var invprod = probs.reduce(function(coll, cur, index, array) {
            return coll * (1 - cur[1]);
        }, 1.0);

        //bounded values
        return Math.min(MAXPROB, Math.max(MINPROB, (prod)/(prod + invprod))); // really?
    },

    test: function(tokens, callback) {

        var start = Date.now(),
            filter = this;

        Step(
            function() {
                filter.getProbabilities(tokens, this);
            },
            function(err, probs) {

                var bestprobs, prob, decision;

                if (err) {
                    callback(err, null);
                    return;
                }

                bestprobs = filter.bestProbabilities(probs);
                prob = filter.combineProbabilities(bestprobs);

                decision = { probability: prob,
                             isSpam: ((prob > SPAM_PROB) ? true : false),
                             bestKeys: bestprobs,
                             elapsed: Date.now() - start};

                callback(null, decision);
            }
        );
    },

    train: function(cat, obj, callback) {

        var filter = this,
            start = Date.now(),
            hash = filter.hashObject(obj),
            tokens = Tokenizer.tokenize(obj);

        filter.db.read("trainrec", hash, function(err, trainrec) {
            if (err instanceof NoSuchThingError) { // Never trained before
                filter.learn(cat, tokens, function(err, results) {
                    if (err) {
                        callback(err, null);
                    } else {
                        trainrec = {cat: cat,
                                    object: JSON.stringify(obj),
                                    date: dateFormat(new Date(), "isoDateTime", true),
                                    elapsed: Date.now() - start};
                        filter.db.save("trainrec", hash, trainrec, callback);
                    }
                });
            } else if (err) { // DB error
                callback(err, null);
            } else if (trainrec.cat === cat) { // trained same; return old training info
                callback(null, trainrec);
            } else if (trainrec.cat === filter.opposite(cat)) { // trained opposite
                // XXX: Do these need to be sequenced...?
                filter.swap(cat, tokens, function(err, results) {
                    if (err) {
                        callback(err, null);
                    } else {
                        trainrec = {cat: cat,
                                    object: JSON.stringify(obj),
                                    date: dateFormat(new Date(), "isoDateTime", true),
                                    elapsed: Date.now() - start};
                        filter.db.save("trainrec", hash, trainrec, callback);
                    }
                });
            }
        });
    },

    hashObject: function(obj) {
        var dig = null;
        var str = JSON.stringify(obj); // Canonicalize? Fuzz? BOTH!?
        var hash = crypto.createHash('md5'); // XXX: might not be there!
        hash.update(str);
        dig = hash.digest('base64');
        // URL-safe
        dig = dig.replace(/\+/g, '-');
        dig = dig.replace(/\//g, '_');
        dig = dig.replace(/=/g, '');
        return dig;
    },

    schema: {'spam': {pkey: 'token'},
             'ham': {pkey: 'token'},
             'prob': {pkey: 'token'},
             'trainrec': {pkey: 'hash'},
             'spamtotal': {pkey: 'service'},
             'hamtotal': {pkey: 'service'}}
};

exports.SpamFilter = SpamFilter;
<commit_msg>let's see those results<commit_after>// Filter function for
//
// Copyright 2011, 2012 StatusNet Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

var databank = require('databank'),
    Databank = databank.Databank,
    NoSuchThingError = databank.NoSuchThingError,
    Step = require('step'),
    crypto = require('crypto'),
    dateFormat = require('dateformat'),
    Tokenizer = require('./tokenizer').Tokenizer;

var RELEVANCE_CUTOFF = 20;
var MINIMUM_OCCURENCES = 3;
var MINPROB = 0.0001;
var MAXPROB = 0.9999;
var DEFAULT_PROB = 0.4; // default probability for unseen values
var SPAM_PROB = 0.90; // cutoff for saying is or isn't
var UP = +1;
var DOWN = -1;
var DIGRAMS = true;
var PREFIXES = true;

var SpamFilter = {

    db: null,

    opposite: function(cat) {
        if (cat === 'ham') {
            return 'spam';
        } else if (cat === 'spam') {
            return 'ham';
        } else {
            throw new Error('Unknown category: ' + cat);
        }
    },

    updateTokenCounts: function(cat, tokens, dir, callback) {

        var filter = this,
            opp = this.opposite(cat),
            catTotalKey = cat + 'total',
            oppTotalKey = opp + 'total',
            catTotal, oppTotal;

        Step(
            function() {
                if (dir === UP) {
                    filter.db.incr(catTotalKey, 1, this);
                } else {
                    filter.db.decr(catTotalKey, 1, this);
                }
            },
            function(err, result) {
                if (err) throw err;
                catTotal = result;
                filter.db.read(oppTotalKey, 1, this);
            },
            function(err, result) {
                var i, group = this.group();
                if (err) {
                    if (err instanceof NoSuchThingError) {
                        oppTotal = 0;
                    } else {
                        throw err;
                    }
                } else {
                    oppTotal = result;
                }
                for (i in tokens) { // Not sure I love this
                    filter.updateTokenCount(cat, tokens[i], dir, catTotal, oppTotal, group());
                }
            },
            function(err, results) {
                if (err) {
                    callback(err, null);
                } else {
                    callback(null, results);
                }
            }
        );
    },

    swap: function(cat, tokens, callback) {

        var filter = this,
            opp = this.opposite(cat),
            catTotalKey = cat + 'total',
            oppTotalKey = opp + 'total',
            catTotal, oppTotal;

        Step(
            function() {
                filter.db.incr(catTotalKey, 1, this);
            },
            function(err, result) {
                if (err) throw err;
                catTotal = result;
                filter.db.decr(oppTotalKey, 1, this);
            },
            function(err, result) {
                var i, group = this.group();
                if (err) {
                    if (err instanceof NoSuchThingError) {
                        oppTotal = 1;
                    } else {
                        throw err;
                    }
                } else {
                    oppTotal = result;
                }
                for (i in tokens) { // Not sure I love this
                    filter.swapTokenCount(cat, tokens[i], catTotal, oppTotal, group());
                }
            },
            function(err, results) {
                if (err) {
                    callback(err, null);
                } else {
                    callback(null, results);
                }
            }
        );
    },

    learn: function(cat, tokens, callback) { // forget that tokens mean cat
        this.updateTokenCounts(cat, tokens, UP, callback);
    },

    forget: function(cat, tokens, callback) { // forget that tokens mean cat
        this.updateTokenCounts(cat, tokens, DOWN, callback);
    },

    updateTokenCount: function(cat, token, dir, catTotal, oppTotal, callback) {
        var opp = this.opposite(cat),
            filter = this,
            catCount, oppCount;

        Step(
            function() {
                if (dir === UP) {
                    filter.db.incr(cat, token, this);
                } else {
                    filter.db.decr(cat, token, this);
                }
            },
            function(err, result) {
                if (err) throw err;
                catCount = result;
                filter.db.read(opp, token, this);
            },
            function(err, result) {
                if (err instanceof NoSuchThingError) {
                    oppCount = 0;
                } else if (err) {
                    callback(err, null);
                } else {
                    oppCount = result;
                }

                if (cat === 'spam') {
                    filter.updateTokenProb(token, catCount, oppCount, catTotal, oppTotal, callback);
                } else {
                    filter.updateTokenProb(token, oppCount, catCount, oppTotal, catTotal, callback);
                }
            }
        );
    },

    swapTokenCount: function(cat, token, catTotal, oppTotal, callback) {
        var opp = this.opposite(cat),
            filter = this,
            catCount, oppCount;

        Step(
            function() {
                filter.db.incr(cat, token, this);
            },
            function(err, result) {
                if (err) throw err;
                catCount = result;
                filter.db.decr(opp, token, this);
            },
            function(err, result) {
                if (err instanceof NoSuchThingError) {
                    oppCount = 0;
                } else if (err) {
                    callback(err, null);
                } else {
                    oppCount = result;
                }

                if (cat === 'spam') {
                    filter.updateTokenProb(token, catCount, oppCount, catTotal, oppTotal, callback);
                } else {
                    filter.updateTokenProb(token, oppCount, catCount, oppTotal, catTotal, callback);
                }
            }
        );
    },

    updateTokenProb: function(token, spamCount, hamCount, spamTotal, hamTotal, callback) {
        var g = 2 * hamCount,
            b = spamCount;

        if (g + b > 5 && spamTotal !== 0 && hamTotal !== 0) {
            var p = Math.max(MINPROB,
                             Math.min(MAXPROB,
                                      Math.min(1, b/spamTotal)/
                                      (Math.min(1, g/hamTotal) + Math.min(1, b/spamTotal))));
            this.db.save('prob', token, p, callback);
        } else {
            callback(null, null);
        }
    },

    getProbabilities: function(tokens, callback) {
        this.db.readAll('prob', tokens, function(err, probs) {
            if (err) {
                callback(err, null);
                return;
            }
            var i, probabilities = [], token;
            for (i = 0; i < tokens.length; i++) {
                token = tokens[i];
                // There's probably a nicer data structure for this
                if (probs[token] === null) {
                    probabilities.push([token, DEFAULT_PROB]);
                } else {
                    probabilities.push([token, probs[token]]);
                }
            }
            callback(null, probabilities);
        });
    },

    bestProbabilities: function(probs) {
        probs.sort(function (a, b) {
            var adist = Math.abs(a[1] - 0.5);
            var bdist = Math.abs(b[1] - 0.5);

            if (adist > bdist) {
                return -1;
            } else if (bdist > adist) {
                return 1;
            } else {
                return 0;
            }
        });

        // Get the most relevant
        return probs.slice(0, Math.min(probs.length, RELEVANCE_CUTOFF));
    },

    combineProbabilities: function(probs) {
        var prod = probs.reduce(function(coll, cur, index, array) {
            return coll * cur[1];
        }, 1.0);

        var invprod = probs.reduce(function(coll, cur, index, array) {
            return coll * (1 - cur[1]);
        }, 1.0);

        //bounded values
        return Math.min(MAXPROB, Math.max(MINPROB, (prod)/(prod + invprod))); // really?
    },

    test: function(tokens, callback) {

        var start = Date.now(),
            filter = this;

        Step(
            function() {
                filter.getProbabilities(tokens, this);
            },
            function(err, probs) {

                var bestprobs, prob, decision;

                if (err) {
                    callback(err, null);
                    return;
                }

                bestprobs = filter.bestProbabilities(probs);
                prob = filter.combineProbabilities(bestprobs);

                decision = { probability: prob,
                             isSpam: ((prob > SPAM_PROB) ? true : false),
                             bestKeys: bestprobs,
                             elapsed: Date.now() - start};

                callback(null, decision);
            }
        );
    },

    train: function(cat, obj, callback) {

        var filter = this,
            start = Date.now(),
            hash = filter.hashObject(obj),
            tokens = Tokenizer.tokenize(obj);

        filter.db.read("trainrec", hash, function(err, trainrec) {
            if (err instanceof NoSuchThingError) { // Never trained before
                filter.learn(cat, tokens, function(err, results) {
                    if (err) {
                        callback(err, null);
                    } else {
                        trainrec = {cat: cat,
                                    object: JSON.stringify(obj),
                                    date: dateFormat(new Date(), "isoDateTime", true),
                                    results: results,
                                    elapsed: Date.now() - start};
                        filter.db.save("trainrec", hash, trainrec, callback);
                    }
                });
            } else if (err) { // DB error
                callback(err, null);
            } else if (trainrec.cat === cat) { // trained same; return old training info
                callback(null, trainrec);
            } else if (trainrec.cat === filter.opposite(cat)) { // trained opposite
                // XXX: Do these need to be sequenced...?
                filter.swap(cat, tokens, function(err, results) {
                    if (err) {
                        callback(err, null);
                    } else {
                        trainrec = {cat: cat,
                                    object: JSON.stringify(obj),
                                    date: dateFormat(new Date(), "isoDateTime", true),
                                    results: results,
                                    elapsed: Date.now() - start};
                        filter.db.save("trainrec", hash, trainrec, callback);
                    }
                });
            }
        });
    },

    hashObject: function(obj) {
        var dig = null;
        var str = JSON.stringify(obj); // Canonicalize? Fuzz? BOTH!?
        var hash = crypto.createHash('md5'); // XXX: might not be there!
        hash.update(str);
        dig = hash.digest('base64');
        // URL-safe
        dig = dig.replace(/\+/g, '-');
        dig = dig.replace(/\//g, '_');
        dig = dig.replace(/=/g, '');
        return dig;
    },

    schema: {'spam': {pkey: 'token'},
             'ham': {pkey: 'token'},
             'prob': {pkey: 'token'},
             'trainrec': {pkey: 'hash'},
             'spamtotal': {pkey: 'service'},
             'hamtotal': {pkey: 'service'}}
};

exports.SpamFilter = SpamFilter;
<|endoftext|>


<commit_before>/* global Uint8Array, Buffer, TextDecoder */
var Transform = require('stream').Transform,
    tools = require('./tools.js'),
    schema = require('./schema.js'),
    debug = require('debug')('ebml:decoder');

var STATE_TAG = 1,
    STATE_SIZE = 2,
    STATE_CONTENT = 3;

function EbmlDecoder(options) {
    options = options || {};
    options.readableObjectMode = true;
    Transform.call(this, options);

    this._buffer = null;
    this._tag_stack = [];
    this._state = STATE_TAG;
    this._cursor = 0;
    this._total = 0;
    this._schema = schema;
}

require('util').inherits(EbmlDecoder, Transform);

EbmlDecoder.prototype._transform = function(chunk, enc, done) {

    if (this._buffer === null) {
        this._buffer = new Uint8Array(chunk);
    } else {
        this._buffer = tools.concatenate(this._buffer, new Uint8Array(chunk));
    }

    while (this._cursor < this._buffer.length) {
        if (this._state === STATE_TAG && !this.readTag()) {
            break;
        }
        if (this._state === STATE_SIZE && !this.readSize()) {
            break;
        }
        if (this._state === STATE_CONTENT && !this.readContent()) {
            break;
        }
    }

    done();
};

EbmlDecoder.prototype.getSchemaInfo = function(tagStr) {
    return this._schema[tagStr] || {
        "type": "unknown",
        "name": "unknown"
    };
};

EbmlDecoder.prototype.readTag = function() {

    debug('parsing tag');

    if (this._cursor >= this._buffer.length) {
        debug('waiting for more data');
        return false;
    }

    var start = this._total;
    var tag = tools.readVint(this._buffer, this._cursor);

    if (tag == null) {
        debug('waiting for more data');
        return false;
    }

    var tagStr = tools.readHexString(this._buffer, this._cursor, this._cursor + tag.length);

    this._cursor += tag.length;
    this._total += tag.length;
    this._state = STATE_SIZE;

    var tagObj = {
        tag: tag.value,
        tagStr: tagStr,
        type: this.getSchemaInfo(tagStr).type,
        name: this.getSchemaInfo(tagStr).name,
        start: start,
        end: start + tag.length
    };

    this._tag_stack.push(tagObj);
    debug('read tag: ' + tagStr);

    return true;
};

EbmlDecoder.prototype.readSize = function() {

    var tagObj = this._tag_stack[this._tag_stack.length - 1];

    debug('parsing size for tag: ' + tagObj.tag.toString(16));

    if (this._cursor >= this._buffer.length) {
        debug('waiting for more data');
        return false;
    }

    var size = tools.readVint(this._buffer, this._cursor);

    if (size == null) {
        debug('waiting for more data');
        return false;
    }

    this._cursor += size.length;
    this._total += size.length;
    this._state = STATE_CONTENT;
    tagObj.dataSize = size.value;

    // unknown size
    if (size.value === -1) {
        tagObj.end = -1;
    } else {
        tagObj.end += size.value + size.length;
    }

    debug('read size: ' + size.value);

    return true;
};

EbmlDecoder.prototype.readContent = function() {

    var tagObj = this._tag_stack[this._tag_stack.length - 1];

    debug('parsing content for tag: ' + tagObj.tag.toString(16));

    if (tagObj.type === 'm') {
        debug('content should be tags');
        this.push(['start', tagObj]);
        this._state = STATE_TAG;
        return true;
    }

    if (this._buffer.length < this._cursor + tagObj.dataSize) {
        debug('got: ' + this._buffer.length);
        debug('need: ' + (this._cursor + tagObj.dataSize));
        debug('waiting for more data');
        return false;
    }

    var data = this._buffer.subarray(this._cursor, this._cursor + tagObj.dataSize);
    this._total += tagObj.dataSize;
    this._state = STATE_TAG;
    this._buffer = this._buffer.subarray(this._cursor + tagObj.dataSize);
    this._cursor = 0;

    this._tag_stack.pop(); // remove the object from the stack

    this.push(['tag', tools.readDataFromTag(tagObj, data)]);

    while (this._tag_stack.length > 0) {
        var topEle = this._tag_stack[this._tag_stack.length - 1];
        if (this._total < topEle.end) {
            break;
        }
        this.push(['end', topEle]);
        this._tag_stack.pop();
    }

    debug('read data: ' + data.toString('hex'));
    return true;
};

module.exports = EbmlDecoder;
<commit_msg>Use the Buffer type to pass data down a Stream pipeline.<commit_after>/* global Uint8Array, Buffer, TextDecoder */
var Transform = require('stream').Transform,
    tools = require('./tools.js'),
    schema = require('./schema.js'),
    debug = require('debug')('ebml:decoder');

var STATE_TAG = 1,
    STATE_SIZE = 2,
    STATE_CONTENT = 3;

function EbmlDecoder(options) {
    options = options || {};
    options.readableObjectMode = true;
    Transform.call(this, options);

    this._buffer = null;
    this._tag_stack = [];
    this._state = STATE_TAG;
    this._cursor = 0;
    this._total = 0;
    this._schema = schema;
}

require('util').inherits(EbmlDecoder, Transform);

EbmlDecoder.prototype._transform = function(chunk, enc, done) {

    if (this._buffer === null) {
        this._buffer = new Uint8Array(chunk);
    } else {
        this._buffer = tools.concatenate(this._buffer, new Uint8Array(chunk));
    }

    while (this._cursor < this._buffer.length) {
        if (this._state === STATE_TAG && !this.readTag()) {
            break;
        }
        if (this._state === STATE_SIZE && !this.readSize()) {
            break;
        }
        if (this._state === STATE_CONTENT && !this.readContent()) {
            break;
        }
    }

    done();
};

EbmlDecoder.prototype.getSchemaInfo = function(tagStr) {
    return this._schema[tagStr] || {
        "type": "unknown",
        "name": "unknown"
    };
};

EbmlDecoder.prototype.readTag = function() {

    debug('parsing tag');

    if (this._cursor >= this._buffer.length) {
        debug('waiting for more data');
        return false;
    }

    var start = this._total;
    var tag = tools.readVint(this._buffer, this._cursor);

    if (tag == null) {
        debug('waiting for more data');
        return false;
    }

    var tagStr = tools.readHexString(this._buffer, this._cursor, this._cursor + tag.length);

    this._cursor += tag.length;
    this._total += tag.length;
    this._state = STATE_SIZE;

    var tagObj = {
        tag: tag.value,
        tagStr: tagStr,
        type: this.getSchemaInfo(tagStr).type,
        name: this.getSchemaInfo(tagStr).name,
        start: start,
        end: start + tag.length
    };

    this._tag_stack.push(tagObj);
    debug('read tag: ' + tagStr);

    return true;
};

EbmlDecoder.prototype.readSize = function() {

    var tagObj = this._tag_stack[this._tag_stack.length - 1];

    debug('parsing size for tag: ' + tagObj.tag.toString(16));

    if (this._cursor >= this._buffer.length) {
        debug('waiting for more data');
        return false;
    }

    var size = tools.readVint(this._buffer, this._cursor);

    if (size == null) {
        debug('waiting for more data');
        return false;
    }

    this._cursor += size.length;
    this._total += size.length;
    this._state = STATE_CONTENT;
    tagObj.dataSize = size.value;

    // unknown size
    if (size.value === -1) {
        tagObj.end = -1;
    } else {
        tagObj.end += size.value + size.length;
    }

    debug('read size: ' + size.value);

    return true;
};

EbmlDecoder.prototype.readContent = function() {

    var tagObj = this._tag_stack[this._tag_stack.length - 1];

    debug('parsing content for tag: ' + tagObj.tag.toString(16));

    if (tagObj.type === 'm') {
        debug('content should be tags');
        this.push(['start', tagObj]);
        this._state = STATE_TAG;
        return true;
    }

    if (this._buffer.length < this._cursor + tagObj.dataSize) {
        debug('got: ' + this._buffer.length);
        debug('need: ' + (this._cursor + tagObj.dataSize));
        debug('waiting for more data');
        return false;
    }

    var data = this._buffer.subarray(this._cursor, this._cursor + tagObj.dataSize);
    this._total += tagObj.dataSize;
    this._state = STATE_TAG;
    this._buffer = this._buffer.subarray(this._cursor + tagObj.dataSize);
    this._cursor = 0;

    this._tag_stack.pop(); // remove the object from the stack

    this.push(['tag', tools.readDataFromTag(tagObj, Buffer.from(data))]);

    while (this._tag_stack.length > 0) {
        var topEle = this._tag_stack[this._tag_stack.length - 1];
        if (this._total < topEle.end) {
            break;
        }
        this.push(['end', topEle]);
        this._tag_stack.pop();
    }

    debug('read data: ' + data.toString('hex'));
    return true;
};

module.exports = EbmlDecoder;
<|endoftext|>


<commit_before>//override filter compilation with these functions.

var BINARY_OPS_TO_SQL_OPS = {
    '==':  '=',
    '!=':  '<>',
    '=~':  'LIKE',
    '!~':  'NOT LIKE'
};

module.exports = {
    initialize: function(options) {
        this.baseQuery = options.baseQuery;
    },
    compile: function(node) {
        return this.visit(node, this.baseQuery);
    },
    visitExpressionFilterTerm: function(node, sql_query) {
        return this.visit(node.expression, sql_query);
    },
    visitBinaryExpression: function(node, sql_query) {
        var self = this;
        var SQL_OP = BINARY_OPS_TO_SQL_OPS[node.operator] || node.operator;

        if (SQL_OP === 'OR' || SQL_OP === 'AND') {
            var whereSpecfic = SQL_OP === 'OR' ? 'orWhere' : 'where';

            return sql_query.where(function() {
                return self.visit(node.left, this);
            })[whereSpecfic](function() {
                return self.visit(node.right, this);
            });
        }

        if (/LIKE/.test(SQL_OP)) {
            node.right.value = node.right.value.replace(/\*/g, '%').replace(/\?/g, '_');
        }

        return sql_query.where(this.visit(node.left), SQL_OP, this.visit(node.right));
    },
    visitUnaryExpression: function(node, sql_query) {
        var self = this;

        switch (node.operator) {
            case 'NOT':
                return sql_query.whereNot(function() {
                    return self.visit(node.expression, this);
                });
            case '*':
                return this.visit(node.expression);
            default:
                throw new Error('Invalid operator: ' + node.operator + '.');
        }
    },
    visitStringLiteral: function(node) {
        return String(node.value);
    },
    visitArrayLiteral: function(node) {
        var self = this;
        return node.elements.map(function(e) {
            return self.visit(e);
        });
    },
    visitMomentConstant: function(node) {
        return new Date(node.value);
    }
};
<commit_msg>filter: update to the AST node change for MomentLiteral<commit_after>//override filter compilation with these functions.

var BINARY_OPS_TO_SQL_OPS = {
    '==':  '=',
    '!=':  '<>',
    '=~':  'LIKE',
    '!~':  'NOT LIKE'
};

module.exports = {
    initialize: function(options) {
        this.baseQuery = options.baseQuery;
    },
    compile: function(node) {
        return this.visit(node, this.baseQuery);
    },
    visitExpressionFilterTerm: function(node, sql_query) {
        return this.visit(node.expression, sql_query);
    },
    visitBinaryExpression: function(node, sql_query) {
        var self = this;
        var SQL_OP = BINARY_OPS_TO_SQL_OPS[node.operator] || node.operator;

        if (SQL_OP === 'OR' || SQL_OP === 'AND') {
            var whereSpecfic = SQL_OP === 'OR' ? 'orWhere' : 'where';

            return sql_query.where(function() {
                return self.visit(node.left, this);
            })[whereSpecfic](function() {
                return self.visit(node.right, this);
            });
        }

        if (/LIKE/.test(SQL_OP)) {
            node.right.value = node.right.value.replace(/\*/g, '%').replace(/\?/g, '_');
        }

        return sql_query.where(this.visit(node.left), SQL_OP, this.visit(node.right));
    },
    visitUnaryExpression: function(node, sql_query) {
        var self = this;

        switch (node.operator) {
            case 'NOT':
                return sql_query.whereNot(function() {
                    return self.visit(node.expression, this);
                });
            case '*':
                return this.visit(node.expression);
            default:
                throw new Error('Invalid operator: ' + node.operator + '.');
        }
    },
    visitStringLiteral: function(node) {
        return String(node.value);
    },
    visitArrayLiteral: function(node) {
        var self = this;
        return node.elements.map(function(e) {
            return self.visit(e);
        });
    },
    visitMomentLiteral: function(node) {
        return new Date(node.value);
    }
};
<|endoftext|>