<commit_before>define( [
            'dojo/_base/declare',
            'dojo/_base/array',
            'JBrowse/Util',
            'JBrowse/Store/LRUCache',
            './Util',
            './Feature'
        ],
        function( declare, array, Util, LRUCache, BAMUtil, BAMFeature ) {

var BAM_MAGIC = 21840194;
var BAI_MAGIC = 21578050;

var dlog = function(){ console.error.apply(console, arguments); };

//
// Binning (transliterated from SAM1.3 spec)
//

/* calculate bin given an alignment covering [beg,end) (zero-based, half-close-half-open) */
function reg2bin(beg, end)
{
    --end;
    if (beg>>14 == end>>14) return ((1<<15)-1)/7 + (beg>>14);
    if (beg>>17 == end>>17) return ((1<<12)-1)/7 + (beg>>17);
    if (beg>>20 == end>>20) return ((1<<9)-1)/7 + (beg>>20);
    if (beg>>23 == end>>23) return ((1<<6)-1)/7 + (beg>>23);
    if (beg>>26 == end>>26) return ((1<<3)-1)/7 + (beg>>26);
    return 0;
}

/* calculate the list of bins that may overlap with region [beg,end) (zero-based) */
var MAX_BIN = (((1<<18)-1)/7);
function reg2bins(beg, end)
{
    var k, list = [];
    --end;
    list.push(0);
    for (k = 1 + (beg>>26); k <= 1 + (end>>26); ++k) list.push(k);
    for (k = 9 + (beg>>23); k <= 9 + (end>>23); ++k) list.push(k);
    for (k = 73 + (beg>>20); k <= 73 + (end>>20); ++k) list.push(k);
    for (k = 585 + (beg>>17); k <= 585 + (end>>17); ++k) list.push(k);
    for (k = 4681 + (beg>>14); k <= 4681 + (end>>14); ++k) list.push(k);
    return list;
}

var Chunk = function(minv,maxv) {
        this.minv = minv;
        this.maxv = maxv;
};

var readInt   = BAMUtil.readInt;
var readVirtualOffset = BAMUtil.readVirtualOffset;

var BamFile = declare( null,

/**
 * @lends JBrowse.Store.SeqFeature.BAM.File
 */
{

    /**
     * Low-level BAM file reading code.
     *
     * Adapted by Robert Buels from bam.js in the Dalliance Genome
     * Explorer which is copyright Thomas Down 2006-2010
     * @constructs
     */
    constructor: function( args ) {
        this.store = args.store;
        this.data  = args.data;
        this.bai   = args.bai;
    },

    init: function( args ) {
        var bam = this;
        var successCallback = args.success || function() {};
        var failCallback = args.failure || function() {};

        this._readBAI( dojo.hitch( this, function() {
            this._readBAMheader( function() {
                successCallback();
            }, failCallback );
        }), failCallback );
    },

    _readBAI: function( successCallback, failCallback ) {
        // Do we really need to fetch the whole thing? :-(
        this.bai.fetch( dojo.hitch( this, function(header) {
            if (!header) {
                dlog("Couldn't access BAI");
                failCallback();
                return;
            }

            var uncba = new Uint8Array(header);
            if( readInt(uncba, 0) != BAI_MAGIC) {
                dlog('Not a BAI file');
                failCallback();
                return;
            }

            var nref = readInt(uncba, 4);

            this.indices = [];

            var p = 8;
            var minVO; // get smallest virtual offset in the indexes, which indicates where the BAM header ends
            for (var ref = 0; ref < nref; ++ref) {
                var blockStart = p;
                var nbin = readInt(uncba, p); p += 4;
                for (var b = 0; b < nbin; ++b) {
                    var bin = readInt(uncba, p);
                    var nchnk = readInt(uncba, p+4);
                    p += 8 + (nchnk * 16);
                }
                var nintv = readInt(uncba, p); p += 4;
                var firstVO = readVirtualOffset(uncba,p);
                if( ! this.minAlignmentVO || this.minAlignmentVO.cmp( firstVO ) < 0 )
                    this.minAlignmentVO = firstVO;
                //console.log( ref, ''+firstVO );
                p += (nintv * 8);
                if (nbin > 0) {
                    this.indices[ref] = new Uint8Array(header, blockStart, p - blockStart);
                }
            }

            successCallback( this.indices, this.minAlignmentVO );
        }));
    },

    _readBAMheader: function( successCallback, failCallback ) {
        this.data.read( 0, this.minAlignmentVO.block, dojo.hitch( this, function(r) {
//        this.data.read(0, (1<<16)-1, dojo.hitch( this, function(r) {
            var unc = BAMUtil.unbgzf(r);
            var uncba = new Uint8Array(unc);

            if( readInt(uncba, 0) != BAM_MAGIC) {
                dlog('Not a BAM file');
                failCallback();
                return;
            }

            var headLen = readInt(uncba, 4);
            // var header = '';
            // for (var i = 0; i < headLen; ++i) {
            //     header += String.fromCharCode(uncba[i + 8]);
            // }

            var nRef = readInt(uncba, headLen + 8);
            var p = headLen + 12;

            this.chrToIndex = {};
            this.indexToChr = [];
            for (var i = 0; i < nRef; ++i) {
                var lName = readInt(uncba, p);
                var name = '';
                for (var j = 0; j < lName-1; ++j) {
                    name += String.fromCharCode(uncba[p + 4 + j]);
                }
                var lRef = readInt(uncba, p + lName + 4);
                // dlog(name + ': ' + lRef);
                this.chrToIndex[name] = i;
                if (name.indexOf('chr') == 0) {
                    this.chrToIndex[name.substring(3)] = i;
                } else {
                    this.chrToIndex['chr' + name] = i;
                }

                this.indexToChr.push(name);

                p = p + 8 + lName;
            }

            successCallback();
        }));
    },

    blocksForRange: function(refId, min, max) {
        var index = this.indices[refId];
        if (!index) {
            return [];
        }

        var intBinsL = reg2bins(min, max);
        var intBins = [];
        for (var i = 0; i < intBinsL.length; ++i) {
            intBins[intBinsL[i]] = true;
        }
        var leafChunks = [], otherChunks = [];

        var nbin = readInt(index, 0);
        var p = 4;
        for (var b = 0; b < nbin; ++b) {
            var bin = readInt(index, p);
            var nchnk = readInt(index, p+4);
    //        dlog('bin=' + bin + '; nchnk=' + nchnk);
            p += 8;
            if (intBins[bin]) {
                for (var c = 0; c < nchnk; ++c) {
                    var cs = readVirtualOffset(index, p);
                    var ce = readVirtualOffset(index, p + 8);
                    (bin < 4681 ? otherChunks : leafChunks).push(new Chunk(cs, ce));
                    p += 16;
                }
            } else {
                p +=  (nchnk * 16);
            }
        }
    //    dlog('leafChunks = ' + miniJSONify(leafChunks));
    //    dlog('otherChunks = ' + miniJSONify(otherChunks));

        var nintv = readInt(index, p);
        var lowest = null;
        var minLin = Math.min(min>>14, nintv - 1), maxLin = Math.min(max>>14, nintv - 1);
        for (var i = minLin; i <= maxLin; ++i) {
            var lb =  readVirtualOffset(index, p + 4 + (i * 8));
            if (!lb) {
                continue;
            }
            if (!lowest || lb.block < lowest.block || lb.offset < lowest.offset) {
                lowest = lb;
            }
        }
        // dlog('Lowest LB = ' + lowest);

        var prunedOtherChunks = [];
        if (lowest != null) {
            for (var i = 0; i < otherChunks.length; ++i) {
                var chnk = otherChunks[i];
                if (chnk.maxv.block >= lowest.block && chnk.maxv.offset >= lowest.offset) {
                    prunedOtherChunks.push(chnk);
                }
            }
        }
        // dlog('prunedOtherChunks = ' + miniJSONify(prunedOtherChunks));
        otherChunks = prunedOtherChunks;

        var intChunks = [];
        for (var i = 0; i < otherChunks.length; ++i) {
            intChunks.push(otherChunks[i]);
        }
        for (var i = 0; i < leafChunks.length; ++i) {
            intChunks.push(leafChunks[i]);
        }

        intChunks.sort(function(c0, c1) {
            var dif = c0.minv.block - c1.minv.block;
            if (dif != 0) {
                return dif;
            } else {
                return c0.minv.offset - c1.minv.offset;
            }
        });
        var mergedChunks = [];
        if (intChunks.length > 0) {
            var cur = intChunks[0];
            for (var i = 1; i < intChunks.length; ++i) {
                var nc = intChunks[i];
                if (nc.minv.block == cur.maxv.block /* && nc.minv.offset == cur.maxv.offset */) { // no point splitting mid-block
                    cur = new Chunk(cur.minv, nc.maxv);
                } else {
                    mergedChunks.push(cur);
                    cur = nc;
                }
            }
            mergedChunks.push(cur);
        }
    //    dlog('mergedChunks = ' + miniJSONify(mergedChunks));

        return mergedChunks;
    },

    fetch: function(chr, min, max, callback) {
        var chrId = this.chrToIndex[chr];
        var chunks;
        if (chrId === undefined) {
            chunks = [];
        } else {
            chunks = this.blocksForRange(chrId, min, max);
            if (!chunks) {
                callback(null, 'Error in index fetch');
            }
        }

        // toString function is used by the cache for making cache keys
        chunks.toString = function() {
            var str = '';
            array.forEach( this, function(c) {
                str += c.minv+'..'+c.maxv+',';
            });
            return str;
        };

        this.featureCache = this.featureCache || new LRUCache({
            name: 'bamFeatureCache',
            fillCallback: dojo.hitch(this, '_fetchChunkFeatures' ),
            sizeFunction: function( features ) {
                return features.length;
            },
            maxSize: 100000 // cache up to 100,000 BAM features
        });

        this.featureCache.get( chunks, function( features ) {
            features = array.filter( features, function( feature ) {
                return ( !( feature.get('end') < min || feature.get('start') > max )
                         && ( chrId === undefined || feature.get('_refID') == chrId ) );
            });
            callback( features );
        });

    },

    _fetchChunkFeatures: function( chunks, callback ) {
        var thisB = this;
        var features = [];
        var chunksProcessed = 0;

        if( ! chunks.length ) {
            callback([]);
            return;
        }

        array.forEach( chunks, function( c ) {
                var fetchMin = c.minv.block;
                var fetchMax = c.maxv.block + (1<<16); // *sigh*

                thisB.data.read(fetchMin, fetchMax - fetchMin + 1, function(r) {

                    var data = BAMUtil.unbgzf(r, c.maxv.block - c.minv.block + 1);

                    thisB.readBamFeatures( new Uint8Array(data), c.minv.offset, features );

                    if( ++chunksProcessed == chunks.length )
                        callback( features );
                });
        });
    },

    readBamFeatures: function(ba, blockStart, sink ) {
        while (true) {
            var blockSize = readInt(ba, blockStart);
            var blockEnd = blockStart + blockSize;
            if (blockEnd >= ba.length) {
                return sink;
            }

            var feature = new BAMFeature({
                    store: this.store,
                    file: this,
                    bytes: { byteArray: ba, start: blockStart, end: blockEnd }
            });

            sink.push(feature);
            blockStart = blockEnd + 4;
        }
        // Exits via top of loop.
    }
});

return BamFile;

});
<commit_msg>rewrite BAM feature-reading loop to stuff itself into a timeout periodically to avoid blocking UI events<commit_after>define( [
            'dojo/_base/declare',
            'dojo/_base/array',
            'JBrowse/Util',
            'JBrowse/Store/LRUCache',
            './Util',
            './Feature'
        ],
        function( declare, array, Util, LRUCache, BAMUtil, BAMFeature ) {

var BAM_MAGIC = 21840194;
var BAI_MAGIC = 21578050;

var dlog = function(){ console.error.apply(console, arguments); };

//
// Binning (transliterated from SAM1.3 spec)
//

/* calculate bin given an alignment covering [beg,end) (zero-based, half-close-half-open) */
function reg2bin(beg, end)
{
    --end;
    if (beg>>14 == end>>14) return ((1<<15)-1)/7 + (beg>>14);
    if (beg>>17 == end>>17) return ((1<<12)-1)/7 + (beg>>17);
    if (beg>>20 == end>>20) return ((1<<9)-1)/7 + (beg>>20);
    if (beg>>23 == end>>23) return ((1<<6)-1)/7 + (beg>>23);
    if (beg>>26 == end>>26) return ((1<<3)-1)/7 + (beg>>26);
    return 0;
}

/* calculate the list of bins that may overlap with region [beg,end) (zero-based) */
var MAX_BIN = (((1<<18)-1)/7);
function reg2bins(beg, end)
{
    var k, list = [];
    --end;
    list.push(0);
    for (k = 1 + (beg>>26); k <= 1 + (end>>26); ++k) list.push(k);
    for (k = 9 + (beg>>23); k <= 9 + (end>>23); ++k) list.push(k);
    for (k = 73 + (beg>>20); k <= 73 + (end>>20); ++k) list.push(k);
    for (k = 585 + (beg>>17); k <= 585 + (end>>17); ++k) list.push(k);
    for (k = 4681 + (beg>>14); k <= 4681 + (end>>14); ++k) list.push(k);
    return list;
}

var Chunk = function(minv,maxv) {
        this.minv = minv;
        this.maxv = maxv;
};

var readInt   = BAMUtil.readInt;
var readVirtualOffset = BAMUtil.readVirtualOffset;

var BamFile = declare( null,

/**
 * @lends JBrowse.Store.SeqFeature.BAM.File
 */
{

    /**
     * Low-level BAM file reading code.
     *
     * Adapted by Robert Buels from bam.js in the Dalliance Genome
     * Explorer which is copyright Thomas Down 2006-2010
     * @constructs
     */
    constructor: function( args ) {
        this.store = args.store;
        this.data  = args.data;
        this.bai   = args.bai;
    },

    init: function( args ) {
        var bam = this;
        var successCallback = args.success || function() {};
        var failCallback = args.failure || function() {};

        this._readBAI( dojo.hitch( this, function() {
            this._readBAMheader( function() {
                successCallback();
            }, failCallback );
        }), failCallback );
    },

    _readBAI: function( successCallback, failCallback ) {
        // Do we really need to fetch the whole thing? :-(
        this.bai.fetch( dojo.hitch( this, function(header) {
            if (!header) {
                dlog("Couldn't access BAI");
                failCallback();
                return;
            }

            var uncba = new Uint8Array(header);
            if( readInt(uncba, 0) != BAI_MAGIC) {
                dlog('Not a BAI file');
                failCallback();
                return;
            }

            var nref = readInt(uncba, 4);

            this.indices = [];

            var p = 8;
            var minVO; // get smallest virtual offset in the indexes, which indicates where the BAM header ends
            for (var ref = 0; ref < nref; ++ref) {
                var blockStart = p;
                var nbin = readInt(uncba, p); p += 4;
                for (var b = 0; b < nbin; ++b) {
                    var bin = readInt(uncba, p);
                    var nchnk = readInt(uncba, p+4);
                    p += 8 + (nchnk * 16);
                }
                var nintv = readInt(uncba, p); p += 4;
                var firstVO = readVirtualOffset(uncba,p);
                if( ! this.minAlignmentVO || this.minAlignmentVO.cmp( firstVO ) < 0 )
                    this.minAlignmentVO = firstVO;
                //console.log( ref, ''+firstVO );
                p += (nintv * 8);
                if (nbin > 0) {
                    this.indices[ref] = new Uint8Array(header, blockStart, p - blockStart);
                }
            }

            successCallback( this.indices, this.minAlignmentVO );
        }));
    },

    _readBAMheader: function( successCallback, failCallback ) {
        this.data.read( 0, this.minAlignmentVO.block, dojo.hitch( this, function(r) {
//        this.data.read(0, (1<<16)-1, dojo.hitch( this, function(r) {
            var unc = BAMUtil.unbgzf(r);
            var uncba = new Uint8Array(unc);

            if( readInt(uncba, 0) != BAM_MAGIC) {
                dlog('Not a BAM file');
                failCallback();
                return;
            }

            var headLen = readInt(uncba, 4);
            // var header = '';
            // for (var i = 0; i < headLen; ++i) {
            //     header += String.fromCharCode(uncba[i + 8]);
            // }

            var nRef = readInt(uncba, headLen + 8);
            var p = headLen + 12;

            this.chrToIndex = {};
            this.indexToChr = [];
            for (var i = 0; i < nRef; ++i) {
                var lName = readInt(uncba, p);
                var name = '';
                for (var j = 0; j < lName-1; ++j) {
                    name += String.fromCharCode(uncba[p + 4 + j]);
                }
                var lRef = readInt(uncba, p + lName + 4);
                // dlog(name + ': ' + lRef);
                this.chrToIndex[name] = i;
                if (name.indexOf('chr') == 0) {
                    this.chrToIndex[name.substring(3)] = i;
                } else {
                    this.chrToIndex['chr' + name] = i;
                }

                this.indexToChr.push(name);

                p = p + 8 + lName;
            }

            successCallback();
        }));
    },

    blocksForRange: function(refId, min, max) {
        var index = this.indices[refId];
        if (!index) {
            return [];
        }

        var intBinsL = reg2bins(min, max);
        var intBins = [];
        for (var i = 0; i < intBinsL.length; ++i) {
            intBins[intBinsL[i]] = true;
        }
        var leafChunks = [], otherChunks = [];

        var nbin = readInt(index, 0);
        var p = 4;
        for (var b = 0; b < nbin; ++b) {
            var bin = readInt(index, p);
            var nchnk = readInt(index, p+4);
    //        dlog('bin=' + bin + '; nchnk=' + nchnk);
            p += 8;
            if (intBins[bin]) {
                for (var c = 0; c < nchnk; ++c) {
                    var cs = readVirtualOffset(index, p);
                    var ce = readVirtualOffset(index, p + 8);
                    (bin < 4681 ? otherChunks : leafChunks).push(new Chunk(cs, ce));
                    p += 16;
                }
            } else {
                p +=  (nchnk * 16);
            }
        }
    //    dlog('leafChunks = ' + miniJSONify(leafChunks));
    //    dlog('otherChunks = ' + miniJSONify(otherChunks));

        var nintv = readInt(index, p);
        var lowest = null;
        var minLin = Math.min(min>>14, nintv - 1), maxLin = Math.min(max>>14, nintv - 1);
        for (var i = minLin; i <= maxLin; ++i) {
            var lb =  readVirtualOffset(index, p + 4 + (i * 8));
            if (!lb) {
                continue;
            }
            if (!lowest || lb.block < lowest.block || lb.offset < lowest.offset) {
                lowest = lb;
            }
        }
        // dlog('Lowest LB = ' + lowest);

        var prunedOtherChunks = [];
        if (lowest != null) {
            for (var i = 0; i < otherChunks.length; ++i) {
                var chnk = otherChunks[i];
                if (chnk.maxv.block >= lowest.block && chnk.maxv.offset >= lowest.offset) {
                    prunedOtherChunks.push(chnk);
                }
            }
        }
        // dlog('prunedOtherChunks = ' + miniJSONify(prunedOtherChunks));
        otherChunks = prunedOtherChunks;

        var intChunks = [];
        for (var i = 0; i < otherChunks.length; ++i) {
            intChunks.push(otherChunks[i]);
        }
        for (var i = 0; i < leafChunks.length; ++i) {
            intChunks.push(leafChunks[i]);
        }

        intChunks.sort(function(c0, c1) {
            var dif = c0.minv.block - c1.minv.block;
            if (dif != 0) {
                return dif;
            } else {
                return c0.minv.offset - c1.minv.offset;
            }
        });
        var mergedChunks = [];
        if (intChunks.length > 0) {
            var cur = intChunks[0];
            for (var i = 1; i < intChunks.length; ++i) {
                var nc = intChunks[i];
                if (nc.minv.block == cur.maxv.block /* && nc.minv.offset == cur.maxv.offset */) { // no point splitting mid-block
                    cur = new Chunk(cur.minv, nc.maxv);
                } else {
                    mergedChunks.push(cur);
                    cur = nc;
                }
            }
            mergedChunks.push(cur);
        }
    //    dlog('mergedChunks = ' + miniJSONify(mergedChunks));

        return mergedChunks;
    },

    fetch: function(chr, min, max, callback) {
        var chrId = this.chrToIndex[chr];
        var chunks;
        if (chrId === undefined) {
            chunks = [];
        } else {
            chunks = this.blocksForRange(chrId, min, max);
            if (!chunks) {
                callback(null, 'Error in index fetch');
            }
        }

        // toString function is used by the cache for making cache keys
        chunks.toString = function() {
            var str = '';
            array.forEach( this, function(c) {
                str += c.minv+'..'+c.maxv+',';
            });
            return str;
        };

        this.featureCache = this.featureCache || new LRUCache({
            name: 'bamFeatureCache',
            fillCallback: dojo.hitch(this, '_fetchChunkFeatures' ),
            sizeFunction: function( features ) {
                return features.length;
            },
            maxSize: 100000 // cache up to 100,000 BAM features
        });

        this.featureCache.get( chunks, function( features ) {
            features = array.filter( features, function( feature ) {
                return ( !( feature.get('end') < min || feature.get('start') > max )
                         && ( chrId === undefined || feature.get('_refID') == chrId ) );
            });
            callback( features );
        });

    },

    _fetchChunkFeatures: function( chunks, callback ) {
        var thisB = this;
        var features = [];
        var chunksProcessed = 0;

        if( ! chunks.length ) {
            callback([]);
            return;
        }

        array.forEach( chunks, function( c ) {
                var fetchMin = c.minv.block;
                var fetchMax = c.maxv.block + (1<<16); // *sigh*

                thisB.data.read(fetchMin, fetchMax - fetchMin + 1, function(r) {

                    var data = BAMUtil.unbgzf(r, c.maxv.block - c.minv.block + 1);

                    thisB.readBamFeatures( new Uint8Array(data), c.minv.offset, features, function() {
                        if( ++chunksProcessed == chunks.length )
                            callback( features );
                    });
                });
        });
    },

    readBamFeatures: function(ba, blockStart, sink, callback ) {
        var that = this;
        var featureCount = 0;

        var maxFeaturesWithoutYielding = 300;

        while ( true ) {
            if( blockStart >= ba.length ) {
                // if we're done, call the callback and return
                callback( sink );
                return;
            }
            else if( featureCount <= maxFeaturesWithoutYielding ) {
                // if we've read no more than 200 features this cycle, read another one
                var blockSize = readInt(ba, blockStart);
                var blockEnd = blockStart + blockSize;

                var feature = new BAMFeature({
                        store: this.store,
                        file: this,
                        bytes: { byteArray: ba, start: blockStart, end: blockEnd }
                });
                sink.push(feature);
                featureCount++;
                blockStart = blockEnd + 4;
            }
            else {
                // if we're not done but we've read a good chunk of
                // features, put the rest of our work into a timeout to continue
                // later, avoiding blocking any UI stuff that's going on
                window.setTimeout( function() {
                    that.readBamFeatures( ba, blockStart, sink, callback );
                }, 1);
                return;
            }
        }
    }
});

return BamFile;

});
<|endoftext|>


<commit_before>define( [
            'dojo/_base/declare',
            'dojo/_base/array',
            './Util',
            'JBrowse/Store/LRUCache'
        ],
        function( declare, array, BAMUtil, LRUCache ) {
//
// Binning (transliterated from SAM1.3 spec)
//

/* calculate bin given an alignment covering [beg,end) (zero-based, half-close-half-open) */
function reg2bin(beg, end)
{
    --end;
    if (beg>>14 == end>>14) return ((1<<15)-1)/7 + (beg>>14);
    if (beg>>17 == end>>17) return ((1<<12)-1)/7 + (beg>>17);
    if (beg>>20 == end>>20) return ((1<<9)-1)/7 + (beg>>20);
    if (beg>>23 == end>>23) return ((1<<6)-1)/7 + (beg>>23);
    if (beg>>26 == end>>26) return ((1<<3)-1)/7 + (beg>>26);
    return 0;
}

/* calculate the list of bins that may overlap with region [beg,end) (zero-based) */
var MAX_BIN = (((1<<18)-1)/7);
function reg2bins(beg, end)
{
    var i = 0, k, list = [];
    --end;
    list.push(0);
    for (k = 1 + (beg>>26); k <= 1 + (end>>26); ++k) list.push(k);
    for (k = 9 + (beg>>23); k <= 9 + (end>>23); ++k) list.push(k);
    for (k = 73 + (beg>>20); k <= 73 + (end>>20); ++k) list.push(k);
    for (k = 585 + (beg>>17); k <= 585 + (end>>17); ++k) list.push(k);
    for (k = 4681 + (beg>>14); k <= 4681 + (end>>14); ++k) list.push(k);
    return list;
}

var SEQRET_DECODER = ['=', 'A', 'C', 'x', 'G', 'x', 'x', 'x', 'T', 'x', 'x', 'x', 'x', 'x', 'x', 'N'];
var CIGAR_DECODER = ['M', 'I', 'D', 'N', 'S', 'H', 'P', '=', 'X', '?', '?', '?', '?', '?', '?', '?'];

var Vob = declare( null, {
    constructor: function(b, o) {
        this.block = b;
        this.offset = o;
    },
    toString: function() {
        return '' + this.block + ':' + this.offset;
    }
});

function readVob(ba, offset) {
    var block = ((ba[offset+6] & 0xff) * 0x100000000) + ((ba[offset+5] & 0xff) * 0x1000000) + ((ba[offset+4] & 0xff) * 0x10000) + ((ba[offset+3] & 0xff) * 0x100) + ((ba[offset+2] & 0xff));
    var bint = (ba[offset+1] << 8) | (ba[offset]);
    if (block == 0 && bint == 0) {
        return null;  // Should only happen in the linear index?
    } else {
        return new Vob(block, bint);
    }
}

var Chunk = declare( null, {
    constructor: function(minv, maxv) {
        this.minv = minv;
        this.maxv = maxv;
    }
});

var BamRecord = declare( null, {} );

var readInt = BAMUtil.readInt;
var readShort = BAMUtil.readInt;

var BamFile = declare( null,

/**
 * @lends JBrowse.Store.SeqFeature.BAM.File
 */
{

    /**
     * Low-level BAM file reading code.
     *
     * Adapted by Robert Buels from bam.js in the Dalliance Genome
     * Explorer which is copyright Thomas Down 2006-2010
     * @constructs
     */
    constructor: function() {},

    blocksForRange: function(refId, min, max) {
        var index = this.indices[refId];
        if (!index) {
            return [];
        }

        var intBinsL = reg2bins(min, max);
        var intBins = [];
        for (var i = 0; i < intBinsL.length; ++i) {
            intBins[intBinsL[i]] = true;
        }
        var leafChunks = [], otherChunks = [];

        var nbin = readInt(index, 0);
        var p = 4;
        for (var b = 0; b < nbin; ++b) {
            var bin = readInt(index, p);
            var nchnk = readInt(index, p+4);
    //        dlog('bin=' + bin + '; nchnk=' + nchnk);
            p += 8;
            if (intBins[bin]) {
                for (var c = 0; c < nchnk; ++c) {
                    var cs = readVob(index, p);
                    var ce = readVob(index, p + 8);
                    (bin < 4681 ? otherChunks : leafChunks).push(new Chunk(cs, ce));
                    p += 16;
                }
            } else {
                p +=  (nchnk * 16);
            }
        }
    //    dlog('leafChunks = ' + miniJSONify(leafChunks));
    //    dlog('otherChunks = ' + miniJSONify(otherChunks));

        var nintv = readInt(index, p);
        var lowest = null;
        var minLin = Math.min(min>>14, nintv - 1), maxLin = Math.min(max>>14, nintv - 1);
        for (var i = minLin; i <= maxLin; ++i) {
            var lb =  readVob(index, p + 4 + (i * 8));
            if (!lb) {
                continue;
            }
            if (!lowest || lb.block < lowest.block || lb.offset < lowest.offset) {
                lowest = lb;
            }
        }
        // dlog('Lowest LB = ' + lowest);

        var prunedOtherChunks = [];
        if (lowest != null) {
            for (var i = 0; i < otherChunks.length; ++i) {
                var chnk = otherChunks[i];
                if (chnk.maxv.block >= lowest.block && chnk.maxv.offset >= lowest.offset) {
                    prunedOtherChunks.push(chnk);
                }
            }
        }
        // dlog('prunedOtherChunks = ' + miniJSONify(prunedOtherChunks));
        otherChunks = prunedOtherChunks;

        var intChunks = [];
        for (var i = 0; i < otherChunks.length; ++i) {
            intChunks.push(otherChunks[i]);
        }
        for (var i = 0; i < leafChunks.length; ++i) {
            intChunks.push(leafChunks[i]);
        }

        intChunks.sort(function(c0, c1) {
            var dif = c0.minv.block - c1.minv.block;
            if (dif != 0) {
                return dif;
            } else {
                return c0.minv.offset - c1.minv.offset;
            }
        });
        var mergedChunks = [];
        if (intChunks.length > 0) {
            var cur = intChunks[0];
            for (var i = 1; i < intChunks.length; ++i) {
                var nc = intChunks[i];
                if (nc.minv.block == cur.maxv.block /* && nc.minv.offset == cur.maxv.offset */) { // no point splitting mid-block
                    cur = new Chunk(cur.minv, nc.maxv);
                } else {
                    mergedChunks.push(cur);
                    cur = nc;
                }
            }
            mergedChunks.push(cur);
        }
    //    dlog('mergedChunks = ' + miniJSONify(mergedChunks));

        return mergedChunks;
    },

    fetch: function(chr, min, max, callback) {
        var chrId = this.chrToIndex[chr];
        var chunks;
        if (chrId === undefined) {
            chunks = [];
        } else {
            chunks = this.blocksForRange(chrId, min, max);
            if (!chunks) {
                callback(null, 'Error in index fetch');
            }
        }

        // toString function is used by the cache for making cache keys
        chunks.toString = function() {
            var str = '';
            array.forEach( this, function(c) {
                str += c.minv+'..'+c.maxv+',';
            });
            return str;
        };

        this.recordCache = this.recordCache || new LRUCache({
            name: 'bamRecordCache',
            fillCallback: dojo.hitch(this, '_fetchChunkRecords' ),
            maxSize: 5000000 // cache up to 5 quasi-MB of BAM records
        });

        this.recordCache.get( chunks, function( records ) {
            records = array.filter( records, function( record ) {
                return (!min || record.pos <= max && record.pos + record.lseq >= min)
                    && (chrId === undefined || record._refID == chrId);
            });
            callback( records );
        });

    },

    _fetchChunkRecords: function( chunks, callback ) {
        var thisB = this;
        var records = [];
        var index = 0;
        var data;

        function tramp() {
            if (index >= chunks.length) {
                return callback(records);
            } else if (!data) {
                // dlog('fetching ' + index);
                var c = chunks[index];
                var fetchMin = c.minv.block;
                var fetchMax = c.maxv.block + (1<<16); // *sigh*
                thisB.data.slice(fetchMin, fetchMax - fetchMin)
                    .fetch(function(r) {
                               data = BAMUtil.unbgzf(r, c.maxv.block - c.minv.block + 1);
                               return tramp();
                           });
                return null;
            } else {
                var ba = new Uint8Array(data);
                thisB.readBamRecords(ba, chunks[index].minv.offset, records );
                data = null;
                ++index;
                return tramp();
            }
        };
        tramp();
    },

    readBamRecords: function(ba, offset, sink ) {
        while (true) {
            var blockSize = readInt(ba, offset);
            var blockEnd = offset + blockSize + 4;
            if (blockEnd >= ba.length) {
                return sink;
            }

            var record = new BamRecord();

            var refID = readInt(ba, offset + 4);
            var pos = readInt(ba, offset + 8);

            var bmn = readInt(ba, offset + 12);
            var bin = (bmn & 0xffff0000) >> 16;
            var mq = (bmn & 0xff00) >> 8;
            var nl = bmn & 0xff;

            var flag_nc = readInt(ba, offset + 16);
            this._decodeFlags( record, (flag_nc & 0xffff0000) >> 16 );

            var numCigarOps = flag_nc & 0xffff;

            var lseq = readInt(ba, offset + 20);
            record.lseq = lseq;

            var nextRef  = readInt(ba, offset + 24);
            var nextPos = readInt(ba, offset + 28);

            var tlen = readInt(ba, offset + 32);

            var readName = '';
            for (var j = 0; j < nl-1; ++j) {
                readName += String.fromCharCode(ba[offset + 36 + j]);
            }

            var p = offset + 36 + nl;

            var cigar = '';
            for (var c = 0; c < numCigarOps; ++c) {
                var cigop = readInt(ba, p);
                cigar = cigar + (cigop>>4) + CIGAR_DECODER[cigop & 0xf];
                p += 4;
            }
            record.cigar = cigar;

            var seq = '';
            var seqBytes = (lseq + 1) >> 1;
            for (var j = 0; j < seqBytes; ++j) {
                var sb = ba[p + j];
                seq += SEQRET_DECODER[(sb & 0xf0) >> 4];
                seq += SEQRET_DECODER[(sb & 0x0f)];
            }
            p += seqBytes;
            record.seq = seq;

            var qseq = '';
            for (var j = 0; j < lseq; ++j) {
                qseq += String.fromCharCode(ba[p + j]);
            }
            p += lseq;
            record.quals = qseq;

            record.pos = pos;
            record.MQ = mq;
            record.readName = readName;
            record.segment = this.indexToChr[refID];
            record._refID = refID;

            while (p < blockEnd) {
                var tag = String.fromCharCode(ba[p]) + String.fromCharCode(ba[p + 1]);
                var type = String.fromCharCode(ba[p + 2]);
                var value;

                if (type == 'A') {
                    value = String.fromCharCode(ba[p + 3]);
                    p += 4;
                } else if (type == 'i' || type == 'I') {
                    value = readInt(ba, p + 3);
                    p += 7;
                } else if (type == 'c' || type == 'C') {
                    value = ba[p + 3];
                    p += 4;
                } else if (type == 's' || type == 'S') {
                    value = readShort(ba, p + 3);
                    p += 5;
                } else if (type == 'f') {
                    throw 'FIXME need floats';
                } else if (type == 'Z') {
                    p += 3;
                    value = '';
                    for (;;) {
                        var cc = ba[p++];
                        if (cc == 0) {
                            break;
                        } else {
                            value += String.fromCharCode(cc);
                        }
                    }
                } else {
                    throw 'Unknown type '+ type;
                }
                record[tag] = value;
            }

            sink.push(record);

            offset = blockEnd;
        }
        // Exits via top of loop.
    },

    /**
     * Decode the BAM flags field and set them in the record.
     */
    _decodeFlags: function( record, flags ) {
        // the following explanations are taken verbatim from the SAM/BAM spec

        // 0x1 template having multiple segments in sequencing
        // If 0x1 is unset, no assumptions can be made about 0x2, 0x8, 0x20,
        // 0x40 and 0x80.
        if( flags & 0x1 ) {
            record.multi_segment_template = true;
            // 0x2 each segment properly aligned according to the aligner
            // 0x8 next segment in the template unmapped
            // 0x20 SEQ of the next segment in the template being reversed

            // 0x40 the first segment in the template
            // 0x80 the last segment in the template
            // * If 0x40 and 0x80 are both set, the segment is part of a linear
            // template, but it is neither the first nor the last segment. If both
            // 0x40 and 0x80 are unset, the index of the segment in the template is
            // unknown. This may happen for a non-linear template or the index is
            // lost in data processing.
        }

        // 0x4 segment unmapped
        // * Bit 0x4 is the only reliable place to tell whether the segment is
        // unmapped. If 0x4 is set, no assumptions can be made about RNAME, POS,
        // CIGAR, MAPQ, bits 0x2, 0x10 and 0x100 and the bit 0x20 of the next
        // segment in the template.
        record.unmapped = !!(flags & 0x4);
        // 0x10 SEQ being reverse complemented
        record.revcom = !!(flags & 0x10);

        // 0x100 secondary alignment
        // * Bit 0x100 marks the alignment not to be used in certain analyses
        // when the tools in use are aware of this bit.
        record.secondary_alignment = !!(flags & 0x100);

        // 0x200 not passing quality controls
        record.qc_failed = !!(flags & 0x200 );
        // 0x400 PCR or optical duplicate
        record.duplicate = !!(flags & 0x400 );
    }
});

return BamFile;

});
<commit_msg>rename the `revcom` BAM record field to `seq_reverse_complemented`<commit_after>define( [
            'dojo/_base/declare',
            'dojo/_base/array',
            './Util',
            'JBrowse/Store/LRUCache'
        ],
        function( declare, array, BAMUtil, LRUCache ) {
//
// Binning (transliterated from SAM1.3 spec)
//

/* calculate bin given an alignment covering [beg,end) (zero-based, half-close-half-open) */
function reg2bin(beg, end)
{
    --end;
    if (beg>>14 == end>>14) return ((1<<15)-1)/7 + (beg>>14);
    if (beg>>17 == end>>17) return ((1<<12)-1)/7 + (beg>>17);
    if (beg>>20 == end>>20) return ((1<<9)-1)/7 + (beg>>20);
    if (beg>>23 == end>>23) return ((1<<6)-1)/7 + (beg>>23);
    if (beg>>26 == end>>26) return ((1<<3)-1)/7 + (beg>>26);
    return 0;
}

/* calculate the list of bins that may overlap with region [beg,end) (zero-based) */
var MAX_BIN = (((1<<18)-1)/7);
function reg2bins(beg, end)
{
    var i = 0, k, list = [];
    --end;
    list.push(0);
    for (k = 1 + (beg>>26); k <= 1 + (end>>26); ++k) list.push(k);
    for (k = 9 + (beg>>23); k <= 9 + (end>>23); ++k) list.push(k);
    for (k = 73 + (beg>>20); k <= 73 + (end>>20); ++k) list.push(k);
    for (k = 585 + (beg>>17); k <= 585 + (end>>17); ++k) list.push(k);
    for (k = 4681 + (beg>>14); k <= 4681 + (end>>14); ++k) list.push(k);
    return list;
}

var SEQRET_DECODER = ['=', 'A', 'C', 'x', 'G', 'x', 'x', 'x', 'T', 'x', 'x', 'x', 'x', 'x', 'x', 'N'];
var CIGAR_DECODER = ['M', 'I', 'D', 'N', 'S', 'H', 'P', '=', 'X', '?', '?', '?', '?', '?', '?', '?'];

var Vob = declare( null, {
    constructor: function(b, o) {
        this.block = b;
        this.offset = o;
    },
    toString: function() {
        return '' + this.block + ':' + this.offset;
    }
});

function readVob(ba, offset) {
    var block = ((ba[offset+6] & 0xff) * 0x100000000) + ((ba[offset+5] & 0xff) * 0x1000000) + ((ba[offset+4] & 0xff) * 0x10000) + ((ba[offset+3] & 0xff) * 0x100) + ((ba[offset+2] & 0xff));
    var bint = (ba[offset+1] << 8) | (ba[offset]);
    if (block == 0 && bint == 0) {
        return null;  // Should only happen in the linear index?
    } else {
        return new Vob(block, bint);
    }
}

var Chunk = declare( null, {
    constructor: function(minv, maxv) {
        this.minv = minv;
        this.maxv = maxv;
    }
});

var BamRecord = declare( null, {} );

var readInt = BAMUtil.readInt;
var readShort = BAMUtil.readInt;

var BamFile = declare( null,

/**
 * @lends JBrowse.Store.SeqFeature.BAM.File
 */
{

    /**
     * Low-level BAM file reading code.
     *
     * Adapted by Robert Buels from bam.js in the Dalliance Genome
     * Explorer which is copyright Thomas Down 2006-2010
     * @constructs
     */
    constructor: function() {},

    blocksForRange: function(refId, min, max) {
        var index = this.indices[refId];
        if (!index) {
            return [];
        }

        var intBinsL = reg2bins(min, max);
        var intBins = [];
        for (var i = 0; i < intBinsL.length; ++i) {
            intBins[intBinsL[i]] = true;
        }
        var leafChunks = [], otherChunks = [];

        var nbin = readInt(index, 0);
        var p = 4;
        for (var b = 0; b < nbin; ++b) {
            var bin = readInt(index, p);
            var nchnk = readInt(index, p+4);
    //        dlog('bin=' + bin + '; nchnk=' + nchnk);
            p += 8;
            if (intBins[bin]) {
                for (var c = 0; c < nchnk; ++c) {
                    var cs = readVob(index, p);
                    var ce = readVob(index, p + 8);
                    (bin < 4681 ? otherChunks : leafChunks).push(new Chunk(cs, ce));
                    p += 16;
                }
            } else {
                p +=  (nchnk * 16);
            }
        }
    //    dlog('leafChunks = ' + miniJSONify(leafChunks));
    //    dlog('otherChunks = ' + miniJSONify(otherChunks));

        var nintv = readInt(index, p);
        var lowest = null;
        var minLin = Math.min(min>>14, nintv - 1), maxLin = Math.min(max>>14, nintv - 1);
        for (var i = minLin; i <= maxLin; ++i) {
            var lb =  readVob(index, p + 4 + (i * 8));
            if (!lb) {
                continue;
            }
            if (!lowest || lb.block < lowest.block || lb.offset < lowest.offset) {
                lowest = lb;
            }
        }
        // dlog('Lowest LB = ' + lowest);

        var prunedOtherChunks = [];
        if (lowest != null) {
            for (var i = 0; i < otherChunks.length; ++i) {
                var chnk = otherChunks[i];
                if (chnk.maxv.block >= lowest.block && chnk.maxv.offset >= lowest.offset) {
                    prunedOtherChunks.push(chnk);
                }
            }
        }
        // dlog('prunedOtherChunks = ' + miniJSONify(prunedOtherChunks));
        otherChunks = prunedOtherChunks;

        var intChunks = [];
        for (var i = 0; i < otherChunks.length; ++i) {
            intChunks.push(otherChunks[i]);
        }
        for (var i = 0; i < leafChunks.length; ++i) {
            intChunks.push(leafChunks[i]);
        }

        intChunks.sort(function(c0, c1) {
            var dif = c0.minv.block - c1.minv.block;
            if (dif != 0) {
                return dif;
            } else {
                return c0.minv.offset - c1.minv.offset;
            }
        });
        var mergedChunks = [];
        if (intChunks.length > 0) {
            var cur = intChunks[0];
            for (var i = 1; i < intChunks.length; ++i) {
                var nc = intChunks[i];
                if (nc.minv.block == cur.maxv.block /* && nc.minv.offset == cur.maxv.offset */) { // no point splitting mid-block
                    cur = new Chunk(cur.minv, nc.maxv);
                } else {
                    mergedChunks.push(cur);
                    cur = nc;
                }
            }
            mergedChunks.push(cur);
        }
    //    dlog('mergedChunks = ' + miniJSONify(mergedChunks));

        return mergedChunks;
    },

    fetch: function(chr, min, max, callback) {
        var chrId = this.chrToIndex[chr];
        var chunks;
        if (chrId === undefined) {
            chunks = [];
        } else {
            chunks = this.blocksForRange(chrId, min, max);
            if (!chunks) {
                callback(null, 'Error in index fetch');
            }
        }

        // toString function is used by the cache for making cache keys
        chunks.toString = function() {
            var str = '';
            array.forEach( this, function(c) {
                str += c.minv+'..'+c.maxv+',';
            });
            return str;
        };

        this.recordCache = this.recordCache || new LRUCache({
            name: 'bamRecordCache',
            fillCallback: dojo.hitch(this, '_fetchChunkRecords' ),
            maxSize: 5000000 // cache up to 5 quasi-MB of BAM records
        });

        this.recordCache.get( chunks, function( records ) {
            records = array.filter( records, function( record ) {
                return (!min || record.pos <= max && record.pos + record.lseq >= min)
                    && (chrId === undefined || record._refID == chrId);
            });
            callback( records );
        });

    },

    _fetchChunkRecords: function( chunks, callback ) {
        var thisB = this;
        var records = [];
        var index = 0;
        var data;

        function tramp() {
            if (index >= chunks.length) {
                return callback(records);
            } else if (!data) {
                // dlog('fetching ' + index);
                var c = chunks[index];
                var fetchMin = c.minv.block;
                var fetchMax = c.maxv.block + (1<<16); // *sigh*
                thisB.data.slice(fetchMin, fetchMax - fetchMin)
                    .fetch(function(r) {
                               data = BAMUtil.unbgzf(r, c.maxv.block - c.minv.block + 1);
                               return tramp();
                           });
                return null;
            } else {
                var ba = new Uint8Array(data);
                thisB.readBamRecords(ba, chunks[index].minv.offset, records );
                data = null;
                ++index;
                return tramp();
            }
        };
        tramp();
    },

    readBamRecords: function(ba, offset, sink ) {
        while (true) {
            var blockSize = readInt(ba, offset);
            var blockEnd = offset + blockSize + 4;
            if (blockEnd >= ba.length) {
                return sink;
            }

            var record = new BamRecord();

            var refID = readInt(ba, offset + 4);
            var pos = readInt(ba, offset + 8);

            var bmn = readInt(ba, offset + 12);
            var bin = (bmn & 0xffff0000) >> 16;
            var mq = (bmn & 0xff00) >> 8;
            var nl = bmn & 0xff;

            var flag_nc = readInt(ba, offset + 16);
            this._decodeFlags( record, (flag_nc & 0xffff0000) >> 16 );

            var numCigarOps = flag_nc & 0xffff;

            var lseq = readInt(ba, offset + 20);
            record.lseq = lseq;

            var nextRef  = readInt(ba, offset + 24);
            var nextPos = readInt(ba, offset + 28);

            var tlen = readInt(ba, offset + 32);

            var readName = '';
            for (var j = 0; j < nl-1; ++j) {
                readName += String.fromCharCode(ba[offset + 36 + j]);
            }

            var p = offset + 36 + nl;

            var cigar = '';
            for (var c = 0; c < numCigarOps; ++c) {
                var cigop = readInt(ba, p);
                cigar = cigar + (cigop>>4) + CIGAR_DECODER[cigop & 0xf];
                p += 4;
            }
            record.cigar = cigar;

            var seq = '';
            var seqBytes = (lseq + 1) >> 1;
            for (var j = 0; j < seqBytes; ++j) {
                var sb = ba[p + j];
                seq += SEQRET_DECODER[(sb & 0xf0) >> 4];
                seq += SEQRET_DECODER[(sb & 0x0f)];
            }
            p += seqBytes;
            record.seq = seq;

            var qseq = '';
            for (var j = 0; j < lseq; ++j) {
                qseq += String.fromCharCode(ba[p + j]);
            }
            p += lseq;
            record.quals = qseq;

            record.pos = pos;
            record.MQ = mq;
            record.readName = readName;
            record.segment = this.indexToChr[refID];
            record._refID = refID;

            while (p < blockEnd) {
                var tag = String.fromCharCode(ba[p]) + String.fromCharCode(ba[p + 1]);
                var type = String.fromCharCode(ba[p + 2]);
                var value;

                if (type == 'A') {
                    value = String.fromCharCode(ba[p + 3]);
                    p += 4;
                } else if (type == 'i' || type == 'I') {
                    value = readInt(ba, p + 3);
                    p += 7;
                } else if (type == 'c' || type == 'C') {
                    value = ba[p + 3];
                    p += 4;
                } else if (type == 's' || type == 'S') {
                    value = readShort(ba, p + 3);
                    p += 5;
                } else if (type == 'f') {
                    throw 'FIXME need floats';
                } else if (type == 'Z') {
                    p += 3;
                    value = '';
                    for (;;) {
                        var cc = ba[p++];
                        if (cc == 0) {
                            break;
                        } else {
                            value += String.fromCharCode(cc);
                        }
                    }
                } else {
                    throw 'Unknown type '+ type;
                }
                record[tag] = value;
            }

            sink.push(record);

            offset = blockEnd;
        }
        // Exits via top of loop.
    },

    /**
     * Decode the BAM flags field and set them in the record.
     */
    _decodeFlags: function( record, flags ) {
        // the following explanations are taken verbatim from the SAM/BAM spec

        // 0x1 template having multiple segments in sequencing
        // If 0x1 is unset, no assumptions can be made about 0x2, 0x8, 0x20,
        // 0x40 and 0x80.
        if( flags & 0x1 ) {
            record.multi_segment_template = true;
            // 0x2 each segment properly aligned according to the aligner
            // 0x8 next segment in the template unmapped
            // 0x20 SEQ of the next segment in the template being reversed

            // 0x40 the first segment in the template
            // 0x80 the last segment in the template
            // * If 0x40 and 0x80 are both set, the segment is part of a linear
            // template, but it is neither the first nor the last segment. If both
            // 0x40 and 0x80 are unset, the index of the segment in the template is
            // unknown. This may happen for a non-linear template or the index is
            // lost in data processing.
        }

        // 0x4 segment unmapped
        // * Bit 0x4 is the only reliable place to tell whether the segment is
        // unmapped. If 0x4 is set, no assumptions can be made about RNAME, POS,
        // CIGAR, MAPQ, bits 0x2, 0x10 and 0x100 and the bit 0x20 of the next
        // segment in the template.
        record.unmapped = !!(flags & 0x4);
        // 0x10 SEQ being reverse complemented
        record.seq_reverse_complemented = !!(flags & 0x10);

        // 0x100 secondary alignment
        // * Bit 0x100 marks the alignment not to be used in certain analyses
        // when the tools in use are aware of this bit.
        record.secondary_alignment = !!(flags & 0x100);

        // 0x200 not passing quality controls
        record.qc_failed = !!(flags & 0x200 );
        // 0x400 PCR or optical duplicate
        record.duplicate = !!(flags & 0x400 );
    }
});

return BamFile;

});
<|endoftext|>


<commit_before>(function() {
  'use strict';
  var module = angular.module('fim.base');
  module.factory('AllGoodsProvider', function(nxt, $q, IndexedEntityProvider) {

    function AllGoodsProvider(api, $scope, pageSize, account) {
      this.init(api, $scope, pageSize, account);
      this.filter = null;
      this.account = account;
    }
    angular.extend(AllGoodsProvider.prototype, IndexedEntityProvider.prototype, {

      uniqueKey: function(good) {
        return good.goods;
      },
      sortFunction: function(a, b) {
        return a.index - b.index;
      },

      getData: function(firstIndex) {
        var deferred = $q.defer();
        var args = {
          firstIndex: firstIndex,
          lastIndex: firstIndex + this.pageSize,
          includeCounts: true,
          requestType: 'getDGSGoods',
          seller: this.account
        }
        if (this.filter) {
          args.query = this.filter;
          args.requestType = 'searchDGSGoods';
          if (!/\*$/.test(args.query)) {
            args.query += '*';
          }
        }
        this.api.engine.socket().callAPIFunction(args).then(deferred.resolve, deferred.reject);
        return deferred.promise;
      },

      dataIterator: function(data) {
        var goods = data.goods;
        var index = index;
        for (var i = 0; i < goods.length; i++) {
          var a = goods[i];
        }
        return new Iterator(goods);
      }
    });
    return AllGoodsProvider;
  });
})();<commit_msg>a.index value changed<commit_after>(function() {
  'use strict';
  var module = angular.module('fim.base');
  module.factory('AllGoodsProvider', function(nxt, $q, IndexedEntityProvider) {

    function AllGoodsProvider(api, $scope, pageSize, account) {
      this.init(api, $scope, pageSize, account);
      this.filter = null;
      this.account = account;
    }
    angular.extend(AllGoodsProvider.prototype, IndexedEntityProvider.prototype, {

      uniqueKey: function(good) {
        return good.goods;
      },
      sortFunction: function(a, b) {
        return a.index - b.index;
      },

      getData: function(firstIndex) {
        var deferred = $q.defer();
        var args = {
          firstIndex: firstIndex,
          lastIndex: firstIndex + this.pageSize,
          includeCounts: true,
          requestType: 'getDGSGoods',
          seller: this.account
        }
        if (this.filter) {
          args.query = this.filter;
          args.requestType = 'searchDGSGoods';
          if (!/\*$/.test(args.query)) {
            args.query += '*';
          }
        }
        this.api.engine.socket().callAPIFunction(args).then(deferred.resolve, deferred.reject);
        return deferred.promise;
      },

      dataIterator: function(data) {
        var goods = data.goods;
        var index = this.entities.length > 0 ? this.entities[this.entities.length - 1].index : 0;
        for (var i = 0; i < goods.length; i++) {
          var a = goods[i];
          a.index = index;
        }
        return new Iterator(goods);
      }
    });
    return AllGoodsProvider;
  });
})();<|endoftext|>


<commit_before>'use strict';

const _ = require('lodash');
const async = require('async');
const child_process = require('child_process');
const fs = require('fs-extra');
const config = require( 'pelias-config' ).generate(require('../schema'));
const logger = require('pelias-logger').get('download');

if (require.main === module) {
  download((err) => {
    if (err) {
      logger.error('Failed to download data', err);
      process.exit(1);
    }
    logger.info('All done!');
  });
}

function download(callback) {
  let sources;

  // if no download sources are specified, default to the planet file
  if (_.isEmpty(config.imports.openstreetmap.download)) {
    sources = [
      'http://planet.openstreetmap.org/pbf/planet-latest.osm.pbf'
    ];
  }
  else {
    sources = _.map(config.imports.openstreetmap.download, (source) => source.sourceURL);
  }

  logger.info(`Downloading sources: ${sources}`);

  fs.ensureDir(config.imports.openstreetmap.datapath, (err) => {
    if (err) {
      logger.error(`error making directory ${config.imports.openstreetmap.datapath}`, err);
      return callback(err);
    }

    async.forEach(
      sources,
      (source, next) => {
        downloadSource(config, source, next);
      },
      callback
    );
  });
}

function downloadSource(config, sourceUrl, callback) {
  const targetDir = config.imports.openstreetmap.datapath;

  logger.debug(`downloading ${sourceUrl} to ${targetDir}`);
  child_process.exec(`cd ${targetDir} && { curl -L -X GET --silent --remote-name ${sourceUrl}; cd -; }`, callback);
}

module.exports = download;<commit_msg>remove unneccessary `cd`<commit_after>'use strict';

const _ = require('lodash');
const async = require('async');
const child_process = require('child_process');
const fs = require('fs-extra');
const config = require( 'pelias-config' ).generate(require('../schema'));
const logger = require('pelias-logger').get('download');

if (require.main === module) {
  download((err) => {
    if (err) {
      logger.error('Failed to download data', err);
      process.exit(1);
    }
    logger.info('All done!');
  });
}

function download(callback) {
  let sources;

  // if no download sources are specified, default to the planet file
  if (_.isEmpty(config.imports.openstreetmap.download)) {
    sources = [
      'http://planet.openstreetmap.org/pbf/planet-latest.osm.pbf'
    ];
  }
  else {
    sources = _.map(config.imports.openstreetmap.download, (source) => source.sourceURL);
  }

  logger.info(`Downloading sources: ${sources}`);

  fs.ensureDir(config.imports.openstreetmap.datapath, (err) => {
    if (err) {
      logger.error(`error making directory ${config.imports.openstreetmap.datapath}`, err);
      return callback(err);
    }

    async.forEach(
      sources,
      (source, next) => {
        downloadSource(config, source, next);
      },
      callback
    );
  });
}

function downloadSource(config, sourceUrl, callback) {
  const targetDir = config.imports.openstreetmap.datapath;

  logger.debug(`downloading ${sourceUrl} to ${targetDir}`);
  child_process.exec(`cd ${targetDir} && { curl -L -X GET --silent --remote-name ${sourceUrl}; }`, callback);
}

module.exports = download;<|endoftext|>


<commit_before>steal('can/util', function (can) {
	var isArray = can.isArray;
	/**
	 * @hide
	 * @page can.Object can.Object
	 * @parent can.util
	 *
	 * @body
	 * Object contains several helper methods that
	 * help compare objects.
	 *
	 * ## same
	 *
	 * Returns true if two objects are similar.
	 *
	 *     can.Object.same({foo: "bar"} , {bar: "foo"}) //-> false
	 *
	 * ## subset
	 *
	 * Returns true if an object is a set of another set.
	 *
	 *     can.Object.subset({}, {foo: "bar"} ) //-> true
	 *
	 * ## subsets
	 *
	 * Returns the subsets of an object
	 *
	 *     can.Object.subsets({userId: 20},
	 *                      [
	 *                       {userId: 20, limit: 30},
	 *                       {userId: 5},
	 *                       {}
	 *                      ])
	 *              //->    [{userId: 20, limit: 30}]
	 */
	can.Object = {};
	/**
	 * @function can.Object.same
	 * @parent can.util
	 * @description Checks if two objects are the same.
	 * @signature `can.Object.same(a, b, compares, aParent, bParent, deep)`
	 * @param {Object} a An object to compare against `b`.
	 * @param {Object} b An object to compare against `a`.
	 * @param {Object} [compares] An object that specifies how to compare properties.
	 * The keys of the `compares` object are names of properties in the objects to compare,
	 * and the values are functions that compare those properties. You can also pass `'i'`
	 * to compare values as case-insensitive strings, or `null` not to compare the properties
	 * at all.
	 * @return {{boolean}} Whether the two objects have the same properties and values.
	 *
	 * @body
	 * This function does not work with objects that create circular references.
	 *
	 * ## Examples
	 * @codestart
	 * can.Object.same({name: "Justin"}, {name: "JUSTIN"}) //-> false
	 *
	 * // ignore the name property
	 * can.Object.same({name: "Brian"}, {name: "JUSTIN"}, {name: null}) //-> true
	 *
	 * // ignore case
	 * can.Object.same({name: "Justin"}, {name: "JUSTIN"}, {name: "i"}) //-> true
	 *
	 * // deep rule
	 * can.Object.same({ person : { name: "Justin" } },
	 *     { person : { name: "JUSTIN" } },
	 *     { person : { name: "i"      } }) //-> true
	 *
	 * // supplied compare function
	 * can.Object.same({age: "Thirty"},
	 *     {age: 30},
	 *     {age: function( a, b ){
	 *     if( a == "Thirty" ) {
	 *         a = 30
	 *     }
	 *     if( b == "Thirty" ) {
	 *         b = 30
	 *     }
	 *     return a === b;
	 * }})      //-> true
	 * @codeend
	 */
	var same = can.Object.same = function (a, b, compares, aParent, bParent, deep) {
		var aType = typeof a,
			aArray = isArray(a),
			comparesType = typeof compares,
			compare;
		if (comparesType === 'string' || compares === null) {
			compares = compareMethods[compares];
			comparesType = 'function';
		}
		if (comparesType === 'function') {
			return compares(a, b, aParent, bParent);
		}
		compares = compares || {};
		if (a === null || b === null) {
			return a === b;
		}
		if (a instanceof Date || b instanceof Date) {
			return a === b;
		}
		if (deep === -1) {
			return aType === 'object' || a === b;
		}
		if (aType !== typeof b || aArray !== isArray(b)) {
			return false;
		}
		if (a === b) {
			return true;
		}
		if (aArray) {
			if (a.length !== b.length) {
				return false;
			}
			for (var i = 0; i < a.length; i++) {
				compare = compares[i] === undefined ? compares['*'] : compares[i];
				if (!same(a[i], b[i], a, b, compare)) {
					return false;
				}
			}
			return true;
		} else if (aType === 'object' || aType === 'function') {
			var bCopy = can.extend({}, b);
			for (var prop in a) {
				compare = compares[prop] === undefined ? compares['*'] : compares[prop];
				if (!same(a[prop], b[prop], compare, a, b, deep === false ? -1 : undefined)) {
					return false;
				}
				delete bCopy[prop];
			}
			// go through bCopy props ... if there is no compare .. return false
			for (prop in bCopy) {
				if (compares[prop] === undefined || !same(undefined, b[prop], compares[prop], a, b, deep === false ? -1 : undefined)) {
					return false;
				}
			}
			return true;
		}
		return false;
	};
	/**
	 * @function can.Object.subsets
	 * @parent can.util
	 * @description Returns the sets in 'sets' that are a subset of checkSet
	 * @codestart
	 * can.Object.subsets({userId: 20},
	 * [
	 *	{userId: 20, limit: 30},
	 *	{userId: 5},
	 *	{}
	 * ]) //-> [{userId: 20, limit: 30}]
	 * @codeend
	 * @signature `can.Object.subsets(checkSet, sets, compares)`
	 * @param {Object} checkSet
	 * @param {Object} sets
	 * @param {Object} compares
	 */
	can.Object.subsets = function (checkSet, sets, compares) {
		var len = sets.length,
			subsets = [];
		for (var i = 0; i < len; i++) {
			//check this subset
			var set = sets[i];
			if (can.Object.subset(checkSet, set, compares)) {
				subsets.push(set);
			}
		}
		return subsets;
	};
	/**
	 * @function can.Object.subset
	 * @parent can.util
	 * @description Compares if subset is a subset of set. Returns true if an object is a set of another set
	 * @codestart
	 * can.Object.subset({}, {foo: "bar"} ) //-> true
	 * @codeend
	 * @signature `can.Object.subset(subset, set, compares)`
	 * @param {Object} subset
	 * @param {Object} set
	 * @param {Object} compares
	 * @returns {Boolean} Whether or not subset is a subset of set
	 */
	can.Object.subset = function (subset, set, compares) {
		// go through set {type: 'folder'} and make sure every property
		// is in subset {type: 'folder', parentId :5}
		// then make sure that set has fewer properties
		// make sure we are only checking 'important' properties
		// in subset (ones that have to have a value)
		compares = compares || {};
		for (var prop in set) {
			if (!same(subset[prop], set[prop], compares[prop], subset, set)) {
				return false;
			}
		}
		return true;
	};
	var compareMethods = {
		'null': function () {
			return true;
		},
		i: function (a, b) {
			return ('' + a)
				.toLowerCase() === ('' + b)
				.toLowerCase();
		},
		eq: function(a, b) {
			return a === b;
		},
		similar: function(a, b) {
			/*jshint eqeqeq:false */
			return a == b;
		}
	};
	compareMethods.eqeq = compareMethods.similar;
	return can.Object;
});
<commit_msg>can.Object.subset<commit_after>steal('can/util', function (can) {
	var isArray = can.isArray;
	/**
	 * @hide
	 * @page can.Object can.Object
	 * @parent can.util
	 *
	 * @body
	 * Object contains several helper methods that
	 * help compare objects.
	 *
	 * ## same
	 *
	 * Returns true if two objects are similar.
	 *
	 *     can.Object.same({foo: "bar"} , {bar: "foo"}) //-> false
	 *
	 * ## subset
	 *
	 * Returns true if an object is a set of another set.
	 *
	 *     can.Object.subset({}, {foo: "bar"} ) //-> true
	 *
	 * ## subsets
	 *
	 * Returns the subsets of an object
	 *
	 *     can.Object.subsets({userId: 20},
	 *                      [
	 *                       {userId: 20, limit: 30},
	 *                       {userId: 5},
	 *                       {}
	 *                      ])
	 *              //->    [{userId: 20, limit: 30}]
	 */
	can.Object = {};
	/**
	 * @function can.Object.same
	 * @parent can.util
	 * @description Checks if two objects are the same.
	 * @signature `can.Object.same(a, b, compares, aParent, bParent, deep)`
	 * @param {Object} a An object to compare against `b`.
	 * @param {Object} b An object to compare against `a`.
	 * @param {Object} [compares] An object that specifies how to compare properties.
	 * The keys of the `compares` object are names of properties in the objects to compare,
	 * and the values are functions that compare those properties. You can also pass `'i'`
	 * to compare values as case-insensitive strings, or `null` not to compare the properties
	 * at all.
	 * @return {{boolean}} Whether the two objects have the same properties and values.
	 *
	 * @body
	 * This function does not work with objects that create circular references.
	 *
	 * ## Examples
	 * @codestart
	 * can.Object.same({name: "Justin"}, {name: "JUSTIN"}) //-> false
	 *
	 * // ignore the name property
	 * can.Object.same({name: "Brian"}, {name: "JUSTIN"}, {name: null}) //-> true
	 *
	 * // ignore case
	 * can.Object.same({name: "Justin"}, {name: "JUSTIN"}, {name: "i"}) //-> true
	 *
	 * // deep rule
	 * can.Object.same({ person : { name: "Justin" } },
	 *     { person : { name: "JUSTIN" } },
	 *     { person : { name: "i"      } }) //-> true
	 *
	 * // supplied compare function
	 * can.Object.same({age: "Thirty"},
	 *     {age: 30},
	 *     {age: function( a, b ){
	 *     if( a == "Thirty" ) {
	 *         a = 30
	 *     }
	 *     if( b == "Thirty" ) {
	 *         b = 30
	 *     }
	 *     return a === b;
	 * }})      //-> true
	 * @codeend
	 */
	var same = can.Object.same = function (a, b, compares, aParent, bParent, deep) {
		var aType = typeof a,
			aArray = isArray(a),
			comparesType = typeof compares,
			compare;
		if (comparesType === 'string' || compares === null) {
			compares = compareMethods[compares];
			comparesType = 'function';
		}
		if (comparesType === 'function') {
			return compares(a, b, aParent, bParent);
		}
		compares = compares || {};
		if (a === null || b === null) {
			return a === b;
		}
		if (a instanceof Date || b instanceof Date) {
			return a === b;
		}
		if (deep === -1) {
			return aType === 'object' || a === b;
		}
		if (aType !== typeof b || aArray !== isArray(b)) {
			return false;
		}
		if (a === b) {
			return true;
		}
		if (aArray) {
			if (a.length !== b.length) {
				return false;
			}
			for (var i = 0; i < a.length; i++) {
				compare = compares[i] === undefined ? compares['*'] : compares[i];
				if (!same(a[i], b[i], a, b, compare)) {
					return false;
				}
			}
			return true;
		} else if (aType === 'object' || aType === 'function') {
			var bCopy = can.extend({}, b);
			for (var prop in a) {
				compare = compares[prop] === undefined ? compares['*'] : compares[prop];
				if (!same(a[prop], b[prop], compare, a, b, deep === false ? -1 : undefined)) {
					return false;
				}
				delete bCopy[prop];
			}
			// go through bCopy props ... if there is no compare .. return false
			for (prop in bCopy) {
				if (compares[prop] === undefined || !same(undefined, b[prop], compares[prop], a, b, deep === false ? -1 : undefined)) {
					return false;
				}
			}
			return true;
		}
		return false;
	};
	/**
	 * @function can.Object.subsets
	 * @parent can.util
	 * @description Returns the sets in 'sets' that are a subset of checkSet
	 * @codestart
	 * can.Object.subsets({userId: 20},
	 * [
	 *	{userId: 20, limit: 30},
	 *	{userId: 5},
	 *	{}
	 * ]) //-> [{userId: 20, limit: 30}]
	 * @codeend
	 * @signature `can.Object.subsets(checkSet, sets, compares)`
	 * @param {Object} checkSet
	 * @param {Object} sets
	 * @param {Object} compares
	 */
	can.Object.subsets = function (checkSet, sets, compares) {
		var len = sets.length,
			subsets = [];
		for (var i = 0; i < len; i++) {
			//check this subset
			var set = sets[i];
			if (can.Object.subset(checkSet, set, compares)) {
				subsets.push(set);
			}
		}
		return subsets;
	};
	/**
	 * @function can.Object.subset
	 * @parent can.util
	 * @description Returns true if an Object is a subset of another Object
	 * @codestart
	 * can.Object.subset({}, {foo: "bar"} ) //-> true
	 * @codeend
	 * @signature `can.Object.subset(subset, set, compares)`
	 * @param {Object} subset
	 * @param {Object} set
	 * @param {Object} compares
	 * @returns {Boolean} Whether or not subset is a subset of set
	 */
	can.Object.subset = function (subset, set, compares) {
		// go through set {type: 'folder'} and make sure every property
		// is in subset {type: 'folder', parentId :5}
		// then make sure that set has fewer properties
		// make sure we are only checking 'important' properties
		// in subset (ones that have to have a value)
		compares = compares || {};
		for (var prop in set) {
			if (!same(subset[prop], set[prop], compares[prop], subset, set)) {
				return false;
			}
		}
		return true;
	};
	var compareMethods = {
		'null': function () {
			return true;
		},
		i: function (a, b) {
			return ('' + a)
				.toLowerCase() === ('' + b)
				.toLowerCase();
		},
		eq: function(a, b) {
			return a === b;
		},
		similar: function(a, b) {
			/*jshint eqeqeq:false */
			return a == b;
		}
	};
	compareMethods.eqeq = compareMethods.similar;
	return can.Object;
});
<|endoftext|>


<commit_before>/*!
 * Collection
 * https://github.com/kobezzza/Collection
 *
 * Released under the MIT license
 * https://github.com/kobezzza/Collection/blob/master/LICENSE
 */

describe('Collection methods with streams', () => {
	const
		{Transform} = require('stream');

	function generateStream(i, method = 'push') {
		const stream = new Transform({
			readableObjectMode: true,
			writableObjectMode: true,
			transform(chunk, enc, cb) {
				cb(null, chunk);
			}
		});

		let
			j = 0;

		(function loop() {
			setTimeout(() => {
				if (i--) {
					stream[method](j++);
					loop();

				} else {
					stream.end();
				}
			}, 50);
		})();

		return stream;
	}

	it('map from a stream', (done) => {
		const
			length = 5,
			dataHandler = jasmine.createSpy();

		$C(generateStream(length)).map((el) => el * 2)
			.addListener('data', dataHandler)
			.addListener('end', () => {
				expect(dataHandler).toHaveBeenCalledTimes(length);
				expect(dataHandler).toHaveBeenCalledWith(0);
				expect(dataHandler).toHaveBeenCalledWith(2);
				expect(dataHandler).toHaveBeenCalledWith(4);
				expect(dataHandler).toHaveBeenCalledWith(6);
				expect(dataHandler).toHaveBeenCalledWith(8);
				expect(dataHandler).not.toHaveBeenCalledWith(10);
				done();
			});
	});

	it('map from a write stream', (done) => {
		const
			length = 5,
			dataHandler = jasmine.createSpy();

		$C(generateStream(length, 'write')).map((el) => el * 2)
			.addListener('data', dataHandler)
			.addListener('end', () => {
				expect(dataHandler).toHaveBeenCalledTimes(length);
				expect(dataHandler).toHaveBeenCalledWith(0);
				expect(dataHandler).toHaveBeenCalledWith(2);
				expect(dataHandler).toHaveBeenCalledWith(4);
				expect(dataHandler).toHaveBeenCalledWith(6);
				expect(dataHandler).toHaveBeenCalledWith(8);
				expect(dataHandler).not.toHaveBeenCalledWith(10);
				done();
			});
	});

	it('map from a stream with parameters', (done) => {
		const
			dataHandler = jasmine.createSpy();

		$C(generateStream(9))
			.filter((el) => el % 2)
			.from(1)
			.count(2)
			.map()

			.addListener('data', dataHandler)
			.addListener('end', () => {
				expect(dataHandler).toHaveBeenCalledTimes(2);
				expect(dataHandler).toHaveBeenCalledWith(3);
				expect(dataHandler).toHaveBeenCalledWith(5);
				done();
			});
	});

	it('map from a stream with .one helper', (done) => {
		const
			dataHandler = jasmine.createSpy();

		$C(generateStream(5))
			.one
			.map()

			.addListener('data', dataHandler)
			.addListener('end', () => {
				expect(dataHandler).toHaveBeenCalledTimes(1);
				expect(dataHandler).toHaveBeenCalledWith(0);
				expect(dataHandler).not.toHaveBeenCalledWith(1);
				done();
			});
	});
});
<commit_msg>Added cases for .reduce<commit_after>/*!
 * Collection
 * https://github.com/kobezzza/Collection
 *
 * Released under the MIT license
 * https://github.com/kobezzza/Collection/blob/master/LICENSE
 */

describe('Collection methods with streams', () => {
	const
		{Transform} = require('stream');

	function generateStream(i, method = 'push') {
		const stream = new Transform({
			readableObjectMode: true,
			writableObjectMode: true,
			transform(chunk, enc, cb) {
				cb(null, chunk);
			}
		});

		let
			j = 0;

		(function loop() {
			setTimeout(() => {
				if (i--) {
					stream[method](j++);
					loop();

				} else {
					stream.end();
				}
			}, 50);
		})();

		return stream;
	}

	it('map from a stream', (done) => {
		const
			length = 5,
			dataHandler = jasmine.createSpy();

		$C(generateStream(length)).map((el) => el * 2)
			.addListener('data', dataHandler)
			.addListener('end', () => {
				expect(dataHandler).toHaveBeenCalledTimes(length);
				expect(dataHandler).toHaveBeenCalledWith(0);
				expect(dataHandler).toHaveBeenCalledWith(2);
				expect(dataHandler).toHaveBeenCalledWith(4);
				expect(dataHandler).toHaveBeenCalledWith(6);
				expect(dataHandler).toHaveBeenCalledWith(8);
				expect(dataHandler).not.toHaveBeenCalledWith(10);
				done();
			});
	});

	it('map from a write stream', (done) => {
		const
			length = 5,
			dataHandler = jasmine.createSpy();

		$C(generateStream(length, 'write')).map((el) => el * 2)
			.addListener('data', dataHandler)
			.addListener('end', () => {
				expect(dataHandler).toHaveBeenCalledTimes(length);
				expect(dataHandler).toHaveBeenCalledWith(0);
				expect(dataHandler).toHaveBeenCalledWith(2);
				expect(dataHandler).toHaveBeenCalledWith(4);
				expect(dataHandler).toHaveBeenCalledWith(6);
				expect(dataHandler).toHaveBeenCalledWith(8);
				expect(dataHandler).not.toHaveBeenCalledWith(10);
				done();
			});
	});

	it('map from a stream with parameters', (done) => {
		const
			dataHandler = jasmine.createSpy();

		$C(generateStream(9))
			.filter((el) => el % 2)
			.from(1)
			.count(2)
			.map()

			.addListener('data', dataHandler)
			.addListener('end', () => {
				expect(dataHandler).toHaveBeenCalledTimes(2);
				expect(dataHandler).toHaveBeenCalledWith(3);
				expect(dataHandler).toHaveBeenCalledWith(5);
				done();
			});
	});

	it('map from a stream with .one helper', (done) => {
		const
			dataHandler = jasmine.createSpy();

		$C(generateStream(5))
			.one
			.map()

			.addListener('data', dataHandler)
			.addListener('end', () => {
				expect(dataHandler).toHaveBeenCalledTimes(1);
				expect(dataHandler).toHaveBeenCalledWith(0);
				expect(dataHandler).not.toHaveBeenCalledWith(1);
				done();
			});
	});

	it('reduce from a stream to an array', async () => {
		expect(
			await $C(generateStream(3)).to([]).reduce((res, el) => {
				res.push(el * 2);
				return res;
			})
		).toEqual([0, 2, 4]);
	});

	it('reduce from a stream to a stream', (done) => {
		const
			length = 5,
			dataHandler = jasmine.createSpy();

		$C(generateStream(length)).toStream().reduce((res, el) => {
			res.write(el * 2);
			return res;
		})
			.addListener('data', dataHandler)
			.addListener('end', () => {
				expect(dataHandler).toHaveBeenCalledTimes(length);
				expect(dataHandler).toHaveBeenCalledWith(0);
				expect(dataHandler).toHaveBeenCalledWith(2);
				expect(dataHandler).toHaveBeenCalledWith(4);
				expect(dataHandler).toHaveBeenCalledWith(6);
				expect(dataHandler).toHaveBeenCalledWith(8);
				expect(dataHandler).not.toHaveBeenCalledWith(10);
				done();
			});
	});

	it('map from reduce from a stream to a stream', (done) => {
		const
			length = 5,
			dataHandler = jasmine.createSpy();

		$C(
			$C(generateStream(length)).toStream().reduce((res, el) => {
				res.write(el);
				return res;
			})
		)
			.map((el) => el * 2)

			.addListener('data', dataHandler)
			.addListener('end', () => {
				expect(dataHandler).toHaveBeenCalledTimes(length);
				expect(dataHandler).toHaveBeenCalledWith(0);
				expect(dataHandler).toHaveBeenCalledWith(2);
				expect(dataHandler).toHaveBeenCalledWith(4);
				expect(dataHandler).toHaveBeenCalledWith(6);
				expect(dataHandler).toHaveBeenCalledWith(8);
				expect(dataHandler).not.toHaveBeenCalledWith(10);
				done();
			});
	});

	it('reduce from a stream to an array with parameters', async () => {
		expect(
			await $C(generateStream(9))
				.filter((el) => el % 2)
				.from(1)
				.count(2)
				.to([])
				.reduce((res, el) => {
					res.push(el);
					return res;
				})
		).toEqual([3, 5]);
	});

	it('reduce from a stream to a stream with parameters', (done) => {
		const
			dataHandler = jasmine.createSpy();

		$C(generateStream(9))
			.filter((el) => el % 2)
			.from(1)
			.count(2)
			.toStream()
			.reduce((res, el) => {
				res.write(el);
				return res;
			})

			.addListener('data', dataHandler)
			.addListener('end', () => {
				expect(dataHandler).toHaveBeenCalledTimes(2);
				expect(dataHandler).toHaveBeenCalledWith(3);
				expect(dataHandler).toHaveBeenCalledWith(5);
				done();
			});
	});

	it('reduce from a stream with .one helper', (done) => {
		const
			dataHandler = jasmine.createSpy();

		$C(generateStream(5))
			.one
			.toStream()
			.reduce((res, el) => {
				res.write(el);
				return res;
			})

			.addListener('data', dataHandler)
			.addListener('end', () => {
				expect(dataHandler).toHaveBeenCalledTimes(1);
				expect(dataHandler).toHaveBeenCalledWith(0);
				expect(dataHandler).not.toHaveBeenCalledWith(1);
				done();
			});
	});
});
<|endoftext|>


<commit_before>/*globals require: false, describe: false, beforeEach: false, it: false, expect: false */
var nodemailer = require('nodemailer');
var smtpTransport = require('nodemailer-smtp-transport');

var SmtpServer = require(__dirname + '/../../lib/smtp');
var Storage = require(__dirname + '/../../lib/storage/memory');

describe('SmtpServer', function() {
  beforeEach(function(cb) {
    var self = this;
    self.storage = new Storage();
    self.smtpServer = new SmtpServer(0, null, self.storage);
    self.smtpServer.run(function() {
      // Create a SMTP transport object
      self.transport = nodemailer.createTransport(smtpTransport({
        port: self.smtpServer.server.server.address().port,
        tls:  {
             rejectUnauthorized: false
        },
        secure: false,
        host: 'localhost'
      }));
      cb();
    });
  });
  afterEach(function(cb) {
    this.smtpServer.close(cb);
  });

  it('Has Working Raw', function(cb) {
    var self = this;
    var message = {
      xMailer: 'xmailer-test-string',
      messageId: "my-random-message-id@example.com",
      // sender info
      from: 'Sender Name <sender@example.com>',
      // Comma separated list of recipients
      to: '"Receiver Name" <nodemailer@disposebox.com>',
      // Subject of the message
      subject: 'My Test Subject ✔', //
      headers: { 'X-Laziness-level': 1000 },
      // plaintext body
      text: 'Hello to myself!\nFrom myself',
    };

    self.transport.sendMail(message, function(err) {
      self.storage.get('_my-random-message-id_example_com_').then(function(message) {
        expect(message).not.toBeNull();
        var raw = message.raw.join('').split(/\n|\r/).filter(function(line) {
          return !line.match(/^Date:/)
        });
        expect(raw).toEqual([
          'Content-Type: text/plain',
          '',
          'From: Sender Name <sender@example.com>',
          '',
          'To: Receiver Name <nodemailer@disposebox.com>',
          '',
          'Subject: My Test Subject =?UTF-8?Q?=E2=9C=94?=',
          '',
          'Message-Id: <my-random-message-id@example.com>',
          '',
          'X-Laziness-Level: 1000',
          '',
          'X-Mailer: xmailer-test-string',
          '',
          'Content-Transfer-Encoding: 7bit',
          '',
          //'Date: Sat, 25 Apr 2015 02:32:30 +0000',
          '',
          'MIME-Version: 1.0',
          '',
          '',
          '',
          'Hello to myself!',
          '',
          'From myself'
        ]);
        cb();
      });
      self.transport.close();
    });

  });
  it('Has working text body', function(cb) {
    var self = this;
    var message = {
      xMailer: 'xmailer-test-string',
      messageId: "my-random-message-id@example.com",
      // sender info
      from: 'Sender Name <sender@example.com>',
      // Comma separated list of recipients
      to: '"Receiver Name" <nodemailer@disposebox.com>',
      // Subject of the message
      subject: 'My Test Subject ✔', //
      headers: { 'X-Laziness-level': 1000 },
      // plaintext body
      text: 'Hello to myself!\nFrom myself',
    };

    self.transport.sendMail(message, function(err) {
      self.transport.close();
      self.storage.get('_my-random-message-id_example_com_').then(function(message) {
        expect(message).not.toBeNull();
        expect(message.body.plain).toEqual('Hello to myself!\nFrom myself');
        cb();
      });
    });

  });
});

<commit_msg>eslint fixes<commit_after>/*globals require: false, describe: false, beforeEach: false, afterEach: false, it: false, expect: false */
var nodemailer = require('nodemailer');
var smtpTransport = require('nodemailer-smtp-transport');

var path = require('path');
var SmtpServer = require(path.join(__dirname, '/../../lib/smtp'));
var Storage = require(path.join(__dirname, '/../../lib/storage/memory'));

describe('SmtpServer', function() {
  beforeEach(function(cb) {
    var self = this;
    self.storage = new Storage();
    self.smtpServer = new SmtpServer(0, null, self.storage);
    self.smtpServer.run(function() {
      // Create a SMTP transport object
      self.transport = nodemailer.createTransport(smtpTransport({
        port: self.smtpServer.server.server.address().port,
        tls: { rejectUnauthorized: false },
        secure: false,
        host: 'localhost'
      }));
      cb();
    });
  });
  afterEach(function(cb) {
    this.smtpServer.close(cb);
  });

  it('Has Working Raw', function(cb) {
    var self = this;
    var message = {
      xMailer: 'xmailer-test-string',
      messageId: "my-random-message-id@example.com",
      // sender info
      from: 'Sender Name <sender@example.com>',
      // Comma separated list of recipients
      to: '"Receiver Name" <nodemailer@disposebox.com>',
      // Subject of the message
      subject: 'My Test Subject ✔', //
      headers: { 'X-Laziness-level': 1000 },
      // plaintext body
      text: 'Hello to myself!\nFrom myself',
    };

    self.transport.sendMail(message, function(err) {
      self.storage.get('_my-random-message-id_example_com_').then(function(message) {
        expect(message).not.toBeNull();
        var raw = message.raw.join('').split(/\n|\r/).filter(function(line) {
          return !line.match(/^Date:/);
        });
        expect(raw).toEqual([
          'Content-Type: text/plain',
          '',
          'From: Sender Name <sender@example.com>',
          '',
          'To: Receiver Name <nodemailer@disposebox.com>',
          '',
          'Subject: My Test Subject =?UTF-8?Q?=E2=9C=94?=',
          '',
          'Message-Id: <my-random-message-id@example.com>',
          '',
          'X-Laziness-Level: 1000',
          '',
          'X-Mailer: xmailer-test-string',
          '',
          'Content-Transfer-Encoding: 7bit',
          '',
          //'Date: Sat, 25 Apr 2015 02:32:30 +0000',
          '',
          'MIME-Version: 1.0',
          '',
          '',
          '',
          'Hello to myself!',
          '',
          'From myself'
        ]);
        cb();
      });
      self.transport.close();
    });

  });
  it('Has working text body', function(cb) {
    var self = this;
    var smtpMessage = {
      xMailer: 'xmailer-test-string',
      messageId: "my-random-message-id@example.com",
      // sender info
      from: 'Sender Name <sender@example.com>',
      // Comma separated list of recipients
      to: '"Receiver Name" <nodemailer@disposebox.com>',
      // Subject of the message
      subject: 'My Test Subject ✔', //
      headers: { 'X-Laziness-level': 1000 },
      // plaintext body
      text: 'Hello to myself!\nFrom myself'
    };

    self.transport.sendMail(smtpMessage, function(err) {
      if (err) { throw err; }
      self.transport.close();
      self.storage.get('_my-random-message-id_example_com_').then(function(message) {
        expect(message).not.toBeNull();
        expect(message.body.plain).toEqual('Hello to myself!\nFrom myself');
        cb();
      });
    });

  });
});

<|endoftext|>


<commit_before>describe('grunt-template-progeny', function () {
	var path, grunt, progeny, fixturePaths, configData;

	path = require('path');
	fixturePaths = {
		a: path.resolve('spec/fixtures/a.less'),
		b: path.resolve('spec/fixtures/b.less'),
		c: path.resolve('spec/fixtures/c.less')
	};

	beforeEach(function () {
		grunt  = require('grunt');
		progeny = require('..')(grunt);

		configData = grunt.config.data;
		configData.path = fixturePaths.a;
	});

	it('should add progeny helper to grunt.template.progeny', function () {
		var reference = grunt.template.progeny;
		expect(reference).toBeDefined();
		expect(reference).toBe(progeny);
	});

	it('should run progeny with LESS files correctly', function () {
		var result;

		expect(function () {
			result = progeny(fixturePaths.a);
		}).not.toThrow();

		expect(result).toEqual([
			fixturePaths.b,
			fixturePaths.c
		]);
	});

	it('should process path templates', function () {
		var result;

		expect(function () {
			result = progeny('<%= path %>');
		}).not.toThrow();

		expect(result).toEqual([
			fixturePaths.b,
			fixturePaths.c
		]);
	});

	it('should function correctly when executed from a grunt template', function () {
		var template = '<%= grunt.template.progeny(path) %>',
			result;

		expect(function () {
			result = grunt.template.process(template);
		}).not.toThrow();

		expect(result).toEqual([
			fixturePaths.b,
			fixturePaths.c
		].join(','));
	});
});
<commit_msg>Update the test spec<commit_after>describe('grunt-template-progeny', function () {
	var path, grunt, progeny, fixturePaths, configData;

	path = require('path');
	fixturePaths = {
		a: path.resolve('spec/fixtures/a.less'),
		b: path.resolve('spec/fixtures/b.less'),
		c: path.resolve('spec/fixtures/c.less')
	};

	beforeEach(function () {
		grunt  = require('grunt');
		progeny = require('..')(grunt);

		configData = grunt.config.data;
		configData.path = fixturePaths.a;
	});

	it('should add progeny helper to grunt.template.progeny', function () {
		var reference = grunt.template.progeny;
		expect(reference).toBeDefined();
		expect(reference).toBe(progeny);
	});

	it('should handle plain paths', function () {
		var expected = '{' + [fixturePaths.b, fixturePaths.c] + ',}',
			result;

		expect(function () {
			result = progeny(fixturePaths.b);
		}).not.toThrow();

		expect(result).toEqual(expected);
	});

	it('should process paths that are, or contain, grunt templates', function () {
		var expected = '{' + [fixturePaths.a, fixturePaths.b, fixturePaths.c] + ',}',
			result;

		expect(function () {
			result = progeny('<%= path %>');
		}).not.toThrow();

		expect(result).toEqual(expected);
	});

	it('should function correctly when executed from a grunt template', function () {
		var template = '<%= grunt.template.progeny(path) %>',
			expected = '{' + [fixturePaths.a, fixturePaths.b, fixturePaths.c] + ',}',
			result;

		expect(function () {
			result = grunt.template.process(template);
		}).not.toThrow();

		expect(result).toEqual(expected);
	});
});
<|endoftext|>


<commit_before>import React, { PropTypes } from 'react';
import Box from 'grommet/components/Box';
import Heading from 'grommet/components/Heading';
import defaultHero from '../../../images/bhamc-hero.jpg';

const PageMarquee = ({ image }) =>
  <Box
    full="horizontal"
    colorIndex="grey-1"
    texture={image || defaultHero}
    size={{ height: 'medium' }}
    align="center"
    justify="center"
  >
    <Heading
      tag="h1"
      strong
      margin="none"
    >
      Belle Haven Animal Medical Centre
    </Heading>
  </Box>;

PageMarquee.propTypes = {
  image: PropTypes.string,
};

export default PageMarquee;
<commit_msg>Add default props to PageMarquee.<commit_after>import React, { PropTypes } from 'react';
import Box from 'grommet/components/Box';
import Heading from 'grommet/components/Heading';
import defaultHero from '../../../images/bhamc-hero.jpg';

const PageMarquee = ({ image }) =>
  <Box
    full="horizontal"
    colorIndex="grey-1"
    texture={image}
    size={{ height: 'medium' }}
    align="center"
    justify="center"
  >
    <Heading
      tag="h1"
      strong
      margin="none"
    >
      Belle Haven Animal Medical Centre
    </Heading>
  </Box>;

PageMarquee.propTypes = {
  image: PropTypes.string,
};

PageMarquee.defaultProps = {
  image: defaultHero,
};

export default PageMarquee;
<|endoftext|>