<commit_before>var ArrayBufferStream = require('./ArrayBufferStream');
var Tone = require('tone');
var log = require('./log');

/*
 * Load wav audio files that have been compressed with the ADPCM format.
 * This is necessary because, while web browsers have native decoders for many audio
 * formats, ADPCM is a non-standard format used by Scratch since its early days.
 * This decoder is based on code from Scratch-Flash:
 * https://github.com/LLK/scratch-flash/blob/master/src/sound/WAVFile.as
 * @constructor
 */
function ADPCMSoundLoader () {
}

/**
 * Load an ADPCM sound file from a URL, decode it, and return a promise
 * with the audio buffer.
 * @param  {string} url - a url pointing to the ADPCM wav file
 * @return {Tone.Buffer}
 */
ADPCMSoundLoader.prototype.load = function (url) {

    return new Promise(function (resolve, reject) {

        var request = new XMLHttpRequest();
        request.open('GET', url, true);
        request.responseType = 'arraybuffer';

        request.onload = function () {
            var audioData = request.response;
            var stream = new ArrayBufferStream(audioData);

            var riffStr = stream.readUint8String(4);
            if (riffStr != 'RIFF') {
                log.warn('incorrect adpcm wav header');
                reject();
            }

            var lengthInHeader = stream.readInt32();
            if ((lengthInHeader + 8) != audioData.byteLength) {
                log.warn('adpcm wav length in header: ' + lengthInHeader + ' is incorrect');
            }

            var wavStr = stream.readUint8String(4);
            if (wavStr != 'WAVE') {
                log.warn('incorrect adpcm wav header');
                reject();
            }

            var formatChunk = this.extractChunk('fmt ', stream);
            this.encoding = formatChunk.readUint16();
            this.channels = formatChunk.readUint16();
            this.samplesPerSecond = formatChunk.readUint32();
            this.bytesPerSecond = formatChunk.readUint32();
            this.blockAlignment = formatChunk.readUint16();
            this.bitsPerSample = formatChunk.readUint16();
            formatChunk.position += 2;  // skip extra header byte count
            this.samplesPerBlock = formatChunk.readUint16();
            this.adpcmBlockSize = ((this.samplesPerBlock - 1) / 2) + 4; // block size in bytes

            var samples = this.imaDecompress(this.extractChunk('data', stream), this.adpcmBlockSize);

            // todo: this line is the only place Tone is used here, should be possible to remove
            var buffer = Tone.context.createBuffer(1, samples.length, this.samplesPerSecond);

            // todo: optimize this? e.g. replace the divide by storing 1/32768 and multiply?
            for (var i=0; i<samples.length; i++) {
                buffer.getChannelData(0)[i] = samples[i] / 32768;
            }

            resolve(buffer);

        }.bind(this);

        request.send();

    }.bind(this));
};

/**
 * Data used by the decompression algorithm
 * @type {Array}
 */
ADPCMSoundLoader.prototype.stepTable = [
    7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
    50, 55, 60, 66, 73, 80, 88, 97, 107, 118, 130, 143, 157, 173, 190, 209, 230,
    253, 279, 307, 337, 371, 408, 449, 494, 544, 598, 658, 724, 796, 876, 963,
    1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066, 2272, 2499, 2749, 3024, 3327,
    3660, 4026, 4428, 4871, 5358, 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487,
    12635, 13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767];

/**
 * Data used by the decompression algorithm
 * @type {Array}
 */
ADPCMSoundLoader.prototype.indexTable = [
    -1, -1, -1, -1, 2, 4, 6, 8,
    -1, -1, -1, -1, 2, 4, 6, 8];

/**
 * Extract a chunk of audio data from the stream, consisting of a set of audio data bytes
 * @param  {string} chunkType - the type of chunk to extract. 'data' or 'fmt' (format)
 * @param  {ArrayBufferStream} stream - an stream containing the audio data
 * @return {ArrayBufferStream} a stream containing the desired chunk
 */
ADPCMSoundLoader.prototype.extractChunk = function (chunkType, stream) {
    stream.position = 12;
    while (stream.position < (stream.getLength() - 8)) {
        var typeStr = stream.readUint8String(4);
        var chunkSize = stream.readInt32();
        if (typeStr == chunkType) {
            var chunk = stream.extract(chunkSize);
            return chunk;
        } else {
            stream.position += chunkSize;
        }
    }
};

/**
 * Decompress sample data using the IMA ADPCM algorithm.
 * Note: Handles only one channel, 4-bits per sample.
 * @param  {ArrayBufferStream} compressedData - a stream of compressed audio samples
 * @param  {number} blockSize - the number of bytes in the stream
 * @return {Int16Array} the uncompressed audio samples
 */
ADPCMSoundLoader.prototype.imaDecompress = function (compressedData, blockSize) {
    var sample, step, code, delta;
    var index = 0;
    var lastByte = -1; // -1 indicates that there is no saved lastByte
    var out = [];

    // Bail and return no samples if we have no data
    if (!compressedData) return out;

    compressedData.position = 0;
    var a = 0;
    while (a==0) {
        if (((compressedData.position % blockSize) == 0) && (lastByte < 0)) { // read block header
            if (compressedData.getBytesAvailable() == 0) break;
            sample = compressedData.readInt16();
            index = compressedData.readUint8();
            compressedData.position++; // skip extra header byte
            if (index > 88) index = 88;
            out.push(sample);
        } else {
            // read 4-bit code and compute delta from previous sample
            if (lastByte < 0) {
                if (compressedData.getBytesAvailable() == 0) break;
                lastByte = compressedData.readUint8();
                code = lastByte & 0xF;
            } else {
                code = (lastByte >> 4) & 0xF;
                lastByte = -1;
            }
            step = this.stepTable[index];
            delta = 0;
            if (code & 4) delta += step;
            if (code & 2) delta += step >> 1;
            if (code & 1) delta += step >> 2;
            delta += step >> 3;
            // compute next index
            index += this.indexTable[code];
            if (index > 88) index = 88;
            if (index < 0) index = 0;
            // compute and output sample
            sample += (code & 8) ? -delta : delta;
            if (sample > 32767) sample = 32767;
            if (sample < -32768) sample = -32768;
            out.push(sample);
        }
    }
    var samples = Int16Array.from(out);
    return samples;
};

module.exports = ADPCMSoundLoader;
<commit_msg>fix JSDoc comment<commit_after>var ArrayBufferStream = require('./ArrayBufferStream');
var Tone = require('tone');
var log = require('./log');

/**
 * Load wav audio files that have been compressed with the ADPCM format.
 * This is necessary because, while web browsers have native decoders for many audio
 * formats, ADPCM is a non-standard format used by Scratch since its early days.
 * This decoder is based on code from Scratch-Flash:
 * https://github.com/LLK/scratch-flash/blob/master/src/sound/WAVFile.as
 * @constructor
 */
function ADPCMSoundLoader () {
}

/**
 * Load an ADPCM sound file from a URL, decode it, and return a promise
 * with the audio buffer.
 * @param  {string} url - a url pointing to the ADPCM wav file
 * @return {Tone.Buffer}
 */
ADPCMSoundLoader.prototype.load = function (url) {

    return new Promise(function (resolve, reject) {

        var request = new XMLHttpRequest();
        request.open('GET', url, true);
        request.responseType = 'arraybuffer';

        request.onload = function () {
            var audioData = request.response;
            var stream = new ArrayBufferStream(audioData);

            var riffStr = stream.readUint8String(4);
            if (riffStr != 'RIFF') {
                log.warn('incorrect adpcm wav header');
                reject();
            }

            var lengthInHeader = stream.readInt32();
            if ((lengthInHeader + 8) != audioData.byteLength) {
                log.warn('adpcm wav length in header: ' + lengthInHeader + ' is incorrect');
            }

            var wavStr = stream.readUint8String(4);
            if (wavStr != 'WAVE') {
                log.warn('incorrect adpcm wav header');
                reject();
            }

            var formatChunk = this.extractChunk('fmt ', stream);
            this.encoding = formatChunk.readUint16();
            this.channels = formatChunk.readUint16();
            this.samplesPerSecond = formatChunk.readUint32();
            this.bytesPerSecond = formatChunk.readUint32();
            this.blockAlignment = formatChunk.readUint16();
            this.bitsPerSample = formatChunk.readUint16();
            formatChunk.position += 2;  // skip extra header byte count
            this.samplesPerBlock = formatChunk.readUint16();
            this.adpcmBlockSize = ((this.samplesPerBlock - 1) / 2) + 4; // block size in bytes

            var samples = this.imaDecompress(this.extractChunk('data', stream), this.adpcmBlockSize);

            // todo: this line is the only place Tone is used here, should be possible to remove
            var buffer = Tone.context.createBuffer(1, samples.length, this.samplesPerSecond);

            // todo: optimize this? e.g. replace the divide by storing 1/32768 and multiply?
            for (var i=0; i<samples.length; i++) {
                buffer.getChannelData(0)[i] = samples[i] / 32768;
            }

            resolve(buffer);

        }.bind(this);

        request.send();

    }.bind(this));
};

/**
 * Data used by the decompression algorithm
 * @type {Array}
 */
ADPCMSoundLoader.prototype.stepTable = [
    7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
    50, 55, 60, 66, 73, 80, 88, 97, 107, 118, 130, 143, 157, 173, 190, 209, 230,
    253, 279, 307, 337, 371, 408, 449, 494, 544, 598, 658, 724, 796, 876, 963,
    1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066, 2272, 2499, 2749, 3024, 3327,
    3660, 4026, 4428, 4871, 5358, 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487,
    12635, 13899, 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767];

/**
 * Data used by the decompression algorithm
 * @type {Array}
 */
ADPCMSoundLoader.prototype.indexTable = [
    -1, -1, -1, -1, 2, 4, 6, 8,
    -1, -1, -1, -1, 2, 4, 6, 8];

/**
 * Extract a chunk of audio data from the stream, consisting of a set of audio data bytes
 * @param  {string} chunkType - the type of chunk to extract. 'data' or 'fmt' (format)
 * @param  {ArrayBufferStream} stream - an stream containing the audio data
 * @return {ArrayBufferStream} a stream containing the desired chunk
 */
ADPCMSoundLoader.prototype.extractChunk = function (chunkType, stream) {
    stream.position = 12;
    while (stream.position < (stream.getLength() - 8)) {
        var typeStr = stream.readUint8String(4);
        var chunkSize = stream.readInt32();
        if (typeStr == chunkType) {
            var chunk = stream.extract(chunkSize);
            return chunk;
        } else {
            stream.position += chunkSize;
        }
    }
};

/**
 * Decompress sample data using the IMA ADPCM algorithm.
 * Note: Handles only one channel, 4-bits per sample.
 * @param  {ArrayBufferStream} compressedData - a stream of compressed audio samples
 * @param  {number} blockSize - the number of bytes in the stream
 * @return {Int16Array} the uncompressed audio samples
 */
ADPCMSoundLoader.prototype.imaDecompress = function (compressedData, blockSize) {
    var sample, step, code, delta;
    var index = 0;
    var lastByte = -1; // -1 indicates that there is no saved lastByte
    var out = [];

    // Bail and return no samples if we have no data
    if (!compressedData) return out;

    compressedData.position = 0;
    var a = 0;
    while (a==0) {
        if (((compressedData.position % blockSize) == 0) && (lastByte < 0)) { // read block header
            if (compressedData.getBytesAvailable() == 0) break;
            sample = compressedData.readInt16();
            index = compressedData.readUint8();
            compressedData.position++; // skip extra header byte
            if (index > 88) index = 88;
            out.push(sample);
        } else {
            // read 4-bit code and compute delta from previous sample
            if (lastByte < 0) {
                if (compressedData.getBytesAvailable() == 0) break;
                lastByte = compressedData.readUint8();
                code = lastByte & 0xF;
            } else {
                code = (lastByte >> 4) & 0xF;
                lastByte = -1;
            }
            step = this.stepTable[index];
            delta = 0;
            if (code & 4) delta += step;
            if (code & 2) delta += step >> 1;
            if (code & 1) delta += step >> 2;
            delta += step >> 3;
            // compute next index
            index += this.indexTable[code];
            if (index > 88) index = 88;
            if (index < 0) index = 0;
            // compute and output sample
            sample += (code & 8) ? -delta : delta;
            if (sample > 32767) sample = 32767;
            if (sample < -32768) sample = -32768;
            out.push(sample);
        }
    }
    var samples = Int16Array.from(out);
    return samples;
};

module.exports = ADPCMSoundLoader;
<|endoftext|>


<commit_before>const WEBDRIVER_LOG_PATH = process.env.npm_package_config_webdriver_log;
const WEBDRIVER_LOG_LEVEL = process.env.npm_package_config_webdriver_log_level || 'INFO';

const AUTH_URL = 'https://ssl.filmweb.pl/login';
const BASE_URL = 'http://www.filmweb.pl';

const USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36';

const webdriver = require('selenium-webdriver');
const phantomjsPath = require('phantomjs-prebuilt').path;
const phantomjsCapabilities = webdriver.Capabilities.phantomjs();

const log = require('./log');
const {
    simulateHumanDelay,
    throttleRequest,
    createQueueItem,
    STALE_NAVIGATION_TRESHOLD,
} = require('./delayUtils');

const browsers = new WeakMap;
const browsersMetadata = new WeakMap;

phantomjsCapabilities.set('phantomjs.binary.path', phantomjsPath);
phantomjsCapabilities.set('phantomjs.page.settings.userAgent', USER_AGENT);
if (WEBDRIVER_LOG_PATH)
{
    phantomjsCapabilities.set('phantomjs.cli.args', [
        '--webdriver-logfile=' + WEBDRIVER_LOG_PATH,
        '--webdriver-loglevel=' + WEBDRIVER_LOG_LEVEL,
    ]);
}

function exitHandler ()
{
    let browser = browsers.get(this);
    if (browser)
    {
        this.destroy();
    }
}

function bootstrap (url)
{
    let browser = browsers.get(this);

    if (! browser)
    {
        return Promise.reject(new Error('Missing browser instance.'));
    }
    else if (url)
    {
        return browser.goToUrl(url);
    }
    else
    {
        return Promise.resolve();
    }
}

function createBrowser ()
{
    log(2, 'Create new browser');

    let browser = new webdriver.Builder()
        .withCapabilities(
            // Overwriteable with SELENIUM_BROWSER env variable
            phantomjsCapabilities
        )
        .build();

    browser.goToUrl = (url) => {
        return browser.getCurrentUrl()
            .then(currentUrl => {
                let lastNavTimestamp = getBrowserMetadata
                    .call(this, 'lastNavTimestamp') || 0;
                if (
                    currentUrl === url &&
                    Date.now() - lastNavTimestamp < STALE_NAVIGATION_TRESHOLD
                )
                {
                    log(3, 'Already at', url);
                }
                else
                {
                    log(3, 'Navigating to', url);
                    return throttleRequest()
                        .then(() => browser.get(url))
                        .then(() => {
                            setBrowserMetadata
                                .call(this, {lastNavTimestamp: Date.now()});
                        });
                }
            });
    };

    browser.querySelector =
        (selector) => browser.findElement(webdriver.By.css(selector));

    browser.waitFor = (selector, timeout, failMessage) => {
        log(3, 'Waiting for', selector);
        return browser.wait(webdriver.until.elementLocated(webdriver.By.css(selector)), timeout, failMessage);
    };

    browser.click = (selector, timeout, failMessage) => {
        log(3, 'Clicking', selector);
        return browser.wait(webdriver.until.elementLocated(webdriver.By.css(selector)), timeout, failMessage)
            .then(element => element.click());
    };

    browser.fillUp = (selector, value, timeout, failMessage) => {
        // Reluctant to log the value itself as it may be a password
        log(3, 'Filling', selector, 'with a value');
        return browser.wait(webdriver.until.elementLocated(webdriver.By.css(selector)), timeout, failMessage)
            .then(element => {
                return element.clear()
                    .then(() => element.sendKeys(value));
            });
    };

    browsers.set(this, browser);
    browsersMetadata.set(this, {});

    // Clean up on exit
    process.on('exit', exitHandler.bind(this));
}

function setBrowserMetadata (values)
{
    let metadata = browsersMetadata.get(this);
    for (let prop in values)
    {
        metadata[prop] = values[prop];
    }
}

function getBrowserMetadata (property)
{
    let metadata = browsersMetadata.get(this) || {};
    if (property === undefined)
    {
        return metadata;
    }
    else
    {
        return metadata[property];
    }
}

// This function will be serialized.
// Do not use any fancy syntax.
function searchDataCollector (context)
{
    var results = {items: []};
    var list;

    var query = function (context, selector) {
        var element = context.querySelector(selector);
        return element || {};
    };

    if (context.querySelector('.searchNoResults'))
    {
        results.total = 0;
        results.page = 0;
        results.totalPages = 0;
    }
    else
    {
        results.nav = query(context, '#searchPaginator .config').textContent;

        list = context.querySelector('.resultsList');
        Array.prototype.forEach.call(list.children, function (listItem) {
            var queryListItem = query.bind(null, listItem);
            var result = {};
            var subList;

            result.url = queryListItem('a.hitTitle').href;
            result.title = queryListItem('a.hitTitle').textContent;
            result.year = queryListItem('a.hitTitle ~ .hitTitle').textContent;
            result.posterUrl = queryListItem('.hitImage img').src;

            result.ratingValue = queryListItem('.rateInfo strong').textContent;
            result.ratingUser = queryListItem('.ribbonButton strong').textContent;

            result.titles = [];
            subList = queryListItem('.sep-line').children;
            Array.prototype.forEach.call(subList, function (subItem) {
                if (! subItem.querySelector('ul'))
                {
                    result.titles.push(subItem.textContent);
                }
            });

            result.directors = [];
            result.cast = [];
            subList = queryListItem('.filmInfo');
            Array.prototype.forEach.call(subList, function (subItem) {
                var peopleList;
                if (/reżyser/i.test(subItem.textContent))
                {
                    subList.activeContainer = results.directors;
                }
                else if (/obsada/i.test(subItem.textContent))
                {
                    subList.activeContainer = results.cast;
                }
                else if (subList.activeContainer)
                {
                    peopleList = query(subItem, 'a');
                    Array.prototype.forEach.call(peopleList, function (person) {
                        subList.activeContainer.push({
                            url: person.href,
                            name: person.textContent,
                        });
                    });
                }
            });

            results.items.push(result);
        });
    }

    return results;
}

function collectSearchResults (element)
{
    let browser = browsers.get(this);
    return browser.executeScript(searchDataCollector, element);
}

function formatRawSearchResults (rawResults)
{
    let results = {
        total: rawResults.total,
        page: rawResults.page,
        totalPages: rawResults.totalPages,
    };

    if (rawResults.nav)
    {
        let rawResultsNav = JSON.parse(rawResults.nav);

        results.total = rawResultsNav.countItems;
        results.page = rawResultsNav.activePage;
        results.totalPages = rawResultsNav.totalPages;
    }

    results.items = rawResults.items.map(formatRawSearchResultItem);

    return results;
}

function formatRawSearchResultItem (rawItem)
{
    let result = {};
    let value;

    result.url = rawItem.url.trim();
    result.title = rawItem.title.trim();
    result.year = rawItem.year.trim().replace(/[()]/g, '');
    result.posterUrl = rawItem.posterUrl.trim();

    result.ratings = {};
    value = parseFloat((rawItem.ratingValue || '').replace(',', '.'));
    result.ratings.average = value || undefined;
    value = parseInt(rawItem.ratingUser, 10);
    result.ratings.user = value || null;

    result.alternativeTitles = rawItem.titles.map(formatRawSearchResultsTitle);
    result.directors = rawItem.directors.map(formatRawSearchResultsPerson);
    result.cast = rawItem.cast.map(formatRawSearchResultsPerson);

    if (rawItem.hasOwnProperty('debug'))
    {
        result.debug = rawItem.debug;
    }

    return result;
}

function formatRawSearchResultsTitle (rawTitle)
{
    const parts = rawTitle.trim().split(' : ');
    const title = {
        value: parts[0],
        countries: [],
    };
    const types = [];

    for (let i = 1; i < parts.length; i += 1)
    {
        let value = parts[i].trim();
        let match = value.match(/^\((.*)\)$/);
        if (match)
        {
            types.push(match[1]);
        }
        else
        {
            title.countries.push(value.replace(/[\/ ]*$/, ''));
        }
    }

    // I haven’t seen any titles with multiple types specified, so we
    // return this as a string, but to support that case, we collect all
    // types and return them.
    if (types.length)
    {
        title.type = types.join(', ');
    }

    return title;
}

function formatRawSearchResultsPerson (rawPerson)
{
    let person = {};
    person.url = rawPerson.url.trim();
    person.name = rawPerson.name.trim();
    return person;
}

function normalizeRatingValue (value)
{
    if (! value)
    {
        value = null;
    }
    else
    {
        value = parseInt(value, 10);
    }
    return value;
}

function normalizeReviewValue (value)
{
    if (value === undefined || value === null)
    {
        value = '';
    }
    else
    {
        value = '' + value;
    }
    return value;
}

function rate (url, value, review)
{
    if (! this.isAuthenticated())
    {
        return Promise.reject(new Error('Rate method can only be used on authenticated object.'));
    }

    let browser = browsers.get(this);

    value = normalizeRatingValue(value);
    review = normalizeReviewValue(review);

    let queueItem = createQueueItem();

    return queueItem.waitForItsTurn
        .then(() => bootstrap.call(this, url))
        .then(() => browser.waitFor('#filmVoteRatingPanel .rateButtons [data-index]'))
        .then(simulateHumanDelay)
        .then(() => browser.querySelector('#filmVoteRatingPanel .rateButtons'))
        .then(element => element.getAttribute('data-rate'))
        .then(normalizeRatingValue)
        .then(currentRating => {
            let promise = Promise.resolve(true);

            if (value === null && currentRating !== null)
            {
                promise = promise
                    .then(simulateHumanDelay)
                    .then(() => browser.click('#filmVoteRatingPanel .rateButtons [data-index="' + currentRating + '"]'))
                    .then(() => browser.waitFor('#filmVoteRatingPanel .rateButtons:not([data-rate])'));
            }
            else if (value !== currentRating)
            {
                promise = promise
                    .then(simulateHumanDelay)
                    .then(() => browser.click('#filmVoteRatingPanel .rateButtons [data-index="' + value + '"]'))
                    .then(() => browser.waitFor('#filmVoteRatingPanel .rateButtons[data-rate="' + value + '"]'));
            }

            if (value !== null)
            {
                promise = promise
                    .then(simulateHumanDelay)
                    .then(() => browser.querySelector('#filmVoteRatingPanel .commentArea'))
                    .then(element => element.getText())
                    .then(normalizeReviewValue)
                    .then(currentReview => {
                        if (currentReview !== review)
                        {
                            return browser.fillUp('#filmVoteRatingPanel .commentArea', review)
                                .then(() => browser.click('#filmVoteRatingPanel .saveComment'))
                                // Wait until comment is saved
                                .then(simulateHumanDelay)
                                .then(() => browser.waitFor('.filmVoteRatingPanelWrapper:not(.commonLoader)'));
                        }
                    });
            }

            return promise;
        })
        .then(queueItem.resolved, queueItem.rejected);
}

class FilmWeb
{
    constructor ()
    {
        createBrowser.call(this);
    }

    isAuthenticated ()
    {
        return !! this.getLogin();
    }

    getLogin ()
    {
        return getBrowserMetadata.call(this, 'login');
    }

    destroy ()
    {
        log(2, 'Destroy a browser');
        let browser = browsers.get(this);
        browser.quit();
        browsers.delete(this);
        browsersMetadata.delete(this);
    }

    search (query, page=1)
    {
        log(1, 'search called', {query, page});

        let browser = browsers.get(this);
        let urlQuery = encodeURIComponent(query);
        page = parseInt(page, 10);
        let url = BASE_URL + '/search?q=' + urlQuery + '&page=' + page;

        let queueItem = createQueueItem();

        return queueItem.waitForItsTurn
            .then(() => bootstrap.call(this, url))
            .then(() => browser.waitFor('.resultsList, .searchNoResults'))
            .then(() => browser.waitFor('.mainCol'))
            .then(collectSearchResults.bind(this))
            .then(formatRawSearchResults)
            .catch(error => {
                log(2, 'search failed', error);
                throw error;
            })
            .then((results) => {
                log(2, 'search finished', {query, page});
                log(4, 'results for FilmWeb.search', {query, page}, ':', results);
                return results;
            })
            .then(queueItem.resolved, queueItem.rejected);
    }

    authenticate (login, password)
    {
        log(1, 'authenticate called', {login, password: '*****'});

        let browser = browsers.get(this);

        setBrowserMetadata.call(this, {login});

        let queueItem = createQueueItem();

        return queueItem.waitForItsTurn
            .then(() => bootstrap.call(this, AUTH_URL))
            .then(simulateHumanDelay)
            .then(() => Promise.all([
                browser.fillUp('[name=j_username]', login),
                browser.fillUp('[name=j_password]', password),
            ]))
            .then(() => browser.click('[type=submit]'))
            .then(() => browser.waitFor('#userHeader .user__name', 2000))
            .then(
                () => log(2, 'authenticate finished', {login}),
                error => {
                    log(2, 'authenticate failed', error.message);
                    throw new Error('Authentication failed.');
                }
            )
            .then(queueItem.resolved, queueItem.rejected);
    }

    rate (url, value, review)
    {
        log(1, 'rate called', {url, value, review});

        return rate.apply(this, arguments)
            .catch(error => {
                log(2, 'rate failed', {error});
                throw error;
            })
            .then(() => log(2, 'rate finished', {url, value, review}));
    }

    removeRating (url)
    {
        log(1, 'removeRating called', {url});

        return rate.apply(this, arguments)
            .catch(error => {
                log(2, 'removeRating failed', error);
                throw error;
            })
            .then(() => log(2, 'removeRating finished', {url}));
    }
}

// Call process.exit when killed with signals to trigger cleanup
process.on('SIGINT', () => process.exit(1));
process.on('SIGHUP', () => process.exit(1));
process.on('SIGQUIT', () => process.exit(1));
process.on('SIGTERM', () => process.exit(1));

module.exports = FilmWeb;
<commit_msg>Fixup collecting directors and cast in search results<commit_after>const WEBDRIVER_LOG_PATH = process.env.npm_package_config_webdriver_log;
const WEBDRIVER_LOG_LEVEL = process.env.npm_package_config_webdriver_log_level || 'INFO';

const AUTH_URL = 'https://ssl.filmweb.pl/login';
const BASE_URL = 'http://www.filmweb.pl';

const USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36';

const webdriver = require('selenium-webdriver');
const phantomjsPath = require('phantomjs-prebuilt').path;
const phantomjsCapabilities = webdriver.Capabilities.phantomjs();

const log = require('./log');
const {
    simulateHumanDelay,
    throttleRequest,
    createQueueItem,
    STALE_NAVIGATION_TRESHOLD,
} = require('./delayUtils');

const browsers = new WeakMap;
const browsersMetadata = new WeakMap;

phantomjsCapabilities.set('phantomjs.binary.path', phantomjsPath);
phantomjsCapabilities.set('phantomjs.page.settings.userAgent', USER_AGENT);
if (WEBDRIVER_LOG_PATH)
{
    phantomjsCapabilities.set('phantomjs.cli.args', [
        '--webdriver-logfile=' + WEBDRIVER_LOG_PATH,
        '--webdriver-loglevel=' + WEBDRIVER_LOG_LEVEL,
    ]);
}

function exitHandler ()
{
    let browser = browsers.get(this);
    if (browser)
    {
        this.destroy();
    }
}

function bootstrap (url)
{
    let browser = browsers.get(this);

    if (! browser)
    {
        return Promise.reject(new Error('Missing browser instance.'));
    }
    else if (url)
    {
        return browser.goToUrl(url);
    }
    else
    {
        return Promise.resolve();
    }
}

function createBrowser ()
{
    log(2, 'Create new browser');

    let browser = new webdriver.Builder()
        .withCapabilities(
            // Overwriteable with SELENIUM_BROWSER env variable
            phantomjsCapabilities
        )
        .build();

    browser.goToUrl = (url) => {
        return browser.getCurrentUrl()
            .then(currentUrl => {
                let lastNavTimestamp = getBrowserMetadata
                    .call(this, 'lastNavTimestamp') || 0;
                if (
                    currentUrl === url &&
                    Date.now() - lastNavTimestamp < STALE_NAVIGATION_TRESHOLD
                )
                {
                    log(3, 'Already at', url);
                }
                else
                {
                    log(3, 'Navigating to', url);
                    return throttleRequest()
                        .then(() => browser.get(url))
                        .then(() => {
                            setBrowserMetadata
                                .call(this, {lastNavTimestamp: Date.now()});
                        });
                }
            });
    };

    browser.querySelector =
        (selector) => browser.findElement(webdriver.By.css(selector));

    browser.waitFor = (selector, timeout, failMessage) => {
        log(3, 'Waiting for', selector);
        return browser.wait(webdriver.until.elementLocated(webdriver.By.css(selector)), timeout, failMessage);
    };

    browser.click = (selector, timeout, failMessage) => {
        log(3, 'Clicking', selector);
        return browser.wait(webdriver.until.elementLocated(webdriver.By.css(selector)), timeout, failMessage)
            .then(element => element.click());
    };

    browser.fillUp = (selector, value, timeout, failMessage) => {
        // Reluctant to log the value itself as it may be a password
        log(3, 'Filling', selector, 'with a value');
        return browser.wait(webdriver.until.elementLocated(webdriver.By.css(selector)), timeout, failMessage)
            .then(element => {
                return element.clear()
                    .then(() => element.sendKeys(value));
            });
    };

    browsers.set(this, browser);
    browsersMetadata.set(this, {});

    // Clean up on exit
    process.on('exit', exitHandler.bind(this));
}

function setBrowserMetadata (values)
{
    let metadata = browsersMetadata.get(this);
    for (let prop in values)
    {
        metadata[prop] = values[prop];
    }
}

function getBrowserMetadata (property)
{
    let metadata = browsersMetadata.get(this) || {};
    if (property === undefined)
    {
        return metadata;
    }
    else
    {
        return metadata[property];
    }
}

// This function will be serialized.
// Do not use any fancy syntax.
function searchDataCollector (context)
{
    var results = {items: []};
    var list;

    var query = function (context, selector) {
        var element = context.querySelector(selector);
        return element || {};
    };

    if (context.querySelector('.searchNoResults'))
    {
        results.total = 0;
        results.page = 0;
        results.totalPages = 0;
    }
    else
    {
        results.nav = query(context, '#searchPaginator .config').textContent;

        list = context.querySelector('.resultsList');
        Array.prototype.forEach.call(list.children, function (listItem) {
            var queryListItem = query.bind(null, listItem);
            var result = {};
            var subList;

            result.url = queryListItem('a.hitTitle').href;
            result.title = queryListItem('a.hitTitle').textContent;
            result.year = queryListItem('a.hitTitle ~ .hitTitle').textContent;
            result.posterUrl = queryListItem('.hitImage img').src;

            result.ratingValue = queryListItem('.rateInfo strong').textContent;
            result.ratingUser = queryListItem('.ribbonButton strong').textContent;

            result.titles = [];
            subList = queryListItem('.sep-line').children;
            Array.prototype.forEach.call(subList, function (subItem) {
                if (! subItem.querySelector('ul'))
                {
                    result.titles.push(subItem.textContent);
                }
            });

            result.directors = [];
            result.cast = [];
            subList = queryListItem('.filmInfo');
            Array.prototype.forEach.call(subList.children, function (subItem) {
                var people;
                if (/reżyser/i.test(subItem.textContent))
                {
                    subList.activeContainer = result.directors;
                }
                else if (/obsada/i.test(subItem.textContent))
                {
                    subList.activeContainer = result.cast;
                }
                else if (subList.activeContainer)
                {
                    people = subItem.querySelectorAll('a');
                    Array.prototype.forEach.call(people, function (person) {
                        subList.activeContainer.push({
                            url: person.href,
                            name: person.textContent,
                        });
                    });
                }
            });

            results.items.push(result);
        });
    }

    return results;
}

function collectSearchResults (element)
{
    let browser = browsers.get(this);
    return browser.executeScript(searchDataCollector, element);
}

function formatRawSearchResults (rawResults)
{
    let results = {
        total: rawResults.total,
        page: rawResults.page,
        totalPages: rawResults.totalPages,
    };

    if (rawResults.nav)
    {
        let rawResultsNav = JSON.parse(rawResults.nav);

        results.total = rawResultsNav.countItems;
        results.page = rawResultsNav.activePage;
        results.totalPages = rawResultsNav.totalPages;
    }

    results.items = rawResults.items.map(formatRawSearchResultItem);

    return results;
}

function formatRawSearchResultItem (rawItem)
{
    let result = {};
    let value;

    result.url = rawItem.url.trim();
    result.title = rawItem.title.trim();
    result.year = rawItem.year.trim().replace(/[()]/g, '');
    result.posterUrl = rawItem.posterUrl.trim();

    result.ratings = {};
    value = parseFloat((rawItem.ratingValue || '').replace(',', '.'));
    result.ratings.average = value || undefined;
    value = parseInt(rawItem.ratingUser, 10);
    result.ratings.user = value || null;

    result.alternativeTitles = rawItem.titles.map(formatRawSearchResultsTitle);
    result.directors = rawItem.directors.map(formatRawSearchResultsPerson);
    result.cast = rawItem.cast.map(formatRawSearchResultsPerson);

    if (rawItem.hasOwnProperty('debug'))
    {
        result.debug = rawItem.debug;
    }

    return result;
}

function formatRawSearchResultsTitle (rawTitle)
{
    const parts = rawTitle.trim().split(' : ');
    const title = {
        value: parts[0],
        countries: [],
    };
    const types = [];

    for (let i = 1; i < parts.length; i += 1)
    {
        let value = parts[i].trim();
        let match = value.match(/^\((.*)\)$/);
        if (match)
        {
            types.push(match[1]);
        }
        else
        {
            title.countries.push(value.replace(/[\/ ]*$/, ''));
        }
    }

    // I haven’t seen any titles with multiple types specified, so we
    // return this as a string, but to support that case, we collect all
    // types and return them.
    if (types.length)
    {
        title.type = types.join(', ');
    }

    return title;
}

function formatRawSearchResultsPerson (rawPerson)
{
    let person = {};
    person.url = rawPerson.url.trim();
    person.name = rawPerson.name.trim();
    return person;
}

function normalizeRatingValue (value)
{
    if (! value)
    {
        value = null;
    }
    else
    {
        value = parseInt(value, 10);
    }
    return value;
}

function normalizeReviewValue (value)
{
    if (value === undefined || value === null)
    {
        value = '';
    }
    else
    {
        value = '' + value;
    }
    return value;
}

function rate (url, value, review)
{
    if (! this.isAuthenticated())
    {
        return Promise.reject(new Error('Rate method can only be used on authenticated object.'));
    }

    let browser = browsers.get(this);

    value = normalizeRatingValue(value);
    review = normalizeReviewValue(review);

    let queueItem = createQueueItem();

    return queueItem.waitForItsTurn
        .then(() => bootstrap.call(this, url))
        .then(() => browser.waitFor('#filmVoteRatingPanel .rateButtons [data-index]'))
        .then(simulateHumanDelay)
        .then(() => browser.querySelector('#filmVoteRatingPanel .rateButtons'))
        .then(element => element.getAttribute('data-rate'))
        .then(normalizeRatingValue)
        .then(currentRating => {
            let promise = Promise.resolve(true);

            if (value === null && currentRating !== null)
            {
                promise = promise
                    .then(simulateHumanDelay)
                    .then(() => browser.click('#filmVoteRatingPanel .rateButtons [data-index="' + currentRating + '"]'))
                    .then(() => browser.waitFor('#filmVoteRatingPanel .rateButtons:not([data-rate])'));
            }
            else if (value !== currentRating)
            {
                promise = promise
                    .then(simulateHumanDelay)
                    .then(() => browser.click('#filmVoteRatingPanel .rateButtons [data-index="' + value + '"]'))
                    .then(() => browser.waitFor('#filmVoteRatingPanel .rateButtons[data-rate="' + value + '"]'));
            }

            if (value !== null)
            {
                promise = promise
                    .then(simulateHumanDelay)
                    .then(() => browser.querySelector('#filmVoteRatingPanel .commentArea'))
                    .then(element => element.getText())
                    .then(normalizeReviewValue)
                    .then(currentReview => {
                        if (currentReview !== review)
                        {
                            return browser.fillUp('#filmVoteRatingPanel .commentArea', review)
                                .then(() => browser.click('#filmVoteRatingPanel .saveComment'))
                                // Wait until comment is saved
                                .then(simulateHumanDelay)
                                .then(() => browser.waitFor('.filmVoteRatingPanelWrapper:not(.commonLoader)'));
                        }
                    });
            }

            return promise;
        })
        .then(queueItem.resolved, queueItem.rejected);
}

class FilmWeb
{
    constructor ()
    {
        createBrowser.call(this);
    }

    isAuthenticated ()
    {
        return !! this.getLogin();
    }

    getLogin ()
    {
        return getBrowserMetadata.call(this, 'login');
    }

    destroy ()
    {
        log(2, 'Destroy a browser');
        let browser = browsers.get(this);
        browser.quit();
        browsers.delete(this);
        browsersMetadata.delete(this);
    }

    search (query, page=1)
    {
        log(1, 'search called', {query, page});

        let browser = browsers.get(this);
        let urlQuery = encodeURIComponent(query);
        page = parseInt(page, 10);
        let url = BASE_URL + '/search?q=' + urlQuery + '&page=' + page;

        let queueItem = createQueueItem();

        return queueItem.waitForItsTurn
            .then(() => bootstrap.call(this, url))
            .then(() => browser.waitFor('.resultsList, .searchNoResults'))
            .then(() => browser.waitFor('.mainCol'))
            .then(collectSearchResults.bind(this))
            .then(formatRawSearchResults)
            .catch(error => {
                log(2, 'search failed', error);
                throw error;
            })
            .then((results) => {
                log(2, 'search finished', {query, page});
                log(4, 'results for FilmWeb.search', {query, page}, ':', results);
                return results;
            })
            .then(queueItem.resolved, queueItem.rejected);
    }

    authenticate (login, password)
    {
        log(1, 'authenticate called', {login, password: '*****'});

        let browser = browsers.get(this);

        setBrowserMetadata.call(this, {login});

        let queueItem = createQueueItem();

        return queueItem.waitForItsTurn
            .then(() => bootstrap.call(this, AUTH_URL))
            .then(simulateHumanDelay)
            .then(() => Promise.all([
                browser.fillUp('[name=j_username]', login),
                browser.fillUp('[name=j_password]', password),
            ]))
            .then(() => browser.click('[type=submit]'))
            .then(() => browser.waitFor('#userHeader .user__name', 2000))
            .then(
                () => log(2, 'authenticate finished', {login}),
                error => {
                    log(2, 'authenticate failed', error.message);
                    throw new Error('Authentication failed.');
                }
            )
            .then(queueItem.resolved, queueItem.rejected);
    }

    rate (url, value, review)
    {
        log(1, 'rate called', {url, value, review});

        return rate.apply(this, arguments)
            .catch(error => {
                log(2, 'rate failed', {error});
                throw error;
            })
            .then(() => log(2, 'rate finished', {url, value, review}));
    }

    removeRating (url)
    {
        log(1, 'removeRating called', {url});

        return rate.apply(this, arguments)
            .catch(error => {
                log(2, 'removeRating failed', error);
                throw error;
            })
            .then(() => log(2, 'removeRating finished', {url}));
    }
}

// Call process.exit when killed with signals to trigger cleanup
process.on('SIGINT', () => process.exit(1));
process.on('SIGHUP', () => process.exit(1));
process.on('SIGQUIT', () => process.exit(1));
process.on('SIGTERM', () => process.exit(1));

module.exports = FilmWeb;
<|endoftext|>


<commit_before>var utils = require('./utils');
var sockets = engine.sockets = {};
var streams = {};

exports.setup = setupSocket;
exports.stream = setupStream;

/**
 * Setup a stream to write over a socket.
 *
 * @public
 * @param {object} The stream object.
 * @param {string} The destination URL.
 */
function setupStream (stream, url) {

    // append parsed url on stream
    if (!url || typeof url === 'string' && !(url = parseUrl(url))) {
        return engine.log('E', new Error('Invalid socket URL.'));
    }

    // create stream id
    stream.id = stream.id || utils.uid(3);

    // save url data
    stream.url = url;

    // create or get a socket
    stream.socket = stream.socket || getSocket(stream);

    // save session on stream
    stream.session = stream.socket.session;

    // send data to socket on data
    stream.data(send);

    // append custom end handler
    stream._end = end;

    // save stream in cache
    streams[stream.id] = stream;

    return stream;
}

/**
 * Get or create a new client Socket.
 *
 * @public
 * @param {string} The target host.
 */
function getSocket (stream) {

    var host = stream.url.host;

    // get socket from cached connections
    if (sockets[host]) {
        sockets[host].streams[stream.id] = stream;
        return sockets[host];
    }

    // create new websocket connection
    var socket = sockets[host] = new WebSocket('ws://' + host + '/');
    socket._cacheKey = host;
    socket = setupSocket(socket);
    socket.streams[stream.id] = stream;

    // setup and return socket
    return socket;
}

/**
 * Setup socket with handlers.
 *
 * @public
 * @param {object} The socket.
 */
function setupSocket (socket) {

    // stream refs
    socket.streams = {};
    socket._buffer = [];

    // session
    socket.session = socket.session || {};

    // setup socket handlers
    socket.onclose = closeHandler;
    socket.onerror = errorHandler;
    socket.onmessage = parseMessage;
    socket.onopen = openHandler;

    return socket;
}

/**
 * Parse a socket URL string.
 *
 * @public
 * @param {string} The destination URL.
 */
function parseUrl (url) {

    url = url.split('/');
    if (url.length < 2) {
        return;
    }

    var urlObject = {};
    if (url.length < 3) {
        urlObject = {
            // TODO server default host??
            host: global.location.host,
            inst: url[0],
            emit: url[1]
        };
    } else {
        urlObject = {
            host: url[0],
            inst: url[1],
            emit: url[2]
        };
    }

    return urlObject;
}

/**
 * Create and send a websocket message.
 *
 * @private
 * @param {object} The error object.
 * @param {object} The data object.
 */
function send (err, data, stream, end) {

    // create message
    var message = [end ? 0 : 1, stream.url.inst, stream.url.emit, stream.id, err ? err.toString() : 0];

    // add the data to the message
    if (typeof data !== 'undefined') {
        message[5] = data;
    }

    // encode message (string)
    try {
        message = JSON.stringify(message);
    } catch (err) {
        return engine.log('E', err);
    }

    // create socket a socket
    if (!stream.socket && stream.url) {
        stream.socket = getSocket(stream);
    }

    // buffer writes if socket is not ready yet
    if (stream.socket.readyState !== 1) {
        return stream.socket._buffer.push(message);
    }

    // send message over socket
    stream.socket.send(message);
}

/**
 * Custom end handler.
 *
 * @private
 */
function end (err, data) {

    // send end signal
    send.call(this._, err, data, this, true);

    // remove stream from cache
    delete streams[this.id];

    // remove stream from socket
    delete this.socket.streams[this.id];
}

/**
 * Prase websocket messages.
 *
 * @public
 * @param {object} The message event object.
 */
function parseMessage (messageEvent) {

    var socket = messageEvent.target;
    var message = messageEvent.data;
    var role = (socket.session || {})[engine.session_role];

    // parse message
    // protocoll: [type, instance, event, id, err, data]

    // TODO broadcast (grouped connections)
    /*
      - how to define/create groups?
      - get group by name
      - get group by session id
      - get group by regexp pattern?
      - get group by role?
      - check group access with session
      - public groups
    */

    try {
        message = JSON.parse(message);
    } catch (error) {
        return engine.log('E', error);
    }

    // extract data from parsed message
    var type = message[0];
    var instance = message[1];
    var event = message[2];
    var id = message[3];
    var err = message[4];
    var data = message[5];

    // check mandatory message values
    if (!event || !instance || !id) {

        // send error back
        message[4] = 'Invalid message.';
        message[5] = 0;
        return socket.send(JSON.stringify(message));
    }

    // get the link object
    var stream = streams[id];

    // write data to stream on write message
    if (type) {

        // create a new stream and emit to flow handlers
        if (!stream) {

            // get module instance
            if (!(instance = engine.instances[instance])) {

                // load an instance
                engine.load(message[1], role, function (err, instance) {

                    if (err) {

                        // send error back
                        message[4] = err;
                        message[5] = 0;
                        return socket.send(JSON.stringify(message));
                    }

                    // create a new stream and save it in cache and write data
                    stream = createStream(instance, event, id, socket, message[1]);
                    stream.write(err, data);
                });
                return;

            // check role and event access on server module instance
            } else if (!(utils.eventAccess(instance, role, event))) {

                // send error back
                message[4] = 'Access denied to instance: ' + message[1];
                message[5] = 0;
                return socket.send(JSON.stringify(message));
            }

            // create a new stream and save it in cache and write data
            if (instance._name === 'custom') {
                console.log('WS MSG NO STREAM:', event, id, instance._access || '@');
                debugger;
            }
            stream = createStream(instance, event, id, socket);
        }

        stream.write(err, data);

    // end stream on end message
    } else if (stream) {
        stream.end(err, data);
    }
}

function createStream (instance, event, id, socket) {

    var stream = instance.flow(event, {
        id: id,
        socket: socket
    });

    // create stream and save it to cache
    sockets[id] = setupStream(stream, {
        inst: instance._name || '@',
        emit: event
    });

    return sockets[id];
}

/**
 * Write bufferd data when a socket is connected.
 *
 * @public
 * @param {object} The open event object.
 */
function openHandler (openEvent) {
    var socket = openEvent.target;

    // send buffered writes to socket
    if (socket._buffer.length) {
        for (var i = 0, l = socket._buffer.length; i < l; ++i) {
            socket.send(socket._buffer[i]);
        }

        // empty buffer
        socket._buffer = [];
    }
}

/**
 * Remove socket from cache and end streams.
 *
 * @public
 * @param {object} The close event object.
 */
function closeHandler (closeEvent) {
    var socket = closeEvent.target;

    for (var stream in socket.streams) {

        // end all streams on the server
        if (!engine.client) {
            socket.streams[stream].end();

        // remove socket ref on the stream
        } else {
            socket.streams[stream].socket = undefined;
        }
    }

    // remove socket from sockets cache
    delete sockets[socket._cacheKey];
}

/**
 * Write socket errors to streams.
 *
 * @public
 * @param {object} The error event object.
 */
function errorHandler (errorEvent) {
    var socket = errorEvent.target;
    for (var stream in socket.streams) {
        stream.write(errorEvent.data);
    }
}
<commit_msg>better socket cacheing, engine socket reload handler<commit_after>var utils = require('./utils');
var streams = {};
var sockets = {};

exports.setup = setupSocket;
exports.stream = setupStream;

engine._r.push(function () {

    // close all websockets
    for (var socket in sockets) {
        sockets[socket].close();
    }

    // remove stream refs
    streams = {};

    // remove socket refs
    sockets = {};
});

/**
 * Setup a stream to write over a socket.
 *
 * @public
 * @param {object} The stream object.
 * @param {string} The destination URL.
 */
function setupStream (stream, url) {

    // append parsed url on stream
    if (!url || typeof url === 'string' && !(url = parseUrl(url))) {
        return engine.log('E', new Error('Invalid socket URL.'));
    }

    // create stream id
    stream.id = stream.id || utils.uid(3);

    // save url data
    stream.url = url;

    // create or get a socket
    stream.socket = stream.socket || getSocket(stream);

    // save session on stream
    stream.session = stream.socket.session;

    // send data to socket on data
    stream.data(send);

    // append custom end handler
    stream._end = end;

    // save stream in cache
    streams[stream.id] = stream;

    return stream;
}

/**
 * Get or create a new client Socket.
 *
 * @public
 * @param {string} The target host.
 */
function getSocket (stream) {

    var host = stream.url.host || global.location.host;
    var socket = sockets[host];

    // get socket from cached connections
    if (socket && socket.readyState < 2) {
        sockets[host].streams[stream.id] = stream;
        return sockets[host];
    }

    // create new websocket connection
    socket = sockets[host] = new WebSocket('ws://' + host + '/');
    socket._cacheKey = host;

    socket = setupSocket(socket);

    socket.streams[stream.id] = stream;

    // setup and return socket
    return socket;
}

/**
 * Setup socket with handlers.
 *
 * @public
 * @param {object} The socket.
 */
function setupSocket (socket) {

    // stream refs
    socket.streams = {};
    socket._buffer = [];

    // session
    socket.session = socket.session || {};

    // setup socket handlers
    socket.onclose = closeHandler;
    socket.onerror = errorHandler;
    socket.onmessage = parseMessage;
    socket.onopen = openHandler;

    return socket;
}

/**
 * Parse a socket URL string.
 *
 * @public
 * @param {string} The destination URL.
 */
function parseUrl (url) {

    url = url.split('/');
    if (url.length < 2) {
        return;
    }

    var urlObject = {};
    if (url.length < 3) {
        urlObject = {
            inst: url[0],
            emit: url[1]
        };
    } else {
        urlObject = {
            host: url[0],
            inst: url[1],
            emit: url[2]
        };
    }

    return urlObject;
}

/**
 * Create and send a websocket message.
 *
 * @private
 * @param {object} The error object.
 * @param {object} The data object.
 */
function send (err, data, stream, end) {

    // create message
    var message = [end ? 0 : 1, stream.url.inst, stream.url.emit, stream.id, err ? err.toString() : 0];

    // add the data to the message
    if (typeof data !== 'undefined') {
        message[5] = data;
    }

    // encode message (string)
    try {
        message = JSON.stringify(message);
    } catch (err) {
        return engine.log('E', err);
    }

    // create a new socket
    if (!stream.socket) {
        stream.socket = getSocket(stream);
    }

    // buffer writes if socket is not ready yet
    if (stream.socket.readyState !== 1) {
        return stream.socket._buffer.push(message);
    }

    // send message over socket
    stream.socket.send(message);
}

/**
 * Custom end handler.
 *
 * @private
 */
function end (err, data) {

    // send end signal
    send.call(this._, err, data, this, true);

    // remove stream from cache
    delete streams[this.id];

    // remove stream from socket
    delete this.socket.streams[this.id];
}

/**
 * Prase websocket messages.
 *
 * @public
 * @param {object} The message event object.
 */
function parseMessage (messageEvent) {

    var socket = messageEvent.target;
    var message = messageEvent.data;
    var role = (socket.session || {})[engine.session_role];

    // parse message
    // protocoll: [type, instance, event, id, err, data]

    // TODO broadcast (grouped connections)
    /*
      - how to define/create groups?
      - get group by name
      - get group by session id
      - get group by regexp pattern?
      - get group by role?
      - check group access with session
      - public groups
    */

    try {
        message = JSON.parse(message);
    } catch (error) {
        return engine.log('E', error);
    }

    // extract data from parsed message
    var type = message[0];
    var instance = message[1];
    var event = message[2];
    var id = message[3];
    var err = message[4];
    var data = message[5];

    // check mandatory message values
    if (!event || !instance || !id) {

        // send error back
        message[4] = 'Invalid message.';
        message[5] = 0;
        return socket.send(JSON.stringify(message));
    }

    // get the link object
    var stream = streams[id];

    // write data to stream on write message
    if (type) {

        // create a new stream and emit to flow handlers
        if (!stream) {

            // get module instance
            if (!(instance = engine.instances[instance])) {

                // load an instance
                engine.load(message[1], role, function (err, instance) {

                    if (err) {

                        // send error back
                        message[4] = err;
                        message[5] = 0;
                        return socket.send(JSON.stringify(message));
                    }

                    // create a new stream and save it in cache and write data
                    stream = createStream(instance, event, id, socket, message[1]);
                    stream.write(err, data);
                });
                return;

            // check role and event access on server module instance
            } else if (!(utils.eventAccess(instance, role, event))) {

                // send error back
                message[4] = 'Access denied to instance: ' + message[1];
                message[5] = 0;
                return socket.send(JSON.stringify(message));
            }

            // create a new stream and save it in cache and write data
            if (instance._name === 'custom') {
                //console.log('WS MSG NO STREAM:', event, id, instance._access || '@');
                debugger;
            }
            stream = createStream(instance, event, id, socket);
        }

        stream.write(err, data);

    // end stream on end message
    } else if (stream) {
        stream.end(err, data);
    }
}

function createStream (instance, event, id, socket) {

    var stream = instance.flow(event, {
        id: id,
        socket: socket
    });

    // create stream and save it to cache
    sockets[id] = setupStream(stream, {
        inst: instance._name || '@',
        emit: event
    });

    return sockets[id];
}

/**
 * Write bufferd data when a socket is connected.
 *
 * @public
 * @param {object} The open event object.
 */
function openHandler (openEvent) {
    var socket = openEvent.target;

    // send buffered writes to socket
    if (socket._buffer.length) {
        for (var i = 0, l = socket._buffer.length; i < l; ++i) {
            socket.send(socket._buffer[i]);
        }

        // empty buffer
        socket._buffer = [];
    }
}

/**
 * Remove socket from cache and end streams.
 *
 * @public
 * @param {object} The close event object.
 */
function closeHandler (closeEvent) {
    var socket = closeEvent.target;

    for (var stream in socket.streams) {

        // end all streams on the server
        if (!engine.client) {
            socket.streams[stream].end();

        // remove socket ref on the stream
        } else {
            delete socket.streams[stream].socket;
        }
    }

    // remove socket from sockets cache
    if (sockets[socket._cacheKey] && sockets[socket._cacheKey].readyState > 0) {
      delete sockets[socket._cacheKey];
    }
}

/**
 * Write socket errors to streams.
 *
 * @public
 * @param {object} The error event object.
 */
function errorHandler (errorEvent) {
    var socket = errorEvent.target;
    for (var stream in socket.streams) {
        stream.write(errorEvent.data);
    }
}
<|endoftext|>


<commit_before>var fs = require('fs');
var path = require('path');
var SVGO = require('svgo');
var dot = require('dot');

// Initialize what will be used for automatic text measurement.
var Canvas = require('canvas');
var canvasElement = new Canvas(0, 0);   // Width and height are irrelevant.
var canvasContext = canvasElement.getContext('2d');
var CanvasFont = Canvas.Font;
try {
  var opensans = new CanvasFont('Verdana',
      path.join(__dirname, 'Verdana.ttf'));
  canvasContext.addFont(opensans);
} catch(e) {}
canvasContext.font = '11px Verdana, "DejaVu Sans"';

// cache templates.
var templates = {};
var templateFiles = fs.readdirSync(path.join(__dirname, 'templates'));
templateFiles.forEach(function(filename) {
  var templateData = fs.readFileSync(
    path.join(__dirname, 'templates', filename)).toString();
  var extension = filename.split('.').pop();
  var style = filename.slice(0, -(('-template.' + extension).length));
  templates[style + '-' + extension] = dot.template(templateData);
});

var colorscheme = require(path.join(__dirname, 'colorscheme.json'));

function optimize(string, callback) {
  var svgo = new SVGO();
  svgo.optimize(string, callback);
}

function makeImage(data, cb) {
  if (data.format !== 'json') {
    data.format = 'svg';
  }
  if (!(data.template + '-' + data.format in templates)) {
    data.template = 'default';
  }
  var template = templates[data.template + '-' + data.format];
  if (data.colorscheme) {
    data.colorA = colorscheme[data.colorscheme].colorA;
    data.colorB = colorscheme[data.colorscheme].colorB;
  }
  data.widths = [
    (canvasContext.measureText(data.text[0]).width|0) + 10,
    (canvasContext.measureText(data.text[1]).width|0) + 10,
  ];
  var result = template(data);
  if (data.format === 'json') {
    cb(result);
  } else {
    // Run the SVG through SVGO.
    optimize(result, function(object) { cb(object.data); });
  }
}

module.exports = makeImage;
<commit_msg>lib: preserve whitespace in templates<commit_after>var fs = require('fs');
var path = require('path');
var SVGO = require('svgo');
var dot = require('dot');

// Initialize what will be used for automatic text measurement.
var Canvas = require('canvas');
var canvasElement = new Canvas(0, 0);   // Width and height are irrelevant.
var canvasContext = canvasElement.getContext('2d');
var CanvasFont = Canvas.Font;
try {
  var opensans = new CanvasFont('Verdana',
      path.join(__dirname, 'Verdana.ttf'));
  canvasContext.addFont(opensans);
} catch(e) {}
canvasContext.font = '11px Verdana, "DejaVu Sans"';

// cache templates.
var templates = {};
var templateFiles = fs.readdirSync(path.join(__dirname, 'templates'));
dot.templateSettings.strip = false;  // Do not strip whitespace.
templateFiles.forEach(function(filename) {
  var templateData = fs.readFileSync(
    path.join(__dirname, 'templates', filename)).toString();
  var extension = filename.split('.').pop();
  var style = filename.slice(0, -(('-template.' + extension).length));
  templates[style + '-' + extension] = dot.template(templateData);
});

var colorscheme = require(path.join(__dirname, 'colorscheme.json'));

function optimize(string, callback) {
  var svgo = new SVGO();
  svgo.optimize(string, callback);
}

function makeImage(data, cb) {
  if (data.format !== 'json') {
    data.format = 'svg';
  }
  if (!(data.template + '-' + data.format in templates)) {
    data.template = 'default';
  }
  var template = templates[data.template + '-' + data.format];
  if (data.colorscheme) {
    data.colorA = colorscheme[data.colorscheme].colorA;
    data.colorB = colorscheme[data.colorscheme].colorB;
  }
  data.widths = [
    (canvasContext.measureText(data.text[0]).width|0) + 10,
    (canvasContext.measureText(data.text[1]).width|0) + 10,
  ];
  var result = template(data);
  if (data.format === 'json') {
    cb(result);
  } else {
    // Run the SVG through SVGO.
    optimize(result, function(object) { cb(object.data); });
  }
}

module.exports = makeImage;
<|endoftext|>


<commit_before>var Logger = require('./Logger');
var debug = Logger.debug;
var verbose = Logger.verbose;
var error = Logger.error;
var info = Logger.info;

/**
 * Register Cluster Events
 *
 * @module
 *
 * @author Kevin Smithson <ksmithson@sazze.com>
 * @author Craig Thayer <cthayer@sazze.com>
 *
 * @param cluster
 */
module.exports = function (cluster) {
  "use strict";

  var forkTimers = {};
  var lifecycleTimers = {};
  var disconnectTimers = {};
  var suicideOverrides = {};

  var config = require('./config');

  // when the master forks a worker
  cluster.on('fork', function (worker) {
    verbose('worker %d forked', worker.process.pid);
    forkTimers[worker.id] = setTimeout(function () {
      forkTimeoutHandler(worker, cluster);
    }, config.timeouts.start);
  });

  // when a worker starts executing
  cluster.on('online', function (worker) {
    verbose('worker %d online', worker.process.pid);

    if (cluster.count == require('./config').n) {
      debug('%d workers are now online', config.n);
    }

    // Create a splay for the lifecycle timers
    var splay = Math.floor((Math.random() * (config.timeouts.stop + config.timeouts.start)) + config.timeouts.stop);
    var maxAge = config.timeouts.maxAge + splay;

    lifecycleTimers[worker.id] = setTimeout(function () {
      lifecycleTimeoutHandler(worker, cluster);
    }, maxAge);

    debug('Setting %d lifecycle timer to %d', worker.process.pid, maxAge);
  });

  // when a worker starts listening
  cluster.on('listening', function (worker, address) {
    verbose('worker %d listening on %s:%d', worker.process.pid, address.address, address.port);

    if (cluster.count == config.n) {
      debug('%d workers are now listening', config.n);
    }

    if (forkTimers[worker.id]) {
      clearTimeout(forkTimers[worker.id]);
      delete forkTimers[worker.id];
    }
  });

  // when a worker disconnects
  cluster.on('disconnect', function (worker) {
    verbose('worker %d disconnect', worker.process.pid);

    if (lifecycleTimers[worker.id]) {
      clearTimeout(lifecycleTimers[worker.id]);
      delete lifecycleTimers[worker.id];
    }

    cluster.isRunning(worker, function(running) {
      if (running) {
        disconnectTimers[worker.id] = setTimeout(function () {
          disconnectTimeoutHandler(worker, cluster);
        }, config.timeouts.stop);
      }
    });
  });

  // when a worker exits
  cluster.on('exit', function (worker, code, signal) {
    verbose('worker %d exit', worker.process.pid);

    if (lifecycleTimers[worker.id]) {
      clearTimeout(lifecycleTimers[worker.id]);
      delete lifecycleTimers[worker.id];
    }

    if (disconnectTimers[worker.id]) {
      clearTimeout(disconnectTimers[worker.id]);
      delete disconnectTimers[worker.id];
    }

    if (worker.suicide) {
      debug('worker %d committed suicide', worker.process.pid);
    }

    debug('worker %d Exit Code: %d', worker.process.pid, code);

    if (signal) {
      debug('worker %d Exit Signal: %s', worker.process.pid, signal);
    }

    if (worker.suicide && suicideOverrides[worker.id]) {
      worker.suicide = false;
      delete suicideOverrides[worker.id];
    }

    if (!worker.suicide) {
      info('worker %d exited.  Restarting....', worker.process.pid);
      cluster.fork();
    }
  });

  /**
   * Called when worker takes too long to fork
   *
   * @private
   * @param worker
   * @param cluster
   */
  function forkTimeoutHandler(worker, cluster) {
    verbose('worker %d is stuck in fork', worker.process.pid);

    if (!cluster.workers[worker.id]) {
      return;
    }

    // something is wrong with this worker.  Kill it!
    suicideOverrides[worker.id] = worker.id;

    cluster.isRunning(worker, function(running) {
      if (!running) {
        return;
      }

      try {
        debug('sending SIGKILL to worker %d', worker.process.pid);
        process.kill(worker.process.pid, 'SIGKILL');
      } catch (e) {
        // this can happen.  don't crash!!
      }
    });
  }

  /**
   * Called when worker has reached the end of it's lifespan
   *
   * @private
   * @param worker
   * @param cluster
   */
  function lifecycleTimeoutHandler(worker, cluster) {
    verbose('worker %d has reached the end of it\'s life', worker.process.pid);

    if (!cluster.workers[worker.id]) {
      return;
    }

    suicideOverrides[worker.id] = worker.id;

    // notify the worker that we're shutting it down
    worker.send('shutdown');

    worker.disconnect();
  }

  /**
   * Called when worker is taking too long to disconnect
   *
   * @private
   * @param worker
   * @param cluster
   */
  function disconnectTimeoutHandler(worker, cluster) {
    verbose('worker %d is stuck in disconnect', worker.process.pid);

    // if this is not a suicide we need to preserve that as worker.kill() will make this a suicide
    if (!worker.suicide) {
      suicideOverrides[worker.id] = worker.id;
    }

    cluster.isRunning(worker, function(running) {
      if (!running) {
        return;
      }

      try {
        debug('sending SIGKILL to worker %d', worker.process.pid);
        process.kill(worker.process.pid, 'SIGKILL');
      } catch (e) {
        // this can happen.  don't crash!!
      }
    });
  }
};<commit_msg>clear fork timers in "online" event instead of listening<commit_after>var Logger = require('./Logger');
var debug = Logger.debug;
var verbose = Logger.verbose;
var error = Logger.error;
var info = Logger.info;

/**
 * Register Cluster Events
 *
 * @module
 *
 * @author Kevin Smithson <ksmithson@sazze.com>
 * @author Craig Thayer <cthayer@sazze.com>
 *
 * @param cluster
 */
module.exports = function (cluster) {
  "use strict";

  var forkTimers = {};
  var lifecycleTimers = {};
  var disconnectTimers = {};
  var suicideOverrides = {};

  var config = require('./config');

  // when the master forks a worker
  cluster.on('fork', function (worker) {
    verbose('worker %d forked', worker.process.pid);
    forkTimers[worker.id] = setTimeout(function () {
      forkTimeoutHandler(worker, cluster);
    }, config.timeouts.start);
  });

  // when a worker starts executing
  cluster.on('online', function (worker) {
    verbose('worker %d online', worker.process.pid);

    if (cluster.count == require('./config').n) {
      debug('%d workers are now online', config.n);
    }

    if (forkTimers[worker.id]) {
      clearTimeout(forkTimers[worker.id]);
      delete forkTimers[worker.id];
    }

    // Create a splay for the lifecycle timers
    var splay = Math.floor((Math.random() * (config.timeouts.stop + config.timeouts.start)) + config.timeouts.stop);
    var maxAge = config.timeouts.maxAge + splay;

    lifecycleTimers[worker.id] = setTimeout(function () {
      lifecycleTimeoutHandler(worker, cluster);
    }, maxAge);

    debug('Setting %d lifecycle timer to %d', worker.process.pid, maxAge);
  });

  // when a worker starts listening
  cluster.on('listening', function (worker, address) {
    verbose('worker %d listening on %s:%d', worker.process.pid, address.address, address.port);

    if (cluster.count == config.n) {
      debug('%d workers are now listening', config.n);
    }
  });

  // when a worker disconnects
  cluster.on('disconnect', function (worker) {
    verbose('worker %d disconnect', worker.process.pid);

    if (lifecycleTimers[worker.id]) {
      clearTimeout(lifecycleTimers[worker.id]);
      delete lifecycleTimers[worker.id];
    }

    cluster.isRunning(worker, function(running) {
      if (running) {
        disconnectTimers[worker.id] = setTimeout(function () {
          disconnectTimeoutHandler(worker, cluster);
        }, config.timeouts.stop);
      }
    });
  });

  // when a worker exits
  cluster.on('exit', function (worker, code, signal) {
    verbose('worker %d exit', worker.process.pid);

    if (lifecycleTimers[worker.id]) {
      clearTimeout(lifecycleTimers[worker.id]);
      delete lifecycleTimers[worker.id];
    }

    if (disconnectTimers[worker.id]) {
      clearTimeout(disconnectTimers[worker.id]);
      delete disconnectTimers[worker.id];
    }

    if (worker.suicide) {
      debug('worker %d committed suicide', worker.process.pid);
    }

    debug('worker %d Exit Code: %d', worker.process.pid, code);

    if (signal) {
      debug('worker %d Exit Signal: %s', worker.process.pid, signal);
    }

    if (worker.suicide && suicideOverrides[worker.id]) {
      worker.suicide = false;
      delete suicideOverrides[worker.id];
    }

    if (!worker.suicide) {
      info('worker %d exited.  Restarting....', worker.process.pid);
      cluster.fork();
    }
  });

  /**
   * Called when worker takes too long to fork
   *
   * @private
   * @param worker
   * @param cluster
   */
  function forkTimeoutHandler(worker, cluster) {
    verbose('worker %d is stuck in fork', worker.process.pid);

    if (!cluster.workers[worker.id]) {
      return;
    }

    // something is wrong with this worker.  Kill it!
    suicideOverrides[worker.id] = worker.id;

    cluster.isRunning(worker, function(running) {
      if (!running) {
        return;
      }

      try {
        debug('sending SIGKILL to worker %d', worker.process.pid);
        process.kill(worker.process.pid, 'SIGKILL');
      } catch (e) {
        // this can happen.  don't crash!!
      }
    });
  }

  /**
   * Called when worker has reached the end of it's lifespan
   *
   * @private
   * @param worker
   * @param cluster
   */
  function lifecycleTimeoutHandler(worker, cluster) {
    verbose('worker %d has reached the end of it\'s life', worker.process.pid);

    if (!cluster.workers[worker.id]) {
      return;
    }

    suicideOverrides[worker.id] = worker.id;

    // notify the worker that we're shutting it down
    worker.send('shutdown');

    worker.disconnect();
  }

  /**
   * Called when worker is taking too long to disconnect
   *
   * @private
   * @param worker
   * @param cluster
   */
  function disconnectTimeoutHandler(worker, cluster) {
    verbose('worker %d is stuck in disconnect', worker.process.pid);

    // if this is not a suicide we need to preserve that as worker.kill() will make this a suicide
    if (!worker.suicide) {
      suicideOverrides[worker.id] = worker.id;
    }

    cluster.isRunning(worker, function(running) {
      if (!running) {
        return;
      }

      try {
        debug('sending SIGKILL to worker %d', worker.process.pid);
        process.kill(worker.process.pid, 'SIGKILL');
      } catch (e) {
        // this can happen.  don't crash!!
      }
    });
  }
};<|endoftext|>