<commit_before>#!/usr/bin/env node

var http = require('http'),
    fs = require('fs'),
    path = require('path'),
    utile = require('utile'),
    libxml = require('libxmljs'),
    Q = require('q');

var stream = null;
var json_arr = [];
var srw_config = { // TODO: limit options
    start: 1,
    limit: 100
};

// args
var argv = require('minimist')(process.argv.slice(2), {
    string: ['q, O, d, f, h'],
    boolean: ['s', 'i', 'a', 'p'],
    alias: {
        'q': 'query',
        'O': 'output',
        'd': 'dir',
        'f': 'format',
        'h': 'host',
        's': 'stream',
        'i': 'include-all',
        'a': 'include-anno',
        'p': 'validate-pids',
    },
    default: {
        'd': 'output',
        'O': 'output',
        'f': 'csv',
        'h': 'devtools.clarin.dk'
    }
});

var delimiter = (argv.f == 'tsv') ? '\t' : '%'; // TODO: custom delimiter
var validate = argv.p;

//console.log('include all: ' + argv.i);
//console.log('include anno: ' + argv.a);
//console.log('validate: ' + validate);

var annotationsOnly = (!argv.i && argv.a);

// eSciDoc 1.3.x SRW Namespaces
var ns_obj = {
    'sru-zr': 'http://www.loc.gov/zing/srw/',
    'escidocItem': 'http://www.escidoc.de/schemas/item/0.10',
    'escidocMetadataRecords': 'http://www.escidoc.de/schemas/metadatarecords/0.5',
    'container': 'http://www.escidoc.de/schemas/container/0.9',
    'escidocContentStreams': 'http://www.escidoc.de/schemas/contentstreams/0.7',
    'escidocComponents': 'http://www.escidoc.de/schemas/components/0.9',
    'version': 'http://escidoc.de/core/01/properties/version/',
    'release': 'http://escidoc.de/core/01/properties/release/',
    'prop': 'http://escidoc.de/core/01/properties/',
    'srel': 'http://escidoc.de/core/01/structural-relations/',
    'relations': 'http://www.escidoc.de/schemas/relations/0.3',
    'xlink': 'http://www.w3.org/1999/xlink'
};

var getProperties = function(obj, name, callback) {
    var props = {}; // Properties object
    props.parent = obj.parent();
    props.name = (name == null) ? obj.name() : name;

    var xpathRoot = (obj.parent() != null) ? '//' : '*/';

    var escidocID_href = obj.attr('href').value();
    props.escidocID = escidocID_href.substring(escidocID_href.indexOf('dkclarin'), escidocID_href.length);

    if (props.escidocID.indexOf('properties') != -1)
        props.escidocID = props.escidocID.substring(0, props.escidocID.indexOf('properties') - 1);

    var contentModelID_href = obj.get(xpathRoot + 'srel:content-model', ns_obj).attr('href').value();
    props.contentModelID = contentModelID_href.substring(contentModelID_href.indexOf('dkclarin'), contentModelID_href.length);

    var obj_pid = obj.get(xpathRoot + 'prop:pid', ns_obj);
    var ver_pid = obj.get(xpathRoot + 'prop:version/version:pid', ns_obj);
    props.ver_no = obj.get(xpathRoot + 'prop:latest-version/version:number', ns_obj).text();

    var last_date = obj.attr('last-modification-date').value();

    props.pid = "none";

    if (ver_pid != null)
        props.pid = ver_pid.text();
    else if (obj_pid != null)
        props.pid = obj_pid.text();
    else
        console.error('No PID value found for ' + props.name + ': ' + props.escidocID);

    if (validate) {
        if (validatePIDVersion(props)) { // PID must be in format of version PID
            console.log('validated found Handle PID: ' + props.pid); // add to member
            return props;
        } else
            throw new Error('Invalid PID handle: ' + props.pid);
    } else
        return props;
}

// Validate HDL PIDs against the version no ref (CLARIN-DK based PID implementaton)
var validatePIDVersion = function(props, callback) {
    if (props.pid.indexOf('hdl:') == 0) {
        var sProps = props.pid.split('-');
        if (sProps.length < 5)
            return true; // object PID
        else
            return (parseInt(sProps[sProps.length - 1], 16) == props.ver_no);
    } else
        return true;
}

// Parse and pull ID data from the XMLDocument
var parse = function(doc) {
    var promises = [];
    var totalRecords = doc.get('//sru-zr:numberOfRecords', ns_obj).text();
    var items = doc.find('//escidocItem:item', ns_obj);
    var containers = doc.find('//container:container', ns_obj);

    console.log('Total records in query: ' + totalRecords);

    if (utile.isArray(items))
        console.log('Found ' + items.length + ' items.');
    else
        console.error('Found 0 items in XMLDocument.');

    if (utile.isArray(containers))
        console.log('Found ' + containers.length + ' containers.');
    else
        console.error('Found 0 containers in XMLDocument.');

    //Add containers to the list
    containers.forEach(function(container) {
        return Q.when(getProperties(container, container.name()), function(props) {
            console.log('retrieved container props data', props.escidocID);
            addMember(props);
        });
    });

    // Add the items to list
    // CSV format: item%{escidocID}%[{objectPID}}|{lastVersionPID}]%{versionNo}
    items.forEach(function(item) {
        if (!annotationsOnly) {
            Q.when(getProperties(item, item.name()), function(props) {
                console.log('retrieved item props data', props.escidocID);
                return addMember(props);
            });
        }

        if (argv.a && item != null) {
            var relations = item.find('relations:relations/relations:relation', ns_obj);

            relations.forEach(function(relation) {
                if (relation != undefined || relation != null) {
                    var relation_type = relation.attr('predicate').value();
                    var relation_href = relation.attr('href').value();
                    var relationObjID = relation_href.substring(relation_href.indexOf('dkclarin'), relation.length);
                    var promise;

                    if (relation_type.indexOf('HasAnnotation') != -1) {
                        // retrieve properties for Annotation item from eSciDoc REST
                        console.log('Found Annotation: ' + relationObjID);

                        promise = retrieveItemProperties(relationObjID).then(function(annoPropsItem) {
                            var props = getProperties(annoPropsItem.root(), "annotation");
                            console.log('retrieved annotation props data', props.escidocID);
                            //console.log('props xml: ' + annoPropsItem.toString());
                            return props;
                        }).then(function(props) {
                            return addMember(props); // add annotation member to file
                        });

                    } else if (relation_type.indexOf('IsAnnotationOf') != -1 ||
                        relation_type.indexOf('isDependentOf') != -1 ||
                        relation_type.indexOf('hasDependent') != -1) { // add annotation member to file including parent ID (annotation)

                        promise = retrieveItemProperties(relationObjID).then(function(annoPropsItem) {
                            var props = getProperties(annoPropsItem.root(), "annotation");
                            console.log('retrieved annotation props data', props.escidocID);
                            return props;
                        }).then(function(propsAnno) {
                            var props = getProperties(item, "item");
                            console.log('retrieved item props data', props.escidocID);
                            return props;
                        }).then(function(props) {
                            return addMember(propsAnno, props.escidocID);
                        });
                    }
                }

                if (promise)
                    promises.push(promise);
            });
        }
    });

    return Q.allSettled(promises).done(function() {
        // iterate over complete SRW result set
        if (Number(totalRecords) >= (srw_config.start + srw_config.limit)) {
            srw_config.start += srw_config.limit;
            return retrieveSRWResult(srw_config)
                .then(function(result) {
                    parse(result);
                });
        } else if (argv.f == 'json') {
            return addMemberToFile(JSON.stringify(json_arr)); // write JSON object to output file
        }
    });
}

/** SRW Query fetch */
var retrieveSRWResult = function(config) {
    return execGET(srw_options(argv.q, config));
}

/* Item Properties */
var retrieveItemProperties = function(escidocID) {
    return execGET(ir_options(escidocID, '/properties'));
}

var execGET = function(options) {
    var deferred = Q.defer();

    http.get(options, function(res) {
        var str = "";
        res.on('data', function(chunk) {
            str += chunk;
        });
        res.on('end', function() {
            if (res.statusCode == "200")
                deferred.resolve(libxml.parseXmlString(str));
            else
                throw new Error("Error code: " + res.statusCode);
        });
    }).on('error', function(e) {
        deferred.reject(e);
    });

    return deferred.promise;
}

/** SRW query options */
var srw_options = function(queryCQL, record_vals) {
    var search_path_limits = "&maximumRecords=" + record_vals.limit + "&startRecord=" + record_vals.start;
    var targetUrl = argv.h;
    var query = (queryCQL != null && queryCQL.length > 0) ? encodeURIComponent(queryCQL.replace(/\s/g, "%20")) + search_path_limits : '';
    query += '&d=' + Date.now();

    return {
        host: (targetUrl.indexOf('http://') != -1) ? targetUrl.replace('http://', '') : targetUrl,
        path: '/srw/search/escidoc_all?query=' + query
    };
};

/* eSciDoc Item REST */
var ir_options = function(escidocID, path) {
    var targetUrl = argv.h;
    return {
        host: (targetUrl.indexOf('http://') != -1) ? targetUrl.replace('http://', '') : targetUrl,
        path: '/ir/item/' + escidocID + path
    };
};

// append props to output file
var addMember = function(props, parentID) {
    if (argv.f == 'csv' || argv.f == 'tsv') {
        var dataArr = [props.name, props.escidocID, props.contentModelID, props.pid, props.ver_no];
        if (parentID != null)
            dataArr.push(parentID);

        addMemberToFile(dataArr.join(delimiter) + '\n');
    } else if (argv.f == 'json') // write output to JSON object
        json_arr.push({
            type: props.name,
            systemID: props.escidocID,
            contentModelID: props.contentModelID,
            PID: props.pid,
            versionNo: props.ver_no,
            parentID: (parentID != null) ? parentID : undefined
        });
    else
        throw new Error('Unsupported format:' + argv.f);
}

// write output to file
var addMemberToFile = function(data) {
    if (stream != null)
        if (stream.write(data))
            console.log('Data written: ' + data);
        else
            console.log('Buffer full waiting for drain..');
    else
        process.send({
            data: data
        });
}

var createStream = function() {
    var is_json = (argv.f == 'json');
    var is_csv = (argv.f == 'csv');
    var is_tsv = (argv.f == 'tsv');

    var file_name = argv.O;
    var file_dir = argv.d;
    var file_path = null;

    if (!argv.s) {
        if (is_json)
            file_path = path.join(file_dir, file_name + '.json');
        else if (is_csv)
            file_path = path.join(file_dir, file_name + '.csv');
        else if (is_tsv)
            file_path = path.join(file_dir, file_name + '.tsv');

        if (stream == null) {
            var deferred = Q.defer();

            stream = fs.createWriteStream(file_path, {
                encoding: 'utf8'
            });

            stream.on('open', function(fd) {
                console.log('opened file: ' + file_path);
                deferred.resolve(file_path);
            });

            stream.on('error', function(err) {
                deferred.reject(err);
            });

            stream.on('drain', function() {
                console.log('Data drained from buffer to file:' + file_path);
                stream.end();
            });

            stream.on('close', function() {
                console.log('Closing file stream.');
            });

            return deferred.promise;
        }
    }
}

var createDir = function(file_dir) {
    if (!fs.existsSync(file_dir)) {
        console.log('Target directory doesn\'t exist: ' + file_dir);
        return Q.nfcall(fs.mkdir, file_dir);
    }
}

module.exports = function() {
    return Q.all([createDir(argv.d), createStream()])
        .then(function() {
            console.log('Using query: ' + argv.q);
            return retrieveSRWResult(srw_config);
        }).then(function(result) {
            return parse(result);
        }).catch(function(err) {
            console.error(err);
        }).done();
}();
<commit_msg>Update xpath for object properties.<commit_after>#!/usr/bin/env node

var http = require('http'),
    fs = require('fs'),
    path = require('path'),
    utile = require('utile'),
    libxml = require('libxmljs'),
    Q = require('q');

var stream = null;
var json_arr = [];
var srw_config = { // TODO: limit options
    start: 1,
    limit: 100
};

// args
var argv = require('minimist')(process.argv.slice(2), {
    string: ['q, O, d, f, h'],
    boolean: ['s', 'i', 'a', 'p'],
    alias: {
        'q': 'query',
        'O': 'output',
        'd': 'dir',
        'f': 'format',
        'h': 'host',
        's': 'stream',
        'i': 'include-all',
        'a': 'include-anno',
        'p': 'validate-pids',
    },
    default: {
        'd': 'output',
        'O': 'output',
        'f': 'csv',
        'h': 'devtools.clarin.dk'
    }
});

var delimiter = (argv.f == 'tsv') ? '\t' : '%'; // TODO: custom delimiter
var validate = argv.p;

//console.log('include all: ' + argv.i);
//console.log('include anno: ' + argv.a);
//console.log('validate: ' + validate);

var annotationsOnly = (!argv.i && argv.a);

// eSciDoc 1.3.x SRW Namespaces
var ns_obj = {
    'sru-zr': 'http://www.loc.gov/zing/srw/',
    'escidocItem': 'http://www.escidoc.de/schemas/item/0.10',
    'escidocMetadataRecords': 'http://www.escidoc.de/schemas/metadatarecords/0.5',
    'container': 'http://www.escidoc.de/schemas/container/0.9',
    'escidocContentStreams': 'http://www.escidoc.de/schemas/contentstreams/0.7',
    'escidocComponents': 'http://www.escidoc.de/schemas/components/0.9',
    'version': 'http://escidoc.de/core/01/properties/version/',
    'release': 'http://escidoc.de/core/01/properties/release/',
    'prop': 'http://escidoc.de/core/01/properties/',
    'srel': 'http://escidoc.de/core/01/structural-relations/',
    'relations': 'http://www.escidoc.de/schemas/relations/0.3',
    'xlink': 'http://www.w3.org/1999/xlink'
};

var getProperties = function(obj, name, callback) {
    var props = {}; // Properties object
    props.parent = obj.parent();
    props.name = (name == null) ? obj.name() : name;

    var xpathRoot = '//';
    if (props.name == 'item' || props.name == 'annotation') xpathRoot = 'escidocItem:properties/'
    else if (props.name == 'container') xpathRoot = 'container:properties/';

    var escidocID_href = obj.attr('href').value();
    props.escidocID = escidocID_href.substring(escidocID_href.indexOf('dkclarin'), escidocID_href.length);

    if (props.escidocID.indexOf('properties') != -1)
        props.escidocID = props.escidocID.substring(0, props.escidocID.indexOf('properties') - 1);

    var contentModelID_href = obj.get(xpathRoot + 'srel:content-model', ns_obj).attr('href').value();
    props.contentModelID = contentModelID_href.substring(contentModelID_href.indexOf('dkclarin'), contentModelID_href.length);

    console.log('xpathRoot: ', xpathRoot);

    var obj_pid = obj.get(xpathRoot + 'prop:pid', ns_obj);
    var ver_pid = obj.get(xpathRoot + 'prop:version/version:pid', ns_obj);
    props.ver_no = obj.get(xpathRoot + 'prop:latest-version/version:number', ns_obj).text();

    var last_date = obj.attr('last-modification-date').value();

    props.pid = "none";

    if (ver_pid != null)
        props.pid = ver_pid.text();
    else if (obj_pid != null)
        props.pid = obj_pid.text();
    else
        console.error('No PID value found for ' + props.name + ': ' + props.escidocID);

    if (validate) {
        if (validatePIDVersion(props)) { // PID must be in format of version PID
            console.log('validated found Handle PID: ' + props.pid); // add to member
            return props;
        } else
            throw new Error('Invalid PID handle: ' + props.pid);
    } else
        return props;
}

// Validate HDL PIDs against the version no ref (CLARIN-DK based PID implementaton)
var validatePIDVersion = function(props, callback) {
    if (props.pid.indexOf('hdl:') == 0) {
        var sProps = props.pid.split('-');
        if (sProps.length < 5)
            return true; // object PID
        else
            return (parseInt(sProps[sProps.length - 1], 16) == props.ver_no);
    } else
        return true;
}

// Parse and pull ID data from the XMLDocument
var parse = function(doc) {
    var promises = [];
    var totalRecords = doc.get('//sru-zr:numberOfRecords', ns_obj).text();
    var items = doc.find('//escidocItem:item', ns_obj);
    var containers = doc.find('//container:container', ns_obj);

    console.log('Total records in query: ' + totalRecords);

    if (utile.isArray(items))
        console.log('Found ' + items.length + ' items.');
    else
        console.error('Found 0 items in XMLDocument.');

    if (utile.isArray(containers))
        console.log('Found ' + containers.length + ' containers.');
    else
        console.error('Found 0 containers in XMLDocument.');

    //Add containers to the list
    for (var i = 0; i < containers.length; i++) {
        var container = containers[i];
        return Q.when(getProperties(container, container.name()), function(props) {
            console.log('retrieved container props data', props.escidocID);
            addMember(props);
        });
    }

    // Add the items to list
    // CSV format: item%{escidocID}%[{objectPID}}|{lastVersionPID}]%{versionNo}
    for (var i = 0; i < items.length; i++) {
        var item = items[i];
        if (!annotationsOnly) {
            Q.when(getProperties(item, item.name()), function(props) {
                console.log('retrieved item props data', props.escidocID);
                return addMember(props);
            });
        }

        if (argv.a && item != null) {
            var relations = item.find('relations:relations/relations:relation', ns_obj);

            relations.forEach(function(relation) {
                if (relation != undefined || relation != null) {
                    var relation_type = relation.attr('predicate').value();
                    var relation_href = relation.attr('href').value();
                    var relationObjID = relation_href.substring(relation_href.indexOf('dkclarin'), relation.length);
                    var promise;

                    if (relation_type.indexOf('HasAnnotation') != -1) {
                        // retrieve properties for Annotation item from eSciDoc REST
                        console.log('Found Annotation: ' + relationObjID);

                        promise = retrieveItemProperties(relationObjID).then(function(annoPropsItem) {
                            var props = getProperties(annoPropsItem.root(), "annotation");
                            console.log('retrieved annotation props data', props.escidocID);
                            //console.log('props xml: ' + annoPropsItem.toString());
                            return props;
                        }).then(function(props) {
                            return addMember(props); // add annotation member to file
                        });

                    } else if (relation_type.indexOf('IsAnnotationOf') != -1 ||
                        relation_type.indexOf('isDependentOf') != -1 ||
                        relation_type.indexOf('hasDependent') != -1) { // add annotation member to file including parent ID (annotation)

                        promise = retrieveItemProperties(relationObjID).then(function(annoPropsItem) {
                            var props = getProperties(annoPropsItem.root(), "annotation");
                            console.log('retrieved annotation props data', props.escidocID);
                            return props;
                        }).then(function(propsAnno) {
                            var props = getProperties(item, "item");
                            console.log('retrieved item props data', props.escidocID);
                            return props;
                        }).then(function(props) {
                            return addMember(propsAnno, props.escidocID);
                        });
                    }
                }

                if (promise)
                    promises.push(promise);
            });
        }
    }

    return Q.allSettled(promises).done(function() {
        // iterate over complete SRW result set
        if (Number(totalRecords) >= (srw_config.start + srw_config.limit)) {
            srw_config.start += srw_config.limit;
            return retrieveSRWResult(srw_config)
                .then(function(result) {
                    parse(result);
                });
        } else if (argv.f == 'json') {
            return addMemberToFile(JSON.stringify(json_arr)); // write JSON object to output file
        }
    });
}

/** SRW Query fetch */
var retrieveSRWResult = function(config) {
    return execGET(srw_options(argv.q, config));
}

/* Item Properties */
var retrieveItemProperties = function(escidocID) {
    return execGET(ir_options(escidocID, '/properties'));
}

var execGET = function(options) {
    var deferred = Q.defer();

    http.get(options, function(res) {
        var str = "";
        res.on('data', function(chunk) {
            str += chunk;
        });
        res.on('end', function() {
            if (res.statusCode == "200")
                deferred.resolve(libxml.parseXmlString(str));
            else
                throw new Error("Error code: " + res.statusCode);
        });
    }).on('error', function(e) {
        deferred.reject(e);
    });

    return deferred.promise;
}

/** SRW query options */
var srw_options = function(queryCQL, record_vals) {
    var search_path_limits = "&maximumRecords=" + record_vals.limit + "&startRecord=" + record_vals.start;
    var targetUrl = argv.h;
    var query = (queryCQL != null && queryCQL.length > 0) ? encodeURIComponent(queryCQL.replace(/\s/g, "%20")) + search_path_limits : '';
    query += '&d=' + Date.now();

    return {
        host: (targetUrl.indexOf('http://') != -1) ? targetUrl.replace('http://', '') : targetUrl,
        path: '/srw/search/escidoc_all?query=' + query
    };
};

/* eSciDoc Item REST */
var ir_options = function(escidocID, path) {
    var targetUrl = argv.h;
    return {
        host: (targetUrl.indexOf('http://') != -1) ? targetUrl.replace('http://', '') : targetUrl,
        path: '/ir/item/' + escidocID + path
    };
};

// append props to output file
var addMember = function(props, parentID) {
    if (argv.f == 'csv' || argv.f == 'tsv') {
        var dataArr = [props.name, props.escidocID, props.contentModelID, props.pid, props.ver_no];
        if (parentID != null)
            dataArr.push(parentID);

        addMemberToFile(dataArr.join(delimiter) + '\n');
    } else if (argv.f == 'json') // write output to JSON object
        json_arr.push({
            type: props.name,
            systemID: props.escidocID,
            contentModelID: props.contentModelID,
            PID: props.pid,
            versionNo: props.ver_no,
            parentID: (parentID != null) ? parentID : undefined
        });
    else
        throw new Error('Unsupported format:' + argv.f);
}

// write output to file
var addMemberToFile = function(data) {
    if (stream != null)
        if (stream.write(data))
            console.log('Data written: ' + data);
        else
            console.log('Buffer full waiting for drain..');
    else
        process.send({
            data: data
        });
}

var createStream = function() {
    var is_json = (argv.f == 'json');
    var is_csv = (argv.f == 'csv');
    var is_tsv = (argv.f == 'tsv');

    var file_name = argv.O;
    var file_dir = argv.d;
    var file_path = null;

    if (!argv.s) {
        if (is_json)
            file_path = path.join(file_dir, file_name + '.json');
        else if (is_csv)
            file_path = path.join(file_dir, file_name + '.csv');
        else if (is_tsv)
            file_path = path.join(file_dir, file_name + '.tsv');

        if (stream == null) {
            var deferred = Q.defer();

            stream = fs.createWriteStream(file_path, {
                encoding: 'utf8'
            });

            stream.on('open', function(fd) {
                console.log('opened file: ' + file_path);
                deferred.resolve(file_path);
            });

            stream.on('error', function(err) {
                deferred.reject(err);
            });

            stream.on('drain', function() {
                console.log('Data drained from buffer to file:' + file_path);
                stream.end();
            });

            stream.on('close', function() {
                console.log('Closing file stream.');
            });

            return deferred.promise;
        }
    }
}

var createDir = function(file_dir) {
    if (!fs.existsSync(file_dir)) {
        console.log('Target directory doesn\'t exist: ' + file_dir);
        return Q.nfcall(fs.mkdir, file_dir);
    }
}

module.exports = function() {
    return Q.all([createDir(argv.d), createStream()])
        .then(function() {
            console.log('Using query: ' + argv.q);
            return retrieveSRWResult(srw_config);
        }).then(function(result) {
            return parse(result);
        }).catch(function(err) {
            console.error(err);
        }).done();
}();
<|endoftext|>


<commit_before>/*
 * Stores page-specific preferences in the body, so it gets saved/loaded with other content
 */

export default class Preferences {

  static get defaults() {
    return  {
      GridSize: {default: 100, short: "grid size"},
      SnapSize: {default: 20, short: "snap size"},
      SnapPaddingSize: {default: 20, short: "padding while snapping size"},
      SnapWindowsInGrid: {default: false, short: "snap windows in grid"},
      ShowFixedBrowser: {default: false, short: "show fixed browser"},
      InteractiveLayer: {default: false, short: "dev methods"},
      ShowDocumentGrid: {default: true, short: "show grid"},
      UseRP19JSX: {default: false, short: "use rp19 implementation for jsx"},
      DisableAExpWorkspace: {default: false, short: "disable AExp in workspace"},
      UseProxiesForAExprs: {default: false, short: "proxy-based Active Expressions"},
      EnableAEDebugging: {default: true, short: "enable Active Expression debugging"},
      SmartAELogging: {default: true, short: "Only log events for interesting Active Expressions"},
      DisableAltGrab: {default: false, short: "disable alt grab with hand"},
      UseAsyncWorkspace: {default: false, short: "support await in eval"},
      OfflineFirst: {default: false, short: "use offline first swx cache"},
      GraffleMode: {default: false, short: "create shapes by drag and hold S/C/F/D"},
      FileIndex: {default: true, short: "local file index"},
      SWXKeepAlive: {default: false, short: "keep SWX alive"},
      LogBoot: {default: false, short: "keep bootlog"},
      PiTerminalURL: {default: "http://localhost:3000/", short: "url of pi terminal"},
      PiTerminalCWD: {default: "", short: "current working directory of pi terminal"},
      PiTerminalSecret: {default: "", short: "pi terminal credentials"},
      ExtraSearchRoots: {default: [], short: "extra search roots"},
      TipOfTheDay: {default: false, short: "show tip of the day on startup"},
      WiderIndentation: {default: false, short: "sets the indentation to 4"},
      CommandModeAsDefault: {default: false, short: "command mode as default in editor"},
      CircumventCodeMirrorModes: {default: false, short: "circumvent code mirror modes"},
      BabylonianProgramming: {default: false, short: "use babylonian programming editor"},
      TabbedWindows: {default: false, short: "use experimental window tabs"},
      SWEDebugging: {default: false, short: "bug showcase in swe lecture"},
      AEXPGraphExperimental: {default: false, short: "AExpr graph experimental"},
    }
  }

  static load() {
    console.info('Load preferences')
  }

  // List all avaiable preferences
  static list () {
    return Object
      .keys(this.defaults)
  }

  static listBooleans () {
    return Object.keys(this.defaults)
      .filter(ea => _.isBoolean(this.defaults[ea].default))
  }

  static shortDescription(preferenceKey) {
    var pref = this.defaults[preferenceKey]
    if (pref && pref.short)
      return pref.short
    else
      return preferenceKey
  }

  /* get preference, consider defaults */
  static get (preferenceKey) {
    if (!preferenceKey) {
      console.error('No preference key was specified')
      return
    }

    let pref = this.read(preferenceKey)
    if (typeof pref === 'string') {
      return JSON.parse(pref)
    }

    pref = this.defaults[preferenceKey]
    if (!pref) {
      console.error(`Preference "${preferenceKey}" does not exist`)
      return
    }

    if(pref.hasOwnProperty('default')) {
      return pref.default
    }
  }

  static set(preferenceKey, value) {
    var pref = this.write(preferenceKey, JSON.stringify(value))
  }

  static get prefsNode() {
    // #BUG: reloading Preferences causes dataset to be not defined anymore
    if (!(window && window.document)) return null;

    let node = document.body.querySelector('.lively-preferences');
    if (!node) {
      console.log("Create prefereces")
      node = document.createElement('div'); // we cannot use custom comps they are async
      node.classList.add("lively-preferences")
      node.classList.add("lively-content")
      document.body.insertBefore(node, document.body.firstChild);
    }
    return node
  }

  static read(preferenceKey) {
    return this.prefsNode && this.prefsNode.dataset ?
      this.prefsNode.dataset[preferenceKey] :
      undefined;
  }

  static write(preferenceKey, preferenceValue) {
    if(!this.prefsNode) { return; }
    if (!this.prefsNode.dataset) {
      this.prefsNode.setAttribute("data-foo", true); // force dataset
    }
    this.prefsNode.dataset[preferenceKey] = preferenceValue;
  }

  static enable(preferenceKey) {
    Preferences.write(preferenceKey, "true")
    this.applyPreference(preferenceKey)
  }

  static disable(preferenceKey) {
    Preferences.write(preferenceKey, "false")
    this.applyPreference(preferenceKey)
  }

  static applyPreference (preferenceKey) {
    if (!preferenceKey) {
      console.error('No preference key was specified')
      return
    }

    const msg = `on${preferenceKey}Preference`
    if (!lively[msg]) {
      // console.warn(`[preference] No event handler registered for "${preferenceKey}"`)
      return
    }

    const config = this.get(preferenceKey)
    lively[msg](config)
  }

  static loadPreferences() {
    Object.keys(Preferences.defaults).forEach(preferenceKey => {
      this.applyPreference(preferenceKey)
    })
  }

  static getURLParameter(theParameter) {
    if (!window) {
      return false;
    }

    var params = window.location.search.substr(1).split('&');

    for (var i = 0; i < params.length; i++) {
      var p=params[i].split('=');
      if (p[0] == theParameter) {
        return decodeURIComponent(p[1]);
      }
    }
    return false;
  }
}

Preferences.load()

<commit_msg>added preference for full logging on gs<commit_after>/*
 * Stores page-specific preferences in the body, so it gets saved/loaded with other content
 */

export default class Preferences {

  static get defaults() {
    return  {
      GridSize: {default: 100, short: "grid size"},
      SnapSize: {default: 20, short: "snap size"},
      SnapPaddingSize: {default: 20, short: "padding while snapping size"},
      SnapWindowsInGrid: {default: false, short: "snap windows in grid"},
      ShowFixedBrowser: {default: false, short: "show fixed browser"},
      InteractiveLayer: {default: false, short: "dev methods"},
      ShowDocumentGrid: {default: true, short: "show grid"},
      UseRP19JSX: {default: false, short: "use rp19 implementation for jsx"},
      DisableAExpWorkspace: {default: false, short: "disable AExp in workspace"},
      UseProxiesForAExprs: {default: false, short: "proxy-based Active Expressions"},
      EnableAEDebugging: {default: true, short: "enable Active Expression debugging"},
      SmartAELogging: {default: true, short: "Only log events for interesting Active Expressions"},
      DisableAltGrab: {default: false, short: "disable alt grab with hand"},
      UseAsyncWorkspace: {default: false, short: "support await in eval"},
      OfflineFirst: {default: false, short: "use offline first swx cache"},
      GraffleMode: {default: false, short: "create shapes by drag and hold S/C/F/D"},
      FileIndex: {default: true, short: "local file index"},
      SWXKeepAlive: {default: false, short: "keep SWX alive"},
      LogBoot: {default: false, short: "keep bootlog"},
      PiTerminalURL: {default: "http://localhost:3000/", short: "url of pi terminal"},
      PiTerminalCWD: {default: "", short: "current working directory of pi terminal"},
      PiTerminalSecret: {default: "", short: "pi terminal credentials"},
      ExtraSearchRoots: {default: [], short: "extra search roots"},
      TipOfTheDay: {default: false, short: "show tip of the day on startup"},
      WiderIndentation: {default: false, short: "sets the indentation to 4"},
      CommandModeAsDefault: {default: false, short: "command mode as default in editor"},
      CircumventCodeMirrorModes: {default: false, short: "circumvent code mirror modes"},
      BabylonianProgramming: {default: false, short: "use babylonian programming editor"},
      TabbedWindows: {default: false, short: "use experimental window tabs"},
      SWEDebugging: {default: false, short: "bug showcase in swe lecture"},
      AEXPGraphExperimental: {default: false, short: "AExpr graph experimental"},
      GSFullLogInfo: {default: false, short: "full log info for GS"},
    }
  }

  static load() {
    console.info('Load preferences')
  }

  // List all avaiable preferences
  static list () {
    return Object
      .keys(this.defaults)
  }

  static listBooleans () {
    return Object.keys(this.defaults)
      .filter(ea => _.isBoolean(this.defaults[ea].default))
  }

  static shortDescription(preferenceKey) {
    var pref = this.defaults[preferenceKey]
    if (pref && pref.short)
      return pref.short
    else
      return preferenceKey
  }

  /* get preference, consider defaults */
  static get (preferenceKey) {
    if (!preferenceKey) {
      console.error('No preference key was specified')
      return
    }

    let pref = this.read(preferenceKey)
    if (typeof pref === 'string') {
      return JSON.parse(pref)
    }

    pref = this.defaults[preferenceKey]
    if (!pref) {
      console.error(`Preference "${preferenceKey}" does not exist`)
      return
    }

    if(pref.hasOwnProperty('default')) {
      return pref.default
    }
  }

  static set(preferenceKey, value) {
    var pref = this.write(preferenceKey, JSON.stringify(value))
  }

  static get prefsNode() {
    // #BUG: reloading Preferences causes dataset to be not defined anymore
    if (!(window && window.document)) return null;

    let node = document.body.querySelector('.lively-preferences');
    if (!node) {
      console.log("Create prefereces")
      node = document.createElement('div'); // we cannot use custom comps they are async
      node.classList.add("lively-preferences")
      node.classList.add("lively-content")
      document.body.insertBefore(node, document.body.firstChild);
    }
    return node
  }

  static read(preferenceKey) {
    return this.prefsNode && this.prefsNode.dataset ?
      this.prefsNode.dataset[preferenceKey] :
      undefined;
  }

  static write(preferenceKey, preferenceValue) {
    if(!this.prefsNode) { return; }
    if (!this.prefsNode.dataset) {
      this.prefsNode.setAttribute("data-foo", true); // force dataset
    }
    this.prefsNode.dataset[preferenceKey] = preferenceValue;
  }

  static enable(preferenceKey) {
    Preferences.write(preferenceKey, "true")
    this.applyPreference(preferenceKey)
  }

  static disable(preferenceKey) {
    Preferences.write(preferenceKey, "false")
    this.applyPreference(preferenceKey)
  }

  static applyPreference (preferenceKey) {
    if (!preferenceKey) {
      console.error('No preference key was specified')
      return
    }

    const msg = `on${preferenceKey}Preference`
    if (!lively[msg]) {
      // console.warn(`[preference] No event handler registered for "${preferenceKey}"`)
      return
    }

    const config = this.get(preferenceKey)
    lively[msg](config)
  }

  static loadPreferences() {
    Object.keys(Preferences.defaults).forEach(preferenceKey => {
      this.applyPreference(preferenceKey)
    })
  }

  static getURLParameter(theParameter) {
    if (!window) {
      return false;
    }

    var params = window.location.search.substr(1).split('&');

    for (var i = 0; i < params.length; i++) {
      var p=params[i].split('=');
      if (p[0] == theParameter) {
        return decodeURIComponent(p[1]);
      }
    }
    return false;
  }
}

Preferences.load()

<|endoftext|>


<commit_before>var request = require("request");

exports.handler = function(event, context) { eventHandler(event, context); };

function eventHandler(event, context) {
  try { if (event.request.type === "LaunchRequest") {
          welcomeOnBoard(function callback(speechResponse) {
            context.succeed(buildResponse(event.session.attributes, speechResponse));
          });
        } else if (event.request.type === "IntentRequest") {
            determineIntent(event.request, function callback(speechResponse) {
              context.succeed(buildResponse(event.session.attributes, speechResponse));
            });
        }
  } catch (e) {
    context.fail("Exception: " + e);
  }
}

function welcomeOnBoard(callback) {
  callback(buildSpeechResponse("welcome aboard", "where to captain?", false));
}

function determineIntent(intentRequest, callback) {
  var intentName = intentRequest.intent.name;
  var planets = ['earth', 'mars', 'girlfriend', 'space', 'orbit', 'stratosphere', 'sun', 'falcon'];
  if (intentName == 'RandomIntent') {
    planet = planets[Math.floor(Math.random() * planets.length)];
    getToken(planet, callback);
  } else if(intentName == 'PlanetIntent') {
    getToken(intentRequest.intent.slots.planet.value, callback);
  } else if (intentName == 'AMAZON.HelpIntent') {
    helpUser(callback);
  } else {
    throw "Invalid intent";
  }
}

function helpUser(callback) {
  callback(buildSpeechResponse("You can go to Mars, Earth or even to the depths of the universe", "Where would you like to go?", true));
}

function callFirebase(planet, callback, token) {
  var url = 'https://fcm.googleapis.com/fcm/send';
  var serverKey = "key=AAAAaI2ZfFw:APA91bGqDh70rNfC8Gtwdxhut5sKhG7td0okEetwnhjWtzvTSC4jJIOReD2nEXkpT4OqMIciJptTxk7Du8MJmvrcW7jTKhiAh7XJYq2kBG2wIQOiwUerx014rpk7nt1JknAS-jdpUJxB";
  var clientToken = token;
  var options = {
    url: url,
    headers: {
      'Authorization': serverKey,
      'Content-Type': 'application/json'
    },
    json: {"to": clientToken,"priority":"high","notification":{"title": planet}}
  };
  request.post(options, function(error, response, body) {
    if(body.success == 1) {
      callback(buildSpeechResponse("this is " + planet, "", true));
    } else {
      callback(buildSpeechResponse("lost in space above all drifting", "", true));
    }
  });
}

function getToken(planet, callback) {
  var url = 'https://spaceship-test.firebaseio.com/browserTokens.json?orderBy="timestamp"&limitToLast=1';
  request.get(url, function(error, response, body) {
    var jsonObj = JSON.parse(body);
    for (var firstKey in jsonObj) break;
    callFirebase(planet, callback, jsonObj[firstKey]['token']);
  });
}

function buildSpeechResponse(output, repromptText, shouldEndSession) {
    return {
        outputSpeech: {
            type: "PlainText",
            text: output
        },
        reprompt: {
            outputSpeech: {
                type: "PlainText",
                text: repromptText
            }
        },
        shouldEndSession: shouldEndSession
    };
}

function buildResponse(sessionAttributes, speechResponse) {
    return {
        version: "1.0",
        sessionAttributes: sessionAttributes,
        response: speechResponse
    };
}
<commit_msg>reorder methods<commit_after>var request = require("request");

exports.handler = function(event, context) { eventHandler(event, context); };

function eventHandler(event, context) {
  try { if (event.request.type === "LaunchRequest") {
          welcomeOnBoard(function callback(speechResponse) {
            context.succeed(buildResponse(event.session.attributes, speechResponse));
          });
        } else if (event.request.type === "IntentRequest") {
            getToken(event.request, function callback(speechResponse) {
              context.succeed(buildResponse(event.session.attributes, speechResponse));
            });
        }
  } catch (e) {
    context.fail("Exception: " + e);
  }
}

function getToken(intentRequest, callback) {
  var url = 'https://spaceship-test.firebaseio.com/browserTokens.json?orderBy="timestamp"&limitToLast=1';
  request.get(url, function(error, response, body) {
    var jsonObj = JSON.parse(body);
    for (var firstKey in jsonObj) break;
    determineIntent(intentRequest, callback, jsonObj[firstKey]['token']);
  });
}

function welcomeOnBoard(callback) {
  callback(buildSpeechResponse("welcome aboard", "where to captain?", false));
}

function determineIntent(intentRequest, callback, token) {
  var intentName = intentRequest.intent.name;
  var planets = ['earth', 'mars', 'girlfriend', 'space', 'orbit', 'stratosphere', 'sun', 'falcon'];
  if (intentName == 'RandomIntent') {
    planet = planets[Math.floor(Math.random() * planets.length)];
    callFirebase(planet, callback, token);
  } else if(intentName == 'PlanetIntent') {
    callFirebase(intentRequest.intent.slots.planet.value, callback, token);
  } else if (intentName == 'AMAZON.HelpIntent') {
    helpUser(callback);
  } else {
    throw "Invalid intent";
  }
}

function helpUser(callback) {
  callback(buildSpeechResponse("You can go to Mars, Earth or even to the depths of the universe", "Where would you like to go?", true));
}

function callFirebase(planet, callback, token) {
  var url = 'https://fcm.googleapis.com/fcm/send';
  var serverKey = "key=AAAAaI2ZfFw:APA91bGqDh70rNfC8Gtwdxhut5sKhG7td0okEetwnhjWtzvTSC4jJIOReD2nEXkpT4OqMIciJptTxk7Du8MJmvrcW7jTKhiAh7XJYq2kBG2wIQOiwUerx014rpk7nt1JknAS-jdpUJxB";
  var clientToken = token;
  var options = {
    url: url,
    headers: {
      'Authorization': serverKey,
      'Content-Type': 'application/json'
    },
    json: {"to": clientToken,"priority":"high","notification":{"title": planet}}
  };
  request.post(options, function(error, response, body) {
    if(body.success == 1) {
      callback(buildSpeechResponse("this is " + planet, "", true));
    } else {
      callback(buildSpeechResponse("lost in space above all drifting", "", true));
    }
  });
}

function buildSpeechResponse(output, repromptText, shouldEndSession) {
    return {
        outputSpeech: {
            type: "PlainText",
            text: output
        },
        reprompt: {
            outputSpeech: {
                type: "PlainText",
                text: repromptText
            }
        },
        shouldEndSession: shouldEndSession
    };
}

function buildResponse(sessionAttributes, speechResponse) {
    return {
        version: "1.0",
        sessionAttributes: sessionAttributes,
        response: speechResponse
    };
}
<|endoftext|>


<commit_before>/* globals jasmine, phantom */
// Verify arguments
if (phantom.args.length === 0) {
    console.log("Simple JasmineBDD test runner for phantom.js");
    console.log("Usage: phantomjs-testrunner.js url_to_runner.html");
    console.log("Accepts http:// and file:// urls");
    console.log("");
    console.log("NOTE: This script depends on jasmine.TrivialReporter being used\non the page, for the DOM elements it creates.\n");
    phantom.exit(2);
}
else {
    var args = phantom.args;
    var fs = require("fs"),
        pages = [],
        page, address, resultsKey, i, l;

    var setupPageFn = function(p, k) {
        return function() {
            overloadPageEvaluate(p);
            setupWriteFileFunction(p, k, fs.separator);
        };
    };

    for (i = 0, l = args.length; i < l; i++) {
        address = args[i];
        console.log("Loading " + address);

        // if provided a url without a protocol, try to use file://
        address = address.indexOf("://") === -1 ? "file://" + address : address;

        // create a WebPage object to work with
        page = require("webpage").create();
        page.url = address;

        // When initialized, inject the reporting functions before the page is loaded
        // (and thus before it will try to utilize the functions)
        resultsKey = "__jr" + Math.ceil(Math.random() * 1000000);
        page.onInitialized = setupPageFn(page, resultsKey);
        page.open(address, processPage(null, page, resultsKey));
        pages.push(page);

        page.onConsoleMessage = logAndWorkAroundDefaultLineBreaking;
    }

    // bail when all pages have been processed
    setInterval(function(){
        var exit_code = 0;
        for (i = 0, l = pages.length; i < l; i++) {
            page = pages[i];
            if (page.__exit_code === null) {
                // wait until later
                return;
            }
            exit_code |= page.__exit_code;
        }
        phantom.exit(exit_code);
    }, 100);
}

// Thanks to hoisting, these helpers are still available when needed above
/**
 * Logs a message. Does not add a line-break for single characters '.' and 'F' or lines ending in ' ...'
 *
 * @param msg
 */
function logAndWorkAroundDefaultLineBreaking(msg) {
    var interpretAsWithoutNewline = /(^(\033\[\d+m)*[\.F](\033\[\d+m)*$)|( \.\.\.$)/;
    if (navigator.userAgent.indexOf("Windows") < 0 && interpretAsWithoutNewline.test(msg)) {
        try {
            var system = require('system');
            system.stdout.write(msg);
        } catch (e) {
            var fs = require('fs');
            fs.write('/dev/stdout', msg, 'w');
        }
    } else {
        console.log(msg);
    }
}

/**
 * Stringifies the function, replacing any %placeholders% with mapped values.
 *
 * @param {function} fn The function to replace occurrences within.
 * @param {object} replacements Key => Value object of string replacements.
 */
function replaceFunctionPlaceholders(fn, replacements) {
    if (replacements && typeof replacements === "object") {
        fn = fn.toString();
        for (var p in replacements) {
            if (replacements.hasOwnProperty(p)) {
                var match = new RegExp("%" + p + "%", "g");
                do {
                    fn = fn.replace(match, replacements[p]);
                } while(fn.indexOf(match) !== -1);
            }
        }
    }
    return fn;
}

/**
 * Replaces the "evaluate" method with one we can easily do substitution with.
 *
 * @param {phantomjs.WebPage} page The WebPage object to overload
 */
function overloadPageEvaluate(page) {
    page._evaluate = page.evaluate;
    page.evaluate = function(fn, replacements) { return page._evaluate(replaceFunctionPlaceholders(fn, replacements)); };
    return page;
}

/** Stubs a fake writeFile function into the test runner.
 *
 * @param {phantomjs.WebPage} page The WebPage object to inject functions into.
 * @param {string} key The name of the global object in which file data should
 *                     be stored for later retrieval.
 */
// TODO: not bothering with error checking for now (closed environment)
function setupWriteFileFunction(page, key, path_separator) {
    page.evaluate(function(){
        window["%resultsObj%"] = {};
        window.fs_path_separator = "%fs_path_separator%";
        window.__phantom_writeFile = function(filename, text) {
            window["%resultsObj%"][filename] = text;
        };
    }, {resultsObj: key, fs_path_separator: path_separator});
}

/**
 * Returns the loaded page's filename => output object.
 *
 * @param {phantomjs.WebPage} page The WebPage object to retrieve data from.
 * @param {string} key The name of the global object to be returned. Should
 *                     be the same key provided to setupWriteFileFunction.
 */
function getXmlResults(page, key) {
    return page.evaluate(function(){
        return window["%resultsObj%"] || {};
    }, {resultsObj: key});
}

/**
 * Processes a page.
 *
 * @param {string} status The status from opening the page via WebPage#open.
 * @param {phantomjs.WebPage} page The WebPage to be processed.
 */
function processPage(status, page, resultsKey) {
    if (status === null && page) {
        page.__exit_code = null;
        return function(stat){
            processPage(stat, page, resultsKey);
        };
    }
    if (status !== "success") {
        console.error("Unable to load resource: " + address);
        page.__exit_code = 2;
    }
    else {
        var isFinished = function() {
            return page.evaluate(function(){
                // if there's a JUnitXmlReporter, return a boolean indicating if it is finished
                if (jasmine && jasmine.JUnitXmlReporter && jasmine.JUnitXmlReporter.started_at !== null) {
                    return jasmine.JUnitXmlReporter.finished_at !== null;
                }
                // otherwise, see if there is anything in a "finished-at" element
                return document.getElementsByClassName("finished-at").length &&
                       document.getElementsByClassName("finished-at")[0].innerHTML.length > 0;
            });
        };
        var getResults = function() {
            return page.evaluate(function(){
                return document.getElementsByClassName("description").length &&
                       document.getElementsByClassName("description")[0].innerHTML.match(/(\d+) spec.* (\d+) failure.*/) ||
                       ["Unable to determine success or failure."];
            });
        };
        var timeout = 60000;
        var loopInterval = 100;
        var ival = setInterval(function(){
            if (isFinished()) {
                // get the results that need to be written to disk
                var fs = require("fs"),
                    xml_results = getXmlResults(page, resultsKey),
                    output;
                for (var filename in xml_results) {
                    if (xml_results.hasOwnProperty(filename) && (output = xml_results[filename]) && typeof(output) === "string") {
                        fs.write(filename, output, "w");
                    }
                }

                // print out a success / failure message of the results
                var results = getResults();
                var failures = Number(results[2]);
                if (failures > 0) {
                    page.__exit_code = 1;
                    clearInterval(ival);
                }
                else {
                    page.__exit_code = 0;
                    clearInterval(ival);
                }
            }
            else {
                timeout -= loopInterval;
                if (timeout <= 0) {
                    console.log('Page has timed out; aborting.');
                    page.__exit_code = 2;
                    clearInterval(ival);
                }
            }
        }, loopInterval);
    }
}
<commit_msg>Update PhantomJS test runner to understand new HtmlReporter output.<commit_after>/* globals jasmine, phantom */
// Verify arguments
if (phantom.args.length === 0) {
    console.log("Simple JasmineBDD test runner for phantom.js");
    console.log("Usage: phantomjs-testrunner.js url_to_runner.html");
    console.log("Accepts http:// and file:// urls");
    console.log("");
    console.log("NOTE: This script depends on jasmine.HtmlReporter being used\non the page, for the DOM elements it creates.\n");
    phantom.exit(2);
}
else {
    var args = phantom.args;
    var fs = require("fs"),
        pages = [],
        page, address, resultsKey, i, l;

    var setupPageFn = function(p, k) {
        return function() {
            overloadPageEvaluate(p);
            setupWriteFileFunction(p, k, fs.separator);
        };
    };

    for (i = 0, l = args.length; i < l; i++) {
        address = args[i];
        console.log("Loading " + address);

        // if provided a url without a protocol, try to use file://
        address = address.indexOf("://") === -1 ? "file://" + address : address;

        // create a WebPage object to work with
        page = require("webpage").create();
        page.url = address;

        // When initialized, inject the reporting functions before the page is loaded
        // (and thus before it will try to utilize the functions)
        resultsKey = "__jr" + Math.ceil(Math.random() * 1000000);
        page.onInitialized = setupPageFn(page, resultsKey);
        page.open(address, processPage(null, page, resultsKey));
        pages.push(page);

        page.onConsoleMessage = logAndWorkAroundDefaultLineBreaking;
    }

    // bail when all pages have been processed
    setInterval(function(){
        var exit_code = 0;
        for (i = 0, l = pages.length; i < l; i++) {
            page = pages[i];
            if (page.__exit_code === null) {
                // wait until later
                return;
            }
            exit_code |= page.__exit_code;
        }
        phantom.exit(exit_code);
    }, 100);
}

// Thanks to hoisting, these helpers are still available when needed above
/**
 * Logs a message. Does not add a line-break for single characters '.' and 'F' or lines ending in ' ...'
 *
 * @param msg
 */
function logAndWorkAroundDefaultLineBreaking(msg) {
    var interpretAsWithoutNewline = /(^(\033\[\d+m)*[\.F](\033\[\d+m)*$)|( \.\.\.$)/;
    if (navigator.userAgent.indexOf("Windows") < 0 && interpretAsWithoutNewline.test(msg)) {
        try {
            var system = require('system');
            system.stdout.write(msg);
        } catch (e) {
            var fs = require('fs');
            fs.write('/dev/stdout', msg, 'w');
        }
    } else {
        console.log(msg);
    }
}

/**
 * Stringifies the function, replacing any %placeholders% with mapped values.
 *
 * @param {function} fn The function to replace occurrences within.
 * @param {object} replacements Key => Value object of string replacements.
 */
function replaceFunctionPlaceholders(fn, replacements) {
    if (replacements && typeof replacements === "object") {
        fn = fn.toString();
        for (var p in replacements) {
            if (replacements.hasOwnProperty(p)) {
                var match = new RegExp("%" + p + "%", "g");
                do {
                    fn = fn.replace(match, replacements[p]);
                } while(fn.indexOf(match) !== -1);
            }
        }
    }
    return fn;
}

/**
 * Replaces the "evaluate" method with one we can easily do substitution with.
 *
 * @param {phantomjs.WebPage} page The WebPage object to overload
 */
function overloadPageEvaluate(page) {
    page._evaluate = page.evaluate;
    page.evaluate = function(fn, replacements) { return page._evaluate(replaceFunctionPlaceholders(fn, replacements)); };
    return page;
}

/** Stubs a fake writeFile function into the test runner.
 *
 * @param {phantomjs.WebPage} page The WebPage object to inject functions into.
 * @param {string} key The name of the global object in which file data should
 *                     be stored for later retrieval.
 */
// TODO: not bothering with error checking for now (closed environment)
function setupWriteFileFunction(page, key, path_separator) {
    page.evaluate(function(){
        window["%resultsObj%"] = {};
        window.fs_path_separator = "%fs_path_separator%";
        window.__phantom_writeFile = function(filename, text) {
            window["%resultsObj%"][filename] = text;
        };
    }, {resultsObj: key, fs_path_separator: path_separator});
}

/**
 * Returns the loaded page's filename => output object.
 *
 * @param {phantomjs.WebPage} page The WebPage object to retrieve data from.
 * @param {string} key The name of the global object to be returned. Should
 *                     be the same key provided to setupWriteFileFunction.
 */
function getXmlResults(page, key) {
    return page.evaluate(function(){
        return window["%resultsObj%"] || {};
    }, {resultsObj: key});
}

/**
 * Processes a page.
 *
 * @param {string} status The status from opening the page via WebPage#open.
 * @param {phantomjs.WebPage} page The WebPage to be processed.
 */
function processPage(status, page, resultsKey) {
    if (status === null && page) {
        page.__exit_code = null;
        return function(stat){
            processPage(stat, page, resultsKey);
        };
    }
    if (status !== "success") {
        console.error("Unable to load resource: " + address);
        page.__exit_code = 2;
    }
    else {
        var isFinished = function() {
            return page.evaluate(function(){
                // if there's a JUnitXmlReporter, return a boolean indicating if it is finished
                if (jasmine && jasmine.JUnitXmlReporter && jasmine.JUnitXmlReporter.started_at !== null) {
                    return jasmine.JUnitXmlReporter.finished_at !== null;
                }
                // otherwise, scrape the DOM for the HtmlReporter "finished in ..." output
                var durElem = document.querySelector(".html-reporter .duration");
                return durElem && durElem.textContent && durElem.textContent.toLowerCase().indexOf("finished in") === 0;
            });
        };
        var getResultsFromHtmlRunner = function() {
            return page.evaluate(function(){
                var resultElem = document.querySelector(".html-reporter .alert .bar");
                return resultElem && resultElem.textContent &&
                    resultElem.textContent.match(/(\d+) spec.* (\d+) failure.*/) ||
                   ["Unable to determine success or failure."];
            });
        };
        var timeout = 60000;
        var loopInterval = 100;
        var ival = setInterval(function(){
            if (isFinished()) {
                // get the results that need to be written to disk
                var fs = require("fs"),
                    xml_results = getXmlResults(page, resultsKey),
                    output;
                for (var filename in xml_results) {
                    if (xml_results.hasOwnProperty(filename) && (output = xml_results[filename]) && typeof(output) === "string") {
                        fs.write(filename, output, "w");
                    }
                }

                // print out a success / failure message of the results
                var results = getResultsFromHtmlRunner();
                var failures = Number(results[2]);
                if (failures > 0) {
                    page.__exit_code = 1;
                    clearInterval(ival);
                }
                else {
                    page.__exit_code = 0;
                    clearInterval(ival);
                }
            }
            else {
                timeout -= loopInterval;
                if (timeout <= 0) {
                    console.log('Page has timed out; aborting.');
                    page.__exit_code = 2;
                    clearInterval(ival);
                }
            }
        }, loopInterval);
    }
}
<|endoftext|>


<commit_before>// declare a new module called 'tokyoRestaurants', and make it require the `ng-admin` module as a dependency
var tokyoRestaurants = angular.module('tokyoRestaurants', ['ng-admin']);

var baseUrl = 'http://trapi.etdev.me/v1/';

// declare a function to run when the module bootstraps (during the 'config' phase)
tokyoRestaurants.config(['NgAdminConfigurationProvider', function (nga) {
    // create an admin application
    var admin = nga.application('Tokyo Restaurants')
      .baseApiUrl('http://trapi.etdev.me/v1/');

    // create area entity, set fields
    var area = nga.entity('areas');
    area.listView().fields([
        nga.field('id'),
        nga.field('name'),
        nga.field('primary_code'),
        nga.field('restaurants', 'embedded_list')
           .targetFields([
             nga.field('id')
                .isDetailLink(true),
             nga.field('name')
                .template('{{value}}'),
             nga.field('rating'),
             nga.field('price_range')
           ])
    ]);

    // create the restaurant list view
    var restaurant = nga.entity('restaurants');

    restaurant.listView().fields([
        nga.field('name')
           .template('{{value}}'),
        nga.field('rating'),
        nga.field('price_range'),
        nga.field('genres')
    ]);

    restaurant.showView().fields([
        nga.field('id'),
        nga.field('name')
    ]);

    // add area entity
    admin.addEntity(area);
    admin.addEntity(restaurant);

    // attach admin entity to DOM and execute
    nga.configure(admin);
}]);
<commit_msg>Add area showView<commit_after>// declare a new module called 'tokyoRestaurants', and make it require the `ng-admin` module as a dependency
var tokyoRestaurants = angular.module('tokyoRestaurants', ['ng-admin']);

var baseUrl = 'http://trapi.etdev.me/v1/';

// declare a function to run when the module bootstraps (during the 'config' phase)
tokyoRestaurants.config(['NgAdminConfigurationProvider', function (nga) {
    // create an admin application
    var admin = nga.application('Tokyo Restaurants')
      //.baseApiUrl('http://localhost:3000/v1/')
      .baseApiUrl('http://trapi.etdev.me/v1/');

    // create area entity, set fields
    var area = nga.entity('areas');
    area.listView().fields([
        nga.field('id'),
        nga.field('name')
    ]).batchActions([]);

    // area editionView
    area.showView().fields([
      nga.field('id'),
      nga.field('name'),
      nga.field('restaurants', 'embedded_list')
          .targetFields([
            nga.field('name')
              .template('{{value}}'),
            nga.field('genres'),
            nga.field('rating'),
            nga.field('price_range'),
          ])
    ]);

    // create the restaurant list view
    var restaurant = nga.entity('restaurants');

    restaurant.listView().fields([
        nga.field('name')
           .template('{{value}}'),
        nga.field('rating'),
        nga.field('price_range'),
        nga.field('genres')
    ]);

    // add area entity
    admin.addEntity(area);
    admin.addEntity(restaurant);

    // attach admin entity to DOM and execute
    nga.configure(admin);
}]);
<|endoftext|>


<commit_before>import sinon from 'sinon';
import path from 'path';
import Apify from '../build/index';
import * as keyValueStore from '../build/key_value_store';

const { utils: { log } } = Apify;

describe('Apify.utils.puppeteer', () => {
    let ll;
    beforeAll(() => {
        ll = log.getLevel();
        log.setLevel(log.LEVELS.ERROR);
    });
    afterAll(() => {
        log.setLevel(ll);
    });

    test('injectFile()', async () => {
        /* eslint-disable no-shadow */
        const browser = await Apify.launchPuppeteer({ headless: true });
        const survive = async (browser) => {
            // Survive navigations
            const page = await browser.newPage();
            let result = await page.evaluate(() => window.injectedVariable === 42);
            expect(result).toBe(false);
            await Apify.utils.puppeteer.injectFile(page, path.join(__dirname, 'data', 'inject_file.txt'), { surviveNavigations: true });
            result = await page.evaluate(() => window.injectedVariable);
            expect(result).toBe(42);
            await page.goto('about:chrome');
            result = await page.evaluate(() => window.injectedVariable);
            expect(result).toBe(42);
            await page.goto('https://www.example.com');
            result = await page.evaluate(() => window.injectedVariable);
            expect(result).toBe(42);
        };
        const remove = async (browser) => {
            // Remove with navigations
            const page = await browser.newPage();
            let result = await page.evaluate(() => window.injectedVariable === 42);
            expect(result).toBe(false);
            await page.goto('about:chrome');
            result = await page.evaluate(() => window.injectedVariable === 42);
            expect(result).toBe(false);
            await Apify.utils.puppeteer.injectFile(page, path.join(__dirname, 'data', 'inject_file.txt'));
            result = await page.evaluate(() => window.injectedVariable);
            expect(result).toBe(42);
            await page.goto('https://www.example.com');
            result = await page.evaluate(() => window.injectedVariable === 42);
            expect(result).toBe(false);
        };
        try {
            await Promise.all([survive(browser), remove(browser)]);
        } finally {
            browser.close();
        }
    });

    test('injectJQuery()', async () => {
        const browser = await Apify.launchPuppeteer({ headless: true });

        try {
            const page = await browser.newPage();
            await page.goto('about:blank');

            // NOTE: Chrome already defines window.$ as alias to document.querySelector(),
            // (https://developers.google.com/web/tools/chrome-devtools/console/command-line-reference#queryselector)
            const result1 = await page.evaluate(() => {
                return {
                    isDefined: window.jQuery !== undefined,
                };
            });
            expect(result1).toEqual({
                isDefined: false,
            });

            await Apify.utils.puppeteer.injectJQuery(page);
            const result2 = await page.evaluate(() => {
                /* global $ */
                return {
                    isDefined: window.jQuery === window.$,
                    text: $('h1').text(),
                };
            });
            expect(result2).toEqual({
                isDefined: true,
                text: '',
            });
        } finally {
            browser.close();
        }
    });

    test('injectUnderscore()', async () => {
        const browser = await Apify.launchPuppeteer({ headless: true });

        try {
            const page = await browser.newPage();
            await page.goto('about:blank');

            const result1 = await page.evaluate(() => {
                return { isDefined: window._ !== undefined };
            });
            expect(result1).toEqual({ isDefined: false });

            await Apify.utils.puppeteer.injectUnderscore(page);
            const result2 = await page.evaluate(() => {
                /* global _ */
                return { isDefined: _.isEmpty({}) };
            });
            expect(result2).toEqual({ isDefined: true });
        } finally {
            browser.close();
        }
    });

    // TODO Remove with 1.0.0. This is here as a backwards comp for moving
    // the function from utils.puppeteer to utils.
    test('enqueueLinks() exists in this namespace', async () => {
        expect(Apify.utils.puppeteer.enqueueLinks).toBeInstanceOf(Function);
        try {
            await Apify.utils.puppeteer.enqueueLinks({ page: {}, $: () => {} });
        } catch (err) {
            expect(err.message).toBe('Only one of the parameters "options.page" or "options.$" must be provided!');
        }
    });

    describe('blockRequests()', () => {
        let browser = null;
        beforeAll(async () => {
            browser = await Apify.launchPuppeteer({ headless: true });
        });
        afterAll(async () => {
            await browser.close();
        });

        test('works with default values', async () => {
            const loadedUrls = [];

            const page = await browser.newPage();
            await Apify.utils.puppeteer.blockRequests(page);
            page.on('response', response => loadedUrls.push(response.url()));
            await page.setContent(`
                <link rel="stylesheet" type="text/css" href="https://example.com/style.css">

            `, { waitUntil: 'load' });
            expect(loadedUrls).toEqual(['https://example.com/script.js']);
        });

        test('works with overridden values', async () => {
            const loadedUrls = [];

            const page = await browser.newPage();
            await Apify.utils.puppeteer.blockRequests(page, {
                urlPatterns: ['.css'],
            });
            page.on('response', response => loadedUrls.push(response.url()));
            await page.setContent(`
                <link rel="stylesheet" type="text/css" href="https://example.com/style.css">

            `, { waitUntil: 'load' });
            expect(loadedUrls).toEqual(expect.arrayContaining([
                'https://example.com/image.png',
                'https://example.com/script.js',
                'https://example.com/image.gif',
            ]));
        });

        test('blockResources() supports default values', async () => {
            const loadedUrls = [];

            const page = await browser.newPage();
            await Apify.utils.puppeteer.blockResources(page);
            page.on('response', response => loadedUrls.push(response.url()));
            await page.setContent(`
                <link rel="stylesheet" type="text/css" href="https://example.com/style.css">

            `, { waitUntil: 'load' });

            expect(loadedUrls).toEqual(expect.arrayContaining([
                'https://example.com/script.js',
            ]));
        });

        test('blockResources() supports nondefault values', async () => {
            const loadedUrls = [];

            const page = await browser.newPage();
            await Apify.utils.puppeteer.blockResources(page, ['script']);
            page.on('response', response => loadedUrls.push(response.url()));
            await page.setContent(`
                <link rel="stylesheet" type="text/css" href="https://example.com/style.css">

            `, { waitUntil: 'load' });

            expect(loadedUrls).toEqual(expect.arrayContaining([
                'https://example.com/style.css',
                'https://example.com/image.png',
            ]));
        });
    });

    test('supports cacheResponses()', async () => {
        const browser = await Apify.launchPuppeteer({ headless: true });
        const cache = {};

        const getResourcesLoadedFromWiki = async () => {
            let downloadedBytes = 0;
            const page = await browser.newPage();
            // Cache all javascript files, png files and svg files
            await Apify.utils.puppeteer.cacheResponses(page, cache, ['.js', /.+\.png/i, /.+\.svg/i]);
            page.on('response', async (response) => {
                if (cache[response.url()]) return;
                try {
                    const buffer = await response.buffer();
                    downloadedBytes += buffer.byteLength;
                } catch (e) {
                    // do nothing
                }
            });
            await page.goto('https://www.wikipedia.org/', { waitUntil: 'networkidle0' });
            await page.close();
            return downloadedBytes;
        };

        try {
            const bytesDownloadedOnFirstRun = await getResourcesLoadedFromWiki();
            const bytesDownloadedOnSecondRun = await getResourcesLoadedFromWiki();
            expect(bytesDownloadedOnSecondRun).toBeLessThan(bytesDownloadedOnFirstRun);
        } finally {
            await browser.close();
        }
    });

    test(
        'cacheResponses() throws when rule with invalid type is provided',
        async () => {
            const mockedPage = {
                setRequestInterception: () => {},
                on: () => {},
            };

            const testRuleType = async (value) => {
                try {
                    await Apify.utils.puppeteer.cacheResponses(mockedPage, {}, [value]);
                } catch (error) {
                    // this is valid path for this test
                    return;
                }

                expect(`Rule '${value}' should have thrown error`).toBe('');
            };
            await testRuleType(0);
            await testRuleType(1);
            await testRuleType(null);
            await testRuleType([]);
            await testRuleType(['']);
            await testRuleType(() => {});
        },
    );

    test('compileScript() works', async () => {
        const { compileScript } = Apify.utils.puppeteer;
        const scriptStringGood = 'await page.goto("about:blank"); return await page.content();';
        const scriptStringBad = 'for const while';
        const script = compileScript(scriptStringGood);

        expect(typeof script).toBe('function');
        expect(script.toString()).toEqual(`async ({ page, request }) => {${scriptStringGood}}`);

        try {
            compileScript(scriptStringBad);
            throw new Error('Should fail.');
        } catch (err) {
            // TODO figure out why the err.message comes out empty in the logs.
            expect(err.message).toMatch(/Unexpected token '?const'?/);
        }
        const browser = await Apify.launchPuppeteer({ headless: true });
        try {
            const page = await browser.newPage();
            const content = await script({ page });
            expect(typeof content).toBe('string');
            expect(content).toBe('');
        } finally {
            browser.close();
        }
    });

    test('gotoExtended() works', async () => {
        const browser = await Apify.launchPuppeteer({ headless: true });

        try {
            const page = await browser.newPage();
            const request = new Apify.Request({
                url: 'https://api.apify.com/v2/browser-info',
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json; charset=utf-8',
                },
                payload: '{ "foo": "bar" }',
            });

            const response = await Apify.utils.puppeteer.gotoExtended(page, request);

            const { method, headers, bodyLength } = JSON.parse(await response.text());
            expect(method).toBe('POST');
            expect(bodyLength).toBe(16);
            expect(headers['content-type']).toBe('application/json; charset=utf-8');
        } finally {
            await browser.close();
        }
    });

    test('infiniteScroll() works', async () => {
        function isAtBottom() {
            return (window.innerHeight + window.pageYOffset) >= document.body.offsetHeight;
        }
        const browser = await Apify.launchPuppeteer({ headless: true });
        try {
            const page = await browser.newPage();
            let count = 0;
            const content = Array(1000).fill(null).map(() => {
                return `<div style="border: 1px solid black">Div number: ${count++}</div>`;
            });
            const contentHTML = `<html><body>${content}</body></html>`;
            await page.setContent(contentHTML);

            const before = await page.evaluate(isAtBottom);
            expect(before).toBe(false);

            await Apify.utils.puppeteer.infiniteScroll(page, { waitForSecs: 0 });

            const after = await page.evaluate(isAtBottom);
            expect(after).toBe(true);
        } finally {
            await browser.close();
        }
    });

    it('saveSnapshot() works', async () => {
        const mock = sinon.mock(keyValueStore);
        const browser = await Apify.launchPuppeteer({ headless: true });
        try {
            const page = await browser.newPage();
            const contentHTML = '
Div number: 1
';
            await page.setContent(contentHTML);

            const screenshot = await page.screenshot({ fullPage: true, type: 'jpeg', screenshotQuality: 60 });

            // Test saving both image and html
            const object = { setValue: async () => {} };
            const stub = sinon.stub(object, 'setValue');

            mock.expects('openKeyValueStore')
                .once()
                .withExactArgs('TEST-STORE')
                .resolves(object);

            await Apify.utils.puppeteer.saveSnapshot(page, { key: 'TEST', keyValueStoreName: 'TEST-STORE', screenshotQuality: 60 });

            expect(stub.calledWithExactly('TEST.jpg', screenshot, { contentType: 'image/jpeg' })).toBe(true);
            expect(stub.calledWithExactly('TEST.html', contentHTML, { contentType: 'text/html' })).toBe(true);

            // Test saving only image
            const object2 = { setValue: async () => {} };
            const stub2 = sinon.stub(object2, 'setValue');
            mock.expects('openKeyValueStore')
                .withExactArgs(null)
                .resolves(object2);

            await Apify.utils.puppeteer.saveSnapshot(page, { saveHtml: false });

            // Default quality is 50
            const screenshot2 = await page.screenshot({ fullPage: true, type: 'jpeg', screenshotQuality: 50 });
            expect(stub2.calledOnceWithExactly('SNAPSHOT.jpg', screenshot2, { contentType: 'image/jpeg' })).toBe(true);

            mock.verify();
        } finally {
            await browser.close();
        }
    });
});
<commit_msg>fixed utils localEmulation.<commit_after>import sinon from 'sinon';
import path from 'path';
import Apify from '../build/index';
import * as keyValueStore from '../build/key_value_store';
import LocalStorageDirEmulator from './local_storage_dir_emulator';

const { utils: { log } } = Apify;

describe('Apify.utils.puppeteer', () => {
    let ll;
    let localStorageEmulator;

    beforeAll(async () => {
        ll = log.getLevel();
        log.setLevel(log.LEVELS.ERROR);
        localStorageEmulator = new LocalStorageDirEmulator();
        await localStorageEmulator.init();
    });

    beforeEach(async () => {
        await localStorageEmulator.clean();
    });

    afterAll(async () => {
        log.setLevel(ll);
        await localStorageEmulator.destroy();
    });

    test('injectFile()', async () => {
        /* eslint-disable no-shadow */
        const browser = await Apify.launchPuppeteer({ headless: true });
        const survive = async (browser) => {
            // Survive navigations
            const page = await browser.newPage();
            let result = await page.evaluate(() => window.injectedVariable === 42);
            expect(result).toBe(false);
            await Apify.utils.puppeteer.injectFile(page, path.join(__dirname, 'data', 'inject_file.txt'), { surviveNavigations: true });
            result = await page.evaluate(() => window.injectedVariable);
            expect(result).toBe(42);
            await page.goto('about:chrome');
            result = await page.evaluate(() => window.injectedVariable);
            expect(result).toBe(42);
            await page.goto('https://www.example.com');
            result = await page.evaluate(() => window.injectedVariable);
            expect(result).toBe(42);
        };
        const remove = async (browser) => {
            // Remove with navigations
            const page = await browser.newPage();
            let result = await page.evaluate(() => window.injectedVariable === 42);
            expect(result).toBe(false);
            await page.goto('about:chrome');
            result = await page.evaluate(() => window.injectedVariable === 42);
            expect(result).toBe(false);
            await Apify.utils.puppeteer.injectFile(page, path.join(__dirname, 'data', 'inject_file.txt'));
            result = await page.evaluate(() => window.injectedVariable);
            expect(result).toBe(42);
            await page.goto('https://www.example.com');
            result = await page.evaluate(() => window.injectedVariable === 42);
            expect(result).toBe(false);
        };
        try {
            await Promise.all([survive(browser), remove(browser)]);
        } finally {
            browser.close();
        }
    });

    test('injectJQuery()', async () => {
        const browser = await Apify.launchPuppeteer({ headless: true });

        try {
            const page = await browser.newPage();
            await page.goto('about:blank');

            // NOTE: Chrome already defines window.$ as alias to document.querySelector(),
            // (https://developers.google.com/web/tools/chrome-devtools/console/command-line-reference#queryselector)
            const result1 = await page.evaluate(() => {
                return {
                    isDefined: window.jQuery !== undefined,
                };
            });
            expect(result1).toEqual({
                isDefined: false,
            });

            await Apify.utils.puppeteer.injectJQuery(page);
            const result2 = await page.evaluate(() => {
                /* global $ */
                return {
                    isDefined: window.jQuery === window.$,
                    text: $('h1').text(),
                };
            });
            expect(result2).toEqual({
                isDefined: true,
                text: '',
            });
        } finally {
            browser.close();
        }
    });

    test('injectUnderscore()', async () => {
        const browser = await Apify.launchPuppeteer({ headless: true });

        try {
            const page = await browser.newPage();
            await page.goto('about:blank');

            const result1 = await page.evaluate(() => {
                return { isDefined: window._ !== undefined };
            });
            expect(result1).toEqual({ isDefined: false });

            await Apify.utils.puppeteer.injectUnderscore(page);
            const result2 = await page.evaluate(() => {
                /* global _ */
                return { isDefined: _.isEmpty({}) };
            });
            expect(result2).toEqual({ isDefined: true });
        } finally {
            browser.close();
        }
    });

    // TODO Remove with 1.0.0. This is here as a backwards comp for moving
    // the function from utils.puppeteer to utils.
    test('enqueueLinks() exists in this namespace', async () => {
        expect(Apify.utils.puppeteer.enqueueLinks).toBeInstanceOf(Function);
        try {
            await Apify.utils.puppeteer.enqueueLinks({ page: {}, $: () => {} });
        } catch (err) {
            expect(err.message).toBe('Only one of the parameters "options.page" or "options.$" must be provided!');
        }
    });

    describe('blockRequests()', () => {
        let browser = null;
        beforeAll(async () => {
            browser = await Apify.launchPuppeteer({ headless: true });
        });
        afterAll(async () => {
            await browser.close();
        });

        test('works with default values', async () => {
            const loadedUrls = [];

            const page = await browser.newPage();
            await Apify.utils.puppeteer.blockRequests(page);
            page.on('response', response => loadedUrls.push(response.url()));
            await page.setContent(`
                <link rel="stylesheet" type="text/css" href="https://example.com/style.css">

            `, { waitUntil: 'load' });
            expect(loadedUrls).toEqual(['https://example.com/script.js']);
        });

        test('works with overridden values', async () => {
            const loadedUrls = [];

            const page = await browser.newPage();
            await Apify.utils.puppeteer.blockRequests(page, {
                urlPatterns: ['.css'],
            });
            page.on('response', response => loadedUrls.push(response.url()));
            await page.setContent(`
                <link rel="stylesheet" type="text/css" href="https://example.com/style.css">

            `, { waitUntil: 'load' });
            expect(loadedUrls).toEqual(expect.arrayContaining([
                'https://example.com/image.png',
                'https://example.com/script.js',
                'https://example.com/image.gif',
            ]));
        });

        test('blockResources() supports default values', async () => {
            const loadedUrls = [];

            const page = await browser.newPage();
            await Apify.utils.puppeteer.blockResources(page);
            page.on('response', response => loadedUrls.push(response.url()));
            await page.setContent(`
                <link rel="stylesheet" type="text/css" href="https://example.com/style.css">

            `, { waitUntil: 'load' });

            expect(loadedUrls).toEqual(expect.arrayContaining([
                'https://example.com/script.js',
            ]));
        });

        test('blockResources() supports nondefault values', async () => {
            const loadedUrls = [];

            const page = await browser.newPage();
            await Apify.utils.puppeteer.blockResources(page, ['script']);
            page.on('response', response => loadedUrls.push(response.url()));
            await page.setContent(`
                <link rel="stylesheet" type="text/css" href="https://example.com/style.css">

            `, { waitUntil: 'load' });

            expect(loadedUrls).toEqual(expect.arrayContaining([
                'https://example.com/style.css',
                'https://example.com/image.png',
            ]));
        });
    });

    test('supports cacheResponses()', async () => {
        const browser = await Apify.launchPuppeteer({ headless: true });
        const cache = {};

        const getResourcesLoadedFromWiki = async () => {
            let downloadedBytes = 0;
            const page = await browser.newPage();
            // Cache all javascript files, png files and svg files
            await Apify.utils.puppeteer.cacheResponses(page, cache, ['.js', /.+\.png/i, /.+\.svg/i]);
            page.on('response', async (response) => {
                if (cache[response.url()]) return;
                try {
                    const buffer = await response.buffer();
                    downloadedBytes += buffer.byteLength;
                } catch (e) {
                    // do nothing
                }
            });
            await page.goto('https://www.wikipedia.org/', { waitUntil: 'networkidle0' });
            await page.close();
            return downloadedBytes;
        };

        try {
            const bytesDownloadedOnFirstRun = await getResourcesLoadedFromWiki();
            const bytesDownloadedOnSecondRun = await getResourcesLoadedFromWiki();
            expect(bytesDownloadedOnSecondRun).toBeLessThan(bytesDownloadedOnFirstRun);
        } finally {
            await browser.close();
        }
    });

    test(
        'cacheResponses() throws when rule with invalid type is provided',
        async () => {
            const mockedPage = {
                setRequestInterception: () => {},
                on: () => {},
            };

            const testRuleType = async (value) => {
                try {
                    await Apify.utils.puppeteer.cacheResponses(mockedPage, {}, [value]);
                } catch (error) {
                    // this is valid path for this test
                    return;
                }

                expect(`Rule '${value}' should have thrown error`).toBe('');
            };
            await testRuleType(0);
            await testRuleType(1);
            await testRuleType(null);
            await testRuleType([]);
            await testRuleType(['']);
            await testRuleType(() => {});
        },
    );

    test('compileScript() works', async () => {
        const { compileScript } = Apify.utils.puppeteer;
        const scriptStringGood = 'await page.goto("about:blank"); return await page.content();';
        const scriptStringBad = 'for const while';
        const script = compileScript(scriptStringGood);

        expect(typeof script).toBe('function');
        expect(script.toString()).toEqual(`async ({ page, request }) => {${scriptStringGood}}`);

        try {
            compileScript(scriptStringBad);
            throw new Error('Should fail.');
        } catch (err) {
            // TODO figure out why the err.message comes out empty in the logs.
            expect(err.message).toMatch(/Unexpected token '?const'?/);
        }
        const browser = await Apify.launchPuppeteer({ headless: true });
        try {
            const page = await browser.newPage();
            const content = await script({ page });
            expect(typeof content).toBe('string');
            expect(content).toBe('');
        } finally {
            browser.close();
        }
    });

    test('gotoExtended() works', async () => {
        const browser = await Apify.launchPuppeteer({ headless: true });

        try {
            const page = await browser.newPage();
            const request = new Apify.Request({
                url: 'https://api.apify.com/v2/browser-info',
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json; charset=utf-8',
                },
                payload: '{ "foo": "bar" }',
            });

            const response = await Apify.utils.puppeteer.gotoExtended(page, request);

            const { method, headers, bodyLength } = JSON.parse(await response.text());
            expect(method).toBe('POST');
            expect(bodyLength).toBe(16);
            expect(headers['content-type']).toBe('application/json; charset=utf-8');
        } finally {
            await browser.close();
        }
    });

    test('infiniteScroll() works', async () => {
        function isAtBottom() {
            return (window.innerHeight + window.pageYOffset) >= document.body.offsetHeight;
        }
        const browser = await Apify.launchPuppeteer({ headless: true });
        try {
            const page = await browser.newPage();
            let count = 0;
            const content = Array(1000).fill(null).map(() => {
                return `<div style="border: 1px solid black">Div number: ${count++}</div>`;
            });
            const contentHTML = `<html><body>${content}</body></html>`;
            await page.setContent(contentHTML);

            const before = await page.evaluate(isAtBottom);
            expect(before).toBe(false);

            await Apify.utils.puppeteer.infiniteScroll(page, { waitForSecs: 0 });

            const after = await page.evaluate(isAtBottom);
            expect(after).toBe(true);
        } finally {
            await browser.close();
        }
    });

    it('saveSnapshot() works', async () => {
        const mock = sinon.mock(keyValueStore);
        const browser = await Apify.launchPuppeteer({ headless: true });
        try {
            const page = await browser.newPage();
            const contentHTML = '
Div number: 1
';
            await page.setContent(contentHTML);

            const screenshot = await page.screenshot({ fullPage: true, type: 'jpeg', screenshotQuality: 60 });

            // Test saving both image and html
            const object = { setValue: async () => {} };
            const stub = sinon.stub(object, 'setValue');

            mock.expects('openKeyValueStore')
                .once()
                .withExactArgs('TEST-STORE')
                .resolves(object);

            await Apify.utils.puppeteer.saveSnapshot(page, { key: 'TEST', keyValueStoreName: 'TEST-STORE', screenshotQuality: 60 });

            expect(stub.calledWithExactly('TEST.jpg', screenshot, { contentType: 'image/jpeg' })).toBe(true);
            expect(stub.calledWithExactly('TEST.html', contentHTML, { contentType: 'text/html' })).toBe(true);

            // Test saving only image
            const object2 = { setValue: async () => {} };
            const stub2 = sinon.stub(object2, 'setValue');
            mock.expects('openKeyValueStore')
                .withExactArgs(null)
                .resolves(object2);

            await Apify.utils.puppeteer.saveSnapshot(page, { saveHtml: false });

            // Default quality is 50
            const screenshot2 = await page.screenshot({ fullPage: true, type: 'jpeg', screenshotQuality: 50 });
            expect(stub2.calledOnceWithExactly('SNAPSHOT.jpg', screenshot2, { contentType: 'image/jpeg' })).toBe(true);

            mock.verify();
        } finally {
            await browser.close();
        }
    });
});
<|endoftext|>