<commit_before>// Look up URLs in SURBL
var url       = require('url');
var dns       = require('dns');
var isIPv4    = require('net').isIPv4;
var net_utils = require('./net_utils.js');

// Default regexps to extract the URIs from the message
var numeric_ip = /\w{3,16}:\/+(\S+@)?(\d+|0[xX][0-9A-Fa-f]+)\.(\d+|0[xX][0-9A-Fa-f]+)\.(\d+|0[xX][0-9A-Fa-f]+)\.(\d+|0[xX][0-9A-Fa-f]+)/gi;
var schemeless = /((?:www\.)?[a-zA-Z0-9][a-zA-Z0-9\-.]+\.(?:aero|arpa|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|xxx|[a-zA-Z]{2}))(?!\w)/gi;
var schemed    = /(\w{3,16}:\/+(?:\S+@)?([a-zA-Z0-9][a-zA-Z0-9\-.]+\.(?:aero|arpa|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|xxx|[a-zA-Z]{2})))(?!\w)/gi;

var lists;
var zones;
var excludes = {};

function check_excludes_list(host) {
    host = host.toLowerCase().split('.').reverse();
    for (var i=0; i < host.length; i++) {
        if (i === 0) {
            var check = host[i];
        }
        else {
            check = [ host[i], check ].join('.');
        }
        if (excludes[check]) {
            return true;
        }
    }
    return false;
}

exports.register = function() {
    // Override regexps if top_level_tlds file is present
    if (net_utils.top_level_tlds && Object.keys(net_utils.top_level_tlds).length) {
        this.logdebug('Building new regexps from TLD file');
        var re_schemeless = '((?:www\\.)?[a-zA-Z0-9][a-zA-Z0-9\\-.]+\\.(?:' +
            Object.keys(net_utils.top_level_tlds).join('|') + '))(?!\\w)';
        schemeless = new RegExp(re_schemeless, 'gi');
        var re_schemed = '(\\w{3,16}:\\/+(?:\\S+@)?([a-zA-Z0-9][a-zA-Z0-9\\-.]+\\.(?:' +
            Object.keys(net_utils.top_level_tlds).join('|') + ')))(?!\\w)';
        schemed = new RegExp(re_schemed, 'gi');
    }
}

exports.load_uri_config = function (next) {
    lists = this.config.get('data.uribl.ini');
    zones = Object.keys(lists);
    if (!zones || !zones.length > 1) {
        this.logerr('aborting: no zones configured');
        return next();
    }
    // Load excludes
    this.config.get('data.uribl.excludes', 'list').forEach(function (domain) {
        excludes[domain.toLowerCase()] = 1;
    });
    // Set defaults
    if (lists.main && !lists.main.max_uris_per_list) {
        lists.main.max_uris_per_list = 20;
    }
}

exports.do_lookups = function (connection, next, hosts, type) {
    var plugin = this;

    if (typeof hosts === 'string') {
        hosts = [ hosts ];
    }
    if (!hosts || (hosts && !hosts.length)) {
        connection.logdebug(plugin, '(' + type + ') no items found for lookup');
        return next();
    }
    else {
        connection.logdebug(plugin, '(' + type + ') found ' + hosts.length + ' items for lookup');
    }
    var queries = {};
    for (var i=0; i < hosts.length; i++) {
        var host = hosts[i].toLowerCase();
        connection.logdebug(plugin, '(' + type + ') checking: ' + host);
        // Make sure we have a valid TLD
        if (!isIPv4(host) && !net_utils.top_level_tlds[(host.split('\.').reverse())[0]]) {
            continue;
        }
        // Check the exclusion list
        if (check_excludes_list(host)) {
            connection.logdebug(plugin, 'skipping excluded domain:' + host);
            continue;
        }
        // Loop through the zones
        for (var j=0; j < zones.length; j++) {
            var zone = zones[j];
            if (zone === 'main') continue;  // skip config
            if (!lists[zone] || (lists[zone] && !/^(?:1|true|yes|enabled|on)$/i.test(lists[zone][type]))) {
                connection.logdebug(plugin, 'skipping zone ' + zone + ' as it does not support lookup type ' + type);
                continue;
            }
            // Convert in-addr.arpa into bare IPv4 lookup
            var arpa = host.split(/\./).reverse();
            if (arpa.shift() === 'arpa' && arpa.shift() === 'in-addr') {
                if (arpa.length < 4) continue; // Only full IP addresses
                host = arpa.join('.');
            }
            var lookup;
            // Handle zones that do not allow IP queries (e.g. Spamhaus DBL)
            if (isIPv4(host)) {
                if (/^(?:1|true|yes|enabled|on)$/i.test(lists[zone].no_ip_lookups)) {
                    connection.logdebug(plugin, 'skipping IP lookup (' + host + ') for zone ' + zone);
                    continue;
                }
                // Skip any private IPs
                if (net_utils.is_rfc1918(host)) continue;
                // Reverse IP for lookup
                lookup = host.split(/\./).reverse().join('.');
            }
            // Handle zones that require host to be stripped to a domain boundary
            else if (/^(?:1|true|yes|enabled|on)$/i.test(lists[zone].strip_to_domain)) {
                lookup = (net_utils.split_hostname(host, 3))[1];
            }
            // Anything else..
            else {
                lookup = host;
            }
            if (!lookup) continue;
            if (!queries[zone]) queries[zone] = {};
            if (Object.keys(queries[zone]).length > lists.main.max_uris_per_list) {
                connection.logwarn(plugin, 'discarding lookup ' + lookup + ' for zone ' +
                              zone + ' maximum query limit reached');
                continue;
            }
            queries[zone][lookup] = 1;
        }
    }

    // Flatten object into array for easier querying
    var queries_to_run = [];
    for (var j=0; j < Object.keys(queries).length; j++) {
        for (var k=0; k < Object.keys(queries[Object.keys(queries)[j]]).length; k++) {
            // host/domain, zone
            queries_to_run.push( [ Object.keys(queries[Object.keys(queries)[j]])[k], Object.keys(queries)[j] ] );
        }
    }
    // Randomize the order a bit
    queries_to_run.sort(Math.round(Math.random())-0.25);

    if(!queries_to_run.length) {
        return next();
    }

    // Perform the lookups
    var plugin = this;
    var pending_queries = 0;
    var called_next = false;

    var timer = setTimeout(function () {
        connection.logdebug(plugin, 'timeout');
        if (!called_next) {
            called_next = true;
            return next();
        }
    }, ((lists.main && lists.main.timeout) ?
        lists.main.timeout : 30) * 1000);

    queries_to_run.forEach(function (query) {
        var lookup = query.join('.');
        // Add root dot if necessary
        if (lookup[lookup.length-1] !== '.') {
            lookup = lookup + '.';
        }
        pending_queries++;
        dns.resolve4(lookup, function(err, addrs) {
            pending_queries--;
            connection.logdebug(plugin, lookup + ' => ' + ((err) ? err : addrs.join(', ')));
            if (!err && !called_next) {
                var skip = false;
                var do_reject = function (msg) {
                    if (!skip && !called_next) {
                        if (!msg) {
                            msg = query[0] + ' blacklisted in ' + query[1];
                        }
                        // Check for custom message
                        if (lists[query[1]] && lists[query[1]].custom_msg) {
                            msg = lists[query[1]].custom_msg.replace(/\{uri\}/g, query[0]).replace(/\{zone\}/g, query[1]);
                        }
                        clearTimeout(timer);
                        called_next = true;
                        return next(DENY, msg);
                    }
                }
                // Optionally validate first result against a regexp
                if (lists[query[1]] && lists[query[1]].validate) {
                    var re = new RegExp(lists[query[1]].validate);
                    if (!re.test(addrs[0])) {
                        connection.logdebug(plugin, 'ignoring result (' + addrs[0] + ') for: ' +
                                lookup + ' as it did not match validation rule');
                        var skip = true;
                    }
                }
                // Check for optional bitmask
                if (lists[query[1]] && lists[query[1]].bitmask) {
                    // A bitmask zone should only return a single result
                    // We only support a bitmask of up to 128 in a single octet
                    var last_octet = new Number((addrs[0].split('.'))[3]);
                    var bitmask = new Number(lists[query[1]].bitmask);
                    if ((last_octet & bitmask) > 0) {
                        connection.loginfo(plugin, 'found ' + query[0] + ' in zone ' + query[1] +
                            ' (' + addrs.join(',') + '; bitmask=' + bitmask + ')');
                        do_reject();
                    } else {
                        connection.logdebug(plugin, 'ignoring result (' + addrs[0] + ') for: ' +
                                lookup + ' as the bitmask did not match');
                        var skip = true;
                    }
                }
                else {
                    connection.loginfo(plugin, 'found ' + query[0] + ' in zone ' + query[1] +
                        ' (' + addrs.join(',') + ')');
                    do_reject();
                }
            }
            if (!called_next && pending_queries === 0) {
                clearTimeout(timer);
                return next();
            }
        });
    });

    if (pending_queries === 0) {
        return next();
    }
}

exports.hook_lookup_rdns = function (next, connection) {
    this.load_uri_config(next);
    var plugin = this;
    dns.reverse(connection.remote_ip, function (err, rdns) {
        if (err) return next();
        plugin.do_lookups(connection, next, rdns, 'rdns');
    });
}

exports.hook_ehlo = function (next, connection, helo) {
    this.load_uri_config(next);
    // Handle IP literals
    var literal;
    if ((literal = /^\[(\d+\.\d+\.\d+\.\d+)\]$/.exec(helo))) {
        this.do_lookups(connection, next, literal[1], 'helo');
    } else {
        this.do_lookups(connection, next, helo, 'helo');
    }
}
exports.hook_helo = exports.hook_ehlo;

exports.hook_mail = function (next, connection, params) {
    this.load_uri_config(next);
    this.do_lookups(connection, next, params[0].host, 'envfrom');
}

exports.hook_data = function (next, connection) {
    // enable mail body parsing
    connection.transaction.parse_body = 1;
    return next();
}

exports.hook_data_post = function (next, connection) {
    this.load_uri_config(next);
    var email_re = /<?[^@]+@([^> ]+)>?/;
    var plugin = this;
    var trans = connection.transaction;

    // From header
    var do_from_header = function (cb) {
        var from = trans.header.get('from');
        var fmatch;
        if (fmatch = email_re.exec(from)) {
            return plugin.do_lookups(connection, cb, fmatch[1], 'from');
        }
        cb();
    }

    // Reply-To header
    var do_replyto_header = function (cb) {
        var replyto = trans.header.get('reply-to');
        var rmatch;
        if (rmatch = email_re.exec(replyto)) {
            return plugin.do_lookups(connection, cb, rmatch[1], 'replyto');
        }
        cb();
    }

    // Message-Id header
    var do_msgid_header = function (cb) {
        var msgid = trans.header.get('message-id');
        var mmatch;
        if (mmatch = /@([^>]+)>/.exec(msgid)) {
            return plugin.do_lookups(connection, cb, mmatch[1], 'msgid');
        }
        cb();
    }

    // Body
    var do_body = function (cb) {
        var urls = {};
        extract_urls(urls, trans.body);
        return plugin.do_lookups(connection, cb, Object.keys(urls), 'body');
    }

    var chain = [ do_from_header, do_replyto_header, do_msgid_header, do_body ];
    var chain_caller = function (code, msg) {
        if (code) {
            return next(code, msg);
        }
        if (!chain.length) {
            return next();
        }
        var next_in_chain = chain.shift();
        next_in_chain(chain_caller);
    }
    chain_caller();
}

function extract_urls (urls, body, connection, self) {
    // extract from body.bodytext
    var match;

    // extract numeric URIs
    while (match = numeric_ip.exec(body.bodytext)) {
        var uri;
        try {
            uri = url.parse(match[0]);
            // Don't reverse the IPs here; we do it in the lookup
            urls[uri.hostname] = uri;
        }
        catch (error) {
            connection.logerror(self, 'parse error: ' + match[0] +
                                      ' ' + error.message);
        }
    }

    // match plain hostname.tld
    while (match = schemeless.exec(body.bodytext)) {
        var uri;
        try {
            uri = url.parse('http://' + match[1]);
            urls[uri.hostname] = uri;
        }
        catch (error) {
            connection.logerror(self, 'parse error: ' + match[1] +
                                      ' ' + error.message);
        }
    }

    // match scheme:// URI
    while (match = schemed.exec(body.bodytext)) {
        var uri;
        try {
            uri = url.parse(match[1]);
            urls[uri.hostname] = uri;
        }
        catch (error) {
            connection.logerror(self, 'parse error: ' + match[1] +
                                      ' ' + error.message);
        }
    }

    // TODO: URIHASH
    // TODO: MAILHASH

    for (var i=0,l=body.children.length; i < l; i++) {
        extract_urls(urls, body.children[i], connection, self);
    }
}
<commit_msg>Complete previous commit<commit_after>// Look up URLs in SURBL
var url       = require('url');
var dns       = require('dns');
var isIPv4    = require('net').isIPv4;
var net_utils = require('./net_utils.js');

// Default regexps to extract the URIs from the message
var numeric_ip = /\w{3,16}:\/+(\S+@)?(\d+|0[xX][0-9A-Fa-f]+)\.(\d+|0[xX][0-9A-Fa-f]+)\.(\d+|0[xX][0-9A-Fa-f]+)\.(\d+|0[xX][0-9A-Fa-f]+)/gi;
var schemeless = /((?:www\.)?[a-zA-Z0-9][a-zA-Z0-9\-.]+\.(?:aero|arpa|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|xxx|[a-zA-Z]{2}))(?!\w)/gi;
var schemed    = /(\w{3,16}:\/+(?:\S+@)?([a-zA-Z0-9][a-zA-Z0-9\-.]+\.(?:aero|arpa|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|xxx|[a-zA-Z]{2})))(?!\w)/gi;

var lists;
var zones;
var excludes = {};

function check_excludes_list(host) {
    host = host.toLowerCase().split('.').reverse();
    for (var i=0; i < host.length; i++) {
        if (i === 0) {
            var check = host[i];
        }
        else {
            check = [ host[i], check ].join('.');
        }
        if (excludes[check]) {
            return true;
        }
    }
    return false;
}

exports.register = function() {
    // Override regexps if top_level_tlds file is present
    if (net_utils.top_level_tlds && Object.keys(net_utils.top_level_tlds).length) {
        this.logdebug('Building new regexps from TLD file');
        var re_schemeless = '((?:www\\.)?[a-zA-Z0-9][a-zA-Z0-9\\-.]+\\.(?:' +
            Object.keys(net_utils.top_level_tlds).join('|') + '))(?!\\w)';
        schemeless = new RegExp(re_schemeless, 'gi');
        var re_schemed = '(\\w{3,16}:\\/+(?:\\S+@)?([a-zA-Z0-9][a-zA-Z0-9\\-.]+\\.(?:' +
            Object.keys(net_utils.top_level_tlds).join('|') + ')))(?!\\w)';
        schemed = new RegExp(re_schemed, 'gi');
    }
}

exports.load_uri_config = function (next) {
    lists = this.config.get('data.uribl.ini');
    zones = Object.keys(lists);
    if (!zones || !zones.length > 1) {
        this.logerr('aborting: no zones configured');
        return next();
    }
    // Load excludes
    this.config.get('data.uribl.excludes', 'list').forEach(function (domain) {
        excludes[domain.toLowerCase()] = 1;
    });
    // Set defaults
    if (lists.main && !lists.main.max_uris_per_list) {
        lists.main.max_uris_per_list = 20;
    }
}

exports.do_lookups = function (connection, next, hosts, type) {
    var plugin = this;

    if (typeof hosts === 'string') {
        hosts = [ hosts ];
    }
    if (!hosts || (hosts && !hosts.length)) {
        connection.logdebug(plugin, '(' + type + ') no items found for lookup');
        return next();
    }
    else {
        connection.logdebug(plugin, '(' + type + ') found ' + hosts.length + ' items for lookup');
    }
    var queries = {};
    for (var i=0; i < hosts.length; i++) {
        var host = hosts[i].toLowerCase();
        connection.logdebug(plugin, '(' + type + ') checking: ' + host);
        // Make sure we have a valid TLD
        if (!isIPv4(host) && !net_utils.top_level_tlds[(host.split('\.').reverse())[0]]) {
            continue;
        }
        // Check the exclusion list
        if (check_excludes_list(host)) {
            connection.logdebug(plugin, 'skipping excluded domain:' + host);
            continue;
        }
        // Loop through the zones
        for (var j=0; j < zones.length; j++) {
            var zone = zones[j];
            if (zone === 'main') continue;  // skip config
            if (!lists[zone] || (lists[zone] && !/^(?:1|true|yes|enabled|on)$/i.test(lists[zone][type]))) {
                connection.logdebug(plugin, 'skipping zone ' + zone + ' as it does not support lookup type ' + type);
                continue;
            }
            // Convert in-addr.arpa into bare IPv4 lookup
            var arpa = host.split(/\./).reverse();
            if (arpa.shift() === 'arpa' && arpa.shift() === 'in-addr') {
                if (arpa.length < 4) continue; // Only full IP addresses
                host = arpa.join('.');
            }
            var lookup;
            // Handle zones that do not allow IP queries (e.g. Spamhaus DBL)
            if (isIPv4(host)) {
                if (/^(?:1|true|yes|enabled|on)$/i.test(lists[zone].no_ip_lookups)) {
                    connection.logdebug(plugin, 'skipping IP lookup (' + host + ') for zone ' + zone);
                    continue;
                }
                // Skip any private IPs
                if (net_utils.is_rfc1918(host)) continue;
                // Reverse IP for lookup
                lookup = host.split(/\./).reverse().join('.');
            }
            // Handle zones that require host to be stripped to a domain boundary
            else if (/^(?:1|true|yes|enabled|on)$/i.test(lists[zone].strip_to_domain)) {
                lookup = (net_utils.split_hostname(host, 3))[1];
            }
            // Anything else..
            else {
                lookup = host;
            }
            if (!lookup) continue;
            if (!queries[zone]) queries[zone] = {};
            if (Object.keys(queries[zone]).length > lists.main.max_uris_per_list) {
                connection.logwarn(plugin, 'discarding lookup ' + lookup + ' for zone ' +
                              zone + ' maximum query limit reached');
                continue;
            }
            queries[zone][lookup] = 1;
        }
    }

    // Flatten object into array for easier querying
    var queries_to_run = [];
    for (var j=0; j < Object.keys(queries).length; j++) {
        for (var k=0; k < Object.keys(queries[Object.keys(queries)[j]]).length; k++) {
            // host/domain, zone
            queries_to_run.push( [ Object.keys(queries[Object.keys(queries)[j]])[k], Object.keys(queries)[j] ] );
        }
    }
    // Randomize the order a bit
    queries_to_run.sort(Math.round(Math.random())-0.25);

    if(!queries_to_run.length) {
        return next();
    }

    // Perform the lookups
    var plugin = this;
    var pending_queries = 0;
    var called_next = false;

    var timer = setTimeout(function () {
        connection.logdebug(plugin, 'timeout');
        if (!called_next) {
            called_next = true;
            return next();
        }
    }, ((lists.main && lists.main.timeout) ?
        lists.main.timeout : 30) * 1000);

    queries_to_run.forEach(function (query) {
        var lookup = query.join('.');
        // Add root dot if necessary
        if (lookup[lookup.length-1] !== '.') {
            lookup = lookup + '.';
        }
        pending_queries++;
        dns.resolve4(lookup, function(err, addrs) {
            pending_queries--;
            connection.logdebug(plugin, lookup + ' => ' + ((err) ? err : addrs.join(', ')));
            if (!err && !called_next) {
                var skip = false;
                var do_reject = function (msg) {
                    if (!skip && !called_next) {
                        if (!msg) {
                            msg = query[0] + ' blacklisted in ' + query[1];
                        }
                        // Check for custom message
                        if (lists[query[1]] && lists[query[1]].custom_msg) {
                            msg = lists[query[1]].custom_msg.replace(/\{uri\}/g, query[0]).replace(/\{zone\}/g, query[1]);
                        }
                        clearTimeout(timer);
                        called_next = true;
                        return next(DENY, msg);
                    }
                }
                // Optionally validate first result against a regexp
                if (lists[query[1]] && lists[query[1]].validate) {
                    var re = new RegExp(lists[query[1]].validate);
                    if (!re.test(addrs[0])) {
                        connection.logdebug(plugin, 'ignoring result (' + addrs[0] + ') for: ' +
                                lookup + ' as it did not match validation rule');
                        var skip = true;
                    }
                }
                // Check for optional bitmask
                if (lists[query[1]] && lists[query[1]].bitmask) {
                    // A bitmask zone should only return a single result
                    // We only support a bitmask of up to 128 in a single octet
                    var last_octet = new Number((addrs[0].split('.'))[3]);
                    var bitmask = new Number(lists[query[1]].bitmask);
                    if ((last_octet & bitmask) > 0) {
                        connection.loginfo(plugin, 'found ' + query[0] + ' in zone ' + query[1] +
                            ' (' + addrs.join(',') + '; bitmask=' + bitmask + ')');
                        do_reject();
                    } else {
                        connection.logdebug(plugin, 'ignoring result (' + addrs[0] + ') for: ' +
                                lookup + ' as the bitmask did not match');
                        var skip = true;
                    }
                }
                else {
                    connection.loginfo(plugin, 'found ' + query[0] + ' in zone ' + query[1] +
                        ' (' + addrs.join(',') + ')');
                    do_reject();
                }
            }
            if (!called_next && pending_queries === 0) {
                clearTimeout(timer);
                return next();
            }
        });
    });

    if (pending_queries === 0) {
        return next();
    }
}

exports.hook_lookup_rdns = function (next, connection) {
    this.load_uri_config(next);
    var plugin = this;
    dns.reverse(connection.remote_ip, function (err, rdns) {
        if (err) return next();
        plugin.do_lookups(connection, next, rdns, 'rdns');
    });
}

exports.hook_ehlo = function (next, connection, helo) {
    this.load_uri_config(next);
    // Handle IP literals
    var literal;
    if ((literal = /^\[(\d+\.\d+\.\d+\.\d+)\]$/.exec(helo))) {
        this.do_lookups(connection, next, literal[1], 'helo');
    } else {
        this.do_lookups(connection, next, helo, 'helo');
    }
}
exports.hook_helo = exports.hook_ehlo;

exports.hook_mail = function (next, connection, params) {
    this.load_uri_config(next);
    this.do_lookups(connection, next, params[0].host, 'envfrom');
}

exports.hook_data = function (next, connection) {
    // enable mail body parsing
    connection.transaction.parse_body = 1;
    return next();
}

exports.hook_data_post = function (next, connection) {
    this.load_uri_config(next);
    var email_re = /<?[^@]+@([^> ]+)>?/;
    var plugin = this;
    var trans = connection.transaction;

    // From header
    var do_from_header = function (cb) {
        var from = trans.header.get('from');
        var fmatch;
        if (fmatch = email_re.exec(from)) {
            return plugin.do_lookups(connection, cb, fmatch[1], 'from');
        }
        cb();
    }

    // Reply-To header
    var do_replyto_header = function (cb) {
        var replyto = trans.header.get('reply-to');
        var rmatch;
        if (rmatch = email_re.exec(replyto)) {
            return plugin.do_lookups(connection, cb, rmatch[1], 'replyto');
        }
        cb();
    }

    // Message-Id header
    var do_msgid_header = function (cb) {
        var msgid = trans.header.get('message-id');
        var mmatch;
        if (mmatch = /@([^>]+)>/.exec(msgid)) {
            return plugin.do_lookups(connection, cb, mmatch[1], 'msgid');
        }
        cb();
    }

    // Body
    var do_body = function (cb) {
        var urls = {};
        extract_urls(urls, trans.body, connection, plugin);
        return plugin.do_lookups(connection, cb, Object.keys(urls), 'body');
    }

    var chain = [ do_from_header, do_replyto_header, do_msgid_header, do_body ];
    var chain_caller = function (code, msg) {
        if (code) {
            return next(code, msg);
        }
        if (!chain.length) {
            return next();
        }
        var next_in_chain = chain.shift();
        next_in_chain(chain_caller);
    }
    chain_caller();
}

function extract_urls (urls, body, connection, self) {
    // extract from body.bodytext
    var match;

    // extract numeric URIs
    while (match = numeric_ip.exec(body.bodytext)) {
        var uri;
        try {
            uri = url.parse(match[0]);
            // Don't reverse the IPs here; we do it in the lookup
            urls[uri.hostname] = uri;
        }
        catch (error) {
            connection.logerror(self, 'parse error: ' + match[0] +
                                      ' ' + error.message);
        }
    }

    // match plain hostname.tld
    while (match = schemeless.exec(body.bodytext)) {
        var uri;
        try {
            uri = url.parse('http://' + match[1]);
            urls[uri.hostname] = uri;
        }
        catch (error) {
            connection.logerror(self, 'parse error: ' + match[1] +
                                      ' ' + error.message);
        }
    }

    // match scheme:// URI
    while (match = schemed.exec(body.bodytext)) {
        var uri;
        try {
            uri = url.parse(match[1]);
            urls[uri.hostname] = uri;
        }
        catch (error) {
            connection.logerror(self, 'parse error: ' + match[1] +
                                      ' ' + error.message);
        }
    }

    // TODO: URIHASH
    // TODO: MAILHASH

    for (var i=0,l=body.children.length; i < l; i++) {
        extract_urls(urls, body.children[i], connection, self);
    }
}
<|endoftext|>


<commit_before>define(["intern",
        "intern!object",
        "intern/dojo/node!leadfoot/helpers/pollUntil",
        "intern/dojo/node!leadfoot/keys",
        "intern/chai!assert",
        "require"
        ], function (intern, registerSuite, pollUntil, keys, assert, require) {

	var loadNextPage = function (remote, listId, pageSize, expectedActiveTextAfterLoad) {
		return remote.pressKeys(keys.PAGE_DOWN)
			.pressKeys(keys.TAB)
			.getActiveElement()
				.getVisibleText()
				.then(function (value) {
					assert.strictEqual(value, "Click to load " + pageSize + " more items");
				})
			.end()
			.pressKeys(keys.SPACE)
			.getActiveElement()
			.getVisibleText()
			.then(function (value) {
				assert.strictEqual(value, expectedActiveTextAfterLoad);
			})
			.end();
	};

	var loadPreviousPage = function (remote, listId, pageSize, expectedActiveTextAfterLoad) {
		return remote.pressKeys(keys.PAGE_UP)
			.pressKeys(keys.SHIFT + keys.TAB)
			.pressKeys(keys.SHIFT) // release shift
			.getActiveElement()
				.getVisibleText()
				.then(function (value) {
					assert.strictEqual(value, "Click to load " + pageSize + " more items");
				})
			.end()
			.pressKeys(keys.SPACE)
			.getActiveElement()
			.getVisibleText()
			.then(function (value) {
				assert.strictEqual(value, expectedActiveTextAfterLoad);
			})
			.end();
	};

	registerSuite({
		name: "Pageable tests",
		"Pageable list keyboard navigation": function () {
			var remote = this.remote;
			// PageableList not currently supported on IE 10
			// see https://github.com/ibm-js/deliteful/issues/280
			if (remote.environmentType.browserName === "internet explorer" &&
					remote.environmentType.version === "10") {
				return this.skip("WARNING: PageableList not supported on IE10");
			}
			if (remote.environmentType.brokenSendKeys || !remote.environmentType.nativeEvents) {
				return this.skip("no keyboard support");
			}
			var listId = "pageable-prog-1";
			return remote
				.get(require.toUrl("./pageable-prog-1.html"))
				.then(pollUntil("return ('ready' in window &&  ready "
						+ "&& document.getElementById('" + listId + "') "
						+ "&& !document.querySelector('#" + listId + " .d-list-container')"
						+	".getAttribute('aria-busy') === false) ? true : null;",
						[],
						intern.config.WAIT_TIMEOUT,
						intern.config.POLL_INTERVAL))
				.getActiveElement()
				// For some reason, tab navigation doesn't succeed on IE if not typing a value before
					.type("test")
				.end()
				.pressKeys(keys.TAB)
				.then(function () {
					return loadNextPage(remote, listId, 20, "Programmatic item of order 20");
				})
				.then(function () {
					return loadNextPage(remote, listId, 20, "Programmatic item of order 59");
				})
				.then(function () {
					return loadNextPage(remote, listId, 20, "Programmatic item of order 79");
				})
				.then(function () {
					return loadNextPage(remote, listId, 20, "Programmatic item of order 99");
				})
				.then(function () {
					return loadNextPage(remote, listId, 20, "Programmatic item of order 99");
				})
				.then(function () {
					return loadPreviousPage(remote, listId, 20, "Programmatic item of order 40");
				})
				.then(function () {
					return loadPreviousPage(remote, listId, 20, "Programmatic item of order 20");
				})
				.then(function () {
					return loadPreviousPage(remote, listId, 20, "Programmatic item of order 0");
				});
		},

		"Pageable categorized list keyboard navigation": function () {
			var remote = this.remote;
			// PageableList not currently supported on IE 10
			// see https://github.com/ibm-js/deliteful/issues/280
			if (remote.environmentType.browserName === "internet explorer" &&
					remote.environmentType.version === "10") {
				return this.skip("WARNING: PageableList not supported on IE10");
			}
			if (remote.environmentType.brokenSendKeys || !remote.environmentType.nativeEvents) {
				return this.skip("no keyboard support");
			}
			var listId = "pageable-prog-2";
			return remote
				.get(require.toUrl("./pageable-prog-2.html"))
				.then(pollUntil("return ('ready' in window &&  ready "
					+ "&& document.getElementById('" + listId + "') "
					+ "&& !document.querySelector('#" + listId + " .d-list-container')"
					+	".getAttribute('aria-busy') === false) ? true : null;",
					[],
					intern.config.WAIT_TIMEOUT,
					intern.config.POLL_INTERVAL))
				.pressKeys(keys.TAB)
				.then(function () {
					return loadNextPage(remote, listId, 25, "Programmatic item of order 25");
				})
				.then(function () {
					return loadNextPage(remote, listId, 25, "Programmatic item of order 73");
				})
				.then(function () {
					return loadNextPage(remote, listId, 25, "Programmatic item of order 99");
				})
				.then(function () {
					return loadNextPage(remote, listId, 25, "Programmatic item of order 99");
				})
				.then(function () {
					return loadPreviousPage(remote, listId, 25, "Category 2");
				})
				.then(function () {
					return loadPreviousPage(remote, listId, 25, "Category 0");
				});
		}
	});
});
<commit_msg>Add comments to make it clear which part of the test failed.<commit_after>define(["intern",
        "intern!object",
        "intern/dojo/node!leadfoot/helpers/pollUntil",
        "intern/dojo/node!leadfoot/keys",
        "intern/chai!assert",
        "require"
        ], function (intern, registerSuite, pollUntil, keys, assert, require) {

	function loadNextPage(remote, listId, pageSize, expectedActiveTextAfterLoad, comment) {
		return remote.pressKeys(keys.PAGE_DOWN)
			.pressKeys(keys.TAB)
			.getActiveElement()
				.getVisibleText()
				.then(function (value) {
					assert.strictEqual(value, "Click to load " + pageSize + " more items", comment + " (a)");
				})
			.end()
			.pressKeys(keys.SPACE)
			.getActiveElement()
			.getVisibleText()
			.then(function (value) {
				assert.strictEqual(value, expectedActiveTextAfterLoad, comment + " (b)");
			})
			.end();
	}

	function loadPreviousPage(remote, listId, pageSize, expectedActiveTextAfterLoad, comment) {
		return remote.pressKeys(keys.PAGE_UP)
			.pressKeys(keys.SHIFT + keys.TAB)
			.pressKeys(keys.SHIFT) // release shift
			.getActiveElement()
				.getVisibleText()
				.then(function (value) {
					assert.strictEqual(value, "Click to load " + pageSize + " more items", comment + " (a)");
				})
			.end()
			.pressKeys(keys.SPACE)
			.getActiveElement()
			.getVisibleText()
			.then(function (value) {
				assert.strictEqual(value, expectedActiveTextAfterLoad, comment + " (b)");
			})
			.end();
	}

	registerSuite({
		name: "Pageable tests",
		"Pageable list keyboard navigation": function () {
			var remote = this.remote;
			// PageableList not currently supported on IE 10
			// see https://github.com/ibm-js/deliteful/issues/280
			if (remote.environmentType.browserName === "internet explorer" &&
					remote.environmentType.version === "10") {
				return this.skip("WARNING: PageableList not supported on IE10");
			}
			if (remote.environmentType.brokenSendKeys || !remote.environmentType.nativeEvents) {
				return this.skip("no keyboard support");
			}
			var listId = "pageable-prog-1";
			return remote
				.get(require.toUrl("./pageable-prog-1.html"))
				.then(pollUntil("return ('ready' in window &&  ready "
						+ "&& document.getElementById('" + listId + "') "
						+ "&& !document.querySelector('#" + listId + " .d-list-container')"
						+	".getAttribute('aria-busy') === false) ? true : null;",
						[],
						intern.config.WAIT_TIMEOUT,
						intern.config.POLL_INTERVAL))
				.getActiveElement()
				// For some reason, tab navigation doesn't succeed on IE if not typing a value before
					.type("test")
				.end()
				.pressKeys(keys.TAB)
				.then(function () {
					return loadNextPage(remote, listId, 20, "Programmatic item of order 20", "loadNext #1");
				})
				.then(function () {
					return loadNextPage(remote, listId, 20, "Programmatic item of order 59", "loadNext #2");
				})
				.then(function () {
					return loadNextPage(remote, listId, 20, "Programmatic item of order 79", "loadNext #3");
				})
				.then(function () {
					return loadNextPage(remote, listId, 20, "Programmatic item of order 99", "loadNext #4");
				})
				.then(function () {
					return loadNextPage(remote, listId, 20, "Programmatic item of order 99", "loadNext #5");
				})
				.then(function () {
					return loadPreviousPage(remote, listId, 20, "Programmatic item of order 40", "loadPrev #1");
				})
				.then(function () {
					return loadPreviousPage(remote, listId, 20, "Programmatic item of order 20", "loadPrev #2");
				})
				.then(function () {
					return loadPreviousPage(remote, listId, 20, "Programmatic item of order 0", "loadPrev #3");
				});
		},

		"Pageable categorized list keyboard navigation": function () {
			var remote = this.remote;
			// PageableList not currently supported on IE 10
			// see https://github.com/ibm-js/deliteful/issues/280
			if (remote.environmentType.browserName === "internet explorer" &&
					remote.environmentType.version === "10") {
				return this.skip("WARNING: PageableList not supported on IE10");
			}
			if (remote.environmentType.brokenSendKeys || !remote.environmentType.nativeEvents) {
				return this.skip("no keyboard support");
			}
			var listId = "pageable-prog-2";
			return remote
				.get(require.toUrl("./pageable-prog-2.html"))
				.then(pollUntil("return ('ready' in window &&  ready "
					+ "&& document.getElementById('" + listId + "') "
					+ "&& !document.querySelector('#" + listId + " .d-list-container')"
					+	".getAttribute('aria-busy') === false) ? true : null;",
					[],
					intern.config.WAIT_TIMEOUT,
					intern.config.POLL_INTERVAL))
				.pressKeys(keys.TAB)
				.then(function () {
					return loadNextPage(remote, listId, 25, "Programmatic item of order 25", "loadNext #1");
				})
				.then(function () {
					return loadNextPage(remote, listId, 25, "Programmatic item of order 73", "loadNext #2");
				})
				.then(function () {
					return loadNextPage(remote, listId, 25, "Programmatic item of order 99", "loadNext #3");
				})
				.then(function () {
					return loadNextPage(remote, listId, 25, "Programmatic item of order 99", "loadNext #4");
				})
				.then(function () {
					return loadPreviousPage(remote, listId, 25, "Category 2", "loadPrev #1");
				})
				.then(function () {
					return loadPreviousPage(remote, listId, 25, "Category 0", "loadPrev #2");
				});
		}
	});
});
<|endoftext|>


<commit_before>const Gio = imports.gi.Gio;
const StyleKnobGenerator = imports.app.compat.styleKnobGenerator;
const Utils = imports.tests.utils;

const TEST_CONTENT_DIR = Utils.get_test_content_srcdir();

let style_knobs_a = {
    'tab_button': {
        'module-background-color': '#7DA443',
    },
    'section_page': {
        'title-font-family': "'Marcellus SC'",
        'title-font-weight': 'bold',
    },
    'no_search_results_page': {
        'title-font-family': "'Marcellus SC'",
        'title-font-weight': 'bold',
    },
    'section_card': {
        'module-background': '#E8E4E1',
        'title-color': '#5A8715',
        'title-font-family': "'Marcellus SC'",
        'title-font-weight': 'bold',
    },
    'article_card': {
        'module-background': '#E8E4E1',
        'title-color': '#5A8715',
        'title-font-family': "'Marcellus SC'",
        'title-font-weight': 'bold',
    }
}

let style_knobs_b = {
    'section_card': {
        'title-color': '#98B8FF',
        'hover-color': '#4573D9',
        'title-font-size': '2.2em',
        'title-padding': '.7em 0em',
        'title-font-family': "'Amatic SC'",
        'title-font-weight': 'bold',
    },
    'article_card': {
        'title-color': '#98B8FF',
        'hover-color': '#4573D9',
    },
    'section_page': {
        'title-font-size': '9.0em',
        'title-padding': '0.5em 0.0em 0.5em 0.7em',
        'title-font-family': "'Amatic SC'",
        'title-font-weight': 'bold',
    },
    'no_search_results_page': {
        'title-font-size': '10.72em',
        'title-font-family': "'Amatic SC'",
        'title-font-weight': 'bold',
    },
};

let style_knobs_reader = {
  'overview_page': {
    'module-background': 'rgba(51, 51, 51, 0.30)',
    'module-background-image': 'linear-gradient(-180deg, rgba(176, 176, 176, 0.20) 1%, alpha(black, 0.30) 100%)',
  },
  'snippet0': {
    'module-font-family': 'ABeeZee',
    'module-font-size': '1em',
    'module-color': '#F2F2F2',
    'title-font-family': 'Abel',
    'title-color': '#E05430',
    'title-font-size': '2.125em',
  },
  'snippet1': {
    'module-font-family': 'ABeeZee',
    'module-font-size': '1em',
    'module-color': '#F2F2F2',
    'title-font-family': 'Roboto',
    'title-color': '#FA770F',
    'title-font-size': '2em',
  },
  'snippet2': {
    'module-font-family': 'ABeeZee',
    'module-font-size': '1em',
    'module-color': '#F2F2F2',
    'title-font-family': 'Dosis',
    'title-color': '#F7C449',
    'title-font-size': '2.25em',
  },
  'done_page': {
    'module-font-family': 'ABeeZee',
    'module-font-size': '1em',
    'module-color': '#F2F2F2',
    'title-font-family': 'Abel',
    'title-color': '#E05430',
    'title-font-size': '2.125em',
  },
  'article_page0': {
    'title-color': '#D95430',
    'title-font-family': 'Abel',
    'title-font-size': '3em',
    'module-color': '#8E421A',
    'module-font-family': 'Roboto',
    'module-font-size': '0.88em',
    'module-font-style': 'italic',
    'title-background-color': '#D95420',
  },
  'article_page1': {
    'title-color': '#CD6E34',
    'title-font-family': 'Lato',
    'title-font-size': '3em',
    'module-color': '#924C22',
    'module-font-family': 'Roboto',
    'module-font-size': '0.75em',
    'module-font-weight': '800',
  },
  'article_page2': {
    'title-color': '#DFAA29',
    'title-font-family': 'Dosis',
    'title-font-size': '3.38em',
    'title-font-weight': '200',
    'title-border-left': '2px solid #F7C464',
    'title-padding-left': '20px',
    'module-color': '#B9922F',
    'module-font-family': 'Roboto',
    'module-font-size': '0.81em',
    'module-font-style': 'italic',
  },
  'reader_card0': {
    'title-color': '#d95430',
    'title-font-family': 'Abel',
    'module-color': '#8e421a',
    'module-font-family': 'Roboto',
    'module-font-style': 'italic',
    'title-background-color': '#D95420',
  },
  'reader_card1': {
    'title-color': '#cd6e34',
    'title-font-family': 'Lato',
    'module-color': '#924c22',
    'module-font-family': 'Roboto',
    'module-font-weight': '800',
  },
  'reader_card2': {
    'title-color': '#dfaa29',
    'title-font-family': 'Dosis',
    'title-font-weight': '200',
    'module-color': '#b9922f',
    'module-font-family': 'Roboto',
    'module-font-style': 'italic',
  }
}

describe('StyleKnobGenerator', function () {
    let css_overrides_a;
    let css_overrides_b;
    let css_overrides_reader;

    beforeEach(function () {
        let file_a = Gio.File.new_for_path(TEST_CONTENT_DIR + 'overrides-a.css');
        css_overrides_a = (file_a.load_contents(null)[1]).toString();
        let file_b = Gio.File.new_for_path(TEST_CONTENT_DIR + 'overrides-b.css');
        css_overrides_b = (file_b.load_contents(null)[1]).toString();
        let file_reader = Gio.File.new_for_path(TEST_CONTENT_DIR + 'overrides-reader.css');
        css_overrides_reader = (file_reader.load_contents(null)[1]).toString();
    });

    it('handles template A overrides', function () {
        let style_knobs = StyleKnobGenerator.get_knobs_from_css(css_overrides_a, 'A');
        expect(style_knobs).toEqual(style_knobs_a);
    });

    it('handles template B overrides', function () {
        let style_knobs = StyleKnobGenerator.get_knobs_from_css(css_overrides_b, 'B');
        expect(style_knobs).toEqual(style_knobs_b);
    });

    it('handles reader app overrides', function () {
        let style_knobs = StyleKnobGenerator.get_knobs_from_css(css_overrides_reader, 'reader');
        print(JSON.stringify(style_knobs, null, 2));
        expect(style_knobs).toEqual(style_knobs_reader);
    });
});
<commit_msg>Remove debugging print statement.<commit_after>const Gio = imports.gi.Gio;
const StyleKnobGenerator = imports.app.compat.styleKnobGenerator;
const Utils = imports.tests.utils;

const TEST_CONTENT_DIR = Utils.get_test_content_srcdir();

let style_knobs_a = {
    'tab_button': {
        'module-background-color': '#7DA443',
    },
    'section_page': {
        'title-font-family': "'Marcellus SC'",
        'title-font-weight': 'bold',
    },
    'no_search_results_page': {
        'title-font-family': "'Marcellus SC'",
        'title-font-weight': 'bold',
    },
    'section_card': {
        'module-background': '#E8E4E1',
        'title-color': '#5A8715',
        'title-font-family': "'Marcellus SC'",
        'title-font-weight': 'bold',
    },
    'article_card': {
        'module-background': '#E8E4E1',
        'title-color': '#5A8715',
        'title-font-family': "'Marcellus SC'",
        'title-font-weight': 'bold',
    }
}

let style_knobs_b = {
    'section_card': {
        'title-color': '#98B8FF',
        'hover-color': '#4573D9',
        'title-font-size': '2.2em',
        'title-padding': '.7em 0em',
        'title-font-family': "'Amatic SC'",
        'title-font-weight': 'bold',
    },
    'article_card': {
        'title-color': '#98B8FF',
        'hover-color': '#4573D9',
    },
    'section_page': {
        'title-font-size': '9.0em',
        'title-padding': '0.5em 0.0em 0.5em 0.7em',
        'title-font-family': "'Amatic SC'",
        'title-font-weight': 'bold',
    },
    'no_search_results_page': {
        'title-font-size': '10.72em',
        'title-font-family': "'Amatic SC'",
        'title-font-weight': 'bold',
    },
};

let style_knobs_reader = {
  'overview_page': {
    'module-background': 'rgba(51, 51, 51, 0.30)',
    'module-background-image': 'linear-gradient(-180deg, rgba(176, 176, 176, 0.20) 1%, alpha(black, 0.30) 100%)',
  },
  'snippet0': {
    'module-font-family': 'ABeeZee',
    'module-font-size': '1em',
    'module-color': '#F2F2F2',
    'title-font-family': 'Abel',
    'title-color': '#E05430',
    'title-font-size': '2.125em',
  },
  'snippet1': {
    'module-font-family': 'ABeeZee',
    'module-font-size': '1em',
    'module-color': '#F2F2F2',
    'title-font-family': 'Roboto',
    'title-color': '#FA770F',
    'title-font-size': '2em',
  },
  'snippet2': {
    'module-font-family': 'ABeeZee',
    'module-font-size': '1em',
    'module-color': '#F2F2F2',
    'title-font-family': 'Dosis',
    'title-color': '#F7C449',
    'title-font-size': '2.25em',
  },
  'done_page': {
    'module-font-family': 'ABeeZee',
    'module-font-size': '1em',
    'module-color': '#F2F2F2',
    'title-font-family': 'Abel',
    'title-color': '#E05430',
    'title-font-size': '2.125em',
  },
  'article_page0': {
    'title-color': '#D95430',
    'title-font-family': 'Abel',
    'title-font-size': '3em',
    'module-color': '#8E421A',
    'module-font-family': 'Roboto',
    'module-font-size': '0.88em',
    'module-font-style': 'italic',
    'title-background-color': '#D95420',
  },
  'article_page1': {
    'title-color': '#CD6E34',
    'title-font-family': 'Lato',
    'title-font-size': '3em',
    'module-color': '#924C22',
    'module-font-family': 'Roboto',
    'module-font-size': '0.75em',
    'module-font-weight': '800',
  },
  'article_page2': {
    'title-color': '#DFAA29',
    'title-font-family': 'Dosis',
    'title-font-size': '3.38em',
    'title-font-weight': '200',
    'title-border-left': '2px solid #F7C464',
    'title-padding-left': '20px',
    'module-color': '#B9922F',
    'module-font-family': 'Roboto',
    'module-font-size': '0.81em',
    'module-font-style': 'italic',
  },
  'reader_card0': {
    'title-color': '#d95430',
    'title-font-family': 'Abel',
    'module-color': '#8e421a',
    'module-font-family': 'Roboto',
    'module-font-style': 'italic',
    'title-background-color': '#D95420',
  },
  'reader_card1': {
    'title-color': '#cd6e34',
    'title-font-family': 'Lato',
    'module-color': '#924c22',
    'module-font-family': 'Roboto',
    'module-font-weight': '800',
  },
  'reader_card2': {
    'title-color': '#dfaa29',
    'title-font-family': 'Dosis',
    'title-font-weight': '200',
    'module-color': '#b9922f',
    'module-font-family': 'Roboto',
    'module-font-style': 'italic',
  }
}

describe('StyleKnobGenerator', function () {
    let css_overrides_a;
    let css_overrides_b;
    let css_overrides_reader;

    beforeEach(function () {
        let file_a = Gio.File.new_for_path(TEST_CONTENT_DIR + 'overrides-a.css');
        css_overrides_a = (file_a.load_contents(null)[1]).toString();
        let file_b = Gio.File.new_for_path(TEST_CONTENT_DIR + 'overrides-b.css');
        css_overrides_b = (file_b.load_contents(null)[1]).toString();
        let file_reader = Gio.File.new_for_path(TEST_CONTENT_DIR + 'overrides-reader.css');
        css_overrides_reader = (file_reader.load_contents(null)[1]).toString();
    });

    it('handles template A overrides', function () {
        let style_knobs = StyleKnobGenerator.get_knobs_from_css(css_overrides_a, 'A');
        expect(style_knobs).toEqual(style_knobs_a);
    });

    it('handles template B overrides', function () {
        let style_knobs = StyleKnobGenerator.get_knobs_from_css(css_overrides_b, 'B');
        expect(style_knobs).toEqual(style_knobs_b);
    });

    it('handles reader app overrides', function () {
        let style_knobs = StyleKnobGenerator.get_knobs_from_css(css_overrides_reader, 'reader');
        expect(style_knobs).toEqual(style_knobs_reader);
    });
});
<|endoftext|>


<commit_before>
var
    util = require('util'),
    fmt = util.format,
    clog = console.log,
    cerr = console.error,
    examiner = require('../lib/examiner'),
    jsforce = require('jsforce')
;

module.exports = {
    auth: auth,
    routes: routes
};

function auth (info, fn) {
    var sfconn = new jsforce.Connection({});

    sfconn.login(
        info.username,
        info.password,
        function(err, uinfo) {
            if (err) {
                cerr('Salesforce auth error (1): %j', err);

                sfconn.login(
                    info.username,
                    info.password + info.securityToken,
                    function (e, uinfo) {
                        if (e) {
                            cerr('Salesforce auth error (2): %j', e);
                            return fn({ ok: false, error: e });
                        }

                        info.conn = sfconn;

                        fn({ ok: true, info: info });
                    }
                );

                return;
            }

            info.conn = sfconn;

            fn({ ok: true, info: info });
        }
    );
}

function routes (conf) {
    var fields = [ ];

    if (!conf.queries) {
        return [];
    }

    return Object.keys(conf.queries).map(
        function (qname) {
            var qtmpl = conf.queries[qname],
                fields = examiner.examineQuery(qtmpl),
                qvars = examiner.findQvars(qtmpl),
                rpath = fmt('/query/%s', qname),
                template = examiner.tmplify(qtmpl),
                rproc = sfQueryRoute(template, qvars)
            ;

            return { method: 'get', path: rpath, proc: rproc, fields: fields }
        }
    );
}

function sfQueryRoute (template, qvars) {
    function mahRoute (req, res) {
        var info = req.iq.get('salesforce'), // req.iq holds the session
            args = [ ],
            qvals = { }
        ;

        function blorg (e) {
            res.send({ ok: false, error: e });
        }

        if (!info) {
            return blorg("Query: SalesForce auth info not found.");
        }

        if (!info.conn) {
            return blorg("Query: No SalesForce connection?!");
        }

        var checkIt = examiner.validateQueryVars(req, qvars);

        if (checkIt.ok) {
            qvals = checkIt.results;
        }
        else {
            return blorg(checkIt.error);
        }

        var qtext = template(qvals),
            conn = info.conn,
            p = conn.query(qtext)
        ;

        // promises, promises...

        p.then(
            function(result) {
                res.send({ ok: true, result: result});
            },
            function (e) {
                cerr('Salesforce query error: %j', e);
                blorg(e);
            }
        );
    }

    return mahRoute;
}

<commit_msg>pluralizing sf plugin<commit_after>
var
    util = require('util'),
    fmt = util.format,
    clog = console.log,
    cerr = console.error,
    examiner = require('../lib/examiner'),
    jsforce = require('jsforce')
;

module.exports = {
    auth: auth,
    routes: routes
};

function auth (info, fn) {
    var sfconn = new jsforce.Connection({});

    sfconn.login(
        info.username,
        info.password,
        function (err, uinfo) {
            if (err) {
                cerr('Salesforce auth error (1): %j', err);

                sfconn.login(
                    info.username,
                    info.password + info.securityToken,
                    function (e, uinfo) {
                        if (e) {
                            cerr('Salesforce auth error (2): %j', e);
                            return fn({ ok: false, error: e });
                        }

                        info.conn = sfconn;

                        fn({ ok: true, info: info });
                    }
                );

                return;
            }

            info.conn = sfconn;

            fn({ ok: true, info: info });
        }
    );
}

function routes (conf) {
    var fields = [ ];

    if (!conf.queries) {
        return [];
    }

    return Object.keys(conf.queries).map(
        function (qname) {
            var qtmpl = conf.queries[qname],
                fields = examiner.examineQuery(qtmpl),
                qvars = examiner.findQvars(qtmpl),
                rpath = fmt('/query/%s', qname),
                template = examiner.tmplify(qtmpl),
                rproc = sfQueryRoute(template, qvars)
            ;

            return { method: 'get', path: rpath, proc: rproc, fields: fields }
        }
    );
}

function sfQueryRoute (template, qvars) {
    function mahRoute (req, res) {
        var info = req.iq.get('salesforce'), // req.iq holds the session
            args = [ ],
            qvals = { }
        ;

        function blorg (e) {
            res.send({ ok: false, error: e });
        }

        if (!info) {
            return blorg("Query: SalesForce auth info not found.");
        }

        if (!info.conn) {
            return blorg("Query: No SalesForce connection?!");
        }

        var checkIt = examiner.validateQueryVars(req, qvars);

        if (checkIt.ok) {
            qvals = checkIt.results;
        }
        else {
            return blorg(checkIt.error);
        }

        var qtext = template(qvals),
            conn = info.conn,
            p = conn.query(qtext)
        ;

        // promises, promises...

        p.then(
            function (results) {
                res.send({ ok: true, results: results });
            },
            function (e) {
                cerr('Salesforce query error: %j', e);
                blorg(e);
            }
        );
    }

    return mahRoute;
}

<|endoftext|>


<commit_before>var async = require('async');

var request = require('util/request').request;
var testUtil = require('util/test');
var misc = require('util/misc');

exports.test_rate_limiting = function(test, assert) {
  var options = {'return_response': true, 'expected_status_codes': [200]}, server = null,
      reqCountPath1 = 0, reqCountPath2 = 0;

  options.headers = {'X-Tenant-Id': '7777', 'X-Auth-Token': 'dev'};
  async.waterfall([
    function startBackendServer(callback) {
      testUtil.getTestHttpServer(9001, '127.0.0.1', function(err, _server) {
        server = _server;

        server.get('/test/a', function(req, res) {
          var code;

          if (req.headers.hasOwnProperty('x-rp-error-code')) {
            code = testUtil.PROXY_ERROR_CODE_TO_HTTP_CODE_MAP[req.headers['x-rp-error-code']];
            res.writeHead(code, {});
            res.end(req.headers['x-rp-error-message']);
            return;
          }

          reqCountPath1++;
          res.writeHead(200, {});
          res.end();
        });

        server.get('/bar', function(req, res) {
          var code;

          if (req.headers.hasOwnProperty('x-rp-error-code')) {
            code = testUtil.PROXY_ERROR_CODE_TO_HTTP_CODE_MAP[req.headers['x-rp-error-code']];
            res.writeHead(code, {});
            res.end(req.headers['x-rp-error-message']);
            return;
          }

          reqCountPath2++;
          res.writeHead(200, {});
          res.end();
        });

        server.post('/rate_limits', function(req, res) {
          assert.equal(req.url, '/rate_limits?foo=bar&bar=baz');
          var buffer = '';

          req.on('data', function(chunk) {
            buffer += chunk;
          });

          req.on('end', function() {
            res.writeHead(200, {'content-type': 'application/json'});
            res.end(buffer);
          });
        });

        testUtil.setupErrorEchoHandlers(server);
        callback();
      });
    },

    function wait(callback) {
      // wait for all the existing rate limits to expire
      setTimeout(callback, 5000);
    },

    function issueRequestsPath1NotRateLimited(callback) {
      async.forEach([1, 2, 3, 4], function(_, callback) {
        request('http://127.0.0.1:9000/test/a', 'GET', null, options, function(err, res) {
          assert.ok(!err);
          assert.equal(res.statusCode, 200);

          callback();
        });
      }, callback);
    },

    function testPath1IsRateLimited(callback) {
      // After 4 requests, path 1 should be rate limited
      request('http://127.0.0.1:9000/test/a', 'GET', null, options, function(err, res) {
        var now = misc.getUnixTimestamp();
        assert.ok(err);
        assert.ok(err.statusCode, 400);

        assert.equal(res.headers['x-ratelimit-path-regex'], '/test/.*');
        assert.equal(res.headers['x-ratelimit-limit'], 4);
        assert.equal(res.headers['x-ratelimit-used'], 4);
        assert.equal(res.headers['x-ratelimit-window'], '4 seconds');
        assert.ok(res.headers['reply-after'] > now);

        assert.match(res.body, /Limit of 4 requests in 4 seconds for path .*? has been reached/i);
        callback();
      });
    },

    function issueRequestsPath2NotRateLimited(callback) {
      async.forEach([6, 7, 8, 9, 10], function(_, callback) {
        request('http://127.0.0.1:9000/bar', 'GET', null, options, function(err, res) {
          assert.ok(!err);
          assert.equal(res.statusCode, 200);
          callback();
        });
      }, callback);
    },

    function testPath2IsRateLimited(callback) {
      // After 10 requests total, path 2 should be rate limited
      request('http://127.0.0.1:9000/bar', 'GET', null, options, function(err, res) {
        assert.ok(err);
        assert.ok(err.statusCode, 400);

        assert.equal(res.headers['x-ratelimit-path-regex'], '/.*');
        assert.equal(res.headers['x-ratelimit-limit'], 10);
        assert.equal(res.headers['x-ratelimit-window'], '4 seconds');

        assert.match(res.body, /Limit of 10 requests in 4 seconds for path .*? has been reached/i);
        callback();
      });
    },

    function wait(callback) {
      // Bucket size is 4 seconds, wait period + 2 seconds before checking
      // the limits again
      setTimeout(callback, (6 * 1000));
    },

    function testPath1IsNotRateLimited(callback) {
      // Limit shouldn't affect this request anymore
      request('http://127.0.0.1:9000/test/a', 'GET', null, options, function(err, res) {
        assert.ok(!err);
        assert.equal(res.statusCode, 200);

        assert.equal(res.headers['x-ratelimit-path-regex'], '/test/.*');
        assert.equal(res.headers['x-ratelimit-limit'], 4);
        assert.equal(res.headers['x-ratelimit-used'], 1);
        assert.equal(res.headers['x-ratelimit-window'], '4 seconds');
        callback();
      });
    },

    function testPath2IsNotRateLimited(callback) {
      // Limit shouldn't affect this request anymore
      request('http://127.0.0.1:9000/bar', 'GET', null, options, function(err, res) {
        assert.ok(!err);
        assert.equal(res.statusCode, 200);

        callback();
      });
    },

    function getLimits(callback) {
      request('http://127.0.0.1:9000/limits?foo=bar&bar=baz', 'GET', null, options, function(err, res) {
        var data = JSON.parse(res.body);

        assert.ok(!err);
        assert.equal(res.statusCode, 200);
        assert.equal(data.length, 2); // 2 rules

        assert.equal(data[0].method, 'GET');
        assert.equal(data[0].path_regex, '/.*');
        assert.equal(data[0].limit, 10);
        assert.equal(data[0].used, 2);
        assert.equal(data[0].period, 4);
        callback();
      });
    }
  ],

  function(err) {
    if (server) {
      server.close();
    }

    assert.equal(reqCountPath1, 5);
    assert.ok([5, 6].indexOf(reqCountPath2) !== -1);
    test.finish();
  });
};
<commit_msg>Add a test case for it.<commit_after>var querystring = require('querystring');

var async = require('async');

var request = require('util/request').request;
var testUtil = require('util/test');
var misc = require('util/misc');

exports.test_rate_limiting = function(test, assert) {
  var options = {'return_response': true, 'expected_status_codes': [200]}, server = null,
      adminOptions = {'return_response': true, 'expected_status_codes': [200]},
      reqCountPath1 = 0, reqCountPath2 = 0;

  options.headers = {'X-Tenant-Id': '7777', 'X-Auth-Token': 'dev'};
  async.waterfall([
    function startBackendServer(callback) {
      testUtil.getTestHttpServer(9001, '127.0.0.1', function(err, _server) {
        server = _server;

        server.get('/test/a', function(req, res) {
          var code;

          if (req.headers.hasOwnProperty('x-rp-error-code')) {
            code = testUtil.PROXY_ERROR_CODE_TO_HTTP_CODE_MAP[req.headers['x-rp-error-code']];
            res.writeHead(code, {});
            res.end(req.headers['x-rp-error-message']);
            return;
          }

          reqCountPath1++;
          res.writeHead(200, {});
          res.end();
        });

        server.get('/bar', function(req, res) {
          var code;

          if (req.headers.hasOwnProperty('x-rp-error-code')) {
            code = testUtil.PROXY_ERROR_CODE_TO_HTTP_CODE_MAP[req.headers['x-rp-error-code']];
            res.writeHead(code, {});
            res.end(req.headers['x-rp-error-message']);
            return;
          }

          reqCountPath2++;
          res.writeHead(200, {});
          res.end();
        });

        server.post('/rate_limits', function(req, res) {
          assert.equal(req.url, '/rate_limits?foo=bar&bar=baz');
          var buffer = '';

          req.on('data', function(chunk) {
            buffer += chunk;
          });

          req.on('end', function() {
            res.writeHead(200, {'content-type': 'application/json'});
            res.end(buffer);
          });
        });

        testUtil.setupErrorEchoHandlers(server);
        callback();
      });
    },

    function wait(callback) {
      // wait for all the existing rate limits to expire
      setTimeout(callback, 5000);
    },

    function issueRequestsPath1NotRateLimited(callback) {
      async.forEach([1, 2, 3, 4], function(_, callback) {
        request('http://127.0.0.1:9000/test/a', 'GET', null, options, function(err, res) {
          assert.ok(!err);
          assert.equal(res.statusCode, 200);

          callback();
        });
      }, callback);
    },

    function testPath1IsRateLimited(callback) {
      // After 4 requests, path 1 should be rate limited
      request('http://127.0.0.1:9000/test/a', 'GET', null, options, function(err, res) {
        var now = misc.getUnixTimestamp();
        assert.ok(err);
        assert.ok(err.statusCode, 400);

        assert.equal(res.headers['x-ratelimit-path-regex'], '/test/.*');
        assert.equal(res.headers['x-ratelimit-limit'], 4);
        assert.equal(res.headers['x-ratelimit-used'], 4);
        assert.equal(res.headers['x-ratelimit-window'], '4 seconds');
        assert.ok(res.headers['reply-after'] > now);

        assert.match(res.body, /Limit of 4 requests in 4 seconds for path .*? has been reached/i);
        callback();
      });
    },

    function issueRequestsPath2NotRateLimited(callback) {
      async.forEach([6, 7, 8, 9, 10], function(_, callback) {
        request('http://127.0.0.1:9000/bar', 'GET', null, options, function(err, res) {
          assert.ok(!err);
          assert.equal(res.statusCode, 200);
          callback();
        });
      }, callback);
    },

    function testPath2IsRateLimited(callback) {
      // After 10 requests total, path 2 should be rate limited
      request('http://127.0.0.1:9000/bar', 'GET', null, options, function(err, res) {
        assert.ok(err);
        assert.ok(err.statusCode, 400);

        assert.equal(res.headers['x-ratelimit-path-regex'], '/.*');
        assert.equal(res.headers['x-ratelimit-limit'], 10);
        assert.equal(res.headers['x-ratelimit-window'], '4 seconds');

        assert.match(res.body, /Limit of 10 requests in 4 seconds for path .*? has been reached/i);
        callback();
      });
    },

    function wait(callback) {
      // Bucket size is 4 seconds, wait period + 2 seconds before checking
      // the limits again
      setTimeout(callback, (6 * 1000));
    },

    function testPath1IsNotRateLimited(callback) {
      // Limit shouldn't affect this request anymore
      request('http://127.0.0.1:9000/test/a', 'GET', null, options, function(err, res) {
        assert.ok(!err);
        assert.equal(res.statusCode, 200);

        assert.equal(res.headers['x-ratelimit-path-regex'], '/test/.*');
        assert.equal(res.headers['x-ratelimit-limit'], 4);
        assert.equal(res.headers['x-ratelimit-used'], 1);
        assert.equal(res.headers['x-ratelimit-window'], '4 seconds');
        callback();
      });
    },

    function testPath2IsNotRateLimited(callback) {
      // Limit shouldn't affect this request anymore
      request('http://127.0.0.1:9000/bar', 'GET', null, options, function(err, res) {
        assert.ok(!err);
        assert.equal(res.statusCode, 200);

        callback();
      });
    },

    function getLimits(callback) {
      request('http://127.0.0.1:9000/limits?foo=bar&bar=baz', 'GET', null, options, function(err, res) {
        var data = JSON.parse(res.body);

        assert.ok(!err);
        assert.equal(res.statusCode, 200);
        assert.equal(data.length, 2); // 2 rules

        assert.equal(data[0].method, 'GET');
        assert.equal(data[0].path_regex, '/.*');
        assert.equal(data[0].limit, 10);
        assert.equal(data[0].used, 2);
        assert.equal(data[0].period, 4);
        callback();
      });
    },

    function useAdminApiToOverrideUserLimits(callback) {
      var url, qs;

      url = 'http://127.0.0.1:8001/v1.0/limits';
      qs = {'userId': '02222', 'pathRegex': '/test/.*', 'value': 8};
      url += '?' + querystring.stringify(qs);

      adminOptions.headers = {'X-Api-Key': 'abcd'};
      request(url, 'POST', null, adminOptions, function(err, res) {
        assert.ok(!err);
        assert.equal(res.statusCode, 200);
        callback();
      });
    },

  function issueRequestsPath1NotRateLimited(callback) {
    options.headers = {'X-Tenant-Id': '02222', 'X-Auth-Token': 'never-cache-this'};
      async.forEach([1, 2, 3, 4, 5, 6, 7, 8], function(_, callback) {
        request('http://127.0.0.1:9000/test/a', 'GET', null, options, function(err, res) {
          assert.ok(!err);
          assert.equal(res.statusCode, 200);

          callback();
        });
      }, callback);
    },

    function testPath1IsRateLimited(callback) {
      // After 9 requests, path 1 should be rate limited
      request('http://127.0.0.1:9000/test/a', 'GET', null, options, function(err, res) {
        var now = misc.getUnixTimestamp();
        assert.ok(err);
        assert.ok(err.statusCode, 400);

        assert.equal(res.headers['x-ratelimit-path-regex'], '/test/.*');
        assert.equal(res.headers['x-ratelimit-limit'], 8);
        assert.equal(res.headers['x-ratelimit-used'], 8);
        assert.equal(res.headers['x-ratelimit-window'], '4 seconds');
        assert.ok(res.headers['reply-after'] > now);

        assert.match(res.body, /Limit of 8 requests in 4 seconds for path .*? has been reached/i);
        callback();
      });
    }
  ],

  function(err) {
    if (server) {
      server.close();
    }

    assert.equal(reqCountPath1, 5 + 8);
    assert.ok([5, 6].indexOf(reqCountPath2) !== -1);
    test.finish();
  });
};
<|endoftext|>


<commit_before>const crypto = require('crypto');
const express = require('express');
const bodyParser = require('body-parser');
const fs = require('fs');
const path = require('path');
const assert = require('assert');

const testAPI = require('../test-api');
const confConsts = require('../../backend/common/conf-constants');
const confMgmt = require('../../backend/api/conf-mgmt');
const security = require('../../backend/api/security');
const securityConsts = require('../../backend/common/security-constants');
const storageUtils = require('../../backend/api/storage-utils');

const webhooks = require('../../backend/api/webhooks');

const PORT = 8181;
const THE_SECRET = 'this is a secret !';

const webhooks2Test = [
	{
		// everything
		id: 1,
		relativePath: '*',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		// disabled
		id: 2,
		relativePath: '*',
		url: `http://localhost:${PORT}`,
		isDisabled: true
	},
	{
		// won't work, path must be started with a slash
		id: 3,
		relativePath: 'common',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		id: 4,
		relativePath: '/common',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		id: 5,
		relativePath: '/common*',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		id: 6,
		relativePath: '!/common',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		id: 7,
		relativePath: '*.js',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		id: 8,
		relativePath: '/common/test.js',
		secret: 'wrong secret',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		id: 9,
		relativePath: '/common/test.js',
		secret: THE_SECRET,
		url: `http://localhost:${PORT}`,
		isDisabled: false
	}
];

const expectingResults = [
	{webhook: '*', target: '/common', action: 'update'},
	{webhook: '/common', target: '/common', action: 'update'},
	{webhook: '/common*', target: '/common', action: 'update'}
];

// --------------------------------------------------------------------------------
// tests
// --------------------------------------------------------------------------------
function timeout() {
	return new Promise((resolve, reject) => {
		setTimeout(_ => {
			resolve();
		}, 1000);
	});
}

function runTests() {
	return storageUtils.mkdir('/common')
		.then(timeout)
		.then(_ => storageUtils.mkdir('/test'))
		.then(_ => storageUtils.saveFileToStorage('/common/test.js', ''))
		;
}

// --------------------------------------------------------------------------------
// webhooks app
// --------------------------------------------------------------------------------

let webServer;

const webhookResults = [];

const app = express();
app.use(bodyParser.json());

function handleWebhook(req, res, next) {
	const payload = JSON.stringify(req.body);
	if (!payload) {
		return next('Request body empty')
	}

	const checksum = req.get(webhooks.SIG_HEADER);
	if (!checksum) {
		return next();
	}

	const hmac = crypto.createHmac('sha1', THE_SECRET);
	const digest = 'sha1=' + hmac.update(payload).digest('hex');
	if (!checksum || !digest || checksum !== digest) {
		return next(`Request body digest (${digest}) did not match ${webhooks.SIG_HEADER} (${checksum})`)
	}

	return next();
}

app.post('/', handleWebhook, function (req, res) {
	webhookResults.push(req.body);
	res.status(200).send('Ok');
});

app.use((err, req, res, next) => {
	if (err) {
		console.error(err);
		res.status(403).send('Webhook failed !!!');
	}
});

function startWebhooksClient() {
	return new Promise((resolve, reject) => {
		webServer = app.listen(PORT, 'localhost', () => {
			resolve();
		}).on('error', err => {
			console.log(err);
			reject(err);
		});
	});
}

// --------------------------------------------------------------------------------
// nexl app
// --------------------------------------------------------------------------------

function init(predefinedNexlJSFIlesDir, tmpNexlJSFilesDir) {
	// adding a webhooks
	return confMgmt.save(webhooks2Test, confConsts.CONF_FILES.WEBHOOKS);
}

function run() {
	return startWebhooksClient()
		.then(runTests);
}

function done() {
	console.log(webhookResults);
	return timeout().then(_ => {
		if (webServer) {
			webServer.close();
		}
		return Promise.resolve();
	});
}

testAPI.startNexlApp(init, run, done);
<commit_msg>- working on webhooks<commit_after>const crypto = require('crypto');
const express = require('express');
const bodyParser = require('body-parser');
const fs = require('fs');
const path = require('path');
const assert = require('assert');

const testAPI = require('../test-api');
const confConsts = require('../../backend/common/conf-constants');
const confMgmt = require('../../backend/api/conf-mgmt');
const security = require('../../backend/api/security');
const securityConsts = require('../../backend/common/security-constants');
const storageUtils = require('../../backend/api/storage-utils');

const webhooks = require('../../backend/api/webhooks');

const PORT = 8181;
const THE_SECRET = 'this is a secret !';

const webhooks2Test = [
	{
		// everything
		id: 0,
		relativePath: '*',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		// won't work, path must be started with a slash
		id: 1,
		relativePath: 'common',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		id: 2,
		relativePath: '/common',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		id: 3,
		relativePath: '/common*',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		id: 4,
		relativePath: '!/common',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		id: 5,
		relativePath: '*.js',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		id: 6,
		relativePath: '/common/test.js',
		url: `http://localhost:${PORT}`,
		isDisabled: false
	},
	{
		id: 7,
		relativePath: '/common/test.js',
		secret: 'wrong secret',
		url: `http://localhost:${PORT}`,
		isDisabled: true
	},
	{
		id: 8,
		relativePath: '/common/test.js',
		secret: THE_SECRET,
		url: `http://localhost:${PORT}`,
		isDisabled: true
	}
];

const expectingResults = [
	{webhook: '*', target: '/common', action: 'update'},
	{webhook: '/common', target: '/common', action: 'update'},
	{webhook: '/common*', target: '/common', action: 'update'}
];

// --------------------------------------------------------------------------------
// tests
// --------------------------------------------------------------------------------
function timeout() {
	return new Promise((resolve, reject) => {
		setTimeout(_ => {
			resolve();
		}, 1000);
	});
}

function updateWebhooks(state) {
	for (let index = 0; index < webhooks2Test.length; index++) {
		webhooks2Test[index].isDisabled = state[index];
	}

	return confMgmt.save(webhooks2Test, confConsts.CONF_FILES.WEBHOOKS);
}

function runTests() {
	return storageUtils.mkdir('/common')
		.then(timeout)
		.then(_ => storageUtils.mkdir('/test'))
		.then(_ => storageUtils.saveFileToStorage('/common/test.js', ''))
		//                           0     1     2     3     4     5     6    7      8
		.then(_ => updateWebhooks([true, false, true, true, true, true, true, true, true]))
		.then(_ => storageUtils.saveFileToStorage('/common/test.js', ''))
		;
}

// --------------------------------------------------------------------------------
// webhooks app
// --------------------------------------------------------------------------------

let webServer;

const webhookResults = [];

const app = express();
app.use(bodyParser.json());

function handleWebhook(req, res, next) {
	const payload = JSON.stringify(req.body);
	if (!payload) {
		return next('Request body empty')
	}

	const checksum = req.get(webhooks.SIG_HEADER);
	if (!checksum) {
		return next();
	}

	const hmac = crypto.createHmac('sha1', THE_SECRET);
	const digest = 'sha1=' + hmac.update(payload).digest('hex');
	if (!checksum || !digest || checksum !== digest) {
		return next(`Request body digest (${digest}) did not match ${webhooks.SIG_HEADER} (${checksum})`)
	}

	return next();
}

app.post('/', handleWebhook, function (req, res) {
	webhookResults.push(req.body);
	res.status(200).send('Ok');
});

app.use((err, req, res, next) => {
	if (err) {
		console.error(err);
		res.status(403).send('Webhook failed !!!');
	}
});

function startWebhooksClient() {
	return new Promise((resolve, reject) => {
		webServer = app.listen(PORT, 'localhost', () => {
			resolve();
		}).on('error', err => {
			console.log(err);
			reject(err);
		});
	});
}

// --------------------------------------------------------------------------------
// nexl app
// --------------------------------------------------------------------------------

function init(predefinedNexlJSFIlesDir, tmpNexlJSFilesDir) {
	// adding a webhooks
	return confMgmt.save(webhooks2Test, confConsts.CONF_FILES.WEBHOOKS);
}

function run() {
	return startWebhooksClient()
		.then(runTests);
}

function done() {
	console.log(webhookResults);
	return timeout().then(_ => {
		if (webServer) {
			webServer.close();
		}
		return Promise.resolve();
	});
}

testAPI.startNexlApp(init, run, done);
<|endoftext|>


<commit_before>var assert = require('assert');

add_dependencies({
    _: 'third/underscore/underscore.js',
    util: 'js/util.js',
    Dict: 'js/dict.js',
    stream_color: 'js/stream_color.js'
});

var stream_data = require('js/stream_data.js');

(function test_basics() {
    var denmark = {
        subscribed: false,
        color: 'blue',
        name: 'Denmark',
        in_home_view: false
    };
    var social = {
        subscribed: true,
        color: 'red',
        name: 'social',
        in_home_view: true,
        invite_only: true
    };
    stream_data.add_sub('Denmark', denmark);
    stream_data.add_sub('social', social);

    assert.equal(stream_data.get_sub('denmark'), denmark);
    assert.equal(stream_data.get_sub('Social'), social);

    assert.deepEqual(stream_data.subscribed_streams(), ['social']);
    assert.deepEqual(stream_data.get_colors(), ['red']);

    assert(stream_data.all_subscribed_streams_are_in_home_view());

    assert(stream_data.is_subscribed('social'));
    assert(stream_data.is_subscribed('Social'));
    assert(!stream_data.is_subscribed('Denmark'));
    assert(!stream_data.is_subscribed('Rome'));

    assert(stream_data.get_invite_only('social'));
    assert(!stream_data.get_invite_only('unknown'));
    assert.equal(stream_data.get_color('social'), 'red');
    assert.equal(stream_data.get_color('unknown'), global.stream_color.default_color);

    assert.equal(stream_data.get_name('denMARK'), 'Denmark');
    assert.equal(stream_data.get_name('unknown Stream'), 'unknown Stream');

    assert(stream_data.in_home_view('social'));
    assert(!stream_data.in_home_view('denmark'));
}());

(function test_get_and_set() {
    stream_data.clear_subscriptions();
    stream_data.add_sub('Denmark', {name: 'Denmark', subscribed: true});
    assert.deepEqual(stream_data.subscribed_streams(), ['Denmark']);
    var info = stream_data.get_stream_info();
    stream_data.clear_subscriptions();
    assert.deepEqual(stream_data.subscribed_streams(), []);
    stream_data.set_stream_info(info);
    assert.deepEqual(stream_data.subscribed_streams(), ['Denmark']);
}());
<commit_msg>node tests: add some asserts for stream_data.delete_sub.<commit_after>var assert = require('assert');

add_dependencies({
    _: 'third/underscore/underscore.js',
    util: 'js/util.js',
    Dict: 'js/dict.js',
    stream_color: 'js/stream_color.js'
});

var stream_data = require('js/stream_data.js');

(function test_basics() {
    var denmark = {
        subscribed: false,
        color: 'blue',
        name: 'Denmark',
        in_home_view: false
    };
    var social = {
        subscribed: true,
        color: 'red',
        name: 'social',
        in_home_view: true,
        invite_only: true
    };
    stream_data.add_sub('Denmark', denmark);
    stream_data.add_sub('social', social);

    assert.equal(stream_data.get_sub('denmark'), denmark);
    assert.equal(stream_data.get_sub('Social'), social);

    assert.deepEqual(stream_data.subscribed_streams(), ['social']);
    assert.deepEqual(stream_data.get_colors(), ['red']);

    assert(stream_data.all_subscribed_streams_are_in_home_view());

    assert(stream_data.is_subscribed('social'));
    assert(stream_data.is_subscribed('Social'));
    assert(!stream_data.is_subscribed('Denmark'));
    assert(!stream_data.is_subscribed('Rome'));

    assert(stream_data.get_invite_only('social'));
    assert(!stream_data.get_invite_only('unknown'));
    assert.equal(stream_data.get_color('social'), 'red');
    assert.equal(stream_data.get_color('unknown'), global.stream_color.default_color);

    assert.equal(stream_data.get_name('denMARK'), 'Denmark');
    assert.equal(stream_data.get_name('unknown Stream'), 'unknown Stream');

    assert(stream_data.in_home_view('social'));
    assert(!stream_data.in_home_view('denmark'));

    // Deleting a subscription makes you unsubscribed from the perspective of
    // the client.
    // Deleting a subscription is case-insensitive.
    stream_data.delete_sub('SOCIAL');
    assert(!stream_data.is_subscribed('social'));
}());

(function test_get_and_set() {
    stream_data.clear_subscriptions();
    stream_data.add_sub('Denmark', {name: 'Denmark', subscribed: true});
    assert.deepEqual(stream_data.subscribed_streams(), ['Denmark']);
    var info = stream_data.get_stream_info();
    stream_data.clear_subscriptions();
    assert.deepEqual(stream_data.subscribed_streams(), []);
    stream_data.set_stream_info(info);
    assert.deepEqual(stream_data.subscribed_streams(), ['Denmark']);
}());
<|endoftext|>


<commit_before>var mysql = require('../../promise.js');
var co = require('co')

co(function * () {
  var c = yield mysql.createConnection({ port: 3306, user: 'root', namedPlaceholders: true });
  var rows = yield c.query('show databases');
  console.log(rows)
  console.log( yield c.execute('select 1+:toAdd as qqq', {toAdd: 10}) );
  yield c.end();

}).then(function() {
  console.log('done');
}, function(err) {
  console.log(err);
  throw err;
});
<commit_msg>Update co.js<commit_after>var mysql = require('mysql/promise');
var co = require('co')

co(function * () {
  var c = yield mysql.createConnection({ port: 3306, user: 'root', namedPlaceholders: true });
  var rows = yield c.query('show databases');
  console.log(rows)
  console.log( yield c.execute('select 1+:toAdd as qqq', {toAdd: 10}) );
  yield c.end();

}).then(function() {
  console.log('done');
}, function(err) {
  console.log(err);
  throw err;
});
<|endoftext|>


<commit_before>// Import React
import React from "react";

// Import Spectacle Core tags
import {
  Appear,
  BlockQuote,
  Cite,
  ComponentPlayground,
  Deck,
  Heading,
  ListItem,
  Image,
  List,
  Quote,
  Slide,
  Fill,
  Fit,
  Text,
  Table,
  TableHeaderItem,
  TableRow,
  TableItem,
  Layout,

} from "spectacle";

// Import image preloader util
import preloader from "spectacle/lib/utils/preloader";

// Import theme
import createTheme from "spectacle/lib/themes/default";

// Require CSS
require("normalize.css");
require("spectacle/lib/themes/default/index.css");

const styles = {
  bg: {
  backgroundColor: "black",
  padding: "20px",
  }
};

const images = {
  city: require("../assets/city.jpg"),
  kat: require("../assets/kat.png"),
  logo: require("../assets/logoandtag.jpg"),
  markdown: require("../assets/markdown.png"),
  bawdoghouse: require("../assets/Drawing-Dog-House-Coloring-Pages.jpg"),
  bdp: require("../assets/black-diamond-plate-dark.jpg"),
  josie: require("../assets/josie3.jpg"),
  keveren: require("../assets/keveren.png"),
  scott: require("../assets/scott.jpg"),
  doug: require("../assets/doug.jpg"),

};

preloader(images);

const theme = createTheme({
  primary: "white",
  secondary: "#1F2022",
  tertiary: "#03A9FC",
  quartenary: "black",
  option1: "orange",
}, {
  primary: { name: "Bangers", googleFont: true },
  secondary: "Helvetica",
  option1: { name: "Bangers", googleFont: true }
});

export default class Presentation extends React.Component {
  render() {
    return (
      <Deck transition={["zoom", "slide"]} transitionDuration={500} theme={theme}>
        <Slide transition={["zoom"]} bgImage={images.bawdoghouse} bgColor="primary">
          <Heading size={1} fit caps lineHeight={1} textColor="option1">
            Doghouse Dodger
          </Heading>
          <Text margin="10px 0 0" textColor="tertiary" size={1} fit bold>
            Taking care of the thought for a thoughtful man!
          </Text>
        </Slide>
        <Slide transition={["zoom"]} bgImage={images.bdp} bgColor="primary">
        <Image src={images.logo}  height="300px"/>
        </Slide>
        <Slide transition={["fade"]} bgImage={images.bdp} bgColor="tertiary">

          <Heading size={2} textColor="option1">Why Doghouse Dodger?</Heading>
          <Heading size={5} textColor="primary">Do these questions sound familiar?</Heading>
            <Appear><Text size={6} textColor="primary">Do you know what today is?</Text></Appear>
            <Appear><Text size={6} textColor="primary">What do you want to eat?</Text></Appear>
            <Appear><Text size={6} textColor="primary">"You don't remember when my birthday is?"</Text></Appear>
            <Appear><Text size={6} textColor="primary">"What's my favorite color?"</Text></Appear>
            <Appear><Text size={6} textColor="primary">"You don't remember when our first date was?"</Text></Appear>

        </Slide>
        <Slide transition={["fade"]} bgImage={images.bdp} bgColor="tertiary">

          <Heading size={3} textFont="primary" textColor="option1">What can Doghouse Dodger do?</Heading>
            <Appear><Text size={6} textColor="primary">Keep information secure with login</Text></Appear>
            <Appear><Text size={6} textColor="primary">Create multiple relationships</Text></Appear>
            <Appear><Text size={6} textColor="primary">Record all personal information</Text></Appear>
            <Appear><Text size={6} textColor="primary">Record restraunts, date activities, goals</Text></Appear>
            <Appear><Text size={6} textColor="primary">Links to all restraunts, date activities, goals</Text></Appear>
            <Appear><Text size={6} textColor="primary">Random Date Generator</Text></Appear>

        </Slide>
        <Slide transition={["fade"]} bgImage={images.bdp} bgColor="tertiary">

            <Heading size={3} textFont="primary" textColor="option1">Future plans for Doghouse Dodger?</Heading>
              <Appear><Text size={6} textColor="primary">Chat group</Text></Appear>
              <Appear><Text size={6} textColor="primary">Text reminders a week out from important dates</Text></Appear>
              <Appear><Text size={6} textColor="primary">Text reminders on important dates</Text></Appear>
              <Appear><Text size={6} textColor="primary">Intigrate important dates with Google Calendar</Text></Appear>
              <Appear><Text size={6} textColor="primary">Emergency button (Auto order flowers, chocolates, etc)</Text></Appear>
              <Appear><Text size={6} textColor="primary"></Text></Appear>

        </Slide>
        <Slide  transition={["slide"]} bgImage={images.bdp} bgColor="tertiary">

          <Heading size={3} textFont="primary"  textColor="option1">Technologies used</Heading>
            <Text size={6} textColor="primary">React</Text>
            <Text size={6} textColor="primary">Redux</Text>
            <Text size={6} textColor="primary">Ruby/Rails</Text>
            <Text size={6} textColor="primary">HTML/CSS</Text>

        </Slide>
        <Slide transition={["fade"]} bgImage={images.bdp} bgColor="tertiary">

            <Heading size={3} textFont="primary" textColor="option1">Challenges</Heading>
              <Appear><Text size={6} textColor="primary">Routing</Text></Appear>
              <Appear><Text size={6} textColor="primary">Carrying relationship Ids through different components</Text></Appear>
              <Appear><Text size={6} textColor="primary">Breaking code when different team members merge</Text></Appear>
              <Appear><Text size={6} textColor="primary">Sticking to Agile/Scrum environment</Text></Appear>

        </Slide>
        <Slide transition={["fade"]} bgImage={images.bdp} bgColor="primary" textColor="tertiary">
        <Heading margin="20px" size={1} fit textColor="option1">
          The Doghouse Dodger Team
        </Heading>
             <Layout align="flex-start center">
               <Fill>

                   <TableRow>
                     <TableItem>
                       <Image src={images.doug} margin="80px 20px 0px 0px" height="200px"/>
                     </TableItem>
                     <TableItem>
                       <Image src={images.josie} margin="80px 20px 0px 20px" height="200px"/>
                     </TableItem>
                     <TableItem>
                       <Image src={images.keveren} margin="80px 20px 0px 20px" height="200px"/>
                     </TableItem>
                     <TableItem>
                       <Image src={images.scott} margin="80px 20px 0px 20px" height="200px"/>
                     </TableItem>
                   </TableRow>
                   <TableRow>
                     <TableItem>Doug</TableItem>
                     <TableItem>Josie</TableItem>
                     <TableItem>Keveren</TableItem>
                     <TableItem>Scott</TableItem>
                   </TableRow>

               </Fill>
             </Layout>
        </Slide>
        <Slide transition={["fade"]} bgColor="secondary" textColor="primary">

            <Quote textColor="primary">Happy Wife,</Quote><Quote textColor="option1"> Happy Life</Quote>
            Author- The Doghouse Dodger

        </Slide>
      </Deck>
    );
  }
}
<commit_msg>change in upcoming features<commit_after>// Import React
import React from "react";

// Import Spectacle Core tags
import {
  Appear,
  BlockQuote,
  Cite,
  ComponentPlayground,
  Deck,
  Heading,
  ListItem,
  Image,
  List,
  Quote,
  Slide,
  Fill,
  Fit,
  Text,
  Table,
  TableHeaderItem,
  TableRow,
  TableItem,
  Layout,

} from "spectacle";

// Import image preloader util
import preloader from "spectacle/lib/utils/preloader";

// Import theme
import createTheme from "spectacle/lib/themes/default";

// Require CSS
require("normalize.css");
require("spectacle/lib/themes/default/index.css");

const styles = {
  bg: {
  backgroundColor: "black",
  padding: "20px",
  }
};

const images = {
  city: require("../assets/city.jpg"),
  kat: require("../assets/kat.png"),
  logo: require("../assets/logoandtag.jpg"),
  markdown: require("../assets/markdown.png"),
  bawdoghouse: require("../assets/Drawing-Dog-House-Coloring-Pages.jpg"),
  bdp: require("../assets/black-diamond-plate-dark.jpg"),
  josie: require("../assets/josie3.jpg"),
  keveren: require("../assets/keveren.png"),
  scott: require("../assets/scott.jpg"),
  doug: require("../assets/doug.jpg"),

};

preloader(images);

const theme = createTheme({
  primary: "white",
  secondary: "#1F2022",
  tertiary: "#03A9FC",
  quartenary: "black",
  option1: "orange",
}, {
  primary: { name: "Bangers", googleFont: true },
  secondary: "Helvetica",
  option1: { name: "Bangers", googleFont: true }
});

export default class Presentation extends React.Component {
  render() {
    return (
      <Deck transition={["zoom", "slide"]} transitionDuration={500} theme={theme}>
        <Slide transition={["zoom"]} bgImage={images.bawdoghouse} bgColor="primary">
          <Heading size={1} fit caps lineHeight={1} textColor="option1">
            Doghouse Dodger
          </Heading>
          <Text margin="10px 0 0" textColor="tertiary" size={1} fit bold>
            Taking care of the thought for a thoughtful man!
          </Text>
        </Slide>
        <Slide transition={["zoom"]} bgImage={images.bdp} bgColor="primary">
        <Image src={images.logo}  height="300px"/>
        </Slide>
        <Slide transition={["fade"]} bgImage={images.bdp} bgColor="tertiary">

          <Heading size={2} textColor="option1">Why Doghouse Dodger?</Heading>
          <Heading size={5} textColor="primary">Do these questions sound familiar?</Heading>
            <Appear><Text size={6} textColor="primary">Do you know what today is?</Text></Appear>
            <Appear><Text size={6} textColor="primary">What do you want to eat?</Text></Appear>
            <Appear><Text size={6} textColor="primary">"You don't remember when my birthday is?"</Text></Appear>
            <Appear><Text size={6} textColor="primary">"What's my favorite color?"</Text></Appear>
            <Appear><Text size={6} textColor="primary">"You don't remember when our first date was?"</Text></Appear>

        </Slide>
        <Slide transition={["fade"]} bgImage={images.bdp} bgColor="tertiary">

          <Heading size={3} textFont="primary" textColor="option1">What can Doghouse Dodger do?</Heading>
            <Appear><Text size={6} textColor="primary">Keep information secure with login</Text></Appear>
            <Appear><Text size={6} textColor="primary">Create multiple relationships</Text></Appear>
            <Appear><Text size={6} textColor="primary">Record all personal information</Text></Appear>
            <Appear><Text size={6} textColor="primary">Record restraunts, date activities, goals</Text></Appear>
            <Appear><Text size={6} textColor="primary">Links to all restraunts, date activities, goals</Text></Appear>
            <Appear><Text size={6} textColor="primary">Random Date Generator</Text></Appear>

        </Slide>
        <Slide transition={["fade"]} bgImage={images.bdp} bgColor="tertiary">

            <Heading size={3} textFont="primary" textColor="option1">Future plans for Doghouse Dodger?</Heading>
              <Appear><Text size={6} textColor="primary">Chat group</Text></Appear>
              <Appear><Text size={6} textColor="primary">Text reminders a week out and day of important dates</Text></Appear>
              <Appear><Text size={6} textColor="primary">Intigrate important dates with Google Calendar</Text></Appear>
              <Appear><Text size={6} textColor="primary">Emergency button (Auto order flowers, chocolates, etc)</Text></Appear>
              <Appear><Text size={6} textColor="primary">Gift tracker</Text></Appear>

        </Slide>
        <Slide  transition={["slide"]} bgImage={images.bdp} bgColor="tertiary">

          <Heading size={3} textFont="primary"  textColor="option1">Technologies used</Heading>
            <Text size={6} textColor="primary">React</Text>
            <Text size={6} textColor="primary">Redux</Text>
            <Text size={6} textColor="primary">Ruby/Rails</Text>
            <Text size={6} textColor="primary">HTML/CSS</Text>

        </Slide>
        <Slide transition={["fade"]} bgImage={images.bdp} bgColor="tertiary">

            <Heading size={3} textFont="primary" textColor="option1">Challenges</Heading>
              <Appear><Text size={6} textColor="primary">Routing</Text></Appear>
              <Appear><Text size={6} textColor="primary">Carrying relationship Ids through different components</Text></Appear>
              <Appear><Text size={6} textColor="primary">Breaking code when different team members merge</Text></Appear>
              <Appear><Text size={6} textColor="primary">Sticking to Agile/Scrum environment</Text></Appear>

        </Slide>
        <Slide transition={["fade"]} bgImage={images.bdp} bgColor="primary" textColor="tertiary">
        <Heading margin="20px" size={1} fit textColor="option1">
          The Doghouse Dodger Team
        </Heading>
             <Layout align="flex-start center">
               <Fill>

                   <TableRow>
                     <TableItem>
                       <Image src={images.doug} margin="80px 20px 0px 0px" height="200px"/>
                     </TableItem>
                     <TableItem>
                       <Image src={images.josie} margin="80px 20px 0px 20px" height="200px"/>
                     </TableItem>
                     <TableItem>
                       <Image src={images.keveren} margin="80px 20px 0px 20px" height="200px"/>
                     </TableItem>
                     <TableItem>
                       <Image src={images.scott} margin="80px 20px 0px 20px" height="200px"/>
                     </TableItem>
                   </TableRow>
                   <TableRow>
                     <TableItem>Doug</TableItem>
                     <TableItem>Josie</TableItem>
                     <TableItem>Keveren</TableItem>
                     <TableItem>Scott</TableItem>
                   </TableRow>

               </Fill>
             </Layout>
        </Slide>
        <Slide transition={["fade"]} bgColor="secondary" textColor="primary">

            <Quote textColor="primary">Happy Wife,</Quote><Quote textColor="option1"> Happy Life</Quote>
            Author- The Doghouse Dodger

        </Slide>
      </Deck>
    );
  }
}
<|endoftext|>