<commit_before>'use strict';

const domino = require('domino');
const underscore = require('underscore');
const uuid = require('cassandra-uuid').TimeUuid;
const HTTPError = require('./util').HTTPError;
const transforms = require('./transforms');
const mUtil = {};

const NS_MAIN = 0;

mUtil.CONTENT_TYPES = {
    mobileSections: { name: 'mobile-sections', version: '0.13.1' },
    media: { name: 'Media', version: '1.0.0' },
    readHtml: { name: 'ReadHtml', version: '0.1.0' },
    summary: { name: 'Summary', version: '1.3.0' },
    definition: { name: 'definition', version: '0.7.4' },
    random: { name: 'random', version: '0.6.0' },
    announcements: { name: 'announcements', version: '0.1.0' },
    compilations: { name: 'compilations', version: '0.1.0' },
    onthisday: { name: 'onthisday', version: '0.3.2' },
    unpublished: { name: 'unpublished', version: '0.0.0' }
};

mUtil.SUMMARY_NS_WHITELIST = [ NS_MAIN ];

mUtil.setContentType = function(res, spec, mainType = 'application/json') {
    if (!spec.name || !spec.version) {
        throw new HTTPError({
            status: 500,
            type: 'invalid_content_type',
            title: 'Only use the allowed content-types',
            detail: `${spec.name}/${spec.version} is not an allowed content-type!`
        });
    }

    res.type(`${mainType}; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/${spec.name}/${spec.version}"`);
};

/**
 * Check if a variable is empty.
 * @param {string} val input value
 * @return {!boolean} true if val is null, undefined, an empty object, an empty array, or an empty
 * string.
 */
mUtil.isEmpty = function(val) {
    return !underscore.isNumber(val)
        && !underscore.isBoolean(val)
        && underscore.isEmpty(val);
};

mUtil.isNonempty = underscore.negate(mUtil.isEmpty);

/**
 * @param {*} val input value
 * @param {*} [fallback] the default value to assign if val is empty
 * @return {*} val if nonempty, else fallback.
*/
mUtil.defaultVal = function(val, fallback) {
    return underscore.isEmpty(val) ? fallback : val;
};

/**
 * @param {*} val input value
 * @return {*} val less empty elements.
*/
mUtil.filterEmpty = function(val) {
    if (Array.isArray(val)) {
        return val.map(mUtil.filterEmpty).filter(mUtil.isNonempty);
    }
    if (underscore.isObject(val)) {
        return underscore.pick(underscore.mapObject(val, mUtil.filterEmpty), mUtil.isNonempty);
    }
    return val;
};

/**
 * Sets the ETag header on the response objecti, comprised of the revision ID and
 * the time UUID. If the latter is not given, the current time stamp is used to
 * generate it.
 * @param {!Object}  response the HTTPResponse object on which to set the header
 * @param {?number} revision the revision integer ID to use
 * @param {?string}  tid      the time UUID to use; optional
 */
mUtil.setETag = function(response, revision, tid) {
    // we want to bail out if the revision hasn't been supplied, except
    // in the case revision === 0 because 0 is actually a valid rev_id
    if (!revision && revision !== 0) {
        return;
    }
    if (!tid) {
        tid = uuid.now().toString();
    }
    response.set('etag', `"${revision}/${tid}"`);
};

/**
 * Convert mobile to canonical domain,
 * e.g., 'en.m.wikipedia.org' => 'en.wikipedia.org'
 */
mUtil.mobileToCanonical = function(domain) {
    return domain.replace('.m.', '.');
};

/**
 * Remove the top-level domain from a domain string, e.g., 'en.wikipedia.org' ->
 * 'en.wikipedia'.
 */
mUtil.removeTLD = function(domain) {
    return domain.split('.').slice(0,2).join('.');
};

/**
 * Get the language of the wiki base don the domain name.
 * Example: 'en.wikipedia.org' -> 'en'.
 */
mUtil.getLanguageFromDomain = function(domain) {
    return domain.split('.')[0];
};

mUtil.removeFragment = function(href) {
    if (href.indexOf('#') > -1) {
        return href.substring(0, href.indexOf('#'));
    }
    return href;
};

mUtil.removeLinkPrefix = function(href) {
    return href.replace(/^\.\//, '');
};

mUtil.extractDbTitleFromAnchor = function(anchor) {
    return anchor && mUtil.removeFragment(mUtil.removeLinkPrefix(anchor.getAttribute('href')));
};

/**
 * Merge two arrays of objects by the specified property.  Modifies arr1 in place and preserves its
 * order.
 * @param {?Object[]} arr1 An array of objects
 * @param {?Object[]} arr2 An array of objects
 * @param {!String} prop the property on which to merge the objects in the arrays
 * @param {!Boolean} push whether to extend arr1 with objects from arr2 for which 'prop' is
 * defined but no object in arr1 has the same value
 * Credit: https://jsfiddle.net/guya/eAWKR/.
 */
mUtil.mergeByProp = function(arr1, arr2, prop, push) {
    if (!arr1 || !arr2) {
        return;
    }

    arr2.forEach((arr2obj) => {
        // Find the object in arr1 with the same value of 'prop' as this object in arr2
        const arr1obj = arr1.find(arr1obj => arr1obj[prop] === arr2obj[prop]);

        // If the object already exists, extend it with the new values from
        // arr2, otherwise conditionally add the new object to arr1.
        if (arr1obj) {
            Object.assign(arr1obj, arr2obj);
        } else if (push) {
            arr1.push(arr2obj);
        }
    });
};

mUtil.getRbPageSummaryUrl = function(restbaseTpl, domain, title) {
    const request = restbaseTpl.expand({
        request: {
            params: {
                domain,
                path: `page/summary/${encodeURIComponent(title)}` },
        }
    });
    return request.uri;
};

/**
 * Builds a dictionary containing the various forms of a page title that a client may need.
 * @param {!Object} title a mediawiki-title Title object constructed from a page title string
 * @param {!Object} meta page metadata
 * @return {!Object} a set of useful page title strings
 */
mUtil.buildTitleDictionary = function(title, meta) {
    return {
        canonical: title.getPrefixedDBKey(),
        normalized: meta.normalizedtitle,
        display: meta.displaytitle,
    };
};

/*
 * Build a page summary
 * @param {!String} domain the request domain
 * @param {!Object} title a mediawiki-title object for the page title
 * @param {!Object} pageData raw page data for the page
 * @param {?boolean} disambiguation whether the page is a disambiguation page
 * @return {!Object} a summary 2.0 spec-compliant page summary object
 */
mUtil.buildSummary = function(domain, title, pageData, disambiguation = false) {
    const page = pageData.page;
    const meta = pageData.meta;
    const isContentModelWikitext = meta.contentmodel === 'wikitext';
    const isWhiteListedNamespace = mUtil.SUMMARY_NS_WHITELIST.includes(meta.ns);
    const isRedirect = meta.redirect;

    if (!isContentModelWikitext) {
        return { code: 204 };
    }

    if (!isWhiteListedNamespace) {
        return { code: 204 };
    }

    if (isRedirect) {
        return { code: 204 };
    }

    const leadText = domino.createDocument(page.sections[0].text);
    const intro = transforms.extractLeadIntroduction(leadText);
    const summary = intro.length ? transforms.summarize(intro) : { extract: '', extract_html: '' };

    return Object.assign({
        code: 200,
        type : disambiguation ? 'disambiguation' : 'standard',
        title: meta.normalizedtitle,
        displaytitle: meta.displaytitle,
        namespace: { id: meta.ns, text: meta.nsText },
        titles: mUtil.buildTitleDictionary(title, meta),
        pageid: meta.id,
        thumbnail: meta.thumbnail,
        originalimage: meta.originalimage,
        lang: meta.lang,
        dir: meta.dir,
        revision: page.revision,
        tid: page.tid,
        timestamp: meta.lastmodified,
        description: meta.description,
        coordinates: meta.geo && {
            lat: meta.geo.latitude,
            lon: meta.geo.longitude
        },
        content_urls: mUtil.buildContentUrls(domain, title, meta),
        api_urls: mUtil.buildApiUrls(domain, title, meta),
    }, summary);
};

mUtil.buildContentUrls = function(domain, title, meta) {
    const mobileBaseUrl = meta.mobileHost;
    return {
        desktop: {
            page: `https://${domain}/wiki/${title.getPrefixedDBKey()}`,
            revisions: `https://${domain}/wiki/${title.getPrefixedDBKey()}?action=history`,
            edit: `https://${domain}/wiki/${title.getPrefixedDBKey()}?action=edit`,
            talk: meta.talkNsText && `https://${domain}/wiki/${meta.talkNsText}:${title.getKey()}`
        },
        mobile: mobileBaseUrl && {
            page: `${mobileBaseUrl}/wiki/${title.getPrefixedDBKey()}`,
            revisions: `${mobileBaseUrl}/wiki/Special:History/${title.getPrefixedDBKey()}`,
            edit: `${mobileBaseUrl}/wiki/${title.getPrefixedDBKey()}?action=edit`,
            talk: meta.talkNsText && `${mobileBaseUrl}/wiki/${meta.talkNsText}:${title.getKey()}`
        }
    };
};

mUtil.buildApiUrls = function(domain, title, meta) {
    return {
        summary: `https://${domain}/api/rest_v1/page/summary/${title.getPrefixedDBKey()}`,
        // read_html: `https://${domain}/api/rest_v1/page/read-html/${title.getPrefixedDBKey()}`,
        // content_html: `https://${domain}/api/rest_v1/page/content-html/${title.getPrefixedDBKey()}`,
        // metadata: `https://${domain}/api/rest_v1/page/metadata/${title.getPrefixedDBKey()}`,
        // references: `https://${domain}/api/rest_v1/page/references/${title.getPrefixedDBKey()}`,
        // media: `https://${domain}/api/rest_v1/page/media/${title.getPrefixedDBKey()}`,
        edit_html: `https://${domain}/api/rest_v1/page/html/${title.getPrefixedDBKey()}`,
        talk_page_html: meta.talkNsText && `https://${domain}/api/rest_v1/page/html/${meta.talkNsText}:${title.getKey()}`
    };
};

mUtil.throw404 = function(message) {
    throw new HTTPError({
        status: 404,
        type: 'not_found',
        title: 'Not found',
        detail: message
    });
};

mUtil.deduplicate = function(array) {
    return Array.from(new Set(array));
};

/* jslint bitwise: true */
/* eslint no-bitwise: ["error", { "allow": ["<<"] }] */
mUtil.hashCode = function(string) {
    return string.split('').reduce((prevHash, currVal) =>
        ((prevHash << 5) - prevHash) + currVal.charCodeAt(0), 0);
};

module.exports = mUtil;
<commit_msg>Bump onthisday version<commit_after>'use strict';

const domino = require('domino');
const underscore = require('underscore');
const uuid = require('cassandra-uuid').TimeUuid;
const HTTPError = require('./util').HTTPError;
const transforms = require('./transforms');
const mUtil = {};

const NS_MAIN = 0;

mUtil.CONTENT_TYPES = {
    mobileSections: { name: 'mobile-sections', version: '0.13.1' },
    media: { name: 'Media', version: '1.0.0' },
    readHtml: { name: 'ReadHtml', version: '0.1.0' },
    summary: { name: 'Summary', version: '1.3.0' },
    definition: { name: 'definition', version: '0.7.4' },
    random: { name: 'random', version: '0.6.0' },
    announcements: { name: 'announcements', version: '0.1.0' },
    compilations: { name: 'compilations', version: '0.1.0' },
    onthisday: { name: 'onthisday', version: '0.3.3' },
    unpublished: { name: 'unpublished', version: '0.0.0' }
};

mUtil.SUMMARY_NS_WHITELIST = [ NS_MAIN ];

mUtil.setContentType = function(res, spec, mainType = 'application/json') {
    if (!spec.name || !spec.version) {
        throw new HTTPError({
            status: 500,
            type: 'invalid_content_type',
            title: 'Only use the allowed content-types',
            detail: `${spec.name}/${spec.version} is not an allowed content-type!`
        });
    }

    res.type(`${mainType}; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/${spec.name}/${spec.version}"`);
};

/**
 * Check if a variable is empty.
 * @param {string} val input value
 * @return {!boolean} true if val is null, undefined, an empty object, an empty array, or an empty
 * string.
 */
mUtil.isEmpty = function(val) {
    return !underscore.isNumber(val)
        && !underscore.isBoolean(val)
        && underscore.isEmpty(val);
};

mUtil.isNonempty = underscore.negate(mUtil.isEmpty);

/**
 * @param {*} val input value
 * @param {*} [fallback] the default value to assign if val is empty
 * @return {*} val if nonempty, else fallback.
*/
mUtil.defaultVal = function(val, fallback) {
    return underscore.isEmpty(val) ? fallback : val;
};

/**
 * @param {*} val input value
 * @return {*} val less empty elements.
*/
mUtil.filterEmpty = function(val) {
    if (Array.isArray(val)) {
        return val.map(mUtil.filterEmpty).filter(mUtil.isNonempty);
    }
    if (underscore.isObject(val)) {
        return underscore.pick(underscore.mapObject(val, mUtil.filterEmpty), mUtil.isNonempty);
    }
    return val;
};

/**
 * Sets the ETag header on the response objecti, comprised of the revision ID and
 * the time UUID. If the latter is not given, the current time stamp is used to
 * generate it.
 * @param {!Object}  response the HTTPResponse object on which to set the header
 * @param {?number} revision the revision integer ID to use
 * @param {?string}  tid      the time UUID to use; optional
 */
mUtil.setETag = function(response, revision, tid) {
    // we want to bail out if the revision hasn't been supplied, except
    // in the case revision === 0 because 0 is actually a valid rev_id
    if (!revision && revision !== 0) {
        return;
    }
    if (!tid) {
        tid = uuid.now().toString();
    }
    response.set('etag', `"${revision}/${tid}"`);
};

/**
 * Convert mobile to canonical domain,
 * e.g., 'en.m.wikipedia.org' => 'en.wikipedia.org'
 */
mUtil.mobileToCanonical = function(domain) {
    return domain.replace('.m.', '.');
};

/**
 * Remove the top-level domain from a domain string, e.g., 'en.wikipedia.org' ->
 * 'en.wikipedia'.
 */
mUtil.removeTLD = function(domain) {
    return domain.split('.').slice(0,2).join('.');
};

/**
 * Get the language of the wiki base don the domain name.
 * Example: 'en.wikipedia.org' -> 'en'.
 */
mUtil.getLanguageFromDomain = function(domain) {
    return domain.split('.')[0];
};

mUtil.removeFragment = function(href) {
    if (href.indexOf('#') > -1) {
        return href.substring(0, href.indexOf('#'));
    }
    return href;
};

mUtil.removeLinkPrefix = function(href) {
    return href.replace(/^\.\//, '');
};

mUtil.extractDbTitleFromAnchor = function(anchor) {
    return anchor && mUtil.removeFragment(mUtil.removeLinkPrefix(anchor.getAttribute('href')));
};

/**
 * Merge two arrays of objects by the specified property.  Modifies arr1 in place and preserves its
 * order.
 * @param {?Object[]} arr1 An array of objects
 * @param {?Object[]} arr2 An array of objects
 * @param {!String} prop the property on which to merge the objects in the arrays
 * @param {!Boolean} push whether to extend arr1 with objects from arr2 for which 'prop' is
 * defined but no object in arr1 has the same value
 * Credit: https://jsfiddle.net/guya/eAWKR/.
 */
mUtil.mergeByProp = function(arr1, arr2, prop, push) {
    if (!arr1 || !arr2) {
        return;
    }

    arr2.forEach((arr2obj) => {
        // Find the object in arr1 with the same value of 'prop' as this object in arr2
        const arr1obj = arr1.find(arr1obj => arr1obj[prop] === arr2obj[prop]);

        // If the object already exists, extend it with the new values from
        // arr2, otherwise conditionally add the new object to arr1.
        if (arr1obj) {
            Object.assign(arr1obj, arr2obj);
        } else if (push) {
            arr1.push(arr2obj);
        }
    });
};

mUtil.getRbPageSummaryUrl = function(restbaseTpl, domain, title) {
    const request = restbaseTpl.expand({
        request: {
            params: {
                domain,
                path: `page/summary/${encodeURIComponent(title)}` },
        }
    });
    return request.uri;
};

/**
 * Builds a dictionary containing the various forms of a page title that a client may need.
 * @param {!Object} title a mediawiki-title Title object constructed from a page title string
 * @param {!Object} meta page metadata
 * @return {!Object} a set of useful page title strings
 */
mUtil.buildTitleDictionary = function(title, meta) {
    return {
        canonical: title.getPrefixedDBKey(),
        normalized: meta.normalizedtitle,
        display: meta.displaytitle,
    };
};

/*
 * Build a page summary
 * @param {!String} domain the request domain
 * @param {!Object} title a mediawiki-title object for the page title
 * @param {!Object} pageData raw page data for the page
 * @param {?boolean} disambiguation whether the page is a disambiguation page
 * @return {!Object} a summary 2.0 spec-compliant page summary object
 */
mUtil.buildSummary = function(domain, title, pageData, disambiguation = false) {
    const page = pageData.page;
    const meta = pageData.meta;
    const isContentModelWikitext = meta.contentmodel === 'wikitext';
    const isWhiteListedNamespace = mUtil.SUMMARY_NS_WHITELIST.includes(meta.ns);
    const isRedirect = meta.redirect;

    if (!isContentModelWikitext) {
        return { code: 204 };
    }

    if (!isWhiteListedNamespace) {
        return { code: 204 };
    }

    if (isRedirect) {
        return { code: 204 };
    }

    const leadText = domino.createDocument(page.sections[0].text);
    const intro = transforms.extractLeadIntroduction(leadText);
    const summary = intro.length ? transforms.summarize(intro) : { extract: '', extract_html: '' };

    return Object.assign({
        code: 200,
        type : disambiguation ? 'disambiguation' : 'standard',
        title: meta.normalizedtitle,
        displaytitle: meta.displaytitle,
        namespace: { id: meta.ns, text: meta.nsText },
        titles: mUtil.buildTitleDictionary(title, meta),
        pageid: meta.id,
        thumbnail: meta.thumbnail,
        originalimage: meta.originalimage,
        lang: meta.lang,
        dir: meta.dir,
        revision: page.revision,
        tid: page.tid,
        timestamp: meta.lastmodified,
        description: meta.description,
        coordinates: meta.geo && {
            lat: meta.geo.latitude,
            lon: meta.geo.longitude
        },
        content_urls: mUtil.buildContentUrls(domain, title, meta),
        api_urls: mUtil.buildApiUrls(domain, title, meta),
    }, summary);
};

mUtil.buildContentUrls = function(domain, title, meta) {
    const mobileBaseUrl = meta.mobileHost;
    return {
        desktop: {
            page: `https://${domain}/wiki/${title.getPrefixedDBKey()}`,
            revisions: `https://${domain}/wiki/${title.getPrefixedDBKey()}?action=history`,
            edit: `https://${domain}/wiki/${title.getPrefixedDBKey()}?action=edit`,
            talk: meta.talkNsText && `https://${domain}/wiki/${meta.talkNsText}:${title.getKey()}`
        },
        mobile: mobileBaseUrl && {
            page: `${mobileBaseUrl}/wiki/${title.getPrefixedDBKey()}`,
            revisions: `${mobileBaseUrl}/wiki/Special:History/${title.getPrefixedDBKey()}`,
            edit: `${mobileBaseUrl}/wiki/${title.getPrefixedDBKey()}?action=edit`,
            talk: meta.talkNsText && `${mobileBaseUrl}/wiki/${meta.talkNsText}:${title.getKey()}`
        }
    };
};

mUtil.buildApiUrls = function(domain, title, meta) {
    return {
        summary: `https://${domain}/api/rest_v1/page/summary/${title.getPrefixedDBKey()}`,
        // read_html: `https://${domain}/api/rest_v1/page/read-html/${title.getPrefixedDBKey()}`,
        // content_html: `https://${domain}/api/rest_v1/page/content-html/${title.getPrefixedDBKey()}`,
        // metadata: `https://${domain}/api/rest_v1/page/metadata/${title.getPrefixedDBKey()}`,
        // references: `https://${domain}/api/rest_v1/page/references/${title.getPrefixedDBKey()}`,
        // media: `https://${domain}/api/rest_v1/page/media/${title.getPrefixedDBKey()}`,
        edit_html: `https://${domain}/api/rest_v1/page/html/${title.getPrefixedDBKey()}`,
        talk_page_html: meta.talkNsText && `https://${domain}/api/rest_v1/page/html/${meta.talkNsText}:${title.getKey()}`
    };
};

mUtil.throw404 = function(message) {
    throw new HTTPError({
        status: 404,
        type: 'not_found',
        title: 'Not found',
        detail: message
    });
};

mUtil.deduplicate = function(array) {
    return Array.from(new Set(array));
};

/* jslint bitwise: true */
/* eslint no-bitwise: ["error", { "allow": ["<<"] }] */
mUtil.hashCode = function(string) {
    return string.split('').reduce((prevHash, currVal) =>
        ((prevHash << 5) - prevHash) + currVal.charCodeAt(0), 0);
};

module.exports = mUtil;
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/*
 * This file implements the /api/obvius route. This route accepts data from
 * Obvius meters, handling parameters passed in form/multipart, GET parameters,
 * or POST body parameters.
 *
 * STATUS mode requests are logged.
 *
 * CONFIGFILEMANIFEST requests are responded to with a dummy manifest which specifies
 * 					  a lack of config files to respond with.
 */

const express = require('express');
const config = require('../config');
const multer = require('multer');
const moment = require('moment');
const md5 = require('md5');
const zlib = require('zlib');
const { log } = require('../log');
const Configfile = require('../models/obvius/Configfile');
const listConfigfiles = require('../services/obvius/listConfigfiles');
const streamBuffers = require('stream-buffers');
const loadLogfileToReadings = require('../services/obvius/loadLogfileToReadings');
const middleware = require('../middleware');
const obvius = require('../util').obvius;
const { getConnection, dropConnection } = require('../db');

const upload = multer({ storage: multer.memoryStorage() });
const router = express.Router();

// Here, the use of upload.array() allows the lowercaseParams middleware to
// integrate form/multipart data into the generic parameter pipeline along with
// POST and GET params.
router.use(upload.any(), middleware.lowercaseAllParamNames);
router.use(middleware.paramsLookupMixin);

/**
 * Inform the client of a failure (406 Not Acceptable), and log it.
 *
 * @param {express.Request} req The Express request object
 * @param {express.Response} res The Express response object
 * @param {string} reason The reason for the failure.
 *
 */
function failure(req, res, reason = '') {
	const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
	log.error(`Obvius protocol request from ${ip} failed due to ${reason}`);

	res.status(406) // 406 Not Acceptable error, as required by Obvius
		.send(`<pre>\n${reason}\n</pre>\n`);
}

/**
 * Inform the client of a success (200 OK).
 *
 * @param {express.Request} req The Express request object
 * @param {express.Response} res The Express response object
 * @param {string} comment Any additional data to be returned to the client.
 *
 */
function success(req, res, comment = '') {
	res.status(200) // 200 OK
		.send(`<pre>\nSUCCESS\n${comment}</pre>\n`);
}

/**
 * Logs a STATUS request for later examination.
 * @param {express.Request} req the request to process (must have the req.param mixin)
 * @param {express.Response} res the response object
 */
function handleStatus(req, res) {
	// Grab the IP of the requester.
	const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
	// These are all the params OED cares about. They just get logged.
	// Note that this route does NOT log the password, for security reasons.
	const paramNames = ['MODE', 'SENDDATATRACE', 'SERIALNUMBER', 'GSMSIGNAL',
		'LOOPNAME', 'UPTIME', 'PERCENTBLOCKSINUSE', 'PERCENTINODESINUSE',
		'UPLOADATTEMPT', 'ACQUISUITEVERSION', 'USRVERSION', 'ROOTVERSION',
		'KERNELVERSION', 'FIRMWAREVERSION', 'BOOTCOUNT', 'BATTERYGOOD'];
	// Build a log entry for this request
	let s = `Handling request from ${ip}\n`;
	for (const paramName of paramNames) {
		if (req.param(paramName) !== false && req.param(paramName) !== undefined) {
			s += `\tGot ${paramName}: ${req.param(paramName)}\n`;
		} else {
			s += `\tNo ${paramName} submitted\n`;
		}
	}
	log.info(s);

	success(req, res);
}

/**
 * Handle an Obvius upload request.
 * Unfortunately the Obvious API does not specify a HTTP verb.
 */
router.all('/', async (req, res) => {
	// Log the IP of the requester
	const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
	log.info(`Received Obvious protocol request from ${ip}`);

	// Attempt to verify the password
	if (!req.param('password')) {
		failure(req, res, 'password parameter is required.');
		return;
	} else if (req.param('password') !== config.obvius.password) {
		failure(req, res, 'password was not correct.');
		return;
	}

	const mode = req.param('mode', false);
	if (mode === false) {
		failure(req, res, 'Request must include mode parameter.');
		return;
	}

	if (mode === obvius.mode.status) {
		handleStatus(req, res);
		return;
	}

	if (mode === obvius.mode.logfile_upload) {
		if (!req.param('serialnumber', false)) {
			failure(req, res, 'Logfile Upload Requires Serial Number');
			return;
		}
		const conn = getConnection();
		for (const fx of req.files) {
			log.info(`Received ${fx.fieldname}: ${fx.originalname}`);
			// Logfiles are always gzipped.
			let data;
			try {
				data = zlib.gunzipSync(fx.buffer);
			} catch (err) {
				log.error(err);
				failure(req, res, `Unable to gunzip incoming buffer: ${err}`);
				return;
			}
			loadLogfileToReadings(req.param('serialnumber'), ip, data, conn);
		}
		success(req, res, 'Logfile Upload IS PROVISIONAL');
		return;
	}

	if (mode === obvius.mode.config_file_download) {
		failure(req, res, 'Config Download Not Implemented');
		return;
	}

	if (mode === obvius.mode.config_file_manifest) {
		const conn = getConnection();
		success(req, res, await listConfigfiles(conn));
		return;
	}

	if (mode === obvius.mode.config_file_upload) {
		// Check required parameters
		if (!req.param('serialnumber', false)) {
			failure(req, res, 'Config Upload Requires Serial Number');
			return;
		}
		if (!req.param('modbusdevice', false)) {
			failure(req, res, 'Config Upload Requires Modbus Device ID');
			return;
		}
		const conn = getConnection();
		for (const fx of req.files) {
			log.info(`Received ${fx.fieldname}: ${fx.originalname}`);

			let data;
			try {
				data = zlib.gunzipSync(fx.buffer).toString('utf-8');
			} catch (error) {
				data = fx.buffer.toString('utf-8');
			}

			const cf = new Configfile(undefined, req.param('serialnumber'), req.param('modbusdevice'), moment(), md5(data), data, true);
			await cf.insert(conn);
			success(req, res, `Acquired config log with (pseudo)filename ${cf.makeFilename()}.`);
		}
		return;
	}

	if (mode === obvius.mode.test) {
		failure(req, res, 'Test Not Implemented');
		return;
	}

	failure(req, res, `Unknown mode '${mode}'`);
});

module.exports = router;
<commit_msg>Escape html to prevent xss attacks<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/*
 * This file implements the /api/obvius route. This route accepts data from
 * Obvius meters, handling parameters passed in form/multipart, GET parameters,
 * or POST body parameters.
 *
 * STATUS mode requests are logged.
 *
 * CONFIGFILEMANIFEST requests are responded to with a dummy manifest which specifies
 * 					  a lack of config files to respond with.
 */

const express = require('express');
const config = require('../config');
const multer = require('multer');
const moment = require('moment');
const md5 = require('md5');
const zlib = require('zlib');
const { log } = require('../log');
const Configfile = require('../models/obvius/Configfile');
const listConfigfiles = require('../services/obvius/listConfigfiles');
const streamBuffers = require('stream-buffers');
const loadLogfileToReadings = require('../services/obvius/loadLogfileToReadings');
const middleware = require('../middleware');
const obvius = require('../util').obvius;
const { getConnection, dropConnection } = require('../db');
const escapeHtml = require('core-js/fn/string/escape-html');

const upload = multer({ storage: multer.memoryStorage() });
const router = express.Router();

// Here, the use of upload.array() allows the lowercaseParams middleware to
// integrate form/multipart data into the generic parameter pipeline along with
// POST and GET params.
router.use(upload.any(), middleware.lowercaseAllParamNames);
router.use(middleware.paramsLookupMixin);

/**
 * Inform the client of a failure (406 Not Acceptable), and log it.
 *
 * @param {express.Request} req The Express request object
 * @param {express.Response} res The Express response object
 * @param {string} reason The reason for the failure.
 *
 */
function failure(req, res, reason = '') {
	reason = escapeHtml(reason); // escape html to sanitize html
	const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
	log.error(`Obvius protocol request from ${ip} failed due to ${reason}`);

	res.status(406) // 406 Not Acceptable error, as required by Obvius
		.send(`<pre>\n${reason}\n</pre>\n`);
}

/**
 * Inform the client of a success (200 OK).
 *
 * @param {express.Request} req The Express request object
 * @param {express.Response} res The Express response object
 * @param {string} comment Any additional data to be returned to the client.
 *
 */
function success(req, res, comment = '') {
	comment = escapeHtml(comment); // escape html to sanitize html
	res.status(200) // 200 OK
		.send(`<pre>\nSUCCESS\n${comment}</pre>\n`);
}

/**
 * Logs a STATUS request for later examination.
 * @param {express.Request} req the request to process (must have the req.param mixin)
 * @param {express.Response} res the response object
 */
function handleStatus(req, res) {
	// Grab the IP of the requester.
	const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
	// These are all the params OED cares about. They just get logged.
	// Note that this route does NOT log the password, for security reasons.
	const paramNames = ['MODE', 'SENDDATATRACE', 'SERIALNUMBER', 'GSMSIGNAL',
		'LOOPNAME', 'UPTIME', 'PERCENTBLOCKSINUSE', 'PERCENTINODESINUSE',
		'UPLOADATTEMPT', 'ACQUISUITEVERSION', 'USRVERSION', 'ROOTVERSION',
		'KERNELVERSION', 'FIRMWAREVERSION', 'BOOTCOUNT', 'BATTERYGOOD'];
	// Build a log entry for this request
	let s = `Handling request from ${ip}\n`;
	for (const paramName of paramNames) {
		if (req.param(paramName) !== false && req.param(paramName) !== undefined) {
			s += `\tGot ${paramName}: ${req.param(paramName)}\n`;
		} else {
			s += `\tNo ${paramName} submitted\n`;
		}
	}
	log.info(s);

	success(req, res);
}

/**
 * Handle an Obvius upload request.
 * Unfortunately the Obvious API does not specify a HTTP verb.
 */
router.all('/', async (req, res) => {
	// Log the IP of the requester
	const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
	log.info(`Received Obvious protocol request from ${ip}`);

	// Attempt to verify the password
	if (!req.param('password')) {
		failure(req, res, 'password parameter is required.');
		return;
	} else if (req.param('password') !== config.obvius.password) {
		failure(req, res, 'password was not correct.');
		return;
	}

	const mode = req.param('mode', false);
	if (mode === false) {
		failure(req, res, 'Request must include mode parameter.');
		return;
	}

	if (mode === obvius.mode.status) {
		handleStatus(req, res);
		return;
	}

	if (mode === obvius.mode.logfile_upload) {
		if (!req.param('serialnumber', false)) {
			failure(req, res, 'Logfile Upload Requires Serial Number');
			return;
		}
		const conn = getConnection();
		for (const fx of req.files) {
			log.info(`Received ${fx.fieldname}: ${fx.originalname}`);
			// Logfiles are always gzipped.
			let data;
			try {
				data = zlib.gunzipSync(fx.buffer);
			} catch (err) {
				log.error(err);
				failure(req, res, `Unable to gunzip incoming buffer: ${err}`);
				return;
			}
			loadLogfileToReadings(req.param('serialnumber'), ip, data, conn);
		}
		success(req, res, 'Logfile Upload IS PROVISIONAL');
		return;
	}

	if (mode === obvius.mode.config_file_download) {
		failure(req, res, 'Config Download Not Implemented');
		return;
	}

	if (mode === obvius.mode.config_file_manifest) {
		const conn = getConnection();
		success(req, res, await listConfigfiles(conn));
		return;
	}

	if (mode === obvius.mode.config_file_upload) {
		// Check required parameters
		if (!req.param('serialnumber', false)) {
			failure(req, res, 'Config Upload Requires Serial Number');
			return;
		}
		if (!req.param('modbusdevice', false)) {
			failure(req, res, 'Config Upload Requires Modbus Device ID');
			return;
		}
		const conn = getConnection();
		for (const fx of req.files) {
			log.info(`Received ${fx.fieldname}: ${fx.originalname}`);

			let data;
			try {
				data = zlib.gunzipSync(fx.buffer).toString('utf-8');
			} catch (error) {
				data = fx.buffer.toString('utf-8');
			}

			const cf = new Configfile(undefined, req.param('serialnumber'), req.param('modbusdevice'), moment(), md5(data), data, true);
			await cf.insert(conn);
			success(req, res, `Acquired config log with (pseudo)filename ${cf.makeFilename()}.`);
		}
		return;
	}

	if (mode === obvius.mode.test) {
		failure(req, res, 'Test Not Implemented');
		return;
	}

	failure(req, res, `Unknown mode '${mode}'`);
});

module.exports = router;
<|endoftext|>


<commit_before>'use strict';

const P = require('bluebird');
const domino = require('domino');
const assert = require('assert');
const underscore = require('underscore');
const uuid = require('cassandra-uuid').TimeUuid;
const HTTPError = require('./util').HTTPError;
const mUtil = {};
const constants = require('./constants');

mUtil.CONTENT_TYPES = {
    html: { name: 'HTML', version: '2.1.0', type: 'text/html' },
    mobileSections: { name: 'mobile-sections', version: '0.14.5', type: 'application/json' },
    media: { name: 'Media', version: '1.4.5', type: 'application/json' },
    mobileHtml: { name: 'Mobile-HTML', version: '1.1.0', type: 'text/html' },
    references: { name: 'References', version: '2.0.0', type: 'application/json' },
    metadata: { name: 'Metadata', version: '1.3.0', type: 'application/json' },
    summary: { name: 'Summary', version: '1.4.2', type: 'application/json' },
    definition: { name: 'definition', version: '0.8.1', type: 'application/json' },
    random: { name: 'Random', version: '0.6.0', type: 'application/json' },
    announcements: { name: 'Announcements', version: '0.3.0', type: 'application/json' },
    onthisday: { name: 'OnThisDay', version: '0.3.3', type: 'application/json' },
    css: { name: 'CSS', version: '2.0.0', type: 'text/css' },
    javascript: { name: 'JavaScript', version: '1.0.0', type: 'text/javascript' },
    availability: { name: 'Availability', version: '1.0.1', type: 'application/json' },
    unpublished: { name: 'Unpublished', version: '0.0.0', type: 'application/json' },
    mobileHtmlOfflineResources: {
        name: 'Mobile-HTML-Offline-Resources',
        version: '1.0.0',
        type: 'application/json'
    },
    talk: { name: 'Talk', version: '0.1.1', type: 'application/json' },
    mediaList: { name: 'MediaList', version: '1.1.0', type: 'application/json' }
};

/**
 * Pass on the Content-Security-Policy HTTP header
 * @param {!Object} response the HTTPResponse object on which to set the headers
 * @param {!string} value the CSP value to set
 */
mUtil.setContentSecurityPolicy = (response, value) => {
    // Note that .set will over-write, not expand
    response.set('content-security-policy', value);
    response.set('x-content-security-policy', value);
    response.set('x-webkit-csp', value);
};

mUtil.getContentTypeString = function(spec) {
    return `${spec.type}; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/${spec.name}/${spec.version}"`;
};

mUtil.setContentType = function(res, spec) {
    if (!spec.name || !spec.version) {
        throw new HTTPError({
            status: 500,
            type: 'invalid_content_type',
            title: 'Only use the allowed content-types',
            detail: `${spec.name}/${spec.version} is not an allowed content-type!`
        });
    }

    res.type(mUtil.getContentTypeString(spec));
};

/**
 * Check if a variable is empty.
 * @param {string} val input value
 * @return {!boolean} true if val is null, undefined, an empty object, an empty array, or an empty
 * string.
 */
mUtil.isEmpty = function(val) {
    return !underscore.isNumber(val)
        && !underscore.isBoolean(val)
        && underscore.isEmpty(val);
};

mUtil.isNonempty = underscore.negate(mUtil.isEmpty);

/**
 * @param {*} val input value
 * @param {*} [fallback] the default value to assign if val is empty
 * @return {*} val if nonempty, else fallback.
*/
mUtil.defaultVal = function(val, fallback) {
    return underscore.isEmpty(val) ? fallback : val;
};

/**
 * @param {*} val input value
 * @return {*} val less empty elements.
*/
mUtil.filterEmpty = function(val) {
    if (Array.isArray(val)) {
        return val.map(mUtil.filterEmpty).filter(mUtil.isNonempty);
    }
    if (underscore.isObject(val)) {
        return underscore.pick(underscore.mapObject(val, mUtil.filterEmpty), mUtil.isNonempty);
    }
    return val;
};

/**
 * Sets the ETag header on the response object, comprised of the revision ID and
 * the time UUID. If the latter is not given, the current time stamp is used to
 * generate it.
 * @param {!Object} response The HTTPResponse object on which to set the header
 * @param {!number|string} revision The revision integer ID to use
 * @param {?string} tid      The time UUID to use; optional
 */
mUtil.setETag = function(response, revision, tid = undefined) {
    assert(['string','number'].includes(typeof revision));
    assert(['string','undefined'].includes(typeof tid));
    if (!tid) {
        tid = uuid.now().toString();
    }
    response.set('etag', `"${revision}/${tid}"`);
};

/**
 * Pass on the Vary and Content-Language HTTP headers from Parsoid if set
 * @param {!Object} response The HTTPResponse object on which to set the headers
 * @param {?number} headers  The incoming response's headers from which to read
 */
mUtil.setLanguageHeaders = (response, headers) => {
    if (headers) {
        // Downcast all keys to lower-case, as the spec doesn't require it.
        const lowerHeaders = Object.keys(headers).reduce((newHeaders, key) => {
            newHeaders[key.toLowerCase()] = headers[key];
            return newHeaders;
        }, {});

        if (lowerHeaders.vary) {
            const vary = lowerHeaders.vary.split(',')
                .map(e => e.trim())
                .filter(e => !/^[Aa]ccept$/.test(e))
                .join(', ');

            // Note that .vary will expand, not over-write
            if (vary) {
                response.vary(vary);
            }
        }
        if (lowerHeaders['content-language']) {
            // Note that .set will over-write, not expand
            response.set('Content-Language', lowerHeaders['content-language']);
        }
    }
};

/**
 * Convert mobile to canonical domain,
 * e.g., 'en.m.wikipedia.org' => 'en.wikipedia.org'
 */
mUtil.mobileToCanonical = function(domain) {
    return domain.replace('.m.', '.');
};

/**
 * Remove the top-level domain from a domain string, e.g., 'en.wikipedia.org' ->
 * 'en.wikipedia'.
 */
mUtil.removeTLD = function(domain) {
    return domain.split('.').slice(0,2).join('.');
};

/**
 * Swaps a domain's lowest-level subdomain with the provided language code.
 * This is assumed to be invalid for provided domains with two or fewer levels, and the domain is
 * returned unchanged.
 * Example: ('en.wikipedia.org', 'de') -> 'de.wikipedia.org'
 * Example: ('de.wikipedia.beta.wmflabs.org', 'ja') -> 'ja.wikipedia.beta.wmflabs.org'
 * Example: ('mediawiki.org', 'es') -> 'mediawiki.org'
 */
mUtil.domainForLangCode = (domain, code) => {
    return domain.split('.').length < 3 ? domain : `${code}${domain.slice(domain.indexOf('.'))}`;
};

mUtil.removeFragment = function(href) {
    if (href.indexOf('#') > -1) {
        return href.substring(0, href.indexOf('#'));
    }
    return href;
};

mUtil.removeLinkPrefix = function(href) {
    return href.replace(/^\.\//, '');
};

mUtil.extractDbTitleFromAnchor = function(anchor) {
    return anchor && mUtil.removeFragment(mUtil.removeLinkPrefix(anchor.getAttribute('href')));
};

mUtil.getRbPageSummaryUrl = function(restbaseTpl, domain, title) {
    const request = restbaseTpl.expand({
        request: {
            params: {
                domain,
                path: `page/summary/${encodeURIComponent(title)}` },
        }
    });
    return request.uri;
};

mUtil.throw404 = function(message) {
    throw new HTTPError({
        status: 404,
        type: 'not_found',
        title: 'Not found',
        detail: message
    });
};

mUtil.deduplicate = function(array) {
    return Array.from(new Set(array));
};

/* jslint bitwise: true */
/* eslint no-bitwise: ["error", { "allow": ["<<"] }] */
mUtil.hashCode = function(string) {
    return string.split('').reduce((prevHash, currVal) =>
        ((prevHash << 5) - prevHash) + currVal.charCodeAt(0), 0);
};

/**
 * Gets the data-mw-section-id for the section containing the element.
 * @param {!Element} el An Element
 * @return {?integer} the data-mw-section-id for the section containing the element
 */
mUtil.getSectionIdForElement = (el) => {
    const section = el.closest('section');
    return section && parseInt(section.getAttribute('data-mw-section-id'), 10);
};

/**
 * Historically, the MediaWiki API indicated boolean values in responses by including the property
 * with a value of "" if true, and omitting the property if false.  JSON format version 2, which
 * we always request, specifies that fields with boolean values should return true boolean 'true'
 * or 'false' values.  However, not all API modules have been updated to implement formatversion=2.
 * This method can be used to handle both cases and to guard against unexpected changes in modules
 * not yet updated.
 * @param {!Object} obj enclosing object
 * @param {!string} prop property with a boolean meaning
 * @return {!boolean} true if the value means true in either API format
 */
mUtil.mwApiTrue = (obj, prop) => {
    return obj[prop] === true || obj[prop] === '';
};

/**
 * Creates a domino Document object from an HTML text string.
 * Takes advantage of functionality provided by domino's internal HTMLParser implementation,
 * but not exposed in its public API via the createDocument() method, to allow parsing an HTML
 * string in chunks.
 * Returns a Promise resolving as a Document rather than synchronously parsing and returning the
 * Document as in Domino's own implementation.
 * @param {?string} html HTML string
 * @return {!Promise} a promise resolving in a Domino HTML document representing the input
 */
mUtil.createDocument = (html) => {
    if (typeof html !== 'string') {
        return P.reject(new HTTPError({
                status: 500,
                type: 'invalid_parameter_type',
                title: 'HTML input should be a string'
        }));
    }

    function _pauseAfter(ms) {
        const start = Date.now();
        return () => (Date.now() - start) >= ms;
    }

    function _processIncremental(parser) {
        return new P((res, rej) => setImmediate(() => {
            try {
                if (parser.process(_pauseAfter(constants.MAX_MS_PER_TICK))) {
                    res(_processIncremental(parser));
                } else {
                    res(parser.document());
                }
            } catch (e) {
                rej(e);
            }
        }));
    }

    return P.resolve(domino.createIncrementalHTMLParser())
    .then((parser) => {
        parser.end(html);
        return _processIncremental(parser);
    });
};

module.exports = mUtil;
<commit_msg>Hygiene: Remove content type definitions for endpoints moved to wikifeeds<commit_after>'use strict';

const P = require('bluebird');
const domino = require('domino');
const assert = require('assert');
const underscore = require('underscore');
const uuid = require('cassandra-uuid').TimeUuid;
const HTTPError = require('./util').HTTPError;
const mUtil = {};
const constants = require('./constants');

mUtil.CONTENT_TYPES = {
    html: { name: 'HTML', version: '2.1.0', type: 'text/html' },
    mobileSections: { name: 'mobile-sections', version: '0.14.5', type: 'application/json' },
    media: { name: 'Media', version: '1.4.5', type: 'application/json' },
    mobileHtml: { name: 'Mobile-HTML', version: '1.1.0', type: 'text/html' },
    references: { name: 'References', version: '2.0.0', type: 'application/json' },
    metadata: { name: 'Metadata', version: '1.3.0', type: 'application/json' },
    summary: { name: 'Summary', version: '1.4.2', type: 'application/json' },
    definition: { name: 'definition', version: '0.8.1', type: 'application/json' },
    random: { name: 'Random', version: '0.6.0', type: 'application/json' },
    css: { name: 'CSS', version: '2.0.0', type: 'text/css' },
    javascript: { name: 'JavaScript', version: '1.0.0', type: 'text/javascript' },
    unpublished: { name: 'Unpublished', version: '0.0.0', type: 'application/json' },
    mobileHtmlOfflineResources: {
        name: 'Mobile-HTML-Offline-Resources',
        version: '1.0.0',
        type: 'application/json'
    },
    talk: { name: 'Talk', version: '0.1.1', type: 'application/json' },
    mediaList: { name: 'MediaList', version: '1.1.0', type: 'application/json' }
};

/**
 * Pass on the Content-Security-Policy HTTP header
 * @param {!Object} response the HTTPResponse object on which to set the headers
 * @param {!string} value the CSP value to set
 */
mUtil.setContentSecurityPolicy = (response, value) => {
    // Note that .set will over-write, not expand
    response.set('content-security-policy', value);
    response.set('x-content-security-policy', value);
    response.set('x-webkit-csp', value);
};

mUtil.getContentTypeString = function(spec) {
    return `${spec.type}; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/${spec.name}/${spec.version}"`;
};

mUtil.setContentType = function(res, spec) {
    if (!spec.name || !spec.version) {
        throw new HTTPError({
            status: 500,
            type: 'invalid_content_type',
            title: 'Only use the allowed content-types',
            detail: `${spec.name}/${spec.version} is not an allowed content-type!`
        });
    }

    res.type(mUtil.getContentTypeString(spec));
};

/**
 * Check if a variable is empty.
 * @param {string} val input value
 * @return {!boolean} true if val is null, undefined, an empty object, an empty array, or an empty
 * string.
 */
mUtil.isEmpty = function(val) {
    return !underscore.isNumber(val)
        && !underscore.isBoolean(val)
        && underscore.isEmpty(val);
};

mUtil.isNonempty = underscore.negate(mUtil.isEmpty);

/**
 * @param {*} val input value
 * @param {*} [fallback] the default value to assign if val is empty
 * @return {*} val if nonempty, else fallback.
*/
mUtil.defaultVal = function(val, fallback) {
    return underscore.isEmpty(val) ? fallback : val;
};

/**
 * @param {*} val input value
 * @return {*} val less empty elements.
*/
mUtil.filterEmpty = function(val) {
    if (Array.isArray(val)) {
        return val.map(mUtil.filterEmpty).filter(mUtil.isNonempty);
    }
    if (underscore.isObject(val)) {
        return underscore.pick(underscore.mapObject(val, mUtil.filterEmpty), mUtil.isNonempty);
    }
    return val;
};

/**
 * Sets the ETag header on the response object, comprised of the revision ID and
 * the time UUID. If the latter is not given, the current time stamp is used to
 * generate it.
 * @param {!Object} response The HTTPResponse object on which to set the header
 * @param {!number|string} revision The revision integer ID to use
 * @param {?string} tid      The time UUID to use; optional
 */
mUtil.setETag = function(response, revision, tid = undefined) {
    assert(['string','number'].includes(typeof revision));
    assert(['string','undefined'].includes(typeof tid));
    if (!tid) {
        tid = uuid.now().toString();
    }
    response.set('etag', `"${revision}/${tid}"`);
};

/**
 * Pass on the Vary and Content-Language HTTP headers from Parsoid if set
 * @param {!Object} response The HTTPResponse object on which to set the headers
 * @param {?number} headers  The incoming response's headers from which to read
 */
mUtil.setLanguageHeaders = (response, headers) => {
    if (headers) {
        // Downcast all keys to lower-case, as the spec doesn't require it.
        const lowerHeaders = Object.keys(headers).reduce((newHeaders, key) => {
            newHeaders[key.toLowerCase()] = headers[key];
            return newHeaders;
        }, {});

        if (lowerHeaders.vary) {
            const vary = lowerHeaders.vary.split(',')
                .map(e => e.trim())
                .filter(e => !/^[Aa]ccept$/.test(e))
                .join(', ');

            // Note that .vary will expand, not over-write
            if (vary) {
                response.vary(vary);
            }
        }
        if (lowerHeaders['content-language']) {
            // Note that .set will over-write, not expand
            response.set('Content-Language', lowerHeaders['content-language']);
        }
    }
};

/**
 * Convert mobile to canonical domain,
 * e.g., 'en.m.wikipedia.org' => 'en.wikipedia.org'
 */
mUtil.mobileToCanonical = function(domain) {
    return domain.replace('.m.', '.');
};

/**
 * Remove the top-level domain from a domain string, e.g., 'en.wikipedia.org' ->
 * 'en.wikipedia'.
 */
mUtil.removeTLD = function(domain) {
    return domain.split('.').slice(0,2).join('.');
};

/**
 * Swaps a domain's lowest-level subdomain with the provided language code.
 * This is assumed to be invalid for provided domains with two or fewer levels, and the domain is
 * returned unchanged.
 * Example: ('en.wikipedia.org', 'de') -> 'de.wikipedia.org'
 * Example: ('de.wikipedia.beta.wmflabs.org', 'ja') -> 'ja.wikipedia.beta.wmflabs.org'
 * Example: ('mediawiki.org', 'es') -> 'mediawiki.org'
 */
mUtil.domainForLangCode = (domain, code) => {
    return domain.split('.').length < 3 ? domain : `${code}${domain.slice(domain.indexOf('.'))}`;
};

mUtil.removeFragment = function(href) {
    if (href.indexOf('#') > -1) {
        return href.substring(0, href.indexOf('#'));
    }
    return href;
};

mUtil.removeLinkPrefix = function(href) {
    return href.replace(/^\.\//, '');
};

mUtil.extractDbTitleFromAnchor = function(anchor) {
    return anchor && mUtil.removeFragment(mUtil.removeLinkPrefix(anchor.getAttribute('href')));
};

mUtil.getRbPageSummaryUrl = function(restbaseTpl, domain, title) {
    const request = restbaseTpl.expand({
        request: {
            params: {
                domain,
                path: `page/summary/${encodeURIComponent(title)}` },
        }
    });
    return request.uri;
};

mUtil.throw404 = function(message) {
    throw new HTTPError({
        status: 404,
        type: 'not_found',
        title: 'Not found',
        detail: message
    });
};

mUtil.deduplicate = function(array) {
    return Array.from(new Set(array));
};

/* jslint bitwise: true */
/* eslint no-bitwise: ["error", { "allow": ["<<"] }] */
mUtil.hashCode = function(string) {
    return string.split('').reduce((prevHash, currVal) =>
        ((prevHash << 5) - prevHash) + currVal.charCodeAt(0), 0);
};

/**
 * Gets the data-mw-section-id for the section containing the element.
 * @param {!Element} el An Element
 * @return {?integer} the data-mw-section-id for the section containing the element
 */
mUtil.getSectionIdForElement = (el) => {
    const section = el.closest('section');
    return section && parseInt(section.getAttribute('data-mw-section-id'), 10);
};

/**
 * Historically, the MediaWiki API indicated boolean values in responses by including the property
 * with a value of "" if true, and omitting the property if false.  JSON format version 2, which
 * we always request, specifies that fields with boolean values should return true boolean 'true'
 * or 'false' values.  However, not all API modules have been updated to implement formatversion=2.
 * This method can be used to handle both cases and to guard against unexpected changes in modules
 * not yet updated.
 * @param {!Object} obj enclosing object
 * @param {!string} prop property with a boolean meaning
 * @return {!boolean} true if the value means true in either API format
 */
mUtil.mwApiTrue = (obj, prop) => {
    return obj[prop] === true || obj[prop] === '';
};

/**
 * Creates a domino Document object from an HTML text string.
 * Takes advantage of functionality provided by domino's internal HTMLParser implementation,
 * but not exposed in its public API via the createDocument() method, to allow parsing an HTML
 * string in chunks.
 * Returns a Promise resolving as a Document rather than synchronously parsing and returning the
 * Document as in Domino's own implementation.
 * @param {?string} html HTML string
 * @return {!Promise} a promise resolving in a Domino HTML document representing the input
 */
mUtil.createDocument = (html) => {
    if (typeof html !== 'string') {
        return P.reject(new HTTPError({
                status: 500,
                type: 'invalid_parameter_type',
                title: 'HTML input should be a string'
        }));
    }

    function _pauseAfter(ms) {
        const start = Date.now();
        return () => (Date.now() - start) >= ms;
    }

    function _processIncremental(parser) {
        return new P((res, rej) => setImmediate(() => {
            try {
                if (parser.process(_pauseAfter(constants.MAX_MS_PER_TICK))) {
                    res(_processIncremental(parser));
                } else {
                    res(parser.document());
                }
            } catch (e) {
                rej(e);
            }
        }));
    }

    return P.resolve(domino.createIncrementalHTMLParser())
    .then((parser) => {
        parser.end(html);
        return _processIncremental(parser);
    });
};

module.exports = mUtil;
<|endoftext|>


<commit_before>var socket = require('socket.io'),
    domain = require('domain'),
    express = require('express'),
    http = require('http'),
    fs = require('fs');
var redis = require('socket.io-redis');
var logger = require(__dirname+'/../../logger/logger'),
    arborescence = require(__dirname+'/../../arborescence'),
    user = require(__dirname + '/../user/models/user');

var ROLES_CONFIG_FILE_PATH = process.cwd() + '/config/roles.json';
var rolesConfig;

var cacheFiles;
var config;

var roleContainsRole = function(activeRole, requiredRole, callback) {
    if (!rolesConfig || rolesConfig.roles.length == 0)
        return callback(ROLES_CONFIG_FILE_PATH + ' is empty');

    for (var i=0; i<rolesConfig.roles.length; i++) {
        if (rolesConfig.roles[i].name == activeRole) {
            if (!rolesConfig.roles[i].children || rolesConfig.roles[i].children.length == 0)
                return callback('access_denied');

            for (var j=0; j<rolesConfig.roles[i].children.length; j++) {
                if (rolesConfig.roles[i].children[j] == requiredRole)
                    return callback();

                roleContainsRole(rolesConfig.roles[i].children[j], requiredRole, function(err) {
                    if (!err)
                        return callback();
                });
            }
        }
    }

    return callback('access_denied');
};

var checkRole = function(userRole, requiredRole, callback) {
    if (!rolesConfig) {
        // Check if config file for roles exists
        try {
            rolesConfig = JSON.parse(fs.readFileSync(ROLES_CONFIG_FILE_PATH));
        }
        catch (err) {
            logger.error(new Error(ROLES_CONFIG_FILE_PATH + ' doesn\'t exists or it\'s not a valid JSON.'), 'Roles');
            process.exit(1);
        }
    }
    return roleContainsRole(userRole, requiredRole, callback);
};

var checkRoles = function(userRole, requiredRoles, callback) {
    if (requiredRoles.length == 1)
        return checkRole(userRole, requiredRoles[0], callback);

    var i=0;
    var done = false;

    async.doWhilst(
        function(callback) {
            if (i >= requiredRoles.length)
                return callback('access_denied');

            checkRole(userRole, requiredRoles[i], function(err) {
                if (!err)
                    done = true;

                i++;
                return callback();
            });
        },
        function() {
            return !done;
        },
        function(err) {
            if (err) {
                if (callback)
                    return callback(err);
                throw err;
            }
            if (callback)
                return callback();
        }
    );
};

var checkAuthentification = function(data, callback) {
    if (!data.user)
        return callback('access_denied');

    if (!data.roles) {
        return callback();
    }
    else {
        if (data.roles.indexOf(data.user.role) > -1)
            return callback();

        checkRoles(data.user.role, data.roles, function(err) {
            if (err)
                return callback(err);

            return callback();
        });
    }
};

var onConnect = function(socket) {
    /**
     * Override socket.on
     */
    if (process.env.NODE_ENV == 'development' || process.env.DEBUG)
        console.log('SOCKET CONNECTED');

    socket.on('disconnect', function() {
        if (process.env.NODE_ENV == 'development' || process.env.DEBUG)
            console.log('SOCKET DISCONNECTED');
    });

    socket.onAux = socket.on;
    socket.on = function (url, options, userCallback) {
        socket.onAux(url, function (data, callback) {
            if (process.env.NODE_ENV == 'development' || process.env.DEBUG) {
                console.log('SOCKET : '+url);
            }

            if (typeof data == 'function') {
                callback = data;
                data = {
                    data: {}
                };
            } else if (!data) {
                data = {
                    data: {}
                };
            }

            user.tokenIsValid(data.token, function (err, user) {
                if (!err && user)
                    data.user = user;

                if (!data.data)
                    data.data = {};
                if (typeof options == 'object' && (options.authentification || options.roles)) {
                    if (options.roles)
                        data.roles = options.roles;

                    checkAuthentification(data, function (err) {
                        if (err)
                            return callback(err);

                        return userCallback(data, callback);
                    });
                }
                else {
                    return options(data, callback);
                }
            });
        });
    };

    if (!cacheFiles) {
        arborescence.getRequiredFiles('socket', function (files) {
            cacheFiles = files;
            arborescence.loadFiles(cacheFiles, socket);
        });
    } else {
        arborescence.loadFiles(cacheFiles, socket);
    }
};

module.exports = function(c) {
    config = c || {};

    if (process.env.NODE_ENV == 'production' || process.env.NODE_ENV == 'preproduction') {
        var sticky = require('socketio-sticky-session');

        sticky(function() {
            // This code will be executed only in slave workers
            var d = domain.create();

            d.on('error', function(err) {
                logger.error(err, 'Socket.io'+':'+config.port);
            });

            d.run(function() {
                var server = require('http').createServer();
                var io = require('socket.io')(server);

                io.on('connect', onConnect);

                return server;
            });
        }).listen(config.port, function() {
            console.log('Socket started on ' + config.port + ' port');
        });
    } else {
        var io;
        if(config.port) {
            io = socket();
        } else {
            var app = express();
            var server = http.createServer(app);
            io = socket(server);
        }
        var d = domain.create();

        d.on('error', function(err) {
            logger.error(err, 'Socket.io'+':'+config.port);
        });

        d.run(function() {
            io.adapter(redis({ host: 'localhost', port: 6379 }));
            io.on('connection', onConnect);
        });
        if(config.port) {
            logger.log('Sockets', ['green'], ' listening on port ', config.port, ['yellow'], '.');

            io.listen(config.port, function() {
                if(typeof callback=='function') cb();
            });
        } else {
            io.listen(8888, function() {
                if(typeof callback=='function') cb();
            });
            return server;
        }
    }
};<commit_msg>socket<commit_after>var socket = require('socket.io'),
    domain = require('domain'),
    express = require('express'),
    http = require('http'),
    fs = require('fs');
var redis = require('socket.io-redis');
var logger = require(__dirname+'/../../logger/logger'),
    arborescence = require(__dirname+'/../../arborescence'),
    user = require(__dirname + '/../user/models/user');

var ROLES_CONFIG_FILE_PATH = process.cwd() + '/config/roles.json';
var rolesConfig;

var cacheFiles;
var config;

var roleContainsRole = function(activeRole, requiredRole, callback) {
    if (!rolesConfig || rolesConfig.roles.length == 0)
        return callback(ROLES_CONFIG_FILE_PATH + ' is empty');

    for (var i=0; i<rolesConfig.roles.length; i++) {
        if (rolesConfig.roles[i].name == activeRole) {
            if (!rolesConfig.roles[i].children || rolesConfig.roles[i].children.length == 0)
                return callback('access_denied');

            for (var j=0; j<rolesConfig.roles[i].children.length; j++) {
                if (rolesConfig.roles[i].children[j] == requiredRole)
                    return callback();

                roleContainsRole(rolesConfig.roles[i].children[j], requiredRole, function(err) {
                    if (!err)
                        return callback();
                });
            }
        }
    }

    return callback('access_denied');
};

var checkRole = function(userRole, requiredRole, callback) {
    if (!rolesConfig) {
        // Check if config file for roles exists
        try {
            rolesConfig = JSON.parse(fs.readFileSync(ROLES_CONFIG_FILE_PATH));
        }
        catch (err) {
            logger.error(new Error(ROLES_CONFIG_FILE_PATH + ' doesn\'t exists or it\'s not a valid JSON.'), 'Roles');
            process.exit(1);
        }
    }
    return roleContainsRole(userRole, requiredRole, callback);
};

var checkRoles = function(userRole, requiredRoles, callback) {
    if (requiredRoles.length == 1)
        return checkRole(userRole, requiredRoles[0], callback);

    var i=0;
    var done = false;

    async.doWhilst(
        function(callback) {
            if (i >= requiredRoles.length)
                return callback('access_denied');

            checkRole(userRole, requiredRoles[i], function(err) {
                if (!err)
                    done = true;

                i++;
                return callback();
            });
        },
        function() {
            return !done;
        },
        function(err) {
            if (err) {
                if (callback)
                    return callback(err);
                throw err;
            }
            if (callback)
                return callback();
        }
    );
};

var checkAuthentification = function(data, callback) {
    if (!data.user)
        return callback('access_denied');

    if (!data.roles) {
        return callback();
    }
    else {
        if (data.roles.indexOf(data.user.role) > -1)
            return callback();

        checkRoles(data.user.role, data.roles, function(err) {
            if (err)
                return callback(err);

            return callback();
        });
    }
};

var onConnect = function(socket) {
    /**
     * Override socket.on
     */
    if (process.env.NODE_ENV == 'development' || process.env.DEBUG)
        console.log('SOCKET CONNECTED');

    socket.on('disconnect', function() {
        if (process.env.NODE_ENV == 'development' || process.env.DEBUG)
            console.log('SOCKET DISCONNECTED');
    });

    socket.onAux = socket.on;
    socket.on = function (url, options, userCallback) {
        socket.onAux(url, function (data, callback) {
            if (process.env.NODE_ENV == 'development' || process.env.DEBUG) {
                console.log('SOCKET : '+url);
            }

            if (typeof data == 'function') {
                callback = data;
                data = {
                    data: {}
                };
            } else if (!data) {
                data = {
                    data: {}
                };
            }

            user.tokenIsValid(data.token, function (err, user) {
                if (!err && user)
                    data.user = user;

                if (!data.data)
                    data.data = {};
                if (typeof options == 'object' && (options.authentification || options.roles)) {
                    if (options.roles)
                        data.roles = options.roles;

                    checkAuthentification(data, function (err) {
                        if (err)
                            return callback(err);

                        return userCallback(data, callback);
                    });
                }
                else {
                    return options(data, callback);
                }
            });
        });
    };

    if (!cacheFiles) {
        arborescence.getRequiredFiles('socket', function (files) {
            cacheFiles = files;
            arborescence.loadFiles(cacheFiles, socket);
        });
    } else {
        arborescence.loadFiles(cacheFiles, socket);
    }
};

module.exports = function(c) {
    config = c || {};

    if (process.env.NODE_ENV == 'production' || process.env.NODE_ENV == 'preproduction') {
        var sticky = require('socketio-sticky-session');

        sticky(function() {
            // This code will be executed only in slave workers

            var server = require('http').createServer();
            var io = require('socket.io')(server);

            io.on('connect', onConnect);

            return server;
        }).listen(config.port, function() {
            console.log('Socket started on ' + config.port + ' port');
        });
    } else {
        var io;
        if(config.port) {
            io = socket();
        } else {
            var app = express();
            var server = http.createServer(app);
            io = socket(server);
        }
        var d = domain.create();

        d.on('error', function(err) {
            logger.error(err, 'Socket.io'+':'+config.port);
        });

        d.run(function() {
            io.adapter(redis({ host: 'localhost', port: 6379 }));
            io.on('connection', onConnect);
        });
        if(config.port) {
            logger.log('Sockets', ['green'], ' listening on port ', config.port, ['yellow'], '.');

            io.listen(config.port, function() {
                if(typeof callback=='function') cb();
            });
        } else {
            io.listen(8888, function() {
                if(typeof callback=='function') cb();
            });
            return server;
        }
    }
};<|endoftext|>


<commit_before>'use strict';

const underscore = require('underscore');
const uuid = require('cassandra-uuid').TimeUuid;
const HTTPError = require('./util').HTTPError;
const mUtil = {};

mUtil.CONTENT_TYPES = {
    html: { name: 'HTML', version: '1.6.0', type: 'text/html' },
    mobileSections: { name: 'mobile-sections', version: '0.13.3', type: 'application/json' },
    media: { name: 'Media', version: '1.0.0', type: 'application/json' },
    readHtml: { name: 'ReadHtml', version: '0.1.0', type: 'text/html' },
    references: { name: 'References', version: '1.0.0', type: 'application/json' },
    metadata: { name: 'Metadata', version: '1.0.0', type: 'application/json' },
    summary: { name: 'Summary', version: '1.3.1', type: 'application/json' },
    definition: { name: 'definition', version: '0.7.4', type: 'application/json' },
    random: { name: 'random', version: '0.6.0', type: 'application/json' },
    announcements: { name: 'announcements', version: '0.1.0', type: 'application/json' },
    compilations: { name: 'compilations', version: '0.1.0', type: 'application/json' },
    onthisday: { name: 'onthisday', version: '0.3.3', type: 'application/json' },
    unpublished: { name: 'unpublished', version: '0.0.0', type: 'application/json' }
};

mUtil.getContentTypeString = function(spec) {
    return `${spec.type}; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/${spec.name}/${spec.version}"`;
};

mUtil.setContentType = function(res, spec) {
    if (!spec.name || !spec.version) {
        throw new HTTPError({
            status: 500,
            type: 'invalid_content_type',
            title: 'Only use the allowed content-types',
            detail: `${spec.name}/${spec.version} is not an allowed content-type!`
        });
    }

    res.type(mUtil.getContentTypeString(spec));
};

/**
 * Check if a variable is empty.
 * @param {string} val input value
 * @return {!boolean} true if val is null, undefined, an empty object, an empty array, or an empty
 * string.
 */
mUtil.isEmpty = function(val) {
    return !underscore.isNumber(val)
        && !underscore.isBoolean(val)
        && underscore.isEmpty(val);
};

mUtil.isNonempty = underscore.negate(mUtil.isEmpty);

/**
 * @param {*} val input value
 * @param {*} [fallback] the default value to assign if val is empty
 * @return {*} val if nonempty, else fallback.
*/
mUtil.defaultVal = function(val, fallback) {
    return underscore.isEmpty(val) ? fallback : val;
};

/**
 * @param {*} val input value
 * @return {*} val less empty elements.
*/
mUtil.filterEmpty = function(val) {
    if (Array.isArray(val)) {
        return val.map(mUtil.filterEmpty).filter(mUtil.isNonempty);
    }
    if (underscore.isObject(val)) {
        return underscore.pick(underscore.mapObject(val, mUtil.filterEmpty), mUtil.isNonempty);
    }
    return val;
};

/**
 * Sets the ETag header on the response objecti, comprised of the revision ID and
 * the time UUID. If the latter is not given, the current time stamp is used to
 * generate it.
 * @param {!Object}  response the HTTPResponse object on which to set the header
 * @param {?number} revision the revision integer ID to use
 * @param {?string}  tid      the time UUID to use; optional
 */
mUtil.setETag = function(response, revision, tid) {
    // we want to bail out if the revision hasn't been supplied, except
    // in the case revision === 0 because 0 is actually a valid rev_id
    if (!revision && revision !== 0) {
        return;
    }
    if (!tid) {
        tid = uuid.now().toString();
    }
    response.set('etag', `"${revision}/${tid}"`);
};

/**
 * Convert mobile to canonical domain,
 * e.g., 'en.m.wikipedia.org' => 'en.wikipedia.org'
 */
mUtil.mobileToCanonical = function(domain) {
    return domain.replace('.m.', '.');
};

/**
 * Remove the top-level domain from a domain string, e.g., 'en.wikipedia.org' ->
 * 'en.wikipedia'.
 */
mUtil.removeTLD = function(domain) {
    return domain.split('.').slice(0,2).join('.');
};

/**
 * Get the language of the wiki base don the domain name.
 * Example: 'en.wikipedia.org' -> 'en'.
 */
mUtil.getLanguageFromDomain = function(domain) {
    return domain.split('.')[0];
};

mUtil.removeFragment = function(href) {
    if (href.indexOf('#') > -1) {
        return href.substring(0, href.indexOf('#'));
    }
    return href;
};

mUtil.removeLinkPrefix = function(href) {
    return href.replace(/^\.\//, '');
};

mUtil.extractDbTitleFromAnchor = function(anchor) {
    return anchor && mUtil.removeFragment(mUtil.removeLinkPrefix(anchor.getAttribute('href')));
};

mUtil.getRbPageSummaryUrl = function(restbaseTpl, domain, title) {
    const request = restbaseTpl.expand({
        request: {
            params: {
                domain,
                path: `page/summary/${encodeURIComponent(title)}` },
        }
    });
    return request.uri;
};

mUtil.throw404 = function(message) {
    throw new HTTPError({
        status: 404,
        type: 'not_found',
        title: 'Not found',
        detail: message
    });
};

mUtil.deduplicate = function(array) {
    return Array.from(new Set(array));
};

/* jslint bitwise: true */
/* eslint no-bitwise: ["error", { "allow": ["<<"] }] */
mUtil.hashCode = function(string) {
    return string.split('').reduce((prevHash, currVal) =>
        ((prevHash << 5) - prevHash) + currVal.charCodeAt(0), 0);
};

/**
 * Gets the data-mw-section-id for the section containing the element.
 * @param {!Element} el An Element
 * @return {?integer} the data-mw-section-id for the section containing the element
 */
mUtil.getSectionIdForElement = (el) => {
    const section = el.closest('section');
    return section && parseInt(section.getAttribute('data-mw-section-id'), 10);
};

module.exports = mUtil;
<commit_msg>Summary: bump version to 1.3.2<commit_after>'use strict';

const underscore = require('underscore');
const uuid = require('cassandra-uuid').TimeUuid;
const HTTPError = require('./util').HTTPError;
const mUtil = {};

mUtil.CONTENT_TYPES = {
    html: { name: 'HTML', version: '1.6.0', type: 'text/html' },
    mobileSections: { name: 'mobile-sections', version: '0.13.3', type: 'application/json' },
    media: { name: 'Media', version: '1.0.0', type: 'application/json' },
    readHtml: { name: 'ReadHtml', version: '0.1.0', type: 'text/html' },
    references: { name: 'References', version: '1.0.0', type: 'application/json' },
    metadata: { name: 'Metadata', version: '1.0.0', type: 'application/json' },
    summary: { name: 'Summary', version: '1.3.2', type: 'application/json' },
    definition: { name: 'definition', version: '0.7.4', type: 'application/json' },
    random: { name: 'random', version: '0.6.0', type: 'application/json' },
    announcements: { name: 'announcements', version: '0.1.0', type: 'application/json' },
    compilations: { name: 'compilations', version: '0.1.0', type: 'application/json' },
    onthisday: { name: 'onthisday', version: '0.3.3', type: 'application/json' },
    unpublished: { name: 'unpublished', version: '0.0.0', type: 'application/json' }
};

mUtil.getContentTypeString = function(spec) {
    return `${spec.type}; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/${spec.name}/${spec.version}"`;
};

mUtil.setContentType = function(res, spec) {
    if (!spec.name || !spec.version) {
        throw new HTTPError({
            status: 500,
            type: 'invalid_content_type',
            title: 'Only use the allowed content-types',
            detail: `${spec.name}/${spec.version} is not an allowed content-type!`
        });
    }

    res.type(mUtil.getContentTypeString(spec));
};

/**
 * Check if a variable is empty.
 * @param {string} val input value
 * @return {!boolean} true if val is null, undefined, an empty object, an empty array, or an empty
 * string.
 */
mUtil.isEmpty = function(val) {
    return !underscore.isNumber(val)
        && !underscore.isBoolean(val)
        && underscore.isEmpty(val);
};

mUtil.isNonempty = underscore.negate(mUtil.isEmpty);

/**
 * @param {*} val input value
 * @param {*} [fallback] the default value to assign if val is empty
 * @return {*} val if nonempty, else fallback.
*/
mUtil.defaultVal = function(val, fallback) {
    return underscore.isEmpty(val) ? fallback : val;
};

/**
 * @param {*} val input value
 * @return {*} val less empty elements.
*/
mUtil.filterEmpty = function(val) {
    if (Array.isArray(val)) {
        return val.map(mUtil.filterEmpty).filter(mUtil.isNonempty);
    }
    if (underscore.isObject(val)) {
        return underscore.pick(underscore.mapObject(val, mUtil.filterEmpty), mUtil.isNonempty);
    }
    return val;
};

/**
 * Sets the ETag header on the response objecti, comprised of the revision ID and
 * the time UUID. If the latter is not given, the current time stamp is used to
 * generate it.
 * @param {!Object}  response the HTTPResponse object on which to set the header
 * @param {?number} revision the revision integer ID to use
 * @param {?string}  tid      the time UUID to use; optional
 */
mUtil.setETag = function(response, revision, tid) {
    // we want to bail out if the revision hasn't been supplied, except
    // in the case revision === 0 because 0 is actually a valid rev_id
    if (!revision && revision !== 0) {
        return;
    }
    if (!tid) {
        tid = uuid.now().toString();
    }
    response.set('etag', `"${revision}/${tid}"`);
};

/**
 * Convert mobile to canonical domain,
 * e.g., 'en.m.wikipedia.org' => 'en.wikipedia.org'
 */
mUtil.mobileToCanonical = function(domain) {
    return domain.replace('.m.', '.');
};

/**
 * Remove the top-level domain from a domain string, e.g., 'en.wikipedia.org' ->
 * 'en.wikipedia'.
 */
mUtil.removeTLD = function(domain) {
    return domain.split('.').slice(0,2).join('.');
};

/**
 * Get the language of the wiki base don the domain name.
 * Example: 'en.wikipedia.org' -> 'en'.
 */
mUtil.getLanguageFromDomain = function(domain) {
    return domain.split('.')[0];
};

mUtil.removeFragment = function(href) {
    if (href.indexOf('#') > -1) {
        return href.substring(0, href.indexOf('#'));
    }
    return href;
};

mUtil.removeLinkPrefix = function(href) {
    return href.replace(/^\.\//, '');
};

mUtil.extractDbTitleFromAnchor = function(anchor) {
    return anchor && mUtil.removeFragment(mUtil.removeLinkPrefix(anchor.getAttribute('href')));
};

mUtil.getRbPageSummaryUrl = function(restbaseTpl, domain, title) {
    const request = restbaseTpl.expand({
        request: {
            params: {
                domain,
                path: `page/summary/${encodeURIComponent(title)}` },
        }
    });
    return request.uri;
};

mUtil.throw404 = function(message) {
    throw new HTTPError({
        status: 404,
        type: 'not_found',
        title: 'Not found',
        detail: message
    });
};

mUtil.deduplicate = function(array) {
    return Array.from(new Set(array));
};

/* jslint bitwise: true */
/* eslint no-bitwise: ["error", { "allow": ["<<"] }] */
mUtil.hashCode = function(string) {
    return string.split('').reduce((prevHash, currVal) =>
        ((prevHash << 5) - prevHash) + currVal.charCodeAt(0), 0);
};

/**
 * Gets the data-mw-section-id for the section containing the element.
 * @param {!Element} el An Element
 * @return {?integer} the data-mw-section-id for the section containing the element
 */
mUtil.getSectionIdForElement = (el) => {
    const section = el.closest('section');
    return section && parseInt(section.getAttribute('data-mw-section-id'), 10);
};

module.exports = mUtil;
<|endoftext|>


<commit_before>var Bluebird = require('bluebird');
var eq = require('deep-equal');
var _ = require('lodash');

var diff = require('./diff');

function Model (collection, attributes, props) {
    this.def('collection', collection);
    this.def('old', {});
    this.def('attributes', {});
    this.def('columns', collection.table.columns);
    this.def('isFromDb', false);

    for (var key in props) {
        this.def(key, props[key]);
    }

    this.bindAccessors();
    _.extend(this, attributes);
}

/**
 * Binds setters and getters for model columns.
 */
Model.prototype.bindAccessors = function () {
    var self = this;
    this.columns.forEach(function (column) {
        var key = column.getName(), value;
        var getter = self.getters[key];
        var setter = self.setters[key];

        Object.defineProperty(self, key, {
            configurable: true,
            enumerable: true,
            get: function () {
                if (typeof getter === 'undefined') {
                    return self.attributes[key];
                } else {
                    return getter.call(self, self.attributes[key]);
                }
            },
            set: function (value) {
                if (typeof setter === 'undefined') {
                    self.attributes[key] = value;
                } else {
                    self.attributes[key] = setter.call(this, value);
                }
            }
        });
    });
};

/**
 * Defines a new non-enumerable property on the model.
 *
 * @param {String}   name
 * @param {*} value
 */
Model.prototype.def = function (name, value) {
    Object.defineProperty(this, name, {
        configurable: true,
        writable: true,
        enumerable: false,
        value: value
    });
};

/**
 * Wipes the sync state and attributes. Mainly for testing ease, but meh,
 * you might find this useful too.
 */
Model.prototype.reset = function () {
    this.old = {};
    this.isFromDb = false;
};

/**
 * Fixes the casing of a column name to match the model definition, since
 * Cassandra columns are case-insensitive but attributes are sensitive.
 * @param  {String} column
 * @return {String}
 */
Model.prototype.fixCasing = function (column) {
    for (var i = 0, l = this.columns.length; i < l; i++) {
        var name = this.columns[i].toString();
        if (name.toLowerCase() === column.toLowerCase()) {
            return name;
        }
    }

    return column;
};

/**
 * Updates the "old" and existing properties. Should be called after the
 * model is updated or read from the database.
 *
 * @param  {Object} attributes
 * @return {Model}
 */
Model.prototype.sync = function (attributes) {
    this.isFromDb = true;

    var self = this;
    Object.keys(attributes).forEach(function (key) {
        var value = attributes[key];
        var name = self.fixCasing(key);

        self.attributes[name] = _.cloneDeep(value);
        self.old[name] = _.cloneDeep(value);
    });

    return this;
};

/**
 * Extends this model's data with anything passed in the arguments. Functions
 * in the same way to lodash's "extend".
 * @return {Model}
 */
Model.prototype.extend = function () {
    var args = new Array(arguments.length + 1);
    var data = args[0] = {};

    var i, l;
    for (i = 0, l = arguments.length; i < l; i++) {
        args[i + 1] = arguments[i];
    }

    // Compact all the arguments to our single "data" argument
    _.extend.apply(null, args);

    // Then apply those over this model.
    for (i = 0, l = this.columns.length; i < l; i++) {
        var column = this.columns[i].toString();
        var item = data[column];
        if (typeof item !== 'undefined') {
            this[column] = item;
        }
    }

    return this;
};

/**
 * Iterates over the model's columns, typecasting each one.
 */
Model.prototype.fixTypes = function () {
    for (var i = 0, l = this.columns.length; i < l; i++) {
        var column = this.columns[i];
        var value = this.attributes[column.toString()];

        this.attributes[column.toString()] = this.collection.typecast(column, value);
    }
};

/**
 * Returns whether the property has changed since it was synced to the
 * database.
 * @param  {String}  property
 * @return {Boolean}
 */
Model.prototype.isDirty = function (property) {
    this.fixTypes();

    return !eq(this.attributes[property], this.old[property]);
};

/**
 * Returns true if every property is synced with the db. False if any
 * are different.
 *
 * @return {Boolean}
 */
Model.prototype.isSynced = function () {
    for (var i = 0, l = this.columns.length; i < l; i++) {
        if (this.isDirty(this.columns[i].toString())) {
            return false;
        }
    }

    return true;
};

/**
 * Converts the model to a plain object
 *
 * @param  {Boolean} useGetters
 * @return {Object}
 */
Model.prototype.toObject = function (useGetters) {
    this.fixTypes();

    var output = {};
    for (var i = 0, l = this.columns.length; i < l; i++) {
        var column = this.columns[i].toString();
        if (typeof useGetters === 'undefined' || useGetters === true) {
            output[column] = this[column];
        } else {
            output[column] = this.attributes[column];
        }
    }

    return output;
};

/**
 * Converts the record to a JSON representation.
 *
 * @return {String}
 */
Model.prototype.toJson = function () {
    return JSON.stringify(this.toObject());
};

/**
 * Persists the model to the database.
 *
 * @param {Boolean=false} force
 * @return {Promise}
 */
Model.prototype.save = function (force) {
    if (!force && this.isSynced()) {
        return Bluebird.resolve(this);
    }

    this.fixTypes();
    if (this.isFromDb) {
        return this.updateExisting();
    } else {
        return this.createNew();
    }
};

/**
 * Deletes this model's record from the database.
 *
 * @param {Boolean=false} force
 * @return {Promise}
 */
Model.prototype.delete = wrapAround('beforeDelete', function (force) {
    if (!this.isFromDb && !force) {
        return Bluebird.resolve();
    }

    return this.addWhereSelf(this.collection.delete()).execute();
}, 'afterDelete');

/**
 * Adds a "where" to the query so that it targets only this model.
 * @param {BaseQuery} query
 * @return {BaseQuery}
 */
Model.prototype.addWhereSelf = function (query) {
    var keys = this.collection.table.keys;
    var pks = keys.partition.concat(keys.compound);

    if (!pks.length) {
        throw new Error('A primary key is required on all models.');
    }

    for (var i = 0, l = pks.length; i < l; i++) {
        query.where(pks[i], '=', this.old[pks[i].toString()]);
    }

    return query;
};

/**
 * INSERTs the model into the database.
 *
 * @return {Promise}
 */
Model.prototype.createNew = wrapAround('beforeCreate', function () {
    var properties = this.toObject(false);
    var self = this;

    return this.collection.insert()
        .data(properties)
        .then(function () {
            self.sync(properties);
            return self;
        });
}, 'afterCreate');

/**
 * UPDATEs a record in the database with the changed properties.
 *
 * @return {Promise}
 */
Model.prototype.updateExisting = wrapAround('beforeUpdate', function () {
    var query = this.addWhereSelf(this.collection.update());
    var self = this;

    for (var i = 0, l = this.columns.length; i < l; i++) {
        var column = this.columns[i];
        var name = column.getName();

        diff(query, column, this.old[name], this.attributes[name]);
    }

    return query.then(function () {
        self.sync(self.toObject(false));
        return self;
    });
}, 'afterUpdate');

/**
 * Wraps the function so that the "before" and "after" middleware
 * is run "around" it.
 *
 * @param  {String}   before
 * @param  {Function} fn
 * @param  {String}   after
 * @return {Promise}
 */
function wrapAround (before, fn, after) {
    return function () {
        var self = this;
        return self.collection
            .run(before, self)
            .then(fn.bind(self))
            .then(function () {
                var resolved = Bluebird.resolve.apply(Bluebird, arguments);

                return self.collection.run(after, self).then(function () {
                    return resolved;
                });
            });
    };
}

module.exports = Model;
<commit_msg>Clean up, slight performance tweaks<commit_after>var Bluebird = require('bluebird');
var eq = require('deep-equal');
var _ = require('lodash');

var diff = require('./diff');

function Model (collection, attributes, props) {
    this.def('collection', collection);
    this.def('old', {});
    this.def('attributes', {});
    this.def('columns', collection.table.columns);
    this.def('isFromDb', false);

    for (var key in props) {
        this.def(key, props[key]);
    }

    this.bindAccessors();
    _.extend(this, attributes);
}

/**
 * Binds setters and getters for model columns.
 */
Model.prototype.bindAccessors = function () {
    var properties = {};

    _.forEach(this.columns, function (column) {
        var key = column.getName();
        var getter = this.getters[key];
        var setter = this.setters[key];

        properties[key] = {
            configurable: true,
            enumerable: true,
            get: function () {
                if (typeof getter === 'undefined') {
                    return this.attributes[key];
                } else {
                    return getter.call(this, this.attributes[key]);
                }
            },
            set: set(key)
        };
    }, this);

    var self = this;
    function set (key) {
        return function (value) {
                if (typeof setter === 'undefined') {
                    this.attributes[key] = value;
                } else {
                    this.attributes[key] = setter.call(this, value);
                }
            };
    }

    Object.defineProperties(this, properties);
};

/**
 * Defines a new non-enumerable property on the model.
 *
 * @param {String}   name
 * @param {*} value
 */
Model.prototype.def = function (name, value) {
    Object.defineProperty(this, name, {
        configurable: true,
        writable: true,
        enumerable: false,
        value: value
    });
};

/**
 * Wipes the sync state and attributes. Mainly for testing ease, but meh,
 * you might find this useful too.
 */
Model.prototype.reset = function () {
    this.old = {};
    this.isFromDb = false;
};

/**
 * Fixes the casing of a column name to match the model definition, since
 * Cassandra columns are case-insensitive but attributes are sensitive.
 * @param  {String} column
 * @return {String}
 */
Model.prototype.fixCasing = function (column) {
    for (var i = 0, l = this.columns.length; i < l; i++) {
        var name = this.columns[i].toString();
        if (name.toLowerCase() === column.toLowerCase()) {
            return name;
        }
    }

    return column;
};

/**
 * Updates the "old" and existing properties. Should be called after the
 * model is updated or read from the database.
 *
 * @param  {Object} attributes
 * @return {Model}
 */
Model.prototype.sync = function (attributes) {
    this.isFromDb = true;

    var self = this;
    Object.keys(attributes).forEach(function (key) {
        var value = attributes[key];
        var name = self.fixCasing(key);

        self.attributes[name] = _.cloneDeep(value);
        self.old[name] = _.cloneDeep(value);
    });

    return this;
};

/**
 * Extends this model's data with anything passed in the arguments. Functions
 * in the same way to lodash's "extend".
 * @return {Model}
 */
Model.prototype.extend = function () {
    var args = new Array(arguments.length + 1);
    var data = args[0] = {};

    var i, l;
    for (i = 0, l = arguments.length; i < l; i++) {
        args[i + 1] = arguments[i];
    }

    // Compact all the arguments to our single "data" argument
    _.extend.apply(null, args);

    // Then apply those over this model.
    for (i = 0, l = this.columns.length; i < l; i++) {
        var column = this.columns[i].toString();
        var item = data[column];
        if (typeof item !== 'undefined') {
            this[column] = item;
        }
    }

    return this;
};

/**
 * Iterates over the model's columns, typecasting each one.
 */
Model.prototype.fixTypes = function () {
    for (var i = 0, l = this.columns.length; i < l; i++) {
        var column = this.columns[i];
        var value = this.attributes[column.toString()];

        this.attributes[column.toString()] = this.collection.typecast(column, value);
    }
};

/**
 * Returns whether the property has changed since it was synced to the
 * database.
 * @param  {String}  property
 * @return {Boolean}
 */
Model.prototype.isDirty = function (property) {
    this.fixTypes();

    return !eq(this.attributes[property], this.old[property]);
};

/**
 * Returns true if every property is synced with the db. False if any
 * are different.
 *
 * @return {Boolean}
 */
Model.prototype.isSynced = function () {
    for (var i = 0, l = this.columns.length; i < l; i++) {
        if (this.isDirty(this.columns[i].toString())) {
            return false;
        }
    }

    return true;
};

/**
 * Converts the model to a plain object
 *
 * @param  {Boolean} useGetters
 * @return {Object}
 */
Model.prototype.toObject = function (useGetters) {
    this.fixTypes();

    var output = {};
    for (var i = 0, l = this.columns.length; i < l; i++) {
        var column = this.columns[i].toString();
        if (typeof useGetters === 'undefined' || useGetters === true) {
            output[column] = this[column];
        } else {
            output[column] = this.attributes[column];
        }
    }

    return output;
};

/**
 * Converts the record to a JSON representation.
 *
 * @return {String}
 */
Model.prototype.toJson = function () {
    return JSON.stringify(this.toObject());
};

/**
 * Persists the model to the database.
 *
 * @param {Boolean=false} force
 * @return {Promise}
 */
Model.prototype.save = function (force) {
    if (!force && this.isSynced()) {
        return Bluebird.resolve(this);
    }

    this.fixTypes();
    if (this.isFromDb) {
        return this.updateExisting();
    } else {
        return this.createNew();
    }
};

/**
 * Deletes this model's record from the database.
 *
 * @param {Boolean=false} force
 * @return {Promise}
 */
Model.prototype.delete = wrapAround('beforeDelete', function (force) {
    if (!this.isFromDb && !force) {
        return Bluebird.resolve();
    }

    return this.addWhereSelf(this.collection.delete()).execute();
}, 'afterDelete');

/**
 * Adds a "where" to the query so that it targets only this model.
 * @param {BaseQuery} query
 * @return {BaseQuery}
 */
Model.prototype.addWhereSelf = function (query) {
    var keys = this.collection.table.keys;
    var pks = keys.partition.concat(keys.compound);

    if (!pks.length) {
        throw new Error('A primary key is required on all models.');
    }

    for (var i = 0, l = pks.length; i < l; i++) {
        query.where(pks[i], '=', this.old[pks[i].toString()]);
    }

    return query;
};

/**
 * INSERTs the model into the database.
 *
 * @return {Promise}
 */
Model.prototype.createNew = wrapAround('beforeCreate', function () {
    var properties = this.toObject(false);
    var self = this;

    return this.collection.insert()
        .data(properties)
        .then(function () {
            self.sync(properties);
            return self;
        });
}, 'afterCreate');

/**
 * UPDATEs a record in the database with the changed properties.
 *
 * @return {Promise}
 */
Model.prototype.updateExisting = wrapAround('beforeUpdate', function () {
    var query = this.addWhereSelf(this.collection.update());
    var self = this;

    for (var i = 0, l = this.columns.length; i < l; i++) {
        var column = this.columns[i];
        var name = column.getName();

        diff(query, column, this.old[name], this.attributes[name]);
    }

    return query.then(function () {
        self.sync(self.toObject(false));
        return self;
    });
}, 'afterUpdate');

/**
 * Wraps the function so that the "before" and "after" middleware
 * is run "around" it.
 *
 * @param  {String}   before
 * @param  {Function} fn
 * @param  {String}   after
 * @return {Promise}
 */
function wrapAround (before, fn, after) {
    return function () {
        var self = this;
        return self.collection
            .run(before, self)
            .then(fn.bind(self))
            .then(function () {
                var resolved = Bluebird.resolve.apply(Bluebird, arguments);

                return self.collection.run(after, self).then(function () {
                    return resolved;
                });
            });
    };
}

module.exports = Model;
<|endoftext|>


<commit_before>import Promise from 'bluebird';
import getWeb3 from '../utils/getWeb3';
import { getTransactionReceiptMined, detectNetwork } from '../utils';

const Web3Service = () => {
    let web3;
    // 	_connected,
    // 	_balance,
    // 	_address,
    // 	_networkId,
    // 	_networkName;

    // let _loaded = false;

    function fetchBalance(addr) {
	if (!addr) {
	    return 0;
	}
	return web3.eth.getBalancePromise(addr);
    }

    async function setup(){
	// Get network provider and web3 instance.
	// See utils/getWeb3 for more info.
	web3 = await getWeb3();
	if (!web3) {
	    throw new Error("Web3 is not connected");
	}
	Promise.promisifyAll(web3.eth, { suffix: "Promise" });
	web3.eth.getTransactionReceiptMined = getTransactionReceiptMined;

	const address = web3.eth.accounts[0];
	const balance = await fetchBalance(address);
	const connected = web3.isConnected();
	const { networkName, networkId } = detectNetwork(web3);

	return {
	    web3,
	    balance,
	    address,
	    connected,
	    networkName,
	    networkId
	};
    }

    // api
    return {
	//isConnected: () => {return _connected && web3.eth.accounts.length > 0;},
	// getBalance: () => _balance,
	//getAddress: () => _address,
	//isLoaded: () => _loaded,
	//getNetworkId: () => _networkId,
	//getNetworkName: () => _networkName,
	//toWei: (n, k) => web3.toWei(n, k),
	//fromWei: (n, k) => web3.fromWei(n,k),
	//toBigNumber: (n) => web3.toBigNumber(n),
	getWeb3: () => web3,
	//getTransactionReceiptMined: (txHash) => web3.eth.getTransactionReceiptMined(txHash, 500),
	setup
    };
}

export default Web3Service();
<commit_msg>web3 service code cleanup<commit_after>import Promise from 'bluebird';
import getWeb3 from '../utils/getWeb3';
import { getTransactionReceiptMined, detectNetwork } from '../utils';

const Web3Service = () => {
    let web3;

    async function setup(){
	// Get network provider and web3 instance.
	// See utils/getWeb3 for more info.
	web3 = await getWeb3();
	if (!web3) {
	    throw new Error("Web3 is not connected");
	}
	Promise.promisifyAll(web3.eth, { suffix: "Promise" });
	web3.eth.getTransactionReceiptMined = getTransactionReceiptMined;

	const address = web3.eth.accounts[0];
	const balance = await web3.eth.getBalancePromise(address);
	const connected = web3.isConnected();
	const { networkName, networkId } = detectNetwork(web3);

	return {
	    web3,
	    balance,
	    address,
	    connected,
	    networkName,
	    networkId
	};
    }

    // api
    return {
	getWeb3: () => web3,
	setup
    };
}

export default Web3Service();
<|endoftext|>