<commit_before>import Promise from 'bluebird';
import querystring from 'querystring';
import urlparse from 'url';

import { getJSON, request } from '../request';
import Media from '../media';

import { Counter } from 'prom-client';

const LOGGER = require('@calzoneman/jsli')('mediaquery/youtube');

let API_KEY = null;
let CACHE = null;

// https://en.wikipedia.org/wiki/ISO_8601#Durations
const DURATION_SCALE = [
    [/(\d+)D/, 24*3600],
    [/(\d+)H/, 3600],
    [/(\d+)M/, 60],
    [/(\d+)S/, 1]
];

/*
 * Parse an ISO 8601 time duration (the format used by YouTube).
 * In the interest of sanity, only days, hours, minutes, and seconds are
 * considered.
 */
function parseDuration(duration) {
    let time = 0;
    for (let [regex, scale] of DURATION_SCALE) {
        let m;
        if (m = duration.match(regex)) {
            time += parseInt(m[1]) * scale;
        }
    }

    return time;
};

const cacheAttemptCount = new Counter({
    name: 'cytube_yt_cache_attempt_count',
    help: 'Number of video lookups eligible for YouTube cache'
});
const cacheFoundCount = new Counter({
    name: 'cytube_yt_cache_found_count',
    help: 'Number of video lookups with a result in the cache'
});
const cacheHitCount = new Counter({
    name: 'cytube_yt_cache_hit_count',
    help: 'Number of video lookups served using the cached result'
});

/*
 * Retrieve metadata for a single YouTube video.
 *
 * Returns a Media object
 */
export async function lookup(id) {
    let cached = null;
    if (CACHE !== null) {
        try {
            cacheAttemptCount.inc();
            cached = await CACHE.get(id, 'yt');
            if (cached !== null) {
                cacheFoundCount.inc();
            }
        } catch (error) {
            LOGGER.error('Error retrieving cached metadata for yt:%s - %s', id, error.stack);
        }
    }

    let media = await _lookupInternal(id, cached);
    if (CACHE !== null) {
        try {
            await CACHE.put(media);
        } catch (error) {
            LOGGER.error('Error updating cached metadata for yt:%s - %s', id, error.stack);
        }
    }

    return media;
}

function getErrorReason(body) {
    try {
        body = JSON.parse(body);
    } catch (error) {
        return null;
    }

    if (!body.error || !body.error.errors) return null;

    return body.error.errors[0].reason;
}

function map403Error(body) {
    switch (getErrorReason(body)) {
        case 'dailyLimitExceeded':
        case 'quotaExceeded':
            return new Error(
                'YouTube videos are temporarily unavailable due to the website ' +
                'exceeding the YouTube API quota.  ' +
                'Quota resets daily at midnight Pacific Time'
            );
        case 'accessNotConfigured':
            return new Error(
                'The YouTube API is not properly configured for this website.  ' +
                'See https://console.developers.google.com/apis/api/youtube.googleapis.com/overview'
            );
        default:
            return new Error(
                'Unable to access YouTube API'
            );
    }
}

function _lookupInternal(id, cached) {
    if (!API_KEY) {
        return Promise.reject(new Error('API key not set for YouTube v3 API'));
    }

    const params = querystring.stringify({
        key: API_KEY,
        part: 'contentDetails,status,snippet',
        id
    });

    const url = `https://www.googleapis.com/youtube/v3/videos?${params}`;
    let headers = {};
    if (cached !== null && cached.meta.etag) {
        headers = {
            'If-None-Match': cached.meta.etag
        };
    }

    return request(url, { headers }).then(res => {
        switch (res.statusCode) {
            case 304:
                cacheHitCount.inc();
                return cached;
            case 400:
                LOGGER.error('YouTube API returned Bad Request: %s', res.data);
                throw new Error('Error calling YouTube API: Bad Request');
            case 403:
                LOGGER.error('YouTube API returned Forbidden: %s', res.data);
                throw map403Error(res.data);
            case 500:
            case 503:
                throw new Error('YouTube API is unavailable.  Please try again later.');
            default:
                if (res.statusCode !== 200) {
                    throw new Error(`Error calling YouTube API: HTTP ${res.statusCode}`);
                }
                break;
        }

        let result;
        try {
            result = JSON.parse(res.data);
        } catch (error) {
            LOGGER.error(
                'YouTube API returned non-JSON response: %s',
                String(res.data).substring(0, 1000)
            );
            throw new Error('Error calling YouTube API: could not decode response as JSON');
        }

        // Sadly, as of the v3 API, YouTube doesn't tell you *why* the request failed.
        if (result.items.length === 0) {
            throw new Error('Video does not exist or is private');
        }

        const video = result.items[0];

        if (!video.status || !video.contentDetails || !video.snippet) {
            LOGGER.info('Incomplete video; assuming deleted video with id=%s', video.id);
            throw new Error('This video is unavailable');
        }

        if (!video.status.embeddable) {
            throw new Error('The uploader has made this video non-embeddable');
        }

        switch (video.status.uploadStatus) {
            case 'deleted':
                throw new Error('This video has been deleted');
            case 'failed':
                throw new Error(
                    'This video is unavailable: ' +
                    video.status.failureReason
                );
            case 'rejected':
                throw new Error(
                    'This video is unavailable: ' +
                    video.status.rejectionReason
                );
            case 'processed':
                break;
            case 'uploaded':
                // For VODs, we must wait for 'processed' before the video
                // metadata is correct.  For livestreams, the status is
                // 'uploaded' while the stream is live, and the metadata
                // is presumably correct (we don't care about duration
                // for livestreams anyways)
                // See calzoneman/sync#710
                if (video.snippet.liveBroadcastContent !== 'live') {
                    throw new Error(
                        'This video has not been processed yet.'
                    );
                }
                break;
            default:
                throw new Error(
                    'This video is not available ' +
                    `(status=${video.status.uploadStatus})`
                );
        }

        const data = {
            id,
            type: 'youtube',
            title: video.snippet.title,
            duration: parseDuration(video.contentDetails.duration),
            meta: {
                thumbnail: video.snippet.thumbnails.default.url,
                etag: result.etag
            }
        };

        if (video.contentDetails.regionRestriction) {
            const restriction = video.contentDetails.regionRestriction;
            if (restriction.blocked) {
                data.meta.blocked = restriction.blocked;
            }
            if (restriction.allowed) {
                data.meta.allowed = restriction.allowed;
            }
        }

        return new Media(data);
    });
}

/*
 * Retrieve metadata for multiple YouTube videos.  As this is intended for use
 * by the search and playlist retrieval, it does not check for failed video IDs.
 * If a video is private, removed, or non-embeddable, its information simply
 * doesn't appear in the results.
 *
 * Returns a list of Media objects
 */
export function lookupMany(ids) {
    if (!API_KEY) {
        return Promise.reject(new Error('API key not set for YouTube v3 API'));
    }

    const params = querystring.stringify({
        key: API_KEY,
        part: 'contentDetails,status,snippet',
        id: ids.join(',')
    });

    const url = `https://www.googleapis.com/youtube/v3/videos?${params}`;

    return getJSON(url).then(result => {
        return result.items.filter(video => {
            if (!video.status || !video.contentDetails || !video.snippet) {
                LOGGER.info('Incomplete video; assuming deleted video with id=%s', video.id);
                return false;
            }

            return video.status.embeddable
        }).map(video => {
            const data = {
                id: video.id,
                type: 'youtube',
                title: video.snippet.title,
                duration: parseDuration(video.contentDetails.duration),
                meta: {
                    thumbnail: video.snippet.thumbnails.default.url
                }
            };

            if (video.contentDetails.regionRestriction) {
                const restriction = video.contentDetails.regionRestriction;
                if (restriction.blocked) {
                    data.meta.blocked = restriction.blocked;
                }
                if (restriction.allowed) {
                    data.meta.allowed = restriction.allowed;
                }
            }

            return new Media(data);
        });
    });
}

/*
 * Search for YouTube videos.  Optionally provide the ID of the page of results
 * to retrieve.
 *
 * Returns { nextPage: (string: next page token), results: (list of Media) }
 */
export function search(query, nextPage = false) {
    if (!API_KEY) {
        return Promise.reject(new Error('API key not set for YouTube v3 API'));
    }

    query = query.replace(/%20/g, '+');

    let params = {
        key: API_KEY,
        part: 'id',
        maxResults: 25,
        q: query,
        type: 'video'
    };

    if (nextPage) {
        params.pageToken = nextPage;
    }

    params = querystring.stringify(params);

    const url = `https://www.googleapis.com/youtube/v3/search?${params}`;

    return getJSON(url).then(result => {
        // https://code.google.com/p/gdata-issues/issues/detail?id=4294
        return lookupMany(result.items.map(item => item.id.videoId)).then(videos => {
            return {
                nextPage: result.nextPageToken || false,
                results: videos
            };
        })
    });
}

/*
 * Retrieve metadata for all items on a YouTube playlist.  For playlists longer
 * than 50 videos, it recurses to retrieve every page of results.
 *
 * Returns a list of Media objects
 */
export function lookupPlaylist(id, nextPage = false) {
    if (!API_KEY) {
        return Promise.reject(new Error('API key not set for YouTube v3 API'));
    }

    let params = {
        key: API_KEY,
        part: 'contentDetails',
        maxResults: 50,
        playlistId: id
    };

    if (nextPage) {
        params.pageToken = nextPage;
    }

    params = querystring.stringify(params);

    const url = `https://www.googleapis.com/youtube/v3/playlistItems?${params}`;

    return getJSON(url).then(result => {
        return lookupMany(result.items.map(item => item.contentDetails.videoId))
                .then(videos => {
            if (result.nextPageToken) {
                // Retrieve the rest of the results, then concatenate them with the current
                // page
                return lookupPlaylist(id, result.nextPageToken).then(
                        other => videos.concat(other));
            } else {
                // No more pages of results
                return videos;
            }
        });
    });
}

/*
 * Attempts to parse a YouTube URL following one of the following forms:
 *   - youtu.be/(video id)
 *   - [www.]youtube.com/watch?v=(video id)
 *   - [www.]youtube.com/playllist?list=(playlist id)
 *
 * Returns {
 *           id: video or playlist id
 *           kind: 'single' or 'playlist'
 *           type: 'youtube'
 *         }
 * or null if the URL is invalid.
 */
export function parseUrl(url) {
    let m = url.match(/^yt:(.*)/);
    if (m) {
        return {
            type: 'youtube',
            kind: 'single',
            id: m[1]
        };
    }

    m = url.match(/^yp:(.*)/);
    if (m) {
        return {
            type: 'youtube',
            kind: 'playlist',
            id: m[1]
        };
    }

    const data = urlparse.parse(url, true);

    if (data.hostname === 'youtu.be') {
        return {
            type: 'youtube',
            kind: 'single',
            id: data.pathname.replace(/^\//, '')
        };
    } else if (!['www.youtube.com', 'youtube.com'].includes(data.hostname)) {
        return null;
    }

    if (data.pathname === '/watch') {
        return {
            type: 'youtube',
            kind: 'single',
            id: data.query.v
        };
    } else if (data.pathname === '/playlist') {
        return {
            type: 'youtube',
            kind: 'playlist',
            id: data.query.list
        };
    } else {
        return null;
    }
}

export function setApiKey(key) {
    API_KEY = key;
}

export function setCache(cache) {
    CACHE = cache;
}
<commit_msg>Made youtube.js check if a video is age-restricted<commit_after>import Promise from 'bluebird';
import querystring from 'querystring';
import urlparse from 'url';

import { getJSON, request } from '../request';
import Media from '../media';

import { Counter } from 'prom-client';

const LOGGER = require('@calzoneman/jsli')('mediaquery/youtube');

let API_KEY = null;
let CACHE = null;

// https://en.wikipedia.org/wiki/ISO_8601#Durations
const DURATION_SCALE = [
    [/(\d+)D/, 24*3600],
    [/(\d+)H/, 3600],
    [/(\d+)M/, 60],
    [/(\d+)S/, 1]
];

/*
 * Parse an ISO 8601 time duration (the format used by YouTube).
 * In the interest of sanity, only days, hours, minutes, and seconds are
 * considered.
 */
function parseDuration(duration) {
    let time = 0;
    for (let [regex, scale] of DURATION_SCALE) {
        let m;
        if (m = duration.match(regex)) {
            time += parseInt(m[1]) * scale;
        }
    }

    return time;
};

const cacheAttemptCount = new Counter({
    name: 'cytube_yt_cache_attempt_count',
    help: 'Number of video lookups eligible for YouTube cache'
});
const cacheFoundCount = new Counter({
    name: 'cytube_yt_cache_found_count',
    help: 'Number of video lookups with a result in the cache'
});
const cacheHitCount = new Counter({
    name: 'cytube_yt_cache_hit_count',
    help: 'Number of video lookups served using the cached result'
});

/*
 * Retrieve metadata for a single YouTube video.
 *
 * Returns a Media object
 */
export async function lookup(id) {
    let cached = null;
    if (CACHE !== null) {
        try {
            cacheAttemptCount.inc();
            cached = await CACHE.get(id, 'yt');
            if (cached !== null) {
                cacheFoundCount.inc();
            }
        } catch (error) {
            LOGGER.error('Error retrieving cached metadata for yt:%s - %s', id, error.stack);
        }
    }

    let media = await _lookupInternal(id, cached);
    if (CACHE !== null) {
        try {
            await CACHE.put(media);
        } catch (error) {
            LOGGER.error('Error updating cached metadata for yt:%s - %s', id, error.stack);
        }
    }

    return media;
}

function getErrorReason(body) {
    try {
        body = JSON.parse(body);
    } catch (error) {
        return null;
    }

    if (!body.error || !body.error.errors) return null;

    return body.error.errors[0].reason;
}

function map403Error(body) {
    switch (getErrorReason(body)) {
        case 'dailyLimitExceeded':
        case 'quotaExceeded':
            return new Error(
                'YouTube videos are temporarily unavailable due to the website ' +
                'exceeding the YouTube API quota.  ' +
                'Quota resets daily at midnight Pacific Time'
            );
        case 'accessNotConfigured':
            return new Error(
                'The YouTube API is not properly configured for this website.  ' +
                'See https://console.developers.google.com/apis/api/youtube.googleapis.com/overview'
            );
        default:
            return new Error(
                'Unable to access YouTube API'
            );
    }
}

function _lookupInternal(id, cached) {
    if (!API_KEY) {
        return Promise.reject(new Error('API key not set for YouTube v3 API'));
    }

    const params = querystring.stringify({
        key: API_KEY,
        part: 'contentDetails,status,snippet',
        id
    });

    const url = `https://www.googleapis.com/youtube/v3/videos?${params}`;
    let headers = {};
    if (cached !== null && cached.meta.etag) {
        headers = {
            'If-None-Match': cached.meta.etag
        };
    }

    return request(url, { headers }).then(res => {
        switch (res.statusCode) {
            case 304:
                cacheHitCount.inc();
                return cached;
            case 400:
                LOGGER.error('YouTube API returned Bad Request: %s', res.data);
                throw new Error('Error calling YouTube API: Bad Request');
            case 403:
                LOGGER.error('YouTube API returned Forbidden: %s', res.data);
                throw map403Error(res.data);
            case 500:
            case 503:
                throw new Error('YouTube API is unavailable.  Please try again later.');
            default:
                if (res.statusCode !== 200) {
                    throw new Error(`Error calling YouTube API: HTTP ${res.statusCode}`);
                }
                break;
        }

        let result;
        try {
            result = JSON.parse(res.data);
        } catch (error) {
            LOGGER.error(
                'YouTube API returned non-JSON response: %s',
                String(res.data).substring(0, 1000)
            );
            throw new Error('Error calling YouTube API: could not decode response as JSON');
        }

        // Sadly, as of the v3 API, YouTube doesn't tell you *why* the request failed.
        if (result.items.length === 0) {
            throw new Error('Video does not exist or is private');
        }

        const video = result.items[0];

        if (!video.status || !video.contentDetails || !video.snippet) {
            LOGGER.info('Incomplete video; assuming deleted video with id=%s', video.id);
            throw new Error('This video is unavailable');
        }

        if (!video.status.embeddable) {
            throw new Error('The uploader has made this video non-embeddable');
        }

        switch (video.status.uploadStatus) {
            case 'deleted':
                throw new Error('This video has been deleted');
            case 'failed':
                throw new Error(
                    'This video is unavailable: ' +
                    video.status.failureReason
                );
            case 'rejected':
                throw new Error(
                    'This video is unavailable: ' +
                    video.status.rejectionReason
                );
            case 'processed':
                break;
            case 'uploaded':
                // For VODs, we must wait for 'processed' before the video
                // metadata is correct.  For livestreams, the status is
                // 'uploaded' while the stream is live, and the metadata
                // is presumably correct (we don't care about duration
                // for livestreams anyways)
                // See calzoneman/sync#710
                if (video.snippet.liveBroadcastContent !== 'live') {
                    throw new Error(
                        'This video has not been processed yet.'
                    );
                }
                break;
            default:
                throw new Error(
                    'This video is not available ' +
                    `(status=${video.status.uploadStatus})`
                );
        }

        const data = {
            id,
            type: 'youtube',
            title: video.snippet.title,
            duration: parseDuration(video.contentDetails.duration),
            meta: {
                thumbnail: video.snippet.thumbnails.default.url,
                etag: result.etag
            }
        };

        if (video.contentDetails.regionRestriction) {
            const restriction = video.contentDetails.regionRestriction;
            if (restriction.blocked) {
                data.meta.blocked = restriction.blocked;
            }
            if (restriction.allowed) {
                data.meta.allowed = restriction.allowed;
            }
        }

        if (video.contentDetails.contentRating) {
            const rating = video.contentDetails.contentRating;

            if (rating.ytRating) {
                data.meta.ytRating = rating.ytRating;
            }
        }

        return new Media(data);
    });
}

/*
 * Retrieve metadata for multiple YouTube videos.  As this is intended for use
 * by the search and playlist retrieval, it does not check for failed video IDs.
 * If a video is private, removed, or non-embeddable, its information simply
 * doesn't appear in the results.
 *
 * Returns a list of Media objects
 */
export function lookupMany(ids) {
    if (!API_KEY) {
        return Promise.reject(new Error('API key not set for YouTube v3 API'));
    }

    const params = querystring.stringify({
        key: API_KEY,
        part: 'contentDetails,status,snippet',
        id: ids.join(',')
    });

    const url = `https://www.googleapis.com/youtube/v3/videos?${params}`;

    return getJSON(url).then(result => {
        return result.items.filter(video => {
            if (!video.status || !video.contentDetails || !video.snippet) {
                LOGGER.info('Incomplete video; assuming deleted video with id=%s', video.id);
                return false;
            }

            return video.status.embeddable
        }).map(video => {
            const data = {
                id: video.id,
                type: 'youtube',
                title: video.snippet.title,
                duration: parseDuration(video.contentDetails.duration),
                meta: {
                    thumbnail: video.snippet.thumbnails.default.url
                }
            };

            if (video.contentDetails.regionRestriction) {
                const restriction = video.contentDetails.regionRestriction;
                if (restriction.blocked) {
                    data.meta.blocked = restriction.blocked;
                }
                if (restriction.allowed) {
                    data.meta.allowed = restriction.allowed;
                }
            }

            return new Media(data);
        });
    });
}

/*
 * Search for YouTube videos.  Optionally provide the ID of the page of results
 * to retrieve.
 *
 * Returns { nextPage: (string: next page token), results: (list of Media) }
 */
export function search(query, nextPage = false) {
    if (!API_KEY) {
        return Promise.reject(new Error('API key not set for YouTube v3 API'));
    }

    query = query.replace(/%20/g, '+');

    let params = {
        key: API_KEY,
        part: 'id',
        maxResults: 25,
        q: query,
        type: 'video'
    };

    if (nextPage) {
        params.pageToken = nextPage;
    }

    params = querystring.stringify(params);

    const url = `https://www.googleapis.com/youtube/v3/search?${params}`;

    return getJSON(url).then(result => {
        // https://code.google.com/p/gdata-issues/issues/detail?id=4294
        return lookupMany(result.items.map(item => item.id.videoId)).then(videos => {
            return {
                nextPage: result.nextPageToken || false,
                results: videos
            };
        })
    });
}

/*
 * Retrieve metadata for all items on a YouTube playlist.  For playlists longer
 * than 50 videos, it recurses to retrieve every page of results.
 *
 * Returns a list of Media objects
 */
export function lookupPlaylist(id, nextPage = false) {
    if (!API_KEY) {
        return Promise.reject(new Error('API key not set for YouTube v3 API'));
    }

    let params = {
        key: API_KEY,
        part: 'contentDetails',
        maxResults: 50,
        playlistId: id
    };

    if (nextPage) {
        params.pageToken = nextPage;
    }

    params = querystring.stringify(params);

    const url = `https://www.googleapis.com/youtube/v3/playlistItems?${params}`;

    return getJSON(url).then(result => {
        return lookupMany(result.items.map(item => item.contentDetails.videoId))
                .then(videos => {
            if (result.nextPageToken) {
                // Retrieve the rest of the results, then concatenate them with the current
                // page
                return lookupPlaylist(id, result.nextPageToken).then(
                        other => videos.concat(other));
            } else {
                // No more pages of results
                return videos;
            }
        });
    });
}

/*
 * Attempts to parse a YouTube URL following one of the following forms:
 *   - youtu.be/(video id)
 *   - [www.]youtube.com/watch?v=(video id)
 *   - [www.]youtube.com/playllist?list=(playlist id)
 *
 * Returns {
 *           id: video or playlist id
 *           kind: 'single' or 'playlist'
 *           type: 'youtube'
 *         }
 * or null if the URL is invalid.
 */
export function parseUrl(url) {
    let m = url.match(/^yt:(.*)/);
    if (m) {
        return {
            type: 'youtube',
            kind: 'single',
            id: m[1]
        };
    }

    m = url.match(/^yp:(.*)/);
    if (m) {
        return {
            type: 'youtube',
            kind: 'playlist',
            id: m[1]
        };
    }

    const data = urlparse.parse(url, true);

    if (data.hostname === 'youtu.be') {
        return {
            type: 'youtube',
            kind: 'single',
            id: data.pathname.replace(/^\//, '')
        };
    } else if (!['www.youtube.com', 'youtube.com'].includes(data.hostname)) {
        return null;
    }

    if (data.pathname === '/watch') {
        return {
            type: 'youtube',
            kind: 'single',
            id: data.query.v
        };
    } else if (data.pathname === '/playlist') {
        return {
            type: 'youtube',
            kind: 'playlist',
            id: data.query.list
        };
    } else {
        return null;
    }
}

export function setApiKey(key) {
    API_KEY = key;
}

export function setCache(cache) {
    CACHE = cache;
}
<|endoftext|>


<commit_before>(function() {
    try {
        // https://developer.chrome.com/docs/extensions/mv2/tut_analytics/
        // https://developer.chrome.com/docs/extensions/mv3/tut_analytics/

        window._gaq = window._gaq || [];
        var _gaq = window._gaq;
        _gaq.push(['_setAccount', 'UA-198813835-1']);
        _gaq.push(['_gat._forceSSL']); // https://stackoverflow.com/questions/37799578/google-analytics-force-https-to-prevent-307-internal-redirect/37799579#37799579
        _gaq.push(['_trackPageview']);

        // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/random
        const getRandomIntInclusive = function (min, max) {
            min = Math.ceil(min);
            max = Math.floor(max);
            return Math.floor(Math.random() * (max - min + 1) + min); //The maximum is inclusive and the minimum is inclusive
        };

        setTimeout(
            function () {
                var ga = document.createElement('script');
                ga.type = 'text/javascript';
                ga.async = true;
                ga.src = 'https://ssl.google-analytics.com/ga.js';
                var s = document.getElementsByTagName('script')[0];
                s.parentNode.insertBefore(ga, s);
            },
            // Load with a delay since this network request is not required for showing the UI of the extension
            getRandomIntInclusive(40 * 1000, 400 * 1000)
        );
    } catch (e) {
        // do nothing
    }
})();
<commit_msg>Improvised actions.js since the background page of extension becomes inactive after few seconds of idle period<commit_after>(function() {
    try {
        // https://developer.chrome.com/docs/extensions/mv2/tut_analytics/
        // https://developer.chrome.com/docs/extensions/mv3/tut_analytics/

        window._gaq = window._gaq || [];
        var _gaq = window._gaq;
        _gaq.push(['_setAccount', 'UA-198813835-1']);
        _gaq.push(['_gat._forceSSL']); // https://stackoverflow.com/questions/37799578/google-analytics-force-https-to-prevent-307-internal-redirect/37799579#37799579

        // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/random
        const getRandomIntInclusive = function (min, max) {
            min = Math.ceil(min);
            max = Math.floor(max);
            return Math.floor(Math.random() * (max - min + 1) + min); //The maximum is inclusive and the minimum is inclusive
        };

        // Some random delay which should not be too high since the background page of the extension can become inactive after a few seconds
        const delay = getRandomIntInclusive(2250, 4500);
        setTimeout(
            function () {
                var ga = document.createElement('script');
                ga.type = 'text/javascript';
                ga.async = true;
                ga.src = 'https://ssl.google-analytics.com/ga.js';
                var s = document.getElementsByTagName('script')[0];
                s.parentNode.insertBefore(ga, s);
            },
            // Load with a delay since this network request is not required for showing the UI of the extension
            delay
        );
    } catch (e) {
        // do nothing
    }
})();
<|endoftext|>


<commit_before>/*
 * Copyright 2008 The Closure Compiler Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * @fileoverview The current draft spec of User Agent Client Hint interface.
 * @see https://wicg.github.io/ua-client-hints/#interface
 * @externs
 */

/**
 * @see https://wicg.github.io/ua-client-hints/#dictdef-navigatoruabrandversion
 * @record
 * @struct
 */
function NavigatorUABrandVersion() {}

/** @type {string} */
NavigatorUABrandVersion.prototype.brand;

/** @type {string} */
NavigatorUABrandVersion.prototype.version;

/**
 * @see https://wicg.github.io/ua-client-hints/#dictdef-uadatavalues
 * @record
 * @struct
 */
function UADataValues() {}

/** @type {string} */
UADataValues.prototype.platform;

/** @type {string} */
UADataValues.prototype.platformVersion;

/** @type {string} */
UADataValues.prototype.architecture;

/** @type {string} */
UADataValues.prototype.model;

/** @type {string} */
UADataValues.prototype.uaFullVersion;

/** @type {string} */
UADataValues.prototype.bitness;

/**
 * @see https://wicg.github.io/ua-client-hints/#navigatoruadata
 * @record
 * @struct
 */
function NavigatorUAData() {}

/** @type {!Array<!NavigatorUABrandVersion>} */
NavigatorUAData.prototype.brands;

/** @type {boolean} */
NavigatorUAData.prototype.mobile;

/**
 * @param {!Array<string>} hints
 * @return {!Promise<!UADataValues>}
 */
NavigatorUAData.prototype.getHighEntropyValues = function(hints) {};

/**
 * @type {?NavigatorUAData}
 * @see https://wicg.github.io/ua-client-hints/#interface
 */
Navigator.prototype.userAgentData;

/**
 * @type {?NavigatorUAData}
 * @see https://wicg.github.io/ua-client-hints/#interface
 */
WorkerNavigator.prototype.userAgentData;
<commit_msg>Add 'platform' to NavigatorUAData<commit_after>/*
 * Copyright 2008 The Closure Compiler Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * @fileoverview The current draft spec of User Agent Client Hint interface.
 * @see https://wicg.github.io/ua-client-hints/#interface
 * @externs
 */

/**
 * @see https://wicg.github.io/ua-client-hints/#dictdef-navigatoruabrandversion
 * @record
 * @struct
 */
function NavigatorUABrandVersion() {}

/** @type {string} */
NavigatorUABrandVersion.prototype.brand;

/** @type {string} */
NavigatorUABrandVersion.prototype.version;

/**
 * @see https://wicg.github.io/ua-client-hints/#dictdef-uadatavalues
 * @record
 * @struct
 */
function UADataValues() {}

/** @type {string} */
UADataValues.prototype.platform;

/** @type {string} */
UADataValues.prototype.platformVersion;

/** @type {string} */
UADataValues.prototype.architecture;

/** @type {string} */
UADataValues.prototype.model;

/** @type {string} */
UADataValues.prototype.uaFullVersion;

/** @type {string} */
UADataValues.prototype.bitness;

/**
 * @see https://wicg.github.io/ua-client-hints/#navigatoruadata
 * @record
 * @struct
 */
function NavigatorUAData() {}

/** @type {!Array<!NavigatorUABrandVersion>} */
NavigatorUAData.prototype.brands;

/** @type {boolean} */
NavigatorUAData.prototype.mobile;

/** @type {string} */
NavigatorUAData.prototype.platform;

/**
 * @param {!Array<string>} hints
 * @return {!Promise<!UADataValues>}
 */
NavigatorUAData.prototype.getHighEntropyValues = function(hints) {};

/**
 * @type {?NavigatorUAData}
 * @see https://wicg.github.io/ua-client-hints/#interface
 */
Navigator.prototype.userAgentData;

/**
 * @type {?NavigatorUAData}
 * @see https://wicg.github.io/ua-client-hints/#interface
 */
WorkerNavigator.prototype.userAgentData;
<|endoftext|>


<commit_before>(function(angular, $, _, undefined) {

  // Cache schema metadata
  var schema = [];
  // Cache fk schema data
  var links = [];
  // Cache list of entities
  var entities = [];
  // Cache list of actions
  var actions = [];

  angular.module('api4').config(function($routeProvider) {
      $routeProvider.when('/api4/:api4entity?/:api4action?', {
        controller: 'Api4Explorer',
        templateUrl: '~/api4/Explorer.html',
        reloadOnSearch: false
      });
    }
  );

  angular.module('api4').controller('Api4Explorer', function($scope, $routeParams, $location, $timeout, crmUiHelp, crmApi4) {
    var ts = $scope.ts = CRM.ts('api4');
    $scope.entities = entities;
    $scope.operators = arrayToSelect2(CRM.vars.api4.operators);
    $scope.actions = actions;
    $scope.fields = [];
    $scope.fieldsAndJoins = [];
    $scope.availableParams = {};
    $scope.params = {};
    var getMetaParams = schema.length ? {} : {schema: ['Entity', 'getFields'], links: ['Entity', 'getLinks']},
      objectParams = {orderBy: 'ASC', values: ''},
      helpTitle = '',
      helpContent = {};
    $scope.helpTitle = '';
    $scope.helpContent = {};
    $scope.entity = $routeParams.api4entity;
    $scope.result = [];
    $scope.status = 'default';
    $scope.loading = false;
    $scope.controls = {};
    $scope.code = {
      php: '',
      javascript: ''
    };

    function ucfirst(str) {
      return str[0].toUpperCase() + str.slice(1);
    }

    function lcfirst(str) {
      return str[0].toLowerCase() + str.slice(1);
    }

    function pluralize(str) {
      switch (str[str.length-1]) {
        case 's':
          return str + 'es';
        case 'y':
          return str.slice(0, -1) + 'ies';
        default:
          return str + 's';
      }
    }

    // Turn a flat array into a select2 array
    function arrayToSelect2(array) {
      var out = [];
      _.each(array, function(item) {
        out.push({id: item, text: item});
      });
      return out;
    }

    // Reformat an existing array of objects for compatibility with select2
    function formatForSelect2(input, container, key, extra, prefix) {
      _.each(input, function(item) {
        var id = (prefix || '') + item[key];
        var formatted = {id: id, text: id};
        if (extra) {
          _.merge(formatted, _.pick(item, extra));
        }
        container.push(formatted);
      });
      return container;
    }

    function entityFields(entity) {
      return _.result(_.findWhere(schema, {name: entity}), 'fields');
    }

    function getFieldList() {
      var fields = [];
      formatForSelect2(entityFields($scope.entity), fields, 'name', ['description', 'required', 'default_value']);
      return fields;
    }

    function addJoins(fieldList) {
      var fields = _.cloneDeep(fieldList),
        fks = _.findWhere(links, {entity: $scope.entity}) || {};
      _.each(fks.links, function(link) {
        var linkFields = entityFields(link.entity);
        if (linkFields) {
          fields.push({
            text: link.alias,
            description: 'Join to ' + link.entity,
            children: formatForSelect2(linkFields, [], 'name', ['description'], link.alias + '.')
          });
        }
      });
      return fields;
    }

    $scope.help = function(title, param) {
      if (!param) {
        $scope.helpTitle = helpTitle;
        $scope.helpContent = helpContent;
      } else {
        $scope.helpTitle = title;
        $scope.helpContent = param;
      }
    };

    $scope.valuesFields = function() {
      var fields = [];
      _.each(_.cloneDeep($scope.fields), function(field, index) {
        if ((field.id === 'id' && $scope.action === 'create') || field.children) {
          return;
        }
        if ($scope.params.values && typeof $scope.params.values[field.id] !== 'undefined') {
          field.disabled = true;
        }
        fields.push(field);
      });
      return fields;
    };

    $scope.selectOptions = function() {
      if ($scope.availableParams.select.options) {
        return arrayToSelect2($scope.availableParams.select.options);
      } else {
        return $scope.fieldsAndJoins;
      }
    };

    $scope.formatSelect2Item = function(row) {
      return _.escape(row.text) +
        (isFieldRequiredForCreate(row) ? ' *' : '') +
        (row.description ? '

' + _.escape(row.description) + '

' : '');
    };

    function isFieldRequiredForCreate(field) {
      return field.required && !field.default_value;
    }

    // Get all params that have been set
    function getParams() {
      var params = {};
      _.each($scope.params, function(param, key) {
        if (param != $scope.availableParams[key].default && !(typeof param === 'object' && _.isEmpty(param))) {
          params[key] = param;
        }
      });
      _.each(objectParams, function(defaultVal, key) {
        if (params[key]) {
          var newParam = {};
          _.each(params[key], function(item) {
            newParam[item[0]] = item[1];
          });
          params[key] = newParam;
        }
      });
      return params;
    }

    function selectAction() {
      $scope.action = $routeParams.api4action;
      $scope.fields = getFieldList();
      $scope.fieldsAndJoins = addJoins($scope.fields);
      if ($scope.action) {
        var actionInfo = _.findWhere(actions, {id: $scope.action});
        _.each(actionInfo.params, function (param, name) {
          var format,
            defaultVal = _.cloneDeep(param.default);
          if (param.type) {
            switch (param.type[0]) {
              case 'int':
              case 'bool':
                format = param.type[0];
                break;

              case 'array':
              case 'object':
                format = 'json';
                break;

              default:
                format = 'raw';
            }
            if (name == 'limit') {
              defaultVal = 25;
            }
            $scope.$bindToRoute({
              expr: 'params["' + name + '"]',
              param: name,
              format: format,
              default: defaultVal,
              deep: name === 'where'
            });
          }
          if (typeof objectParams[name] !== 'undefined') {
            $scope.$watch('params.' + name, function(values) {
              // Remove empty values
              _.each(values, function(clause, index) {
                if (!clause[0]) {
                  $scope.params[name].splice(index, 1);
                }
              });
            }, true);
            $scope.$watch('controls.' + name, function(value) {
              var field = value;
              $timeout(function() {
                if (field) {
                  var defaultOp = objectParams[name];
                  if (_.isEmpty($scope.params[name])) {
                    $scope.params[name] = [[field, defaultOp]];
                  } else {
                    $scope.params[name].push([field, defaultOp]);
                  }
                  $scope.controls[name] = null;
                }
              });
            });
          }
        });
        $scope.availableParams = actionInfo.params;
      }
      writeCode();
    }

    function stringify(value, trim) {
      var str = JSON.stringify(value).replace(/,/g, ', ');
      if (trim) {
        str = str.slice(1, -1);
      }
      return str.trim();
    }

    function writeCode() {
      var code = {
        php: ts('Select an entity and action'),
        javascript: ''
      },
        entity = $scope.entity,
        action = $scope.action,
        params = getParams(),
        result = 'result';
      if ($scope.entity && $scope.action) {
        if (action.slice(0, 3) === 'get') {
          result = lcfirst(action.replace(/s$/, '').slice(3) || entity);
        }
        var results = lcfirst(pluralize(result)),
          paramCount = _.size(params),
          i = 0;
        code.javascript = "CRM.api4('" + entity + "', '" + action + "', {";
        _.each(params, function(param, key) {
          code.javascript += "\n  " + key + ': ' + stringify(param) +
            (++i < paramCount ? ',' : '');
          if (key === 'checkPermissions') {
            code.javascript += ' // IGNORED: permissions are always enforced from client-side requests';
          }
        });
        code.javascript += "\n}).done(function(" + results + ") {\n  // do something with " + results + " array\n});";
        if (entity.substr(0, 7) !== 'Custom_') {
          code.php = '$' + results + " = \\Civi\\Api4\\" + entity + '::' + action + '()';
        } else {
          code.php = '$' + results + " = \\Civi\\Api4\\CustomValue()::" + action + "('" + entity.substr(7) + "')";
        }
        _.each(params, function(param, key) {
          var val = '';
          if (typeof objectParams[key] !== 'undefined') {
            _.each(param, function(item, index) {
              val = stringify(index) + ', ' + stringify(item);
              code.php += "\n  ->add" + ucfirst(key).replace(/s$/, '') + '(' + val + ')';
            });
          } else if (key === 'where') {
            _.each(param, function (clause) {
              if (clause[0] === 'AND' || clause[0] === 'OR' || clause[0] === 'NOT') {
                code.php += "\n  ->addClause('" + clause[0] + "', " + stringify(clause[1], true) + ')';
              } else {
                code.php += "\n  ->addWhere(" + stringify(clause, true) + ")";
              }
            });
          } else {
            code.php += "\n  ->set" + ucfirst(key) + '(' + stringify(param) + ')';
          }
        });
        code.php += "\n  ->execute();\nforeach ($" + results + ' as $' + result + ') {\n  // do something\n}';
      }
      $scope.code = code;
    }

    $scope.execute = function() {
      $scope.status = 'warning';
      $scope.loading = true;
      crmApi4($scope.entity, $scope.action, getParams())
        .then(function(data) {
          var meta = {length: data.length},
            result = JSON.stringify(data, null, 2);
          data.length = 0;
          _.assign(meta, data);
          $scope.loading = false;
          $scope.status = 'success';
          $scope.result = [JSON.stringify(meta).replace('{', '').replace(/}$/, ''), result];
        }, function(data) {
          $scope.loading = false;
          $scope.status = 'danger';
          $scope.result = [JSON.stringify(data, null, 2)];
        });
    };

    function fetchMeta() {
      crmApi4(getMetaParams)
        .then(function(data) {
          if (data.schema) {
            schema = data.schema;
            entities.length = 0;
            formatForSelect2(schema, entities, 'name', ['description']);
            if ($scope.entity && !$scope.action) {
              showEntityHelp($scope.entity);
            }
          }
          if (data.links) {
            links = data.links;
          }
          if (data.actions) {
            formatForSelect2(data.actions, actions, 'name', ['description', 'params']);
            selectAction();
          }
        });
    }

    // Help for an entity with no action selected
    function showEntityHelp(entity) {
      var entityInfo = _.findWhere(schema, {name: entity});
      $scope.helpTitle = helpTitle = $scope.entity;
      $scope.helpContent = helpContent = {
        description: entityInfo.description,
        comment: entityInfo.comment
      };
    }

    if (!$scope.entity) {
      $scope.helpTitle = helpTitle = ts('Help');
      $scope.helpContent = helpContent = {description: ts('Welcome to the api explorer.'), comment: ts('Select an entity to begin.')};
      if (getMetaParams.schema) {
        fetchMeta();
      }
    } else if (!actions.length) {
      if (getMetaParams.schema) {
        entities.push({id: $scope.entity, text: $scope.entity});
      }
      getMetaParams.actions = [$scope.entity, 'getActions'];
      fetchMeta();
    } else {
      selectAction();
    }

    if ($scope.entity && schema.length) {
      showEntityHelp($scope.entity);
    }

    // Update route when changing entity
    $scope.$watch('entity', function(newVal, oldVal) {
      if (oldVal !== newVal) {
        // Flush actions cache to re-fetch for new entity
        actions = [];
        $location.url('/api4/' + newVal);
      }
    });

    // Update route when changing actions
    $scope.$watch('action', function(newVal, oldVal) {
      if ($scope.entity && $routeParams.api4action !== newVal && !_.isUndefined(newVal)) {
        $location.url('/api4/' + $scope.entity + '/' + newVal);
      } else if (newVal) {
        $scope.helpTitle = helpTitle = $scope.entity + '::' + newVal;
        $scope.helpContent = helpContent = _.pick(_.findWhere(actions, {id: newVal}), ['description', 'comment']);
      }
    });

    $scope.$watch('params', writeCode, true);
    writeCode();

  });

  angular.module('api4').directive('crmApi4WhereClause', function($timeout) {
    return {
      scope: {
        data: '=crmApi4WhereClause'
      },
      templateUrl: '~/api4/WhereClause.html',
      link: function (scope, element, attrs) {
        var ts = scope.ts = CRM.ts('api4');
        scope.newClause = '';
        scope.conjunctions = ['AND', 'OR', 'NOT'];

        scope.addGroup = function(op) {
          scope.data.where.push([op, []]);
        };

        scope.removeGroup = function() {
          scope.data.groupParent.splice(scope.data.groupIndex, 1);
        };

        scope.onSort = function(event, ui) {
          $('.api4-where-fieldset').toggleClass('api4-sorting', event.type === 'sortstart');
          $('.api4-input.form-inline').css('margin-left', '');
        };

        // Indent clause while dragging between nested groups
        scope.onSortOver = function(event, ui) {
          var offset = 0;
          if (ui.sender) {
            offset = $(ui.placeholder).offset().left - $(ui.sender).offset().left;
          }
          $('.api4-input.form-inline.ui-sortable-helper').css('margin-left', '' + offset + 'px');
        };

        scope.$watch('newClause', function(value) {
          var field = value;
          $timeout(function() {
            if (field) {
              scope.data.where.push([field, '=', '']);
              scope.newClause = null;
            }
          });
        });
        scope.$watch('data.where', function(values) {
          // Remove empty values
          _.each(values, function(clause, index) {
            if (typeof clause !== 'undefined' && !clause[0]) {
              values.splice(index, 1);
            }
          });
        }, true);
      }
    };
  });

  // Collapsible optgroups for select2
  $(function() {
    $('body')
      .on('select2-open', function(e) {
        if ($(e.target).hasClass('collapsible-optgroups')) {
          $('#select2-drop')
            .off('.collapseOptionGroup')
            .addClass('collapsible-optgroups-enabled')
            .on('click.collapseOptionGroup', '.select2-result-with-children > .select2-result-label', function() {
              $(this).parent().toggleClass('optgroup-expanded');
            });
        }
      })
     .on('select2-close', function() {
        $('#select2-drop').off('.collapseOptionGroup').removeClass('collapsible-optgroups-enabled');
      });
  });
})(angular, CRM.$, CRM._);
<commit_msg>Explorer: Write cli code<commit_after>(function(angular, $, _, undefined) {

  // Cache schema metadata
  var schema = [];
  // Cache fk schema data
  var links = [];
  // Cache list of entities
  var entities = [];
  // Cache list of actions
  var actions = [];

  angular.module('api4').config(function($routeProvider) {
      $routeProvider.when('/api4/:api4entity?/:api4action?', {
        controller: 'Api4Explorer',
        templateUrl: '~/api4/Explorer.html',
        reloadOnSearch: false
      });
    }
  );

  angular.module('api4').controller('Api4Explorer', function($scope, $routeParams, $location, $timeout, crmUiHelp, crmApi4) {
    var ts = $scope.ts = CRM.ts('api4');
    $scope.entities = entities;
    $scope.operators = arrayToSelect2(CRM.vars.api4.operators);
    $scope.actions = actions;
    $scope.fields = [];
    $scope.fieldsAndJoins = [];
    $scope.availableParams = {};
    $scope.params = {};
    var getMetaParams = schema.length ? {} : {schema: ['Entity', 'getFields'], links: ['Entity', 'getLinks']},
      objectParams = {orderBy: 'ASC', values: ''},
      helpTitle = '',
      helpContent = {};
    $scope.helpTitle = '';
    $scope.helpContent = {};
    $scope.entity = $routeParams.api4entity;
    $scope.result = [];
    $scope.status = 'default';
    $scope.loading = false;
    $scope.controls = {};
    $scope.code = {
      php: '',
      javascript: '',
      cli: ''
    };

    function ucfirst(str) {
      return str[0].toUpperCase() + str.slice(1);
    }

    function lcfirst(str) {
      return str[0].toLowerCase() + str.slice(1);
    }

    function pluralize(str) {
      switch (str[str.length-1]) {
        case 's':
          return str + 'es';
        case 'y':
          return str.slice(0, -1) + 'ies';
        default:
          return str + 's';
      }
    }

    // Turn a flat array into a select2 array
    function arrayToSelect2(array) {
      var out = [];
      _.each(array, function(item) {
        out.push({id: item, text: item});
      });
      return out;
    }

    // Reformat an existing array of objects for compatibility with select2
    function formatForSelect2(input, container, key, extra, prefix) {
      _.each(input, function(item) {
        var id = (prefix || '') + item[key];
        var formatted = {id: id, text: id};
        if (extra) {
          _.merge(formatted, _.pick(item, extra));
        }
        container.push(formatted);
      });
      return container;
    }

    function entityFields(entity) {
      return _.result(_.findWhere(schema, {name: entity}), 'fields');
    }

    function getFieldList() {
      var fields = [];
      formatForSelect2(entityFields($scope.entity), fields, 'name', ['description', 'required', 'default_value']);
      return fields;
    }

    function addJoins(fieldList) {
      var fields = _.cloneDeep(fieldList),
        fks = _.findWhere(links, {entity: $scope.entity}) || {};
      _.each(fks.links, function(link) {
        var linkFields = entityFields(link.entity);
        if (linkFields) {
          fields.push({
            text: link.alias,
            description: 'Join to ' + link.entity,
            children: formatForSelect2(linkFields, [], 'name', ['description'], link.alias + '.')
          });
        }
      });
      return fields;
    }

    $scope.help = function(title, param) {
      if (!param) {
        $scope.helpTitle = helpTitle;
        $scope.helpContent = helpContent;
      } else {
        $scope.helpTitle = title;
        $scope.helpContent = param;
      }
    };

    $scope.valuesFields = function() {
      var fields = [];
      _.each(_.cloneDeep($scope.fields), function(field, index) {
        if ((field.id === 'id' && $scope.action === 'create') || field.children) {
          return;
        }
        if ($scope.params.values && typeof $scope.params.values[field.id] !== 'undefined') {
          field.disabled = true;
        }
        fields.push(field);
      });
      return fields;
    };

    $scope.selectOptions = function() {
      if ($scope.availableParams.select.options) {
        return arrayToSelect2($scope.availableParams.select.options);
      } else {
        return $scope.fieldsAndJoins;
      }
    };

    $scope.formatSelect2Item = function(row) {
      return _.escape(row.text) +
        (isFieldRequiredForCreate(row) ? ' *' : '') +
        (row.description ? '

' + _.escape(row.description) + '

' : '');
    };

    function isFieldRequiredForCreate(field) {
      return field.required && !field.default_value;
    }

    // Get all params that have been set
    function getParams() {
      var params = {};
      _.each($scope.params, function(param, key) {
        if (param != $scope.availableParams[key].default && !(typeof param === 'object' && _.isEmpty(param))) {
          params[key] = param;
        }
      });
      _.each(objectParams, function(defaultVal, key) {
        if (params[key]) {
          var newParam = {};
          _.each(params[key], function(item) {
            newParam[item[0]] = item[1];
          });
          params[key] = newParam;
        }
      });
      return params;
    }

    function selectAction() {
      $scope.action = $routeParams.api4action;
      $scope.fields = getFieldList();
      $scope.fieldsAndJoins = addJoins($scope.fields);
      if ($scope.action) {
        var actionInfo = _.findWhere(actions, {id: $scope.action});
        _.each(actionInfo.params, function (param, name) {
          var format,
            defaultVal = _.cloneDeep(param.default);
          if (param.type) {
            switch (param.type[0]) {
              case 'int':
              case 'bool':
                format = param.type[0];
                break;

              case 'array':
              case 'object':
                format = 'json';
                break;

              default:
                format = 'raw';
            }
            if (name == 'limit') {
              defaultVal = 25;
            }
            $scope.$bindToRoute({
              expr: 'params["' + name + '"]',
              param: name,
              format: format,
              default: defaultVal,
              deep: name === 'where'
            });
          }
          if (typeof objectParams[name] !== 'undefined') {
            $scope.$watch('params.' + name, function(values) {
              // Remove empty values
              _.each(values, function(clause, index) {
                if (!clause[0]) {
                  $scope.params[name].splice(index, 1);
                }
              });
            }, true);
            $scope.$watch('controls.' + name, function(value) {
              var field = value;
              $timeout(function() {
                if (field) {
                  var defaultOp = objectParams[name];
                  if (_.isEmpty($scope.params[name])) {
                    $scope.params[name] = [[field, defaultOp]];
                  } else {
                    $scope.params[name].push([field, defaultOp]);
                  }
                  $scope.controls[name] = null;
                }
              });
            });
          }
        });
        $scope.availableParams = actionInfo.params;
      }
      writeCode();
    }

    function stringify(value, trim) {
      var str = JSON.stringify(value).replace(/,/g, ', ');
      if (trim) {
        str = str.slice(1, -1);
      }
      return str.trim();
    }

    function writeCode() {
      var code = {
        php: ts('Select an entity and action'),
        javascript: '',
        cli: ''
      },
        entity = $scope.entity,
        action = $scope.action,
        params = getParams(),
        result = 'result';
      if ($scope.entity && $scope.action) {
        if (action.slice(0, 3) === 'get') {
          result = lcfirst(action.replace(/s$/, '').slice(3) || entity);
        }
        var results = lcfirst(pluralize(result)),
          paramCount = _.size(params),
          i = 0;

        // Write javascript
        code.javascript = "CRM.api4('" + entity + "', '" + action + "', {";
        _.each(params, function(param, key) {
          code.javascript += "\n  " + key + ': ' + stringify(param) +
            (++i < paramCount ? ',' : '');
          if (key === 'checkPermissions') {
            code.javascript += ' // IGNORED: permissions are always enforced from client-side requests';
          }
        });
        code.javascript += "\n}).done(function(" + results + ") {\n  // do something with " + results + " array\n});";

        // Write php code
        if (entity.substr(0, 7) !== 'Custom_') {
          code.php = '$' + results + " = \\Civi\\Api4\\" + entity + '::' + action + '()';
        } else {
          code.php = '$' + results + " = \\Civi\\Api4\\CustomValue()::" + action + "('" + entity.substr(7) + "')";
        }
        _.each(params, function(param, key) {
          var val = '';
          if (typeof objectParams[key] !== 'undefined') {
            _.each(param, function(item, index) {
              val = stringify(index) + ', ' + stringify(item);
              code.php += "\n  ->add" + ucfirst(key).replace(/s$/, '') + '(' + val + ')';
            });
          } else if (key === 'where') {
            _.each(param, function (clause) {
              if (clause[0] === 'AND' || clause[0] === 'OR' || clause[0] === 'NOT') {
                code.php += "\n  ->addClause('" + clause[0] + "', " + stringify(clause[1], true) + ')';
              } else {
                code.php += "\n  ->addWhere(" + stringify(clause, true) + ")";
              }
            });
          } else {
            code.php += "\n  ->set" + ucfirst(key) + '(' + stringify(param) + ')';
          }
        });
        code.php += "\n  ->execute();\nforeach ($" + results + ' as $' + result + ') {\n  // do something\n}';

        // Write cli code
        code.cli = 'cv api4 ' + entity + '.' + action + " '" + stringify(params) + "'";
      }
      $scope.code = code;
    }

    $scope.execute = function() {
      $scope.status = 'warning';
      $scope.loading = true;
      crmApi4($scope.entity, $scope.action, getParams())
        .then(function(data) {
          var meta = {length: data.length},
            result = JSON.stringify(data, null, 2);
          data.length = 0;
          _.assign(meta, data);
          $scope.loading = false;
          $scope.status = 'success';
          $scope.result = [JSON.stringify(meta).replace('{', '').replace(/}$/, ''), result];
        }, function(data) {
          $scope.loading = false;
          $scope.status = 'danger';
          $scope.result = [JSON.stringify(data, null, 2)];
        });
    };

    function fetchMeta() {
      crmApi4(getMetaParams)
        .then(function(data) {
          if (data.schema) {
            schema = data.schema;
            entities.length = 0;
            formatForSelect2(schema, entities, 'name', ['description']);
            if ($scope.entity && !$scope.action) {
              showEntityHelp($scope.entity);
            }
          }
          if (data.links) {
            links = data.links;
          }
          if (data.actions) {
            formatForSelect2(data.actions, actions, 'name', ['description', 'params']);
            selectAction();
          }
        });
    }

    // Help for an entity with no action selected
    function showEntityHelp(entity) {
      var entityInfo = _.findWhere(schema, {name: entity});
      $scope.helpTitle = helpTitle = $scope.entity;
      $scope.helpContent = helpContent = {
        description: entityInfo.description,
        comment: entityInfo.comment
      };
    }

    if (!$scope.entity) {
      $scope.helpTitle = helpTitle = ts('Help');
      $scope.helpContent = helpContent = {description: ts('Welcome to the api explorer.'), comment: ts('Select an entity to begin.')};
      if (getMetaParams.schema) {
        fetchMeta();
      }
    } else if (!actions.length) {
      if (getMetaParams.schema) {
        entities.push({id: $scope.entity, text: $scope.entity});
      }
      getMetaParams.actions = [$scope.entity, 'getActions'];
      fetchMeta();
    } else {
      selectAction();
    }

    if ($scope.entity && schema.length) {
      showEntityHelp($scope.entity);
    }

    // Update route when changing entity
    $scope.$watch('entity', function(newVal, oldVal) {
      if (oldVal !== newVal) {
        // Flush actions cache to re-fetch for new entity
        actions = [];
        $location.url('/api4/' + newVal);
      }
    });

    // Update route when changing actions
    $scope.$watch('action', function(newVal, oldVal) {
      if ($scope.entity && $routeParams.api4action !== newVal && !_.isUndefined(newVal)) {
        $location.url('/api4/' + $scope.entity + '/' + newVal);
      } else if (newVal) {
        $scope.helpTitle = helpTitle = $scope.entity + '::' + newVal;
        $scope.helpContent = helpContent = _.pick(_.findWhere(actions, {id: newVal}), ['description', 'comment']);
      }
    });

    $scope.$watch('params', writeCode, true);
    writeCode();

  });

  angular.module('api4').directive('crmApi4WhereClause', function($timeout) {
    return {
      scope: {
        data: '=crmApi4WhereClause'
      },
      templateUrl: '~/api4/WhereClause.html',
      link: function (scope, element, attrs) {
        var ts = scope.ts = CRM.ts('api4');
        scope.newClause = '';
        scope.conjunctions = ['AND', 'OR', 'NOT'];

        scope.addGroup = function(op) {
          scope.data.where.push([op, []]);
        };

        scope.removeGroup = function() {
          scope.data.groupParent.splice(scope.data.groupIndex, 1);
        };

        scope.onSort = function(event, ui) {
          $('.api4-where-fieldset').toggleClass('api4-sorting', event.type === 'sortstart');
          $('.api4-input.form-inline').css('margin-left', '');
        };

        // Indent clause while dragging between nested groups
        scope.onSortOver = function(event, ui) {
          var offset = 0;
          if (ui.sender) {
            offset = $(ui.placeholder).offset().left - $(ui.sender).offset().left;
          }
          $('.api4-input.form-inline.ui-sortable-helper').css('margin-left', '' + offset + 'px');
        };

        scope.$watch('newClause', function(value) {
          var field = value;
          $timeout(function() {
            if (field) {
              scope.data.where.push([field, '=', '']);
              scope.newClause = null;
            }
          });
        });
        scope.$watch('data.where', function(values) {
          // Remove empty values
          _.each(values, function(clause, index) {
            if (typeof clause !== 'undefined' && !clause[0]) {
              values.splice(index, 1);
            }
          });
        }, true);
      }
    };
  });

  // Collapsible optgroups for select2
  $(function() {
    $('body')
      .on('select2-open', function(e) {
        if ($(e.target).hasClass('collapsible-optgroups')) {
          $('#select2-drop')
            .off('.collapseOptionGroup')
            .addClass('collapsible-optgroups-enabled')
            .on('click.collapseOptionGroup', '.select2-result-with-children > .select2-result-label', function() {
              $(this).parent().toggleClass('optgroup-expanded');
            });
        }
      })
     .on('select2-close', function() {
        $('#select2-drop').off('.collapseOptionGroup').removeClass('collapsible-optgroups-enabled');
      });
  });
})(angular, CRM.$, CRM._);
<|endoftext|>


<commit_before>angular.module('siyfion.sfTypeahead', [])
  .directive('sfTypeahead', function () {
    return {
      restrict: 'ACE',
      scope: {
        datasets: '=',
        ngModel: '='
      },
      link: function (scope, element) {
        var localChange = false;
        element.typeahead(scope.datasets);

        function updateScope(object, datum, dataset) {
          // for some reason $apply will place [Object] into element, this hacks around it
          var preserveVal = element.val();
          scope.$apply(function(){
            localChange = true;
            scope.ngModel = datum;
            scope.selectedDataset = dataset;
          });
          element.val(preserveVal);
        }

        // Updates the ngModel binding when a value is manually selected from the dropdown.
        element.bind('typeahead:selected', updateScope);

        // Updates the ngModel binding when a query is autocompleted.
        element.bind('typeahead:autocompleted', updateScope);

        // Updates the ngModel binding when the user manually enters some text
        element.bind('input', function () {
          scope.$apply(function () {
            localChange = true;
            var value = element.val();
            scope.ngModel = value;
          });
        });

        // Updates typeahead when ngModel changed.
        scope.$watch('ngModel', function (newVal) {
          var valueKey;
          if (localChange) {
            localChange = false;
            return;
          }

          if ($.isArray(scope.datasets)) {
            for (var i = 0; i < scope.datasets.length; i++) {
              if (scope.datasets[i].name ==  scope.selectedDataset) {
                valueKey = scope.datasets[i].valueKey;
                break;
              }
            }
          } else {
            valueKey = scope.datasets.valueKey;
          }

          if (newVal && valueKey && newVal.hasOwnProperty(valueKey)) {
            newVal = newVal[valueKey];
          }
          element.typeahead('setQuery', newVal || '');
        });
      }
    };
  });

<commit_msg>Updated some code layout to maintain consistency.<commit_after>angular.module('siyfion.sfTypeahead', [])
  .directive('sfTypeahead', function () {
    return {
      restrict: 'ACE',
      scope: {
        datasets: '=',
        ngModel: '='
      },
      link: function (scope, element) {
        var localChange = false;
        element.typeahead(scope.datasets);

        function updateScope (object, datum, dataset) {
          // for some reason $apply will place [Object] into element, this hacks around it
          var preserveVal = element.val();
          scope.$apply(function () {
            localChange = true;
            scope.ngModel = datum;
            scope.selectedDataset = dataset;
          });
          element.val(preserveVal);
        }

        // Updates the ngModel binding when a value is manually selected from the dropdown.
        element.bind('typeahead:selected', updateScope);

        // Updates the ngModel binding when a query is autocompleted.
        element.bind('typeahead:autocompleted', updateScope);

        // Updates the ngModel binding when the user manually enters some text
        element.bind('input', function () {
          scope.$apply(function () {
            localChange = true;
            var value = element.val();
            scope.ngModel = value;
          });
        });

        // Updates typeahead when ngModel changed.
        scope.$watch('ngModel', function (newVal) {
          var valueKey;
          if (localChange) {
            localChange = false;
            return;
          }

          if ($.isArray(scope.datasets)) {
            for (var i = 0; i < scope.datasets.length; i++) {
              if (scope.datasets[i].name ==  scope.selectedDataset) {
                valueKey = scope.datasets[i].valueKey;
                break;
              }
            }
          } else {
            valueKey = scope.datasets.valueKey;
          }

          if (newVal && valueKey && newVal.hasOwnProperty(valueKey)) {
            newVal = newVal[valueKey];
          }
          element.typeahead('setQuery', newVal || '');
        });
      }
    };
  });

<|endoftext|>


<commit_before>'use strict';

// Utils
const { logError } = require('../utils/log');

const Datastore = require('nedb-promise');

const User = new Datastore({
	autoload: true,
	filename: 'data/User.db'
});

User.ensureIndex({
	fieldName: 'id',
	unique: true
});

User.ensureIndex({
	fieldName: 'status',
});

const addUser = ({ id, first_name = '', last_name = '', username = '' }) =>
	User.update(
		{ id },
		{ first_name, id, last_name, status: 'member', username, warns: [] },
		{ upsert: true })
		.catch(logError);

const isUser = ({ id }) =>
	User.findOne({ id });

const getUser = user =>
	User.findOne(user);

const admin = ({ id }) =>
	User.update(
		{ id },
		{ $set: { status: 'admin' } });

const getAdmins = () =>
	User.find({ status: 'admin' });

const unadmin = ({ id }) =>
	User.update({ id }, { $set: { status: 'member' } });

const isAdmin = ({ id }) =>
	User.findOne({ id, status: 'admin' });

const ban = ({ id }, ban_reason) =>
	User.update({ id }, { $set: { ban_reason, status: 'banned', warns: [] } });

const unban = ({ id }) =>
	User.update({ id }, { $set: { banReason: '', status: 'member' } });

const isBanned = ({ id }) =>
	User.findOne({ id, status: 'banned' })
		.then(user => user ? user.ban_reason : null);

const warn = ({ id }, reason) =>
	User.update({ id }, { $push: { warns: reason } });

const unwarn = ({ id }) =>
	User.update({ id }, { $pop: { warns: 1 } });

const nowarns = ({ id }) =>
	User.update({ id }, { $set: { warns: [] } });

const getWarns = ({ id }) =>
	User.findOne({ id })
		.then(user => user && user.warns.length > 0
			? user.warns
			: null);

module.exports = {
	addUser,
	admin,
	ban,
	getAdmins,
	getUser,
	getWarns,
	isAdmin,
	isBanned,
	isUser,
	nowarns,
	unadmin,
	unban,
	unwarn,
	warn
};
<commit_msg>Remove warnings on unban instead of on ban<commit_after>'use strict';

// Utils
const { logError } = require('../utils/log');

const Datastore = require('nedb-promise');

const User = new Datastore({
	autoload: true,
	filename: 'data/User.db'
});

User.ensureIndex({
	fieldName: 'id',
	unique: true
});

User.ensureIndex({
	fieldName: 'status',
});

const addUser = ({ id, first_name = '', last_name = '', username = '' }) =>
	User.update(
		{ id },
		{ first_name, id, last_name, status: 'member', username, warns: [] },
		{ upsert: true })
		.catch(logError);

const isUser = ({ id }) =>
	User.findOne({ id });

const getUser = user =>
	User.findOne(user);

const admin = ({ id }) =>
	User.update(
		{ id },
		{ $set: { status: 'admin' } });

const getAdmins = () =>
	User.find({ status: 'admin' });

const unadmin = ({ id }) =>
	User.update({ id }, { $set: { status: 'member' } });

const isAdmin = ({ id }) =>
	User.findOne({ id, status: 'admin' });

const ban = ({ id }, ban_reason) =>
	User.update({ id }, { $set: { ban_reason, status: 'banned' } });

const unban = ({ id }) =>
	User.update(
		{ id },
		{ $set: { ban_reason: null, status: 'member', warns: [] } });

const isBanned = ({ id }) =>
	User.findOne({ id, status: 'banned' })
		.then(user => user ? user.ban_reason : null);

const warn = ({ id }, reason) =>
	User.update({ id }, { $push: { warns: reason } });

const unwarn = ({ id }) =>
	User.update({ id }, { $pop: { warns: 1 } });

const nowarns = ({ id }) =>
	User.update({ id }, { $set: { warns: [] } });

const getWarns = ({ id }) =>
	User.findOne({ id })
		.then(user => user && user.warns.length > 0
			? user.warns
			: null);

module.exports = {
	addUser,
	admin,
	ban,
	getAdmins,
	getUser,
	getWarns,
	isAdmin,
	isBanned,
	isUser,
	nowarns,
	unadmin,
	unban,
	unwarn,
	warn
};
<|endoftext|>


<commit_before>/* eslint no-console:0, prefer-spread:0 */
"use strict";

var childProcess = require("child_process");
var fs = require("fs");
var http = require("http");
var jspngopt = require("jspngopt");
var net = require("net");
var pako = require("pako");
var path = require("path");
var selenium = require("selenium-webdriver");
var firefox = require("selenium-webdriver/firefox");

var app = require("../../server");
var data = require("../../test/screenshotter/ss_data");

var dstDir = path.normalize(
    path.join(__dirname, "..", "..", "test", "screenshotter", "images"));

//////////////////////////////////////////////////////////////////////
// Process command line arguments

var opts = require("nomnom")
    .option("browser", {
        abbr: "b",
        "default": "firefox",
        help: "Name of the browser to use",
    })
    .option("container", {
        abbr: "c",
        type: "string",
        help: "Name or ID of a running docker container to contact",
    })
    .option("seleniumURL", {
        full: "selenium-url",
        help: "Full URL of the Selenium web driver",
    })
    .option("seleniumIP", {
        full: "selenium-ip",
        help: "IP address of the Selenium web driver",
    })
    .option("seleniumPort", {
        full: "selenium-port",
        "default": 4444,
        help: "Port number of the Selenium web driver",
    })
    .option("katexURL", {
        full: "katex-url",
        help: "Full URL of the KaTeX development server",
    })
    .option("katexIP", {
        full: "katex-ip",
        "default": "localhost",
        help: "Full URL of the KaTeX development server",
    })
    .option("katexPort", {
        full: "katex-port",
        help: "Port number of the KaTeX development server",
    })
    .option("include", {
        abbr: "i",
        help: "Comma-separated list of test cases to process",
    })
    .option("exclude", {
        abbr: "x",
        help: "Comma-separated list of test cases to exclude",
    })
    .option("verify", {
        flag: true,
        help: "Check whether screenshot matches current file content",
    })
    .parse();

var listOfCases;
if (opts.include) {
    listOfCases = opts.include.split(",");
} else {
    listOfCases = Object.keys(data);
}
if (opts.exclude) {
    var exclude = opts.exclude.split(",");
    listOfCases = listOfCases.filter(function(key) {
        return exclude.indexOf(key) === -1;
    });
}

var seleniumURL = opts.seleniumURL;
var seleniumIP = opts.seleniumIP;
var seleniumPort = opts.seleniumPort;
var katexURL = opts.katexURL;
var katexIP = opts.katexIP;
var katexPort = opts.katexPort;

//////////////////////////////////////////////////////////////////////
// Work out connection to selenium docker container

function check(err) {
    if (!err) {
        return;
    }
    console.error(err);
    console.error(err.stack);
    process.exit(1);
}

function dockerCmd() {
    var args = Array.prototype.slice.call(arguments);
    return childProcess.execFileSync(
        "docker", args, { encoding: "utf-8" }).replace(/\n$/, "");
}

if (!seleniumURL && opts.container) {
    try {
        // When using boot2docker, seleniumIP and katexIP are distinct.
        seleniumIP = childProcess.execFileSync(
            "boot2docker", ["ip"], { encoding: "utf-8" }).replace(/\n$/, "");
        var config = childProcess.execFileSync(
            "boot2docker", ["config"], { encoding: "utf-8" });
        config = (/^HostIP = "(.*)"$/m).exec(config);
        if (!config) {
            console.error("Failed to find HostIP");
            process.exit(2);
        }
        katexIP = config[1];
    } catch (e) {
        seleniumIP = katexIP = dockerCmd(
            "inspect", "-f", "{{.NetworkSettings.Gateway}}", opts.container);
    }
    seleniumPort = dockerCmd("port", opts.container, seleniumPort);
    seleniumPort = seleniumPort.replace(/^.*:/, "");
}
if (!seleniumURL && seleniumIP) {
    seleniumURL = "http://" + seleniumIP + ":" + seleniumPort + "/wd/hub";
}
if (seleniumURL) {
    console.log("Selenium driver at " + seleniumURL);
} else {
    console.log("Selenium driver in local session");
}

process.nextTick(startServer);
var attempts = 0;

//////////////////////////////////////////////////////////////////////
// Start up development server

var devServer = null;
var minPort = 32768;
var maxPort = 61000;

function startServer() {
    if (katexURL || katexPort) {
        process.nextTick(tryConnect);
        return;
    }
    var port = Math.floor(Math.random() * (maxPort - minPort)) + minPort;
    var server = http.createServer(app).listen(port);
    server.once("listening", function() {
        devServer = server;
        katexPort = port;
        attempts = 0;
        process.nextTick(tryConnect);
    });
    server.on("error", function(err) {
        if (devServer !== null) { // error after we started listening
            throw err;
        } else if (++attempts > 50) {
            throw new Error("Failed to start up dev server");
        } else {
            process.nextTick(startServer);
        }
    });
}

//////////////////////////////////////////////////////////////////////
// Wait for container to become ready

function tryConnect() {
    if (!katexURL) {
        katexURL = "http://" + katexIP + ":" + katexPort + "/";
        console.log("KaTeX URL is " + katexURL);
    }
    if (!seleniumIP) {
        process.nextTick(buildDriver);
        return;
    }
    var sock = net.connect({
        host: seleniumIP,
        port: +seleniumPort,
    });
    sock.on("connect", function() {
        sock.end();
        attempts = 0;
        process.nextTick(buildDriver);
    }).on("error", function() {
        if (++attempts > 50) {
            throw new Error("Failed to connect selenium server.");
        }
        setTimeout(tryConnect, 200);
    });
}

//////////////////////////////////////////////////////////////////////
// Build the web driver

var driver;
function buildDriver() {
    var builder = new selenium.Builder().forBrowser(opts.browser);
    var ffProfile = new firefox.Profile();
    ffProfile.setPreference(
        "browser.startup.homepage_override.mstone", "ignore");
    ffProfile.setPreference("browser.startup.page", 0);
    var ffOptions = new firefox.Options().setProfile(ffProfile);
    builder.setFirefoxOptions(ffOptions);
    if (seleniumURL) {
        builder.usingServer(seleniumURL);
    }
    driver = builder.build();
    driver.manage().timeouts().setScriptTimeout(3000).then(function() {
        setSize(targetW, targetH);
    });
}

//////////////////////////////////////////////////////////////////////
// Set the screen size

var targetW = 1024;
var targetH = 768;
function setSize(reqW, reqH) {
    return driver.manage().window().setSize(reqW, reqH).then(function() {
        return driver.takeScreenshot();
    }).then(function(img) {
        img = imageDimensions(img);
        var actualW = img.width;
        var actualH = img.height;
        if (actualW === targetW && actualH === targetH) {
            process.nextTick(takeScreenshots);
            return;
        }
        if (++attempts > 5) {
            throw new Error("Failed to set window size correctly.");
        }
        return setSize(targetW + reqW - actualW, targetH + reqH - actualH);
    }, check);
}

function imageDimensions(img) {
    var buf = new Buffer(img, "base64");
    return {
        buf: buf,
        width: buf.readUInt32BE(16),
        height: buf.readUInt32BE(20),
    };
}

//////////////////////////////////////////////////////////////////////
// Take the screenshots

var countdown = listOfCases.length;

var exitStatus = 0;
var listOfFailed = [];

function takeScreenshots() {
    listOfCases.forEach(takeScreenshot);
}

function takeScreenshot(key) {
    var itm = data[key];
    if (!itm) {
        console.error("Test case " + key + " not known!");
        return;
    }

    var file = path.join(dstDir, key + "-" + opts.browser + ".png");
    var retry = 0;
    var loadExpected = null;
    if (opts.verify) {
        loadExpected = promisify(fs.readFile, file);
    }

    var url = katexURL + "test/screenshotter/test.html?" + itm.query;
    driver.get(url);
    driver.takeScreenshot().then(haveScreenshot).then(function() {
        if (--countdown === 0) {
            if (listOfFailed.length) {
                console.error("Failed: " + listOfFailed.join(" "));
            }
            // devServer.close(cb) will take too long.
            process.exit(exitStatus);
        }
    }, check);

    function haveScreenshot(img) {
        img = imageDimensions(img);
        if (img.width !== targetW || img.height !== targetH) {
            throw new Error("Excpected " + targetW + " x " + targetH +
                            ", got " + img.width + "x" + img.height);
        }
        if (key === "Lap" && opts.browser === "firefox" &&
            img.buf[0x32] === 0xf8) {
            /* There is some strange non-determinism with this case,
             * causing slight vertical shifts.  The first difference
             * is at offset 0x32, where one file has byte 0xf8 and
             * the other has something else.  By using a different
             * output file name for one of these cases, we accept both.
             */
            key += "_alt";
            file = path.join(dstDir, key + "-" + opts.browser + ".png");
            if (loadExpected) {
                loadExpected = promisify(fs.readFile, file);
            }
        }
        var opt = new jspngopt.Optimizer({
            pako: pako,
        });
        var buf = opt.bufferSync(img.buf);
        if (loadExpected) {
            return loadExpected.then(function(expected) {
                if (!buf.equals(expected)) {
                    if (++retry === 5) {
                        console.error("FAIL! " + key);
                        listOfFailed.push(key);
                        exitStatus = 3;
                    } else {
                        console.log("error " + key);
                        driver.get(url);
                        browserSideWait(500 * retry);
                        return driver.takeScreenshot().then(haveScreenshot);
                    }
                } else {
                    console.log("* ok  " + key);
                }
            });
        } else {
            return promisify(fs.writeFile, file, buf).then(function() {
                console.log(key);
            });
        }
    }
}

// Wait using a timeout call in the browser, to ensure that the wait
// time doesn't start before the page has reportedly been loaded.
function browserSideWait(milliseconds) {
    // The last argument (arguments[1] here) is the callback to selenium
    return driver.executeAsyncScript(
        "window.setTimeout(arguments[1], arguments[0]);",
        milliseconds);
}

// Turn node callback style into a call returning a promise,
// like Q.nfcall but using Selenium promises instead of Q ones.
// Second and later arguments are passed to the function named in the
// first argument, and a callback is added as last argument.
function promisify(f) {
    var args = Array.prototype.slice.call(arguments, 1);
    var deferred = new selenium.promise.Deferred();
    args.push(function(err, val) {
        if (err) {
            deferred.reject(err);
        } else {
            deferred.fulfill(val);
        }
    });
    f.apply(null, args);
    return deferred.promise;
}
<commit_msg>Add --wait option to screenshotter (#510)<commit_after>/* eslint no-console:0, prefer-spread:0 */
"use strict";

var childProcess = require("child_process");
var fs = require("fs");
var http = require("http");
var jspngopt = require("jspngopt");
var net = require("net");
var pako = require("pako");
var path = require("path");
var selenium = require("selenium-webdriver");
var firefox = require("selenium-webdriver/firefox");

var app = require("../../server");
var data = require("../../test/screenshotter/ss_data");

var dstDir = path.normalize(
    path.join(__dirname, "..", "..", "test", "screenshotter", "images"));

//////////////////////////////////////////////////////////////////////
// Process command line arguments

var opts = require("nomnom")
    .option("browser", {
        abbr: "b",
        "default": "firefox",
        help: "Name of the browser to use",
    })
    .option("container", {
        abbr: "c",
        type: "string",
        help: "Name or ID of a running docker container to contact",
    })
    .option("seleniumURL", {
        full: "selenium-url",
        help: "Full URL of the Selenium web driver",
    })
    .option("seleniumIP", {
        full: "selenium-ip",
        help: "IP address of the Selenium web driver",
    })
    .option("seleniumPort", {
        full: "selenium-port",
        "default": 4444,
        help: "Port number of the Selenium web driver",
    })
    .option("katexURL", {
        full: "katex-url",
        help: "Full URL of the KaTeX development server",
    })
    .option("katexIP", {
        full: "katex-ip",
        "default": "localhost",
        help: "Full URL of the KaTeX development server",
    })
    .option("katexPort", {
        full: "katex-port",
        help: "Port number of the KaTeX development server",
    })
    .option("include", {
        abbr: "i",
        help: "Comma-separated list of test cases to process",
    })
    .option("exclude", {
        abbr: "x",
        help: "Comma-separated list of test cases to exclude",
    })
    .option("verify", {
        flag: true,
        help: "Check whether screenshot matches current file content",
    })
    .option("wait", {
        help: "Wait this many seconds between page load and screenshot",
    })
    .parse();

var listOfCases;
if (opts.include) {
    listOfCases = opts.include.split(",");
} else {
    listOfCases = Object.keys(data);
}
if (opts.exclude) {
    var exclude = opts.exclude.split(",");
    listOfCases = listOfCases.filter(function(key) {
        return exclude.indexOf(key) === -1;
    });
}

var seleniumURL = opts.seleniumURL;
var seleniumIP = opts.seleniumIP;
var seleniumPort = opts.seleniumPort;
var katexURL = opts.katexURL;
var katexIP = opts.katexIP;
var katexPort = opts.katexPort;

//////////////////////////////////////////////////////////////////////
// Work out connection to selenium docker container

function check(err) {
    if (!err) {
        return;
    }
    console.error(err);
    console.error(err.stack);
    process.exit(1);
}

function dockerCmd() {
    var args = Array.prototype.slice.call(arguments);
    return childProcess.execFileSync(
        "docker", args, { encoding: "utf-8" }).replace(/\n$/, "");
}

if (!seleniumURL && opts.container) {
    try {
        // When using boot2docker, seleniumIP and katexIP are distinct.
        seleniumIP = childProcess.execFileSync(
            "boot2docker", ["ip"], { encoding: "utf-8" }).replace(/\n$/, "");
        var config = childProcess.execFileSync(
            "boot2docker", ["config"], { encoding: "utf-8" });
        config = (/^HostIP = "(.*)"$/m).exec(config);
        if (!config) {
            console.error("Failed to find HostIP");
            process.exit(2);
        }
        katexIP = config[1];
    } catch (e) {
        seleniumIP = katexIP = dockerCmd(
            "inspect", "-f", "{{.NetworkSettings.Gateway}}", opts.container);
    }
    seleniumPort = dockerCmd("port", opts.container, seleniumPort);
    seleniumPort = seleniumPort.replace(/^.*:/, "");
}
if (!seleniumURL && seleniumIP) {
    seleniumURL = "http://" + seleniumIP + ":" + seleniumPort + "/wd/hub";
}
if (seleniumURL) {
    console.log("Selenium driver at " + seleniumURL);
} else {
    console.log("Selenium driver in local session");
}

process.nextTick(startServer);
var attempts = 0;

//////////////////////////////////////////////////////////////////////
// Start up development server

var devServer = null;
var minPort = 32768;
var maxPort = 61000;

function startServer() {
    if (katexURL || katexPort) {
        process.nextTick(tryConnect);
        return;
    }
    var port = Math.floor(Math.random() * (maxPort - minPort)) + minPort;
    var server = http.createServer(app).listen(port);
    server.once("listening", function() {
        devServer = server;
        katexPort = port;
        attempts = 0;
        process.nextTick(tryConnect);
    });
    server.on("error", function(err) {
        if (devServer !== null) { // error after we started listening
            throw err;
        } else if (++attempts > 50) {
            throw new Error("Failed to start up dev server");
        } else {
            process.nextTick(startServer);
        }
    });
}

//////////////////////////////////////////////////////////////////////
// Wait for container to become ready

function tryConnect() {
    if (!katexURL) {
        katexURL = "http://" + katexIP + ":" + katexPort + "/";
        console.log("KaTeX URL is " + katexURL);
    }
    if (!seleniumIP) {
        process.nextTick(buildDriver);
        return;
    }
    var sock = net.connect({
        host: seleniumIP,
        port: +seleniumPort,
    });
    sock.on("connect", function() {
        sock.end();
        attempts = 0;
        process.nextTick(buildDriver);
    }).on("error", function() {
        if (++attempts > 50) {
            throw new Error("Failed to connect selenium server.");
        }
        setTimeout(tryConnect, 200);
    });
}

//////////////////////////////////////////////////////////////////////
// Build the web driver

var driver;
function buildDriver() {
    var builder = new selenium.Builder().forBrowser(opts.browser);
    var ffProfile = new firefox.Profile();
    ffProfile.setPreference(
        "browser.startup.homepage_override.mstone", "ignore");
    ffProfile.setPreference("browser.startup.page", 0);
    var ffOptions = new firefox.Options().setProfile(ffProfile);
    builder.setFirefoxOptions(ffOptions);
    if (seleniumURL) {
        builder.usingServer(seleniumURL);
    }
    driver = builder.build();
    driver.manage().timeouts().setScriptTimeout(3000).then(function() {
        setSize(targetW, targetH);
    });
}

//////////////////////////////////////////////////////////////////////
// Set the screen size

var targetW = 1024;
var targetH = 768;
function setSize(reqW, reqH) {
    return driver.manage().window().setSize(reqW, reqH).then(function() {
        return driver.takeScreenshot();
    }).then(function(img) {
        img = imageDimensions(img);
        var actualW = img.width;
        var actualH = img.height;
        if (actualW === targetW && actualH === targetH) {
            process.nextTick(takeScreenshots);
            return;
        }
        if (++attempts > 5) {
            throw new Error("Failed to set window size correctly.");
        }
        return setSize(targetW + reqW - actualW, targetH + reqH - actualH);
    }, check);
}

function imageDimensions(img) {
    var buf = new Buffer(img, "base64");
    return {
        buf: buf,
        width: buf.readUInt32BE(16),
        height: buf.readUInt32BE(20),
    };
}

//////////////////////////////////////////////////////////////////////
// Take the screenshots

var countdown = listOfCases.length;

var exitStatus = 0;
var listOfFailed = [];

function takeScreenshots() {
    listOfCases.forEach(takeScreenshot);
}

function takeScreenshot(key) {
    var itm = data[key];
    if (!itm) {
        console.error("Test case " + key + " not known!");
        return;
    }

    var file = path.join(dstDir, key + "-" + opts.browser + ".png");
    var retry = 0;
    var loadExpected = null;
    if (opts.verify) {
        loadExpected = promisify(fs.readFile, file);
    }

    var url = katexURL + "test/screenshotter/test.html?" + itm.query;
    driver.get(url);
    if (opts.wait) {
        browserSideWait(1000 * opts.wait);
    }
    driver.takeScreenshot().then(haveScreenshot).then(function() {
        if (--countdown === 0) {
            if (listOfFailed.length) {
                console.error("Failed: " + listOfFailed.join(" "));
            }
            // devServer.close(cb) will take too long.
            process.exit(exitStatus);
        }
    }, check);

    function haveScreenshot(img) {
        img = imageDimensions(img);
        if (img.width !== targetW || img.height !== targetH) {
            throw new Error("Excpected " + targetW + " x " + targetH +
                            ", got " + img.width + "x" + img.height);
        }
        if (key === "Lap" && opts.browser === "firefox" &&
            img.buf[0x32] === 0xf8) {
            /* There is some strange non-determinism with this case,
             * causing slight vertical shifts.  The first difference
             * is at offset 0x32, where one file has byte 0xf8 and
             * the other has something else.  By using a different
             * output file name for one of these cases, we accept both.
             */
            key += "_alt";
            file = path.join(dstDir, key + "-" + opts.browser + ".png");
            if (loadExpected) {
                loadExpected = promisify(fs.readFile, file);
            }
        }
        var opt = new jspngopt.Optimizer({
            pako: pako,
        });
        var buf = opt.bufferSync(img.buf);
        if (loadExpected) {
            return loadExpected.then(function(expected) {
                if (!buf.equals(expected)) {
                    if (++retry === 5) {
                        console.error("FAIL! " + key);
                        listOfFailed.push(key);
                        exitStatus = 3;
                    } else {
                        console.log("error " + key);
                        driver.get(url);
                        browserSideWait(500 * retry);
                        return driver.takeScreenshot().then(haveScreenshot);
                    }
                } else {
                    console.log("* ok  " + key);
                }
            });
        } else {
            return promisify(fs.writeFile, file, buf).then(function() {
                console.log(key);
            });
        }
    }
}

// Wait using a timeout call in the browser, to ensure that the wait
// time doesn't start before the page has reportedly been loaded.
function browserSideWait(milliseconds) {
    // The last argument (arguments[1] here) is the callback to selenium
    return driver.executeAsyncScript(
        "window.setTimeout(arguments[1], arguments[0]);",
        milliseconds);
}

// Turn node callback style into a call returning a promise,
// like Q.nfcall but using Selenium promises instead of Q ones.
// Second and later arguments are passed to the function named in the
// first argument, and a callback is added as last argument.
function promisify(f) {
    var args = Array.prototype.slice.call(arguments, 1);
    var deferred = new selenium.promise.Deferred();
    args.push(function(err, val) {
        if (err) {
            deferred.reject(err);
        } else {
            deferred.fulfill(val);
        }
    });
    f.apply(null, args);
    return deferred.promise;
}
<|endoftext|>