<commit_before>import path from 'path';
import fs from 'fs';
import url from 'url';
import handlebars from 'handlebars';
import Bottleneck from 'bottleneck';
import slug from 'slug';
slug.defaults.mode = 'rfc3986';
import FixtureParser from './fixtureParser.js';
import BrowserParser from './browserParser.js';
import FirefoxVersionParser from './firefoxVersionParser.js';
import CanIUseParser from './canIUseParser.js';
import cache from './cache.js';

const fixtureDir = path.resolve('./features');
const fixtureParser = new FixtureParser(fixtureDir);
const browserParser = new BrowserParser();
const firefoxVersionParser = new FirefoxVersionParser();
const canIUseParser = new CanIUseParser();
let validationWarnings;

function validateWarning(msg) {
  validationWarnings.push(msg);
}

function normalizeStatus(status, browser) {
  switch (status.trim().toLowerCase()) {
    case '':
    case 'unknown':
      return 'unknown';
    case 'no active development':
    case 'not currently planned':
    case 'not considering':
    case 'not-planned':
      return 'not-planned';
    case 'deprecated':
    case 'no longer pursuing':
    case 'removed':
      return 'deprecated';
    case 'under consideration':
    case 'proposed':
    case 'under-consideration':
      return 'under-consideration';
    case 'in development':
    case 'behind a flag':
    case 'prototyping':
    case 'preview release':
    case 'in-development':
      return 'in-development';
    case 'shipped':
    case 'enabled by default':
    case 'done':
    case 'partial support':
    case 'prefixed':
      return 'shipped';
    default:
      throw new Error('Unmapped status: "' + status + '" for "' + browser + '"');
  }
}

class BrowserFeature {
  constructor(data) {
    this.data = data;
  }

  get status() {
    return normalizeStatus(this._rawStatus, this.name);
  }
}

class ChromeBrowserFeature extends BrowserFeature {
  constructor(data) {
    super(data);
    this.name = 'chrome';
  }

  get _rawStatus() {
    return this.data.impl_status_chrome;
  }
  get url() {
    return url.format({
      host: 'www.chromestatus.com',
      pathname: '/feature/' + this.data.id,
      protocol: 'https:',
    });
  }
}

ChromeBrowserFeature.defaultUrl = 'https://www.chromestatus.com/features';

class OperaBrowserFeature extends ChromeBrowserFeature {
  constructor(data) {
    super(data);
    this.name = 'opera';
  }

  get _rawStatus() {
    return this.data.shipped_opera_milestone ? 'shipped' : '';
  }
}

class WebKitBrowserFeature extends BrowserFeature {
  constructor(data) {
    super(data);
    this.name = 'webkit';
  }
  get _rawStatus() {
    return this.data.status ? this.data.status.status : '';
  }
  get url() {
    return url.format({
      host: 'www.webkit.org',
      pathname: '/status.html',
      hash: '#' + this.data.type + '-' + slug(this.data.name),
      protocol: 'https:',
    });
  }
}

WebKitBrowserFeature.defaultUrl = 'https://www.webkit.org/status.html';

class IEBrowserFeature extends BrowserFeature {
  constructor(data) {
    super(data);
    this.name = 'ie';
  }
  get _rawStatus() {
    return this.data.ieStatus.text;
  }
  get url() {
    return url.format({
      host: 'dev.modern.ie',
      pathname: '/platform/status/' + this.data.name.replace(/[^a-zA-Z0-9]/g, '').toLowerCase(),
      protocol: 'https:',
    });
  }
}

IEBrowserFeature.defaultUrl = 'https://dev.modern.ie/platform/status/';

const allBrowserFeatures = [
  ['chrome', 'chrome', ChromeBrowserFeature],
  ['opera', 'chrome', OperaBrowserFeature],
  ['webkit', 'webkit', WebKitBrowserFeature],
  ['ie', 'ie', IEBrowserFeature],
];

function populateBrowserFeatureData(browserData, features) {
  features.forEach((feature) => {
    allBrowserFeatures.map(([key, relKey, BrowserFeatureConstructor]) => {
      if (!feature[key + '_status']) {
        feature[key + '_status'] = 'unknown';
      } else {
        feature[key + '_status'] = normalizeStatus(feature[key + '_status'], key);
      }
      feature[key + '_url'] = BrowserFeatureConstructor.defaultUrl;

      if (!feature[relKey + '_ref']) {
        return;
      }

      const browserFeatureData = browserData[relKey].get(feature[relKey + '_ref']);
      if (!browserFeatureData) {
        throw new Error('Wrong value for ' + relKey + '_ref in ' + feature.file);
      }

      const browserFeature = new BrowserFeatureConstructor(browserFeatureData);
      feature[key + '_status'] = browserFeature.status;
      feature[key + '_url'] = browserFeature.url;
    });
  });
}

function populateSpecStatus(browserData, features) {
  features.forEach((feature) => {
    const browserFeatureData = browserData.chrome.get(feature.chrome_ref);
    if (!browserFeatureData || !browserFeatureData.standardization) {
      return;
    }
    let normalized;
    const status = browserFeatureData.standardization.text;
    switch (status) {
      case 'De-facto standard':
        normalized = 'de-facto-standard';
        break;
      case 'Editor\'s draft':
        normalized = 'editors-draft';
        break;
      case 'Established standard':
        normalized = 'established-standard';
        break;
      case 'No public standards discussion':
        normalized = 'no-public-discussion';
        break;
      case 'Public discussion':
        normalized = 'public-discussion';
        break;
      case 'Working draft or equivalent':
        normalized = 'working-draft-or-equivalent';
        break;
      default:
        validateWarning('Unmapped standardization status: ' + status);
        normalized = 'invalid';
        break;
    }
    feature.spec_status = normalized;
  });
}

// Bugzilla has a limit on concurrent connections. I haven't found what the
// limit is, but 20 seems to work.
const bugzillaBottleneck = new Bottleneck(20);

function bugzillaFetch(bugzillaUrl, options) {
  return bugzillaBottleneck.schedule(cache.readJson, bugzillaUrl, options.cacheDir);
}

function getBugzillaBugData(bugId, options) {
  return bugzillaFetch('https://bugzilla.mozilla.org/rest/bug?id=' + bugId + '&include_fields=' + options.include_fields.join(','), options)
  .then((json) => {
    if (!json.bugs.length) {
      throw new Error('Bug not found(secure bug?)');
    }
    return json.bugs[0];
  })
  .catch((reason) => {
    validateWarning('Failed to get bug data for: ' + bugId + ': ' + reason);
    return null;
  });
}

function populateBugzillaData(features, options) {
  return Promise.all(features.map((feature) => {
    if (!feature.bugzilla) {
      return null;
    }
    return getBugzillaBugData(feature.bugzilla, Object.assign(options, { include_fields: ['status', 'depends_on', 'id'] }))
    .then((bugData) => {
      if (!bugData) {
        feature.bugzilla_status = null;
        return null;
      }
      feature.bugzilla_status = bugData.status;
      feature.bugzilla_resolved_count = (bugData.status === 'RESOLVED') ? 1 : 0;
      // Add one to show status of the tracking bug itself.
      feature.bugzilla_dependant_count = bugData.depends_on.length + 1;
      if (!bugData.depends_on.length) {
        return null;
      }
      // Check all the dependent bugs to count how many are resolved.
      return bugzillaFetch('https://bugzilla.mozilla.org/rest/bug?id=' + bugData.depends_on.join(',') + '&status=RESOLVED&include_fields=id', options)
      .then((dependentResult) => {
        feature.bugzilla_resolved_count += dependentResult.bugs.length;
      });
    });
  }));
}

function populateFirefoxStatus(versions, features) {
  features.forEach((feature) => {
    if (!isNaN(feature.firefox_status)) {
      const version = parseInt(feature.firefox_status, 10);
      feature.firefox_version = version;
      if (version <= versions.aurora) {
        feature.firefox_status = 'shipped';
      } else {
        feature.firefox_status = 'in-development';
      }

      if (version <= versions.stable) {
        feature.firefox_channel = 'release';
      } else if (version === versions.beta) {
        feature.firefox_channel = 'beta';
      } else if (version === versions.aurora) {
        feature.firefox_channel = 'developer-edition';
      } else if (version === versions.nightly) {
        feature.firefox_channel = 'nightly';
      }
    } else {
      feature.firefox_status = normalizeStatus(feature.firefox_status, 'firefox');
    }
  });
}

function populateCanIUsePercent(canIUseData, features) {
  features.forEach((feature) => {
    if (!feature.caniuse_ref) {
      validateWarning(feature.file + ': missing caniuse_ref');
      return;
    }
    const data = canIUseData.data[feature.caniuse_ref];
    if (!data) {
      validateWarning(feature.file + ': invalid caniuse_ref ' + feature.caniuse_ref);
      return;
    }
    feature.caniuse_usage_perc_y = data.usage_perc_y;
    feature.caniuse_usage_perc_a = data.usage_perc_a;
    feature.caniuse_usage_perc_total = Math.round(data.usage_perc_y + data.usage_perc_a);
  });
}

function validateFeatureInput(features) {
  // We could potentially use a real JSON schema, but we'd still have to do
  // uniqueness checks ourselves.
  const schema = {
    file: {
      required: true,
      unique: true,
    },
    title: {
      required: true,
      unique: true,
    },
    summary: {
      required: true,
      unique: true,
    },
    slug: {
      required: true,
      unique: true,
    },
    category: {
    },
    bugzilla: {
      required: true,
      unique: true,
    },
    firefox_status: {
      required: true,
    },
    mdn_url: {
      required: true,
      unique: true,
    },
    spec_url: {
      required: true,
      unique: true,
    },
    spec_repo: {
    },
    spec_status: {
    },
    chrome_ref: {
      required: true,
      unique: true,
    },
    chrome_status: {
    },
    chrome_footnote: {
    },
    opera_status: {
    },
    opera_footnote: {
    },
    webkit_ref: {
      required: true,
      unique: true,
    },
    webkit_status: {
    },
    webkit_footnote: {
    },
    ie_ref: {
      required: true,
      unique: true,
    },
    ie_status: {
    },
    ie_footnote: {
    },
    caniuse_ref: {
      unique: true,
    },
  };
  const uniques = {};
  features.forEach((feature) => {
    const properties = Object.keys(feature);
    for (const propertyName of properties) {
      if (!(propertyName in schema)) {
        validateWarning(feature.file + ': unknown property "' + propertyName + '"');
      }
    }

    for (const key of Object.keys(schema)) {
      const value = feature[key];

      if (schema[key].required && !value) {
        validateWarning(feature.file + ': missing required property "' + key + '"');
      }

      if (schema[key].unique && typeof value !== 'undefined') {
        if (!(key in uniques)) {
          uniques[key] = {};
        }
        const duplicate = uniques[key][value];
        if (duplicate) {
          validateWarning(feature.file + ': duplicate value "' + value + '" for key "' + key + '", previously defined in ' + duplicate);
        } else {
          uniques[key][value] = feature.file;
        }
      }
    }
  });
}

let alt = null;
handlebars.registerHelper('alt', (state, field, variance) => {
  if (!alt) {
    alt = JSON.parse(fs.readFileSync('./src/tpl/alt.json'));
  }
  const value = alt[field][state] || null;
  if (!value || typeof variance !== 'string') {
    return value;
  }
  return value[variance];
});

handlebars.registerHelper('if_eq', function comparison(left, right, opts) { // No fat-arrow since we want don't want lexical 'this'
  if (left === right) {
    return opts.fn(this);
  }
  return opts.inverse(this);
});

function buildIndex(status) {
  const templateContents = fs.readFileSync('src/tpl/index.html', {
    encoding: 'utf-8',
  });
  const featureLinksContents = fs.readFileSync('src/tpl/featureLinksPartial.html', {
    encoding: 'utf-8',
  });
  const featureStatusContents = fs.readFileSync('src/tpl/featureStatusPartial.html', {
    encoding: 'utf-8',
  });
  handlebars.registerHelper('featureStatusName', function featureStatusName() {
    return this.slug + '-status';
  });
  handlebars.registerHelper('featureLinksName', function featureLinksName() {
    return this.slug + '-links';
  });
  status.features.forEach((featureData) => {
    handlebars.registerPartial(
        featureData.slug + '-status',
        handlebars.compile(featureStatusContents)(featureData));
    handlebars.registerPartial(
        featureData.slug + '-links',
        handlebars.compile(featureLinksContents)(featureData));
  });
  return Promise.resolve(handlebars.compile(templateContents)(status));
}

function buildFeatures(status) {
  const templateContents = fs.readFileSync('src/tpl/feature.html', {
    encoding: 'utf-8',
  });
  const featureLinksContents = fs.readFileSync('src/tpl/featureLinksPartial.html', {
    encoding: 'utf-8',
  });
  const featureStatusContents = fs.readFileSync('src/tpl/featureStatusPartial.html', {
    encoding: 'utf-8',
  });
  const promises = status.features.map(function(feature) {
    handlebars.registerPartial('featureStatus',
        handlebars.compile(featureStatusContents)(feature));
    handlebars.registerPartial('featureLinks',
        handlebars.compile(featureLinksContents)(feature));
    return {
      slug: feature.slug,
      contents: handlebars.compile(templateContents)(feature),
    };
  });
  return Promise.all(promises);
}

function buildStatus(options) {
  validationWarnings = [];
  return Promise.all([
    fixtureParser.read(),
    browserParser.read(options),
    firefoxVersionParser.read(options),
    canIUseParser.read(options),
  ]).then(() => {
    validateFeatureInput(fixtureParser.results);
    return populateBugzillaData(fixtureParser.results, options);
  }).then(() => {
    populateFirefoxStatus(firefoxVersionParser.results, fixtureParser.results);
    populateBrowserFeatureData(browserParser.results, fixtureParser.results);
    populateSpecStatus(browserParser.results, fixtureParser.results);
    populateCanIUsePercent(canIUseParser.results, fixtureParser.results);
    const data = {
      created: (new Date()).toISOString(),
      features: fixtureParser.results,
      firefoxVersions: firefoxVersionParser.results,
    };
    if (validationWarnings.length) {
      console.warn('Validation warnings: ');
      validationWarnings.forEach((warning) => {
        console.warn('\t' + warning);
      });
    }
    return data;
  });
}

export default {
  buildStatus,
  buildIndex,
  buildFeatures,
};
<commit_msg>moved the repeating constants out of the function<commit_after>import path from 'path';
import fs from 'fs';
import url from 'url';
import handlebars from 'handlebars';
import Bottleneck from 'bottleneck';
import slug from 'slug';
slug.defaults.mode = 'rfc3986';
import FixtureParser from './fixtureParser.js';
import BrowserParser from './browserParser.js';
import FirefoxVersionParser from './firefoxVersionParser.js';
import CanIUseParser from './canIUseParser.js';
import cache from './cache.js';

const fixtureDir = path.resolve('./features');
const fixtureParser = new FixtureParser(fixtureDir);
const browserParser = new BrowserParser();
const firefoxVersionParser = new FirefoxVersionParser();
const canIUseParser = new CanIUseParser();
let validationWarnings;

function validateWarning(msg) {
  validationWarnings.push(msg);
}

function normalizeStatus(status, browser) {
  switch (status.trim().toLowerCase()) {
    case '':
    case 'unknown':
      return 'unknown';
    case 'no active development':
    case 'not currently planned':
    case 'not considering':
    case 'not-planned':
      return 'not-planned';
    case 'deprecated':
    case 'no longer pursuing':
    case 'removed':
      return 'deprecated';
    case 'under consideration':
    case 'proposed':
    case 'under-consideration':
      return 'under-consideration';
    case 'in development':
    case 'behind a flag':
    case 'prototyping':
    case 'preview release':
    case 'in-development':
      return 'in-development';
    case 'shipped':
    case 'enabled by default':
    case 'done':
    case 'partial support':
    case 'prefixed':
      return 'shipped';
    default:
      throw new Error('Unmapped status: "' + status + '" for "' + browser + '"');
  }
}

class BrowserFeature {
  constructor(data) {
    this.data = data;
  }

  get status() {
    return normalizeStatus(this._rawStatus, this.name);
  }
}

class ChromeBrowserFeature extends BrowserFeature {
  constructor(data) {
    super(data);
    this.name = 'chrome';
  }

  get _rawStatus() {
    return this.data.impl_status_chrome;
  }
  get url() {
    return url.format({
      host: 'www.chromestatus.com',
      pathname: '/feature/' + this.data.id,
      protocol: 'https:',
    });
  }
}

ChromeBrowserFeature.defaultUrl = 'https://www.chromestatus.com/features';

class OperaBrowserFeature extends ChromeBrowserFeature {
  constructor(data) {
    super(data);
    this.name = 'opera';
  }

  get _rawStatus() {
    return this.data.shipped_opera_milestone ? 'shipped' : '';
  }
}

class WebKitBrowserFeature extends BrowserFeature {
  constructor(data) {
    super(data);
    this.name = 'webkit';
  }
  get _rawStatus() {
    return this.data.status ? this.data.status.status : '';
  }
  get url() {
    return url.format({
      host: 'www.webkit.org',
      pathname: '/status.html',
      hash: '#' + this.data.type + '-' + slug(this.data.name),
      protocol: 'https:',
    });
  }
}

WebKitBrowserFeature.defaultUrl = 'https://www.webkit.org/status.html';

class IEBrowserFeature extends BrowserFeature {
  constructor(data) {
    super(data);
    this.name = 'ie';
  }
  get _rawStatus() {
    return this.data.ieStatus.text;
  }
  get url() {
    return url.format({
      host: 'dev.modern.ie',
      pathname: '/platform/status/' + this.data.name.replace(/[^a-zA-Z0-9]/g, '').toLowerCase(),
      protocol: 'https:',
    });
  }
}

IEBrowserFeature.defaultUrl = 'https://dev.modern.ie/platform/status/';

const allBrowserFeatures = [
  ['chrome', 'chrome', ChromeBrowserFeature],
  ['opera', 'chrome', OperaBrowserFeature],
  ['webkit', 'webkit', WebKitBrowserFeature],
  ['ie', 'ie', IEBrowserFeature],
];

function populateBrowserFeatureData(browserData, features) {
  features.forEach((feature) => {
    allBrowserFeatures.map(([key, relKey, BrowserFeatureConstructor]) => {
      if (!feature[key + '_status']) {
        feature[key + '_status'] = 'unknown';
      } else {
        feature[key + '_status'] = normalizeStatus(feature[key + '_status'], key);
      }
      feature[key + '_url'] = BrowserFeatureConstructor.defaultUrl;

      if (!feature[relKey + '_ref']) {
        return;
      }

      const browserFeatureData = browserData[relKey].get(feature[relKey + '_ref']);
      if (!browserFeatureData) {
        throw new Error('Wrong value for ' + relKey + '_ref in ' + feature.file);
      }

      const browserFeature = new BrowserFeatureConstructor(browserFeatureData);
      feature[key + '_status'] = browserFeature.status;
      feature[key + '_url'] = browserFeature.url;
    });
  });
}

function populateSpecStatus(browserData, features) {
  features.forEach((feature) => {
    const browserFeatureData = browserData.chrome.get(feature.chrome_ref);
    if (!browserFeatureData || !browserFeatureData.standardization) {
      return;
    }
    let normalized;
    const status = browserFeatureData.standardization.text;
    switch (status) {
      case 'De-facto standard':
        normalized = 'de-facto-standard';
        break;
      case 'Editor\'s draft':
        normalized = 'editors-draft';
        break;
      case 'Established standard':
        normalized = 'established-standard';
        break;
      case 'No public standards discussion':
        normalized = 'no-public-discussion';
        break;
      case 'Public discussion':
        normalized = 'public-discussion';
        break;
      case 'Working draft or equivalent':
        normalized = 'working-draft-or-equivalent';
        break;
      default:
        validateWarning('Unmapped standardization status: ' + status);
        normalized = 'invalid';
        break;
    }
    feature.spec_status = normalized;
  });
}

// Bugzilla has a limit on concurrent connections. I haven't found what the
// limit is, but 20 seems to work.
const bugzillaBottleneck = new Bottleneck(20);

function bugzillaFetch(bugzillaUrl, options) {
  return bugzillaBottleneck.schedule(cache.readJson, bugzillaUrl, options.cacheDir);
}

function getBugzillaBugData(bugId, options) {
  return bugzillaFetch('https://bugzilla.mozilla.org/rest/bug?id=' + bugId + '&include_fields=' + options.include_fields.join(','), options)
  .then((json) => {
    if (!json.bugs.length) {
      throw new Error('Bug not found(secure bug?)');
    }
    return json.bugs[0];
  })
  .catch((reason) => {
    validateWarning('Failed to get bug data for: ' + bugId + ': ' + reason);
    return null;
  });
}

function populateBugzillaData(features, options) {
  return Promise.all(features.map((feature) => {
    if (!feature.bugzilla) {
      return null;
    }
    return getBugzillaBugData(feature.bugzilla, Object.assign(options, { include_fields: ['status', 'depends_on', 'id'] }))
    .then((bugData) => {
      if (!bugData) {
        feature.bugzilla_status = null;
        return null;
      }
      feature.bugzilla_status = bugData.status;
      feature.bugzilla_resolved_count = (bugData.status === 'RESOLVED') ? 1 : 0;
      // Add one to show status of the tracking bug itself.
      feature.bugzilla_dependant_count = bugData.depends_on.length + 1;
      if (!bugData.depends_on.length) {
        return null;
      }
      // Check all the dependent bugs to count how many are resolved.
      return bugzillaFetch('https://bugzilla.mozilla.org/rest/bug?id=' + bugData.depends_on.join(',') + '&status=RESOLVED&include_fields=id', options)
      .then((dependentResult) => {
        feature.bugzilla_resolved_count += dependentResult.bugs.length;
      });
    });
  }));
}

function populateFirefoxStatus(versions, features) {
  features.forEach((feature) => {
    if (!isNaN(feature.firefox_status)) {
      const version = parseInt(feature.firefox_status, 10);
      feature.firefox_version = version;
      if (version <= versions.aurora) {
        feature.firefox_status = 'shipped';
      } else {
        feature.firefox_status = 'in-development';
      }

      if (version <= versions.stable) {
        feature.firefox_channel = 'release';
      } else if (version === versions.beta) {
        feature.firefox_channel = 'beta';
      } else if (version === versions.aurora) {
        feature.firefox_channel = 'developer-edition';
      } else if (version === versions.nightly) {
        feature.firefox_channel = 'nightly';
      }
    } else {
      feature.firefox_status = normalizeStatus(feature.firefox_status, 'firefox');
    }
  });
}

function populateCanIUsePercent(canIUseData, features) {
  features.forEach((feature) => {
    if (!feature.caniuse_ref) {
      validateWarning(feature.file + ': missing caniuse_ref');
      return;
    }
    const data = canIUseData.data[feature.caniuse_ref];
    if (!data) {
      validateWarning(feature.file + ': invalid caniuse_ref ' + feature.caniuse_ref);
      return;
    }
    feature.caniuse_usage_perc_y = data.usage_perc_y;
    feature.caniuse_usage_perc_a = data.usage_perc_a;
    feature.caniuse_usage_perc_total = Math.round(data.usage_perc_y + data.usage_perc_a);
  });
}

function validateFeatureInput(features) {
  // We could potentially use a real JSON schema, but we'd still have to do
  // uniqueness checks ourselves.
  const schema = {
    file: {
      required: true,
      unique: true,
    },
    title: {
      required: true,
      unique: true,
    },
    summary: {
      required: true,
      unique: true,
    },
    slug: {
      required: true,
      unique: true,
    },
    category: {
    },
    bugzilla: {
      required: true,
      unique: true,
    },
    firefox_status: {
      required: true,
    },
    mdn_url: {
      required: true,
      unique: true,
    },
    spec_url: {
      required: true,
      unique: true,
    },
    spec_repo: {
    },
    spec_status: {
    },
    chrome_ref: {
      required: true,
      unique: true,
    },
    chrome_status: {
    },
    chrome_footnote: {
    },
    opera_status: {
    },
    opera_footnote: {
    },
    webkit_ref: {
      required: true,
      unique: true,
    },
    webkit_status: {
    },
    webkit_footnote: {
    },
    ie_ref: {
      required: true,
      unique: true,
    },
    ie_status: {
    },
    ie_footnote: {
    },
    caniuse_ref: {
      unique: true,
    },
  };
  const uniques = {};
  features.forEach((feature) => {
    const properties = Object.keys(feature);
    for (const propertyName of properties) {
      if (!(propertyName in schema)) {
        validateWarning(feature.file + ': unknown property "' + propertyName + '"');
      }
    }

    for (const key of Object.keys(schema)) {
      const value = feature[key];

      if (schema[key].required && !value) {
        validateWarning(feature.file + ': missing required property "' + key + '"');
      }

      if (schema[key].unique && typeof value !== 'undefined') {
        if (!(key in uniques)) {
          uniques[key] = {};
        }
        const duplicate = uniques[key][value];
        if (duplicate) {
          validateWarning(feature.file + ': duplicate value "' + value + '" for key "' + key + '", previously defined in ' + duplicate);
        } else {
          uniques[key][value] = feature.file;
        }
      }
    }
  });
}

let alt = null;
handlebars.registerHelper('alt', (state, field, variance) => {
  if (!alt) {
    alt = JSON.parse(fs.readFileSync('./src/tpl/alt.json'));
  }
  const value = alt[field][state] || null;
  if (!value || typeof variance !== 'string') {
    return value;
  }
  return value[variance];
});

handlebars.registerHelper('if_eq', function comparison(left, right, opts) { // No fat-arrow since we want don't want lexical 'this'
  if (left === right) {
    return opts.fn(this);
  }
  return opts.inverse(this);
});

// status partial
const featureStatusContents = fs.readFileSync('src/tpl/featureStatusPartial.html', {
  encoding: 'utf-8',
});
// links partial is needed as the feature page is build in slightly different
// way (no summary), we might also switch off links when embedded
const featureLinksContents = fs.readFileSync('src/tpl/featureLinksPartial.html', {
  encoding: 'utf-8',
});

function buildIndex(status) {
  const templateContents = fs.readFileSync('src/tpl/index.html', {
    encoding: 'utf-8',
  });
  handlebars.registerHelper('featureStatusName', function featureStatusName() {
    return this.slug + '-status';
  });
  handlebars.registerHelper('featureLinksName', function featureLinksName() {
    return this.slug + '-links';
  });
  status.features.forEach((featureData) => {
    // register partials for each feature
    handlebars.registerPartial(
        featureData.slug + '-status',
        handlebars.compile(featureStatusContents)(featureData));
    handlebars.registerPartial(
        featureData.slug + '-links',
        handlebars.compile(featureLinksContents)(featureData));
  });
  return Promise.resolve(handlebars.compile(templateContents)(status));
}

function buildFeatures(status) {
  const templateContents = fs.readFileSync('src/tpl/feature.html', {
    encoding: 'utf-8',
  });
  const promises = status.features.map(function(feature) {
    handlebars.registerPartial('featureStatus',
        handlebars.compile(featureStatusContents)(feature));
    handlebars.registerPartial('featureLinks',
        handlebars.compile(featureLinksContents)(feature));
    return {
      slug: feature.slug,
      contents: handlebars.compile(templateContents)(feature),
    };
  });
  return Promise.all(promises);
}

function buildStatus(options) {
  validationWarnings = [];
  return Promise.all([
    fixtureParser.read(),
    browserParser.read(options),
    firefoxVersionParser.read(options),
    canIUseParser.read(options),
  ]).then(() => {
    validateFeatureInput(fixtureParser.results);
    return populateBugzillaData(fixtureParser.results, options);
  }).then(() => {
    populateFirefoxStatus(firefoxVersionParser.results, fixtureParser.results);
    populateBrowserFeatureData(browserParser.results, fixtureParser.results);
    populateSpecStatus(browserParser.results, fixtureParser.results);
    populateCanIUsePercent(canIUseParser.results, fixtureParser.results);
    const data = {
      created: (new Date()).toISOString(),
      features: fixtureParser.results,
      firefoxVersions: firefoxVersionParser.results,
    };
    if (validationWarnings.length) {
      console.warn('Validation warnings: ');
      validationWarnings.forEach((warning) => {
        console.warn('\t' + warning);
      });
    }
    return data;
  });
}

export default {
  buildStatus,
  buildIndex,
  buildFeatures,
};
<|endoftext|>


<commit_before>'use strict';

var path = require('path');
var del = require('delete');
var Time = require('time-diff');
var watch = require('base-watch');
var prettify = require('gulp-prettify');
var drafts = require('gulp-drafts');
var extname = require('gulp-extname');
var ignore = require('gulp-ignore');
var debug = require('debug')('assemble:docs');
var browserSync = require('browser-sync').create();
var permalinks = require('assemble-permalinks');
var merge = require('mixin-deep');
var through = require('through2');

var collections = require('./support/plugins/assemble-collections');
var viewEvents = require('./support/plugins/view-events');
var redirects = require('./support/plugins/redirects');
var manifest = require('./support/plugins/manifest');
var versions = require('./support/plugins/versions');
var getDest = require('./support/plugins/get-dest');
var utils = require('./support/utils');
var pkg = require('../package');
var assemble = require('..');

/**
 * Create our assemble appplication
 */

var app = assemble();

app.time = new Time();
app.use(viewEvents('permalink'));
app.use(collections());
app.use(permalinks());
app.use(getDest());
app.use(watch());

app.on('error', function(err) {
  console.log(err.stack);
});

/**
 * Common variables
 */

var build = {
  dest: path.resolve.bind(path, __dirname, '../_gh_pages')
};

/**
 * Middleware
 */

var permalinkCache = {};
app.onLoad(/./, function(file, next) {
  debug('onLoad for %s', file.path);
  if (permalinkCache[file.path] === true) {
    debug('creating permalink context for %s', file.path);
    file.permalink(file.options.permalink);
  }
  next();
});

app.onPermalink(/./, function(file, next) {
  debug('onPermalink for %s', file.path);
  permalinkCache[file.path] = true;
  file.data = merge({category: 'docs'}, app.cache.data, file.data);
  next();
});

/**
 * Customize how templates are stored. This changes the
 * key of each template, so it's easier to lookup later
 */

app.option('renameKey', function(fp) {
  if (fp.indexOf(__dirname) === -1) {
    return fp;
  }
  return path.basename(fp, path.extname(fp));
});

/**
 * Load some "global" data
 */

app.data({
  docs: {
    content: 'https://github.com/assemble/assemble/tree/master/docs/content'
  },
  site: {
    title: 'Assemble Docs',
    version: pkg.version,
    base: ':destBase/en/v:site.version',
    categories: {
      docs: {
        title: 'Docs',
        description: 'Assemble documentation',
        category: 'docs',
        filter: navFilter('docs')
      },
      api: {
        title: 'API',
        description: 'Assemble API Documentation',
        category: 'api',
        filter: navFilter('api')
      },
      recipes: {
        title: 'Recipes',
        description: 'Assemble recipes',
        category: 'recipes',
        filter: navFilter('recipes')
      },
      subjects: {
        title: 'Subjects',
        description: 'Advanced subjects on using assemble',
        category: 'subjects',
        filter: navFilter('subjects')
      }
    }
  },
  destBase: '_gh_pages',
  assets: ':site.base/assets',
  links: [{
    dest: 'assemble',
    collection: 'docs',
    text: 'Assemble'
  }]
});

/**
 * Create a custom view collection
 */

var permalinkOpts = {
  slug: function() {
    if (this.category === 'docs') {
      return this.filename + '.html';
    }
    var rel = this.relative;
    var ext = path.extname(rel);
    return path.join(rel.substr(0, rel.length - ext.length), 'index.html');
  }
};

function navFilter(category) {
  return function(item) {
    if (item.data.category !== category) {
      return false;
    }
    return (typeof item.data.draft === 'undefined' || item.data.draft === false);
  };
}

app.create('docs', {layout: 'body'})
  .preRender(/\.md/, function(file, next) {
    var category = file.data.category;
    if (!category) return next();
    file.data = merge({
      section: app.data('site.categories.' + category),
      layout: category === 'recipes' ? 'recipe' : file.data.layout
    }, file.data);
    next();
  })
  .use(permalinks(':site.base/:slug()', permalinkOpts));

app.create('redirects', {
  renameKey: function(key, view) {
    return view ? view.path : key;
  }
});

/**
 * Load helpers
 */

app.helpers('support/helpers/*.js');

/**
 * Clean out the current version's built files
 */

app.task('clean', function(cb) {
  del(build.dest('en/v' + pkg.version), {force: true}, cb);
});

/**
 * Build manifest for a specific version.
 * This is in case an old version didn't have a manifest.
 * Run with `assemble manifest --version 0.6.0`
 */

app.task('manifest', function(cb) {
  // get the version from argv.orig (because of the dots in versions)
  var version = app.argv.orig.version;
  if (typeof version === 'undefined' || typeof version === 'boolean') {
    cb(new Error('--version needs to be specified when running `assemble manifest`'));
    return;
  }

  var dir = build.dest('en/v' + version);
  if (!utils.exists(dir)) {
    cb(new Error('Unable to find "' + dir + '" for version "' + version + '"'));
    return;
  }

  return app.src([dir + '/**/*.html'])
    .on('err', console.log)
    .pipe(manifest(app, {
      renamePath: function(fp) {
        return 'en/v' + version + '/' + fp;
      }
    }))
    .pipe(ignore.include('manifest.json'))
    .pipe(app.dest(function(file) {
      file.base = file.dirname;
      return dir;
    }));
});

/**
 * Build up a redirects.json file
 */

app.task('redirects', function() {
  return app.src(build.dest('en/*/manifest.json'))
    .pipe(redirects(app))
    .pipe(versions(app))
    .pipe(ignore.include(['redirects.json', 'versions.json']))
    .pipe(app.dest(function(file) {
      file.base = file.dirname;
      return 'data';
    }))
    .pipe(app.dest(build.dest()));
});

/**
 * Re-load templates when triggered by watch
 */

app.task('load', function* () {
  app.partials('templates/partials/**/*.hbs', {cwd: __dirname});
  app.layouts('templates/layouts/**/*.hbs', {cwd: __dirname});
  app.docs(['content/**/*.md'], {cwd: __dirname});
});

/**
 * Static server
 */

app.task('serve', function() {
  browserSync.init({
    port: 8080,
    startPath: 'docs.html',
    server: {
      baseDir: build.dest()
    }
  });
});

/**
 * Create redirect views from paths > "data/redirects.json"
 */

app.task('generate-redirects', function() {
  app.data('data/redirects.json', { namespace: 'redirects' });
  var redirects = app.data('redirects');

  for(var key in redirects) {
    var to = redirects[key];
    app.redirect(key, {
      path: key,
      data: {
        title: key,
        layout: 'redirect',
        redirect: { seconds: 0, url: to }
      },
      content: ''
    });
  }

  return app.toStream('redirects')
    .on('error', console.error)
    .pipe(app.renderFile('hbs'))
    .on('error', console.error)
    // .pipe(prettify())
    .pipe(extname())
    .pipe(app.dest(build.dest()));
});

/**
 * Only push files through the stream that have changed.
 * This is useful when using `assemble dev`
 */

var changed = [];
function changedFilter(key, view) {
  if (!changed.length) return true;
  return changed.filter(function(fp) {
    return view.path.substr(view.path.length - fp.length) === fp;
  }).length;
}

/**
 * Build the assemble docs
 */

app.task('build', ['load'], function() {
  return app.toStream('docs', changedFilter).on('error', console.log)
    .pipe(drafts())
    .pipe(app.renderFile()).on('error', console.log)
    // .pipe(prettify())
    .pipe(extname())
    .pipe(through.obj(function(file, enc, next) {
      file.path = file.data.permalink || file.path;
      file.base = '_gh_pages';
      next(null, file);
    }))
    .pipe(manifest(app, {
      base: 'en/v' + pkg.version,
      renamePath: function(fp) {
        var idx = fp.indexOf('en/v' + pkg.version);
        if (idx === -1) return fp;
        return fp.substr(idx);
      }
    }))
    .pipe(app.dest(function(file) {
      return build.dest();
    }))
    .pipe(browserSync.stream());
});

/**
 * Copy assets
 */

app.task('assets', function() {
  return app.copy(['assets/**/*'], build.dest('en/v' + pkg.version + '/assets'))
    .pipe(browserSync.stream());
});

/**
 * Watch files for changes
 */

app.task('watch', function() {
  app.watch('templates/**/*.hbs', 'build');
  app.watch('assets/**/*', 'assets');
  var watcher = app.watch('content/**/*.md');

  console.log('watching docs templates');

  // manually handle the change event build
  // up a list of actual changed files and kick off
  // the build. This is useful with `assemble dev`
  watcher.on('change', function(fp) {
    changed.push(fp);
    app.build(['build'], function() {
      changed = [];
    });
  });
});

/**
 * Dev: alias for building, serving, and watching docs.
 */

app.task('dev', ['default'], app.parallel(['serve', 'watch']));

/**
 * Production: alias for cleaning and building docs.
 */

app.task('default', [
  'clean',
  'build',
  'assets',
  'redirects',
  'generate-redirects'
]);

/**
 * Expose the `app` instance
 */

module.exports = app;
<commit_msg>load async helpers trim `.md` extension from keys to assign in finding views<commit_after>'use strict';

var path = require('path');
var del = require('delete');
var Time = require('time-diff');
var watch = require('base-watch');
var prettify = require('gulp-prettify');
var drafts = require('gulp-drafts');
var extname = require('gulp-extname');
var ignore = require('gulp-ignore');
var debug = require('debug')('assemble:docs');
var browserSync = require('browser-sync').create();
var permalinks = require('assemble-permalinks');
var merge = require('mixin-deep');
var through = require('through2');

var collections = require('./support/plugins/assemble-collections');
var viewEvents = require('./support/plugins/view-events');
var redirects = require('./support/plugins/redirects');
var manifest = require('./support/plugins/manifest');
var versions = require('./support/plugins/versions');
var getDest = require('./support/plugins/get-dest');
var utils = require('./support/utils');
var pkg = require('../package');
var assemble = require('..');

/**
 * Create our assemble appplication
 */

var app = assemble();

app.time = new Time();
app.use(viewEvents('permalink'));
app.use(collections());
app.use(permalinks());
app.use(getDest());
app.use(watch());

app.on('error', function(err) {
  console.log(err.stack);
});

/**
 * Common variables
 */

var build = {
  dest: path.resolve.bind(path, __dirname, '../_gh_pages')
};

/**
 * Middleware
 */

var permalinkCache = {};
app.onLoad(/./, function(file, next) {
  debug('onLoad for %s', file.path);
  if (permalinkCache[file.path] === true) {
    debug('creating permalink context for %s', file.path);
    file.permalink(file.options.permalink);
  }
  next();
});

app.onPermalink(/./, function(file, next) {
  debug('onPermalink for %s', file.path);
  permalinkCache[file.path] = true;
  file.data = merge({category: 'docs'}, app.cache.data, file.data);
  next();
});

/**
 * Customize how templates are stored. This changes the
 * key of each template, so it's easier to lookup later
 */

app.option('renameKey', function(fp) {
  if (fp.indexOf(__dirname) === -1) {
    if (/\.md$/.test(fp)) {
      return fp.substr(0, fp.length - 3);
    }
    return fp;
  }
  return path.basename(fp, path.extname(fp));
});

/**
 * Load some "global" data
 */

app.data({
  docs: {
    content: 'https://github.com/assemble/assemble/tree/master/docs/content'
  },
  site: {
    title: 'Assemble Docs',
    version: pkg.version,
    base: ':destBase/en/v:site.version',
    categories: {
      docs: {
        title: 'Docs',
        description: 'Assemble documentation',
        category: 'docs',
        filter: navFilter('docs')
      },
      api: {
        title: 'API',
        description: 'Assemble API Documentation',
        category: 'api',
        filter: navFilter('api')
      },
      recipes: {
        title: 'Recipes',
        description: 'Assemble recipes',
        category: 'recipes',
        filter: navFilter('recipes')
      },
      subjects: {
        title: 'Subjects',
        description: 'Advanced subjects on using assemble',
        category: 'subjects',
        filter: navFilter('subjects')
      }
    }
  },
  destBase: '_gh_pages',
  assets: ':site.base/assets',
  links: [{
    dest: 'assemble',
    collection: 'docs',
    text: 'Assemble'
  }]
});

/**
 * Create a custom view collection
 */

var permalinkOpts = {
  slug: function() {
    if (this.category === 'docs') {
      return this.filename + '.html';
    }
    var rel = this.relative;
    var ext = path.extname(rel);
    return path.join(rel.substr(0, rel.length - ext.length), 'index.html');
  }
};

function navFilter(category) {
  return function(item) {
    if (item.data.category !== category) {
      return false;
    }
    return (typeof item.data.draft === 'undefined' || item.data.draft === false);
  };
}

app.create('docs', {layout: 'body'})
  .preRender(/\.md/, function(file, next) {
    var category = file.data.category;
    if (!category) return next();
    file.data = merge({
      section: app.data('site.categories.' + category),
      layout: category === 'recipes' ? 'recipe' : file.data.layout
    }, file.data);
    next();
  })
  .use(permalinks(':site.base/:slug()', permalinkOpts));

app.create('redirects', {
  renameKey: function(key, view) {
    return view ? view.path : key;
  }
});

/**
 * Load helpers
 */

app.helpers('support/helpers/*.js');
app.asyncHelpers('support/async-helpers/*.js');

/**
 * Clean out the current version's built files
 */

app.task('clean', function(cb) {
  del(build.dest('en/v' + pkg.version), {force: true}, cb);
});

/**
 * Build manifest for a specific version.
 * This is in case an old version didn't have a manifest.
 * Run with `assemble manifest --version 0.6.0`
 */

app.task('manifest', function(cb) {
  // get the version from argv.orig (because of the dots in versions)
  var version = app.argv.orig.version;
  if (typeof version === 'undefined' || typeof version === 'boolean') {
    cb(new Error('--version needs to be specified when running `assemble manifest`'));
    return;
  }

  var dir = build.dest('en/v' + version);
  if (!utils.exists(dir)) {
    cb(new Error('Unable to find "' + dir + '" for version "' + version + '"'));
    return;
  }

  return app.src([dir + '/**/*.html'])
    .on('err', console.log)
    .pipe(manifest(app, {
      renamePath: function(fp) {
        return 'en/v' + version + '/' + fp;
      }
    }))
    .pipe(ignore.include('manifest.json'))
    .pipe(app.dest(function(file) {
      file.base = file.dirname;
      return dir;
    }));
});

/**
 * Build up a redirects.json file
 */

app.task('redirects', function() {
  return app.src(build.dest('en/*/manifest.json'))
    .pipe(redirects(app))
    .pipe(versions(app))
    .pipe(ignore.include(['redirects.json', 'versions.json']))
    .pipe(app.dest(function(file) {
      file.base = file.dirname;
      return 'data';
    }))
    .pipe(app.dest(build.dest()));
});

/**
 * Re-load templates when triggered by watch
 */

app.task('load', function* () {
  app.partials('templates/partials/**/*.hbs', {cwd: __dirname});
  app.layouts('templates/layouts/**/*.hbs', {cwd: __dirname});
  app.docs(['content/**/*.md'], {cwd: __dirname});
});

/**
 * Static server
 */

app.task('serve', function() {
  browserSync.init({
    port: 8080,
    startPath: 'docs.html',
    server: {
      baseDir: build.dest()
    }
  });
});

/**
 * Create redirect views from paths > "data/redirects.json"
 */

app.task('generate-redirects', function() {
  app.data('data/redirects.json', { namespace: 'redirects' });
  var redirects = app.data('redirects');

  for(var key in redirects) {
    var to = redirects[key];
    app.redirect(key, {
      path: key,
      data: {
        title: key,
        layout: 'redirect',
        redirect: { seconds: 0, url: to }
      },
      content: ''
    });
  }

  return app.toStream('redirects')
    .on('error', console.error)
    .pipe(app.renderFile('hbs'))
    .on('error', console.error)
    // .pipe(prettify())
    .pipe(extname())
    .pipe(app.dest(build.dest()));
});

/**
 * Only push files through the stream that have changed.
 * This is useful when using `assemble dev`
 */

var changed = [];
function changedFilter(key, view) {
  if (!changed.length) return true;
  return changed.filter(function(fp) {
    return view.path.substr(view.path.length - fp.length) === fp;
  }).length;
}

/**
 * Build the assemble docs
 */

app.task('build', ['load'], function() {
  return app.toStream('docs', changedFilter).on('error', console.log)
    .pipe(drafts())
    .pipe(app.renderFile()).on('error', console.log)
    // .pipe(prettify())
    .pipe(extname())
    .pipe(through.obj(function(file, enc, next) {
      file.path = file.data.permalink || file.path;
      file.base = '_gh_pages';
      next(null, file);
    }))
    .pipe(manifest(app, {
      base: 'en/v' + pkg.version,
      renamePath: function(fp) {
        var idx = fp.indexOf('en/v' + pkg.version);
        if (idx === -1) return fp;
        return fp.substr(idx);
      }
    }))
    .pipe(app.dest(function(file) {
      return build.dest();
    }))
    .pipe(browserSync.stream());
});

/**
 * Copy assets
 */

app.task('assets', function() {
  return app.copy(['assets/**/*'], build.dest('en/v' + pkg.version + '/assets'))
    .pipe(browserSync.stream());
});

/**
 * Watch files for changes
 */

app.task('watch', function() {
  app.watch('templates/**/*.hbs', 'build');
  app.watch('assets/**/*', 'assets');
  var watcher = app.watch('content/**/*.md');

  console.log('watching docs templates');

  // manually handle the change event build
  // up a list of actual changed files and kick off
  // the build. This is useful with `assemble dev`
  watcher.on('change', function(fp) {
    changed.push(fp);
    app.build(['build'], function() {
      changed = [];
    });
  });
});

/**
 * Dev: alias for building, serving, and watching docs.
 */

app.task('dev', ['default'], app.parallel(['serve', 'watch']));

/**
 * Production: alias for cleaning and building docs.
 */

app.task('default', [
  'clean',
  'build',
  'assets',
  'redirects',
  'generate-redirects'
]);

/**
 * Expose the `app` instance
 */

module.exports = app;
<|endoftext|>


<commit_before>/* eslint-disable no-await-in-loop */

import {
  RECEIVE_REVISIONS,
  REVISIONS_LOADING,
  RECEIVE_COURSE_SCOPED_REVISIONS,
  COURSE_SCOPED_REVISIONS_LOADING,
  SORT_REVISIONS,
  API_FAIL
} from '../constants';
import { fetchWikidataLabelsForRevisions } from './wikidata_actions';
import logErrorMessage from '../utils/log_error_message';
import request from '../utils/request';
import moment from 'moment';
import { stringify } from 'query-string';
import { ORESSupportedWiki, PageAssessmentGrades, PageAssessmentSupportedWiki } from '../utils/article_finder_language_mappings';
import { url } from '../utils/wiki_utils';
import { chunk, flatten } from 'lodash-es';
import { queryUrl } from '../utils/article_finder_utils';

const fetchAll = async (API_URL, params, continue_str) => {
  const allData = [];
  let continueToken;
  let hasMore = true;
  while (hasMore) {
    let response;
    if (continueToken) {
      params[continue_str] = continueToken[continue_str];
      params.continue = continueToken.continue;
    }
    try {
      response = await request(`${API_URL}?${stringify(params)}&origin=*`);
      if (!response.ok) {
        throw response;
      }
    } catch (e) {
      return allData;
    }
    const json = await response.json();
    allData.push(...json.query.usercontribs);
    if (json.continue) {
      continueToken = json.continue;
    } else {
      hasMore = false;
    }
  }
  return allData;
};
const fetchAllRevisions = async (API_URL, days, usernames, wiki, course_start, last_date) => {
  let ucend;
  if (moment(last_date).subtract(days, 'days').isBefore(course_start)) {
    ucend = moment(course_start).format();
  } else {
    ucend = moment(last_date).subtract(days, 'days').format();
  }
  // eslint-disable-next-line no-console
  console.log(`Fetching revisions between ${moment(last_date).format()} and ${ucend} -> ${days} days from ${API_URL}`);
  // since a max of 50 users are allowed in one query
  const usernamesChunks = chunk(usernames, 50);
  const usernamePromises = [];
  /* eslint-disable no-restricted-syntax */
  for (const usernameChunk of usernamesChunks) {
    const params = {
      action: 'query',
      format: 'json',
      list: 'usercontribs',
      ucuser: usernameChunk.join('|'),
      ucprop: 'ids|title|sizediff|timestamp',
      uclimit: 50,
      ucend,
      ucstart: moment(last_date).format(),
      ucdir: 'older',
    };
    usernamePromises.push(fetchAll(API_URL, params, 'uccontinue'));
  }
  const values = await Promise.all(usernamePromises);
  const revisions = flatten(values);
  return revisions;
};

const fetchClassFromRevisionsOfWiki = async (wiki_url, revisionsOfWiki) => {
  // remove duplicates -> each article occurs only once after this
  const prefix = `https://${wiki_url}`;
  const API_URL = `${prefix}/w/api.php`;
  // eslint-disable-next-line no-console
  console.log(`Fetching page assessments from ${wiki_url}`);

  const uniqueArticles = [...new Set(revisionsOfWiki.filter(revision => revision.ns === 0).map(revision => revision.title))];
  const allRatings = [];
  const chunks = chunk(uniqueArticles, 50);
  for (const uniqueArticlesChunk of chunks) {
    const params = {
      action: 'query',
      format: 'json',
      titles: uniqueArticlesChunk.join('|'),
      prop: 'pageassessments',
      pasubprojects: false,
      palimit: 500
    };
    const response = await request(`${API_URL}?${stringify(params)}&origin=*`);
    const pages = (await response.json())?.query?.pages;
    if (pages) { allRatings.push(pages); }
  }
  if (!allRatings) {
    // no ratings found
    return;
  }
  // merge list of objects of ratings into a single object
  const ratings = Object.assign({}, ...allRatings);
  const assessments = {};
  /* eslint-disable no-restricted-syntax */
  for (const revision of revisionsOfWiki) {
    const assessment = {};
    // if pageassessments exists
    if (ratings?.[revision.pageid]?.pageassessments) {
      // pick the first key of the object pageassessments
      let rating;
      for (const value of Object.values(ratings[revision.pageid].pageassessments)) {
        if (value.class) {
          rating = value.class;
          break;
        }
      }
      if (rating) {
        const mapping = PageAssessmentGrades[revision.wiki.project][revision.wiki.language][rating];
        if (mapping) {
          assessment.rating_num = mapping.score;
          assessment.pretty_rating = mapping.pretty;
          assessment.rating = mapping.class;
        }
      }
    }
    assessments[revision.revid] = assessment;
  }
  return assessments;
};

const getReferences = (item) => {
  const features = item?.articlequality?.features;
  return features?.['feature.wikitext.revision.ref_tags']
        || features?.['feature.len(<datasource.wikidatawiki.revision.references>)']
        || features?.['feature.enwiki.revision.shortened_footnote_templates']
        || 0;
};

const fetchReferencesAdded = async (prevReferences, wikiMap) => {
  // eslint-disable-next-line no-restricted-syntax
  const referencesPromises = [];
  // eslint-disable-next-line no-restricted-syntax
  for (const [wiki_url, revisionsOfWiki] of wikiMap) {
    referencesPromises.push(fetchReferencesAddedFromWiki(wiki_url, revisionsOfWiki));
  }
  const resolvedValues = await Promise.all(referencesPromises);

  // merge array of objects into one object
  const allReferences = Object.assign({}, ...resolvedValues);

  // return after merging previous References and current
  return Object.assign({}, ...[allReferences, prevReferences]);
};

const fetchReferencesAddedFromWiki = async (wiki_url, revisions) => {
  // eslint-disable-next-line no-console
  console.log(`Fetching references information from ${wiki_url}`);

  const list = wiki_url.split('.');
  let wiki;
  if (list.length === 3) {
    wiki = { language: list[0], project: list[1] };
  } else {
    wiki = { project: list[0] };
  }
  if (
    !ORESSupportedWiki.projects.includes(wiki.project)
  && !ORESSupportedWiki.languages.includes(wiki.language)
  ) {
    // wiki does not support ORES
    return;
  }
  let suffix;
  if (wiki.project === 'wikidata') {
    suffix = 'wikidatawiki';
  } else {
    suffix = `${wiki.language}wiki`;
  }
  const API_URL = `http://ores.wikimedia.org/v3/scores/${suffix}`;
  const revids = revisions.filter(revision => revision.ns === 0).map(revision => `${revision.parentid}|${revision.revid}`);
  const chunks = chunk(revids, 25);
  // eslint-disable-next-line no-restricted-syntax

  const promises = [];
  // eslint-disable-next-line no-restricted-syntax
  for (const revid_chunk of chunks) {
    const params = {
      revids: revid_chunk.join('|'),
      features: true,
      models: 'articlequality'
    };

    promises.push(queryUrl(`${API_URL}`, params));
  }
  // get the scores and remove all undefined values
  // this is an array of objects
  const values = (await Promise.all(promises)).map(data => data[suffix].scores).filter(item => item);

  // merge the array of objects into one object
  const combinedObject = Object.assign({}, ...values);

  const referencesAdded = {};
  // eslint-disable-next-line no-restricted-syntax
  for (const revision of revisions) {
    const references = getReferences(combinedObject?.[revision.revid]);
    if (references) {
      referencesAdded[revision.revid] = references - getReferences(combinedObject?.[revision.parentid]);
    }
  }
  return referencesAdded;
};

const fetchRevisionsAndReferences = async (prevReferences, prevAssessments, revisions, dispatch) => {
  const wikiMap = new Map();
  // eslint-disable-next-line no-restricted-syntax
  for (const revision of revisions) {
    if (
      PageAssessmentSupportedWiki?.[revision.wiki.project]?.includes(revision.wiki.language)
      || ORESSupportedWiki.projects.includes(revision.wiki.project)
      || ORESSupportedWiki.languages.includes(revision.wiki.language)
    ) {
      if (wikiMap.has(url(revision.wiki))) {
        const value = wikiMap.get(url(revision.wiki));
        value.push(revision);
        wikiMap.set(url(revision.wiki), value);
      } else {
        const value = [];
        value.push(revision);
        wikiMap.set(url(revision.wiki), value);
      }
    }
  }
  fetchClassFromRevisions(prevAssessments, wikiMap).then((assessments) => {
    // eslint-disable-next-line no-console
    console.log('Successfully fetched all page assessment information');
    dispatch({
      type: 'RECEIVE_ASSESSMENTS',
      data: { assessments }
    });
  }
  );

  fetchReferencesAdded(prevReferences, wikiMap).then((referencesAdded) => {
    // eslint-disable-next-line no-console
    console.log('Successfully fetched all references information');
    dispatch({
      type: 'RECEIVE_REFERENCES',
      data: { referencesAdded }
    });
  });
};

const fetchClassFromRevisions = async (prevAssessments, wikiMap) => {
  // eslint-disable-next-line no-restricted-syntax
  const assessmentsPromises = [];
  // eslint-disable-next-line no-restricted-syntax
  for (const [wiki_url, revisionsOfWiki] of wikiMap) {
    assessmentsPromises.push(fetchClassFromRevisionsOfWiki(wiki_url, revisionsOfWiki));
  }
  const resolvedValues = await Promise.all(assessmentsPromises);

  // merge all the assessments
  const allAssessments = Object.assign({}, ...resolvedValues);

  // merge the previous and current assessments
  return Object.assign({}, ...[allAssessments, prevAssessments]);
};

const fetchRevisionsFromWiki = async (days, wiki, usernames, course_start, last_date) => {
  const prefix = `https://${url(wiki)}`;
  const API_URL = `${prefix}/w/api.php`;
  const revisions = await fetchAllRevisions(API_URL, days, usernames, wiki, course_start, last_date);
  /* eslint-disable no-restricted-syntax */
  for (const revision of revisions) {
    revision.wiki = wiki;
    const diff_params = {
      title: revision.title,
      diff: 'prev',
      oldid: revision.parentid
    };

    // url for the diff
    revision.url = `${prefix}/w/index.php?${stringify(diff_params)}`;

    // main article url - we use stringify to ensure that its encoded properly
    revision.article_url = `${prefix}/w/index.php?${stringify({ title: revision.title })}`;

    // to maintain the old structure of the revision object
    revision.characters = revision.sizediff;
    revision.mw_rev_id = revision.revid;
    revision.id = revision.revid;
    revision.revisor = revision.user;
    revision.date = revision.timestamp;
    revision.mw_page_id = revision.pageid;
  }
  /* eslint-enable no-restricted-syntax */
  return { revisions, wiki };
};

const fetchRevisionsFromUsers = async (course, users, days, last_date) => {
  const usernames = users.map(user => user.username);

  let revisions = [];
  const wikiPromises = [];
  /* eslint-disable no-restricted-syntax */

  // request until we find 50 revisions or the date is outside the course duration
  while (revisions.length < 50 && moment(last_date).isAfter(course.start)) {
    for (const wiki of course.wikis) {
      wikiPromises.push(fetchRevisionsFromWiki(days, wiki, usernames, course.start, last_date));
    }
    const resolvedValues = await Promise.all(wikiPromises);
    for (const value of resolvedValues) {
      const { revisions: items } = value;
      revisions.push(...items);
    }
    last_date = moment(last_date).subtract(days, 'days').format();
    if (revisions.length < 50) {
      days *= 3;
    }
  }
  // remove duplicates
  // they occur because dates overlap and sometimes the same revision is included twice
  revisions = [...new Map(revisions.map(v => [v.id, v])).values()];

  /* eslint-enable no-restricted-syntax */
  return { revisions, last_date };
};

const fetchRevisionsPromise = async (course, limit, isCourseScoped, users, last_date, prevAssessments, prevReferences, dispatch) => {
  if (!isCourseScoped) {
    const { revisions, last_date: new_last_date } = await fetchRevisionsFromUsers(course, users, 7, last_date);
    if (course.revisions) {
      course.revisions = course.revisions.concat(revisions);
    } else {
      course.revisions = revisions;
    }
    course.revisions = course.revisions.sort((revision1, revision2) => {
      const date1 = new Date(revision1.date);
      const date2 = new Date(revision2.date);
      return date2.getTime() - date1.getTime();
    });
    // we don't await this. When the assessments/references get laoded, the action is dispatched
    fetchRevisionsAndReferences(prevReferences, prevAssessments, revisions, dispatch);
    return { course, last_date: new_last_date };
  }
  const response = await request(`/courses/${course.slug}/revisions.json?limit=${limit}&course_scoped=${isCourseScoped}`);
  if (!response.ok) {
    logErrorMessage(response);
    const data = await response.text();
    response.responseText = data;
    throw response;
  }
  return response.json();
};

export const fetchRevisions = (course, limit, isCourseScoped = false) => async (dispatch, getState) => {
  let actionType;
  if (isCourseScoped) {
    actionType = RECEIVE_COURSE_SCOPED_REVISIONS;
    dispatch({ type: COURSE_SCOPED_REVISIONS_LOADING });
  } else {
    actionType = RECEIVE_REVISIONS;
    dispatch({ type: REVISIONS_LOADING });
  }
  const state = getState();
  const users = state.users.users;
  if (users.length === 0) {
    course.revisions = [];
    dispatch({
      type: actionType,
      data: { course },
      limit: limit
    });
    return;
  }
  return (
    fetchRevisionsPromise(course, limit, isCourseScoped, users, state.revisions.last_date, state.revisions.assessments, state.revisions.referencesAdded, dispatch)
      .then((resp) => {
        dispatch({
          type: actionType,
          data: resp,
          limit: limit
        });
        // Now that we received the revisions data, query wikidata.org for the labels
        // of any Wikidata entries that are among the revisions.
        fetchWikidataLabelsForRevisions(resp.course.revisions, dispatch);
      })
      .catch(response => (dispatch({ type: API_FAIL, data: response })))
  );
};

export const sortRevisions = key => ({ type: SORT_REVISIONS, key: key });
<commit_msg>used direct wiki link instead of index.php<commit_after>/* eslint-disable no-await-in-loop */

import {
  RECEIVE_REVISIONS,
  REVISIONS_LOADING,
  RECEIVE_COURSE_SCOPED_REVISIONS,
  COURSE_SCOPED_REVISIONS_LOADING,
  SORT_REVISIONS,
  API_FAIL
} from '../constants';
import { fetchWikidataLabelsForRevisions } from './wikidata_actions';
import logErrorMessage from '../utils/log_error_message';
import request from '../utils/request';
import moment from 'moment';
import { stringify } from 'query-string';
import { ORESSupportedWiki, PageAssessmentGrades, PageAssessmentSupportedWiki } from '../utils/article_finder_language_mappings';
import { url } from '../utils/wiki_utils';
import { chunk, flatten } from 'lodash-es';
import { queryUrl } from '../utils/article_finder_utils';

const fetchAll = async (API_URL, params, continue_str) => {
  const allData = [];
  let continueToken;
  let hasMore = true;
  while (hasMore) {
    let response;
    if (continueToken) {
      params[continue_str] = continueToken[continue_str];
      params.continue = continueToken.continue;
    }
    try {
      response = await request(`${API_URL}?${stringify(params)}&origin=*`);
      if (!response.ok) {
        throw response;
      }
    } catch (e) {
      return allData;
    }
    const json = await response.json();
    allData.push(...json.query.usercontribs);
    if (json.continue) {
      continueToken = json.continue;
    } else {
      hasMore = false;
    }
  }
  return allData;
};
const fetchAllRevisions = async (API_URL, days, usernames, wiki, course_start, last_date) => {
  let ucend;
  if (moment(last_date).subtract(days, 'days').isBefore(course_start)) {
    ucend = moment(course_start).format();
  } else {
    ucend = moment(last_date).subtract(days, 'days').format();
  }
  // eslint-disable-next-line no-console
  console.log(`Fetching revisions between ${moment(last_date).format()} and ${ucend} -> ${days} days from ${API_URL}`);
  // since a max of 50 users are allowed in one query
  const usernamesChunks = chunk(usernames, 50);
  const usernamePromises = [];
  /* eslint-disable no-restricted-syntax */
  for (const usernameChunk of usernamesChunks) {
    const params = {
      action: 'query',
      format: 'json',
      list: 'usercontribs',
      ucuser: usernameChunk.join('|'),
      ucprop: 'ids|title|sizediff|timestamp',
      uclimit: 50,
      ucend,
      ucstart: moment(last_date).format(),
      ucdir: 'older',
    };
    usernamePromises.push(fetchAll(API_URL, params, 'uccontinue'));
  }
  const values = await Promise.all(usernamePromises);
  const revisions = flatten(values);
  return revisions;
};

const fetchClassFromRevisionsOfWiki = async (wiki_url, revisionsOfWiki) => {
  // remove duplicates -> each article occurs only once after this
  const prefix = `https://${wiki_url}`;
  const API_URL = `${prefix}/w/api.php`;
  // eslint-disable-next-line no-console
  console.log(`Fetching page assessments from ${wiki_url}`);

  const uniqueArticles = [...new Set(revisionsOfWiki.filter(revision => revision.ns === 0).map(revision => revision.title))];
  const allRatings = [];
  const chunks = chunk(uniqueArticles, 50);
  for (const uniqueArticlesChunk of chunks) {
    const params = {
      action: 'query',
      format: 'json',
      titles: uniqueArticlesChunk.join('|'),
      prop: 'pageassessments',
      pasubprojects: false,
      palimit: 500
    };
    const response = await request(`${API_URL}?${stringify(params)}&origin=*`);
    const pages = (await response.json())?.query?.pages;
    if (pages) { allRatings.push(pages); }
  }
  if (!allRatings) {
    // no ratings found
    return;
  }
  // merge list of objects of ratings into a single object
  const ratings = Object.assign({}, ...allRatings);
  const assessments = {};
  /* eslint-disable no-restricted-syntax */
  for (const revision of revisionsOfWiki) {
    const assessment = {};
    // if pageassessments exists
    if (ratings?.[revision.pageid]?.pageassessments) {
      // pick the first key of the object pageassessments
      let rating;
      for (const value of Object.values(ratings[revision.pageid].pageassessments)) {
        if (value.class) {
          rating = value.class;
          break;
        }
      }
      if (rating) {
        const mapping = PageAssessmentGrades[revision.wiki.project][revision.wiki.language][rating];
        if (mapping) {
          assessment.rating_num = mapping.score;
          assessment.pretty_rating = mapping.pretty;
          assessment.rating = mapping.class;
        }
      }
    }
    assessments[revision.revid] = assessment;
  }
  return assessments;
};

const getReferences = (item) => {
  const features = item?.articlequality?.features;
  return features?.['feature.wikitext.revision.ref_tags']
        || features?.['feature.len(<datasource.wikidatawiki.revision.references>)']
        || features?.['feature.enwiki.revision.shortened_footnote_templates']
        || 0;
};

const fetchReferencesAdded = async (prevReferences, wikiMap) => {
  // eslint-disable-next-line no-restricted-syntax
  const referencesPromises = [];
  // eslint-disable-next-line no-restricted-syntax
  for (const [wiki_url, revisionsOfWiki] of wikiMap) {
    referencesPromises.push(fetchReferencesAddedFromWiki(wiki_url, revisionsOfWiki));
  }
  const resolvedValues = await Promise.all(referencesPromises);

  // merge array of objects into one object
  const allReferences = Object.assign({}, ...resolvedValues);

  // return after merging previous References and current
  return Object.assign({}, ...[allReferences, prevReferences]);
};

const fetchReferencesAddedFromWiki = async (wiki_url, revisions) => {
  // eslint-disable-next-line no-console
  console.log(`Fetching references information from ${wiki_url}`);

  const list = wiki_url.split('.');
  let wiki;
  if (list.length === 3) {
    wiki = { language: list[0], project: list[1] };
  } else {
    wiki = { project: list[0] };
  }
  if (
    !ORESSupportedWiki.projects.includes(wiki.project)
  && !ORESSupportedWiki.languages.includes(wiki.language)
  ) {
    // wiki does not support ORES
    return;
  }
  let suffix;
  if (wiki.project === 'wikidata') {
    suffix = 'wikidatawiki';
  } else {
    suffix = `${wiki.language}wiki`;
  }
  const API_URL = `http://ores.wikimedia.org/v3/scores/${suffix}`;
  const revids = revisions.filter(revision => revision.ns === 0).map(revision => `${revision.parentid}|${revision.revid}`);
  const chunks = chunk(revids, 25);
  // eslint-disable-next-line no-restricted-syntax

  const promises = [];
  // eslint-disable-next-line no-restricted-syntax
  for (const revid_chunk of chunks) {
    const params = {
      revids: revid_chunk.join('|'),
      features: true,
      models: 'articlequality'
    };

    promises.push(queryUrl(`${API_URL}`, params));
  }
  // get the scores and remove all undefined values
  // this is an array of objects
  const values = (await Promise.all(promises)).map(data => data[suffix].scores).filter(item => item);

  // merge the array of objects into one object
  const combinedObject = Object.assign({}, ...values);

  const referencesAdded = {};
  // eslint-disable-next-line no-restricted-syntax
  for (const revision of revisions) {
    const references = getReferences(combinedObject?.[revision.revid]);
    if (references) {
      referencesAdded[revision.revid] = references - getReferences(combinedObject?.[revision.parentid]);
    }
  }
  return referencesAdded;
};

const fetchRevisionsAndReferences = async (prevReferences, prevAssessments, revisions, dispatch) => {
  const wikiMap = new Map();
  // eslint-disable-next-line no-restricted-syntax
  for (const revision of revisions) {
    if (
      PageAssessmentSupportedWiki?.[revision.wiki.project]?.includes(revision.wiki.language)
      || ORESSupportedWiki.projects.includes(revision.wiki.project)
      || ORESSupportedWiki.languages.includes(revision.wiki.language)
    ) {
      if (wikiMap.has(url(revision.wiki))) {
        const value = wikiMap.get(url(revision.wiki));
        value.push(revision);
        wikiMap.set(url(revision.wiki), value);
      } else {
        const value = [];
        value.push(revision);
        wikiMap.set(url(revision.wiki), value);
      }
    }
  }
  fetchClassFromRevisions(prevAssessments, wikiMap).then((assessments) => {
    // eslint-disable-next-line no-console
    console.log('Successfully fetched all page assessment information');
    dispatch({
      type: 'RECEIVE_ASSESSMENTS',
      data: { assessments }
    });
  }
  );

  fetchReferencesAdded(prevReferences, wikiMap).then((referencesAdded) => {
    // eslint-disable-next-line no-console
    console.log('Successfully fetched all references information');
    dispatch({
      type: 'RECEIVE_REFERENCES',
      data: { referencesAdded }
    });
  });
};

const fetchClassFromRevisions = async (prevAssessments, wikiMap) => {
  // eslint-disable-next-line no-restricted-syntax
  const assessmentsPromises = [];
  // eslint-disable-next-line no-restricted-syntax
  for (const [wiki_url, revisionsOfWiki] of wikiMap) {
    assessmentsPromises.push(fetchClassFromRevisionsOfWiki(wiki_url, revisionsOfWiki));
  }
  const resolvedValues = await Promise.all(assessmentsPromises);

  // merge all the assessments
  const allAssessments = Object.assign({}, ...resolvedValues);

  // merge the previous and current assessments
  return Object.assign({}, ...[allAssessments, prevAssessments]);
};

const fetchRevisionsFromWiki = async (days, wiki, usernames, course_start, last_date) => {
  const prefix = `https://${url(wiki)}`;
  const API_URL = `${prefix}/w/api.php`;
  const revisions = await fetchAllRevisions(API_URL, days, usernames, wiki, course_start, last_date);
  /* eslint-disable no-restricted-syntax */
  for (const revision of revisions) {
    revision.wiki = wiki;
    const diff_params = {
      title: revision.title,
      diff: 'prev',
      oldid: revision.parentid
    };

    // url for the diff
    revision.url = `${prefix}/w/index.php?${stringify(diff_params)}`;

    // main article url - we use stringify to ensure that its encoded properly
    revision.article_url = `${prefix}/wiki/${encodeURIComponent(revision.title)}`;

    // to maintain the old structure of the revision object
    revision.characters = revision.sizediff;
    revision.mw_rev_id = revision.revid;
    revision.id = revision.revid;
    revision.revisor = revision.user;
    revision.date = revision.timestamp;
    revision.mw_page_id = revision.pageid;
  }
  /* eslint-enable no-restricted-syntax */
  return { revisions, wiki };
};

const fetchRevisionsFromUsers = async (course, users, days, last_date) => {
  const usernames = users.map(user => user.username);

  let revisions = [];
  const wikiPromises = [];
  /* eslint-disable no-restricted-syntax */

  // request until we find 50 revisions or the date is outside the course duration
  while (revisions.length < 50 && moment(last_date).isAfter(course.start)) {
    for (const wiki of course.wikis) {
      wikiPromises.push(fetchRevisionsFromWiki(days, wiki, usernames, course.start, last_date));
    }
    const resolvedValues = await Promise.all(wikiPromises);
    for (const value of resolvedValues) {
      const { revisions: items } = value;
      revisions.push(...items);
    }
    last_date = moment(last_date).subtract(days, 'days').format();
    if (revisions.length < 50) {
      days *= 3;
    }
  }
  // remove duplicates
  // they occur because dates overlap and sometimes the same revision is included twice
  revisions = [...new Map(revisions.map(v => [v.id, v])).values()];

  /* eslint-enable no-restricted-syntax */
  return { revisions, last_date };
};

const fetchRevisionsPromise = async (course, limit, isCourseScoped, users, last_date, prevAssessments, prevReferences, dispatch) => {
  if (!isCourseScoped) {
    const { revisions, last_date: new_last_date } = await fetchRevisionsFromUsers(course, users, 7, last_date);
    if (course.revisions) {
      course.revisions = course.revisions.concat(revisions);
    } else {
      course.revisions = revisions;
    }
    course.revisions = course.revisions.sort((revision1, revision2) => {
      const date1 = new Date(revision1.date);
      const date2 = new Date(revision2.date);
      return date2.getTime() - date1.getTime();
    });
    // we don't await this. When the assessments/references get laoded, the action is dispatched
    fetchRevisionsAndReferences(prevReferences, prevAssessments, revisions, dispatch);
    return { course, last_date: new_last_date };
  }
  const response = await request(`/courses/${course.slug}/revisions.json?limit=${limit}&course_scoped=${isCourseScoped}`);
  if (!response.ok) {
    logErrorMessage(response);
    const data = await response.text();
    response.responseText = data;
    throw response;
  }
  return response.json();
};

export const fetchRevisions = (course, limit, isCourseScoped = false) => async (dispatch, getState) => {
  let actionType;
  if (isCourseScoped) {
    actionType = RECEIVE_COURSE_SCOPED_REVISIONS;
    dispatch({ type: COURSE_SCOPED_REVISIONS_LOADING });
  } else {
    actionType = RECEIVE_REVISIONS;
    dispatch({ type: REVISIONS_LOADING });
  }
  const state = getState();
  const users = state.users.users;
  if (users.length === 0) {
    course.revisions = [];
    dispatch({
      type: actionType,
      data: { course },
      limit: limit
    });
    return;
  }
  return (
    fetchRevisionsPromise(course, limit, isCourseScoped, users, state.revisions.last_date, state.revisions.assessments, state.revisions.referencesAdded, dispatch)
      .then((resp) => {
        dispatch({
          type: actionType,
          data: resp,
          limit: limit
        });
        // Now that we received the revisions data, query wikidata.org for the labels
        // of any Wikidata entries that are among the revisions.
        fetchWikidataLabelsForRevisions(resp.course.revisions, dispatch);
      })
      .catch(response => (dispatch({ type: API_FAIL, data: response })))
  );
};

export const sortRevisions = key => ({ type: SORT_REVISIONS, key: key });
<|endoftext|>


<commit_before>
import cloneImpl from "./clone"
import equalsImpl from "./equals"

/**
 * Clones the provided value using the structured clone algorithm.
 *
 * @param {*} value The value to clone.
 * @return {*} The created clone of the value.
 * @see https://developer.mozilla.org/en-US/docs/Web/Guide/API/DOM/The_structured_clone_algorithm
 * @see http://www.w3.org/html/wg/drafts/html/master/infrastructure.html#safe-passing-of-structured-data
 */
export let clone = cloneImpl

/**
 * Compares the provided values to determine whether they are equal.
 *
 * @param {*} value1 The 1st value to compare.
 * @param {*} value2 The 2nd value to compare.
 * @return {boolean} {@code true} if the values are equal.
 */
export let equals = equalsImpl

/**
 * Serializes the provided Indexed DB key to a string.
 *
 * @param {(number|string|Date|Array)} key The key to serialize.
 * @return {string} Serialized key.
 */
export function serializeKey(key) {
  switch (typeof key) {
    case "number":
    case "string":
      return JSON.stringify(key)
  }

  if (key instanceof Date) {
    return `Date(${key.valueOf()})`;
  }

  return key.map(keyPart => serializeKey(keyPart)).join(",")
}

/**
 * Retrieves the primary key from the provided entity.
 *
 * @param {AbstractEntity} entity The entity.
 * @param {(string|string[])} keyPath The primary key key path.
 * @return {(number|string|Date|Array)} The primary key of the entity.
 */
export function getPrimaryKey(entity, keyPath) {
  if (keyPath instanceof Array) {
    return keyPath.map(fieldPath => getField(entity, fieldPath))
  }

  return getField(entity, keyPath)
}

/**
 * Sets the primary key of the provided entity to the provided value.
 *
 * @param {AbstractEntity} entity The entity that should have its primary key
 *        set.
 * @param {(string|string[])} keyPath The primary key key path.
 * @param {(number|string|Date|Array)} primaryKey The primary key to set.
 */
export function setPrimaryKey(entity, keyPath, primaryKey) {
  if (keyPath instanceof Array) {
    let isPrimaryKeyValid = (primaryKey instanceof Array) &&
        (keyPath.length !== primaryKey.length)
    if (!isPrimaryKeyValid) {
      throw new Error("The number of elements of the primary key array must " +
          "match the number of field paths in the key path")
    }

    for (let i = 0; i < keyPath.length; i++) {
      setField(entity, keyPath[i], primaryKey[i])
    }
  } else {
    setField(entity, keyPath, primaryKey)
  }
}

/**
 * Tests whether the provided class is a valid entity class. The function
 * throws an error if the class is invalid.
 *
 * @template {T} extends AbstractEntity
 * @param {function(new: T, data: Object<string, *>)} entityClass
 * @throws {Error} Thrown if the provided class extends the
 *         {@linkcode AbstractEntity} class, but does not define the static
 *         {@code objectStore} property.
 * @throws {TypeError} Thrown if the provided class does not extend the
 *         {@linkcode AbstractEntity} class.
 */
export function validateEntityClass(entityClass) {
  if (!isSubClass(entityClass, AbstractEntity)) {
    throw new TypeError("The provided class must be an entity class - it " +
      "must extend the AbstractEntity class")
  }

  if (!specifiesObjectStore(entityClass)) {
    throw new Error("The entity class must specify the objectStore property " +
      "that returns a non-empty string specifying the name of the object " +
      "store")
  }
}

/**
 * Tests whether the specified sub-class extends the specified super-class.
 * Note that according to this function, any class is its own sub-class and
 * super-class.
 *
 * @param {function(new: *)} subClass The sub-class to test.
 * @param {function(new: *)} superClass The super-class to test against.
 * @return {boolean} If the provided sub-class extends the specified
 *         super-class.
 */
export function isSubClass(subClass, superClass) {
  let prototype = subClass.prototype

  while (prototype) {
    if (prototype === superClass.prototype) {
      return true
    }

    prototype = Object.getPrototypeOf(prototype)
  }

  return false
}

/**
 * Returns {@code true} if the provided entity class has a valid static
 * {@code objectStore} property that returns a non-empty string.
 *
 * @param {function(new: AbstractEntity, data: Object<string, *>)} entityClass
 *        The entity class to validate.
 * @return {boolean} {@code true} if the provided entity class has a valid
 *         static {@code objectStore} property.
 */
export function specifiesObjectStore(entityClass) {
  return entityClass.hasOwnProperty("objectStore") &&
      (typeof entityClass.objectStore === "string") &&
      !!entityClass.objectStore
}

/**
 * Returns the value of the field at the specified field path of the provided
 * object.
 *
 * @param {Object} object The object from which the field path value should be
 *        extracted.
 * @param {string} fieldPath The path to the field from which the value should
 *        be returned. The field path must be a sequence of valid ECMAScript
 *        identifiers separated by dots.
 * @return {*} The value of the field at the specified field path.
 */
function getField(object, fieldPath) {
  let currentObject = object

  for (let fieldName of fieldPath.split(".")) {
    if (!currentObject.hasOwnProperty(fieldName)) {
      return undefined
    }

    currentObject = currentObject[fieldName]
  }

  return currentObject
}

/**
 * Sets the field at the specified field path to the provided value in the
 * provided object.
 *
 * The function creates the field path out of empty plain objects if it does
 * not already exist.
 *
 * @param {Object} object The object in which the specified field path should
 *        be set to the provided value.
 * @param {string} fieldPath The field path at which the value should be set.
 *        The field path must be a sequence of valid ECMAScript identifiers
 *        separated by dots.
 * @param {*} value The value to set.
 */
function setField(object, fieldPath, value) {
  let currentObject = object
  let fieldNames = fieldPath.split(".")

  for (let fieldName of fieldNames.slice(0, -1)) {
    if (!currentObject.hasOwnProperty(fieldName)) {
      currentObject[fieldName] = {}
    }

    currentObject = currentObject[fieldName]
  }

  currentObject[fieldNames.pop()] = value
}
<commit_msg>fixed setting a compound primary key on an entity<commit_after>
import cloneImpl from "./clone"
import equalsImpl from "./equals"

/**
 * Clones the provided value using the structured clone algorithm.
 *
 * @param {*} value The value to clone.
 * @return {*} The created clone of the value.
 * @see https://developer.mozilla.org/en-US/docs/Web/Guide/API/DOM/The_structured_clone_algorithm
 * @see http://www.w3.org/html/wg/drafts/html/master/infrastructure.html#safe-passing-of-structured-data
 */
export let clone = cloneImpl

/**
 * Compares the provided values to determine whether they are equal.
 *
 * @param {*} value1 The 1st value to compare.
 * @param {*} value2 The 2nd value to compare.
 * @return {boolean} {@code true} if the values are equal.
 */
export let equals = equalsImpl

/**
 * Serializes the provided Indexed DB key to a string.
 *
 * @param {(number|string|Date|Array)} key The key to serialize.
 * @return {string} Serialized key.
 */
export function serializeKey(key) {
  switch (typeof key) {
    case "number":
    case "string":
      return JSON.stringify(key)
  }

  if (key instanceof Date) {
    return `Date(${key.valueOf()})`;
  }

  return key.map(keyPart => serializeKey(keyPart)).join(",")
}

/**
 * Retrieves the primary key from the provided entity.
 *
 * @param {AbstractEntity} entity The entity.
 * @param {(string|string[])} keyPath The primary key key path.
 * @return {(number|string|Date|Array)} The primary key of the entity.
 */
export function getPrimaryKey(entity, keyPath) {
  if (keyPath instanceof Array) {
    return keyPath.map(fieldPath => getField(entity, fieldPath))
  }

  return getField(entity, keyPath)
}

/**
 * Sets the primary key of the provided entity to the provided value.
 *
 * @param {AbstractEntity} entity The entity that should have its primary key
 *        set.
 * @param {(string|string[])} keyPath The primary key key path.
 * @param {(number|string|Date|Array)} primaryKey The primary key to set.
 */
export function setPrimaryKey(entity, keyPath, primaryKey) {
  if (keyPath instanceof Array) {
    let isPrimaryKeyValid = (primaryKey instanceof Array) &&
        (keyPath.length === primaryKey.length)
    if (!isPrimaryKeyValid) {
      throw new Error("The number of elements of the primary key array must " +
          "match the number of field paths in the key path")
    }

    for (let i = 0; i < keyPath.length; i++) {
      setField(entity, keyPath[i], primaryKey[i])
    }
  } else {
    setField(entity, keyPath, primaryKey)
  }
}

/**
 * Tests whether the provided class is a valid entity class. The function
 * throws an error if the class is invalid.
 *
 * @template {T} extends AbstractEntity
 * @param {function(new: T, data: Object<string, *>)} entityClass
 * @throws {Error} Thrown if the provided class extends the
 *         {@linkcode AbstractEntity} class, but does not define the static
 *         {@code objectStore} property.
 * @throws {TypeError} Thrown if the provided class does not extend the
 *         {@linkcode AbstractEntity} class.
 */
export function validateEntityClass(entityClass) {
  if (!isSubClass(entityClass, AbstractEntity)) {
    throw new TypeError("The provided class must be an entity class - it " +
      "must extend the AbstractEntity class")
  }

  if (!specifiesObjectStore(entityClass)) {
    throw new Error("The entity class must specify the objectStore property " +
      "that returns a non-empty string specifying the name of the object " +
      "store")
  }
}

/**
 * Tests whether the specified sub-class extends the specified super-class.
 * Note that according to this function, any class is its own sub-class and
 * super-class.
 *
 * @param {function(new: *)} subClass The sub-class to test.
 * @param {function(new: *)} superClass The super-class to test against.
 * @return {boolean} If the provided sub-class extends the specified
 *         super-class.
 */
export function isSubClass(subClass, superClass) {
  let prototype = subClass.prototype

  while (prototype) {
    if (prototype === superClass.prototype) {
      return true
    }

    prototype = Object.getPrototypeOf(prototype)
  }

  return false
}

/**
 * Returns {@code true} if the provided entity class has a valid static
 * {@code objectStore} property that returns a non-empty string.
 *
 * @param {function(new: AbstractEntity, data: Object<string, *>)} entityClass
 *        The entity class to validate.
 * @return {boolean} {@code true} if the provided entity class has a valid
 *         static {@code objectStore} property.
 */
export function specifiesObjectStore(entityClass) {
  return entityClass.hasOwnProperty("objectStore") &&
      (typeof entityClass.objectStore === "string") &&
      !!entityClass.objectStore
}

/**
 * Returns the value of the field at the specified field path of the provided
 * object.
 *
 * @param {Object} object The object from which the field path value should be
 *        extracted.
 * @param {string} fieldPath The path to the field from which the value should
 *        be returned. The field path must be a sequence of valid ECMAScript
 *        identifiers separated by dots.
 * @return {*} The value of the field at the specified field path.
 */
function getField(object, fieldPath) {
  let currentObject = object

  for (let fieldName of fieldPath.split(".")) {
    if (!currentObject.hasOwnProperty(fieldName)) {
      return undefined
    }

    currentObject = currentObject[fieldName]
  }

  return currentObject
}

/**
 * Sets the field at the specified field path to the provided value in the
 * provided object.
 *
 * The function creates the field path out of empty plain objects if it does
 * not already exist.
 *
 * @param {Object} object The object in which the specified field path should
 *        be set to the provided value.
 * @param {string} fieldPath The field path at which the value should be set.
 *        The field path must be a sequence of valid ECMAScript identifiers
 *        separated by dots.
 * @param {*} value The value to set.
 */
function setField(object, fieldPath, value) {
  let currentObject = object
  let fieldNames = fieldPath.split(".")

  for (let fieldName of fieldNames.slice(0, -1)) {
    if (!currentObject.hasOwnProperty(fieldName)) {
      currentObject[fieldName] = {}
    }

    currentObject = currentObject[fieldName]
  }

  currentObject[fieldNames.pop()] = value
}
<|endoftext|>


<commit_before>import React from 'react';
import PropTypes from 'prop-types';
import _ from 'lodash';

// Listen for escape key being pressed and call `onEscape`
export default class EscapeListener extends React.Component {
  constructor(props) {
    super(props);
    this.onKeyDown = this.onKeyDown.bind(this);
    this.onClick = this.onClick.bind(this);
  }

  componentDidMount() {
    document.addEventListener('keydown', this.onKeyDown);
  }

  componentWillUnmount() {
    document.removeEventListener('keydown', this.onKeyDown);
  }

  onKeyDown(e) {
    const {onEscape} = this.props;
    if (e.which === 27) onEscape();
  }

  onClick(e) {
    const {escapeOnUnhandledClick, onEscape} = this.props;
    if (escapeOnUnhandledClick) onEscape();
  }

  render() {
    const {className, children, style, escapeOnUnhandledClick} = this.props;
    const classNameString = _.compact([className, 'EscapeListener']).join(' ');
    return (

{children}

    );
  }
}
EscapeListener.propTypes = {
  onEscape: PropTypes.func.isRequired,
  children: PropTypes.node.isRequired,
  escapeOnUnhandledClick: PropTypes.bool,
  className: PropTypes.string,
  style: PropTypes.object
};

<commit_msg>backing out global listeners in escapelistener<commit_after>import React from 'react';
import PropTypes from 'prop-types';
import _ from 'lodash';

// Listen for escape key being pressed and call `onEscape`
export default class EscapeListener extends React.Component {
  constructor(props) {
    super(props);
    this.onKeyDown = this.onKeyDown.bind(this);
    this.onClick = this.onClick.bind(this);
  }

  onKeyDown(e) {
    const {onEscape} = this.props;
    if (e.which === 27) onEscape();
  }

  onClick(e) {
    const {escapeOnUnhandledClick, onEscape} = this.props;
    if (escapeOnUnhandledClick) onEscape();
  }

  render() {
    const {className, children, style, escapeOnUnhandledClick} = this.props;
    const classNameString = _.compact([className, 'EscapeListener']).join(' ');
    return (

{children}

    );
  }
}
EscapeListener.propTypes = {
  onEscape: PropTypes.func.isRequired,
  children: PropTypes.node.isRequired,
  escapeOnUnhandledClick: PropTypes.bool,
  className: PropTypes.string,
  style: PropTypes.object
};

<|endoftext|>


<commit_before>var ignoreUrlChange = false;
var ignoreWaypoints = true;

var deHash = function (hashString) {
  return hashString.slice(1);
};

Session.setDefault('urlHash', location.hash);
$(window).on('hashchange', function () {
  Session.set('urlHash', location.hash);
});

// need an actual function to only debounce inside the if statement
var actuallyUpdateUrl = _.debounce(function (el) {
  var docsType = Session.get("fullApi") ? "full" : "basic";
  ignoreUrlChange = true;
  window.location.replace("#/" + docsType + "/" + el.id);
}, 100);

var updateUrlFromWaypoint = function (el) {
  if (! ignoreWaypoints) {
    actuallyUpdateUrl(el);
  }
};

Meteor.startup(function () {
  Tracker.autorun(function () {
    var docsType = Session.get("fullApi") ? "full" : "basic";
    Tracker.afterFlush(function () {
      $.waypoints('destroy');

      setTimeout(function () {
        $('.main-content [id]').each(function (i, el) {
          if (! $("#nav [href='#/" + docsType + '/' + el.id + "']").get(0)) {
            // only add waypoints to things that have sidebar links
            return;
          }

          $(el).waypoint(function() {
            updateUrlFromWaypoint(this);
          }, { context: $('.main-content') });
        });
      }, 0);
    });
  });
});

Tracker.autorun(function () {
  // returns a "location" like object with all of the url parts
  var current = Session.get('urlHash');

  // If the URL changes from a waypoint, do nothing
  if (ignoreUrlChange) {
    ignoreUrlChange = false;
    return;
  }

  // If the URL changes, close the sidebar
  Session.set("sidebarOpen", false);

  // redirect routes with no trailing slash
  if (current === "#/basic") {
    navigate("#/basic/");
    return;
  } else if (current === "#/full") {
    navigate("#/full/");
    return;
  }

  if (current.match(/^#\/basic\//)) {
    Session.set("fullApi", false);
  } else if (current.match(/^#\/full\//)) {
    Session.set("fullApi", true);
  } else {
    if (current) {
      // XXX COMPAT WITH old docs
      navigate("#/full/" + deHash(current));
    } else {
      navigate("#/basic/");
    }
    return;
  }

  Tracker.afterFlush(function () {
    setTimeout(function () {
      var targetLocation;

      var id = current.split('/')[2];

      if (! id) {
        // will scroll to the very top
        id = "main";
      }

      var selector = '#' + id;
      var foundElement = $(selector);
      if (foundElement.get(0)) {
        targetLocation = $(".main-content").scrollTop() + foundElement.offset().top - $(".main-content").offset().top;
      }

      ignoreWaypoints = true;
      $(".main-content").animate({
          scrollTop: targetLocation
      }, 500, function () {
        ignoreWaypoints = false;
      });
    }, 0);
  });
});
<commit_msg>Remove weird 'underscore' waypoints leftovers<commit_after>var ignoreUrlChange = false;
var ignoreWaypoints = true;

var deHash = function (hashString) {
  return hashString.slice(1);
};

Session.setDefault('urlHash', location.hash);
$(window).on('hashchange', function () {
  Session.set('urlHash', location.hash);
});

// need an actual function to only debounce inside the if statement
var actuallyUpdateUrl = _.debounce(function (el) {
  var docsType = Session.get("fullApi") ? "full" : "basic";
  ignoreUrlChange = true;
  window.location.replace("#/" + docsType + "/" + el.id);
}, 100);

var updateUrlFromWaypoint = function (el) {
  if (! ignoreWaypoints) {
    actuallyUpdateUrl(el);
  }
};

Meteor.startup(function () {
  Tracker.autorun(function () {
    var docsType = Session.get("fullApi") ? "full" : "basic";
    Tracker.afterFlush(function () {
      $.waypoints('destroy');

      setTimeout(function () {
        $('.main-content :not(.hidden) [id]').each(function (i, el) {
          if (! $("#nav [href='#/" + docsType + '/' + el.id + "']").get(0)) {
            // only add waypoints to things that have sidebar links
            return;
          }

          $(el).waypoint(function() {
            updateUrlFromWaypoint(this);
          }, { context: $('.main-content') });
        });
      }, 0);
    });
  });
});

Tracker.autorun(function () {
  // returns a "location" like object with all of the url parts
  var current = Session.get('urlHash');

  // If the URL changes from a waypoint, do nothing
  if (ignoreUrlChange) {
    ignoreUrlChange = false;
    return;
  }

  // If the URL changes, close the sidebar
  Session.set("sidebarOpen", false);

  // redirect routes with no trailing slash
  if (current === "#/basic") {
    navigate("#/basic/");
    return;
  } else if (current === "#/full") {
    navigate("#/full/");
    return;
  }

  if (current.match(/^#\/basic\//)) {
    Session.set("fullApi", false);
  } else if (current.match(/^#\/full\//)) {
    Session.set("fullApi", true);
  } else {
    if (current) {
      // XXX COMPAT WITH old docs
      navigate("#/full/" + deHash(current));
    } else {
      navigate("#/basic/");
    }
    return;
  }

  Tracker.afterFlush(function () {
    setTimeout(function () {
      var targetLocation;

      var id = current.split('/')[2];

      if (! id) {
        // will scroll to the very top
        id = "main";
      }

      var selector = '#' + id;
      var foundElement = $(selector);
      if (foundElement.get(0)) {
        targetLocation = $(".main-content").scrollTop() + foundElement.offset().top - $(".main-content").offset().top;
      }

      ignoreWaypoints = true;
      $(".main-content").animate({
          scrollTop: targetLocation
      }, 500, function () {
        ignoreWaypoints = false;
      });
    }, 0);
  });
});
<|endoftext|>