<commit_before>
// inspired by https://github.com/NYTimes/Emphasis

import SparkMD5 from 'spark-md5';
import sbd from 'sbd';
import shortid from 'shortid';
import levenshtein from 'fast-levenshtein';
import escapeRegex from 'escape-regex-string';

const TEXT_NODE = 3;
const SHOW_TEXT = 4;
// welcome to a world of horrors
// this is here because I don't trust \s to be correct across browsers
// TODO: if we can ascertain that it is reliable enough, or make Babel transforms it correctly with /u
// inline this whole mess
const SPACE = '[ \\f\\n\\r\\t\\v\\u00a0\\u1680\\u180e\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000\\ufeff]';
const NOT_SPACE = SPACE.replace('[', '[^');
const RE_ONLY_SPACE = new RegExp('^' + SPACE + '+$');
const RE_SPACES = new RegExp(SPACE + '+');
const RE_SPACES_CAPTURE = new RegExp('(' + SPACE + '+)');
const RE_SPACES_GLOBAL = new RegExp(SPACE + '+', 'g');
const RE_SPACE_GLOBAL = new RegExp(SPACE, 'g');
const RE_NOT_SPACES = new RegExp(NOT_SPACE + '+');
const RE_TRIM_LEFT = new RegExp('^' + SPACE + '*');
const RE_TRIM_RIGHT = new RegExp(SPACE + '*$');
const RE_TRIM_LEFT_CAPTURE = new RegExp('^(' + SPACE + '*)');
const RE_TRIM_RIGHT_CAPTURE = new RegExp('(' + SPACE + '*)$');
const RE_TRIM = new RegExp('^' + SPACE + '*|' + SPACE + '*$', 'g');
const leftTrim = (str) => String(str).replace(RE_TRIM_LEFT, '');
const rightTrim = (str) => String(str).replace(RE_TRIM_RIGHT, '');
const trim = (str) => String(str).replace(RE_TRIM, '');
export let citeable = ['P', 'LI', 'DD', 'DT', 'BLOCKQUOTE', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6',
                       'FIGCAPTION', 'CAPTION', 'ASIDE', 'SECTION', 'ARTICLE', 'BODY', 'DIV', 'MAIN']
;

/**
 * From a block element, generate a Key
 * - Break block element text content into Sentences
 * - Take first and last Sentences
 * - Sometimes the same. That's ok.
 * - First character from the first three words of each sentence
 * - Each 6 char key refers to specific block element
 */
export function createKey ($el) {
  let key = '';
  let len = 6;
  let txt = trim(($el.textContent || '').replace(/[^\w\. ]+/gui, ''));

  if (txt && txt.length > 1) {
    let lines = sbd.sentences(txt).map(x => trim(x)).filter(x => x);
    if (lines.length) {
      let first = lines[0].match(/\S+/gu).slice(0, (len/2));
      let last = lines[lines.length-1].match(/\S+/gu).slice(0, (len/2));
      let k = first.concat(last);

      let max = (k.length > len) ? len : k.length;

      for (var i=0; i < max; i++) {
        key += k[i].substring(0, 1);
      }
    }
  }

  return key;
}

// create a md5 hash for the trimmed content of the given element
// XXX broken: normalise the text
export function createHash ($el) {
  return SparkMD5.hash(trim($el.textContent));
}

// given a range, find the enclosing block element that is part of our whitelist
export function getScope (range) {
  var $scope = range.commonAncestorContainer;
  if ($scope.nodeType === TEXT_NODE) {
    $scope = $scope.parentElement; //get closest Element
  };

  // get closest citeable element
  while (!~citeable.indexOf($scope.tagName)) {
    $scope = $scope.parentElement;
    if ($scope.tagName === 'HTML') {
      return;
    }
  }
  return $scope;
}

// given a range and a scope, returns a data structure with all the details needed to reconstruct it
export function serializeRange (range, $scope = getScope(range)) {
  if (!$scope) return;

  let offsets = getOffsets(range, $scope);

  return {
    $scope: $scope,
    sha1: createHash($scope),
    key: createKey($scope),
    startOffset: offsets.startOffset,
    endOffset: offsets.endOffset,
    text: trim(range.toString())
  };
}

// serialise the selection as a range
export function serializeSelection () {
  let selection = window.getSelection();
  if (!selection.isCollapsed) return serializeRange(selection.getRangeAt(0));
}

// given a scope and start/end offsets that ignore white space, returns a range that takes the
// white space into account and captures that content
// XXX broken: needs to use the offset helpers
export function rangeFromOffsets ($scope, startOffset, endOffset) {
  let node
  ,   it = document.createNodeIterator($scope, SHOW_TEXT, null, true)
  ,   acc = 0
  ,   startNode, endNode, relStartOffset, relEndOffset
  ;

  // XXX
  // is it all supposed to only work with trimming?
  // this algorithm only works with trimming
  while (node = it.nextNode()) {
    // let textContent = node.textContent.trim();
    let textContent = node.textContent.replace(/\s+/g, '');
    // console.log('textContent', textContent, acc, textContent.length, startOffset);
    if (relStartOffset === undefined && ((acc + textContent.length) >= startOffset)) {
      startNode = node;
      // console.log('startNode', node);
      // we add back the effect of having trimmed the textContent
      relStartOffset = Math.max(0, startOffset - acc + (startNode.textContent.length -
                                                        startNode.textContent.replace(/^\s+/, '').length));
      // console.log('start offset', relStartOffset, startOffset, acc, startNode.textContent.length, startNode.textContent.replace(/^\s+/, '').length);
    }
    if (relEndOffset === undefined && ((acc + textContent.length) >= endOffset)) {
      endNode = node;
      // console.log('endNode', node);
      // ditto
      relEndOffset = Math.max(0, endOffset - acc + (endNode.textContent.length -
                                                    endNode.textContent.replace(/^\s+/, '').length));
      console.log('end offset', relEndOffset, endOffset, acc, endNode.textContent.length, endNode.textContent.replace(/^\s+/, '').length);
      break;
    }
    acc += textContent.length;
  }
  // console.log(relStartOffset, relEndOffset);

  var range = document.createRange();
  range.setStart(startNode, relStartOffset);
  range.setEnd(endNode, relEndOffset);

  return range;
};

// given a target key and a list of candidates, find either an exact match, or one with the smallest
// Levenshtein edit distance. If no candidate has an edit distance less than three, the match is
// undefined.
export function findKey (target, candidates) {
  let x = { index: undefined, value: undefined, lev: undefined };

  for (let i = 0; i < candidates.length; i++) {
    if (target === candidates[i]) {
      return { index: i, value: candidates[i], lev: 0 };
    }
    // look for 1st closest Match
    else {
      let ls = levenshtein.get(target.slice(0, 3), candidates[i].slice(0, 3));
      let le = levenshtein.get(target.slice(-3), candidates[i].slice(-3));
      let lev = ls + le;
      if (lev < 3 && ((x.lev === undefined) || (lev < x.lev)) ) {
        x.index = i;
        x.value = candidates[i];
        x.lev = lev;
      }
    }
  }

  return x;
};

// Given a document object, add data-id, data-hash, and data-key to all citeables.
// Only call this if you have control over the document's lifecycle.
export function addIdentifiers ($doc) {
  Array.prototype.forEach.call($doc.body.getElementsByTagName('*'), $el => {
    if (~citeable.indexOf($el.tagName)) {
      $el.setAttribute('data-id', shortid.generate());
      $el.setAttribute('data-hash', createHash($el));
      $el.setAttribute('data-key', createKey($el));
    }
  });
  return $doc;
};

function textNodeFromNode (container) {
    if (container.nodeType === TEXT_NODE) return container;
    for (let i = 0; i < container.childNodes.length; i++) {
      if (container.childNodes[i].nodeType === TEXT_NODE) return container.childNodes[i];
  }
}

function trimmedLength (nodes, index) {
  return nodes.slice(0, index).reduce(function (a, b) {
    return a + trim(b.textContent).length;
  }, 0);
}

// Given a range and an element scope, return the start and end offsets into the text that ignore
// white space.
// XXX broken: needs to use the offset helpers
export function getOffsets (range, $scope) {
  let startTextNode = textNodeFromNode(range.startContainer)
  ,   endTextNode = textNodeFromNode(range.endContainer)
  ;

  let node, indStartTextNode, indEndTextNode, ind = 0, textNodes = []
  ,   it = document.createNodeIterator($scope, SHOW_TEXT, null, true);
  while (node = it.nextNode()) {
    textNodes.push(node);
    if (node === startTextNode) {
      indStartTextNode = ind;
    }
    if (node === endTextNode) {
      indEndTextNode = ind;
      break;
    }
    ind++;
  }

  // get the offset without taking white space into account
  let trimmedOffset = (nodes, index, rangeOffset, anchorNode) => {
      let baseOffset = trimmedLength(nodes, index);
      // we subtract the effect of having trimmed the textContent
      if (rangeOffset !== undefined) {
        baseOffset += rangeOffset - (anchorNode.textContent.length -
                                     anchorNode.textContent.replace(/^\s+/, '').length);
      }
      return Math.max(0, baseOffset);
    }
  , startOffset = trimmedOffset(textNodes, indStartTextNode, range.startOffset, startTextNode)
  , endOffset = trimmedOffset(textNodes, indEndTextNode, range.endOffset, endTextNode)
  ;

  return { startOffset: startOffset, endOffset: endOffset };
}

// XXX broken: ditto
export function getChildOffsets ($parent, $child) {
  let startTextNode = textNodeFromNode($child);

  let node, indStartTextNode, ind = 0, textNodes = [];
  let it = document.createNodeIterator($parent, SHOW_TEXT, null, true);
  while (node = it.nextNode()) {
    textNodes.push(node);
    if (node === startTextNode) {
      indStartTextNode = ind;
      break;
    }
    ind++;
  }

  let startOffset = trimmedLength(textNodes, indStartTextNode);
  let endOffset = trimmedLength(textNodes, indStartTextNode + 1);
  return { startOffset: startOffset, endOffset: endOffset };
};

// given a scope and a string, it will find all instances of that string within the
// scope, and use that to create white-space-independent ranges
export function getRangesFromText ($scope, text) {
  // make the text safe to search, but spaces in it need to match \s+
  text = escapeRegex(trim(text)).replace(RE_SPACES_GLOBAL, SPACE + '+');
  let re = new RegExp(text, 'gi')
    , tc = $scope.textContent
  ;
  //  We need to get an offset that ignores whitespace, BUT the regex needs to match
  //  if it contains whitespace.
  // So we:
  //  get all match indices while trimming *nothing*;
  //  then, for each index, remove the amount of WS that is *before* it, get the length of the
  //  match (so that any space becomes a match for \s+), and remove the space from the length of the
  //  match
  let result, matchIndexes = [];
  while ((result = re.exec(tc)) !== null) {
    matchIndexes.push({ index: result.index, length: result[0].length - (result[0].match(RE_SPACE_GLOBAL) || []).length });
  }

  return matchIndexes.map(function (match) {
    match.index -= (tc.substr(0, match.index).match(RE_SPACE_GLOBAL) || []).length;
    console.log('numbers are', match.index, match.length);
    return rangeFromOffsets($scope, match.index, match.index + match.length);
  });
};

// returns text that has been trimmed and with all white space normalised to space
export function normalizeText (text) {
  return trim(text).replace(RE_SPACES, ' ');
}

// Takes a raw offset into a raw text and returns the offset of the same character in a normalised
// text.
export function normalizeOffset (rawOffset, rawText) {
  let workText = rawText.substring(0, rawOffset);
  // we have to special-case the leading space because trim() does more than \s
  // the length different once left-trim and space normalisation have happened
  let delta = workText.length - workText.replace(RE_TRIM_LEFT, '').replace(RE_SPACES, ' ').length;
  return rawOffset - delta;
}

// Takes a normalised offset and a raw text, and returns the corresponding raw offset
export function denormalizeOffset (normOffset, rawText) {
  normOffset = parseInt(normOffset, 10);
  // process the text into blocks that are either:
  //  - simple non-white-space text
  //  - simple white space
  // in each case the block is given a normalised length, which is how long it would be in normal
  // text, and a raw length, which is its actual length. The first and last white space blocks gets
  // normal lengths of 0, others of 1.
  // then for each block, if the normal offset would be in the block we compute it, otherwise we
  // increment our raw and normal offsets and move to the next block
  let leftTrimMatch = rawText.match(RE_TRIM_LEFT_CAPTURE)
    , rightTrimMatch = rawText.match(RE_TRIM_RIGHT_CAPTURE)
  ;
  rawText = trim(rawText);
  let blocks = rawText.split(RE_SPACES_CAPTURE).map(b => {
    return { rawLength: b.length, normalLength: RE_ONLY_SPACE.test(b) ? 1 : b.length };
  });
  blocks.unshift({ rawLength: leftTrimMatch[1].length, normalLength: 0 });
  blocks.push({ rawLength: rightTrimMatch[1].length, normalLength: 0 });

  let rawOffset = 0
    , normalisingOffset = 0
  ;
  for (var i = 0; i < blocks.length; i++) {
    let block = blocks[i];
    if (block.normalLength + normalisingOffset >= normOffset) {
      return rawOffset + (block.rawLength - block.normalLength) + (normOffset - normalisingOffset);
    }
    else {
      rawOffset += block.rawLength;
      normalisingOffset += block.normalLength;
    }
  }
  return rawOffset;
}

if (typeof window === 'object') window.WebVerse = exports;
<commit_msg>doc tweak<commit_after>
// inspired by https://github.com/NYTimes/Emphasis

import SparkMD5 from 'spark-md5';
import sbd from 'sbd';
import shortid from 'shortid';
import levenshtein from 'fast-levenshtein';
import escapeRegex from 'escape-regex-string';

const TEXT_NODE = 3;
const SHOW_TEXT = 4;
// welcome to a world of horrors
// this is here because I don't trust \s to be correct across browsers
// TODO: if we can ascertain that it is reliable enough, or make Babel transforms it correctly with /u
// inline this whole mess
const SPACE = '[ \\f\\n\\r\\t\\v\\u00a0\\u1680\\u180e\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000\\ufeff]';
const NOT_SPACE = SPACE.replace('[', '[^');
const RE_ONLY_SPACE = new RegExp('^' + SPACE + '+$');
const RE_SPACES = new RegExp(SPACE + '+');
const RE_SPACES_CAPTURE = new RegExp('(' + SPACE + '+)');
const RE_SPACES_GLOBAL = new RegExp(SPACE + '+', 'g');
const RE_SPACE_GLOBAL = new RegExp(SPACE, 'g');
const RE_NOT_SPACES = new RegExp(NOT_SPACE + '+');
const RE_TRIM_LEFT = new RegExp('^' + SPACE + '*');
const RE_TRIM_RIGHT = new RegExp(SPACE + '*$');
const RE_TRIM_LEFT_CAPTURE = new RegExp('^(' + SPACE + '*)');
const RE_TRIM_RIGHT_CAPTURE = new RegExp('(' + SPACE + '*)$');
const RE_TRIM = new RegExp('^' + SPACE + '*|' + SPACE + '*$', 'g');
const leftTrim = (str) => String(str).replace(RE_TRIM_LEFT, '');
const rightTrim = (str) => String(str).replace(RE_TRIM_RIGHT, '');
const trim = (str) => String(str).replace(RE_TRIM, '');
export let citeable = ['P', 'LI', 'DD', 'DT', 'BLOCKQUOTE', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6',
                       'FIGCAPTION', 'CAPTION', 'ASIDE', 'SECTION', 'ARTICLE', 'BODY', 'DIV', 'MAIN']
;

/**
 * From a block element, generate a Key
 * - Break block element text content into Sentences
 * - Take first and last Sentences
 * - Sometimes the same. That's ok.
 * - First character from the first three words of each sentence
 * - Each 6 char key refers to specific block element
 */
export function createKey ($el) {
  let key = '';
  let len = 6;
  let txt = trim(($el.textContent || '').replace(/[^\w\. ]+/gui, ''));

  if (txt && txt.length > 1) {
    let lines = sbd.sentences(txt).map(x => trim(x)).filter(x => x);
    if (lines.length) {
      let first = lines[0].match(/\S+/gu).slice(0, (len/2));
      let last = lines[lines.length-1].match(/\S+/gu).slice(0, (len/2));
      let k = first.concat(last);

      let max = (k.length > len) ? len : k.length;

      for (var i=0; i < max; i++) {
        key += k[i].substring(0, 1);
      }
    }
  }

  return key;
}

// create a md5 hash for the trimmed content of the given element
// XXX broken: normalise the text
export function createHash ($el) {
  return SparkMD5.hash(trim($el.textContent));
}

// given a range, find the enclosing block element that is part of our whitelist
export function getScope (range) {
  var $scope = range.commonAncestorContainer;
  if ($scope.nodeType === TEXT_NODE) {
    $scope = $scope.parentElement; //get closest Element
  };

  // get closest citeable element
  while (!~citeable.indexOf($scope.tagName)) {
    $scope = $scope.parentElement;
    if ($scope.tagName === 'HTML') {
      return;
    }
  }
  return $scope;
}

// given a range and a scope, returns a data structure with all the details needed to reconstruct it
export function serializeRange (range, $scope = getScope(range)) {
  if (!$scope) return;

  let offsets = getOffsets(range, $scope);

  return {
    $scope: $scope,
    sha1: createHash($scope),
    key: createKey($scope),
    startOffset: offsets.startOffset,
    endOffset: offsets.endOffset,
    text: trim(range.toString())
  };
}

// serialise the selection as a range
export function serializeSelection () {
  let selection = window.getSelection();
  if (!selection.isCollapsed) return serializeRange(selection.getRangeAt(0));
}

// given a scope and start/end offsets that ignore white space, returns a range that takes the
// white space into account and captures that content
// XXX broken: needs to use the offset helpers
export function rangeFromOffsets ($scope, startOffset, endOffset) {
  let node
  ,   it = document.createNodeIterator($scope, SHOW_TEXT, null, true)
  ,   acc = 0
  ,   startNode, endNode, relStartOffset, relEndOffset
  ;

  // XXX
  // is it all supposed to only work with trimming?
  // this algorithm only works with trimming
  while (node = it.nextNode()) {
    // let textContent = node.textContent.trim();
    let textContent = node.textContent.replace(/\s+/g, '');
    // console.log('textContent', textContent, acc, textContent.length, startOffset);
    if (relStartOffset === undefined && ((acc + textContent.length) >= startOffset)) {
      startNode = node;
      // console.log('startNode', node);
      // we add back the effect of having trimmed the textContent
      relStartOffset = Math.max(0, startOffset - acc + (startNode.textContent.length -
                                                        startNode.textContent.replace(/^\s+/, '').length));
      // console.log('start offset', relStartOffset, startOffset, acc, startNode.textContent.length, startNode.textContent.replace(/^\s+/, '').length);
    }
    if (relEndOffset === undefined && ((acc + textContent.length) >= endOffset)) {
      endNode = node;
      // console.log('endNode', node);
      // ditto
      relEndOffset = Math.max(0, endOffset - acc + (endNode.textContent.length -
                                                    endNode.textContent.replace(/^\s+/, '').length));
      console.log('end offset', relEndOffset, endOffset, acc, endNode.textContent.length, endNode.textContent.replace(/^\s+/, '').length);
      break;
    }
    acc += textContent.length;
  }
  // console.log(relStartOffset, relEndOffset);

  var range = document.createRange();
  range.setStart(startNode, relStartOffset);
  range.setEnd(endNode, relEndOffset);

  return range;
};

// given a target key and a list of candidates, find either an exact match, or one with the smallest
// Levenshtein edit distance. If no candidate has an edit distance less than three, the match is
// undefined.
export function findKey (target, candidates) {
  let x = { index: undefined, value: undefined, lev: undefined };

  for (let i = 0; i < candidates.length; i++) {
    if (target === candidates[i]) {
      return { index: i, value: candidates[i], lev: 0 };
    }
    // look for 1st closest Match
    else {
      let ls = levenshtein.get(target.slice(0, 3), candidates[i].slice(0, 3));
      let le = levenshtein.get(target.slice(-3), candidates[i].slice(-3));
      let lev = ls + le;
      if (lev < 3 && ((x.lev === undefined) || (lev < x.lev)) ) {
        x.index = i;
        x.value = candidates[i];
        x.lev = lev;
      }
    }
  }

  return x;
};

// Given a document object, add data-id, data-hash, and data-key to all citeables.
// Only call this if you have control over the document's lifecycle.
export function addIdentifiers ($doc) {
  Array.prototype.forEach.call($doc.body.getElementsByTagName('*'), $el => {
    if (~citeable.indexOf($el.tagName)) {
      $el.setAttribute('data-id', shortid.generate());
      $el.setAttribute('data-hash', createHash($el));
      $el.setAttribute('data-key', createKey($el));
    }
  });
  return $doc;
};

function textNodeFromNode (container) {
    if (container.nodeType === TEXT_NODE) return container;
    for (let i = 0; i < container.childNodes.length; i++) {
      if (container.childNodes[i].nodeType === TEXT_NODE) return container.childNodes[i];
  }
}

function trimmedLength (nodes, index) {
  return nodes.slice(0, index).reduce(function (a, b) {
    return a + trim(b.textContent).length;
  }, 0);
}

// Given a range and an element scope, return the start and end offsets into the text that ignore
// white space.
// XXX broken: needs to use the offset helpers
export function getOffsets (range, $scope) {
  let startTextNode = textNodeFromNode(range.startContainer)
  ,   endTextNode = textNodeFromNode(range.endContainer)
  ;

  let node, indStartTextNode, indEndTextNode, ind = 0, textNodes = []
  ,   it = document.createNodeIterator($scope, SHOW_TEXT, null, true);
  while (node = it.nextNode()) {
    textNodes.push(node);
    if (node === startTextNode) {
      indStartTextNode = ind;
    }
    if (node === endTextNode) {
      indEndTextNode = ind;
      break;
    }
    ind++;
  }

  // get the offset without taking white space into account
  let trimmedOffset = (nodes, index, rangeOffset, anchorNode) => {
      let baseOffset = trimmedLength(nodes, index);
      // we subtract the effect of having trimmed the textContent
      if (rangeOffset !== undefined) {
        baseOffset += rangeOffset - (anchorNode.textContent.length -
                                     anchorNode.textContent.replace(/^\s+/, '').length);
      }
      return Math.max(0, baseOffset);
    }
  , startOffset = trimmedOffset(textNodes, indStartTextNode, range.startOffset, startTextNode)
  , endOffset = trimmedOffset(textNodes, indEndTextNode, range.endOffset, endTextNode)
  ;

  return { startOffset: startOffset, endOffset: endOffset };
}

// XXX broken: ditto
export function getChildOffsets ($parent, $child) {
  let startTextNode = textNodeFromNode($child);

  let node, indStartTextNode, ind = 0, textNodes = [];
  let it = document.createNodeIterator($parent, SHOW_TEXT, null, true);
  while (node = it.nextNode()) {
    textNodes.push(node);
    if (node === startTextNode) {
      indStartTextNode = ind;
      break;
    }
    ind++;
  }

  let startOffset = trimmedLength(textNodes, indStartTextNode);
  let endOffset = trimmedLength(textNodes, indStartTextNode + 1);
  return { startOffset: startOffset, endOffset: endOffset };
};

// given a scope and a string, it will find all instances of that string within the
// scope, and use that to create white-space-independent ranges
// XXX broken: use offset helpers
export function getRangesFromText ($scope, text) {
  // make the text safe to search, but spaces in it need to match \s+
  text = escapeRegex(trim(text)).replace(RE_SPACES_GLOBAL, SPACE + '+');
  let re = new RegExp(text, 'gi')
    , tc = $scope.textContent
  ;
  //  We need to get an offset that ignores whitespace, BUT the regex needs to match
  //  if it contains whitespace.
  // So we:
  //  get all match indices while trimming *nothing*;
  //  then, for each index, remove the amount of WS that is *before* it, get the length of the
  //  match (so that any space becomes a match for \s+), and remove the space from the length of the
  //  match
  let result, matchIndexes = [];
  while ((result = re.exec(tc)) !== null) {
    matchIndexes.push({ index: result.index, length: result[0].length - (result[0].match(RE_SPACE_GLOBAL) || []).length });
  }

  return matchIndexes.map(function (match) {
    match.index -= (tc.substr(0, match.index).match(RE_SPACE_GLOBAL) || []).length;
    console.log('numbers are', match.index, match.length);
    return rangeFromOffsets($scope, match.index, match.index + match.length);
  });
};

// returns text that has been trimmed and with all white space normalised to space
export function normalizeText (text) {
  return trim(text).replace(RE_SPACES, ' ');
}

// Takes a raw offset into a raw text and returns the offset of the same character in a normalised
// text.
export function normalizeOffset (rawOffset, rawText) {
  let workText = rawText.substring(0, rawOffset);
  // the length difference once left-trim and space normalisation have happened
  let delta = workText.length - workText.replace(RE_TRIM_LEFT, '').replace(RE_SPACES, ' ').length;
  return rawOffset - delta;
}

// Takes a normalised offset and a raw text, and returns the corresponding raw offset
export function denormalizeOffset (normOffset, rawText) {
  normOffset = parseInt(normOffset, 10);
  // process the text into blocks that are either:
  //  - simple non-white-space text
  //  - simple white space
  // in each case the block is given a normalised length, which is how long it would be in normal
  // text, and a raw length, which is its actual length. The first and last white space blocks gets
  // normal lengths of 0, others of 1.
  // then for each block, if the normal offset would be in the block we compute it, otherwise we
  // increment our raw and normal offsets and move to the next block
  let leftTrimMatch = rawText.match(RE_TRIM_LEFT_CAPTURE)
    , rightTrimMatch = rawText.match(RE_TRIM_RIGHT_CAPTURE)
  ;
  rawText = trim(rawText);
  let blocks = rawText.split(RE_SPACES_CAPTURE).map(b => {
    return { rawLength: b.length, normalLength: RE_ONLY_SPACE.test(b) ? 1 : b.length };
  });
  blocks.unshift({ rawLength: leftTrimMatch[1].length, normalLength: 0 });
  blocks.push({ rawLength: rightTrimMatch[1].length, normalLength: 0 });

  let rawOffset = 0
    , normalisingOffset = 0
  ;
  for (var i = 0; i < blocks.length; i++) {
    let block = blocks[i];
    if (block.normalLength + normalisingOffset >= normOffset) {
      return rawOffset + (block.rawLength - block.normalLength) + (normOffset - normalisingOffset);
    }
    else {
      rawOffset += block.rawLength;
      normalisingOffset += block.normalLength;
    }
  }
  return rawOffset;
}

if (typeof window === 'object') window.WebVerse = exports;
<|endoftext|>


<commit_before>/* global process */
/* eslint no-console: 0 */

'use strict'

process.env.FORCE_COLOR = true // Force colors for packages that depend on https://www.npmjs.com/package/supports-color

const path = require('path')
const sgf = require('staged-git-files')
const appRoot = require('app-root-path')
const Listr = require('listr')
const cosmiconfig = require('cosmiconfig')

const packageJson = require(appRoot.resolve('package.json')) // eslint-disable-line
const runScript = require('./runScript')
const resolvePaths = require('./resolvePaths')
const generateTasks = require('./generateTasks')

cosmiconfig(packageJson.name, {
    rc: 'lintstaged'
})
    .then((result) => {
        // result.config is the parsed configuration object
        // result.filepath is the path to the config file that was found
        const config = result.config
        const concurrent = config.concurrent || true

        // If git-root is defined -> set git root as sgf's cwd
        if ('git-root' in config) {
            sgf.cwd = path.resolve(config['git-root'])
        }

        sgf('ACM', (err, files) => {
            if (err) {
                console.error(err)
            }

            const tasks = generateTasks(config, resolvePaths(files))
                .map(task => ({
                    title: `Running tasks for ${ task.pattern }`,
                    task: () => (new Listr(runScript(task.commands, task.fileList, packageJson)))
                }))

            if (tasks.length) {
                new Listr(tasks, { concurrent }).run().catch((error) => {
                    console.error(error)
                    process.exit(1)
                })
            }
        })
    })
    .catch((parsingError) => {
        console.error(`Could not parse ${ packageJson.name } config.
Make sure you have created it. See ${ packageJson.homepage }.

${ parsingError }
`)
        process.exit(1)
    })

<commit_msg>Fixed rc file name<commit_after>/* global process */
/* eslint no-console: 0 */

'use strict'

process.env.FORCE_COLOR = true // Force colors for packages that depend on https://www.npmjs.com/package/supports-color

const path = require('path')
const sgf = require('staged-git-files')
const appRoot = require('app-root-path')
const Listr = require('listr')
const cosmiconfig = require('cosmiconfig')

const packageJson = require(appRoot.resolve('package.json')) // eslint-disable-line
const runScript = require('./runScript')
const resolvePaths = require('./resolvePaths')
const generateTasks = require('./generateTasks')

cosmiconfig(packageJson.name, {
    rc: '.lintstagedrc'
})
    .then((result) => {
        // result.config is the parsed configuration object
        // result.filepath is the path to the config file that was found
        const config = result.config
        const concurrent = config.concurrent || true

        // If git-root is defined -> set git root as sgf's cwd
        if ('git-root' in config) {
            sgf.cwd = path.resolve(config['git-root'])
        }

        sgf('ACM', (err, files) => {
            if (err) {
                console.error(err)
            }

            const tasks = generateTasks(config, resolvePaths(files))
                .map(task => ({
                    title: `Running tasks for ${ task.pattern }`,
                    task: () => (new Listr(runScript(task.commands, task.fileList, packageJson)))
                }))

            if (tasks.length) {
                new Listr(tasks, { concurrent }).run().catch((error) => {
                    console.error(error)
                    process.exit(1)
                })
            }
        })
    })
    .catch((parsingError) => {
        console.error(`Could not parse ${ packageJson.name } config.
Make sure you have created it. See ${ packageJson.homepage }.

${ parsingError }
`)
        process.exit(1)
    })

<|endoftext|>


<commit_before>var request = require('request');
var endsWith = require("ends-with")
var escape = require('escape-html');
var getType = require("./analysis")
var fs = require("fs")
var _ = require("lodash")
const path = require('path');
var mkdirp = require('mkdirp')

// not used anymore, using chrome extension to intercept request
var rewriteHtml = require("./rewriteHtml")

var connect = require('connect');
var http = require('http');
var bodyParser = require('body-parser')
var url = require("url")

var program = require('commander');

process.title = "Glasswing Server"

program
  .version('1.0.0')
  .option('-p, --port [port]', 'Glasswing port - not supported yet', 9500)
  .option('-p, --save [path]', 'Directory to save data in')
  .parse(process.argv);

var port = program.port

var app = connect();

var Compiler = require("./Compiler")
var compiler = new Compiler()

var level = require('level')

function KeyValueStore(storeIn){
    console.log("creating lvel db", storeIn)
    this.db = level(storeIn)
}
KeyValueStore.prototype.put = function(key, value, callback){
    this.db.put(key, value, function (err) {
        if (err) {
            console.log("LevelDB put err", err)
        }
        callback()
    })
}
KeyValueStore.prototype.get = function(key, callback){
    this.db.get(key, function (err, value) {
        if (err) {

        }
        callback(value)
    })
}

function beautifyJS(code){
    const prettier = require("prettier");
    const options= {
    // Indent lines with tabs
    useTabs: false,

    // Fit code within this line limit
    printWidth: 80,

    // Number of spaces it should use per tab
    tabWidth: 2,

    // If true, will use single instead of double quotes
    singleQuote: false,

    // Controls the printing of trailing commas wherever possible. Valid options:
    // "none" - No trailing commas
    // "es5"  - Trailing commas where valid in ES5 (objects, arrays, etc)
    // "all"  - Trailing commas wherever possible (function arguments)
    //
    // NOTE: Above is only available in 0.19.0 and above. Previously this was
    // a boolean argument.
    trailingComma: "none",

    // Controls the printing of spaces inside object literals
    bracketSpacing: true,

    // If true, puts the `>` of a multi-line jsx element at the end of
    // the last line instead of being alone on the next line
    jsxBracketSameLine: false,

    // Which parser to use. Valid options are "flow" and "babylon"
    parser: "babylon",

    // Whether to add a semicolon at the end of every line (semi: true),
    // or only at the beginning of lines that may introduce ASI failures (semi: false)
    semi: true
        }

    try {
        return prettier.format(code, options);
    } catch(err) {
        console.log("Prettier error:", err)
        return code
    }
}

var async = require("async")

function DataStore(options){
    console.log("new datastore with", options.url)
    // random b/c sometiems we have multipel stroes with same id.. shoudl really be same store
    if (!options.dbFileName) {
        this.dbFileName = encodeURIComponent(options.url).slice(-100)
    } else {
        this.dbFileName = options.dbFileName
    }
    this.values = new KeyValueStore(saveToDir + "/" + this.dbFileName)

    this.url = options.url
    this.locations = options.locations
    this.code = options.code
}
DataStore.prototype.reportValue = function(data, callback){
    this.getValues(data.valueId, (values) => {
        values.push(data.value)
        this.values.put(data.valueId.toString(), JSON.stringify(values), callback)
    })
}
DataStore.prototype.getValues = function(valueId, callback){
    this.values.get(valueId, (values) => {
        if (values) {
            values = JSON.parse(values)
        } else {
            values = []
        }
        callback(values)
    })
}
DataStore.prototype.serialize = function(){
    return {
        dbFileName: this.dbFileName,
        url: this.url,
        locations: this.locations,
        code: this.code
    }
}
DataStore.deserialize =  function(data){
    var store = new DataStore(data)
    return store
}

var urlToScriptId = {}

var dataStores = {}
function getDataStore(scriptId){
    return dataStores[scriptId]
}

var scriptIdCounter = 1

const resById = {}

var saveToDir = null
var saveTo = null
if (program.save) {
    saveToDir = program.save
    mkdirp.sync(program.save)
    saveTo = program.save + "/data.json"
} else {
    console.error("--save option is required, pass in a directory like /tmp/glasswing1")
    process.exit()
}

if (saveTo) {
    try {
        var data = JSON.parse(fs.readFileSync(saveTo).toString())
    } catch (err) {}

    if (data) {
        scriptIdCounter = data.scriptIdCounter
        urlToScriptId = data.urlToScriptId
        dataStores = _.mapValues(data.stores, s => {
            return DataStore.deserialize(s)
        })
        console.log("loaded urlToScriptId", urlToScriptId)
    }
}

function pathFromRoot(p){
    return path.join(__dirname + "/../", p)
}

app.use( bodyParser.json({limit: "300mb"}) );
app.use(function(req, res){
    var url = req.url.split("?")[0]

    if (url.indexOf("/node_modules/") !== -1) {
        var filePath = pathFromRoot(url.replace(/\.\./g, ""))
        var fileContent = fs.readFileSync(path.join(__dirname + "/../", url.replace(/\.\./g, "")))
        res.end(fileContent).toString()
        return
    }

    if (url === "/__jscb/bundle.js") {
        res.end(fs.readFileSync(pathFromRoot("src/ui/dist/bundle.js")).toString())
        return
    }

    if (url.indexOf("/__jscb/getValues") !== -1) {
        var parts = url.replace("/__jscb/getValues/", "").split("/").map(parseFloat)
        var scriptId = parts[0]
        var valueId = parts[1]

        var info = getDataStore(scriptId)
        info.getValues(valueId, function(values){
            res.end(JSON.stringify(values))
        })

        return
    }

    if (url.indexOf("/__jscb/getLocations") !== -1) {
        var scriptId = parseFloat(url.replace("/__jscb/getLocations/", ""))

        var info = getDataStore(scriptId)
        res.end(JSON.stringify(info.locations))

        return
    }

    if (url.indexOf("/__jscb/getCode") !== -1) {
        var scriptId = parseFloat(url.replace("/__jscb/getCode/", ""))

        var info = getDataStore(scriptId)
        res.end(info.code)

        return
    }

    if (req.url.indexOf("/browse") !== -1) {
        var url = decodeURIComponent(req.url).replace("/browse?", "")

        console.log("Url", url)
        console.log(urlToScriptId)
        var scriptId = urlToScriptId[url]
        var info = getDataStore(scriptId)
        if (!info){
            res.end("No data for this file has been collected. Load a web page that loads this file")
        } else {
            renderInfo(info, scriptId, text => res.end(text))

        }

    }

    if (url.indexOf("/__jscb/fetchFunctionCode") !== -1) {
        var parts = url.split("/")
        var locationId = parseFloat(parts.pop())
        var scriptId = parseFloat(parts.pop())
        var store = dataStores[scriptId]
        var loc = store.locations[locationId]
        var json = {
            text: store.code.slice(loc.start, loc.end),
            url: "/browse?" + encodeURIComponent(store.url) + "#" + loc.loc.start.line
        }
        res.end(JSON.stringify(json))
        return
    }

    if (url.indexOf("/request") !== -1) {
        var id = url.split("/")[2]
        console.log("request started", id, decodeURIComponent(req.url.split("?")[1]))

        resById[id] = res
        return
    }
    if (url.indexOf("/response") !== -1) {
        var id = url.split("/")[2]
        console.log("Response", id, req.body.url)

        var response = req.body.response
        if (endsWith(req.body.url, ".js") || req.body.requestType === "script") {
            var scriptId = urlToScriptId[req.body.url]
            if (!scriptId){
                scriptId = scriptIdCounter
                scriptIdCounter++
                urlToScriptId[req.body.url] = scriptId
            }

            response = beautifyJS(response)

            var started = new Date()
            var compiled = compiler.compile(response, {
                scriptId
            })
            var ended = new Date()
            var ms = ended.valueOf() - started.valueOf()
            console.log("Compiling " + req.body.url + " took "  + ms + "ms")

            if (!dataStores[scriptId]) {
                dataStores[scriptId] = new DataStore({
                    code: response,
                    locations: compiled.locations,
                    url: req.body.url
                })
            } else {
                console.log("re-using datastore for url", req.body.url)
            }

            response = compiled.code
        }

        var pre = fs.readFileSync(pathFromRoot("src/browser.js")).toString().replace("{{port}}", port) + "\n\n"
        response = pre + response

        if (!req.body.returnProcessedContent) {
            var interval = setInterval(function(){
                if (resById[id]) {

                    resById[id].end(pre + response)
                    clearInterval(interval)
                } else {
                    console.log("no request yet for" + id, "waiting...")
                }

            }, 100)
            res.end("OK")
        } else {
            res.end(response)
        }

        return
    }

    if (url === "/") {
        var html = fs.readFileSync(__dirname + "/ui/home.html").toString()
        console.log("urlToScriptId", urlToScriptId)
        var scriptDataCollected = Object.keys(urlToScriptId).length  > 0
        var fileLinks
        if(scriptDataCollected) {
            fileLinks = Object.keys(urlToScriptId).map(url => {
                console.log(urlToScriptId)
                console.log("Url", url)
                var scriptId = urlToScriptId[url]
                var store = dataStores[scriptId]
                var content = ""
                if (store === undefined) {
                    content = "No value store found"
                } else {
                    var values = Object.keys(store.values).length
                    var locations = Object.keys(store.locations).length
                    // rough percentage b/c funcitonlocations are also locations
                    var roughPercentage = Math.round(values / locations * 100 * 10) /10
                    content = `~${roughPercentage}%`
                }

                return `

                        ${escape(url)}

                        ${content}

                `
            }).join("")
            fileLinks = "
"  +
                `<thead><th>File</th><th style="min-width: 150px">Locations with values</th></thead>`
                + fileLinks + "
"
        } else {
            fileLinks = "
No data collected. Load a website and then click the Glassdoor Chrome extension button in the top right of your browser.
"
        }

        res.end(html.replace("{{fileLinks}}", fileLinks))
        return
    }

    if (url.indexOf("__jscb/reportValues") !== -1) {
        res.setHeader("Access-Control-Allow-Origin", "*")
        res.setHeader("Access-Control-Allow-Headers", "Content-Type, Access-Control-Allow-Headers, Authorization, X-Requested-With")
        if (!req.body.length) {
            res.end()
            return
        }
        console.log("Received " + req.body.length + " values")

        async.eachSeries(req.body, function iteratee(data, callback) {
            var dataStore = getDataStore(data.scriptId)
            dataStore.reportValue(data, callback)
        });

        if (saveTo) {
            var data = {
                stores: _.mapValues(dataStores, s => s.serialize()),
                scriptIdCounter: scriptIdCounter,
                urlToScriptId: urlToScriptId
            }

            var stringifiedData = JSON.stringify(data, null, 4)
            var mb = Math.round(stringifiedData.length / 1024 / 1024)
            console.log("Saving data to " + saveTo + ": " + mb + "MB")
            fs.writeFileSync(saveTo, stringifiedData)
        }

        res.end('{"status": "success"}')
    }
});

function renderInfo(info,scriptId, cb){
    var res = {}

    var valueIds = Object.keys(info.locations)
    // async.eachSeries(valueIds, function iteratee(valueId, callback) {
    //     info.getValues(valueId, function(values){
    //         res[valueId] = {
    //             type: null, // types are out of scope for now
    //             examples: values
    //         }
    //         callback()
    //     })
    // }, function(){
        // });
    console.log("DONE")
        fileName = _.last(info.url.split("/"))

        // window.values = JSON.parse(decodeURI("${encodeURI(JSON.stringify(res))}"));
    var valueEmbeds = `
        window.scriptId =${scriptId}

    `

    cb(fs.readFileSync(__dirname + "/ui/file.html").toString()
        .replace("{{valueEmbeds}}", valueEmbeds)
        .replace("{{fileName}}", fileName))

}

function renderInfoOldUnused(info){
    var m = new MagicString(info.code)
    var errors = []
    Object.keys(info.locations).forEach(function(id){
        var loc = info.locations[id]
        try {
            if (loc.type === "call") {
                m.insertLeft(loc.end, "OPENTAGspan data-value-id='" + id + "' style='background: red; color: white;border-radius: 4px;padding: 2;font-size: 12px'CLOSETAG" +
                    "RET"
                + "OPENTAG/spanCLOSETAG")
            }
            else {
                var end = loc.end
                if (loc.type === "returnStatement") {
                    end = loc.start + "return".length
                }
                m.overwrite(loc.start, end, "OPENTAGspan data-value-id='" + id + "' style='border-bottom: 1px solid red'CLOSETAG" +
                    (loc.type === "returnStatement" ? "return" : info.code.slice(loc.start, loc.end) )

                + "OPENTAG/spanCLOSETAG")
            }
        } catch (err) {
            errors.push(err)
        }
    })

    var res = {}
    Object.keys(info.values).forEach(function(key){
        var values = info.values[key]
        if (values.length === 0) {
            res[key] = null
        } else {
            res[key] = {
                type: null, // types are out of scope for now
                examples: values.slice(0, 1)
            }
        }
    })

    return `
    <meta charset="utf-8" />

${m.toString().replace(/</g, "<").replace(/>/g, ">").replace(/OPENTAG/g, "<").replace(/CLOSETAG/g, ">")}

ERRORS:
${errors.join("
")}

        `
}

//create node.js http server and listen on port
http.createServer(app).listen(port);
console.log("Listening on http://localhost:" + port)
<commit_msg>Improve perf logging.<commit_after>var request = require('request');
var endsWith = require("ends-with")
var escape = require('escape-html');
var getType = require("./analysis")
var fs = require("fs")
var _ = require("lodash")
const path = require('path');
var mkdirp = require('mkdirp')

// not used anymore, using chrome extension to intercept request
var rewriteHtml = require("./rewriteHtml")

var connect = require('connect');
var http = require('http');
var bodyParser = require('body-parser')
var url = require("url")

var program = require('commander');

process.title = "Glasswing Server"

program
  .version('1.0.0')
  .option('-p, --port [port]', 'Glasswing port - not supported yet', 9500)
  .option('-p, --save [path]', 'Directory to save data in')
  .parse(process.argv);

var port = program.port

var app = connect();

var Compiler = require("./Compiler")
var compiler = new Compiler()

var level = require('level')

function KeyValueStore(storeIn){
    console.log("creating lvel db", storeIn)
    this.db = level(storeIn)
}
KeyValueStore.prototype.put = function(key, value, callback){
    this.db.put(key, value, function (err) {
        if (err) {
            console.log("LevelDB put err", err)
        }
        callback()
    })
}
KeyValueStore.prototype.get = function(key, callback){
    this.db.get(key, function (err, value) {
        if (err) {

        }
        callback(value)
    })
}

function beautifyJS(code){
    const prettier = require("prettier");
    const options= {
    // Indent lines with tabs
    useTabs: false,

    // Fit code within this line limit
    printWidth: 80,

    // Number of spaces it should use per tab
    tabWidth: 2,

    // If true, will use single instead of double quotes
    singleQuote: false,

    // Controls the printing of trailing commas wherever possible. Valid options:
    // "none" - No trailing commas
    // "es5"  - Trailing commas where valid in ES5 (objects, arrays, etc)
    // "all"  - Trailing commas wherever possible (function arguments)
    //
    // NOTE: Above is only available in 0.19.0 and above. Previously this was
    // a boolean argument.
    trailingComma: "none",

    // Controls the printing of spaces inside object literals
    bracketSpacing: true,

    // If true, puts the `>` of a multi-line jsx element at the end of
    // the last line instead of being alone on the next line
    jsxBracketSameLine: false,

    // Which parser to use. Valid options are "flow" and "babylon"
    parser: "babylon",

    // Whether to add a semicolon at the end of every line (semi: true),
    // or only at the beginning of lines that may introduce ASI failures (semi: false)
    semi: true
        }

    try {
        return prettier.format(code, options);
    } catch(err) {
        console.log("Prettier error:", err)
        return code
    }
}

var async = require("async")

var perf = {}
function logPerfStart(label){
    perf[label] = new Date()
}
function logPerfEnd(label){
    console.log(label, "took", new Date().valueOf() - perf[label].valueOf(), "ms")
}

function DataStore(options){
    console.log("new datastore with", options.url)
    // random b/c sometiems we have multipel stroes with same id.. shoudl really be same store
    if (!options.dbFileName) {
        this.dbFileName = encodeURIComponent(options.url).slice(-100)
    } else {
        this.dbFileName = options.dbFileName
    }
    this.values = new KeyValueStore(saveToDir + "/" + this.dbFileName)

    this.url = options.url
    this.locations = options.locations
    this.code = options.code
}
DataStore.prototype.reportValue = function(data, callback){
    this.getValues(data.valueId, (values) => {
        values.push(data.value)
        this.values.put(data.valueId.toString(), JSON.stringify(values), callback)
    })
}
DataStore.prototype.getValues = function(valueId, callback){
    this.values.get(valueId, (values) => {
        if (values) {
            values = JSON.parse(values)
        } else {
            values = []
        }
        callback(values)
    })
}
DataStore.prototype.serialize = function(){
    return {
        dbFileName: this.dbFileName,
        url: this.url,
        locations: this.locations,
        code: this.code
    }
}
DataStore.deserialize =  function(data){
    var store = new DataStore(data)
    return store
}

var urlToScriptId = {}

var dataStores = {}
function getDataStore(scriptId){
    return dataStores[scriptId]
}

var scriptIdCounter = 1

const resById = {}

var saveToDir = null
var saveTo = null
if (program.save) {
    saveToDir = program.save
    mkdirp.sync(program.save)
    saveTo = program.save + "/data.json"
} else {
    console.error("--save option is required, pass in a directory like /tmp/glasswing1")
    process.exit()
}

if (saveTo) {
    try {
        var data = JSON.parse(fs.readFileSync(saveTo).toString())
    } catch (err) {}

    if (data) {
        scriptIdCounter = data.scriptIdCounter
        urlToScriptId = data.urlToScriptId
        dataStores = _.mapValues(data.stores, s => {
            return DataStore.deserialize(s)
        })
        console.log("loaded urlToScriptId", urlToScriptId)
    }
}

function pathFromRoot(p){
    return path.join(__dirname + "/../", p)
}

app.use( bodyParser.json({limit: "300mb"}) );
app.use(function(req, res){
    var url = req.url.split("?")[0]

    if (url.indexOf("/node_modules/") !== -1) {
        var filePath = pathFromRoot(url.replace(/\.\./g, ""))
        var fileContent = fs.readFileSync(path.join(__dirname + "/../", url.replace(/\.\./g, "")))
        res.end(fileContent).toString()
        return
    }

    if (url === "/__jscb/bundle.js") {
        res.end(fs.readFileSync(pathFromRoot("src/ui/dist/bundle.js")).toString())
        return
    }

    if (url.indexOf("/__jscb/getValues") !== -1) {
        var parts = url.replace("/__jscb/getValues/", "").split("/").map(parseFloat)
        var scriptId = parts[0]
        var valueId = parts[1]

        var info = getDataStore(scriptId)
        info.getValues(valueId, function(values){
            res.end(JSON.stringify(values))
        })

        return
    }

    if (url.indexOf("/__jscb/getLocations") !== -1) {
        var scriptId = parseFloat(url.replace("/__jscb/getLocations/", ""))

        var info = getDataStore(scriptId)
        res.end(JSON.stringify(info.locations))

        return
    }

    if (url.indexOf("/__jscb/getCode") !== -1) {
        var scriptId = parseFloat(url.replace("/__jscb/getCode/", ""))

        var info = getDataStore(scriptId)
        res.end(info.code)

        return
    }

    if (req.url.indexOf("/browse") !== -1) {
        var url = decodeURIComponent(req.url).replace("/browse?", "")

        console.log("Url", url)
        console.log(urlToScriptId)
        var scriptId = urlToScriptId[url]
        var info = getDataStore(scriptId)
        if (!info){
            res.end("No data for this file has been collected. Load a web page that loads this file")
        } else {
            renderInfo(info, scriptId, text => res.end(text))

        }

    }

    if (url.indexOf("/__jscb/fetchFunctionCode") !== -1) {
        var parts = url.split("/")
        var locationId = parseFloat(parts.pop())
        var scriptId = parseFloat(parts.pop())
        var store = dataStores[scriptId]
        var loc = store.locations[locationId]
        var json = {
            text: store.code.slice(loc.start, loc.end),
            url: "/browse?" + encodeURIComponent(store.url) + "#" + loc.loc.start.line
        }
        res.end(JSON.stringify(json))
        return
    }

    if (url.indexOf("/request") !== -1) {
        var id = url.split("/")[2]
        console.log("request started", id, decodeURIComponent(req.url.split("?")[1]))

        resById[id] = res
        return
    }
    if (url.indexOf("/response") !== -1) {
        var id = url.split("/")[2]
        console.log("Response", id, req.body.url)

        var response = req.body.response
        if (endsWith(req.body.url, ".js") || req.body.requestType === "script") {
            var scriptId = urlToScriptId[req.body.url]
            if (!scriptId){
                scriptId = scriptIdCounter
                scriptIdCounter++
                urlToScriptId[req.body.url] = scriptId
            }

            logPerfStart("Prettify " + req.body.url)
            response = beautifyJS(response)
            logPerfEnd("Prettify " + req.body.url)

            logPerfStart("Compile " + req.body.url)
            var compiled = compiler.compile(response, {
                scriptId
            })
            logPerfEnd("Compile " + req.body.url)

            if (!dataStores[scriptId]) {
                dataStores[scriptId] = new DataStore({
                    code: response,
                    locations: compiled.locations,
                    url: req.body.url
                })
            } else {
                console.log("re-using datastore for url", req.body.url)
            }

            response = compiled.code
        }

        var pre = fs.readFileSync(pathFromRoot("src/browser.js")).toString().replace("{{port}}", port) + "\n\n"
        response = pre + response

        if (!req.body.returnProcessedContent) {
            var interval = setInterval(function(){
                if (resById[id]) {

                    resById[id].end(pre + response)
                    clearInterval(interval)
                } else {
                    console.log("no request yet for" + id, "waiting...")
                }

            }, 100)
            res.end("OK")
        } else {
            res.end(response)
        }

        return
    }

    if (url === "/") {
        var html = fs.readFileSync(__dirname + "/ui/home.html").toString()
        console.log("urlToScriptId", urlToScriptId)
        var scriptDataCollected = Object.keys(urlToScriptId).length  > 0
        var fileLinks
        if(scriptDataCollected) {
            fileLinks = Object.keys(urlToScriptId).map(url => {
                console.log(urlToScriptId)
                console.log("Url", url)
                var scriptId = urlToScriptId[url]
                var store = dataStores[scriptId]
                var content = ""
                if (store === undefined) {
                    content = "No value store found"
                } else {
                    var values = Object.keys(store.values).length
                    var locations = Object.keys(store.locations).length
                    // rough percentage b/c funcitonlocations are also locations
                    var roughPercentage = Math.round(values / locations * 100 * 10) /10
                    content = `~${roughPercentage}%`
                }

                return `

                        ${escape(url)}

                        ${content}

                `
            }).join("")
            fileLinks = "
"  +
                `<thead><th>File</th><th style="min-width: 150px">Locations with values</th></thead>`
                + fileLinks + "
"
        } else {
            fileLinks = "
No data collected. Load a website and then click the Glassdoor Chrome extension button in the top right of your browser.
"
        }

        res.end(html.replace("{{fileLinks}}", fileLinks))
        return
    }

    if (url.indexOf("__jscb/reportValues") !== -1) {
        res.setHeader("Access-Control-Allow-Origin", "*")
        res.setHeader("Access-Control-Allow-Headers", "Content-Type, Access-Control-Allow-Headers, Authorization, X-Requested-With")
        if (!req.body.length) {
            res.end()
            return
        }
        console.log("Received " + req.body.length + " values")

        logPerfStart("Save " + req.body.length + "values")
        async.eachSeries(req.body, function iteratee(data, callback) {
            var dataStore = getDataStore(data.scriptId)
            dataStore.reportValue(data, callback)
        }, function(){
            logPerfEnd("Save " + req.body.length + "values")
            if (saveTo) {
                var data = {
                    stores: _.mapValues(dataStores, s => s.serialize()),
                    scriptIdCounter: scriptIdCounter,
                    urlToScriptId: urlToScriptId
                }

                var stringifiedData = JSON.stringify(data, null, 4)
                var mb = Math.round(stringifiedData.length / 1024 / 1024)
                console.log("Saving data to " + saveTo + ": " + mb + "MB")
                fs.writeFileSync(saveTo, stringifiedData)
            }

            res.end('{"status": "success"}')
        });

    }
});

function renderInfo(info,scriptId, cb){
    var res = {}

    var valueIds = Object.keys(info.locations)
    // async.eachSeries(valueIds, function iteratee(valueId, callback) {
    //     info.getValues(valueId, function(values){
    //         res[valueId] = {
    //             type: null, // types are out of scope for now
    //             examples: values
    //         }
    //         callback()
    //     })
    // }, function(){
        // });
    console.log("DONE")
        fileName = _.last(info.url.split("/"))

        // window.values = JSON.parse(decodeURI("${encodeURI(JSON.stringify(res))}"));
    var valueEmbeds = `
        window.scriptId =${scriptId}

    `

    cb(fs.readFileSync(__dirname + "/ui/file.html").toString()
        .replace("{{valueEmbeds}}", valueEmbeds)
        .replace("{{fileName}}", fileName))

}

function renderInfoOldUnused(info){
    var m = new MagicString(info.code)
    var errors = []
    Object.keys(info.locations).forEach(function(id){
        var loc = info.locations[id]
        try {
            if (loc.type === "call") {
                m.insertLeft(loc.end, "OPENTAGspan data-value-id='" + id + "' style='background: red; color: white;border-radius: 4px;padding: 2;font-size: 12px'CLOSETAG" +
                    "RET"
                + "OPENTAG/spanCLOSETAG")
            }
            else {
                var end = loc.end
                if (loc.type === "returnStatement") {
                    end = loc.start + "return".length
                }
                m.overwrite(loc.start, end, "OPENTAGspan data-value-id='" + id + "' style='border-bottom: 1px solid red'CLOSETAG" +
                    (loc.type === "returnStatement" ? "return" : info.code.slice(loc.start, loc.end) )

                + "OPENTAG/spanCLOSETAG")
            }
        } catch (err) {
            errors.push(err)
        }
    })

    var res = {}
    Object.keys(info.values).forEach(function(key){
        var values = info.values[key]
        if (values.length === 0) {
            res[key] = null
        } else {
            res[key] = {
                type: null, // types are out of scope for now
                examples: values.slice(0, 1)
            }
        }
    })

    return `
    <meta charset="utf-8" />

${m.toString().replace(/</g, "<").replace(/>/g, ">").replace(/OPENTAG/g, "<").replace(/CLOSETAG/g, ">")}

ERRORS:
${errors.join("
")}

        `
}

//create node.js http server and listen on port
http.createServer(app).listen(port);
console.log("Listening on http://localhost:" + port)
<|endoftext|>


<commit_before>'use strict'

/**
 * @module serverless-plugin-warmup
 *
 * @see {@link https://serverless.com/framework/docs/providers/aws/guide/plugins/}
 *
 * @requires 'bluebird'
 * @requires 'fs-extra'
 * @requires 'path'
 * */
const BbPromise = require('bluebird')
const fs = BbPromise.promisifyAll(require('fs-extra'))
const path = require('path')

/**
 * @classdesc Keep your lambdas warm during Winter
 * @class WarmUP
 * */
class WarmUP {
  /**
   * @description Serverless Warm Up
   * @constructor
   *
   * @param {!Object} serverless - Serverless object
   * @param {!Object} options - Serverless options
   * */
  constructor (serverless, options) {
    /** Serverless variables */
    this.serverless = serverless
    this.options = options
    this.custom = this.serverless.service.custom
    this.provider = this.serverless.getProvider('aws')

    this.hooks = {
      'after:package:initialize': this.afterPackageInitialize.bind(this),
      'after:package:createDeploymentArtifacts': this.afterCreateDeploymentArtifacts.bind(this),
      'after:deploy:deploy': this.afterDeployFunctions.bind(this)
    }
  }

  /**
   * @description After package initialize hook. Create warmer function and add it to the service.
   *
   * @fulfil {} — Warm up set
   * @reject {Error} Warm up error
   *
   * @return {(boolean|Promise)}
   * */
  afterPackageInitialize () {
    this.configPlugin()
    return this.createWarmer()
  }

  /**
   * @description After create deployment artifacts. Clean prefix folder.
   *
   * @fulfil {} — Optimization finished
   * @reject {Error} Optimization error
   *
   * @return {Promise}
   * */
  afterCreateDeploymentArtifacts () {
    return this.cleanFolder()
  }

  /**
   * @description After deploy functions hooks
   *
   * @fulfil {} — Functions warmed up sucessfuly
   * @reject {Error} Functions couldn't be warmed up
   *
   * @return {Promise}
   * */
  afterDeployFunctions () {
    if (this.warmup.prewarm) {
      this.configPlugin()
      return this.warmUpFunctions()
    }
  }

  /**
   * @description Configure the plugin based on the context of serverless.yml
   *
   * @return {}
   * */
  configPlugin () {
    /** Set warm up folder, file and handler paths */
    this.folderName = '_warmup'
    this.pathFolder = this.getPath(this.folderName)
    this.pathFile = this.pathFolder + '/index.js'
    this.pathHandler = this.folderName + '/index.warmUp'

    /** Default options */
    this.warmup = {
      memorySize: 128,
      name: this.serverless.service.service + '-' + this.options.stage + '-warmup-plugin',
      schedule: 'rate(5 minutes)',
      timeout: 10,
      prewarm: false
    }

    /** Set global custom options */
    if (!this.custom || !this.custom.warmup) {
      return
    }

    /** Memory size */
    if (typeof this.custom.warmup.memorySize === 'number') {
      this.warmup.memorySize = this.custom.warmup.memorySize
    }

    /** Function name */
    if (typeof this.custom.warmup.name === 'string') {
      this.warmup.name = this.custom.warmup.name
    }

    /** Schedule expression */
    if (typeof this.custom.warmup.schedule === 'string') {
      this.warmup.schedule = this.custom.warmup.schedule
    }

    /** Timeout */
    if (typeof this.custom.warmup.timeout === 'number') {
      this.warmup.timeout = this.custom.warmup.timeout
    }

    /** Pre-warm */
    if (typeof this.custom.warmup.prewarm === 'boolean') {
      this.warmup.prewarm = this.custom.warmup.prewarm
    }
  }

  /**
   * @description After create deployment artifacts
   *
   * @param {string} file — File path
   *
   * @return {String} Absolute file path
   * */
  getPath (file) {
    return path.join(this.serverless.config.servicePath, file)
  }

  /**
   * @description Clean prefix folder
   *
   * @fulfil {} — Folder cleaned
   * @reject {Error} File system error
   *
   * @return {Promise}
   * */
  cleanFolder () {
    return fs.removeAsync(this.pathFolder)
  }

  /**
   * @description Warm up functions
   *
   * @fulfil {} — Warm up function created and added to service
   * @reject {Error} Warm up error
   *
   * @return {Promise}
   * */
  createWarmer () {
    /** Get functions */
    const allFunctions = this.serverless.service.getAllFunctions()

    /** Filter functions for warm up */
    return BbPromise.filter(allFunctions, (functionName) => {
      const functionObject = this.serverless.service.getFunction(functionName)

      /** Function needs to be warm */
      if (functionObject.warmup === true) {
        return functionObject
      }
    }).then((functionNames) => {
      /** Skip writing if no functions need to be warm */
      if (!functionNames.length) {
        /** Log no warmup */
        this.serverless.cli.log('WarmUP: no lambda to warm')
        return true
      }

      /** Write warm up function */
      return this.createWarmUpFunctionArtifact(functionNames)
    }).then((skip) => {
      /** Add warm up function to service */
      if (skip !== true) {
        return this.addWarmUpFunctionToService()
      }
    })
  }

  /**
   * @description Write warm up ES6 function
   *
   * @param {Array} functionNames - Function names
   *
   * @fulfil {} — Warm up function created
   * @reject {Error} Warm up error
   *
   * @return {Promise}
   * */
  createWarmUpFunctionArtifact (functionNames) {
    /** Log warmup start */
    this.serverless.cli.log('WarmUP: setting ' + functionNames.length + ' lambdas to be warm')

    /** Get function names */
    functionNames = functionNames.map((functionName) => {
      const functionObject = this.serverless.service.getFunction(functionName)
      this.serverless.cli.log('WarmUP: ' + functionObject.name)
      return functionObject.name
    })

    /** Write function invoke promises and push to array */
    const warmUpFunction = `"use strict";

      /** Generated by Serverless WarmUP Plugin at ${new Date().toISOString()} */
      const aws = require("aws-sdk");
      aws.config.region = "${this.serverless.service.provider.region}";
      const lambda = new aws.Lambda();
      const functionNames = "${functionNames.join()}".split(",");
      module.exports.warmUp = (event, context, callback) => {
        let invokes = [];
        let errors = 0;
        console.log("Warm Up Start");
        functionNames.forEach((functionName) => {
          const params = {
            FunctionName: functionName,
            InvocationType: "RequestResponse",
            LogType: "None",
            Qualifier: process.env.SERVERLESS_ALIAS || "$LATEST",
            Payload: JSON.stringify({ source: "serverless-plugin-warmup" })
          };
          invokes.push(lambda.invoke(params).promise().then((data) => {
            console.log("Warm Up Invoke Success: " + functionName, data);
          }, (error) => {
            errors++;
            console.log("Warm Up Invoke Error: " + functionName, error);
          }));
        });
        Promise.all(invokes).then(() => {
          console.log("Warm Up Finished with " + errors + " invoke errors");
          callback();
        });
      }`

    /** Write warm up file */
    return fs.outputFileAsync(this.pathFile, warmUpFunction)
  }

  /**
   * @description Add warm up function to service
   *
   * @return {Object} Warm up service function object
   * */
  addWarmUpFunctionToService () {
    /** SLS warm up function */
    this.serverless.service.functions.warmUpPlugin = {
      description: 'Serverless WarmUP Plugin',
      events: [
        {
          schedule: this.warmup.schedule
        }
      ],
      handler: this.pathHandler,
      memorySize: this.warmup.memorySize,
      name: this.warmup.name,
      runtime: 'nodejs6.10',
      package: {
        individually: true,
        exclude: ['**'],
        include: [this.folderName + '/**']
      },
      timeout: this.warmup.timeout
    }

    /** Return service function object */
    return this.serverless.service.functions.warmUpPlugin
  }

  /**
   * @description Warm up the functions immediately after deployment
   *
   * @fulfil {} — Functions warmed up sucessfuly
   * @reject {Error} Functions couldn't be warmed up
   *
   * @return {Promise}
   * */
  warmUpFunctions () {
    this.serverless.cli.log('WarmUP: Pre-warming up your functions')

    const params = {
      FunctionName: this.warmup.name,
      InvocationType: 'RequestResponse',
      LogType: 'None',
      Qualifier: process.env.SERVERLESS_ALIAS || '$LATEST',
      Payload: JSON.stringify({ source: 'serverless-plugin-warmup' })
    }

    return this.provider.request('Lambda', 'invoke', params)
      .then(data => this.serverless.cli.log('WarmUp: Functions sucessfuly pre-warmed'))
      .catch(error => this.serverless.cli.log('WarmUp: Error while pre-warming functions', error))
  }
}

/** Export WarmUP class */
module.exports = WarmUP
<commit_msg>fix(config): config must run before condition<commit_after>'use strict'

/**
 * @module serverless-plugin-warmup
 *
 * @see {@link https://serverless.com/framework/docs/providers/aws/guide/plugins/}
 *
 * @requires 'bluebird'
 * @requires 'fs-extra'
 * @requires 'path'
 * */
const BbPromise = require('bluebird')
const fs = BbPromise.promisifyAll(require('fs-extra'))
const path = require('path')

/**
 * @classdesc Keep your lambdas warm during Winter
 * @class WarmUP
 * */
class WarmUP {
  /**
   * @description Serverless Warm Up
   * @constructor
   *
   * @param {!Object} serverless - Serverless object
   * @param {!Object} options - Serverless options
   * */
  constructor (serverless, options) {
    /** Serverless variables */
    this.serverless = serverless
    this.options = options
    this.custom = this.serverless.service.custom
    this.provider = this.serverless.getProvider('aws')

    this.hooks = {
      'after:package:initialize': this.afterPackageInitialize.bind(this),
      'after:package:createDeploymentArtifacts': this.afterCreateDeploymentArtifacts.bind(this),
      'after:deploy:deploy': this.afterDeployFunctions.bind(this)
    }
  }

  /**
   * @description After package initialize hook. Create warmer function and add it to the service.
   *
   * @fulfil {} — Warm up set
   * @reject {Error} Warm up error
   *
   * @return {(boolean|Promise)}
   * */
  afterPackageInitialize () {
    this.configPlugin()
    return this.createWarmer()
  }

  /**
   * @description After create deployment artifacts. Clean prefix folder.
   *
   * @fulfil {} — Optimization finished
   * @reject {Error} Optimization error
   *
   * @return {Promise}
   * */
  afterCreateDeploymentArtifacts () {
    return this.cleanFolder()
  }

  /**
   * @description After deploy functions hooks
   *
   * @fulfil {} — Functions warmed up sucessfuly
   * @reject {Error} Functions couldn't be warmed up
   *
   * @return {Promise}
   * */
  afterDeployFunctions () {
    this.configPlugin()
    if (this.warmup.prewarm) {
      return this.warmUpFunctions()
    }
  }

  /**
   * @description Configure the plugin based on the context of serverless.yml
   *
   * @return {}
   * */
  configPlugin () {
    /** Set warm up folder, file and handler paths */
    this.folderName = '_warmup'
    this.pathFolder = this.getPath(this.folderName)
    this.pathFile = this.pathFolder + '/index.js'
    this.pathHandler = this.folderName + '/index.warmUp'

    /** Default options */
    this.warmup = {
      memorySize: 128,
      name: this.serverless.service.service + '-' + this.options.stage + '-warmup-plugin',
      schedule: 'rate(5 minutes)',
      timeout: 10,
      prewarm: false
    }

    /** Set global custom options */
    if (!this.custom || !this.custom.warmup) {
      return
    }

    /** Memory size */
    if (typeof this.custom.warmup.memorySize === 'number') {
      this.warmup.memorySize = this.custom.warmup.memorySize
    }

    /** Function name */
    if (typeof this.custom.warmup.name === 'string') {
      this.warmup.name = this.custom.warmup.name
    }

    /** Schedule expression */
    if (typeof this.custom.warmup.schedule === 'string') {
      this.warmup.schedule = this.custom.warmup.schedule
    }

    /** Timeout */
    if (typeof this.custom.warmup.timeout === 'number') {
      this.warmup.timeout = this.custom.warmup.timeout
    }

    /** Pre-warm */
    if (typeof this.custom.warmup.prewarm === 'boolean') {
      this.warmup.prewarm = this.custom.warmup.prewarm
    }
  }

  /**
   * @description After create deployment artifacts
   *
   * @param {string} file — File path
   *
   * @return {String} Absolute file path
   * */
  getPath (file) {
    return path.join(this.serverless.config.servicePath, file)
  }

  /**
   * @description Clean prefix folder
   *
   * @fulfil {} — Folder cleaned
   * @reject {Error} File system error
   *
   * @return {Promise}
   * */
  cleanFolder () {
    return fs.removeAsync(this.pathFolder)
  }

  /**
   * @description Warm up functions
   *
   * @fulfil {} — Warm up function created and added to service
   * @reject {Error} Warm up error
   *
   * @return {Promise}
   * */
  createWarmer () {
    /** Get functions */
    const allFunctions = this.serverless.service.getAllFunctions()

    /** Filter functions for warm up */
    return BbPromise.filter(allFunctions, (functionName) => {
      const functionObject = this.serverless.service.getFunction(functionName)

      /** Function needs to be warm */
      if (functionObject.warmup === true) {
        return functionObject
      }
    }).then((functionNames) => {
      /** Skip writing if no functions need to be warm */
      if (!functionNames.length) {
        /** Log no warmup */
        this.serverless.cli.log('WarmUP: no lambda to warm')
        return true
      }

      /** Write warm up function */
      return this.createWarmUpFunctionArtifact(functionNames)
    }).then((skip) => {
      /** Add warm up function to service */
      if (skip !== true) {
        return this.addWarmUpFunctionToService()
      }
    })
  }

  /**
   * @description Write warm up ES6 function
   *
   * @param {Array} functionNames - Function names
   *
   * @fulfil {} — Warm up function created
   * @reject {Error} Warm up error
   *
   * @return {Promise}
   * */
  createWarmUpFunctionArtifact (functionNames) {
    /** Log warmup start */
    this.serverless.cli.log('WarmUP: setting ' + functionNames.length + ' lambdas to be warm')

    /** Get function names */
    functionNames = functionNames.map((functionName) => {
      const functionObject = this.serverless.service.getFunction(functionName)
      this.serverless.cli.log('WarmUP: ' + functionObject.name)
      return functionObject.name
    })

    /** Write function invoke promises and push to array */
    const warmUpFunction = `"use strict";

      /** Generated by Serverless WarmUP Plugin at ${new Date().toISOString()} */
      const aws = require("aws-sdk");
      aws.config.region = "${this.serverless.service.provider.region}";
      const lambda = new aws.Lambda();
      const functionNames = "${functionNames.join()}".split(",");
      module.exports.warmUp = (event, context, callback) => {
        let invokes = [];
        let errors = 0;
        console.log("Warm Up Start");
        functionNames.forEach((functionName) => {
          const params = {
            FunctionName: functionName,
            InvocationType: "RequestResponse",
            LogType: "None",
            Qualifier: process.env.SERVERLESS_ALIAS || "$LATEST",
            Payload: JSON.stringify({ source: "serverless-plugin-warmup" })
          };
          invokes.push(lambda.invoke(params).promise().then((data) => {
            console.log("Warm Up Invoke Success: " + functionName, data);
          }, (error) => {
            errors++;
            console.log("Warm Up Invoke Error: " + functionName, error);
          }));
        });
        Promise.all(invokes).then(() => {
          console.log("Warm Up Finished with " + errors + " invoke errors");
          callback();
        });
      }`

    /** Write warm up file */
    return fs.outputFileAsync(this.pathFile, warmUpFunction)
  }

  /**
   * @description Add warm up function to service
   *
   * @return {Object} Warm up service function object
   * */
  addWarmUpFunctionToService () {
    /** SLS warm up function */
    this.serverless.service.functions.warmUpPlugin = {
      description: 'Serverless WarmUP Plugin',
      events: [
        {
          schedule: this.warmup.schedule
        }
      ],
      handler: this.pathHandler,
      memorySize: this.warmup.memorySize,
      name: this.warmup.name,
      runtime: 'nodejs6.10',
      package: {
        individually: true,
        exclude: ['**'],
        include: [this.folderName + '/**']
      },
      timeout: this.warmup.timeout
    }

    /** Return service function object */
    return this.serverless.service.functions.warmUpPlugin
  }

  /**
   * @description Warm up the functions immediately after deployment
   *
   * @fulfil {} — Functions warmed up sucessfuly
   * @reject {Error} Functions couldn't be warmed up
   *
   * @return {Promise}
   * */
  warmUpFunctions () {
    this.serverless.cli.log('WarmUP: Pre-warming up your functions')

    const params = {
      FunctionName: this.warmup.name,
      InvocationType: 'RequestResponse',
      LogType: 'None',
      Qualifier: process.env.SERVERLESS_ALIAS || '$LATEST',
      Payload: JSON.stringify({ source: 'serverless-plugin-warmup' })
    }

    return this.provider.request('Lambda', 'invoke', params)
      .then(data => this.serverless.cli.log('WarmUp: Functions sucessfuly pre-warmed'))
      .catch(error => this.serverless.cli.log('WarmUp: Error while pre-warming functions', error))
  }
}

/** Export WarmUP class */
module.exports = WarmUP
<|endoftext|>


<commit_before>var es = require('event-stream')
  , gutil = require('gulp-util')
  , BufferStreams = require('bufferstreams')
  , ttf2eot = require('ttf2eot')
;

// File level transform function
function ttf2eotTransform(opt) {
  // Return a callback function handling the buffered content
  return function(err, buf, cb) {

    // Handle any error
    if(err) {
      cb(new gutil.PluginError('ttf2eot', err, {showStack: true}));
    }

    // Use the buffered content
      try {
        buf = new Buffer(ttf2eot(new Uint8Array(buf)).buffer);
        cb(null, buf);
      } catch(err) {
        cb(new gutil.PluginError('ttf2eot', err, {showStack: true}));
      }

  };
}

// Plugin function
function ttf2eotGulp() {

  return es.map(function (file, callback) {
    file.path = gutil.replaceExtension(file.path, ".eot");

    // Buffers
    if(file.isBuffer()) {
      try {
        file.contents = new Buffer(ttf2eot(
          new Uint8Array(file.contents)
        ).buffer);
        callback(null, file);
      } catch(err) {
        callback(new gutil.PluginError('ttf2eot', err, {showStack: true}));
      }

    // Streams
    } else {
      file.contents = file.contents.pipe(new BufferStreams(ttf2eotTransform()));
      callback(null, file);
    }
  });

};

// Export the file level transform function for other plugins usage
ttf2eotGulp.fileTransform = ttf2eotTransform;

// Export the plugin main function
module.exports = ttf2eotGulp;

<commit_msg>Suppressing event-stream dependency + do nothing when file contents are null<commit_after>var PassThrough = require('stream').PassThrough
  , gutil = require('gulp-util')
  , BufferStreams = require('bufferstreams')
  , ttf2eot = require('ttf2eot')
;

// File level transform function
function ttf2eotTransform(opt) {
  // Return a callback function handling the buffered content
  return function(err, buf, cb) {

    // Handle any error
    if(err) {
      cb(new gutil.PluginError('ttf2eot', err, {showStack: true}));
    }

    // Use the buffered content
      try {
        buf = new Buffer(ttf2eot(new Uint8Array(buf)).buffer);
        cb(null, buf);
      } catch(err) {
        cb(new gutil.PluginError('ttf2eot', err, {showStack: true}));
      }

  };
}

// Plugin function
function ttf2eotGulp() {

  var stream = new PassThrough({objectMode: true});
  stream.on('data', function(file) {
    if(file.isNull()) return;

    file.path = gutil.replaceExtension(file.path, ".eot");

    // Buffers
    if(file.isBuffer()) {
      try {
        file.contents = new Buffer(ttf2eot(
          new Uint8Array(file.contents)
        ).buffer);
      } catch(err) {
        stream.emit('error', new gutil.PluginError('ttf2eot', err, {
          showStack: true
        }));
      }

    // Streams
    } else {
      file.contents = file.contents.pipe(new BufferStreams(ttf2eotTransform()));
    }

  });
  return stream;

};

// Export the file level transform function for other plugins usage
ttf2eotGulp.fileTransform = ttf2eotTransform;

// Export the plugin main function
module.exports = ttf2eotGulp;

<|endoftext|>


<commit_before>import './fix/safari';
import 'skatejs-named-slots';
import 'document-register-element';
<commit_msg>refactor: Add comments and eslint igores.<commit_after>/* eslint import/imports-first: 0 */

// We load the Safari fix before document-register-element because DRE
// overrides attachShadow() and calls back the one it finds on HTMLElement.
import './fix/safari';
import 'skatejs-named-slots';
import 'document-register-element';
<|endoftext|>


<commit_before>const aws = require('aws-sdk');
const plpr = require('plpr');
const fs = require('fs');
const pgn = require('pg-query-normalizer');

const es = require('./elasticsearch');
const bq = require('./bq');

const rds = new aws.RDS();

const DBInstanceIdentifier = process.env.AWS_DB_INSTANCE_IDENTIFIER;
const logLinePrefix = process.env.PSQL_LOG_LINE_PREFIX;
const backend = process.env.BACKEND_SERVICE;

async function downloadLogFile() {
  const params = {
    DBInstanceIdentifier,
  };
  if (typeof DBInstanceIdentifier === 'undefined') throw new Error('You have to export AWS RDS DB instance name to "AWS_DB_INSTANCE_IDENTIFIER"');

  try {
    const describeDBLogFilesPromise = rds.describeDBLogFiles(params).promise();
    const data = await describeDBLogFilesPromise;
    let obj = JSON.parse(JSON.stringify(data)).DescribeDBLogFiles;
    const files = obj.map(value => value.LogFileName);

    const filename = process.env.AWS_DB_LOGFILE_NAME;
    params.LogFileName = filename;
    if (typeof filename === 'undefined') {
      params.LogFileName = files[files.length - 2];
    }

    console.log(`Downloading: ${params.LogFileName}...`);
    let next = true;
    let marker = '0';
    let raw = '';
    while (next) {
      params.Marker = marker;
      const downloadLogFilePortionPromise = rds.downloadDBLogFilePortion(params).promise();
      const resp = await downloadLogFilePortionPromise; // eslint-disable-line no-await-in-loop

      obj = JSON.parse(JSON.stringify(resp));
      next = obj.AdditionalDataPending;
      marker = obj.Marker;
      raw += obj.LogFileData;
    }
    return raw;
  } catch (error) {
    throw error;
  }
}

function dataReceiver() {
  const filepath = process.env.LOCAL_LOGFILE_PATH;
  if (typeof filepath === 'undefined') {
    return downloadLogFile();
  }
  const data = fs.readFileSync(filepath, 'utf-8');
  return Promise.resolve(data);
}

function normalizeQueries(logs) {
  const newLogs = [];
  for (let i = 0, len = logs.length; i < len; i += 1) {
    const normalized = pgn(logs[i].query);
    newLogs[i] = logs[i];
    newLogs[i].normalized_query = normalized;
  }
  return newLogs;
}

exports.handler = (event, context, callback) => {
  dataReceiver().then((data) => {
    if (typeof logLinePrefix === 'undefined') throw new Error('You have to set PostgreSQL log_line_prefix in PSQL_LOG_LINE_PREFIX');

    const logs = plpr(data, logLinePrefix);
    if (logs.length === 0) {
      return callback(null, 'success');
    }

    console.log(`Insert data length: ${logs.length}`);

    let inserter = null;
    switch (backend) {
      case 'es':
        inserter = es;
        break;
      case 'bq':
        inserter = bq;
        break;
      default:
        throw new Error('You have to export Backend Service "es" (Elasticsearch) or "bq" (BigQuery) to "BACKEND_SERVICE"');
    }

    const normalized = normalizeQueries(logs);
    return inserter(normalized, DBInstanceIdentifier).then(() => callback(null, 'success')).catch(err => callback(err, 'error'));
  }).catch(err => callback(err, 'error'));
};
<commit_msg>Add eslint-disable-line no-console<commit_after>const aws = require('aws-sdk');
const plpr = require('plpr');
const fs = require('fs');
const pgn = require('pg-query-normalizer');

const es = require('./elasticsearch');
const bq = require('./bq');

const rds = new aws.RDS();

const DBInstanceIdentifier = process.env.AWS_DB_INSTANCE_IDENTIFIER;
const logLinePrefix = process.env.PSQL_LOG_LINE_PREFIX;
const backend = process.env.BACKEND_SERVICE;

async function downloadLogFile() {
  const params = {
    DBInstanceIdentifier,
  };
  if (typeof DBInstanceIdentifier === 'undefined') throw new Error('You have to export AWS RDS DB instance name to "AWS_DB_INSTANCE_IDENTIFIER"');

  try {
    const describeDBLogFilesPromise = rds.describeDBLogFiles(params).promise();
    const data = await describeDBLogFilesPromise;
    let obj = JSON.parse(JSON.stringify(data)).DescribeDBLogFiles;
    const files = obj.map(value => value.LogFileName);

    const filename = process.env.AWS_DB_LOGFILE_NAME;
    params.LogFileName = filename;
    if (typeof filename === 'undefined') {
      params.LogFileName = files[files.length - 2];
    }
    console.log(`Downloading: ${params.LogFileName}...`); // eslint-disable-line no-console

    let next = true;
    let marker = '0';
    let raw = '';
    while (next) {
      params.Marker = marker;
      const downloadLogFilePortionPromise = rds.downloadDBLogFilePortion(params).promise();
      const resp = await downloadLogFilePortionPromise; // eslint-disable-line no-await-in-loop

      obj = JSON.parse(JSON.stringify(resp));
      next = obj.AdditionalDataPending;
      marker = obj.Marker;
      raw += obj.LogFileData;
    }
    return raw;
  } catch (error) {
    throw error;
  }
}

function dataReceiver() {
  const filepath = process.env.LOCAL_LOGFILE_PATH;
  if (typeof filepath === 'undefined') {
    return downloadLogFile();
  }
  const data = fs.readFileSync(filepath, 'utf-8');
  return Promise.resolve(data);
}

function normalizeQueries(logs) {
  const newLogs = [];
  for (let i = 0, len = logs.length; i < len; i += 1) {
    const normalized = pgn(logs[i].query);
    newLogs[i] = logs[i];
    newLogs[i].normalized_query = normalized;
  }
  return newLogs;
}

exports.handler = (event, context, callback) => {
  dataReceiver().then((data) => {
    if (typeof logLinePrefix === 'undefined') throw new Error('You have to set PostgreSQL log_line_prefix in PSQL_LOG_LINE_PREFIX');

    const logs = plpr(data, logLinePrefix);
    if (logs.length === 0) {
      return callback(null, 'success');
    }
    console.log(`Insert data length: ${logs.length}`); // eslint-disable-line no-console

    let inserter = null;
    switch (backend) {
      case 'es':
        inserter = es;
        break;
      case 'bq':
        inserter = bq;
        break;
      default:
        throw new Error('You have to export Backend Service "es" (Elasticsearch) or "bq" (BigQuery) to "BACKEND_SERVICE"');
    }

    const normalized = normalizeQueries(logs);
    return inserter(normalized, DBInstanceIdentifier).then(() => callback(null, 'success')).catch(err => callback(err, 'error'));
  }).catch(err => callback(err, 'error'));
};
<|endoftext|>