<commit_before>var express = require('express');
var url = require('url');
var net = require('net');
var path = require('path');
var tls = require('tls');
var fs = require('fs');
var socks = require('socksv5');
var EventEmitter = require('events').EventEmitter;
var extend = require('util')._extend;
var util = require('./util');
var logger = require('./util/logger');
var rules = require('./rules');
var dispatch = require('./https');
var httpsUtil = require('./https/util');
var rulesUtil = require('./rules/util');
var initDataServer = require('./util/data-server');
var initLogServer = require('./util/log-server');
var pluginMgr = require('./plugins');
var config = require('./config');
var LOCALHOST = '127.0.0.1';
var WHISTLE_POLICY_HEADER = 'x-whistle-policy';
var TUNNEL_HOST_RE = /^[^:\/]+\.[^:\/]+:\d+$/;
var index = 0;

function tunnelProxy(server, proxy) {
//see: https://github.com/joyent/node/issues/9272
  if (typeof tls.checkServerIdentity == 'function') {
    var checkServerIdentity = tls.checkServerIdentity;
    tls.checkServerIdentity = function() {
      try {
        return checkServerIdentity.apply(this, arguments);
      } catch(err) {
        logger.error(err);
        return err;
      }
    };
  }

  process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';
  server.on('connect', function(req, reqSocket, head) {//ws, wss, https proxy
    var tunnelUrl = req.fullUrl = util.setProtocol(TUNNEL_HOST_RE.test(req.url) ? req.url : req.headers.host, true);
    var options;
    var parseUrl = function (_url, port) {
      _url = _url || tunnelUrl;
      options = req.options = url.parse(_url);
      options.port = options.port || port || 443;
      return options;
    };
    parseUrl();
    tunnelUrl = req.fullUrl = 'tunnel://' + options.host;
    var resSocket, proxySocket, responsed;
    req.isTunnel = true;
    req.clientIp = util.getClientIp(req) || LOCALHOST;
    var hostname = options.hostname;
    var _rules = req.rules = rules.resolveRules(tunnelUrl);
    rules.resolveFileRules(req, function() {
      var filter = req.filter;
      var disable = req.disable;
      var plugin = pluginMgr.getPluginByRuleUrl(util.rule.getUrl(_rules.rule));
      var rulesMgr = plugin && plugin.rulesMgr;
      util.mergeRules(req, rulesMgr && rulesMgr.resolveRules(tunnelUrl));
      var useTunnelPolicy = req.headers[WHISTLE_POLICY_HEADER] == 'tunnel';
      if (!useTunnelPolicy && (config.isLocalUIUrl(hostname) || config.isPluginUrl(hostname) || ((filter.https || filter.tunnel
          || rulesUtil.properties.get('interceptHttpsConnects')) && !filter.intercept && !disable.intercept))) {
        dispatch(reqSocket, hostname, proxy);
        sendEstablished();
        return;
      }

      var reqEmitter = new EventEmitter();
      var headers = req.headers;
      var reqData = {
        ip: req.clientIp,
        method: util.toUpperCase(req.method) || 'CONNECT',
        httpVersion: req.httpVersion || '1.1',
        headers: headers
      };
      var resData = {headers: {}};
      var data = reqEmitter.data = {
        url: options.host,
        startTime: Date.now(),
        rules: _rules,
        req: reqData,
        res: resData,
        isHttps: true
      };
      abortIfUnavailable(reqSocket);
      if (!rulesUtil.properties.get('hideHttpsConnects') && !filter.hide) {
        proxy.emit('request', reqEmitter);
      }

      if (disable.tunnel) {
        return reqSocket.destroy();
      }

      req.reqId = ++index;
      plugin = pluginMgr.resolveWhistlePlugins(req);
      pluginMgr.getTunnelRules(req, function(_rulesMgr) {
        if (_rulesMgr) {
          rulesMgr = _rulesMgr;
          util.mergeRules(req, rulesMgr.resolveRules(tunnelUrl));
          plugin = pluginMgr.getPluginByRuleUrl(util.rule.getUrl(_rules.rule));
          if (disable.tunnel) {
            return reqSocket.destroy();
          }
        }

        if (filter.rule) {
          delete _rules.rule;
          plugin = null;
        }

        var tunnelPort, proxyUrl;
        if(plugin) {
          tunnelPort = plugin.ports && plugin.ports.tunnelPort;
          if (!tunnelPort) {
            return emitError(new Error('No plugin.tunnelServer'));
          }
          proxyUrl = 'proxy://127.0.0.1:' + tunnelPort;
        }

        var realUrl = _rules.rule && _rules.rule.url.replace('https:', 'tunnel:');
        if (/^tunnel:\/\//.test(realUrl) && realUrl != tunnelUrl) {
          tunnelUrl = realUrl;
          data.realUrl = realUrl.replace('tunnel://', '');
          parseUrl();
        }

        rules.getProxy(tunnelUrl, tunnelPort || proxyUrl ? null : req, function() {
          if (!proxyUrl) {
            proxyUrl = !tunnelPort && _rules.proxy ? _rules.proxy.matcher : null;
          }

          if (proxyUrl) {
            var isSocks = /^socks:\/\//.test(proxyUrl);
            var _url = 'http:' + util.removeProtocol(proxyUrl);
            data.proxy = true;
            resolveHost(_url, function(ip) {
              options = parseUrl(_url, isSocks ? 1080 : 80);
              if (options.port == config.port && util.isLocalAddress(ip)) {
                return emitError(new Error('Can not proxy to itself'));
              }
              var onConnect = function(_proxySocket) {
                proxySocket = _proxySocket;
                abortIfUnavailable(proxySocket);
                reqSocket.pipe(proxySocket).pipe(reqSocket);
                sendEstablished();
              };
              var dstOptions = url.parse(tunnelUrl);
              dstOptions.proxyHost = ip;
              dstOptions.proxyPort = parseInt(options.port, 10);
              dstOptions.port = dstOptions.port || 443;
              dstOptions.host = dstOptions.hostname;
              var _headers = extend({}, headers);
              if (req.clientIp) {
                _headers[pluginMgr.CLIENT_IP_HEADER] = req.clientIp;
              }
              var ruleValue = tunnelPort && util.getMatcherValue(_rules.rule);
              if (ruleValue) {
                _headers[pluginMgr.RULE_VALUE_HEADER] = encodeURIComponent(ruleValue);
              }
              _headers.host = dstOptions.hostname + ':' + (dstOptions.port || 443);
              dstOptions.headers = _headers;
              if (isSocks) {
                dstOptions.proxyPort = options.port || 1080;
                dstOptions.localDNS = false;
                dstOptions.auths = config.getAuths(options);
                socks.connect(dstOptions, onConnect).on('error', emitError);
              } else {
                dstOptions.proxyPort = options.port || 80;
                dstOptions.proxyAuth = options.auth;
                config.connect(dstOptions, onConnect).on('error', emitError);
              }
            });
          } else {
            tunnel();
          }
        });
      });

      function tunnel() {
        resolveHost(tunnelUrl, function(ip, port) {
          resData.ip = port ? ip + ':' + port : ip;
          resSocket = net.connect(port || options.port, ip, function() {
            resSocket.pipe(reqSocket).pipe(resSocket);
            sendEstablished();
          });
          abortIfUnavailable(resSocket);
        });
      }

      function resolveHost(url, callback) {
        rules.resolveHost(url, function(err, ip, port, host) {
          if (host) {
            _rules.host = host;
          }
          data.requestTime = data.dnsTime = Date.now();
          resData.ip = ip || LOCALHOST;
          reqEmitter.emit('send', data);
          err ? emitError(err) : callback(ip, port);
        }, rulesMgr);
      }

      function abortIfUnavailable(socket) {
        socket.on('error', emitError).on('close', emitError);
      }

      function sendEstablished() {
        reqSocket.write('HTTP/1.1 200 Connection Established\r\nProxy-Agent: ' + config.name + '\r\n\r\n');
        responsed = true;
        if (reqEmitter) {
          data.responseTime = data.endTime = Date.now();
          resData.statusCode = 200;
          reqEmitter.emit('response', data);
          reqEmitter.emit('end', data);
        }

        return reqSocket;
      }

      function emitError(err) {
        if (responsed) {
          return;
        }
        responsed = true;
        resSocket && resSocket.destroy();
        proxySocket && proxySocket.destroy();
        reqSocket.destroy();
        data.responseTime = data.endTime = Date.now();

        if (!resData.ip) {
          resData.ip = LOCALHOST;
        }

        if (err) {
          err = new Error('Aborted');
          data.reqError = true;
          resData.statusCode ='aborted';
          reqData.body = util.getErrorStack(err);
          reqEmitter.emit('abort', data);
        } else {
          data.resError = true;
          resData.statusCode = resData.statusCode || 502;
          resData.body = util.getErrorStack(err);
          util.emitError(reqEmitter, data);
        }
      }
    });
  });

  return server;
}

function proxy(callback) {
  var app = express();
  var server = app.listen(config.port, callback);
  var proxyEvents = new EventEmitter();
  var middlewares = ['./init',
'../biz']
.concat(require('./inspectors'))
.concat(config.middlewares)
.concat(require('./handlers'));

  proxyEvents.config = config;
  app.logger = logger;
  middlewares.forEach(function(mw) {
    mw && app.use((typeof mw == 'string' ? require(mw) : mw).bind(proxyEvents));
  });

  exportInterfaces(proxyEvents);
  tunnelProxy(server, proxyEvents);
  initDataServer(proxyEvents);
  initLogServer(proxyEvents);
  require('../biz/init')(proxyEvents);
  config.debug && rules.disableDnsCache();
  config.rules && rulesUtil.parseRules(config.rules);
  if (config.values) {
    Object.keys(config.values).forEach(function(name) {
      rulesUtil.values.add(name, config.values[name]);
    });
  }
  return proxyEvents;
}

function exportInterfaces(obj) {
  obj.rules = rules;
  obj.util = util;
  obj.rulesUtil = rulesUtil;
  obj.httpsUtil = httpsUtil;
  obj.pluginMgr = pluginMgr;
  obj.logger = logger;
  return obj;
}

process.on('uncaughtException', function(err){
  var stack = util.getErrorStack(err);
  fs.writeFileSync(path.join(process.cwd(), config.name + '.log'), '\r\n' + stack + '\r\n', {flag: 'a'});
  /*eslint no-console: "off"*/
  console.error(stack);
  process.exit(1);
});

module.exports = exportInterfaces(proxy);
<commit_msg>refactor: Supports intercept HTTPS by plugin<commit_after>var express = require('express');
var url = require('url');
var net = require('net');
var path = require('path');
var tls = require('tls');
var fs = require('fs');
var socks = require('socksv5');
var EventEmitter = require('events').EventEmitter;
var extend = require('util')._extend;
var util = require('./util');
var logger = require('./util/logger');
var rules = require('./rules');
var dispatch = require('./https');
var httpsUtil = require('./https/util');
var rulesUtil = require('./rules/util');
var initDataServer = require('./util/data-server');
var initLogServer = require('./util/log-server');
var pluginMgr = require('./plugins');
var config = require('./config');
var LOCALHOST = '127.0.0.1';
var WHISTLE_POLICY_HEADER = 'x-whistle-policy';
var TUNNEL_HOST_RE = /^[^:\/]+\.[^:\/]+:\d+$/;
var index = 0;

function tunnelProxy(server, proxy) {
//see: https://github.com/joyent/node/issues/9272
  if (typeof tls.checkServerIdentity == 'function') {
    var checkServerIdentity = tls.checkServerIdentity;
    tls.checkServerIdentity = function() {
      try {
        return checkServerIdentity.apply(this, arguments);
      } catch(err) {
        logger.error(err);
        return err;
      }
    };
  }

  process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';
  server.on('connect', function(req, reqSocket, head) {//ws, wss, https proxy
    var tunnelUrl = req.fullUrl = util.setProtocol(TUNNEL_HOST_RE.test(req.url) ? req.url : req.headers.host, true);
    var options;
    var parseUrl = function (_url, port) {
      _url = _url || tunnelUrl;
      options = req.options = url.parse(_url);
      options.port = options.port || port || 443;
      return options;
    };
    parseUrl();
    tunnelUrl = req.fullUrl = 'tunnel://' + options.host;
    var resSocket, proxySocket, responsed, reqEmitter;
    req.isTunnel = true;
    req.clientIp = util.getClientIp(req) || LOCALHOST;
    req.reqId = ++index;
    var hostname = options.hostname;
    var _rules = req.rules = rules.resolveRules(tunnelUrl);
    rules.resolveFileRules(req, function() {
      plugin = pluginMgr.resolveWhistlePlugins(req);
      pluginMgr.getTunnelRules(req, function(_rulesMgr) {
        if (_rulesMgr) {
          rulesMgr = _rulesMgr;
          util.mergeRules(req, rulesMgr.resolveRules(tunnelUrl));
          plugin = pluginMgr.getPluginByRuleUrl(util.rule.getUrl(_rules.rule));
        }
        var filter = req.filter;
        var disable = req.disable;
        if (disable.tunnel) {
          return reqSocket.destroy();
        }
        var plugin = pluginMgr.getPluginByRuleUrl(util.rule.getUrl(_rules.rule));
        var rulesMgr = plugin && plugin.rulesMgr;
        util.mergeRules(req, rulesMgr && rulesMgr.resolveRules(tunnelUrl));
        var useTunnelPolicy = req.headers[WHISTLE_POLICY_HEADER] == 'tunnel';
        if (!useTunnelPolicy && (config.isLocalUIUrl(hostname) || config.isPluginUrl(hostname) || ((filter.https || filter.tunnel
            || rulesUtil.properties.get('interceptHttpsConnects')) && !filter.intercept && !disable.intercept))) {
          dispatch(reqSocket, hostname, proxy);
          sendEstablished();
          return;
        }

        reqEmitter = new EventEmitter();
        var headers = req.headers;
        var reqData = {
          ip: req.clientIp,
          method: util.toUpperCase(req.method) || 'CONNECT',
          httpVersion: req.httpVersion || '1.1',
          headers: headers
        };
        var resData = {headers: {}};
        var data = reqEmitter.data = {
          url: options.host,
          startTime: Date.now(),
          rules: _rules,
          req: reqData,
          res: resData,
          isHttps: true
        };
        abortIfUnavailable(reqSocket);
        if (!rulesUtil.properties.get('hideHttpsConnects') && !filter.hide) {
          proxy.emit('request', reqEmitter);
        }

        if (disable.tunnel) {
          return reqSocket.destroy();
        }

        if (filter.rule) {
          delete _rules.rule;
          plugin = null;
        }

        var tunnelPort, proxyUrl;
        if(plugin) {
          tunnelPort = plugin.ports && plugin.ports.tunnelPort;
          if (!tunnelPort) {
            return emitError(new Error('No plugin.tunnelServer'));
          }
          proxyUrl = 'proxy://127.0.0.1:' + tunnelPort;
        }

        var realUrl = _rules.rule && _rules.rule.url.replace('https:', 'tunnel:');
        if (/^tunnel:\/\//.test(realUrl) && realUrl != tunnelUrl) {
          tunnelUrl = realUrl;
          data.realUrl = realUrl.replace('tunnel://', '');
          parseUrl();
        }

        rules.getProxy(tunnelUrl, tunnelPort || proxyUrl ? null : req, function() {
          if (!proxyUrl) {
            proxyUrl = !tunnelPort && _rules.proxy ? _rules.proxy.matcher : null;
          }

          if (proxyUrl) {
            var isSocks = /^socks:\/\//.test(proxyUrl);
            var _url = 'http:' + util.removeProtocol(proxyUrl);
            data.proxy = true;
            resolveHost(_url, function(ip) {
              options = parseUrl(_url, isSocks ? 1080 : 80);
              if (options.port == config.port && util.isLocalAddress(ip)) {
                return emitError(new Error('Can not proxy to itself'));
              }
              var onConnect = function(_proxySocket) {
                proxySocket = _proxySocket;
                abortIfUnavailable(proxySocket);
                reqSocket.pipe(proxySocket).pipe(reqSocket);
                sendEstablished();
              };
              var dstOptions = url.parse(tunnelUrl);
              dstOptions.proxyHost = ip;
              dstOptions.proxyPort = parseInt(options.port, 10);
              dstOptions.port = dstOptions.port || 443;
              dstOptions.host = dstOptions.hostname;
              var _headers = extend({}, headers);
              if (req.clientIp) {
                _headers[pluginMgr.CLIENT_IP_HEADER] = req.clientIp;
              }
              var ruleValue = tunnelPort && util.getMatcherValue(_rules.rule);
              if (ruleValue) {
                _headers[pluginMgr.RULE_VALUE_HEADER] = encodeURIComponent(ruleValue);
              }
              _headers.host = dstOptions.hostname + ':' + (dstOptions.port || 443);
              dstOptions.headers = _headers;
              if (isSocks) {
                dstOptions.proxyPort = options.port || 1080;
                dstOptions.localDNS = false;
                dstOptions.auths = config.getAuths(options);
                socks.connect(dstOptions, onConnect).on('error', emitError);
              } else {
                dstOptions.proxyPort = options.port || 80;
                dstOptions.proxyAuth = options.auth;
                config.connect(dstOptions, onConnect).on('error', emitError);
              }
            });
          } else {
            tunnel();
          }
        });
      });

      function tunnel() {
        resolveHost(tunnelUrl, function(ip, port) {
          resData.ip = port ? ip + ':' + port : ip;
          resSocket = net.connect(port || options.port, ip, function() {
            resSocket.pipe(reqSocket).pipe(resSocket);
            sendEstablished();
          });
          abortIfUnavailable(resSocket);
        });
      }

      function resolveHost(url, callback) {
        rules.resolveHost(url, function(err, ip, port, host) {
          if (host) {
            _rules.host = host;
          }
          data.requestTime = data.dnsTime = Date.now();
          resData.ip = ip || LOCALHOST;
          reqEmitter.emit('send', data);
          err ? emitError(err) : callback(ip, port);
        }, rulesMgr);
      }

      function abortIfUnavailable(socket) {
        socket.on('error', emitError).on('close', emitError);
      }

      function sendEstablished() {
        reqSocket.write('HTTP/1.1 200 Connection Established\r\nProxy-Agent: ' + config.name + '\r\n\r\n');
        responsed = true;
        if (reqEmitter) {
          data.responseTime = data.endTime = Date.now();
          resData.statusCode = 200;
          reqEmitter.emit('response', data);
          reqEmitter.emit('end', data);
        }

        return reqSocket;
      }

      function emitError(err) {
        if (responsed) {
          return;
        }
        responsed = true;
        resSocket && resSocket.destroy();
        proxySocket && proxySocket.destroy();
        reqSocket.destroy();
        data.responseTime = data.endTime = Date.now();

        if (!resData.ip) {
          resData.ip = LOCALHOST;
        }

        if (err) {
          err = new Error('Aborted');
          data.reqError = true;
          resData.statusCode ='aborted';
          reqData.body = util.getErrorStack(err);
          reqEmitter.emit('abort', data);
        } else {
          data.resError = true;
          resData.statusCode = resData.statusCode || 502;
          resData.body = util.getErrorStack(err);
          util.emitError(reqEmitter, data);
        }
      }
    });
  });

  return server;
}

function proxy(callback) {
  var app = express();
  var server = app.listen(config.port, callback);
  var proxyEvents = new EventEmitter();
  var middlewares = ['./init',
'../biz']
.concat(require('./inspectors'))
.concat(config.middlewares)
.concat(require('./handlers'));

  proxyEvents.config = config;
  app.logger = logger;
  middlewares.forEach(function(mw) {
    mw && app.use((typeof mw == 'string' ? require(mw) : mw).bind(proxyEvents));
  });

  exportInterfaces(proxyEvents);
  tunnelProxy(server, proxyEvents);
  initDataServer(proxyEvents);
  initLogServer(proxyEvents);
  require('../biz/init')(proxyEvents);
  config.debug && rules.disableDnsCache();
  config.rules && rulesUtil.parseRules(config.rules);
  if (config.values) {
    Object.keys(config.values).forEach(function(name) {
      rulesUtil.values.add(name, config.values[name]);
    });
  }
  return proxyEvents;
}

function exportInterfaces(obj) {
  obj.rules = rules;
  obj.util = util;
  obj.rulesUtil = rulesUtil;
  obj.httpsUtil = httpsUtil;
  obj.pluginMgr = pluginMgr;
  obj.logger = logger;
  return obj;
}

process.on('uncaughtException', function(err){
  var stack = util.getErrorStack(err);
  fs.writeFileSync(path.join(process.cwd(), config.name + '.log'), '\r\n' + stack + '\r\n', {flag: 'a'});
  /*eslint no-console: "off"*/
  console.error(stack);
  process.exit(1);
});

module.exports = exportInterfaces(proxy);
<|endoftext|>


<commit_before>// @flow
/* eslint-disable class-methods-use-this */
import React, { PureComponent } from 'react';
import Rx from 'rxjs/Rx';
import CodeMirror from 'react-codemirror';
import CM from 'codemirror';

import 'codemirror/addon/hint/show-hint';
import 'codemirror/addon/hint/anyword-hint';
import 'codemirror/addon/search/search';
import 'codemirror/addon/search/searchcursor';
import 'codemirror/addon/search/match-highlighter.js';
import 'codemirror/addon/search/matchesonscrollbar.js';
import 'codemirror/addon/edit/matchbrackets';
import 'codemirror/addon/edit/closebrackets';
import 'codemirror/addon/dialog/dialog';
import 'codemirror/addon/comment/comment.js';
import 'codemirror/addon/selection/mark-selection.js';

import 'codemirror/mode/python/python';
import 'codemirror/mode/ruby/ruby';
import 'codemirror/mode/javascript/javascript';
import 'codemirror/mode/css/css';
import 'codemirror/mode/julia/julia';
import 'codemirror/mode/r/r';
import 'codemirror/mode/clike/clike';
import 'codemirror/mode/shell/shell';
import 'codemirror/mode/sql/sql';
import 'codemirror/mode/markdown/markdown';
import 'codemirror/mode/gfm/gfm';

const oldFindNext = CM.commands.findNext;
CM.commands.findNext = (editor) => {
  const before = editor.getCursor();
  oldFindNext(editor);
  const after = editor.getCursor();

  if (before.line > after.line ||
      (before.line === after.line && before.char > after.char) ) {
    CM.signal(editor, 'wrapNext');
  }
};

const oldFindPrev = CM.commands.findPrev;
CM.commands.findPrev = (editor) => {
  const before = editor.getCursor();
  oldFindPrev(editor);
  const after = editor.getCursor();

  if (before.line < after.line ||
      (before.line === after.line && before.char < after.char) ) {
    CM.signal(editor, 'wrapPrev');
  }
};

import { findInNotebook } from '../../../actions';

CM.keyMap.default["Ctrl-H"] = "replace";
CM.keyMap.default["Cmd-H"] = "replace";
delete CM.keyMap.default["Shift-Ctrl-F"];

import './codemirror-ipython';
import excludedIntelliSenseTriggerKeys from './excludedIntelliSenseKeys';
import { codeComplete, pick } from './complete';

type WrapperProps = {
  id: string,
  input: any,
  editorFocused: boolean,
  setFocusedEditor: (editor) => void,
  cellFocused: boolean,
  completion: boolean,
  focusAbove: () => void,
  focusBelow: () => void,
  wrapNext: () => void,
  wrapPrev: () => void,
  theme: string,
  channels: any,
  cursorBlinkRate: number,
  executionState: 'idle' | 'starting' | 'not connected',
  language: string,
  onChange: (text: string) => void,
  onFocusChange: (focused: boolean) => void,
  searchText: string,
}

type FunctionalComponent<P> = (props: P) => React.Element<*>
type ClassComponent<P> = Class<React.Component<void, P, void>>

type CodeMirrorHOC = (
  E: ClassComponent<*> | FunctionalComponent<*>,
  C?: { [key: string]: any } | null
) => ClassComponent<WrapperProps>

const CodeMirrorWrapper: CodeMirrorHOC = (EditorView, customOptions = null) =>
  class CodeMirrorEditor extends PureComponent<void, WrapperProps, void> {
    codemirror: Object;
    getCodeMirrorOptions: (p: WrapperProps) => Object;
    goLineUpOrEmit: (editor: Object) => void;
    goLineDownOrEmit: (editor: Object) => void;
    hint: (editor: Object, cb: Function) => void;

    static contextTypes = {
      store: React.PropTypes.object,
    };

    constructor(): void {
      super();

      this.hint = this.completions.bind(this);
      this.hint.async = true;
      this.clearSearchInNotebook = this.clearSearchInNotebook.bind(this);
    }

    componentDidMount(): void {
      const { editorFocused, executionState, focusAbove, focusBelow, wrapNext, wrapPrev } = this.props;
      const cm = this.codemirror.getCodeMirror();

      // On first load, if focused, set codemirror to focus
      if (editorFocused) {
        this.codemirror.focus();
        this.props.setFocusedEditor(cm);
      }

      cm.on('topBoundary', focusAbove);
      cm.on('bottomBoundary', focusBelow);
      cm.on('wrapNext', wrapNext);
      cm.on('wrapPrev', wrapPrev);
//       this.codemirror.commands.findNext = this.findNextOrEmit;
//       cm.commands.findNext = this.findNextOrEmit;
//       cm.commands.findNext1 = null;
//       cm.commands.xyz = null;

      const keyupEvents = Rx.Observable.fromEvent(cm, 'keyup', (editor, ev) => ({ editor, ev }));

      keyupEvents
        .switchMap(i => Rx.Observable.of(i))
        .subscribe(({ editor, ev }) => {
          const cursor = editor.getDoc().getCursor();
          const token = editor.getTokenAt(cursor);

          if (!editor.state.completionActive &&
              !excludedIntelliSenseTriggerKeys[(ev.keyCode || ev.which).toString()] &&
              (token.type === 'tag' || token.type === 'variable' || token.string === ' ' ||
               token.string === '<' || token.string === '/' || token.string === '.') && executionState === 'idle') {
            editor.execCommand('autocomplete', { completeSingle: false });
          }
        });
    }

    componentDidUpdate(prevProps: WrapperProps): void {
      const cm = this.codemirror.getCodeMirror();
      const { cursorBlinkRate, editorFocused, theme, searchText } = this.props;

      if (prevProps.theme !== theme) {
        cm.refresh();
      }

      if (prevProps.searchText !== searchText) {
        if (searchText && searchText.length > 0) {
//           cm.markText({line: 0, ch: 0}, {line: 0, ch: 4}, {className: "text-highlight-background"});
//           cm.addOverlay({token: (stream) => {
//               if (stream.match(searchText))
//                 return 'matchhighlight';
//               stream.next();
//               stream.skipTo(searchText.charAt(0)) || stream.skipToEnd();
//             }
//           });
          const queryCaseInsensitive = (query) => (typeof query == "string" && query == query.toLowerCase());
          if (cm.state.search && cm.state.search.overlay) {
            cm.removeOverlay(cm.state.search.overlay, queryCaseInsensitive(searchText));
          }

          let query;
          const caseInsensitive = false;
          // https://codemirror.net/addon/search/search.js
          if (true) // is string query
            query = new RegExp(searchText.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&"), caseInsensitive ? "gi" : "g");
          else // regex query
            query = new RegExp(searchText.source, searchText.ignoreCase ? "gi" : "g");
          cm.state.search = {
            posFrom: null,
            posTo: null,
            lastQuery: query,
            query: query,
            queryText: searchText,
            overlay: {token: (stream) => {
              query.lastIndex = stream.pos;
              const match = query.exec(stream.string);
              if (match && match.index == stream.pos) {
                stream.pos += match[0].length || 1;
                return "searching";
              } else if (match) {
                stream.pos = match.index;
              } else {
                stream.skipToEnd();
              }
            }},
          };
          cm.addOverlay(cm.state.search.overlay);
//           cm.execCommand('findPersistent');
//           cm.execCommand('findPersistentNext');
//           cm.execCommand('find');
          cm.state.search.posFrom = cm.state.search.posTo = cm.getCursor();
          cm.execCommand('findNext');
        } else {
          // clear the search
          cm.execCommand('clearSearch');
        }
      }

      if (prevProps.editorFocused !== editorFocused) {
        if (editorFocused) {
          this.codemirror.focus();
          this.props.setFocusedEditor(cm);
        } else {
          cm.getInputField().blur();
        }
      }

      if (prevProps.cursorBlinkRate !== cursorBlinkRate) {
        cm.setOption('cursorBlinkRate', cursorBlinkRate);
        if (editorFocused) {
          // code mirror doesn't change the blink rate immediately, we have to
          // move the cursor, or unfocus and refocus the editor to get the blink
          // rate to update - so here we do that (unfocus and refocus)
          cm.getInputField().blur();
          cm.focus();
        }
      }
    }

    completions(editor: Object, callback: Function): void {
      const { completion, channels } = this.props;
      if (completion) {
        codeComplete(channels, editor)
          .subscribe(callback);
      }
    }

    getCodeMirrorOptions({ cursorBlinkRate, language }: WrapperProps): Object {
      return {
        autoCloseBrackets: true,
        mode: language || 'python',
        lineNumbers: false,
        lineWrapping: true,
        matchBrackets: true,
        theme: 'composition',
        autofocus: false,
        hintOptions: {
          hint: this.hint,
          completeSingle: false, // In automatic autocomplete mode we don't want override
          extraKeys: {
            Right: pick,
          },
        },
        extraKeys: {
          'Ctrl-Space': 'autocomplete',
          Tab: editor => editor.execCommand('insertSoftTab'),
          Up: this.goLineUpOrEmit,
          Down: this.goLineDownOrEmit,
          'Cmd-/': 'toggleComment',
          'Ctrl-/': 'toggleComment',
          'Esc': this.clearSearchInNotebook,
        },
        indentUnit: 4,
        cursorBlinkRate,
        ...customOptions
      };
    }

    goLineDownOrEmit(editor: Object): void {
      const cursor = editor.getCursor();
      const lastLineNumber = editor.lastLine();
      const lastLine = editor.getLine(lastLineNumber);
      if (cursor.line === lastLineNumber &&
          cursor.ch === lastLine.length &&
          !editor.somethingSelected()) {
        CM.signal(editor, 'bottomBoundary');
      } else {
        editor.execCommand('goLineDown');
      }
    }

    goLineUpOrEmit(editor: Object): void {
      const cursor = editor.getCursor();
      if (cursor.line === 0 && cursor.ch === 0 && !editor.somethingSelected()) {
        CM.signal(editor, 'topBoundary');
      } else {
        editor.execCommand('goLineUp');
      }
    }

    clearSearchInNotebook(editor: Object): void {
      this.context.store.dispatch(findInNotebook());
      // clear search in case it was a local in-cell search
      editor.execCommand('clearSearch');
    }

    render(): React.Element<*> {
      const { input, onChange, onFocusChange } = this.props;
      const options = this.getCodeMirrorOptions(this.props);

      return (
        <EditorView {...this.props}>
          <CodeMirror
            value={input}
            ref={(el) => { this.codemirror = el; }}
            className="cell_cm"
            options={options}
            onChange={onChange}
            onClick={() => {this.codemirror.focus(); this.props.setFocusedEditor(this.codemirror.getCodeMirror());}}
            onFocusChange={onFocusChange}
          />
        </EditorView>
      );
    }
  };

export default CodeMirrorWrapper;
<commit_msg>remove diag<commit_after>// @flow
/* eslint-disable class-methods-use-this */
import React, { PureComponent } from 'react';
import Rx from 'rxjs/Rx';
import CodeMirror from 'react-codemirror';
import CM from 'codemirror';

import 'codemirror/addon/hint/show-hint';
import 'codemirror/addon/hint/anyword-hint';
import 'codemirror/addon/search/search';
import 'codemirror/addon/search/searchcursor';
import 'codemirror/addon/search/match-highlighter.js';
import 'codemirror/addon/search/matchesonscrollbar.js';
import 'codemirror/addon/edit/matchbrackets';
import 'codemirror/addon/edit/closebrackets';
import 'codemirror/addon/dialog/dialog';
import 'codemirror/addon/comment/comment.js';
import 'codemirror/addon/selection/mark-selection.js';

import 'codemirror/mode/python/python';
import 'codemirror/mode/ruby/ruby';
import 'codemirror/mode/javascript/javascript';
import 'codemirror/mode/css/css';
import 'codemirror/mode/julia/julia';
import 'codemirror/mode/r/r';
import 'codemirror/mode/clike/clike';
import 'codemirror/mode/shell/shell';
import 'codemirror/mode/sql/sql';
import 'codemirror/mode/markdown/markdown';
import 'codemirror/mode/gfm/gfm';

const oldFindNext = CM.commands.findNext;
CM.commands.findNext = (editor) => {
  const before = editor.getCursor();
  oldFindNext(editor);
  const after = editor.getCursor();

  if (before.line > after.line ||
      (before.line === after.line && before.char > after.char) ) {
    CM.signal(editor, 'wrapNext');
  }
};

const oldFindPrev = CM.commands.findPrev;
CM.commands.findPrev = (editor) => {
  const before = editor.getCursor();
  oldFindPrev(editor);
  const after = editor.getCursor();

  if (before.line < after.line ||
      (before.line === after.line && before.char < after.char) ) {
    CM.signal(editor, 'wrapPrev');
  }
};

import { findInNotebook } from '../../../actions';

CM.keyMap.default["Ctrl-H"] = "replace";
CM.keyMap.default["Cmd-H"] = "replace";
delete CM.keyMap.default["Shift-Ctrl-F"];

import './codemirror-ipython';
import excludedIntelliSenseTriggerKeys from './excludedIntelliSenseKeys';
import { codeComplete, pick } from './complete';

type WrapperProps = {
  id: string,
  input: any,
  editorFocused: boolean,
  setFocusedEditor: (editor) => void,
  cellFocused: boolean,
  completion: boolean,
  focusAbove: () => void,
  focusBelow: () => void,
  wrapNext: () => void,
  wrapPrev: () => void,
  theme: string,
  channels: any,
  cursorBlinkRate: number,
  executionState: 'idle' | 'starting' | 'not connected',
  language: string,
  onChange: (text: string) => void,
  onFocusChange: (focused: boolean) => void,
  searchText: string,
}

type FunctionalComponent<P> = (props: P) => React.Element<*>
type ClassComponent<P> = Class<React.Component<void, P, void>>

type CodeMirrorHOC = (
  E: ClassComponent<*> | FunctionalComponent<*>,
  C?: { [key: string]: any } | null
) => ClassComponent<WrapperProps>

const CodeMirrorWrapper: CodeMirrorHOC = (EditorView, customOptions = null) =>
  class CodeMirrorEditor extends PureComponent<void, WrapperProps, void> {
    codemirror: Object;
    getCodeMirrorOptions: (p: WrapperProps) => Object;
    goLineUpOrEmit: (editor: Object) => void;
    goLineDownOrEmit: (editor: Object) => void;
    hint: (editor: Object, cb: Function) => void;

    static contextTypes = {
      store: React.PropTypes.object,
    };

    constructor(): void {
      super();

      this.hint = this.completions.bind(this);
      this.hint.async = true;
      this.clearSearchInNotebook = this.clearSearchInNotebook.bind(this);
    }

    componentDidMount(): void {
      const { editorFocused, executionState, focusAbove, focusBelow, wrapNext, wrapPrev } = this.props;
      const cm = this.codemirror.getCodeMirror();

      // On first load, if focused, set codemirror to focus
      if (editorFocused) {
        this.codemirror.focus();
        this.props.setFocusedEditor(cm);
      }

      cm.on('topBoundary', focusAbove);
      cm.on('bottomBoundary', focusBelow);
      cm.on('wrapNext', wrapNext);
      cm.on('wrapPrev', wrapPrev);
//       this.codemirror.commands.findNext = this.findNextOrEmit;
//       cm.commands.findNext = this.findNextOrEmit;
//       cm.commands.findNext1 = null;
//       cm.commands.xyz = null;

      const keyupEvents = Rx.Observable.fromEvent(cm, 'keyup', (editor, ev) => ({ editor, ev }));

      keyupEvents
        .switchMap(i => Rx.Observable.of(i))
        .subscribe(({ editor, ev }) => {
          const cursor = editor.getDoc().getCursor();
          const token = editor.getTokenAt(cursor);

          if (!editor.state.completionActive &&
              !excludedIntelliSenseTriggerKeys[(ev.keyCode || ev.which).toString()] &&
              (token.type === 'tag' || token.type === 'variable' || token.string === ' ' ||
               token.string === '<' || token.string === '/' || token.string === '.') && executionState === 'idle') {
            editor.execCommand('autocomplete', { completeSingle: false });
          }
        });
    }

    componentDidUpdate(prevProps: WrapperProps): void {
      const cm = this.codemirror.getCodeMirror();
      const { cursorBlinkRate, editorFocused, theme, searchText } = this.props;

      if (prevProps.theme !== theme) {
        cm.refresh();
      }

      if (prevProps.searchText !== searchText) {
        if (searchText && searchText.length > 0) {
//           cm.markText({line: 0, ch: 0}, {line: 0, ch: 4}, {className: "text-highlight-background"});
//           cm.addOverlay({token: (stream) => {
//               if (stream.match(searchText))
//                 return 'matchhighlight';
//               stream.next();
//               stream.skipTo(searchText.charAt(0)) || stream.skipToEnd();
//             }
//           });
          const queryCaseInsensitive = (query) => (typeof query == "string" && query == query.toLowerCase());
          if (cm.state.search && cm.state.search.overlay) {
            cm.removeOverlay(cm.state.search.overlay, queryCaseInsensitive(searchText));
          }

          let query;
          const caseInsensitive = false;
          // https://codemirror.net/addon/search/search.js
          if (true) // is string query
            query = new RegExp(searchText.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&"), caseInsensitive ? "gi" : "g");
          else // regex query
            query = new RegExp(searchText.source, searchText.ignoreCase ? "gi" : "g");

          cm.state.search = {
            posFrom: null,
            posTo: null,
            lastQuery: query,
            query: query,
            queryText: searchText,
            overlay: {token: (stream) => {
              query.lastIndex = stream.pos;
              const match = query.exec(stream.string);
              if (match && match.index == stream.pos) {
                stream.pos += match[0].length || 1;
                return "searching";
              } else if (match) {
                stream.pos = match.index;
              } else {
                stream.skipToEnd();
              }
            }},
          };
          cm.addOverlay(cm.state.search.overlay);
          cm.state.search.posFrom = cm.state.search.posTo = cm.getCursor();
          cm.execCommand('findNext');
        } else {
          // clear the search
          cm.execCommand('clearSearch');
        }
      }

      if (prevProps.editorFocused !== editorFocused) {
        if (editorFocused) {
          this.codemirror.focus();
          this.props.setFocusedEditor(cm);
        } else {
          cm.getInputField().blur();
        }
      }

      if (prevProps.cursorBlinkRate !== cursorBlinkRate) {
        cm.setOption('cursorBlinkRate', cursorBlinkRate);
        if (editorFocused) {
          // code mirror doesn't change the blink rate immediately, we have to
          // move the cursor, or unfocus and refocus the editor to get the blink
          // rate to update - so here we do that (unfocus and refocus)
          cm.getInputField().blur();
          cm.focus();
        }
      }
    }

    completions(editor: Object, callback: Function): void {
      const { completion, channels } = this.props;
      if (completion) {
        codeComplete(channels, editor)
          .subscribe(callback);
      }
    }

    getCodeMirrorOptions({ cursorBlinkRate, language }: WrapperProps): Object {
      return {
        autoCloseBrackets: true,
        mode: language || 'python',
        lineNumbers: false,
        lineWrapping: true,
        matchBrackets: true,
        theme: 'composition',
        autofocus: false,
        hintOptions: {
          hint: this.hint,
          completeSingle: false, // In automatic autocomplete mode we don't want override
          extraKeys: {
            Right: pick,
          },
        },
        extraKeys: {
          'Ctrl-Space': 'autocomplete',
          Tab: editor => editor.execCommand('insertSoftTab'),
          Up: this.goLineUpOrEmit,
          Down: this.goLineDownOrEmit,
          'Cmd-/': 'toggleComment',
          'Ctrl-/': 'toggleComment',
          'Esc': this.clearSearchInNotebook,
        },
        indentUnit: 4,
        cursorBlinkRate,
        ...customOptions
      };
    }

    goLineDownOrEmit(editor: Object): void {
      const cursor = editor.getCursor();
      const lastLineNumber = editor.lastLine();
      const lastLine = editor.getLine(lastLineNumber);
      if (cursor.line === lastLineNumber &&
          cursor.ch === lastLine.length &&
          !editor.somethingSelected()) {
        CM.signal(editor, 'bottomBoundary');
      } else {
        editor.execCommand('goLineDown');
      }
    }

    goLineUpOrEmit(editor: Object): void {
      const cursor = editor.getCursor();
      if (cursor.line === 0 && cursor.ch === 0 && !editor.somethingSelected()) {
        CM.signal(editor, 'topBoundary');
      } else {
        editor.execCommand('goLineUp');
      }
    }

    clearSearchInNotebook(editor: Object): void {
      this.context.store.dispatch(findInNotebook());
      // clear search in case it was a local in-cell search
      editor.execCommand('clearSearch');
    }

    render(): React.Element<*> {
      const { input, onChange, onFocusChange } = this.props;
      const options = this.getCodeMirrorOptions(this.props);

      return (
        <EditorView {...this.props}>
          <CodeMirror
            value={input}
            ref={(el) => { this.codemirror = el; }}
            className="cell_cm"
            options={options}
            onChange={onChange}
            onClick={() => {this.codemirror.focus(); this.props.setFocusedEditor(this.codemirror.getCodeMirror());}}
            onFocusChange={onFocusChange}
          />
        </EditorView>
      );
    }
  };

export default CodeMirrorWrapper;
<|endoftext|>


<commit_before>var Schema = require("truffle-contract-schema");
var fs = require("fs-extra");
var path = require("path");
var async = require("async");
var _ = require("lodash");

function Artifactor(contracts_build_directory) {
  this.contracts_build_directory = contracts_build_directory;
};

Artifactor.prototype.save = function(options, extra_options) {
  var self = this;

  return new Promise(function(accept, reject) {
    options = Schema.normalizeOptions(options, extra_options);

    if (options.contract_name == null) {
      return reject("You must specify a contract name.");
    }

    var filename = path.resolve(path.join(self.contracts_build_directory, options.contract_name + ".json"));

    fs.readFile(filename, {encoding: "utf8"}, function(err, json) {
      // No need to handle the error. If the file doesn't exist then we'll start afresh
      // with a new binary (see generateBinary()).
      var existing_binary;

      if (!err) {
        try {
          existing_binary = JSON.parse(json);
        } catch (e) {
          // Do nothing
        }
      }

      var final_binary;
      try {
        final_binary = Schema.generateBinary(options, existing_binary);
      } catch (e) {
        return reject(e);
      }

      fs.outputFile(filename, JSON.stringify(final_binary, null, 2), "utf8", function(err) {
        if (err) return reject(err);
        accept();
      });
    });
  });
};

Artifactor.prototype.saveAll = function(contracts, options) {
  var self = this;
  options = options || {};

  if (Array.isArray(contracts)) {
    var arr = contracts;
    contracts = {};
    arr.forEach(function(contract) {
      contracts[contract.contract_name] = contract;
    });
  }

  return new Promise(function(accept, reject) {
    var destination = self.contracts_build_directory;

    fs.stat(destination, function(err, stat) {
      if (err) {
        return reject(new Error("Desination " + destination + " doesn't exist!"));
      }

      async.each(Object.keys(contracts), function(contract_name, done) {
        var contract_data = contracts[contract_name];
        contract_data.contract_name = contract_data.contract_name || contract_name;

        var filename = path.join(destination, contract_name + ".json");

        // Add the contract name to our extra options, without editing
        // the options object on its own.
        options = _.extend({}, options, {contract_name: contract_name});

        // Finally save the contract.
        self.save(contract_data, options).then(done).catch(done);
      }, function(err) {
        if (err) return reject(err);
        accept();
      });
    });
  });
};

module.exports = Artifactor;
<commit_msg>Remove unnecessary code<commit_after>var Schema = require("truffle-contract-schema");
var fs = require("fs-extra");
var path = require("path");
var async = require("async");
var _ = require("lodash");

function Artifactor(contracts_build_directory) {
  this.contracts_build_directory = contracts_build_directory;
};

Artifactor.prototype.save = function(options, extra_options) {
  var self = this;

  return new Promise(function(accept, reject) {
    options = Schema.normalizeOptions(options, extra_options);

    if (options.contract_name == null) {
      return reject("You must specify a contract name.");
    }

    var filename = path.resolve(path.join(self.contracts_build_directory, options.contract_name + ".json"));

    fs.readFile(filename, {encoding: "utf8"}, function(err, json) {
      // No need to handle the error. If the file doesn't exist then we'll start afresh
      // with a new binary (see generateBinary()).
      var existing_binary;

      if (!err) {
        try {
          existing_binary = JSON.parse(json);
        } catch (e) {
          // Do nothing
        }
      }

      var final_binary;
      try {
        final_binary = Schema.generateBinary(options, existing_binary);
      } catch (e) {
        return reject(e);
      }

      fs.outputFile(filename, JSON.stringify(final_binary, null, 2), "utf8", function(err) {
        if (err) return reject(err);
        accept();
      });
    });
  });
};

Artifactor.prototype.saveAll = function(contracts, options) {
  var self = this;
  options = options || {};

  if (Array.isArray(contracts)) {
    var arr = contracts;
    contracts = {};
    arr.forEach(function(contract) {
      contracts[contract.contract_name] = contract;
    });
  }

  return new Promise(function(accept, reject) {
    var destination = self.contracts_build_directory;

    fs.stat(destination, function(err, stat) {
      if (err) {
        return reject(new Error("Desination " + destination + " doesn't exist!"));
      }

      async.each(Object.keys(contracts), function(contract_name, done) {
        var contract_data = contracts[contract_name];
        contract_data.contract_name = contract_data.contract_name || contract_name;

        var filename = path.join(destination, contract_name + ".json");

        // Finally save the contract.
        self.save(contract_data).then(done).catch(done);
      }, function(err) {
        if (err) return reject(err);
        accept();
      });
    });
  });
};

module.exports = Artifactor;
<|endoftext|>


<commit_before>export default (connection, Sequelize) => {
  const GroupMember = connection.define('groupMember', {
    userId: {
      type: Sequelize.UUID,
      allowNull: false,
    },
    // References userId in User Model
    groupId: {
      type: Sequelize.UUID,
      allowNull: false
    }
  }, {
    paranoid: true
  });
  return GroupMember;
};
<commit_msg>Add groupMember (group membership) model<commit_after>export default (connection, Sequelize) => {
  const GroupMember = connection.define('groupMember', {
    membershipId: {
      type: Sequelize.UUID,
      defaultValue: Sequelize.UUIDV4,
      primaryKey: true
    },
    // References userId in the User model
    userId: {
      type: Sequelize.UUID,
      allowNull: false,
    },
    // References groupId in the Group Model
    groupId: {
      type: Sequelize.UUID,
      allowNull: false
    }
  }, {
    paranoid: true
  });
  return GroupMember;
};
<|endoftext|>


<commit_before>/* @flow */

import 'babel-polyfill'
import path from 'path'

import { Builder } from '../src/main'
import type { Message } from '../src/types'

describe('Builder', () => {
  let builder: Builder
  let messages: Array<Message>

  beforeEach(async (done) => {
    const options = { ignoreCache: true, severity: 'error' }
    messages = []
    builder = await Builder.create(path.resolve(__dirname, 'fixtures', 'foo.tex'), options, message => { messages.push(message) })
    done()
  })

  it('verifies that Builder run returns expected true', async (done) => {
    expect(await builder.run('build')).toBeTruthy()
    expect(messages).toEqual([])
    done()
  }, 10000)
})
<commit_msg>Add log message display in spec<commit_after>/* @flow */

import 'babel-polyfill'
import path from 'path'

import { Builder } from '../src/main'
import type { Message } from '../src/types'

describe('Builder', () => {
  let builder: Builder
  let messages: Array<Message>

  beforeEach(async (done) => {
    const options = { ignoreCache: true, severity: 'error', reportLogMesssages: true }
    messages = []
    builder = await Builder.create(path.resolve(__dirname, 'fixtures', 'foo.tex'), options, message => { messages.push(message) })
    done()
  })

  it('verifies that Builder run returns expected true', async (done) => {
    expect(await builder.run('build')).toBeTruthy()
    expect(messages).toEqual([])
    done()
  }, 10000)
})
<|endoftext|>


<commit_before>var url       = require('url');
var qs        = require('querystring');
var http      = require('http');
var EventVat  = require('eventvat');
var request   = require('request');
var streamify = require('streamify');
var Hash      = require('hashish');
var FORMATS   = require('./formats');
var parseTime = require('./util').parseTime;

var INFO_URL = 'http://www.youtube.com/get_video_info?hl=en_US&el=detailpage&video_id=';
var KEYS_TO_SPLIT = [
  'keywords'
, 'fmt_list'
, 'fexp'
, 'watermark'
, 'ad_channel_code_overlay'
];

// Cache for recently looked up video infos.
var cache = ytdl.cache = new EventVat({ autoexpire: 30 });

/**
 * Checks cache to see if info for this video has been fetched recently.
 * If not, get it, then download the video once the info is found.
 *
 * @param {String} link
 * @param {Object} options
 * @return {ReadableStream}
 */
function ytdl(link, options) {
  var id = url.parse(link, true).query.v;
  var stream = streamify({
    superCtor: http.ClientResponse,
    readable: true,
    writable: false
  });

  options = options || {};
  options.quality = options.quality || 'highest';

  if (cache.exists(id)) {
    var info = cache.get(id);
    process.nextTick(function() {
      download(options, stream, info);
    });

  } else {
    ytdl.getInfo(link, options, function(err, info) {
      if (err) {
        stream.emit('error', err);
        stream.readable = false;
        return;
      }

      download(options, stream, info);
    });
  }

  return stream;
}

/**
 * Downloads a video after retrieving video info.
 *
 * @param {Object} options
 * @param {ReadableStream} stream
 * @param {Object} info
 * @return {ReadableStream}
 */
function download(options, stream, info) {
  var formats = info.formats;
  if (options.filter) {
    formats = formats.filter(options.filter);
  }

  var format;
  switch (options.quality) {
    case 'highest':
      format = formats[0];
      break;

    case 'lowest':
      format = formats[formats.length - 1];
      break;

    default:
      format = formats.filter(function(format) {
        return format.itag === '' + options.quality;
      })[0];

  }

  if (!format) {
    stream.emit('error', new Error('No such format found'));
    return;
  }

  format = Hash.copy(format);
  var parsedUrl = url.parse(format.url, true);
  var requestOptions = Hash.copy(options);
  delete requestOptions.quality;
  delete requestOptions.start;
  delete requestOptions.filter;

  if (options.start) {
    parsedUrl.query.begin = parseTime(options.start);
  }

  format.url = requestOptions.url = url.format(parsedUrl);

  // Start downloading the video.
  var res = request(requestOptions);
  res.on('response', function(res) {
    if (res.statusCode !== 200) {
      return stream.emit('error', new Error('status code ' + res.statusCode));
    }

    format.size = res.headers['content-length'];
    stream.emit('info', info, format);
  });
  stream.resolve(res);
}

/**
 * Gets info from a video.
 *
 * @param {String} link
 * @param {Object} requestOptions
 * @param {Function(Error, Object)} callback
 */
ytdl.getInfo = function getInfo(link, requestOptions, callback) {
  if (typeof requestOptions === 'function') {
    callback = requestOptions;
    requestOptions = {};
  } else {
    requestOptions = Hash.copy(requestOptions);
  }

  var linkParsed = url.parse(link, true);
  var id = linkParsed.hostname === 'youtu.be'
    ? linkParsed.pathname.slice(1) : linkParsed.query.v;

  if (!id) {
    process.nextTick(function() {
      callback(new Error('Video ID not found'));
    });
    return;
  }

  requestOptions.url = INFO_URL + id;

  request(requestOptions, function(err, res, body) {
    if (err) return callback(err);
    if (res.statusCode !== 200) {
      return callback(new Error('Video not found: ' + link));
    }

    var info = qs.parse(body);

    if (info.status === 'fail') {
      callback(new Error('Error ' + info.errorcode + ': ' + info.reason));
      return;
    }

    // Split some keys by commas.
    KEYS_TO_SPLIT.forEach(function(key) {
      if (!info[key]) return;
      info[key] = info[key].split(',').filter(function(v) { return v !== ''; });
    });

    // Convert some strings to javascript numbers and booleans.
    info = Hash.map(info, function(val) {
      var intVal = parseInt(val, 10);
      var floatVal = parseFloat(val, 10);

      if (intVal.toString() === val) {
        return intVal;
      } else if (floatVal.toString() === val) {
        return floatVal;
      } else if (val === 'True') {
        return true;
      } else if (val === 'False') {
        return false;
      } else {
        return val;
      }
    });

    info.fmt_list = info.fmt_list.map(function(format) {
      return format.split('/');
    });

    info.formats = info.url_encoded_fmt_stream_map
      .split(',')
      .map(function(format) {
        var data = qs.parse(format);
        if (data.sig) {
          data.url += '&signature=' + data.sig;
        }
        format = FORMATS[data.itag];

        if (!format) {
          err = new Error('No such format for itag ' + data.itag + ' found');
          return;
        }

        Hash(format).forEach(function(val, key) {
          data[key] = val;
        });

        return data;
      })
      ;

    delete info.url_encoded_fmt_stream_map;

    if (err) {
      return callback(err);
    }

    info.video_verticals = info.video_verticals
      .slice(1, -1)
      .split(', ')
      .filter(function(val) { return val !== ''; })
      .map(function(val) { return parseInt(val, 10); })
      ;

    cache.set(id, info);
    callback(null, info);
  });
};

/**
 * export
 */
module.exports = ytdl;
<commit_msg>Handle case if fmt_list or url_encoded_fmt_stream_map is empty<commit_after>var url       = require('url');
var qs        = require('querystring');
var http      = require('http');
var EventVat  = require('eventvat');
var request   = require('request');
var streamify = require('streamify');
var Hash      = require('hashish');
var FORMATS   = require('./formats');
var parseTime = require('./util').parseTime;

var INFO_URL = 'http://www.youtube.com/get_video_info?hl=en_US&el=detailpage&video_id=';
var KEYS_TO_SPLIT = [
  'keywords'
, 'fmt_list'
, 'fexp'
, 'watermark'
, 'ad_channel_code_overlay'
];

// Cache for recently looked up video infos.
var cache = ytdl.cache = new EventVat({ autoexpire: 30 });

/**
 * Checks cache to see if info for this video has been fetched recently.
 * If not, get it, then download the video once the info is found.
 *
 * @param {String} link
 * @param {Object} options
 * @return {ReadableStream}
 */
function ytdl(link, options) {
  var id = url.parse(link, true).query.v;
  var stream = streamify({
    superCtor: http.ClientResponse,
    readable: true,
    writable: false
  });

  options = options || {};
  options.quality = options.quality || 'highest';

  if (cache.exists(id)) {
    var info = cache.get(id);
    process.nextTick(function() {
      download(options, stream, info);
    });

  } else {
    ytdl.getInfo(link, options, function(err, info) {
      if (err) {
        stream.emit('error', err);
        stream.readable = false;
        return;
      }

      download(options, stream, info);
    });
  }

  return stream;
}

/**
 * Downloads a video after retrieving video info.
 *
 * @param {Object} options
 * @param {ReadableStream} stream
 * @param {Object} info
 * @return {ReadableStream}
 */
function download(options, stream, info) {
  var formats = info.formats;
  if (options.filter) {
    formats = formats.filter(options.filter);
  }

  var format;
  switch (options.quality) {
    case 'highest':
      format = formats[0];
      break;

    case 'lowest':
      format = formats[formats.length - 1];
      break;

    default:
      format = formats.filter(function(format) {
        return format.itag === '' + options.quality;
      })[0];

  }

  if (!format) {
    stream.emit('error', new Error('No such format found'));
    return;
  }

  format = Hash.copy(format);
  var parsedUrl = url.parse(format.url, true);
  var requestOptions = Hash.copy(options);
  delete requestOptions.quality;
  delete requestOptions.start;
  delete requestOptions.filter;

  if (options.start) {
    parsedUrl.query.begin = parseTime(options.start);
  }

  format.url = requestOptions.url = url.format(parsedUrl);

  // Start downloading the video.
  var res = request(requestOptions);
  res.on('response', function(res) {
    if (res.statusCode !== 200) {
      return stream.emit('error', new Error('status code ' + res.statusCode));
    }

    format.size = res.headers['content-length'];
    stream.emit('info', info, format);
  });
  stream.resolve(res);
}

/**
 * Gets info from a video.
 *
 * @param {String} link
 * @param {Object} requestOptions
 * @param {Function(Error, Object)} callback
 */
ytdl.getInfo = function getInfo(link, requestOptions, callback) {
  if (typeof requestOptions === 'function') {
    callback = requestOptions;
    requestOptions = {};
  } else {
    requestOptions = Hash.copy(requestOptions);
  }

  var linkParsed = url.parse(link, true);
  var id = linkParsed.hostname === 'youtu.be'
    ? linkParsed.pathname.slice(1) : linkParsed.query.v;

  if (!id) {
    process.nextTick(function() {
      callback(new Error('Video ID not found'));
    });
    return;
  }

  requestOptions.url = INFO_URL + id;

  request(requestOptions, function(err, res, body) {
    if (err) return callback(err);
    if (res.statusCode !== 200) {
      return callback(new Error('Video not found: ' + link));
    }

    var info = qs.parse(body);

    if (info.status === 'fail') {
      callback(new Error('Error ' + info.errorcode + ': ' + info.reason));
      return;
    }

    // Split some keys by commas.
    KEYS_TO_SPLIT.forEach(function(key) {
      if (!info[key]) return;
      info[key] = info[key].split(',').filter(function(v) { return v !== ''; });
    });

    // Convert some strings to javascript numbers and booleans.
    info = Hash.map(info, function(val) {
      var intVal = parseInt(val, 10);
      var floatVal = parseFloat(val, 10);

      if (intVal.toString() === val) {
        return intVal;
      } else if (floatVal.toString() === val) {
        return floatVal;
      } else if (val === 'True') {
        return true;
      } else if (val === 'False') {
        return false;
      } else {
        return val;
      }
    });

    if (info.fmt_list) {
      info.fmt_list = info.fmt_list.map(function(format) {
        return format.split('/');
      });
    } else {
      info.fmt_list = [];
    }

    if (info.url_encoded_fmt_stream_map) {
      info.formats = info.url_encoded_fmt_stream_map
        .split(',')
        .map(function(format) {
          var data = qs.parse(format);
          if (data.sig) {
            data.url += '&signature=' + data.sig;
          }
          format = FORMATS[data.itag];

          if (!format) {
            err = new Error('No such format for itag ' + data.itag + ' found');
            return;
          }

          Hash(format).forEach(function(val, key) {
            data[key] = val;
          });

          return data;
        });
    } else {
      info.formats = [];
    }

    delete info.url_encoded_fmt_stream_map;

    if (err) {
      return callback(err);
    }

    info.video_verticals = info.video_verticals
      .slice(1, -1)
      .split(', ')
      .filter(function(val) { return val !== ''; })
      .map(function(val) { return parseInt(val, 10); })
      ;

    cache.set(id, info);
    callback(null, info);
  });
};

/**
 * export
 */
module.exports = ytdl;
<|endoftext|>


<commit_before>import JSON5 from 'json5'
import limitConcurrency from 'limit-concurrency-decorator'
import { BaseError } from 'make-error'
import {
  endsWith,
  findKey,
  forEach,
  get,
  identity,
  map,
  mapValues,
  mean,
  sum,
  uniq,
  zipWith,
} from 'lodash'

import { parseDateTime } from './xapi'

export class FaultyGranularity extends BaseError {}

// -------------------------------------------------------------------

// according to https://xapi-project.github.io/xen-api/metrics.html
// The values are stored at intervals of:
//  - 5 seconds for the past 10 minutes
//  - one minute for the past 2 hours
//  - one hour for the past week
//  - one day for the past year
const RRD_STEP_SECONDS = 5
const RRD_STEP_MINUTES = 60
const RRD_STEP_HOURS = 3600
const RRD_STEP_DAYS = 86400

const RRD_STEP_FROM_STRING = {
  seconds: RRD_STEP_SECONDS,
  minutes: RRD_STEP_MINUTES,
  hours: RRD_STEP_HOURS,
  days: RRD_STEP_DAYS,
}

// points = intervalInSeconds / step
const RRD_POINTS_PER_STEP = {
  [RRD_STEP_SECONDS]: 120,
  [RRD_STEP_MINUTES]: 120,
  [RRD_STEP_HOURS]: 168,
  [RRD_STEP_DAYS]: 366,
}

// -------------------------------------------------------------------
// Utils
// -------------------------------------------------------------------

// Return current local timestamp in seconds
function getCurrentTimestamp() {
  return Date.now() / 1000
}

function convertNanToNull(value) {
  return isNaN(value) ? null : value
}

async function getServerTimestamp(xapi, hostRef) {
  const serverLocalTime = await xapi.call('host.get_servertime', hostRef)
  return Math.floor(parseDateTime(serverLocalTime).getTime() / 1e3)
}

// -------------------------------------------------------------------
// Stats
// -------------------------------------------------------------------

const computeValues = (dataRow, legendIndex, transformValue = identity) =>
  map(dataRow, ({ values }) =>
    transformValue(convertNanToNull(values[legendIndex]))
  )

const combineStats = (stats, path, combineValues) =>
  zipWith(...map(stats, path), (...values) => combineValues(values))

// It browse the object in depth and initialise it's properties
// The targerPath can be a string or an array containing the depth
// targetPath: [a, b, c] => a.b.c
const getValuesFromDepth = (obj, targetPath) => {
  if (typeof targetPath === 'string') {
    return (obj[targetPath] = [])
  }

  forEach(targetPath, (path, key) => {
    if (obj[path] === undefined) {
      obj = obj[path] = targetPath.length - 1 === key ? [] : {}
      return
    }
    obj = obj[path]
  })
  return obj
}

const testMetric = (test, type) =>
  typeof test === 'string'
    ? test === type
    : typeof test === 'function'
    ? test(type)
    : test.exec(type)

const findMetric = (metrics, metricType) => {
  let testResult
  let metric

  forEach(metrics, (current, key) => {
    if (current.test === undefined) {
      const newValues = findMetric(current, metricType)

      metric = newValues.metric
      if (metric !== undefined) {
        testResult = newValues.testResult
        return false
      }
    } else if ((testResult = testMetric(current.test, metricType))) {
      metric = current
      return false
    }
  })

  return { metric, testResult }
}

// -------------------------------------------------------------------

// The metrics:
//  test: can be a function, regexp or string, default to: currentKey
//  getPath: default to: () => currentKey
//  transformValue: default to: identity
const STATS = {
  host: {
    load: {
      test: 'loadavg',
    },
    memoryFree: {
      test: 'memory_free_kib',
      transformValue: value => value * 1024,
    },
    memory: {
      test: 'memory_total_kib',
      transformValue: value => value * 1024,
    },
    cpus: {
      test: /^cpu(\d+)$/,
      getPath: matches => ['cpus', matches[1]],
      transformValue: value => value * 1e2,
    },
    pifs: {
      rx: {
        test: /^pif_eth(\d+)_rx$/,
        getPath: matches => ['pifs', 'rx', matches[1]],
      },
      tx: {
        test: /^pif_eth(\d+)_tx$/,
        getPath: matches => ['pifs', 'tx', matches[1]],
      },
    },
    iops: {
      r: {
        test: /^iops_read_(\w+)$/,
        getPath: matches => ['iops', 'r', matches[1]],
      },
      w: {
        test: /^iops_write_(\w+)$/,
        getPath: matches => ['iops', 'w', matches[1]],
      },
    },
    ioThroughput: {
      r: {
        test: /^io_throughput_read_(\w+)$/,
        getPath: matches => ['ioThroughput', 'r', matches[1]],
        transformValue: value => value * 2 ** 20,
      },
      w: {
        test: /^io_throughput_write_(\w+)$/,
        getPath: matches => ['ioThroughput', 'w', matches[1]],
        transformValue: value => value * 2 ** 20,
      },
    },
    latency: {
      r: {
        test: /^read_latency_(\w+)$/,
        getPath: matches => ['latency', 'r', matches[1]],
        transformValue: value => value / 1e3,
      },
      w: {
        test: /^write_latency_(\w+)$/,
        getPath: matches => ['latency', 'w', matches[1]],
        transformValue: value => value / 1e3,
      },
    },
    iowait: {
      test: /^iowait_(\w+)$/,
      getPath: matches => ['iowait', matches[1]],
    },
  },
  vm: {
    memoryFree: {
      test: 'memory_internal_free',
      transformValue: value => value * 1024,
    },
    memory: {
      test: metricType => endsWith(metricType, 'memory'),
    },
    cpus: {
      test: /^cpu(\d+)$/,
      getPath: matches => ['cpus', matches[1]],
      transformValue: value => value * 1e2,
    },
    vifs: {
      rx: {
        test: /^vif_(\d+)_rx$/,
        getPath: matches => ['vifs', 'rx', matches[1]],
      },
      tx: {
        test: /^vif_(\d+)_tx$/,
        getPath: matches => ['vifs', 'tx', matches[1]],
      },
    },
    xvds: {
      r: {
        test: /^vbd_xvd(.)_read$/,
        getPath: matches => ['xvds', 'r', matches[1]],
      },
      w: {
        test: /^vbd_xvd(.)_write$/,
        getPath: matches => ['xvds', 'w', matches[1]],
      },
    },
    iops: {
      r: {
        test: /^vbd_xvd(.)_iops_read$/,
        getPath: matches => ['iops', 'r', matches[1]],
      },
      w: {
        test: /^vbd_xvd(.)_iops_write$/,
        getPath: matches => ['iops', 'w', matches[1]],
      },
    },
  },
}

// -------------------------------------------------------------------

export default class XapiStats {
  constructor() {
    this._statsByObject = {}
  }

  // Execute one http request on a XenServer for get stats
  // Return stats (Json format) or throws got exception
  @limitConcurrency(3)
  _getJson(xapi, host, timestamp, step) {
    return xapi
      .getResource('/rrd_updates', {
        host,
        query: {
          cf: 'AVERAGE',
          host: 'true',
          interval: step,
          json: 'true',
          start: timestamp,
        },
      })
      .then(response => response.readAll().then(JSON5.parse))
  }

  async _getNextTimestamp(xapi, host, step) {
    const currentTimeStamp = await getServerTimestamp(xapi, host.$ref)
    const maxDuration = step * RRD_POINTS_PER_STEP[step]
    const lastTimestamp = get(this._statsByObject, [
      host.uuid,
      step,
      'endTimestamp',
    ])

    if (
      lastTimestamp === undefined ||
      currentTimeStamp - lastTimestamp + step > maxDuration
    ) {
      return currentTimeStamp - maxDuration + step
    }
    return lastTimestamp
  }

  _getStats(hostUuid, step, vmUuid) {
    const hostStats = this._statsByObject[hostUuid][step]

    // Return host stats
    if (vmUuid === undefined) {
      return {
        interval: step,
        ...hostStats,
      }
    }

    // Return vm stats
    return {
      interval: step,
      endTimestamp: hostStats.endTimestamp,
      ...this._statsByObject[vmUuid][step],
    }
  }

  async _getAndUpdateStats(xapi, { host, vmUuid, granularity }) {
    const step =
      granularity === undefined
        ? RRD_STEP_SECONDS
        : RRD_STEP_FROM_STRING[granularity]

    if (step === undefined) {
      throw new FaultyGranularity(
        `Unknown granularity: '${granularity}'. Use 'seconds', 'minutes', 'hours', or 'days'.`
      )
    }

    // Limit the number of http requests
    const hostUuid = host.uuid

    if (
      !(
        vmUuid !== undefined &&
        get(this._statsByObject, [vmUuid, step]) === undefined
      ) &&
      get(this._statsByObject, [hostUuid, step, 'localTimestamp']) + step >
        getCurrentTimestamp()
    ) {
      return this._getStats(hostUuid, step, vmUuid)
    }

    const timestamp = await this._getNextTimestamp(xapi, host, step)
    const json = await this._getJson(xapi, host, timestamp, step)
    if (json.meta.step !== step) {
      throw new FaultyGranularity(
        `Unable to get the true granularity: ${json.meta.step}`
      )
    }

    // It exists data
    if (json.data.length !== 0) {
      // Warning: Sometimes, the json.xport.meta.start value does not match with the
      // timestamp of the oldest data value
      // So, we use the timestamp of the oldest data value !
      const startTimestamp = json.data[json.meta.rows - 1].t
      const endTimestamp = get(this._statsByObject, [
        hostUuid,
        step,
        'endTimestamp',
      ])

      const statsOffset = endTimestamp - startTimestamp + step
      if (endTimestamp !== undefined && statsOffset > 0) {
        const parseOffset = statsOffset / step
        // Remove useless data
        // Note: Older values are at end of json.data.row
        json.data.splice(json.data.length - parseOffset)
      }

      // It exists useful data
      if (json.data.length > 0) {
        // reorder data
        json.data.reverse()
        forEach(json.meta.legend, (legend, index) => {
          const [, type, uuid, metricType] = /^AVERAGE:([^:]+):(.+):(.+)$/.exec(
            legend
          )

          const metrics = STATS[type]
          if (metrics === undefined) {
            return
          }

          const { metric, testResult } = findMetric(metrics, metricType)

          if (metric === undefined) {
            return
          }

          const path =
            metric.getPath !== undefined
              ? metric.getPath(testResult)
              : [findKey(metrics, metric)]

          const metricValues = getValuesFromDepth(this._statsByObject, [
            uuid,
            step,
            'stats',
            ...path,
          ])

          metricValues.push(
            ...computeValues(json.data, index, metric.transformValue)
          )

          // remove older Values
          metricValues.splice(
            0,
            metricValues.length - RRD_POINTS_PER_STEP[step]
          )
        })
      }
    }

    // Update timestamp
    const hostStats = this._statsByObject[hostUuid][step]
    hostStats.endTimestamp = json.meta.end
    hostStats.localTimestamp = getCurrentTimestamp()
    return this._getStats(hostUuid, step, vmUuid)
  }

  getHostStats(xapi, hostId, granularity) {
    return this._getAndUpdateStats(xapi, {
      host: xapi.getObject(hostId),
      granularity,
    })
  }

  async getVmStats(xapi, vmId, granularity) {
    const vm = xapi.getObject(vmId)
    const host = vm.$resident_on
    if (!host) {
      throw new Error(`VM ${vmId} is halted or host could not be found.`)
    }

    return this._getAndUpdateStats(xapi, {
      host,
      vmUuid: vm.uuid,
      granularity,
    })
  }

  async getSrStats(xapi, srId, granularity) {
    const sr = xapi.getObject(srId)

    const hostsStats = {}
    await Promise.all(
      map(uniq(map(sr.$PBDs, 'host')), hostId =>
        this.getHostStats(xapi, hostId, granularity).then(stats => {
          hostsStats[xapi.getObject(hostId).name_label] = stats
        })
      )
    )

    const srShortUUID = sr.uuid.slice(0, 8)
    return {
      interval: hostsStats[Object.keys(hostsStats)[0]].interval,
      endTimestamp: Math.max(...map(hostsStats, 'endTimestamp')),
      localTimestamp: Math.min(...map(hostsStats, 'localTimestamp')),
      stats: {
        iops: {
          r: combineStats(hostsStats, `stats.iops.r[${srShortUUID}]`, sum),
          w: combineStats(hostsStats, `stats.iops.w[${srShortUUID}]`, sum),
        },
        ioThroughput: {
          r: combineStats(
            hostsStats,
            `stats.ioThroughput.r[${srShortUUID}]`,
            sum
          ),
          w: combineStats(
            hostsStats,
            `stats.ioThroughput.w[${srShortUUID}]`,
            sum
          ),
        },
        latency: {
          r: combineStats(hostsStats, `stats.latency.r[${srShortUUID}]`, mean),
          w: combineStats(hostsStats, `stats.latency.w[${srShortUUID}]`, mean),
        },
        iowait: mapValues(hostsStats, `stats.iowait[${srShortUUID}]`),
      },
    }
  }
}
<commit_msg>chore(xo-server/xapi-stats): add documentation (#4031)<commit_after>import JSON5 from 'json5'
import limitConcurrency from 'limit-concurrency-decorator'
import { BaseError } from 'make-error'
import {
  endsWith,
  findKey,
  forEach,
  get,
  identity,
  map,
  mapValues,
  mean,
  sum,
  uniq,
  zipWith,
} from 'lodash'

import { parseDateTime } from './xapi'

export class FaultyGranularity extends BaseError {}

// -------------------------------------------------------------------

// according to https://xapi-project.github.io/xen-api/metrics.html
// The values are stored at intervals of:
//  - 5 seconds for the past 10 minutes
//  - one minute for the past 2 hours
//  - one hour for the past week
//  - one day for the past year
const RRD_STEP_SECONDS = 5
const RRD_STEP_MINUTES = 60
const RRD_STEP_HOURS = 3600
const RRD_STEP_DAYS = 86400

const RRD_STEP_FROM_STRING = {
  seconds: RRD_STEP_SECONDS,
  minutes: RRD_STEP_MINUTES,
  hours: RRD_STEP_HOURS,
  days: RRD_STEP_DAYS,
}

// points = intervalInSeconds / step
const RRD_POINTS_PER_STEP = {
  [RRD_STEP_SECONDS]: 120,
  [RRD_STEP_MINUTES]: 120,
  [RRD_STEP_HOURS]: 168,
  [RRD_STEP_DAYS]: 366,
}

// -------------------------------------------------------------------
// Utils
// -------------------------------------------------------------------

// Return current local timestamp in seconds
function getCurrentTimestamp() {
  return Date.now() / 1000
}

function convertNanToNull(value) {
  return isNaN(value) ? null : value
}

async function getServerTimestamp(xapi, hostRef) {
  const serverLocalTime = await xapi.call('host.get_servertime', hostRef)
  return Math.floor(parseDateTime(serverLocalTime).getTime() / 1e3)
}

// -------------------------------------------------------------------
// Stats
// -------------------------------------------------------------------

const computeValues = (dataRow, legendIndex, transformValue = identity) =>
  map(dataRow, ({ values }) =>
    transformValue(convertNanToNull(values[legendIndex]))
  )

const combineStats = (stats, path, combineValues) =>
  zipWith(...map(stats, path), (...values) => combineValues(values))

// It browse the object in depth and initialise it's properties
// The targerPath can be a string or an array containing the depth
// targetPath: [a, b, c] => a.b.c
const getValuesFromDepth = (obj, targetPath) => {
  if (typeof targetPath === 'string') {
    return (obj[targetPath] = [])
  }

  forEach(targetPath, (path, key) => {
    if (obj[path] === undefined) {
      obj = obj[path] = targetPath.length - 1 === key ? [] : {}
      return
    }
    obj = obj[path]
  })
  return obj
}

const testMetric = (test, type) =>
  typeof test === 'string'
    ? test === type
    : typeof test === 'function'
    ? test(type)
    : test.exec(type)

const findMetric = (metrics, metricType) => {
  let testResult
  let metric

  forEach(metrics, (current, key) => {
    if (current.test === undefined) {
      const newValues = findMetric(current, metricType)

      metric = newValues.metric
      if (metric !== undefined) {
        testResult = newValues.testResult
        return false
      }
    } else if ((testResult = testMetric(current.test, metricType))) {
      metric = current
      return false
    }
  })

  return { metric, testResult }
}

// -------------------------------------------------------------------

// The metrics:
//  test: can be a function, regexp or string, default to: currentKey
//  getPath: default to: () => currentKey
//  transformValue: default to: identity
const STATS = {
  host: {
    load: {
      test: 'loadavg',
    },
    memoryFree: {
      test: 'memory_free_kib',
      transformValue: value => value * 1024,
    },
    memory: {
      test: 'memory_total_kib',
      transformValue: value => value * 1024,
    },
    cpus: {
      test: /^cpu(\d+)$/,
      getPath: matches => ['cpus', matches[1]],
      transformValue: value => value * 1e2,
    },
    pifs: {
      rx: {
        test: /^pif_eth(\d+)_rx$/,
        getPath: matches => ['pifs', 'rx', matches[1]],
      },
      tx: {
        test: /^pif_eth(\d+)_tx$/,
        getPath: matches => ['pifs', 'tx', matches[1]],
      },
    },
    iops: {
      r: {
        test: /^iops_read_(\w+)$/,
        getPath: matches => ['iops', 'r', matches[1]],
      },
      w: {
        test: /^iops_write_(\w+)$/,
        getPath: matches => ['iops', 'w', matches[1]],
      },
    },
    ioThroughput: {
      r: {
        test: /^io_throughput_read_(\w+)$/,
        getPath: matches => ['ioThroughput', 'r', matches[1]],
        transformValue: value => value * 2 ** 20,
      },
      w: {
        test: /^io_throughput_write_(\w+)$/,
        getPath: matches => ['ioThroughput', 'w', matches[1]],
        transformValue: value => value * 2 ** 20,
      },
    },
    latency: {
      r: {
        test: /^read_latency_(\w+)$/,
        getPath: matches => ['latency', 'r', matches[1]],
        transformValue: value => value / 1e3,
      },
      w: {
        test: /^write_latency_(\w+)$/,
        getPath: matches => ['latency', 'w', matches[1]],
        transformValue: value => value / 1e3,
      },
    },
    iowait: {
      test: /^iowait_(\w+)$/,
      getPath: matches => ['iowait', matches[1]],
    },
  },
  vm: {
    memoryFree: {
      test: 'memory_internal_free',
      transformValue: value => value * 1024,
    },
    memory: {
      test: metricType => endsWith(metricType, 'memory'),
    },
    cpus: {
      test: /^cpu(\d+)$/,
      getPath: matches => ['cpus', matches[1]],
      transformValue: value => value * 1e2,
    },
    vifs: {
      rx: {
        test: /^vif_(\d+)_rx$/,
        getPath: matches => ['vifs', 'rx', matches[1]],
      },
      tx: {
        test: /^vif_(\d+)_tx$/,
        getPath: matches => ['vifs', 'tx', matches[1]],
      },
    },
    xvds: {
      r: {
        test: /^vbd_xvd(.)_read$/,
        getPath: matches => ['xvds', 'r', matches[1]],
      },
      w: {
        test: /^vbd_xvd(.)_write$/,
        getPath: matches => ['xvds', 'w', matches[1]],
      },
    },
    iops: {
      r: {
        test: /^vbd_xvd(.)_iops_read$/,
        getPath: matches => ['iops', 'r', matches[1]],
      },
      w: {
        test: /^vbd_xvd(.)_iops_write$/,
        getPath: matches => ['iops', 'w', matches[1]],
      },
    },
  },
}

// -------------------------------------------------------------------

// RRD
// json: {
//   meta: {
//     start: Number,
//     step: Number,
//     end: Number,
//     rows: Number,
//     columns: Number,
//     legend: String[rows]
//   },
//   data: Item[columns] // Item = { t: Number, values: Number[rows] }
// }

// Local cache
// _statsByObject : {
//   [uuid]: {
//     [step]: {
//       endTimestamp: Number, // the timestamp of the last statistic point
//       interval: Number, // step
//       stats: {
//         [metric1]: Number[],
//         [metric2]: {
//           [subMetric]: Number[],
//         }
//       }
//     }
//   }
// }
export default class XapiStats {
  constructor() {
    this._statsByObject = {}
  }

  // Execute one http request on a XenServer for get stats
  // Return stats (Json format) or throws got exception
  @limitConcurrency(3)
  _getJson(xapi, host, timestamp, step) {
    return xapi
      .getResource('/rrd_updates', {
        host,
        query: {
          cf: 'AVERAGE',
          host: 'true',
          interval: step,
          json: 'true',
          start: timestamp,
        },
      })
      .then(response => response.readAll().then(JSON5.parse))
  }

  async _getNextTimestamp(xapi, host, step) {
    const currentTimeStamp = await getServerTimestamp(xapi, host.$ref)
    const maxDuration = step * RRD_POINTS_PER_STEP[step]
    const lastTimestamp = get(this._statsByObject, [
      host.uuid,
      step,
      'endTimestamp',
    ])

    if (
      lastTimestamp === undefined ||
      currentTimeStamp - lastTimestamp + step > maxDuration
    ) {
      return currentTimeStamp - maxDuration + step
    }
    return lastTimestamp
  }

  _getStats(hostUuid, step, vmUuid) {
    const hostStats = this._statsByObject[hostUuid][step]

    // Return host stats
    if (vmUuid === undefined) {
      return {
        interval: step,
        ...hostStats,
      }
    }

    // Return vm stats
    return {
      interval: step,
      endTimestamp: hostStats.endTimestamp,
      ...this._statsByObject[vmUuid][step],
    }
  }

  async _getAndUpdateStats(xapi, { host, vmUuid, granularity }) {
    const step =
      granularity === undefined
        ? RRD_STEP_SECONDS
        : RRD_STEP_FROM_STRING[granularity]

    if (step === undefined) {
      throw new FaultyGranularity(
        `Unknown granularity: '${granularity}'. Use 'seconds', 'minutes', 'hours', or 'days'.`
      )
    }

    // Limit the number of http requests
    const hostUuid = host.uuid

    if (
      !(
        vmUuid !== undefined &&
        get(this._statsByObject, [vmUuid, step]) === undefined
      ) &&
      get(this._statsByObject, [hostUuid, step, 'localTimestamp']) + step >
        getCurrentTimestamp()
    ) {
      return this._getStats(hostUuid, step, vmUuid)
    }

    const timestamp = await this._getNextTimestamp(xapi, host, step)
    const json = await this._getJson(xapi, host, timestamp, step)
    if (json.meta.step !== step) {
      throw new FaultyGranularity(
        `Unable to get the true granularity: ${json.meta.step}`
      )
    }

    // It exists data
    if (json.data.length !== 0) {
      // Warning: Sometimes, the json.xport.meta.start value does not match with the
      // timestamp of the oldest data value
      // So, we use the timestamp of the oldest data value !
      const startTimestamp = json.data[json.meta.rows - 1].t
      const endTimestamp = get(this._statsByObject, [
        hostUuid,
        step,
        'endTimestamp',
      ])

      const statsOffset = endTimestamp - startTimestamp + step
      if (endTimestamp !== undefined && statsOffset > 0) {
        const parseOffset = statsOffset / step
        // Remove useless data
        // Note: Older values are at end of json.data.row
        json.data.splice(json.data.length - parseOffset)
      }

      // It exists useful data
      if (json.data.length > 0) {
        // reorder data
        json.data.reverse()
        forEach(json.meta.legend, (legend, index) => {
          const [, type, uuid, metricType] = /^AVERAGE:([^:]+):(.+):(.+)$/.exec(
            legend
          )

          const metrics = STATS[type]
          if (metrics === undefined) {
            return
          }

          const { metric, testResult } = findMetric(metrics, metricType)

          if (metric === undefined) {
            return
          }

          const path =
            metric.getPath !== undefined
              ? metric.getPath(testResult)
              : [findKey(metrics, metric)]

          const metricValues = getValuesFromDepth(this._statsByObject, [
            uuid,
            step,
            'stats',
            ...path,
          ])

          metricValues.push(
            ...computeValues(json.data, index, metric.transformValue)
          )

          // remove older Values
          metricValues.splice(
            0,
            metricValues.length - RRD_POINTS_PER_STEP[step]
          )
        })
      }
    }

    // Update timestamp
    const hostStats = this._statsByObject[hostUuid][step]
    hostStats.endTimestamp = json.meta.end
    hostStats.localTimestamp = getCurrentTimestamp()
    return this._getStats(hostUuid, step, vmUuid)
  }

  getHostStats(xapi, hostId, granularity) {
    return this._getAndUpdateStats(xapi, {
      host: xapi.getObject(hostId),
      granularity,
    })
  }

  async getVmStats(xapi, vmId, granularity) {
    const vm = xapi.getObject(vmId)
    const host = vm.$resident_on
    if (!host) {
      throw new Error(`VM ${vmId} is halted or host could not be found.`)
    }

    return this._getAndUpdateStats(xapi, {
      host,
      vmUuid: vm.uuid,
      granularity,
    })
  }

  async getSrStats(xapi, srId, granularity) {
    const sr = xapi.getObject(srId)

    const hostsStats = {}
    await Promise.all(
      map(uniq(map(sr.$PBDs, 'host')), hostId =>
        this.getHostStats(xapi, hostId, granularity).then(stats => {
          hostsStats[xapi.getObject(hostId).name_label] = stats
        })
      )
    )

    const srShortUUID = sr.uuid.slice(0, 8)
    return {
      interval: hostsStats[Object.keys(hostsStats)[0]].interval,
      endTimestamp: Math.max(...map(hostsStats, 'endTimestamp')),
      localTimestamp: Math.min(...map(hostsStats, 'localTimestamp')),
      stats: {
        iops: {
          r: combineStats(hostsStats, `stats.iops.r[${srShortUUID}]`, sum),
          w: combineStats(hostsStats, `stats.iops.w[${srShortUUID}]`, sum),
        },
        ioThroughput: {
          r: combineStats(
            hostsStats,
            `stats.ioThroughput.r[${srShortUUID}]`,
            sum
          ),
          w: combineStats(
            hostsStats,
            `stats.ioThroughput.w[${srShortUUID}]`,
            sum
          ),
        },
        latency: {
          r: combineStats(hostsStats, `stats.latency.r[${srShortUUID}]`, mean),
          w: combineStats(hostsStats, `stats.latency.w[${srShortUUID}]`, mean),
        },
        iowait: mapValues(hostsStats, `stats.iowait[${srShortUUID}]`),
      },
    }
  }
}
<|endoftext|>