<commit_before>// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

/**
 * This file contains helpers to load the correct path to various scripts and
 * directories. It is the Node equivalent of devtools_paths.py.
 *
 * Note that not all paths implemented in devtools_paths.py are implemented
 * here. Please add any paths you need that are missing.
 */

const path = require('path');
const os = require('os');

/**
 * You would think we can use __filename here but we cannot because __filename
 * has any symlinks resolved. This means we can't use it to tell if the user is
 * using the external repo with a standalone build setup because the symlink
 * from chromium/src/third_party/devtools-frontend => devtools-frontend repo
 * gets resolved by Node before it gives us __filename.
 *
 * We can use process.argv[1], which is the path to the file currently being
 * executed without any symlinks resolution. If we assume that file is always in
 * the devtools-frontend repository/directory, we can use that file as the
 * starting point for figuring out if we're in Chromium or not. until we find
 * the scripts directory, at which point we've found this file and can use it
 * for all subsequent logic.
 *
 * e.g. the user executes a script: scripts/test/run_lint_check_css.js
 *
 * process.argv[1] =
 * /full/path/devtools-frontend/src/scripts/test/run_lint_check_css.js
 */
const PATH_TO_EXECUTED_FILE = process.argv[1];

function pathIsMostTopLevelPath(filePath) {
  /**
   * On Linux/Mac, if we do path.dirname(X) as many times as possible, it will
   * eventually equal `/`. On Windows, it will end up equalling C:\, and
   * path.dirname('C:\') === 'C:\', so we use that to figure out if we've made
   * it as far up the tree as we can.
   */
  return filePath === path.sep || path.dirname(filePath) === filePath;
}

const _lookUpCaches = new Map(
    [['chromium', null]],
);
/**
 * This function figures out if we're within a chromium directory, and therefore
 * we are in the integrated workflow mode, rather than working in a standalone
 * devtools-frontend repository.
 */
function isInChromiumDirectory() {
  const cached = _lookUpCaches.get('chromium');
  if (cached) {
    return cached;
  }

  let potentialChromiumDir = PATH_TO_EXECUTED_FILE;
  let isInChromium = false;
  while (!pathIsMostTopLevelPath(potentialChromiumDir)) {
    potentialChromiumDir = path.dirname(potentialChromiumDir);
    if (path.basename(potentialChromiumDir) === 'chromium') {
      isInChromium = true;
      break;
    }
  }
  const result = {isInChromium, chromiumDirectory: potentialChromiumDir};
  _lookUpCaches.set('chromium', result);
  return result;
}
/**
 * Returns the path to the root of the devtools-frontend repository.
 *
 * If we're in Chromium, this will be /path/to/chromium/src/third_party/devtools-frontend/src
 * If it's standalone, it will be /path/to/devtools-frontend
 */
function devtoolsRootPath() {
  const nodeScriptFileThatIsBeingExecuted = PATH_TO_EXECUTED_FILE;
  let devtoolsRootFolder = nodeScriptFileThatIsBeingExecuted;
  while (path.basename(devtoolsRootFolder) !== 'devtools-frontend') {
    devtoolsRootFolder = path.dirname(devtoolsRootFolder);
    // We reached the end and can't find devtools-frontend.
    if (pathIsMostTopLevelPath(devtoolsRootFolder)) {
      throw new Error(
          'Could not find devtools-frontend in path. If you have cloned the repository to a different directory name, it will not work.');
    }
  }
  // In Chromium the path to the source code for devtools-frontend is:
  // third_party/devtools-frontend/src
  const {isInChromium} = isInChromiumDirectory();
  if (isInChromium) {
    return path.join(devtoolsRootFolder, 'src');
  }

  // But if you're in a standalone repo it's just the devtools-frontend folder.
  return devtoolsRootFolder;
}

/**
 * Returns the path to the root of the main repository we're in.
 * if we're in Chromium, this is /path/to/chromium/src
 * if we're in standalone, this is /path/to/devtools-frontend
 *
 * Note this is different to devtoolsRootPath(), which always returns the path
 * to the devtools-frontend source code.
 */
function rootPath() {
  const {isInChromium, chromiumDirectory} = isInChromiumDirectory();
  if (isInChromium) {
    return path.join(chromiumDirectory, 'src');
  }
  return devtoolsRootPath();
}

/**
 * Path to the third_party directory. Used because if we're running in Chromium
 * land we need to use e.g. the Node executable from Chromium's third_party
 * directory, not from the devtools-frontend third_party directory.
 */
function thirdPartyPath() {
  return path.join(rootPath(), 'third_party');
}

function nodePath() {
  const paths = {
    'darwin': path.join('mac', 'node-darwin-x64', 'bin', 'node'),
    'linux': path.join('linux', 'node-linux-x64', 'bin', 'node'),
    'win32': path.join('win', 'node.exe'),
  };
  return path.join(thirdPartyPath(), 'node', paths[os.platform()]);
}

/**
 * The path to the devtools-frontend node_modules folder.
 */
function nodeModulesPath() {
  return path.join(devtoolsRootPath(), 'node_modules');
}

function stylelintExecutablePath() {
  return path.join(nodeModulesPath(), 'stylelint', 'bin', 'stylelint.js');
}

function mochaExecutablePath() {
  return path.join(nodeModulesPath(), 'mocha', 'bin', 'mocha');
}

function downloadedChromeBinaryPath() {
  const paths = {
    'linux': path.join('chrome-linux', 'chrome'),
    'darwin': path.join('chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium'),
    'win32': path.join('chrome-win', 'chrome.exe'),
  };
  return path.join(thirdPartyPath(), 'chrome', paths[os.platform()]);
}

module.exports = {
  thirdPartyPath,
  nodePath,
  devtoolsRootPath,
  nodeModulesPath,
  mochaExecutablePath,
  stylelintExecutablePath,
  downloadedChromeBinaryPath
};
<commit_msg>Make isInChromiumDirectory also work when inside chromium<commit_after>// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

/**
 * This file contains helpers to load the correct path to various scripts and
 * directories. It is the Node equivalent of devtools_paths.py.
 *
 * Note that not all paths implemented in devtools_paths.py are implemented
 * here. Please add any paths you need that are missing.
 */

const path = require('path');
const os = require('os');

/**
 * You would think we can use __filename here but we cannot because __filename
 * has any symlinks resolved. This means we can't use it to tell if the user is
 * using the external repo with a standalone build setup because the symlink
 * from chromium/src/third_party/devtools-frontend => devtools-frontend repo
 * gets resolved by Node before it gives us __filename.
 *
 * We can use process.argv[1], which is the path to the file currently being
 * executed without any symlinks resolution. If we assume that file is always in
 * the devtools-frontend repository/directory, we can use that file as the
 * starting point for figuring out if we're in Chromium or not. until we find
 * the scripts directory, at which point we've found this file and can use it
 * for all subsequent logic.
 *
 * e.g. the user executes a script: scripts/test/run_lint_check_css.js
 *
 * process.argv[1] =
 * /full/path/devtools-frontend/src/scripts/test/run_lint_check_css.js
 */
const PATH_TO_EXECUTED_FILE = process.argv[1];

function pathIsMostTopLevelPath(filePath) {
  /**
   * On Linux/Mac, if we do path.dirname(X) as many times as possible, it will
   * eventually equal `/`. On Windows, it will end up equalling C:\, and
   * path.dirname('C:\') === 'C:\', so we use that to figure out if we've made
   * it as far up the tree as we can.
   */
  return filePath === path.sep || path.dirname(filePath) === filePath;
}

const _lookUpCaches = new Map(
    [['chromium', null]],
);
/**
 * This function figures out if we're within a chromium directory, and therefore
 * we are in the integrated workflow mode, rather than working in a standalone
 * devtools-frontend repository.
 */
function isInChromiumDirectory() {
  const cached = _lookUpCaches.get('chromium');
  if (cached) {
    return cached;
  }

  const normalizedPath = PATH_TO_EXECUTED_FILE.split(path.sep).join('/');
  const isInChromium = normalizedPath.includes('chromium/src/third_party/devtools-frontend');
  const potentialChromiumDir = PATH_TO_EXECUTED_FILE.substring(0, PATH_TO_EXECUTED_FILE.indexOf('chromium') + 8);
  const result = {isInChromium, chromiumDirectory: potentialChromiumDir};
  _lookUpCaches.set('chromium', result);
  return result;
}
/**
 * Returns the path to the root of the devtools-frontend repository.
 *
 * If we're in Chromium, this will be /path/to/chromium/src/third_party/devtools-frontend/src
 * If it's standalone, it will be /path/to/devtools-frontend
 */
function devtoolsRootPath() {
  const nodeScriptFileThatIsBeingExecuted = PATH_TO_EXECUTED_FILE;
  let devtoolsRootFolder = nodeScriptFileThatIsBeingExecuted;
  while (path.basename(devtoolsRootFolder) !== 'devtools-frontend') {
    devtoolsRootFolder = path.dirname(devtoolsRootFolder);
    // We reached the end and can't find devtools-frontend.
    if (pathIsMostTopLevelPath(devtoolsRootFolder)) {
      throw new Error(
          'Could not find devtools-frontend in path. If you have cloned the repository to a different directory name, it will not work.');
    }
  }
  // In Chromium the path to the source code for devtools-frontend is:
  // third_party/devtools-frontend/src
  const {isInChromium} = isInChromiumDirectory();
  if (isInChromium) {
    return path.join(devtoolsRootFolder, 'src');
  }

  // But if you're in a standalone repo it's just the devtools-frontend folder.
  return devtoolsRootFolder;
}

/**
 * Returns the path to the root of the main repository we're in.
 * if we're in Chromium, this is /path/to/chromium/src
 * if we're in standalone, this is /path/to/devtools-frontend
 *
 * Note this is different to devtoolsRootPath(), which always returns the path
 * to the devtools-frontend source code.
 */
function rootPath() {
  const {isInChromium, chromiumDirectory} = isInChromiumDirectory();
  if (isInChromium) {
    return path.join(chromiumDirectory, 'src');
  }
  return devtoolsRootPath();
}

/**
 * Path to the third_party directory. Used because if we're running in Chromium
 * land we need to use e.g. the Node executable from Chromium's third_party
 * directory, not from the devtools-frontend third_party directory.
 */
function thirdPartyPath() {
  return path.join(rootPath(), 'third_party');
}

function nodePath() {
  const paths = {
    'darwin': path.join('mac', 'node-darwin-x64', 'bin', 'node'),
    'linux': path.join('linux', 'node-linux-x64', 'bin', 'node'),
    'win32': path.join('win', 'node.exe'),
  };
  return path.join(thirdPartyPath(), 'node', paths[os.platform()]);
}

/**
 * The path to the devtools-frontend node_modules folder.
 */
function nodeModulesPath() {
  return path.join(devtoolsRootPath(), 'node_modules');
}

function stylelintExecutablePath() {
  return path.join(nodeModulesPath(), 'stylelint', 'bin', 'stylelint.js');
}

function mochaExecutablePath() {
  return path.join(nodeModulesPath(), 'mocha', 'bin', 'mocha');
}

function downloadedChromeBinaryPath() {
  const paths = {
    'linux': path.join('chrome-linux', 'chrome'),
    'darwin': path.join('chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium'),
    'win32': path.join('chrome-win', 'chrome.exe'),
  };
  return path.join(thirdPartyPath(), 'chrome', paths[os.platform()]);
}

module.exports = {
  thirdPartyPath,
  nodePath,
  devtoolsRootPath,
  nodeModulesPath,
  mochaExecutablePath,
  stylelintExecutablePath,
  downloadedChromeBinaryPath
};
<|endoftext|>


<commit_before>#!/usr/bin/env node

module.exports = function (context) {
    var IosSDKVersion = "OpenTok-iOS-2.7.1";
    var downloadFile = require('./downloadFile.js'),
        exec = require('./exec/exec.js'),
        Q = context.requireCordovaModule('q'),
        deferral = new Q.defer();
    console.log('Downloading OpenTok iOS SDK');
    downloadFile('https://s3.amazonaws.com/artifact.tokbox.com/rel/ios-sdk/' + IosSDKVersion + '.tar.bz2',
        './' + IosSDKVersion + '.tar.bz2', function (err) {
            if (!err) {
                console.log('downloaded');
                exec('tar -zxvf ./' + IosSDKVersion + '.tar.bz2', function (err, out, code) {
                    console.log('expanded');
                    var frameworkDir = context.opts.plugin.dir + '/src/ios/';
                    exec('mv ./' + IosSDKVersion + '/OpenTok.framework ' + frameworkDir, function (err, out, code) {
                        console.log('moved OpenTok.framework into ' + frameworkDir);
                        exec('rm -r ./' + IosSDKVersion, function (err, out, code) {
                            console.log('Removed extracted dir');
                            exec('rm ./' + IosSDKVersion + '.tar.bz2', function (err, out, code) {
                                console.log('Removed downloaded SDK');
                                deferral.resolve();
                            });
                        });
                    });
                });
            }
        });
    return deferral.promise;
};
<commit_msg>Update downloadiOSSDK.js<commit_after>#!/usr/bin/env node

module.exports = function (context) {
    var IosSDKVersion = "OpenTok-iOS-2.8.3";
    var downloadFile = require('./downloadFile.js'),
        exec = require('./exec/exec.js'),
        Q = context.requireCordovaModule('q'),
        deferral = new Q.defer();
    console.log('Downloading OpenTok iOS SDK');
    downloadFile('https://s3.amazonaws.com/artifact.tokbox.com/rel/ios-sdk/' + IosSDKVersion + '.tar.bz2',
        './' + IosSDKVersion + '.tar.bz2', function (err) {
            if (!err) {
                console.log('downloaded');
                exec('tar -zxvf ./' + IosSDKVersion + '.tar.bz2', function (err, out, code) {
                    console.log('expanded');
                    var frameworkDir = context.opts.plugin.dir + '/src/ios/';
                    exec('mv ./' + IosSDKVersion + '/OpenTok.framework ' + frameworkDir, function (err, out, code) {
                        console.log('moved OpenTok.framework into ' + frameworkDir);
                        exec('rm -r ./' + IosSDKVersion, function (err, out, code) {
                            console.log('Removed extracted dir');
                            exec('rm ./' + IosSDKVersion + '.tar.bz2', function (err, out, code) {
                                console.log('Removed downloaded SDK');
                                deferral.resolve();
                            });
                        });
                    });
                });
            }
        });
    return deferral.promise;
};
<|endoftext|>


<commit_before>/**
 * The gulp-cmd plugin for embedding style text in JavaScript code
 */

'use strict';

var styleNode;
var doc = document;
var toString = ({}).property.toString;
var head = doc.getElementsByTagName('head')[0] || doc.documentElement;

/**
 * Is string
 * @param value
 * @returns {boolean}
 */

function isString(value){
  return {}.toString.call(value) === "[object String]";
}

module.exports = function (cssText, imports){
  var element;

  imports = isString(imports) ? imports : '';

  // Don't share styleNode when id is spectied
  if (!styleNode) {
    element = doc.createElement('style');

    // Adds to DOM first to avoid the css hack invalid
    head.appendChild(element);
  } else {
    element = styleNode;
  }

  // IE
  if (element.styleSheet !== undefined) {

    // http://support.microsoft.com/kb/262161
    if (doc.getElementsByTagName('style').length > 31) {
      throw new Error('Exceed the maximal count of style tags in IE');
    }

    element.styleSheet.cssText = imports + element.styleSheet.cssText + cssText;
  }
  // W3C
  else {
    element.insertBefore(doc.createTextNode(imports), element.firstChild);
    element.appendChild(doc.createTextNode(cssText));
  }

  if (!id) {
    styleNode = element;
  }
};
<commit_msg>add source<commit_after>/**
 * The gulp-cmd plugin for embedding style text in JavaScript code
 */

'use strict';

var styleNode;
var doc = document;
var toString = ({}).property.toString;
var head = doc.getElementsByTagName('head')[0] || doc.documentElement;

/**
 * Is string
 * @param value
 * @returns {boolean}
 */

function isString(value){
  return {}.toString.call(value) === "[object String]";
}

// Exports
module.exports = function (cssText, imports){
  var element;

  imports = isString(imports) ? imports : '';

  // Don't share styleNode when id is spectied
  if (!styleNode) {
    element = doc.createElement('style');

    // Adds to DOM first to avoid the css hack invalid
    head.appendChild(element);

    // Cache style node
    styleNode = element;
  } else {
    element = styleNode;
  }

  // IE
  if (element.styleSheet !== undefined) {

    // http://support.microsoft.com/kb/262161
    if (doc.getElementsByTagName('style').length > 31) {
      throw new Error('Exceed the maximal count of style tags in IE');
    }

    element.styleSheet.cssText = imports + element.styleSheet.cssText + cssText;
  }
  // W3C
  else {
    element.insertBefore(doc.createTextNode(imports), element.firstChild);
    element.appendChild(doc.createTextNode(cssText));
  }
};
<|endoftext|>


<commit_before>#!/usr/bin/env node

/**
 * `tail` source files listed in app/config.js.
 *
 * Usage:
 * - To use config/config.js source list: app/tail.js
 * - To use external source list: app/tail.js path/to/config.js
 *
 * process.send() is occasionally used to coordinate flow with test scripts.
 */

'use strict';

(function() {
  var program = require('commander');
  program
    .option('-c, --config <file>', '/path/to/config.js', null)
    .option('-t, --test <#>', 'Exit after # expected lines for unit tests', Number, 0)
    .option('-q, --quiet')
    .option('-v, --verbose')
    .parse(process.argv);

  require(__dirname + '/modules/diana.js');
  var spawn = require('child_process').spawn,
      config = diana.getConfig(program.config),
      parsers = diana.requireModule('parsers/parsers'),
      procLog = diana.createUtilLogger('tail.js');

  // To support maximum line count for --test.
  var lineCount = 0;

  /**
   * Configure the monitor.
   */
  var Monitor = function(source) {
    // Log source attribute object from config/app.js 'sources' list.
    this.source = _.clone(source);

    // ChildProcess object.
    this.tail = null;

    this.log = diana.createUtilLogger(this.source.path, program.quiet);
  };

  /**
   * Start a new `tail` instance and attach event handlers.
   */
  Monitor.prototype.start = function() {
    this.log('spawning tail');

    this.tail = spawn('tail', ['--bytes=0', '-F', this.source.path]);

    var monitor = this;

    this.tail.stdout.on('data', function(data) {
      var lines = data.toString().replace(/\n$/, '').split("\n");
      lineCount += lines.length;

      if (program.verbose) {
        monitor.log('data=[%s] length={%d]', data.toString(), lines.length);
      }

      parsers.parseAndInsert({source: monitor.source, lines: lines}, function() {
        // Support maximum line count for --test.
        if (program.test > 0 && lineCount >= program.test) {
          process.exit();
        }
      });
    });

    this.tail.on('exit', function(code, signal) {
      monitor.log("tail exited with code %d, signal %s", code, signal);

      // Auto-restart.
      monitor.start();

      if (program.test) {
        process.send('MONITOR_RESTART');
      }
    });

    // Kill `tail` before parent process exits.
    var killTail = function() {
      process.removeListener('END_MONITOR', killTail);
      if (monitor.tail) {
        monitor.log('killing pid %d ...', monitor.tail.pid);
        monitor.tail.kill('SIGKILL');
        monitor.log('killed');
        monitor.tail = null;
      }

      if (program.test) {
        if (!process.listeners('END_MONITOR').length) {
          process.send('MONITORS_ENDED');
        }
      }
    };
    process.on('END_MONITOR', killTail);

    this.log('pid %d', this.tail.pid);
  };

  /**
   * Seed initial `tail` set based on config/app.js 'sources' list.
   */
  var startAllMonitors = function() {
    _.each(config.sources, function(source) {
      (new Monitor(source)).start();
    });
  };

  /**
   * Trigger monitors to clean up their `tail` instances.
   */
  var endAllMonitors = function() {
    process.emit('END_MONITOR');
  };
  process.on('exit', endAllMonitors);
  process.on('SIGINT', function() {
    procLog('SIGINT received');
    process.exit();
  });
  process.on('SIGTERM', function() {
    procLog('SIGTERM received');
    process.exit();
  });
  process.on('uncaughtException', function(err) {
    procLog('uncaught exception: %s', err.stack ? err.stack : '');
    process.exit();
  });

  // Test mode -- wait until instructed by parent process.
  if (program.test) {
    process.on('message', function(message) {
      if ('START_TEST' == message) {
        startAllMonitors();
        process.send('MONITORS_STARTED');
      }
    });
  // Normal mode -- start immediately.
  } else {
    startAllMonitors();
  }
})();
<commit_msg>Set tail.js max event listeners to match source list.<commit_after>#!/usr/bin/env node

/**
 * `tail` source files listed in app/config.js.
 *
 * Usage:
 * - To use config/config.js source list: app/tail.js
 * - To use external source list: app/tail.js path/to/config.js
 *
 * process.send() is occasionally used to coordinate flow with test scripts.
 */

'use strict';

(function() {
  var program = require('commander');
  program
    .option('-c, --config <file>', '/path/to/config.js', null)
    .option('-t, --test <#>', 'Exit after # expected lines for unit tests', Number, 0)
    .option('-q, --quiet')
    .option('-v, --verbose')
    .parse(process.argv);

  require(__dirname + '/modules/diana.js');
  var spawn = require('child_process').spawn,
      config = diana.getConfig(program.config),
      parsers = diana.requireModule('parsers/parsers'),
      procLog = diana.createUtilLogger('tail.js');

  // To support maximum line count for --test.
  var lineCount = 0;

  /**
   * Configure the monitor.
   */
  var Monitor = function(source) {
    // Log source attribute object from config/app.js 'sources' list.
    this.source = _.clone(source);

    // ChildProcess object.
    this.tail = null;

    this.log = diana.createUtilLogger(this.source.path, program.quiet);
  };

  /**
   * Start a new `tail` instance and attach event handlers.
   */
  Monitor.prototype.start = function() {
    this.log('spawning tail');

    this.tail = spawn('tail', ['--bytes=0', '-F', this.source.path]);

    var monitor = this;

    this.tail.stdout.on('data', function(data) {
      var lines = data.toString().replace(/\n$/, '').split("\n");
      lineCount += lines.length;

      if (program.verbose) {
        monitor.log('data=[%s] length={%d]', data.toString(), lines.length);
      }

      parsers.parseAndInsert({source: monitor.source, lines: lines}, function() {
        // Support maximum line count for --test.
        if (program.test > 0 && lineCount >= program.test) {
          process.exit();
        }
      });
    });

    this.tail.on('exit', function(code, signal) {
      monitor.log("tail exited with code %d, signal %s", code, signal);

      // Auto-restart.
      monitor.start();

      if (program.test) {
        process.send('MONITOR_RESTART');
      }
    });

    // Kill `tail` before parent process exits.
    var killTail = function() {
      process.removeListener('END_MONITOR', killTail);
      if (monitor.tail) {
        monitor.log('killing pid %d ...', monitor.tail.pid);
        monitor.tail.kill('SIGKILL');
        monitor.log('killed');
        monitor.tail = null;
      }

      if (program.test) {
        if (!process.listeners('END_MONITOR').length) {
          process.send('MONITORS_ENDED');
        }
      }
    };
    process.on('END_MONITOR', killTail);

    this.log('pid %d', this.tail.pid);
  };

  /**
   * Seed initial `tail` set based on config/app.js 'sources' list.
   */
  var startAllMonitors = function() {
    // Avoid EventEmitter mem leak false alarms about > 10.
    process.setMaxListeners(config.sources.length);

    _.each(config.sources, function(source) {
      (new Monitor(source)).start();
    });
  };

  /**
   * Trigger monitors to clean up their `tail` instances.
   */
  var endAllMonitors = function() {
    process.emit('END_MONITOR');
  };
  process.on('exit', endAllMonitors);
  process.on('SIGINT', function() {
    procLog('SIGINT received');
    process.exit();
  });
  process.on('SIGTERM', function() {
    procLog('SIGTERM received');
    process.exit();
  });
  process.on('uncaughtException', function(err) {
    procLog('uncaught exception: %s', err.stack ? err.stack : '');
    process.exit();
  });

  // Test mode -- wait until instructed by parent process.
  if (program.test) {
    process.on('message', function(message) {
      if ('START_TEST' == message) {
        startAllMonitors();
        process.send('MONITORS_STARTED');
      }
    });
  // Normal mode -- start immediately.
  } else {
    startAllMonitors();
  }
})();
<|endoftext|>


<commit_before>/*global URL:false,Blob:false*/
import React, { Component } from 'react';
import { hterm, lib as htermLib } from 'hterm-umd';

hterm.defaultStorage = new htermLib.Storage.Memory();

// override double click behavior to copy
const oldMouse = hterm.Terminal.prototype.onMouse_;
hterm.Terminal.prototype.onMouse_ = function (e) {
  if ('dblclick' === e.type) {
    console.log('[hyperterm+hterm] ignore double click');
    return;
  }
  return oldMouse.call(this, e);
};

// there's no option to turn off the size overlay
hterm.Terminal.prototype.overlaySize = function () {};

// fixing a bug in hterm where a double click triggers
// a non-collapsed selection whose text is '', and results
// in an infinite copy loop
hterm.Terminal.prototype.copySelectionToClipboard = function () {
  var text = this.getSelectionText();
  if (text != null && text !== '') {
    this.copyStringToClipboard(text);
  }
};

// passthrough all the commands that are meant to control
// hyperterm and not the terminal itself
const oldKeyDown = hterm.Keyboard.prototype.onKeyDown_;
hterm.Keyboard.prototype.onKeyDown_ = function (e) {
  if (e.metaKey) {
    return;
  }
  return oldKeyDown.call(this, e);
};

const oldKeyPress = hterm.Keyboard.prototype.onKeyPress_;
hterm.Keyboard.prototype.onKeyPress_ = function (e) {
  if (e.metaKey) {
    return;
  }
  return oldKeyPress.call(this, e);
};

const domainRegex = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/;

export default class Term extends Component {

  componentDidMount () {
    this.term = new hterm.Terminal();

    // the first term that's created has unknown size
    // subsequent new tabs have size
    if (this.props.cols) {
      this.term.realizeSize_(this.props.cols, this.props.rows);
    }

    this.term.prefs_.set('font-family', 'Menlo');
    this.term.prefs_.set('font-size', 11);
    this.term.prefs_.set('cursor-color', '#F81CE5');
    this.term.prefs_.set('enable-clipboard-notice', false);
    this.term.prefs_.set('background-color', '#000');

    this.term.prefs_.set('user-css', URL.createObjectURL(new Blob([`
      .cursor-node[focus="false"] {
        border-width: 1px !important;
      }
    `]), { type: 'text/css' }));

    this.term.onTerminalReady = () => {
      const io = this.term.io.push();
      io.onVTKeystroke = io.sendString = (str) => {
        this.props.onData(str);
      };
      io.onTerminalResize = (cols, rows) => {
        this.props.onResize({ cols, rows });
      };
    };
    this.term.decorate(this.refs.term);
    this.term.installKeyboard();
  }

  getTermDocument () {
    return this.term.document_;
  }

  shouldComponentUpdate (nextProps) {
    if (this.props.url !== nextProps.url) {
      // when the url prop changes, we make sure
      // the terminal starts or stops ignoring
      // key input so that it doesn't conflict
      // with the <webview>
      if (nextProps.url) {
        const io = this.term.io.push();
        io.onVTKeystroke = io.sendString = (str) => {
          if (1 === str.length && 3 === str.charCodeAt(0) /* Ctrl + C */) {
            this.props.onURLAbort();
          }
        };
      } else {
        this.term.io.pop();
      }
      return true;
    }

    return false;
  }

  write (data) {
    const match = data.match(/bash: ((https?:\/\/)|(\/\/))?(.*): ((command not found)|(No such file or directory))/);
    if (match) {
      const url = match[4];
      // extract the domain portion from the url
      const domain = url.split('/')[0];
      if (domainRegex.test(domain)) {
        this.props.onURL(toURL(url));
        return;
      }
    }
    this.term.io.print(data);
  }

  focus () {
    this.term.focus();
  }

  clear () {
    const { term } = this;

    // we re-implement `wipeContents` to preserve the line
    // and cursor position that the client is in.
    // otherwise the user ends up with a completely clear
    // screen which is really strange
    term.scrollbackRows_.length = 0;
    term.scrollPort_.resetCache();

    [term.primaryScreen_, term.alternateScreen_].forEach(function (screen) {
      const bottom = screen.getHeight();
      if (bottom > 0) {
        term.renumberRows_(0, bottom);

        const x = screen.cursorPosition.column;
        const y = screen.cursorPosition.row;

        if (x === 0) {
          // Empty screen, nothing to do.
          return;
        }

        // here we move the row that the user was focused on
        // to the top of the screen
        term.moveRows_(y, 1, 0);

        for (let i = 1; i < bottom; i++) {
          screen.setCursorPosition(i, 0);
          screen.clearCursorRow();
        }

        // we restore the cursor position
        screen.setCursorPosition(0, x);
      }
    });

    term.syncCursorPosition_();
    term.scrollPort_.invalidate();

    // this will avoid a bug where the `wipeContents`
    // hterm API doens't send the scroll to the top
    this.term.scrollPort_.redraw_();
  }

  componentWillUnmount () {
    // there's no need to manually destroy
    // as all the events are attached to the iframe
    // which gets removed
  }

  render () {
    return

      { this.props.url
        ? <webview
            src={this.props.url}
            style={{
              background: '#000',
              position: 'absolute',
              top: 0,
              left: 0,
              display: 'inline-flex',
              width: '100%',
              height: '100%'
            }}></webview>
        : null
      }

;
  }

}

function toURL (domain) {
  if (/^https?:\/\//.test(domain)) {
    return domain;
  }

  if ('//' === domain.substr(0, 2)) {
    return domain;
  }

  return 'http://' + domain;
}
<commit_msg>term: cross-platform font family settings (#26)<commit_after>/*global URL:false,Blob:false*/
import React, { Component } from 'react';
import { hterm, lib as htermLib } from 'hterm-umd';

hterm.defaultStorage = new htermLib.Storage.Memory();

// override double click behavior to copy
const oldMouse = hterm.Terminal.prototype.onMouse_;
hterm.Terminal.prototype.onMouse_ = function (e) {
  if ('dblclick' === e.type) {
    console.log('[hyperterm+hterm] ignore double click');
    return;
  }
  return oldMouse.call(this, e);
};

// there's no option to turn off the size overlay
hterm.Terminal.prototype.overlaySize = function () {};

// fixing a bug in hterm where a double click triggers
// a non-collapsed selection whose text is '', and results
// in an infinite copy loop
hterm.Terminal.prototype.copySelectionToClipboard = function () {
  var text = this.getSelectionText();
  if (text != null && text !== '') {
    this.copyStringToClipboard(text);
  }
};

// passthrough all the commands that are meant to control
// hyperterm and not the terminal itself
const oldKeyDown = hterm.Keyboard.prototype.onKeyDown_;
hterm.Keyboard.prototype.onKeyDown_ = function (e) {
  if (e.metaKey) {
    return;
  }
  return oldKeyDown.call(this, e);
};

const oldKeyPress = hterm.Keyboard.prototype.onKeyPress_;
hterm.Keyboard.prototype.onKeyPress_ = function (e) {
  if (e.metaKey) {
    return;
  }
  return oldKeyPress.call(this, e);
};

const domainRegex = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/;

export default class Term extends Component {

  componentDidMount () {
    this.term = new hterm.Terminal();

    // the first term that's created has unknown size
    // subsequent new tabs have size
    if (this.props.cols) {
      this.term.realizeSize_(this.props.cols, this.props.rows);
    }

    this.term.prefs_.set('font-family', "Menlo, 'DejaVu Sans Mono', 'Lucida Console', monospace");
    this.term.prefs_.set('font-size', 11);
    this.term.prefs_.set('cursor-color', '#F81CE5');
    this.term.prefs_.set('enable-clipboard-notice', false);
    this.term.prefs_.set('background-color', '#000');

    this.term.prefs_.set('user-css', URL.createObjectURL(new Blob([`
      .cursor-node[focus="false"] {
        border-width: 1px !important;
      }
    `]), { type: 'text/css' }));

    this.term.onTerminalReady = () => {
      const io = this.term.io.push();
      io.onVTKeystroke = io.sendString = (str) => {
        this.props.onData(str);
      };
      io.onTerminalResize = (cols, rows) => {
        this.props.onResize({ cols, rows });
      };
    };
    this.term.decorate(this.refs.term);
    this.term.installKeyboard();
  }

  getTermDocument () {
    return this.term.document_;
  }

  shouldComponentUpdate (nextProps) {
    if (this.props.url !== nextProps.url) {
      // when the url prop changes, we make sure
      // the terminal starts or stops ignoring
      // key input so that it doesn't conflict
      // with the <webview>
      if (nextProps.url) {
        const io = this.term.io.push();
        io.onVTKeystroke = io.sendString = (str) => {
          if (1 === str.length && 3 === str.charCodeAt(0) /* Ctrl + C */) {
            this.props.onURLAbort();
          }
        };
      } else {
        this.term.io.pop();
      }
      return true;
    }

    return false;
  }

  write (data) {
    const match = data.match(/bash: ((https?:\/\/)|(\/\/))?(.*): ((command not found)|(No such file or directory))/);
    if (match) {
      const url = match[4];
      // extract the domain portion from the url
      const domain = url.split('/')[0];
      if (domainRegex.test(domain)) {
        this.props.onURL(toURL(url));
        return;
      }
    }
    this.term.io.print(data);
  }

  focus () {
    this.term.focus();
  }

  clear () {
    const { term } = this;

    // we re-implement `wipeContents` to preserve the line
    // and cursor position that the client is in.
    // otherwise the user ends up with a completely clear
    // screen which is really strange
    term.scrollbackRows_.length = 0;
    term.scrollPort_.resetCache();

    [term.primaryScreen_, term.alternateScreen_].forEach(function (screen) {
      const bottom = screen.getHeight();
      if (bottom > 0) {
        term.renumberRows_(0, bottom);

        const x = screen.cursorPosition.column;
        const y = screen.cursorPosition.row;

        if (x === 0) {
          // Empty screen, nothing to do.
          return;
        }

        // here we move the row that the user was focused on
        // to the top of the screen
        term.moveRows_(y, 1, 0);

        for (let i = 1; i < bottom; i++) {
          screen.setCursorPosition(i, 0);
          screen.clearCursorRow();
        }

        // we restore the cursor position
        screen.setCursorPosition(0, x);
      }
    });

    term.syncCursorPosition_();
    term.scrollPort_.invalidate();

    // this will avoid a bug where the `wipeContents`
    // hterm API doens't send the scroll to the top
    this.term.scrollPort_.redraw_();
  }

  componentWillUnmount () {
    // there's no need to manually destroy
    // as all the events are attached to the iframe
    // which gets removed
  }

  render () {
    return

      { this.props.url
        ? <webview
            src={this.props.url}
            style={{
              background: '#000',
              position: 'absolute',
              top: 0,
              left: 0,
              display: 'inline-flex',
              width: '100%',
              height: '100%'
            }}></webview>
        : null
      }

;
  }

}

function toURL (domain) {
  if (/^https?:\/\//.test(domain)) {
    return domain;
  }

  if ('//' === domain.substr(0, 2)) {
    return domain;
  }

  return 'http://' + domain;
}
<|endoftext|>


<commit_before>/**
 *  \details © 2011  Open Ximdex Evolution SL [http://www.ximdex.org]
 *
 *  Ximdex a Semantic Content Management System (CMS)
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as published
 *  by the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  See the Affero GNU General Public License for more details.
 *  You should have received a copy of the Affero GNU General Public License
 *  version 3 along with Ximdex (see LICENSE file).
 *
 *  If not, visit http://gnu.org/licenses/agpl-3.0.html.
 *
 *  @author Ximdex DevTeam <dev@ximdex.com>
 *  @version $Revision$
 */
ximdexInstallerApp.controller('InstallDatabaseController', ["$timeout", '$scope', '$attrs', 'installerService', "$q", "$window",
 function($timeout, $scope, $attrs, installerService, $q, $window) {

    $scope.error=false;
    $scope.submit = false;
    $scope.root_user="root";
    $scope.installed=false;
    $scope.name = $attrs.ximInstallInstanceName;
    $scope.overwrite = false;
    $scope.host="localhost";
    $scope.port="3306";

    /* Causes a warning. Check, at the first, if host & port are right.
    installerService.sendAction("checkHost").then(function(response) {
        if (response.data.success){
            $scope.host=response.data.host;
            $scope.port=response.data.port;
            $scope.hostCheck = true;
        }else{
            $scope.hostCheck = 'host';
        }

    });*/

    $scope.sendForm = function(){
        if ($scope.installed){
            $scope.addUser();
        }else{
            if ($scope.overwrite){
                $scope.installDataBase();
            }
            else{
                $scope.processForm();
            }
        }
    }

    $scope.processForm = function(){
        if ($scope.formDataBase.name.$error.pattern){
            $scope.dbErrors = "Invalid database name. Only alphanumeric and underline characters are valid.";
            $scope.hostCheck = "exist_db";
        }else{
            $scope.genericErrors="";
            $scope.dbErrors="";
            $scope.loading = true;
            var index = 0;
            $scope.checkRootUser();
        }
    };

    $scope.checkRootUser = function(){

        var params = "user="+$scope.root_user;
        params += "&pass="+$scope.root_pass;
        params += "&host="+$scope.host;
        params += "&port="+$scope.port;
        installerService.sendAction("checkUser",params).then(function(response) {
        if (response.data.success){
            $scope.checkExistDataBase();

        }else{
            var error = response.data.errors.toLowerCase();
            if (error.indexOf("unknown mysql server host")>-1){
                $scope.hostCheck = "host";
            }else{
                $scope.hostCheck = "root_user";
            }
            $scope.loading = false;
            $scope.genericErrors = response.data.errors;
        }

    });
    };

    $scope.checkExistDataBase = function(){
        var params = "user="+$scope.root_user;
        params += "&pass="+$scope.root_pass;
        params += "&host="+$scope.host;
        params += "&port="+$scope.port;
        params += "&name="+$scope.name;
        installerService.sendAction("checkExistDataBase",params).then(function(response) {
        if (response.data.success){
            $scope.installDataBase();
        }else{
            $scope.dbErrors = $scope.name+" database already exists. Overwrite it?";
            $scope.hostCheck = "exist_db";
            $scope.overwrite = true;
            $scope.loading=false;
        }
     });
    };

    $scope.installDataBase = function(){
        if ($scope.overwrite)
            $scope.loadingOverwrite = true;
        var params = "user="+$scope.root_user;
        params += "&pass="+$scope.root_pass;
        params += "&host="+$scope.host;
        params += "&port="+$scope.port;
        params += "&name="+$scope.name;
        installerService.sendAction("createDataBase",params).then(function(response) {
        if ($scope.overwrite)
            $scope.loadingOverwrite = false;
        else
            $scope.loading=false;
        if (response.data.success){
            $scope.installed = true;
            $scope.user = $scope.name;
        }else{
            $scope.error = response.data.errors;
        }
    });

    }

    $scope.$watch("name", function(){
        $scope.overwrite = false;
        $scope.dbErrors = false;
        $scope.hostCheck=true;
    });

    $scope.$watch("host", function(){
        $scope.genericErrors = false;
        $scope.hostCheck=true;
    });
    $scope.$watch("port", function(){
        $scope.genericErrors = false;
        $scope.hostCheck=true;
    });

    $scope.$watch("root_user", function(){
        $scope.genericErrors = false;
        $scope.hostCheck=true;
    });

    $scope.$watch("root_pass", function(){
        $scope.genericErrors = false;
        $scope.hostCheck=true;
    });

     $scope.addUser = function(){
        $scope.loadingAddUser = true;
        var params = "user="+$scope.user;
        params += "&pass="+$scope.pass;
        params += "&host="+$scope.host;
        params += "&port="+$scope.port;
        params += "&name="+$scope.name;
        params += "&root_user="+$scope.root_user;
        params += "&root_pass="+$scope.root_pass;
        installerService.sendAction("addUser",params).then(function(response) {
            location.reload();
    });

    }

}]);<commit_msg>Corregido bug en la instalación, el parámetro password de mysql se envía vacío si no existe<commit_after>/**
 *  \details © 2011  Open Ximdex Evolution SL [http://www.ximdex.org]
 *
 *  Ximdex a Semantic Content Management System (CMS)
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as published
 *  by the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  See the Affero GNU General Public License for more details.
 *  You should have received a copy of the Affero GNU General Public License
 *  version 3 along with Ximdex (see LICENSE file).
 *
 *  If not, visit http://gnu.org/licenses/agpl-3.0.html.
 *
 *  @author Ximdex DevTeam <dev@ximdex.com>
 *  @version $Revision$
 */
ximdexInstallerApp.controller('InstallDatabaseController', ["$timeout", '$scope', '$attrs', 'installerService', "$q", "$window",
    function ($timeout, $scope, $attrs, installerService, $q, $window) {

        $scope.error = false;
        $scope.submit = false;
        $scope.root_user = "root";
        $scope.installed = false;
        $scope.name = $attrs.ximInstallInstanceName;
        $scope.overwrite = false;
        $scope.host = "localhost";
        $scope.port = "3306";
        $scope.root_pass = "";

        /* Causes a warning. Check, at the first, if host & port are right.
         installerService.sendAction("checkHost").then(function(response) {
         if (response.data.success){
         $scope.host=response.data.host;
         $scope.port=response.data.port;
         $scope.hostCheck = true;
         }else{
         $scope.hostCheck = 'host';
         }

         });*/

        $scope.sendForm = function () {
            if ($scope.installed) {
                $scope.addUser();
            } else {
                if ($scope.overwrite) {
                    $scope.installDataBase();
                }
                else {
                    $scope.processForm();
                }
            }
        }

        $scope.processForm = function () {
            if ($scope.formDataBase.name.$error.pattern) {
                $scope.dbErrors = "Invalid database name. Only alphanumeric and underline characters are valid.";
                $scope.hostCheck = "exist_db";
            } else {
                $scope.genericErrors = "";
                $scope.dbErrors = "";
                $scope.loading = true;
                var index = 0;
                $scope.checkRootUser();
            }
        };

        $scope.checkRootUser = function () {

            var params = "user=" + $scope.root_user;
            params += "&pass=" + $scope.root_pass;
            params += "&host=" + $scope.host;
            params += "&port=" + $scope.port;
            installerService.sendAction("checkUser", params).then(function (response) {
                if (response.data.success) {
                    $scope.checkExistDataBase();

                } else {
                    var error = response.data.errors.toLowerCase();
                    if (error.indexOf("unknown mysql server host") > -1) {
                        $scope.hostCheck = "host";
                    } else {
                        $scope.hostCheck = "root_user";
                    }
                    $scope.loading = false;
                    $scope.genericErrors = response.data.errors;
                }

            });
        };

        $scope.checkExistDataBase = function () {
            var params = "user=" + $scope.root_user;
            params += "&pass=" + $scope.root_pass;
            params += "&host=" + $scope.host;
            params += "&port=" + $scope.port;
            params += "&name=" + $scope.name;
            installerService.sendAction("checkExistDataBase", params).then(function (response) {
                if (response.data.success) {
                    $scope.installDataBase();
                } else {
                    $scope.dbErrors = $scope.name + " database already exists. Overwrite it?";
                    $scope.hostCheck = "exist_db";
                    $scope.overwrite = true;
                    $scope.loading = false;
                }
            });
        };

        $scope.installDataBase = function () {
            if ($scope.overwrite)
                $scope.loadingOverwrite = true;
            var params = "user=" + $scope.root_user;
            params += "&pass=" + $scope.root_pass;
            params += "&host=" + $scope.host;
            params += "&port=" + $scope.port;
            params += "&name=" + $scope.name;
            installerService.sendAction("createDataBase", params).then(function (response) {
                if ($scope.overwrite)
                    $scope.loadingOverwrite = false;
                else
                    $scope.loading = false;
                if (response.data.success) {
                    $scope.installed = true;
                    $scope.user = $scope.name;
                } else {
                    $scope.error = response.data.errors;
                }
            });

        }

        $scope.$watch("name", function () {
            $scope.overwrite = false;
            $scope.dbErrors = false;
            $scope.hostCheck = true;
        });

        $scope.$watch("host", function () {
            $scope.genericErrors = false;
            $scope.hostCheck = true;
        });
        $scope.$watch("port", function () {
            $scope.genericErrors = false;
            $scope.hostCheck = true;
        });

        $scope.$watch("root_user", function () {
            $scope.genericErrors = false;
            $scope.hostCheck = true;
        });

        $scope.$watch("root_pass", function () {
            $scope.genericErrors = false;
            $scope.hostCheck = true;
        });

        $scope.addUser = function () {
            $scope.loadingAddUser = true;
            var params = "user=" + $scope.user;
            params += "&pass=" + $scope.pass;
            params += "&host=" + $scope.host;
            params += "&port=" + $scope.port;
            params += "&name=" + $scope.name;
            params += "&root_user=" + $scope.root_user;
            params += "&root_pass=" + $scope.root_pass;
            installerService.sendAction("addUser", params).then(function (response) {
                location.reload();
            });

        }

    }]);<|endoftext|>


<commit_before>import Ansi from 'ansi-to-html';
import hasAnsi from 'has-ansi';

let ansi;
let $timeout;
let $sce;
let $compile;
let $scope;

const record = {};

const EVENT_START_TASK = 'playbook_on_task_start';
const EVENT_START_PLAY = 'playbook_on_play_start';
const EVENT_STATS_PLAY = 'playbook_on_stats';

const EVENT_GROUPS = [
    EVENT_START_TASK,
    EVENT_START_PLAY
];

const TIME_EVENTS = [
    EVENT_START_TASK,
    EVENT_START_PLAY,
    EVENT_STATS_PLAY
];

function JobsIndexController (job, _$sce_, _$timeout_, _$scope_, _$compile_) {
    ansi = new Ansi();
    $timeout = _$timeout_;
    $sce = _$sce_;
    $compile = _$compile_;
    $scope = _$scope_;

    const vm = this || {};
    const events = job.get('related.job_events.results');
    const html = $sce.trustAsHtml(parseEvents(events));

    vm.toggle = toggle;

    $timeout(() => {
        const table = $('#result-table');

        table.html($sce.getTrustedHtml(html));
        $compile(table.contents())($scope);
    });

    console.log(record);
}

function parseEvents (events) {
    events.sort(orderByLineNumber);

    return events.reduce((html, event) => `${html}${parseLine(event)}`, '');
}

function orderByLineNumber (a, b) {
    if (a.start_line > b.start_line) {
        return 1;
    }

    if (a.start_line < b.start_line) {
        return -1;
    }

    return 0;
}

function parseLine (event) {
    if (!event || !event.stdout) {
        return '';
    }

    const { stdout } = event;
    const lines = stdout.split('\r\n');

    let eventLn = event.start_line;
    let ln = event.start_line + 1;

    if (lines[0] === '') {
        ln++;
    }

    record[ln] = {
        line: ln,
        id: event.id,
        uuid: event.uuid,
        level: event.event_level,
        start: event.start_line,
        end: event.end_line,
        isTruncated: (event.end_line - event.start_line) > lines.length
    };

    if (record[ln].isTruncated) {
        record[ln].truncatedAt = event.start_line + lines.length;
    }

    if (EVENT_GROUPS.includes(event.event)) {
        record[ln].parent = true;
    }

    const current = record[ln];

    return lines.reduce((html, line, i) => {
        eventLn++;

        const time = getTime(event, i);
        const group = getGroup(event, i);
        const isLastLine = i === lines.length - 1;

        if (current.isTruncated && isLastLine) {
            return `${html}${createRow(eventLn, line, time, group)}${createTruncatedRow(event.id)}`;
        }

        return `${html}${createRow(eventLn, line, time, group)}`;
    }, '');
}

function createTruncatedRow (id) {
    return `

            ...

`;
}

function createRow (ln, content, time, group) {
    content = hasAnsi(content) ? ansi.toHtml(content) : content;

    let expand = '';
    if (group.parent) {
        expand = '';
    }

    return `

            ${expand}
            ${ln}
            ${content}
            ${time}

`;
}

function getGroup (event, i) {
    const group = {};

    if (EVENT_GROUPS.includes(event.event) && i === 1) {
        group.parent = true;
        group.classList = `parent parent-${event.event_level}`;
        group.id = i;
    } else {
        group.classList = '';
    }

    group.level = event.event_level;

    return group;
}

function getTime (event, i) {
    if (!TIME_EVENTS.includes(event.event) || i !== 1) {
        return '';
    }

    const date = new Date(event.created);

    return `${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`;
}

function toggle (id) {
    console.log(id);
}

JobsIndexController.$inject = ['job', '$sce', '$timeout', '$scope', '$compile'];

module.exports = JobsIndexController;
<commit_msg>Use event record with output template<commit_after>import Ansi from 'ansi-to-html';
import hasAnsi from 'has-ansi';

let ansi;
let $timeout;
let $sce;
let $compile;
let $scope;

const record = {};

const EVENT_START_TASK = 'playbook_on_task_start';
const EVENT_START_PLAY = 'playbook_on_play_start';
const EVENT_STATS_PLAY = 'playbook_on_stats';

const EVENT_GROUPS = [
    EVENT_START_TASK,
    EVENT_START_PLAY
];

const TIME_EVENTS = [
    EVENT_START_TASK,
    EVENT_START_PLAY,
    EVENT_STATS_PLAY
];

function JobsIndexController (job, _$sce_, _$timeout_, _$scope_, _$compile_) {
    $timeout = _$timeout_;
    $sce = _$sce_;
    $compile = _$compile_;
    $scope = _$scope_;
    ansi = new Ansi();

    const vm = this || {};
    const events = job.get('related.job_events.results');
    const html = $sce.trustAsHtml(parseEvents(events));

    vm.toggle = toggle;

    $timeout(() => {
        const table = $('#result-table');

        table.html($sce.getTrustedHtml(html));
        $compile(table.contents())($scope);
    });

    console.log(record);
}

function parseEvents (events) {
    events.sort(orderByLineNumber);

    return events.reduce((html, event) => `${html}${parseLine(event)}`, '');
}

function orderByLineNumber (a, b) {
    if (a.start_line > b.start_line) {
        return 1;
    }

    if (a.start_line < b.start_line) {
        return -1;
    }

    return 0;
}

function parseLine (event) {
    if (!event || !event.stdout) {
        return '';
    }

    const { stdout } = event;
    const lines = stdout.split('\r\n');

    let eventLine = event.start_line;
    let displayLine = event.start_line + 1;

    if (lines[0] === '') {
        displayLine++;
    }

    record[displayLine] = {
        line: displayLine,
        id: event.id,
        uuid: event.uuid,
        level: event.event_level,
        start: event.start_line,
        end: event.end_line,
        isTruncated: (event.end_line - event.start_line) > lines.length,
    };

    if (record[displayLine].isTruncated) {
        record[displayLine].truncatedAt = event.start_line + lines.length;
    }

    if (EVENT_GROUPS.includes(event.event)) {
        record[displayLine].isParent = true;
    }

    if (TIME_EVENTS.includes(event.event)) {
        record[displayLine].time = getTime(event.created);
    }

    const current = record[displayLine];

    return lines.reduce((html, line, i) => {
        eventLine++;

        const isLastLine = i === lines.length - 1;
        let append = createRow(eventLine, line, current);

        if (current.isTruncated && isLastLine) {
            append += createRow();
        }

        return `${html}${append}`;
    }, '');
}

function createRow (ln, content, current) {
    let expand = '';
    let timestamp = '';
    let toggleRow = '';
    let classList = '';

    content = content || '';

    if (hasAnsi(content)) {
        content = ansi.toHtml(content);
    }

    if (current) {
        if (current.line === ln) {
            if (current.isParent) {
                expand = '';
                toggleRow = `<td class="at-Stdout-toggle" ng-click="vm.toggle(${ln})">${expand}</td>`;
            }

            if (current.time) {
                timestamp = current.time;
            }
        } else {
            classList += `child-of-${current.line}`;
        }
    }

    if (!toggleRow) {
        toggleRow = '	';
    }

    if (!ln) {
        ln = '...';
    }

    return `

            ${toggleRow}
            ${ln}
            ${content}
            ${timestamp}

`;
}

function getTime (created) {
    const date = new Date(created);

    return `${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`;
}

function toggle (line) {
    const lines = document.getElementsByClassName(`child-of-${line}`);
    console.log(lines);
}

JobsIndexController.$inject = ['job', '$sce', '$timeout', '$scope', '$compile'];

module.exports = JobsIndexController;
<|endoftext|>


<commit_before>/**
 * Standalone script to import FrameNet lexical units to MongoDB.
 */
const AnnotationSet = require('noframenet-core').AnnotationSet;
const Frame = require('noframenet-core').Frame;
const FrameElement = require('noframenet-core').FrameElement;
const Label = require('noframenet-core').Label;
const Pattern = require('noframenet-core').Pattern;
const Sentence = require('noframenet-core').Sentence;
const ValenceUnit = require('noframenet-core').ValenceUnit;
const ProgressBar = require('ascii-progress');
const toJsonixLabelArray = require('./../utils/jsonixUtils').toJsonixLabelArray;
const toJsonixLayerArray = require('./../utils/jsonixUtils').toJsonixLayerArray;
const toJsonixLexUnitSentenceArray = require('./../utils/jsonixUtils').toJsonixLexUnitSentenceArray;
const toJsonixPatternAnnoSetArray = require('./../utils/jsonixUtils').toJsonixPatternAnnoSetArray;
const toJsonixPatternArray = require('./../utils/jsonixUtils').toJsonixPatternArray;
const toJsonixSentenceAnnoSetArray = require('./../utils/jsonixUtils').toJsonixSentenceAnnoSetArray;
const toJsonixValenceUnitArray = require('./../utils/jsonixUtils').toJsonixValenceUnitArray;
const config = require('./../config');
const driver = require('./../db/mongoose');
const marshaller = require('./../marshalling/unmarshaller');
const utils = require('./../utils/utils');
const mongoose = require('mongoose');
mongoose.Promise = require('bluebird');

const logger = config.logger;

function convertToValenceUnits(jsonixPattern, valenceUnitsMap,
                               feName2IDmap) {
  return toJsonixValenceUnitArray(jsonixPattern).map((jsonixValenceUnit) => {
    const feID = feName2IDmap.get(jsonixValenceUnit.fe);
    if (!feID) {
      throw new Error(`FE is undefined: ${jsonixValenceUnit.fe}`);
    } else {
      const key = `${feID}#${jsonixValenceUnit.pt}#${jsonixValenceUnit.gf}`;
      // Note that here feID is a Number as we are using the original
      // FrameNet id. So we don't need to stringify feID
      let valenceUnit;
      if (!valenceUnitsMap.has(key)) {
        valenceUnit = new ValenceUnit({
          FE: feID,
          PT: jsonixValenceUnit.pt,
          GF: jsonixValenceUnit.gf,
        });
        valenceUnitsMap.set(key, valenceUnit.toObject());
      } else {
        valenceUnit = valenceUnitsMap.get(key);
      }
      return valenceUnit;
    }
  });
}

function processPatterns(jsonixLexUnit, annoSetID2PatternIDmap, patternsMap,
                         valenceUnitsMap, feName2IDmap) {
  // Import patterns and valenceUnits.
  // Add all info regading patterns and lexUnits to AnnoSet objects
  toJsonixPatternArray(jsonixLexUnit).forEach((jsonixPattern) => {
    try {
      const vus = convertToValenceUnits(jsonixPattern, valenceUnitsMap,
                                        feName2IDmap);
      if (vus.length === 0) {
        logger.error('vus.length === 0');
      }
      vus.forEach((vu) => {
        //logger.info(`vu._id = ${vu._id}, vu.id = ${vu.id}`);
        if (!vu._id) {
          logger.error(`vu._id does not exist: ${vu._id}`);
        } else if (vu._id === null || vu._id === undefined) {
          logger.error(`vu._id is null or undefined: ${vu._id}`);
        }
      });
      if (vus.length === 0) {
        logger.error('vus.length === 0');
      }
      const key = vus.map(vu => vu._id.toString()).sort().join('#');
      let pattern;
      if (!patternsMap.has(key)) {
        pattern = new Pattern({
          valenceUnits: vus.map(vu => vu._id),
        });
        patternsMap.set(key, pattern.toObject());
      } else {
        pattern = patternsMap.get(key);
      }
      toJsonixPatternAnnoSetArray(jsonixPattern).forEach((jsonixAnnoSet) => {
        annoSetID2PatternIDmap.set(jsonixAnnoSet.id, pattern._id);
      });
    } catch (err) {
      logger.verbose(`${err.message} in lexUnit #${jsonixLexUnit.value.id}`);
    }
  });
}

function convertToLabels(jsonixAnnoSet) {
  return toJsonixLayerArray(jsonixAnnoSet)
    .map(jsonixLayer => toJsonixLabelArray(jsonixLayer)
      .map(jsonixLabel => new Label({
        name: jsonixLabel.name,
        type: jsonixLayer.name,
        rank: jsonixLayer.rank,
        startPos: jsonixLabel.start,
        endPos: jsonixLabel.end,
      }).toObject()))
    .reduce((a, b) => a.concat(b));
}

function convertToAnnoSets(jsonixSentence, lexUnitId, labels) {
  return toJsonixSentenceAnnoSetArray(jsonixSentence).map(jsonixAnnoSet => new AnnotationSet({
    _id: jsonixAnnoSet.id,
    lexUnit: lexUnitId,
    sentence: jsonixSentence.id,
    labels: convertToLabels(jsonixAnnoSet)
      .map((label) => {
        labels.push(label);
        return label._id;
      }),
  }).toObject());
}

function convertToSentences(jsonixLexUnit, annotationSets, labels) {
  return toJsonixLexUnitSentenceArray(jsonixLexUnit)
    .map((jsonixSentence) => {
      annotationSets.push(...convertToAnnoSets(jsonixSentence, jsonixLexUnit.value.id, labels));
      return new Sentence({
        _id: jsonixSentence.id,
        text: jsonixSentence.text,
        paragraphNumber: jsonixSentence.paragNo,
        sentenceNumber: jsonixSentence.sentNo,
        aPos: jsonixSentence.aPos,
      }).toObject();
    });
}

// Lemma and Lexeme information is updated via
// importLemmasAndLexemes script
function processLexUnit(jsonixLexUnit, annoSetID2PatternIDmap, annotationSets,
                        labels, patternsMap, sentences, valenceUnitsMap,
                        feName2IDmap) {
  sentences.push(...convertToSentences(jsonixLexUnit, annotationSets, labels));
  processPatterns(jsonixLexUnit, annoSetID2PatternIDmap, patternsMap,
                  valenceUnitsMap, feName2IDmap);
}

// A Map of FEName -> full noframenet-core.FrameElement for a given Frame
async function getFEName2IDmap(frameID) {
  const feMap = new Map();
  const frame = await Frame.findById(frameID);
  if (!frame) {
    logger.error(`frame is null for _id = ${frameID}`);
  }
  const fes = await FrameElement.find().where('_id').in(frame.frameElements);
  fes.forEach((fe) => {
    feMap.set(fe.name, fe._id);
  });
  return feMap;
}

async function convertToObjects(batch, uniques) {
  const data = {
    annotationSets: [],
    labels: [],
    sentences: [],
  };
  await Promise.all(batch.map(async (file) => {
    const jsonixLexUnit = await marshaller.unmarshall(file);
    const feName2IDmap = await getFEName2IDmap(jsonixLexUnit.value.frameID);
    try {
      processLexUnit(jsonixLexUnit, uniques.annoSetID2PatternIDmap,
                     data.annotationSets, data.labels, uniques.patternsMap,
                     data.sentences, uniques.valenceUnitsMap, feName2IDmap);
    } catch (err) {
      logger.verbose(`Ill-formed lexUnit detected: ID = ${jsonixLexUnit.value.id}`);
      logger.debug(err);
    }
  }));
  return data;
}

async function saveArraysToDb(data) {
  await AnnotationSet.collection.insertMany(data.annotationSets);
  await Label.collection.insertMany(data.labels);
  await Sentence.collection.insertMany(data.sentences);
}

async function saveMapsToDb(maps) {
  await Pattern.collection.insertMany(Array.from(maps.patternsMap.values()));
  await ValenceUnit.collection.insertMany(
    Array.from(maps.valenceUnitsMap.values()));
  logger.info('Updating annotatioSets\' pattern references...');
  const annoSetProgressBar = new ProgressBar({
    total: maps.annoSetID2PatternIDmap.size,
    clean: true,
  });
  let count = 0;
  let tickCount = 0;
  for (const entry of maps.annoSetID2PatternIDmap) {
    const annoSetId = entry[0];
    const patternId = entry[1];
    await AnnotationSet.collection.update({ _id: annoSetId },
                                          { $set: { pattern: patternId } });
    count += 1;
    if (count === 1000) {
      annoSetProgressBar.tick(1000);
      tickCount += 1000;
      count = 0;
    } if (count === maps.annoSetID2PatternIDmap.size) {
      annoSetProgressBar.tick(maps.annoSetID2PatternIDmap.size - tickCount);
    }
  }
}

async function importBatchSet(batchSet) {
  let counter = 1;
  const lexUnitProgressBar = new ProgressBar({
    total: batchSet.length,
    clean: true,
  });
  logger.info('Importing lexical units by batch...');
  const uniques = {
    annoSetID2PatternIDmap: new Map(),
    patternsMap: new Map(),
    valenceUnitsMap: new Map(),
  };
  for (const batch of batchSet) {
    logger.debug(`Importing lexUnit batch ${counter} out of ${batchSet.length}...`);
    const data = await convertToObjects(batch, uniques);
    try {
      await saveArraysToDb(data);
    } catch (err) {
      logger.error(err);
      logger.info('Exiting NoFrameNet');
      process.exit(1);
    }
    counter += 1;
    lexUnitProgressBar.tick();
  }
  try {
    await saveMapsToDb(uniques);
  } catch (err) {
    logger.error(err);
    logger.info('Exiting NoFrameNet');
    process.exit(1);
  }
}

async function importLexUnitsOnceConnectedToDb(lexUnitDir, chunkSize) {
  let batchSet;
  try {
    batchSet = await utils.filterAndChunk(lexUnitDir, chunkSize);
  } catch (err) {
    logger.error(err);
    logger.info('Exiting NoFrameNet');
    process.exit(1);
  }
  await importBatchSet(batchSet);
}

async function importLexUnits(lexUnitDir, chunkSize, dbUri) {
  await driver.connectToDatabase(dbUri);
  await importLexUnitsOnceConnectedToDb(lexUnitDir, chunkSize);
  await mongoose.disconnect();
}

if (require.main === module) {
  const startTime = process.hrtime();
  const dbUri = config.dbUri;
  const lexUnitDir = config.frameNetDir.concat('lu');
  const lexUnitChunkSize = config.lexUnitChunkSize;
  importLexUnits(lexUnitDir, lexUnitChunkSize, dbUri).then(() => logger.info(`Import process completed in ${process.hrtime(startTime)[0]}s`));
}

module.exports = {
  importLexUnitsOnceConnectedToDb,
};
<commit_msg>Started refactoring with annoSetsMap and sentencesMap<commit_after>/**
 * Standalone script to import FrameNet lexical units to MongoDB.
 */
const AnnotationSet = require('noframenet-core').AnnotationSet;
const Frame = require('noframenet-core').Frame;
const FrameElement = require('noframenet-core').FrameElement;
const Label = require('noframenet-core').Label;
const Pattern = require('noframenet-core').Pattern;
const Sentence = require('noframenet-core').Sentence;
const ValenceUnit = require('noframenet-core').ValenceUnit;
const ProgressBar = require('ascii-progress');
const toJsonixLabelArray = require('./../utils/jsonixUtils').toJsonixLabelArray;
const toJsonixLayerArray = require('./../utils/jsonixUtils').toJsonixLayerArray;
const toJsonixLexUnitSentenceArray = require('./../utils/jsonixUtils').toJsonixLexUnitSentenceArray;
const toJsonixPatternAnnoSetArray = require('./../utils/jsonixUtils').toJsonixPatternAnnoSetArray;
const toJsonixPatternArray = require('./../utils/jsonixUtils').toJsonixPatternArray;
const toJsonixSentenceAnnoSetArray = require('./../utils/jsonixUtils').toJsonixSentenceAnnoSetArray;
const toJsonixValenceUnitArray = require('./../utils/jsonixUtils').toJsonixValenceUnitArray;
const config = require('./../config');
const driver = require('./../db/mongoose');
const marshaller = require('./../marshalling/unmarshaller');
const utils = require('./../utils/utils');
const mongoose = require('mongoose');
mongoose.Promise = require('bluebird');

const logger = config.logger;

function convertToValenceUnits(jsonixPattern, valenceUnitsMap,
                               feName2IDmap) {
  return toJsonixValenceUnitArray(jsonixPattern).map((jsonixValenceUnit) => {
    const feID = feName2IDmap.get(jsonixValenceUnit.fe);
    if (!feID) {
      throw new Error(`FE is undefined: ${jsonixValenceUnit.fe}`);
    } else {
      const key = `${feID}#${jsonixValenceUnit.pt}#${jsonixValenceUnit.gf}`;
      // Note that here feID is a Number as we are using the original
      // FrameNet id. So we don't need to stringify feID
      let valenceUnit;
      if (!valenceUnitsMap.has(key)) {
        valenceUnit = new ValenceUnit({
          FE: feID,
          PT: jsonixValenceUnit.pt,
          GF: jsonixValenceUnit.gf,
        });
        valenceUnitsMap.set(key, valenceUnit.toObject());
      } else {
        valenceUnit = valenceUnitsMap.get(key);
      }
      return valenceUnit;
    }
  });
}

function processPatterns(jsonixLexUnit, annoSetID2PatternIDmap, patternsMap,
                         valenceUnitsMap, feName2IDmap) {
  // Import patterns and valenceUnits.
  // Add all info regading patterns and lexUnits to AnnoSet objects
  toJsonixPatternArray(jsonixLexUnit).forEach((jsonixPattern) => {
    try {
      const vus = convertToValenceUnits(jsonixPattern, valenceUnitsMap,
                                        feName2IDmap);
      if (vus.length === 0) {
        logger.error('vus.length === 0');
      }
      vus.forEach((vu) => {
        //logger.info(`vu._id = ${vu._id}, vu.id = ${vu.id}`);
        if (!vu._id) {
          logger.error(`vu._id does not exist: ${vu._id}`);
        } else if (vu._id === null || vu._id === undefined) {
          logger.error(`vu._id is null or undefined: ${vu._id}`);
        }
      });
      if (vus.length === 0) {
        logger.error('vus.length === 0');
      }
      const key = vus.map(vu => vu._id.toString()).sort().join('#');
      let pattern;
      if (!patternsMap.has(key)) {
        pattern = new Pattern({
          valenceUnits: vus.map(vu => vu._id),
        });
        patternsMap.set(key, pattern.toObject());
      } else {
        pattern = patternsMap.get(key);
      }
      toJsonixPatternAnnoSetArray(jsonixPattern).forEach((jsonixAnnoSet) => {
        annoSetID2PatternIDmap.set(jsonixAnnoSet.id, pattern._id);
      });
    } catch (err) {
      logger.verbose(`${err.message} in lexUnit #${jsonixLexUnit.value.id}`);
    }
  });
}

function convertToLabels(jsonixAnnoSet) {
  return toJsonixLayerArray(jsonixAnnoSet)
    .map(jsonixLayer => toJsonixLabelArray(jsonixLayer)
      .map(jsonixLabel => new Label({
        name: jsonixLabel.name,
        type: jsonixLayer.name,
        rank: jsonixLayer.rank,
        startPos: jsonixLabel.start,
        endPos: jsonixLabel.end,
      }).toObject()))
    .reduce((a, b) => a.concat(b));
}

function convertToAnnoSets(jsonixSentence, lexUnitId, labels) {
  return toJsonixSentenceAnnoSetArray(jsonixSentence).map(jsonixAnnoSet => new AnnotationSet({
    _id: jsonixAnnoSet.id,
    lexUnit: lexUnitId,
    sentence: jsonixSentence.id,
    labels: convertToLabels(jsonixAnnoSet)
      .map((label) => {
        labels.push(label);
        return label._id;
      }),
  }).toObject());
}

function convertToSentences(jsonixLexUnit, annotationSets, labels) {
  return toJsonixLexUnitSentenceArray(jsonixLexUnit)
    .map((jsonixSentence) => {
      annotationSets.push(...convertToAnnoSets(jsonixSentence, jsonixLexUnit.value.id, labels));
      return new Sentence({
        _id: jsonixSentence.id,
        text: jsonixSentence.text,
        paragraphNumber: jsonixSentence.paragNo,
        sentenceNumber: jsonixSentence.sentNo,
        aPos: jsonixSentence.aPos,
      }).toObject();
    });
}

function processSentences(jsonixLexUnit, annoSetsMap, sentencesMap, labels) {
  toJsonixLexUnitSentenceArray(jsonixLexUnit).forEach((jsonixSentence) => {

  });
}

// Lemma and Lexeme information is updated via
// importLemmasAndLexemes script
function processLexUnit(jsonixLexUnit, annoSetsMap, patternsMap, sentencesMap,
                        valenceUnitsMap, labels, feName2IDmap) {
  //sentences.push(...convertToSentences(jsonixLexUnit, annotationSets, labels));
  processSentences(jsonixLexUnit, annoSetsMap, sentencesMap, labels);
  processPatterns(jsonixLexUnit, annoSetID2PatternIDmap, patternsMap,
                  valenceUnitsMap, feName2IDmap);
}

// A Map of FEName -> full noframenet-core.FrameElement for a given Frame
async function getFEName2IDmap(frameID) {
  const feMap = new Map();
  const frame = await Frame.findById(frameID);
  if (!frame) {
    logger.error(`frame is null for _id = ${frameID}`);
  }
  const fes = await FrameElement.find().where('_id').in(frame.frameElements);
  fes.forEach((fe) => {
    feMap.set(fe.name, fe._id);
  });
  return feMap;
}

async function convertToObjects(batch, uniques) {
  const data = {
    //annotationSets: [],
    labels: [],
    //sentences: [],
  };
  await Promise.all(batch.map(async (file) => {
    const jsonixLexUnit = await marshaller.unmarshall(file);
    const feName2IDmap = await getFEName2IDmap(jsonixLexUnit.value.frameID);
    try {
      processLexUnit(jsonixLexUnit, uniques.annoSetID2PatternIDmap,
                     data.annotationSets, data.labels, uniques.patternsMap,
                     data.sentences, uniques.valenceUnitsMap, feName2IDmap);
    } catch (err) {
      logger.verbose(`Ill-formed lexUnit detected: ID = ${jsonixLexUnit.value.id}`);
      logger.debug(err);
    }
  }));
  return data;
}

async function saveArraysToDb(data) {
  //await AnnotationSet.collection.insertMany(data.annotationSets);
  await Label.collection.insertMany(data.labels);
  //await Sentence.collection.insertMany(data.sentences);
}

async function saveMapsToDb(maps) {
  await AnnotationSet.collection.insertMany(
    Array.from(maps.annoSetsMap.values()));
  await Pattern.collection.insertMany(Array.from(maps.patternsMap.values()));
  await Sentence.collection.insertMany(Array.from(maps.sentencesMap.values()));
  await ValenceUnit.collection.insertMany(
    Array.from(maps.valenceUnitsMap.values()));
  /*logger.info('Updating annotatioSets\' pattern references...');
  const annoSetProgressBar = new ProgressBar({
    total: maps.annoSetID2PatternIDmap.size,
    clean: true,
  });
  let count = 0;
  let tickCount = 0;
  for (const entry of maps.annoSetID2PatternIDmap) {
    const annoSetId = entry[0];
    const patternId = entry[1];
    await AnnotationSet.collection.update({ _id: annoSetId },
                                          { $set: { pattern: patternId } });
    count += 1;
    if (count === 1000) {
      annoSetProgressBar.tick(1000);
      tickCount += 1000;
      count = 0;
    } if (count === maps.annoSetID2PatternIDmap.size) {
      annoSetProgressBar.tick(maps.annoSetID2PatternIDmap.size - tickCount);
    }
  }*/
}

async function importBatchSet(batchSet) {
  let counter = 1;
  const lexUnitProgressBar = new ProgressBar({
    total: batchSet.length,
    clean: true,
  });
  logger.info('Importing lexical units by batch...');
  const uniques = {
    //annoSetID2PatternIDmap: new Map(),
    annoSetsMap: new Map(),
    patternsMap: new Map(),
    sentencesMap: new Map(),
    valenceUnitsMap: new Map(),
  };
  for (const batch of batchSet) {
    logger.debug(`Importing lexUnit batch ${counter} out of ${batchSet.length}...`);
    const data = await convertToObjects(batch, uniques);
    try {
      await saveArraysToDb(data);
    } catch (err) {
      logger.error(err);
      logger.info('Exiting NoFrameNet');
      process.exit(1);
    }
    counter += 1;
    lexUnitProgressBar.tick();
  }
  try {
    await saveMapsToDb(uniques);
  } catch (err) {
    logger.error(err);
    logger.info('Exiting NoFrameNet');
    process.exit(1);
  }
}

async function importLexUnitsOnceConnectedToDb(lexUnitDir, chunkSize) {
  let batchSet;
  try {
    batchSet = await utils.filterAndChunk(lexUnitDir, chunkSize);
  } catch (err) {
    logger.error(err);
    logger.info('Exiting NoFrameNet');
    process.exit(1);
  }
  await importBatchSet(batchSet);
}

async function importLexUnits(lexUnitDir, chunkSize, dbUri) {
  await driver.connectToDatabase(dbUri);
  await importLexUnitsOnceConnectedToDb(lexUnitDir, chunkSize);
  await mongoose.disconnect();
}

if (require.main === module) {
  const startTime = process.hrtime();
  const dbUri = config.dbUri;
  const lexUnitDir = config.frameNetDir.concat('lu');
  const lexUnitChunkSize = config.lexUnitChunkSize;
  importLexUnits(lexUnitDir, lexUnitChunkSize, dbUri).then(() => logger.info(`Import process completed in ${process.hrtime(startTime)[0]}s`));
}

module.exports = {
  importLexUnitsOnceConnectedToDb,
};
<|endoftext|>


<commit_before>File = function(name){
    this.name = name
}
File.prototype = {
    write: function(text){
        localStorage.setItem(this.name, text);
    },
    read: function(){
        return localStorage.getItem(this.name);
    },
    append: function(text){
        this.write(this.read() + text);
    }
};

function In(stream, end){
    this.stream = stream;
    this.index = 0;
    this.depth = 0;
    this.end = end
    this.killer = null;
}
In.prototype = {
    wrap: function(callback){
        var self = this;
        return function(line){
            if(self.end){
                try{
                    callback(line);
                }catch(e){
                    self.end(e);
                }
            }else{
                callback(line)
            }
        }
    },
    readln: function(callback){
        if(this.killer){
            throw this.killer
        }
        if(this.index < this.stream.lines.length){
            this.index ++;
            this.wrap(callback)(this.stream.lines[this.index-1]);
        }
        else{
            this.index ++;
            this.stream.line_listener.push(this.wrap(callback));
        }
    },
    read: function(callback){
        if(this.killer){
            throw this.killer
        }
        if(this.index < this.stream.lines.length || this.depth < this.stream.line.length ){
            prevind = this.index;
            prevdepth = this.depth;
            this.index = this.stream.lines.length;
            this.depth = this.stream.line.length;
            this.wrap(callback)(this.stream.lines.slice(prevind).join("\n") + "\n" + this.stream.line.slice(prevdepth));

        }
        else{
            this.index = this.stream.lines.length;
            this.depth = this.stream.line.length;
            this.stream.listener.push(this.wrap(callback));
        }
    },
    kill: function(error){
        this.killer = error;
    },
};
function Stream(){
    this.lines = [];
    this.line = "";
    this.line_listener = [];
    this.listener = [];

};
Stream.prototype = {
    write: function(text){
        var lines = text.split("\n")
        for(var i = 0; i < lines.length-1; i++){
            this.line += lines[i]
            this.endln()
        }
        this.line += lines[lines.length-1]
        var callbacks = this.listener;
        this.listener = [];
        for(var i = 0; i < callbacks.length; i++){
            callbacks[i](text)
        }
    },
    writeln: function(text){
        this.write(text+"\n")
    },
    endln: function(){
        var line_listener = this.line_listener;
        this.line_listener = [];
        for(var i = 0; i < line_listener.length; i++){
            line_listener[i](this.line);
        }
        this.lines.push(this.line)
        this.line = ""
    },
    reader: function(end){
        return new In(this, end);
    },
};
<commit_msg>stream uses pseudofile to store lines<commit_after>PseudoFile = function(){
    this.arr = [];
}
PseudoFile.prototype = {
    write: function(text){
        this.arr = text.split("\n");
    },
    append: function(line){
        this.arr.push(line);
    },
    read: function(){
        return this.arr.join("\n");
    },
    readLnAt: function(indice){
        return this.arr[indice];
    },
    readLnFrom: function(indice){
        return this.arr.slice(indice)
    },
    get length(){
        return this.arr.length;
    },
};

File = function(name){
    this.name = name
}
File.prototype = {
    write: function(text){
        localStorage.setItem(this.name, text);
    },
    append: function(text){
        this.write(this.read() + text);
    },
    read: function(){
        return localStorage.getItem(this.name);
    },
    readLnAt: function(indice){
        return this.read().split("\n")[indice];
    },
    readLnFrom: function(indice){
        return this.read().split("\n").slice(indice);
    },
    get length(){
        return (this.read().match(/\n/g) || []).length
    },
};
function In(stream, end){
    this.stream = stream;
    this.index = 0;
    this.depth = 0;
    this.end = end
    this.killer = null;
}
In.prototype = {
    wrap: function(callback){
        var self = this;
        return function(line){
            if(self.end){
                try{
                    callback(line);
                }catch(e){
                    self.end(e);
                }
            }else{
                callback(line)
            }
        }
    },
    readln: function(callback){
        if(this.killer){
            throw this.killer
        }
        if(this.index < this.stream.lines.length){
            this.index ++;
            this.wrap(callback)(this.stream.lines.readLnAt(this.index-1));
        }
        else{
            this.index ++;
            this.stream.line_listener.push(this.wrap(callback));
        }
    },
    read: function(callback){
        if(this.killer){
            throw this.killer
        }
        if(this.index < this.stream.lines.length || this.depth < this.stream.line.length ){
            prevind = this.index;
            prevdepth = this.depth;
            this.index = this.stream.lines.length;
            this.depth = this.stream.line.length;
            this.wrap(callback)(this.stream.lines.readLnFrom(prevind).join("\n") + "\n" + this.stream.line.slice(prevdepth));
        }
        else{
            this.index = this.stream.lines.length;
            this.depth = this.stream.line.length;
            this.stream.listener.push(this.wrap(callback));
        }
    },
    kill: function(error){
        this.killer = error;
    },
};
function Stream(){
    this.lines = new PseudoFile();
    this.line = "";
    this.line_listener = [];
    this.listener = [];

};
Stream.prototype = {
    write: function(text){
        var lines = text.split("\n")
        for(var i = 0; i < lines.length-1; i++){
            this.line += lines[i]
            this.endln()
        }
        this.line += lines[lines.length-1]
        var callbacks = this.listener;
        this.listener = [];
        for(var i = 0; i < callbacks.length; i++){
            callbacks[i](text)
        }
    },
    writeln: function(text){
        this.write(text+"\n")
    },
    endln: function(){
        var line_listener = this.line_listener;
        this.line_listener = [];
        for(var i = 0; i < line_listener.length; i++){
            line_listener[i](this.line);
        }
        this.lines.append(this.line)
        this.line = "";
    },
    reader: function(end){
        return new In(this, end);
    },
};
function FileStream(filename){
    Stream.call(this);
    this.lines = new File(filename);
}
FileStream.prototype = Object.create(Stream.prototype)
<|endoftext|>