<commit_before>/*
 * grabber
 * https://github.com/ezoapp/grabber.git
 *
 * Copyright (c) 2014
 * Licensed under the MIT license.
 */

'use strict';

var _ = require('underscore'),
  request = require('request'),
  fs = require('fs'),
  cheerio = require('cheerio'),
  urllib = require('url'),
  pathlib = require('path'),
  glob = require('glob'),
  wrench = require('wrench'),
  hashids = new(require('hashids'))(new Date().getTime() + 'T');

function Grabber(data, options, logger, originUrl) {
  this.$ = null;
  this.data = data;
  this.baseUrl = '';
  this.tasklist = [];
  this.options = _.extend({
    dstDir: 'www',
    baseDir: '',
    tmplDir: '',
    indexFile: 'index.html'
  }, options || {});
  this.logger = (logger) ? logger : console;
  this.originUrl = new RegExp(originUrl, 'i');
  this.streamlist = [];
}

Grabber.prototype = (function () {

  var RES_TAG = {
    'href': ['link'], //,'a'],
    'src': ['img', 'script'], //, 'iframe'],
    'data-background-image': ['a']
  },
    USE_LOCAL_RES = {
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile-1.4.2.min.js': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile-1.4.2.min.js',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile-1.4.2.min.css',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile.external-png-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile.external-png-1.4.2.min.css',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile.icons-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile.icons-1.4.2.min.min.css',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile.inline-png-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile.inline-png-1.4.2.min.min.css',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile.inline-svg-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile.inline-svg-1.4.2.min.css',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile.structure-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile.structure-1.4.2.min.css',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile.theme-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile.theme-1.4.2.min.css'
      //          ,
      //          'http://jqmdesigner.appspot.com/1.4.2/gk/lib/jquery.mobile/1.4.2/flatui/jquery.mobile.flatui.css': 'gk/lib/jquery.mobile/1.4.2/flatui/jquery.mobile.flatui.css',
      //          'http://jqmdesigner.appspot.com/1.4.2/components/jquery.gk/jquery.gk.min.js': 'components/jquery.gk/jquery.gk.min.js',
      //          'http://jqmdesigner.appspot.com/1.4.2/components/gk-loader/gk-loader.js': 'components/gk-loader/gk-loader.js'
    },
    EXCLUDE_GKCOM = ['jqgrid', 'editor', 'niceditor', 'tab-panel'],
    EXCLUDE_GKLIB = ['ace', 'ace-min', 'backbone', 'bootstrap', 'codemirror', 'gk', 'htmlparser', 'jquery', 'jquery.cookie', 'jquery.mobile', 'jquery.nicescroll', 'jquery.ui.tabs', 'jqgrid', 'miniscroll-js', 'nicedit', 'require', 'require-css', 'require-text', 'split-pane'],
    EXCLUDE_COMS = ['ace', 'ace-builds', 'gk-editor', 'jquery', 'jquery.gk', 'jqgrid', 'widget-ext', 'widget-jquerymobile'];

  function run(cb) {
    var self = this,
      url = self.data;
    if (isValidUrl(url)) {
      self.baseUrl = pathlib.dirname(url);
      request(url, function (error, response, body) {
        if (!error && response.statusCode === 200) {
          self.$ = cheerio.load(body);
          self._(parse)(function (path) {
            cb(null, path);
          });
        } else {
          cb({
            error: error,
            response: response
          });
        }
      });
    } else {
      self.$ = cheerio.load(url);
      self._(parse)(function (path) {
        cb(null, path)
      });
    }
  }

  function close() {
    var self = this;
    _.each(self.streamlist, function (ele) {
      ele.end();
    });
  }

  function parse(callback) {
    var self = this,
      $ = self.$,
      dstDir = self.options.dstDir,
      comDir = '',
      gkTags = [],
      downloadProc = function () {
        self._(fetchFile)(function () {
          fs.writeFileSync(dstDir + '/' + self.options.indexFile, self._(replaceHtml)());
          self._(packageAll)(callback);
        });
      };
    _.each(RES_TAG, function (tags, attr) {
      _.each(tags, function (tag) {
        _.each($(tag), function (ele, idx) {
          var url = ele.attribs[attr];
          if (url) {
            if (isValidUrl(url)) {
              var realUrl = self._(replaceWithLocal)(url);
              ele.attribs[attr] = realUrl;
            }
            self._(queueDownload)(ele, attr);
            if (ele.attribs['components']) {
              // keep in dstDir
              //                comDir = pathlib.normalize(dstDir + '/' + urllib.parse(url).pathname + '/../../');
              gkTags = ele.attribs['components'].split(/[\s,]+/);
            }
          }
        });
      });
    });
    if (fs.existsSync(dstDir)) {
      wrench.rmdirSyncRecursive(dstDir);
    }

    self._(copyTmpl)(function () {
      if (gkTags.length) {
        self._(copyCom)(comDir, function () {
          downloadProc();
        });
      } else {
        downloadProc();
      }
    });

  }

  function queueDownload(ele, attr) {
    var self = this;
    if (_.isString(ele)) {
      self.logger.log('debug', '[queued] ' + ele);
      self.tasklist.push({
        url: ele
      });
    } else {
      self.logger.log('debug', '[queued] ' + ele.attribs[attr]);
      self.tasklist.push({
        ele: ele,
        attr: attr
      });
    }
  }

  function copyTmpl(callback) {
    var self = this,
      srcDir = self.options.tmplDir,
      dstDir = self.options.dstDir;
    wrench.mkdirSyncRecursive(dstDir);
    self.logger.log('debug', '[copyTmpl] ' + srcDir + ' -> ' + dstDir);
    wrench.copyDirSyncRecursive(srcDir, dstDir, {
      forceDelete: true
    });
    callback();
  }

  function copyCom(comDir, callback) {
    var self = this,
      srcDir = self.options.baseDir,
      dstDir = comDir || self.options.dstDir,
      //          comJs = {
      //            pattern: 'com/*\.js',
      //            exclude: _.map(EXCLUDE_GKCOM, function (com) {
      //              return 'com/' + com + '.js';
      //            })
      //          },
      //          coms = {
      //            pattern: 'com/*[^\.js]',
      //            exclude: _.map(EXCLUDE_GKCOM, function (com) {
      //              return 'com/' + com;
      //            })
      //          },
      //          libs = {
      //            pattern: 'lib/*',
      //            exclude: _.map(EXCLUDE_GKLIB, function (lib) {
      //              return 'lib/' + lib;
      //            })
      //          },
      coms = {
        pattern: './*',
        exclude: _.map(EXCLUDE_COMS, function (com) {
          return './' + com;
        })
      };
    wrench.mkdirSyncRecursive(dstDir + '/components');
    _.each([coms], function (col) {
      scanDir(srcDir + '/components', col.pattern, col.exclude, function (err, matches) {
        _.each(matches, function (f) {
          var src = srcDir + '/components/' + f,
            dst = dstDir + '/components/' + f,
            isFile = fs.lstatSync(src).isFile();
          self.logger.log('debug', '[copyCom] ' + src + ' -> ' + dst);

          wrench.mkdirSyncRecursive(pathlib.dirname(dst));
          if (isFile) {
            fs.writeFileSync(dst, fs.readFileSync(src));
          } else {
            wrench.copyDirSyncRecursive(src, dst, {
              forceDelete: true
            });
          }
        });
      });
    });
    callback();
  }

  function fetchFile(callback) {
    var self = this,
      tasklist = self.tasklist,
      srcDir = self.options.baseDir,
      dstDir = self.options.dstDir,
      logger = self.logger,
      count = 0,
      checkFinish = function () {
        count++;
        if (count >= tasklist.length) {
          logger.log('debug', '[fetchFile] finish...');
          callback();
        }
      };
    _.each(tasklist, function (t, idx) {
      var url = t.ele.attribs[t.attr];
      if (isValidUrl(url)) {
        self._(download)(idx, url, dstDir, function (err, path) {
          err ? logger.log('error', '[download] failed: ' + url + ', err: ' + err) : (t.path = path);
          checkFinish();
        });
      } else if (url.indexOf('data') !== 0) {
        self._(copyLocal)(idx, srcDir + '/' + url, dstDir + '/' + url, dstDir, function (err, path) {
          err ? logger.log('error', '[copy] failed: ' + url + ', err: ' + err) : (t.path = pathlib.relative(dstDir, path));
          checkFinish();
        })
      } else {
        checkFinish();
      }
    });
  }

  function replaceHtml() {
    var self = this,
      tasklist = self.tasklist;
    _.each(tasklist, function (t) {
      if (t.ele && t.attr && t.path) {
        t.ele.attribs[t.attr] = t.path;
      }
    });
    return self.$.html();
  }

  function packageAll(callback) {
    var self = this;
    callback(self.options.dstDir);
  }

  return {
    constructor: Grabber,
    run: run,
    close: close,
    _: function (privateMethod) {
      var self = this;
      return function () {
        return privateMethod.apply(self, arguments);
      };
    }
  };

  function replaceWithLocal(url) {
    if (this.originUrl.test(url)) {
      return url.replace(this.originUrl, '');
    }
    return (USE_LOCAL_RES[url]) ? (USE_LOCAL_RES[url]) : url;
  }

  function copyLocal(sn, srcPath, dstPath, dstDir, callback) {
    try {
      wrench.mkdirSyncRecursive(pathlib.dirname(dstPath));
    } catch (e) {
      dstPath = dstDir + '/' + id + pathlib.extname(dstPath);
    }
    try {
      if (fs.existsSync(dstPath)) {
        this.logger.log('debug', '[copyLocal] skip ' + dstPath);
      } else {
        fs.writeFileSync(dstPath, fs.readFileSync(srcPath));
        this.logger.log('debug', '[copyLocal] ' + srcPath + ' -> ' + dstPath);
      }
      callback(null, dstPath);
    } catch (e) {
      callback(e);
    }
  }

  function download(sn, url, dstDir, callback) {
    var path = urllib.parse(url).pathname,
      id = hashids.encrypt(sn),
      logger = this.logger,
      streamlist = this.streamlist,
      filepath;
    request.head(url, function (error, response, body) {
      if (!error) {
        if (fs.existsSync(dstDir + '/' + path)) {
          filepath = dstDir + '/' + pathlib.dirname(path) + '/' + pathlib.basename(path, pathlib.extname(path)) + id + pathlib.extname(path);
        } else {
          filepath = dstDir + '/' + path;
        }
        filepath = pathlib.normalize(filepath);
        try {
          wrench.mkdirSyncRecursive(pathlib.dirname(filepath));
        } catch (e) {
          filepath = dstDir + '/' + id + pathlib.extname(path);
        }
        var ws = fs.createWriteStream(filepath);
        ws.on('finish', function () {
          streamlist.splice(streamlist.indexOf(ws), 1);
        });
        streamlist.push(ws);
        request(url).pipe(ws);
        logger.log('debug', '[download] ' + url + ' -> ' + filepath);
        callback(error, pathlib.relative(dstDir, filepath));
      } else {
        callback(error);
      }
    });
  }

  function scanDir(baseDir, pattern, excludes, callback) {
    glob(pattern, {
      cwd: baseDir
    }, function (err, matches) {
      callback(err, _.difference(matches, excludes));
    });
  }

  function isValidUrl(str) {
    var url = urllib.parse(str);
    return url.protocol && url.protocol.indexOf('http') === 0;
  }

}());

exports.Grabber = Grabber;
<commit_msg>add socket.io<commit_after>/*
 * grabber
 * https://github.com/ezoapp/grabber.git
 *
 * Copyright (c) 2014
 * Licensed under the MIT license.
 */

'use strict';

var _ = require('underscore'),
  request = require('request'),
  fs = require('fs'),
  cheerio = require('cheerio'),
  urllib = require('url'),
  pathlib = require('path'),
  glob = require('glob'),
  wrench = require('wrench'),
  hashids = new(require('hashids'))(new Date().getTime() + 'T');

function Grabber(data, options, logger, originUrl, socket) {
  this.$ = null;
  this.data = data;
  this.baseUrl = '';
  this.tasklist = [];
  this.options = _.extend({
    dstDir: 'www',
    baseDir: '',
    tmplDir: '',
    indexFile: 'index.html'
  }, options || {});
  this.logger = (logger) ? logger : console;
  this.originUrl = new RegExp(originUrl, 'i');
  this.streamlist = [];
  this.socket = socket;
}

Grabber.prototype = (function () {

  var RES_TAG = {
    'href': ['link'], //,'a'],
    'src': ['img', 'script'], //, 'iframe'],
    'data-background-image': ['a']
  },
    USE_LOCAL_RES = {
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile-1.4.2.min.js': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile-1.4.2.min.js',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile-1.4.2.min.css',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile.external-png-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile.external-png-1.4.2.min.css',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile.icons-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile.icons-1.4.2.min.min.css',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile.inline-png-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile.inline-png-1.4.2.min.min.css',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile.inline-svg-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile.inline-svg-1.4.2.min.css',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile.structure-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile.structure-1.4.2.min.css',
      'http://code.jquery.com/mobile/1.4.2/jquery.mobile.theme-1.4.2.min.css': 'gk/lib/jquery.mobile/1.4.2/jquery.mobile.theme-1.4.2.min.css'
      //          ,
      //          'http://jqmdesigner.appspot.com/1.4.2/gk/lib/jquery.mobile/1.4.2/flatui/jquery.mobile.flatui.css': 'gk/lib/jquery.mobile/1.4.2/flatui/jquery.mobile.flatui.css',
      //          'http://jqmdesigner.appspot.com/1.4.2/components/jquery.gk/jquery.gk.min.js': 'components/jquery.gk/jquery.gk.min.js',
      //          'http://jqmdesigner.appspot.com/1.4.2/components/gk-loader/gk-loader.js': 'components/gk-loader/gk-loader.js'
    },
    EXCLUDE_GKCOM = ['jqgrid', 'editor', 'niceditor', 'tab-panel'],
    EXCLUDE_GKLIB = ['ace', 'ace-min', 'backbone', 'bootstrap', 'codemirror', 'gk', 'htmlparser', 'jquery', 'jquery.cookie', 'jquery.mobile', 'jquery.nicescroll', 'jquery.ui.tabs', 'jqgrid', 'miniscroll-js', 'nicedit', 'require', 'require-css', 'require-text', 'split-pane'],
    EXCLUDE_COMS = ['ace', 'ace-builds', 'gk-editor', 'jquery', 'jquery.gk', 'jqgrid', 'widget-ext', 'widget-jquerymobile'];

  function run(cb) {
    var self = this,
      url = self.data;
    if (isValidUrl(url)) {
      self.baseUrl = pathlib.dirname(url);
      request(url, function (error, response, body) {
        if (!error && response.statusCode === 200) {
          self.$ = cheerio.load(body);
          self._(parse)(function (path) {
            cb(null, path);
          });
        } else {
          cb({
            error: error,
            response: response
          });
        }
      });
    } else {
      self.$ = cheerio.load(url);
      self._(parse)(function (path) {
        cb(null, path)
      });
    }
  }

  function close() {
    var self = this;
    _.each(self.streamlist, function (ele) {
      ele.end();
    });
  }

  function parse(callback) {
    var self = this,
      $ = self.$,
      dstDir = self.options.dstDir,
      comDir = '',
      gkTags = [],
      downloadProc = function () {
        self.socket.emit('progress', 60);
        self._(fetchFile)(function () {
          fs.writeFileSync(dstDir + '/' + self.options.indexFile, self._(replaceHtml)());
          self._(packageAll)(callback);
        });
      };
    _.each(RES_TAG, function (tags, attr) {
      _.each(tags, function (tag) {
        _.each($(tag), function (ele, idx) {
          var url = ele.attribs[attr];
          if (url) {
            if (isValidUrl(url)) {
              var realUrl = self._(replaceWithLocal)(url);
              ele.attribs[attr] = realUrl;
            }
            self._(queueDownload)(ele, attr);
            if (ele.attribs['components']) {
              // keep in dstDir
              //                comDir = pathlib.normalize(dstDir + '/' + urllib.parse(url).pathname + '/../../');
              gkTags = ele.attribs['components'].split(/[\s,]+/);
            }
          }
        });
      });
    });
    if (fs.existsSync(dstDir)) {
      wrench.rmdirSyncRecursive(dstDir);
    }

    self.socket.emit('progress', 20);

    self._(copyTmpl)(function () {
      self.socket.emit('progress', 40);
      if (gkTags.length) {
        self._(copyCom)(comDir, function () {
          downloadProc();
        });
      } else {
        downloadProc();
      }
    });

  }

  function queueDownload(ele, attr) {
    var self = this;
    if (_.isString(ele)) {
      self.logger.log('debug', '[queued] ' + ele);
      self.tasklist.push({
        url: ele
      });
    } else {
      self.logger.log('debug', '[queued] ' + ele.attribs[attr]);
      self.tasklist.push({
        ele: ele,
        attr: attr
      });
    }
  }

  function copyTmpl(callback) {
    var self = this,
      srcDir = self.options.tmplDir,
      dstDir = self.options.dstDir;
    wrench.mkdirSyncRecursive(dstDir);
    self.logger.log('debug', '[copyTmpl] ' + srcDir + ' -> ' + dstDir);
    wrench.copyDirSyncRecursive(srcDir, dstDir, {
      forceDelete: true
    });
    callback();
  }

  function copyCom(comDir, callback) {
    var self = this,
      srcDir = self.options.baseDir,
      dstDir = comDir || self.options.dstDir,
      //          comJs = {
      //            pattern: 'com/*\.js',
      //            exclude: _.map(EXCLUDE_GKCOM, function (com) {
      //              return 'com/' + com + '.js';
      //            })
      //          },
      //          coms = {
      //            pattern: 'com/*[^\.js]',
      //            exclude: _.map(EXCLUDE_GKCOM, function (com) {
      //              return 'com/' + com;
      //            })
      //          },
      //          libs = {
      //            pattern: 'lib/*',
      //            exclude: _.map(EXCLUDE_GKLIB, function (lib) {
      //              return 'lib/' + lib;
      //            })
      //          },
      coms = {
        pattern: './*',
        exclude: _.map(EXCLUDE_COMS, function (com) {
          return './' + com;
        })
      };
    wrench.mkdirSyncRecursive(dstDir + '/components');
    _.each([coms], function (col) {
      scanDir(srcDir + '/components', col.pattern, col.exclude, function (err, matches) {
        _.each(matches, function (f) {
          var src = srcDir + '/components/' + f,
            dst = dstDir + '/components/' + f,
            isFile = fs.lstatSync(src).isFile();
          self.logger.log('debug', '[copyCom] ' + src + ' -> ' + dst);

          wrench.mkdirSyncRecursive(pathlib.dirname(dst));
          if (isFile) {
            fs.writeFileSync(dst, fs.readFileSync(src));
          } else {
            wrench.copyDirSyncRecursive(src, dst, {
              forceDelete: true
            });
          }
        });
      });
    });
    callback();
  }

  function fetchFile(callback) {
    var self = this,
      tasklist = self.tasklist,
      srcDir = self.options.baseDir,
      dstDir = self.options.dstDir,
      logger = self.logger,
      count = 0,
      checkFinish = function () {
        count++;
        if (count >= tasklist.length) {
          logger.log('debug', '[fetchFile] finish...');
          callback();
        }
      };
    _.each(tasklist, function (t, idx) {
      var url = t.ele.attribs[t.attr];
      if (isValidUrl(url)) {
        self._(download)(idx, url, dstDir, function (err, path) {
          err ? logger.log('error', '[download] failed: ' + url + ', err: ' + err) : (t.path = path);
          checkFinish();
        });
      } else if (url.indexOf('data') !== 0) {
        self._(copyLocal)(idx, srcDir + '/' + url, dstDir + '/' + url, dstDir, function (err, path) {
          err ? logger.log('error', '[copy] failed: ' + url + ', err: ' + err) : (t.path = pathlib.relative(dstDir, path));
          checkFinish();
        })
      } else {
        checkFinish();
      }
    });
  }

  function replaceHtml() {
    var self = this,
      tasklist = self.tasklist;
    _.each(tasklist, function (t) {
      if (t.ele && t.attr && t.path) {
        t.ele.attribs[t.attr] = t.path;
      }
    });
    return self.$.html();
  }

  function packageAll(callback) {
    var self = this;
    callback(self.options.dstDir);
  }

  return {
    constructor: Grabber,
    run: run,
    close: close,
    _: function (privateMethod) {
      var self = this;
      return function () {
        return privateMethod.apply(self, arguments);
      };
    }
  };

  function replaceWithLocal(url) {
    if (this.originUrl.test(url)) {
      return url.replace(this.originUrl, '');
    }
    return (USE_LOCAL_RES[url]) ? (USE_LOCAL_RES[url]) : url;
  }

  function copyLocal(sn, srcPath, dstPath, dstDir, callback) {
    try {
      wrench.mkdirSyncRecursive(pathlib.dirname(dstPath));
    } catch (e) {
      dstPath = dstDir + '/' + id + pathlib.extname(dstPath);
    }
    try {
      if (fs.existsSync(dstPath)) {
        this.logger.log('debug', '[copyLocal] skip ' + dstPath);
      } else {
        fs.writeFileSync(dstPath, fs.readFileSync(srcPath));
        this.logger.log('debug', '[copyLocal] ' + srcPath + ' -> ' + dstPath);
      }
      callback(null, dstPath);
    } catch (e) {
      callback(e);
    }
  }

  function download(sn, url, dstDir, callback) {
    var path = urllib.parse(url).pathname,
      id = hashids.encrypt(sn),
      logger = this.logger,
      streamlist = this.streamlist,
      filepath;
    request.head(url, function (error, response, body) {
      if (!error) {
        if (fs.existsSync(dstDir + '/' + path)) {
          filepath = dstDir + '/' + pathlib.dirname(path) + '/' + pathlib.basename(path, pathlib.extname(path)) + id + pathlib.extname(path);
        } else {
          filepath = dstDir + '/' + path;
        }
        filepath = pathlib.normalize(filepath);
        try {
          wrench.mkdirSyncRecursive(pathlib.dirname(filepath));
        } catch (e) {
          filepath = dstDir + '/' + id + pathlib.extname(path);
        }
        var ws = fs.createWriteStream(filepath);
        ws.on('finish', function () {
          streamlist.splice(streamlist.indexOf(ws), 1);
        });
        streamlist.push(ws);
        request(url).pipe(ws);
        logger.log('debug', '[download] ' + url + ' -> ' + filepath);
        callback(error, pathlib.relative(dstDir, filepath));
      } else {
        callback(error);
      }
    });
  }

  function scanDir(baseDir, pattern, excludes, callback) {
    glob(pattern, {
      cwd: baseDir
    }, function (err, matches) {
      callback(err, _.difference(matches, excludes));
    });
  }

  function isValidUrl(str) {
    var url = urllib.parse(str);
    return url.protocol && url.protocol.indexOf('http') === 0;
  }

}());

exports.Grabber = Grabber;
<|endoftext|>


<commit_before>/*******************************************************************************
 * Author: Hovig Ohannessian - Initial implementation
 *******************************************************************************/
(function(window) {
	'use strict'

	var Gyroscope = {};
	var x = 0, y = 0;
	var alpha = 0, beta = 0, gamma = 0;
	var upperLeftCoordsX = 0, upperLeftCoordsY = 0;
	var lowerRightCoordsX = 0, l owerRightCoordsY = 0;
	var coefX = 0, coefY = 0;
	var lpfilterX = new LowPassFilter();
	var lpfilterY = new LowPassFilter();
	var gyroMath = new GyroMath();
	var socket = io();

	function defineGyro() {
		// Read Gyroscope sensor data
		Gyroscope.read = function() {
			var coords = {alpha: 0, beta: 0, gamma: 0};
			if(window.DeviceOrientationEvent) {
				window.addEventListener("deviceorientation", function () {
				coords = { alpha: event.alpha, beta: event.beta, gamma: event.gamma }
				socket.emit('coords', JSON.stringify(coords));
				gyroObjects(event.alpha, event.beta, event.gamma)
				}, true);
			}
		};

		// Display converted Gyroscope coordinates
		Gyroscope.execute = function(imgName) {
			setInterval(function() {
				var img = document.getElementById(imgName);
				var delay = 10;
				x = lpfilterX.process(Math.abs(lowerRightValues.lowerRightCoordsX - alpha)*coefValues.coefX);
				y = lpfilterY.process(Math.abs(lowerRightValues.lowerRightCoordsY - beta)*coefValues.coefY);

				if(coefValues.coefX > 0){
					img.style.left = x + "px";
					img.style.top = y + "px";
					convertedGyro(x, y);
				}
			}, delay);
		};

		// Main socket.on needed to start
		Gyroscope.socketOn = function(name, callback) {
			socket.on('Coords', function(msg){
				msg = JSON.parse(msg);
				alpha = gyroMath.check(msg.alpha);
				beta = msg.beta;
			});
			socket.on('CaliberateUpperLeft', function(msg){
				msg = JSON.parse(msg);
				upperLeftCoordsX = msg.alpha;
				upperLeftCoordsY = msg.beta;
				upperLeftCoords(upperLeftCoordsX, upperLeftCoordsY)
			});
			socket.on('CaliberateLowerRight', function(msg){
				msg = JSON.parse(msg);
				lowerRightCoordsX = msg.alpha;
				lowerRightCoordsY = msg.beta;
				lowerRightCoords(lowerRightCoordsX, lowerRightCoordsY);
			});
		};

		Gyroscope.ajax = function(url, method) {
			$.ajax({
				url: url,
				method: method
			});
		};

		Gyroscope.innerHTML = function(name, value) {
			document.getElementById(name).innerHTML = value;
		};

		Gyroscope.source = function(name, value) {
			document.getElementById(name).src = value;
		};
		return Gyroscope;
	}

	if(typeof(Gyroscope) === 'undefined') {
		window.Gyroscope = defineGyro();
	}
})(window);

// Gyroscope values sent from mobile phone
function gyroObjects(alpha, beta, gamma) {
	var gyroValues = {};
	gyroValues['alpha'] = alpha;
	gyroValues['beta'] = beta;
	gyroValues['gamma'] = gamma;
	return gyroValues;
}

// Gyroscope coordinates on browser
function convertedGyro(x, y) {
	var gyroBrowser = {};
	gyroBrowser['x'] = x;
	gyroBrowser['y'] = y;
	return gyroBrowser;
}

// Coeffiecient calculated and stored
function coefficientValues(coefX, coefY) {
	var coefValues = {};
	// "reset" the lowerRightCoordsX if > than upperLeftCoordsX on the trigonometric circle(substracting 360°)
	if(lowerRightValues.lowerRightCoordsX > upperLeftValues.upperLeftCoordsX) gyroMath.revert(true);
	lowerRightValues.lowerRightCoordsX = gyroMath.check(lowerRightValues.lowerRightCoordsX);
	coefX = document.documentElement.clientWidth/Math.abs(lowerRightValues.lowerRightCoordsX - upperLeftValues.upperLeftCoordsX);
	coefY = document.documentElement.clientHeight/Math.abs(lowerRightValues.lowerRightCoordsY - upperLeftValues.upperLeftCoordsY);
	coefValues['coefX'] = coefX;
	coefValues['coefY'] = coefY;
	return coefValues;
}

// Upper left coords values for calibration
function upperLeftCoords(upperLeftCoordsX, upperLeftCoordsY) {
	var upperLeftValues = {};
	upperLeftValues['upperLeftCoordsX'] = upperLeftCoordsX;
	upperLeftValues['upperLeftCoordsY'] = upperLeftCoordsY;
	return upperLeftValues;
}

// Lower right coords values for calibration
function lowerRightCoords(lowerRightCoordsX, lowerRightCoordsY) {
	var lowerRightValues = {};
	lowerRightValues['lowerRightCoordsX'] = lowerRightCoordsX;
	lowerRightValues['lowerRightCoordsY'] = lowerRightCoordsY;
	return lowerRightValues;
}

// Low pass filter to reduce the noise in gyro for better positioning
function LowPassFilter() {
    this.a = 0.8;
    this.b = 1.2 - this.a;
    this.g = 0;
}

LowPassFilter.prototype.process = function (input) {
    this.g = (input * this.b);
    return this.g;
}

// Re-calculate alpha for degree-to-pixel conversion
function GyroMath(revert) {
    this.x = false;
}

GyroMath.prototype.check = function(value) {
	if(this.revert && value > upperLeftCoordsX) {
		//trigonometric circle position + upper left x coordinate
		return Math.abs(360 - value) + upperLeftCoordsX;
	}
	return value;
}

GyroMath.prototype.revert = function(value) {
	this.x = value;
}

// Mouse event coordinates on browser
function showMouseEventCoords(event) {
	var mouseX = event.clientX;
	var mouseY = event.clientY;
	document.getElementById("mouseX").innerHTML = mouseX;
	document.getElementById("mouseY").innerHTML = mouseY;
}<commit_msg>Update gyrolib.js<commit_after>/*******************************************************************************
 * Author: Hovig Ohannessian - Initial implementation
 *******************************************************************************/
(function(window) {
	'use strict'

	var Gyroscope = {};
	var x = 0, y = 0;
	var alpha = 0, beta = 0, gamma = 0;
	var upperLeftCoordsX = 0, upperLeftCoordsY = 0;
	var lowerRightCoordsX = 0, l owerRightCoordsY = 0;
	var coefX = 0, coefY = 0;
	var lpfilterX = new LowPassFilter();
	var lpfilterY = new LowPassFilter();
	var gyroMath = new GyroMath();
	var socket = io();

	function defineGyro() {
		// Read Gyroscope sensor data
		Gyroscope.read = function() {
			var coords = {alpha: 0, beta: 0, gamma: 0};
			if(window.DeviceOrientationEvent) {
				window.addEventListener("deviceorientation", function () {
				coords = { alpha: event.alpha, beta: event.beta, gamma: event.gamma }
				socket.emit('coords', JSON.stringify(coords));
				gyroObjects(event.alpha, event.beta, event.gamma)
				}, true);
			}
		};

		// Display converted Gyroscope coordinates
		Gyroscope.execute = function(imgName) {
			setInterval(function() {
				var img = document.getElementById(imgName);
				var delay = 10;
				x = lpfilterX.process(Math.abs(lowerRightValues.lowerRightCoordsX - alpha)*coefValues.coefX);
				y = lpfilterY.process(Math.abs(lowerRightValues.lowerRightCoordsY - beta)*coefValues.coefY);

				if(coefValues.coefX > 0){
					img.style.left = x + "px";
					img.style.top = y + "px";
					convertedGyro(x, y);
				}
			}, delay);
		};

		// Main socket.on needed to start
		Gyroscope.socketOn = function() {
			socket.on('Coords', function(msg){
				msg = JSON.parse(msg);
				alpha = gyroMath.check(msg.alpha);
				beta = msg.beta;
			});
			socket.on('CaliberateUpperLeft', function(msg){
				msg = JSON.parse(msg);
				upperLeftCoordsX = msg.alpha;
				upperLeftCoordsY = msg.beta;
				upperLeftCoords(upperLeftCoordsX, upperLeftCoordsY)
			});
			socket.on('CaliberateLowerRight', function(msg){
				msg = JSON.parse(msg);
				lowerRightCoordsX = msg.alpha;
				lowerRightCoordsY = msg.beta;
				lowerRightCoords(lowerRightCoordsX, lowerRightCoordsY);
			});
		};

		Gyroscope.ajax = function(url, method) {
			$.ajax({
				url: url,
				method: method
			});
		};

		Gyroscope.innerHTML = function(name, value) {
			document.getElementById(name).innerHTML = value;
		};

		Gyroscope.source = function(name, value) {
			document.getElementById(name).src = value;
		};
		return Gyroscope;
	}

	if(typeof(Gyroscope) === 'undefined') {
		window.Gyroscope = defineGyro();
	}
})(window);

// Gyroscope values sent from mobile phone
function gyroObjects(alpha, beta, gamma) {
	var gyroValues = {};
	gyroValues['alpha'] = alpha;
	gyroValues['beta'] = beta;
	gyroValues['gamma'] = gamma;
	return gyroValues;
}

// Gyroscope coordinates on browser
function convertedGyro(x, y) {
	var gyroBrowser = {};
	gyroBrowser['x'] = x;
	gyroBrowser['y'] = y;
	return gyroBrowser;
}

// Coeffiecient calculated and stored
function coefficientValues(coefX, coefY) {
	var coefValues = {};
	// "reset" the lowerRightCoordsX if > than upperLeftCoordsX on the trigonometric circle(substracting 360°)
	if(lowerRightValues.lowerRightCoordsX > upperLeftValues.upperLeftCoordsX) gyroMath.revert(true);
	lowerRightValues.lowerRightCoordsX = gyroMath.check(lowerRightValues.lowerRightCoordsX);
	coefX = document.documentElement.clientWidth/Math.abs(lowerRightValues.lowerRightCoordsX - upperLeftValues.upperLeftCoordsX);
	coefY = document.documentElement.clientHeight/Math.abs(lowerRightValues.lowerRightCoordsY - upperLeftValues.upperLeftCoordsY);
	coefValues['coefX'] = coefX;
	coefValues['coefY'] = coefY;
	return coefValues;
}

// Upper left coords values for calibration
function upperLeftCoords(upperLeftCoordsX, upperLeftCoordsY) {
	var upperLeftValues = {};
	upperLeftValues['upperLeftCoordsX'] = upperLeftCoordsX;
	upperLeftValues['upperLeftCoordsY'] = upperLeftCoordsY;
	return upperLeftValues;
}

// Lower right coords values for calibration
function lowerRightCoords(lowerRightCoordsX, lowerRightCoordsY) {
	var lowerRightValues = {};
	lowerRightValues['lowerRightCoordsX'] = lowerRightCoordsX;
	lowerRightValues['lowerRightCoordsY'] = lowerRightCoordsY;
	return lowerRightValues;
}

// Low pass filter to reduce the noise in gyro for better positioning
function LowPassFilter() {
    this.a = 0.8;
    this.b = 1.2 - this.a;
    this.g = 0;
}

LowPassFilter.prototype.process = function (input) {
    this.g = (input * this.b);
    return this.g;
}

// Re-calculate alpha for degree-to-pixel conversion
function GyroMath(revert) {
    this.x = false;
}

GyroMath.prototype.check = function(value) {
	if(this.revert && value > upperLeftCoordsX) {
		//trigonometric circle position + upper left x coordinate
		return Math.abs(360 - value) + upperLeftCoordsX;
	}
	return value;
}

GyroMath.prototype.revert = function(value) {
	this.x = value;
}

// Mouse event coordinates on browser
function showMouseEventCoords(event) {
	var mouseX = event.clientX;
	var mouseY = event.clientY;
	document.getElementById("mouseX").innerHTML = mouseX;
	document.getElementById("mouseY").innerHTML = mouseY;
}
<|endoftext|>


<commit_before>
function handle( req, res, done )
{
    if( !req.flick ) {
        return done();
    }

    var payload = req.flick.payload,
        repository = payload.repository.owner.name + '/' + payload.repository.name,
        index = 0,
        self = this;

    function next( err )
    {
        var handler = self.stack[ index++ ];

        if( !handler || err ) {
            return done( err );
        }

        if( !~[ '*', repository ].indexOf( handler.repository ) ) {
            return next();
        }

        handler.handle( req, res, next );
    }

    next();
}

exports.handle = handle;

function use( repository, fn )
{
    if( !fn ) {
        fn = repository;
        repository = '*';
    }

    this.stack.push( {
        repository: repository,
        handle: fn
    } );
}

exports.use = use;
<commit_msg>Call done() only if it has been supplied.<commit_after>
function handle( req, res, done )
{
    if( !req.flick ) {
        return done ? done() : undefined;
    }

    var payload = req.flick.payload,
        repository = payload.repository.owner.name + '/' + payload.repository.name,
        index = 0,
        self = this;

    function next( err )
    {
        var handler = self.stack[ index++ ];

        if( !handler || err ) {
            return done( err );
        }

        if( !~[ '*', repository ].indexOf( handler.repository ) ) {
            return next();
        }

        handler.handle( req, res, next );
    }

    next();
}

exports.handle = handle;

function use( repository, fn )
{
    if( !fn ) {
        fn = repository;
        repository = '*';
    }

    this.stack.push( {
        repository: repository,
        handle: fn
    } );
}

exports.use = use;
<|endoftext|>


<commit_before>const Path = require('node:path');
const { readFile } = require('node:fs/promises');
const { promisify } = require('node:util');
const { request } = require('undici');

const debug = require('debug')('express-dom');
const { BufferListStream } = require('bl');

exports.view = async function view(mw, settings, req, res) {
	if (settings.input != null) return;
	const view = settings.view || req.path;

	if (Buffer.isBuffer(view)) {
		return update(view);
	}
	if (req.protocol == "about:" && req.path == "blank") {
		return update('<html/>');
	} else if (
		(typeof view == "string" && /^https?:\/\//.test(view))
		||
		(view.protocol && view.hostname)
	) {
		const { body, statusCode } = await request(view, {
			headers: {
				// TODO UserAgent here ? do we really care ?
				Accept: 'text/html'
			}
		});
		res.status(statusCode);
		return update(body);
	}
	if (typeof view.pipe == "function") {
		return bufferPipe(view).then(update);
	}
	if (typeof view != "string") {
		throw new Error("unknown input type");
	}
	if (view.startsWith('<')) {
		debug('View is html string');
		return update(view);
	}
	let views = settings.views ?? req.app?.get('views') ?? [];
	if (!Array.isArray(views)) views = [views];
	if (views.length == 0) {
		throw new Error("Please set settings.views or app views");
	}

	return tryRoot(views.slice(), view).then(update);

	function update(input) {
		settings.input = input;
		return settings;
	}
};

function tryRoot(views, view) {
	if (views.length == 0) throw new Error("view not found " + view);
	const root = views.shift();

	let path = view.indexOf(root) === 0 ? view : Path.join(root, view);
	if (path.indexOf(root) !== 0) {
		throw new Error("Path outside views dir\n" + root + '\n' + view);
	}
	if (Path.extname(path) != ".html") path += ".html";
	debug("read view from", path);
	return readFile(path).catch(() => {
		return tryRoot(views, view);
	});
}

function bufferPipe(ps) {
	return promisify(cb => {
		ps.pipe(BufferListStream(cb));
	})();
}

exports.prioritize = function(mw, settings, request) {
	if (!settings.priority) settings.priority = 0;
	if (request.xhr) settings.priority++;
};

exports.develop = function(mw, settings, request) {
	if (request.query.develop !== undefined) {
		settings.location.searchParams.delete('develop');
		settings.load.disable = true;
	}
};
<commit_msg>Style<commit_after>const Path = require('node:path');
const { readFile } = require('node:fs/promises');
const { promisify } = require('node:util');
const { request } = require('undici');

const debug = require('debug')('express-dom');
const { BufferListStream } = require('bl');

exports.view = async function view(mw, settings, req, res) {
	if (settings.input != null) return;
	const view = settings.view || req.path;

	if (Buffer.isBuffer(view)) {
		return update(view);
	}
	if (req.protocol == "about:" && req.path == "blank") {
		return update('<html/>');
	} else if (
		(typeof view == "string" && /^https?:\/\//.test(view))
		||
		(view.protocol && view.hostname)
	) {
		const { body, statusCode } = await request(view, {
			headers: {
				// TODO UserAgent here ? do we really care ?
				Accept: 'text/html'
			}
		});
		res.status(statusCode);
		return update(body);
	}
	if (typeof view.pipe == "function") {
		return bufferPipe(view).then(update);
	}
	if (typeof view != "string") {
		throw new Error("unknown input type");
	}
	if (view.startsWith('<')) {
		debug('View is html string');
		return update(view);
	}
	let views = settings.views ?? req.app?.get('views') ?? [];
	if (!Array.isArray(views)) views = [views];
	if (views.length == 0) {
		throw new Error("Please set settings.views or app views");
	}

	return tryRoot(views.slice(), view).then(update);

	function update(input) {
		settings.input = input;
		return settings;
	}
};

function tryRoot(views, view) {
	if (views.length == 0) throw new Error("view not found " + view);
	const root = views.shift();

	let path = view.indexOf(root) === 0 ? view : Path.join(root, view);
	if (path.indexOf(root) !== 0) {
		throw new Error("Path outside views dir\n" + root + '\n' + view);
	}
	if (Path.extname(path) != ".html") path += ".html";
	debug("read view from", path);
	return readFile(path).catch(() => {
		return tryRoot(views, view);
	});
}

function bufferPipe(ps) {
	return promisify(cb => {
		ps.pipe(BufferListStream(cb));
	})();
}

exports.prioritize = function(mw, settings, { xhr }) {
	if (!settings.priority) settings.priority = 0;
	if (xhr) settings.priority++;
};

exports.develop = function (mw, settings, { query }) {
	if (query.develop !== undefined) {
		settings.location.searchParams.delete('develop');
		settings.load.disable = true;
	}
};
<|endoftext|>


<commit_before>var Path = require('path');
var pify = require('pify');
var readFile = pify(require('fs').readFile);
var debug = require('debug')('express-dom');
var BufferList = require('bl');
var http = require('http');
var https = require('https');
var URL = require('url');
var dom = require('..');
var PassThrough = require('stream').PassThrough;

exports.view = function view(mw, settings, request, response) {
	if (settings.input != null) return;
	var view = settings.view || request.path;

	if (Buffer.isBuffer(view)) {
		return update(view);
	}
	if (
		(typeof view == "string" && /^https?:\/\//.test(view))
		||
		(view.protocol && view.hostname)
	) {
		return fetchUrl(view).then(function(ps) {
			return bufferPipe(ps);
		}).then(update);
	}
	if (typeof view.pipe == "function") {
		return bufferPipe(view).then(update);
	}
	if (typeof view != "string") {
		return Promise.reject(new Error("unknown input type"));
	}
	if (view.startsWith('<')) {
		debug('View is html string');
		return update(view);
	}
	var views = settings.views || request.app && request.app.get('views') || [];
	if (!Array.isArray(views)) views = [views];
	if (views.length == 0) {
		return Promise.reject(new Error("Please set settings.views or app views"));
	}

	return tryRoot(views.slice(), view).then(update);

	function update(input) {
		settings.input = input;
	}
};

function tryRoot(views, view) {
	if (views.length == 0) return Promise.reject(new Error("view not found " + view));
	var root = views.shift();

	var path = Path.join(root, view);
	if (path.indexOf(root) !== 0) return Promise.reject(
		new Error("Path outside views dir\n" + root + '\n' + view)
	);
	if (Path.extname(path) != ".html") path += ".html";
	debug("read view from", path);
	return readFile(path).catch(function(err) {
		return tryRoot(views, view);
	});
}

function bufferPipe(ps) {
	return pify(function(cb) {
		ps.pipe(BufferList(cb));
	})();
}

function fetchUrl(url) {
	var obj = typeof url == "string" ? URL.parse(url) : Object.assign({}, url);
	obj.headers = Object.assign({
		"User-Agent": dom.navigator.userAgent
	}, obj.headers);
	obj.href = URL.format(obj);
	obj.path = URL.parse(obj.href).path;

	var ps = new PassThrough();
	debug("requesting", obj.href);
	return new Promise(function(resolve, reject) {
		var req = (obj.protocol == "https:" ? https : http).request(obj, function(res) {
			if (res.statusCode >= 400) return reject(new Error(`Error loading subrequest`));
			if (res.headers.location) {
				var location = URL.resolve(obj.href, res.headers.location);
				var redirObj = URL.parse(location);
				try {
					req.abort();
				} catch(ex) {
				}
				redirObj.redirects = (obj.redirects || 0) + 1;
				if (redirObj.redirects >= 5) {
					debug("Too many redirects");
					return reject(new Error(`Too many redirects ${obj.href}`));
				} else {
					debug("Redirecting to", obj.href);
					fetchUrl(redirObj).then(resolve).catch(reject);
				}
			} else {
				debug("pipe url response", res.statusCode);
				res.setEncoding('utf-8');
				res.pipe(ps);
				resolve(ps);
			}
		}).end();
	});
}

exports.prioritize = function(mw, settings, request, response) {
	if (!settings.priority) settings.priority = 0;
	if (request.xhr) settings.priority++;
};

exports.develop = function(mw, settings, request, response) {
	settings.load.disable = request.query.develop !== undefined;
};
<commit_msg>Warn about missing userAgent<commit_after>var Path = require('path');
var pify = require('pify');
var readFile = pify(require('fs').readFile);
var debug = require('debug')('express-dom');
var BufferList = require('bl');
var http = require('http');
var https = require('https');
var URL = require('url');
var dom = require('..');
var PassThrough = require('stream').PassThrough;

exports.view = function view(mw, settings, request, response) {
	if (settings.input != null) return;
	var view = settings.view || request.path;

	if (Buffer.isBuffer(view)) {
		return update(view);
	}
	if (
		(typeof view == "string" && /^https?:\/\//.test(view))
		||
		(view.protocol && view.hostname)
	) {
		return fetchUrl(view).then(function(ps) {
			return bufferPipe(ps);
		}).then(update);
	}
	if (typeof view.pipe == "function") {
		return bufferPipe(view).then(update);
	}
	if (typeof view != "string") {
		return Promise.reject(new Error("unknown input type"));
	}
	if (view.startsWith('<')) {
		debug('View is html string');
		return update(view);
	}
	var views = settings.views || request.app && request.app.get('views') || [];
	if (!Array.isArray(views)) views = [views];
	if (views.length == 0) {
		return Promise.reject(new Error("Please set settings.views or app views"));
	}

	return tryRoot(views.slice(), view).then(update);

	function update(input) {
		settings.input = input;
	}
};

function tryRoot(views, view) {
	if (views.length == 0) return Promise.reject(new Error("view not found " + view));
	var root = views.shift();

	var path = Path.join(root, view);
	if (path.indexOf(root) !== 0) return Promise.reject(
		new Error("Path outside views dir\n" + root + '\n' + view)
	);
	if (Path.extname(path) != ".html") path += ".html";
	debug("read view from", path);
	return readFile(path).catch(function(err) {
		return tryRoot(views, view);
	});
}

function bufferPipe(ps) {
	return pify(function(cb) {
		ps.pipe(BufferList(cb));
	})();
}

function fetchUrl(url) {
	var obj = typeof url == "string" ? URL.parse(url) : Object.assign({}, url);
	if (!dom.navigator.userAgent) {
		console.warn("Missing dom.navigator.userAgent property");
	} else {
		obj.headers = Object.assign({
			"User-Agent": dom.navigator.userAgent
		}, obj.headers);
	}
	obj.href = URL.format(obj);
	obj.path = URL.parse(obj.href).path;

	var ps = new PassThrough();
	debug("requesting", obj.href);
	return new Promise(function(resolve, reject) {
		var req = (obj.protocol == "https:" ? https : http).request(obj, function(res) {
			if (res.statusCode >= 400) return reject(new Error(`Error loading subrequest`));
			if (res.headers.location) {
				var location = URL.resolve(obj.href, res.headers.location);
				var redirObj = URL.parse(location);
				try {
					req.abort();
				} catch(ex) {
				}
				redirObj.redirects = (obj.redirects || 0) + 1;
				if (redirObj.redirects >= 5) {
					debug("Too many redirects");
					return reject(new Error(`Too many redirects ${obj.href}`));
				} else {
					debug("Redirecting to", obj.href);
					fetchUrl(redirObj).then(resolve).catch(reject);
				}
			} else {
				debug("pipe url response", res.statusCode);
				res.setEncoding('utf-8');
				res.pipe(ps);
				resolve(ps);
			}
		}).end();
	});
}

exports.prioritize = function(mw, settings, request, response) {
	if (!settings.priority) settings.priority = 0;
	if (request.xhr) settings.priority++;
};

exports.develop = function(mw, settings, request, response) {
	settings.load.disable = request.query.develop !== undefined;
};
<|endoftext|>


<commit_before>(function() {

    var i18n = require('./i18nextWrapper')
      , i18nWT = require('./i18nextWTWrapper')
      , fs = require('fs')
      , filesync = require('./filesync')
      , Cookies = require('cookies');

    if (typeof module !== 'undefined' && module.exports) {
        module.exports = i18n;
    }

    var resStore
      , backend
      , orgInit = i18n.init
      , registeredAppHelpers;

    i18n.init = function(options, cb) {
        if (typeof options === 'function') {
            cb = options;
            options = {};
        }

        options = options || {};
        options.resSetPath = options.resSetPath || 'locales/__lng__/__ns__.json';
        options.sendMissing = options.saveMissing || false;
        options.detectLngFromPath = options.detectLngFromPath === undefined ? false : options.detectLngFromPath;
        options.forceDetectLngFromPath = options.forceDetectLngFromPath !== true ? false : options.forceDetectLngFromPath;
        options.supportedLngs = [];
        options.ignoreRoutes = options.ignoreRoutes || [];
        options.cookieName = options.cookieName || 'i18next';
        resStore = options.resStore ? options.resStore : {};

        // obsolete
        if (options.useSync) console.log('options.useSync is obsolete use i18next.backend(...) instead.');

        var o = i18n.functions.extend(i18n.options, options);

        // extend functions with applyReplacement
        var applyReplacement = function(string, replacementHash) {
            i18n.functions.each(replacementHash, function(key, value) {
                string = string.replace([o.interpolationPrefix, key, o.interpolationSuffix].join(''), value);
            });
            return string;
        };
        i18n.functions.extend(i18n.functions, {
            applyReplacement: applyReplacement
        });

        if (!backend) backend = filesync;

        i18n.functions.extend(i18n.sync, backend);

        if (typeof i18n.sync.configure === 'function') {
            i18n.sync.configure(resStore, o, i18n.functions);
        }

        orgInit(options, cb);
    };

    i18n.backend = function(sync) {
        backend = sync;
    };

    i18n.handle = function(req, res, next) {
        var ignore = i18n.options.ignoreRoutes;
        for (var i = 0, len = ignore.length; i < len; i++) {
            if (req.path.indexOf(ignore[i]) > -1) {
                return next();
            }
        }

        var locale = i18n.detectLanguage(req, res);

        // set locale & i18n in req
        req.locale = req.lng = req.language = locale;

        // assert t function returns always translation
        // in given lng inside this request
        var t = function(key, options) {
            options = options || {};
            options.lng = options.lng || req.lng || i18n.lng();
            return i18n.t(key, options);
        };

        var i18nDummy = {
            t: t,
            translate: t,
            lng: function() { return locale; },
            locale: function() { return locale; },
            language: function() { return locale; }
        };

        // assert for req
        req.i18n = i18nDummy;
        req.t = req.t || t;

        // assert for res -> template
        if (registeredAppHelpers) {
            if (res.locals) {
                res.locals.t = t;
                res.locals.i18n = i18n;
            }
        }

        i18n.setLng(locale, function() {
            if (i18n.options.useCookie) i18n.persistCookie(req, res, i18n.lng());
            next();
        });
    };

    i18n.registerAppHelper = function(app) {
        registeredAppHelpers = true;

        if(app.helpers) {
          app.helpers({
            t: i18n.t,
            i18n: {
                t: i18n.t,
                translate: i18n.t,
                lng: i18n.lng,
                locale: i18n.lng,
                language: i18n.lng
            }
          });
        } else {
            app.locals.t = i18n.t;
            app.locals.i18n = {
                t: i18n.t,
                translate: i18n.t,
                lng: i18n.lng,
                locale: i18n.lng,
                language: i18n.lng
            };
        }

        return i18n;
    };

    i18n.serveClientScript = function(app) {
        app.get('/i18next/i18next.js', function(req, res) {
            var filename = (process.env.NODE_ENV !== 'production' || process.env.DEBUG) ? __dirname + '/dep/i18next-' + i18n.Version + '.js' : __dirname + '/dep/i18next-' + i18n.Version + '.min.js';

            res.sendfile(filename);
        });

        return i18n;
    };

    i18n.serveWebTranslate = function(app, options) {
        i18nWT.serve(app, options);

        return i18n;
    };

    i18n.serveDynamicResources = function(app) {
        app.get('/locales/resources.json', function(req, res) {

            var resources = {};

            res.contentType('json');

            var languages = req.query.lng ? req.query.lng.split(' ') : []
              , opts = { ns: { namespaces: req.query.ns ? req.query.ns.split(' ') : [] } };

            i18n.sync.load(languages, opts, function() {

                languages.forEach(function(lng) {
                    if (!resources[lng]) resources[lng] = {};

                    opts.ns.namespaces.forEach(function(ns) {
                        if (!resources[lng][ns]) resources[lng][ns] = resStore[lng][ns] || {};
                    });
                });

                res.send(resources);
            });
        });

        return i18n;
    };

    i18n.serveMissingKeyRoute = function(app, authenticated) {
        app.post('/locales/add/:lng/:ns', function(req, res) {
            if (authenticated && !authenticated(req, res)) {
                res.end();
                return;
            }

            var lng = req.params.lng;
            var ns = req.params.ns;

            for (var m in req.body) {
                i18n.sync.postMissing(lng, ns, m, req.body[m]);
            }

            res.json('ok');
        });

        return i18n;
    };

    i18n.serveChangeKeyRoute = function(app, authenticated) {
        app.post('/locales/change/:lng/:ns', function(req, res) {
            if (authenticated && !authenticated(req, res)) {
                res.end();
                return;
            }

            var lng = req.params.lng;
            var ns = req.params.ns;

            for (var m in req.body) {
                i18n.sync.postChange(lng, ns, m, req.body[m]);
            }

            res.json('ok');
        });

        return i18n;
    };

    i18n.serveRemoveKeyRoute = function(app) {
        app.post('/locales/remove/:lng/:ns', function(req, res) {
            if (authenticated && !authenticated(req, res)) {
                res.end();
                return;
            }

            var lng = req.params.lng;
            var ns = req.params.ns;

            for (var m in req.body) {
                i18n.sync.postRemove(lng, ns, m, req.body[m]);
            }

            res.json('ok');
        });

        return i18n;
    };

    i18n.persistCookie = function(req, res, locale) {
        var cookies = new Cookies(req, res);
        var expirationDate = new Date();
        expirationDate.setFullYear(expirationDate.getFullYear() + 1);

        cookies.set(i18n.options.cookieName, locale, { expires: expirationDate, httpOnly : false });
    };

    i18n.addRoute = function(route, lngs, app, verb, fc) {
        if (typeof verb === 'function') {
            fc = verb;
            verb = 'get';
        }

        for (var i = 0, li = lngs.length; i < li; i++) {
            var parts = route.split('/');
            var locRoute = [];
            for (var y = 0, ly = parts.length; y < ly; y++) {
                var part = parts[y];
                if (part.indexOf(':') === 0 || part === '') {
                    locRoute.push(part);
                } else {
                    locRoute.push(i18n.t(part, { lng: lngs[i] }));
                }
            }

            app[verb || 'get'](locRoute.join('/'), fc);
        }
    };

})();
<commit_msg>added missing authenticated argument<commit_after>(function() {

    var i18n = require('./i18nextWrapper')
      , i18nWT = require('./i18nextWTWrapper')
      , fs = require('fs')
      , filesync = require('./filesync')
      , Cookies = require('cookies');

    if (typeof module !== 'undefined' && module.exports) {
        module.exports = i18n;
    }

    var resStore
      , backend
      , orgInit = i18n.init
      , registeredAppHelpers;

    i18n.init = function(options, cb) {
        if (typeof options === 'function') {
            cb = options;
            options = {};
        }

        options = options || {};
        options.resSetPath = options.resSetPath || 'locales/__lng__/__ns__.json';
        options.sendMissing = options.saveMissing || false;
        options.detectLngFromPath = options.detectLngFromPath === undefined ? false : options.detectLngFromPath;
        options.forceDetectLngFromPath = options.forceDetectLngFromPath !== true ? false : options.forceDetectLngFromPath;
        options.supportedLngs = [];
        options.ignoreRoutes = options.ignoreRoutes || [];
        options.cookieName = options.cookieName || 'i18next';
        resStore = options.resStore ? options.resStore : {};

        // obsolete
        if (options.useSync) console.log('options.useSync is obsolete use i18next.backend(...) instead.');

        var o = i18n.functions.extend(i18n.options, options);

        // extend functions with applyReplacement
        var applyReplacement = function(string, replacementHash) {
            i18n.functions.each(replacementHash, function(key, value) {
                string = string.replace([o.interpolationPrefix, key, o.interpolationSuffix].join(''), value);
            });
            return string;
        };
        i18n.functions.extend(i18n.functions, {
            applyReplacement: applyReplacement
        });

        if (!backend) backend = filesync;

        i18n.functions.extend(i18n.sync, backend);

        if (typeof i18n.sync.configure === 'function') {
            i18n.sync.configure(resStore, o, i18n.functions);
        }

        orgInit(options, cb);
    };

    i18n.backend = function(sync) {
        backend = sync;
    };

    i18n.handle = function(req, res, next) {
        var ignore = i18n.options.ignoreRoutes;
        for (var i = 0, len = ignore.length; i < len; i++) {
            if (req.path.indexOf(ignore[i]) > -1) {
                return next();
            }
        }

        var locale = i18n.detectLanguage(req, res);

        // set locale & i18n in req
        req.locale = req.lng = req.language = locale;

        // assert t function returns always translation
        // in given lng inside this request
        var t = function(key, options) {
            options = options || {};
            options.lng = options.lng || req.lng || i18n.lng();
            return i18n.t(key, options);
        };

        var i18nDummy = {
            t: t,
            translate: t,
            lng: function() { return locale; },
            locale: function() { return locale; },
            language: function() { return locale; }
        };

        // assert for req
        req.i18n = i18nDummy;
        req.t = req.t || t;

        // assert for res -> template
        if (registeredAppHelpers) {
            if (res.locals) {
                res.locals.t = t;
                res.locals.i18n = i18n;
            }
        }

        i18n.setLng(locale, function() {
            if (i18n.options.useCookie) i18n.persistCookie(req, res, i18n.lng());
            next();
        });
    };

    i18n.registerAppHelper = function(app) {
        registeredAppHelpers = true;

        if(app.helpers) {
          app.helpers({
            t: i18n.t,
            i18n: {
                t: i18n.t,
                translate: i18n.t,
                lng: i18n.lng,
                locale: i18n.lng,
                language: i18n.lng
            }
          });
        } else {
            app.locals.t = i18n.t;
            app.locals.i18n = {
                t: i18n.t,
                translate: i18n.t,
                lng: i18n.lng,
                locale: i18n.lng,
                language: i18n.lng
            };
        }

        return i18n;
    };

    i18n.serveClientScript = function(app) {
        app.get('/i18next/i18next.js', function(req, res) {
            var filename = (process.env.NODE_ENV !== 'production' || process.env.DEBUG) ? __dirname + '/dep/i18next-' + i18n.Version + '.js' : __dirname + '/dep/i18next-' + i18n.Version + '.min.js';

            res.sendfile(filename);
        });

        return i18n;
    };

    i18n.serveWebTranslate = function(app, options) {
        i18nWT.serve(app, options);

        return i18n;
    };

    i18n.serveDynamicResources = function(app) {
        app.get('/locales/resources.json', function(req, res) {

            var resources = {};

            res.contentType('json');

            var languages = req.query.lng ? req.query.lng.split(' ') : []
              , opts = { ns: { namespaces: req.query.ns ? req.query.ns.split(' ') : [] } };

            i18n.sync.load(languages, opts, function() {

                languages.forEach(function(lng) {
                    if (!resources[lng]) resources[lng] = {};

                    opts.ns.namespaces.forEach(function(ns) {
                        if (!resources[lng][ns]) resources[lng][ns] = resStore[lng][ns] || {};
                    });
                });

                res.send(resources);
            });
        });

        return i18n;
    };

    i18n.serveMissingKeyRoute = function(app, authenticated) {
        app.post('/locales/add/:lng/:ns', function(req, res) {
            if (authenticated && !authenticated(req, res)) {
                res.end();
                return;
            }

            var lng = req.params.lng;
            var ns = req.params.ns;

            for (var m in req.body) {
                i18n.sync.postMissing(lng, ns, m, req.body[m]);
            }

            res.json('ok');
        });

        return i18n;
    };

    i18n.serveChangeKeyRoute = function(app, authenticated) {
        app.post('/locales/change/:lng/:ns', function(req, res) {
            if (authenticated && !authenticated(req, res)) {
                res.end();
                return;
            }

            var lng = req.params.lng;
            var ns = req.params.ns;

            for (var m in req.body) {
                i18n.sync.postChange(lng, ns, m, req.body[m]);
            }

            res.json('ok');
        });

        return i18n;
    };

    i18n.serveRemoveKeyRoute = function(app, authenticated) {
        app.post('/locales/remove/:lng/:ns', function(req, res) {
            if (authenticated && !authenticated(req, res)) {
                res.end();
                return;
            }

            var lng = req.params.lng;
            var ns = req.params.ns;

            for (var m in req.body) {
                i18n.sync.postRemove(lng, ns, m, req.body[m]);
            }

            res.json('ok');
        });

        return i18n;
    };

    i18n.persistCookie = function(req, res, locale) {
        var cookies = new Cookies(req, res);
        var expirationDate = new Date();
        expirationDate.setFullYear(expirationDate.getFullYear() + 1);

        cookies.set(i18n.options.cookieName, locale, { expires: expirationDate, httpOnly : false });
    };

    i18n.addRoute = function(route, lngs, app, verb, fc) {
        if (typeof verb === 'function') {
            fc = verb;
            verb = 'get';
        }

        for (var i = 0, li = lngs.length; i < li; i++) {
            var parts = route.split('/');
            var locRoute = [];
            for (var y = 0, ly = parts.length; y < ly; y++) {
                var part = parts[y];
                if (part.indexOf(':') === 0 || part === '') {
                    locRoute.push(part);
                } else {
                    locRoute.push(i18n.t(part, { lng: lngs[i] }));
                }
            }

            app[verb || 'get'](locRoute.join('/'), fc);
        }
    };

})();
<|endoftext|>


<commit_before>"use strict";

var fs = require("fs");
var console = require("console");
var util = require("util");
var path = require("path");
var childProcess = require("child_process");
var AdmZip = require("adm-zip");
var phantomjs = require("phantomjs-prebuilt");
var binPath = phantomjs.path;

var sheetTypeInfo = {
    scss: {
        varPrefix: "$",
        mixinPrefixFormat: "@mixin %sclasses"
    },
    less: {
        varPrefix: "@",
        mixinPrefixFormat: ".%sclasses()"
    }
};

function buildStylesheet(selection, type, with_map) {
    var typeInfo = sheetTypeInfo[type];
    if (!typeInfo) {
        throw new Error("Invalid stylesheet type: " + type);
    }
    var prefix = selection.preferences.fontPref.prefix;
    var contentDeclarations = "";
    var mainMixin = "";
    var sassMapItems = [];
    selection.icons.forEach(function(iconEntry) {
        var props = iconEntry.properties;
        contentDeclarations += util.format("%s%s%s-content: \"\\%s\";\n", typeInfo.varPrefix, prefix, props.name, props.code.toString(16));
        mainMixin += util.format(".%s%s:before { content: %s%s%s-content; }\n", prefix, props.name, typeInfo.varPrefix, prefix, props.name);
        sassMapItems.push(util.format("'%s': '\\%s'", props.name, props.code.toString(16)));
    });
    var output = "";
    output += "// Script-generated file, do not modify by hand\n\n";
    output += contentDeclarations;
    output += "\n";
    if (type === "scss" && with_map) {
        output += util.format("%s%sitem-map: (%s);\n\n", typeInfo.varPrefix, prefix, sassMapItems.join(", "));
    }
    output += util.format(typeInfo.mixinPrefixFormat + " {\n", prefix);
    output += "    " + mainMixin.replace(/\n(?=.+?)/g, "\n    ");
    output += "}\n";
    return output;
}

function buildScssSheet(selection, with_map) {
    return buildStylesheet(selection, "scss", with_map);
}

function buildLessSheet(selection) {
    return buildStylesheet(selection, "less");
}

function buildProject(projectFilePath, cb) {
    var childArgs = [
        "--load-images=false",
        "--ssl-protocol=any",
        path.join(__dirname, "icomoon.phantom.js"),
        projectFilePath
    ];
    childProcess.execFile(binPath, childArgs, {maxBuffer: 102400*1024}, function(err, stdout, stderr) {
        if (err) {
            cb(err);
            return;
        }
        var result;
        try {
            var zipBuf = new Buffer(stdout, "base64");
            var zip = new AdmZip(zipBuf);
            var zipEntries = zip.getEntries(); // an array of ZipEntry records

            result = {
                zip: zipBuf,
                fonts: {}
            };
            zipEntries.forEach(function(zipEntry) {
                if (/.+\/.*\.(ttf|woff|eot|svg)$/.test(zipEntry.entryName)) {
                    result.fonts[zipEntry.name] = {
                        path: zipEntry.entryName,
                        data: zip.readFile(zipEntry),
                    };
                } else if (/^[^\/]+\.css$/.test(zipEntry.entryName)) {
                    result.css = zip.readAsText(zipEntry);
                } else if ("selection.json" === zipEntry.entryName) {
                    result.selection = JSON.parse(zip.readAsText(zipEntry));
                }
            });
            result.scss = buildScssSheet(result.selection);
            result.scss_with_map = buildScssSheet(result.selection, true);
            result.less = buildLessSheet(result.selection);
        } catch (e) {
            cb(e);
            return;
        }
        cb(null, result);
    });
}

module.exports = {
    buildProject: buildProject,
    buildScssSheet: buildScssSheet,
    buildLessSheet: buildLessSheet,
};
<commit_msg>Support multiple class names<commit_after>"use strict";

var fs = require("fs");
var console = require("console");
var util = require("util");
var path = require("path");
var childProcess = require("child_process");
var AdmZip = require("adm-zip");
var phantomjs = require("phantomjs-prebuilt");
var binPath = phantomjs.path;

var sheetTypeInfo = {
    scss: {
        varPrefix: "$",
        mixinPrefixFormat: "@mixin %sclasses"
    },
    less: {
        varPrefix: "@",
        mixinPrefixFormat: ".%sclasses()"
    }
};

function buildStylesheet(selection, type, with_map) {
    var typeInfo = sheetTypeInfo[type];
    if (!typeInfo) {
        throw new Error("Invalid stylesheet type: " + type);
    }
    var prefix = selection.preferences.fontPref.prefix;
    var contentDeclarations = "";
    var mainMixin = "";
    var sassMapItems = [];
    selection.icons.forEach(function(iconEntry) {
        var props = iconEntry.properties;
        var splittedNames = props.name.split(",");
        for (let name of splittedNames) {
            name = name.trim();
            contentDeclarations += util.format("%s%s%s-content: \"\\%s\";\n", typeInfo.varPrefix, prefix, name, props.code.toString(16));
            mainMixin += util.format(".%s%s:before { content: %s%s%s-content; }\n", prefix, name, typeInfo.varPrefix, prefix, props.name);
            sassMapItems.push(util.format("'%s': '\\%s'", name, props.code.toString(16)));
        }
    });
    var output = "";
    output += "// Script-generated file, do not modify by hand\n\n";
    output += contentDeclarations;
    output += "\n";
    if (type === "scss" && with_map) {
        output += util.format("%s%sitem-map: (%s);\n\n", typeInfo.varPrefix, prefix, sassMapItems.join(", "));
    }
    output += util.format(typeInfo.mixinPrefixFormat + " {\n", prefix);
    output += "    " + mainMixin.replace(/\n(?=.+?)/g, "\n    ");
    output += "}\n";
    return output;
}

function buildScssSheet(selection, with_map) {
    return buildStylesheet(selection, "scss", with_map);
}

function buildLessSheet(selection) {
    return buildStylesheet(selection, "less");
}

function buildProject(projectFilePath, cb) {
    var childArgs = [
        "--load-images=false",
        "--ssl-protocol=any",
        path.join(__dirname, "icomoon.phantom.js"),
        projectFilePath
    ];
    childProcess.execFile(binPath, childArgs, {maxBuffer: 102400*1024}, function(err, stdout, stderr) {
        if (err) {
            cb(err);
            return;
        }
        var result;
        try {
            var zipBuf = new Buffer(stdout, "base64");
            var zip = new AdmZip(zipBuf);
            var zipEntries = zip.getEntries(); // an array of ZipEntry records

            result = {
                zip: zipBuf,
                fonts: {}
            };
            zipEntries.forEach(function(zipEntry) {
                if (/.+\/.*\.(ttf|woff|eot|svg)$/.test(zipEntry.entryName)) {
                    result.fonts[zipEntry.name] = {
                        path: zipEntry.entryName,
                        data: zip.readFile(zipEntry),
                    };
                } else if (/^[^\/]+\.css$/.test(zipEntry.entryName)) {
                    result.css = zip.readAsText(zipEntry);
                } else if ("selection.json" === zipEntry.entryName) {
                    result.selection = JSON.parse(zip.readAsText(zipEntry));
                }
            });
            result.scss = buildScssSheet(result.selection);
            result.scss_with_map = buildScssSheet(result.selection, true);
            result.less = buildLessSheet(result.selection);
        } catch (e) {
            cb(e);
            return;
        }
        cb(null, result);
    });
}

module.exports = {
    buildProject: buildProject,
    buildScssSheet: buildScssSheet,
    buildLessSheet: buildLessSheet,
};
<|endoftext|>


<commit_before>var fsStack = [];

/**
 * Добавить содержимое файла в стек вставок в шаблон по заданному URL
 *
 * @expose
 * @param {string} base - базовый путь к файлу
 * @param {string} url - путь к файлу
 * @param {string} nl - символ перевода строки
 * @param {?string=} [opt_type] - тип рендеринга шаблонов
 * @return {(string|boolean)}
 */
Snakeskin.include = function (base, url, nl, opt_type) {
	/* istanbul ignore if */
	if (!IS_NODE) {
		return false;
	}

	var fs = require('fs'),
		path = require('path'),
		glob = require('glob')['sync'];

	var s = ADV_LEFT_BLOCK + LEFT_BLOCK,
		e = RIGHT_BLOCK;

	try {
		let extname = path['extname'](url),
			include = Snakeskin.LocalVars.include;

		let src = path['resolve'](
			path['dirname'](base),
			url + (extname ? '' : '.ss')
		);

		(/\*/.test(src) ? glob(src) : [src]).forEach((src) => {
			src = path['normalize'](src);

			if (!include[src]) {
				include[src] = true;
				let file = fs['readFileSync'](src).toString();

				fsStack.push(
					`${s}__setFile__ ${escapeBackslash(src)}${e}` +

					(opt_type ?
						`${s}__setSSFlag__ renderAs '${opt_type}'${e}` : '') +

					`${startWhiteSpaceRgxp.test(file) ? '' : nl}` +

					file +

					`${endWhiteSpaceRgxp.test(file) ? '' : `${nl}${s}__cutLine__${e}`}` +
					`${s}__endSetFile__${e}`
				);
			}
		});

		return true;

	} catch (err) {
		fsStack.push(`${s}__setError__ ${err.message}${e}`);
	}

	return false;
};
<commit_msg>Исправление ошибки<commit_after>var fsStack = [];

/**
 * Добавить содержимое файла в стек вставок в шаблон по заданному URL
 *
 * @expose
 * @param {string} base - базовый путь к файлу
 * @param {string} url - путь к файлу
 * @param {string} nl - символ перевода строки
 * @param {?string=} [opt_type] - тип рендеринга шаблонов
 * @return {(string|boolean)}
 */
Snakeskin.include = function (base, url, nl, opt_type) {
	/* istanbul ignore if */
	if (!IS_NODE) {
		return false;
	}

	var fs = require('fs'),
		path = require('path'),
		glob = require('glob')['sync'];

	var s = ADV_LEFT_BLOCK + LEFT_BLOCK,
		e = RIGHT_BLOCK;

	try {
		let extname = path['extname'](url),
			include = Snakeskin.LocalVars.include;

		let src = path['resolve'](
			path['dirname'](base),
			url + (extname ? '' : '.ss')
		);

		(/\*/.test(src) ? glob(src) : [src]).forEach((src) => {
			src = path['normalize'](src);

			if (!include[src]) {
				include[src] = true;
				let file = fs['readFileSync'](src).toString();

				fsStack.push(
					`${s}__setFile__ ${src}${e}` +

					(opt_type ?
						`${s}__setSSFlag__ renderAs '${opt_type}'${e}` : '') +

					`${startWhiteSpaceRgxp.test(file) ? '' : nl}` +

					file +

					`${endWhiteSpaceRgxp.test(file) ? '' : `${nl}${s}__cutLine__${e}`}` +
					`${s}__endSetFile__${e}`
				);
			}
		});

		return true;

	} catch (err) {
		fsStack.push(`${s}__setError__ ${err.message}${e}`);
	}

	return false;
};
<|endoftext|>


<commit_before>'use strict';

var shell = require('shelljs');
var fs = require('fs');
var chalk = require('chalk');
var utils = require('./utils');
var series = require('async-series');

var progress = new utils.Progress();

exports.init = function(options) {
  progress.start();

  series([
      checkPermissions,
      utils.checkNpmPermission,
      function(done) { cloneMean(options, done); },
      printMeanIntro,
      function(done) { addGitRemote(options.name, done); },
    ],function(err) {
      if (err){ throw err; }
      nextSteps(options.name);
    }
    );
  progress.stop();

    //checkoutStableTag(name); // Checkout to stable tag noted in the cli.
    //nextSteps(name); // Guide the user about the next steps

};

function checkPermissions(done){
  if (utils.isWin){
    console.log('On windows platform - Please check permissions independently');
    console.log('All permissions should be run with the local users permissions');
  }
  if (process.getuid) {
   var uid = process.getuid();
   if (  uid === 0){
     console.log(chalk.red('Installation of mean should not be done as root! bailing out'));
     throw('RUNNING AS ROOT');

   }
 }
  done();
}

function printMeanIntro(done) {
  var logo = fs.readFileSync(__dirname + '/../img/logo.txt');
  console.log(logo.toString());
  done();
}

function cloneMean(options, done){

  // TODO - Possibly remove depdendency of git all together and download and optionally add a remote.
  if (!shell.which('git')) return console.log(chalk.red('Prerequisite not installed: git'));
  var name = options.name;
  var source = (options.git ? 'git@github.com:linnovate/mean.git' : 'http://github.com/linnovate/mean.git');

  // Allow specifying specific repo
  if (options.repo) {
    source = options.repo;
  }

  // If full clone is specified do a normal clone - default is a shallow --depth 1 clone"
  var shallow = (options.full ? '' : ' --depth 1 ');

  // Install the specified branch or the stableTag - No more MASTER!
  //var branch = options.branch ? options.branch : stableTag;
  var branch = options.branch;
  source = branch + ' ' + source + ' "' + name + '"';
  var gitCmd = 'git clone ' + shallow + ' -b' + source;
  console.log(chalk.green('Cloning branch: %s into destination folder:'), options.branch, name);
  console.log(chalk.green(gitCmd));

  // Running clone
  shell.exec(gitCmd , function(status, output) {
    if (status) {
      throw output;
    }
    console.log();
    done();

  });
}

function addGitRemote(name, done) {
  shell.exec('cd '+ name + ' && git remote rename origin upstream', function (status, error) {
    if (!status) {
      console.log('  Added the "remote" upstream origin');
      console.log();
      done();
    } else {
      throw error;
    }
  });
}

/*
function checkoutStableTag(name){
  utils.loadPackageJson('./' + name + '/package.json', function (err, data) {
    var stableTag = 'v'+ data.mean;
    console.log('xxx' ,data.mean);
    var tagCmd = 'cd ' + name + '&& git checkout  ' + stableTag;
    shell.exec(tagCmd);
    console.log(chalk.green('Checked out to stable Tag - '), stableTag);
  });

}
*/

function nextSteps(name) {
  // show the next steps after the person has seen the start/help text so sleep 1 second.
  setTimeout(function(){
    console.log('  Install node package dependencies:');
    console.log('    $ ' + chalk.green('cd %s && npm install'), name);
    console.log('  Bower install should be triggered for client side dependencies.');
    console.log('  If it did not run invoke it manually...');
    console.log('    $ ' + chalk.green('cd %s && bower install'), name);
    console.log();
    console.log('  Run the app by running:');
    console.log('    $ ' + chalk.green('cd %s and then run..'), name);
    console.log('    $ '+ chalk.green('gulp'));
    console.log();
  },1000);
}
<commit_msg>typo in https - fixed #1126<commit_after>'use strict';

var shell = require('shelljs');
var fs = require('fs');
var chalk = require('chalk');
var utils = require('./utils');
var series = require('async-series');

var progress = new utils.Progress();

exports.init = function(options) {
  progress.start();

  series([
      checkPermissions,
      utils.checkNpmPermission,
      function(done) { cloneMean(options, done); },
      printMeanIntro,
      function(done) { addGitRemote(options.name, done); },
    ],function(err) {
      if (err){ throw err; }
      nextSteps(options.name);
    }
    );
  progress.stop();

    //checkoutStableTag(name); // Checkout to stable tag noted in the cli.
    //nextSteps(name); // Guide the user about the next steps

};

function checkPermissions(done){
  if (utils.isWin){
    console.log('On windows platform - Please check permissions independently');
    console.log('All permissions should be run with the local users permissions');
  }
  if (process.getuid) {
   var uid = process.getuid();
   if (  uid === 0){
     console.log(chalk.red('Installation of mean should not be done as root! bailing out'));
     throw('RUNNING AS ROOT');

   }
 }
  done();
}

function printMeanIntro(done) {
  var logo = fs.readFileSync(__dirname + '/../img/logo.txt');
  console.log(logo.toString());
  done();
}

function cloneMean(options, done){

  // TODO - Possibly remove depdendency of git all together and download and optionally add a remote.
  if (!shell.which('git')) return console.log(chalk.red('Prerequisite not installed: git'));
  var name = options.name;
  var source = (options.git ? 'git@github.com:linnovate/mean.git' : 'https://github.com/linnovate/mean.git');

  // Allow specifying specific repo
  if (options.repo) {
    source = options.repo;
  }

  // If full clone is specified do a normal clone - default is a shallow --depth 1 clone"
  var shallow = (options.full ? '' : ' --depth 1 ');

  // Install the specified branch or the stableTag - No more MASTER!
  //var branch = options.branch ? options.branch : stableTag;
  var branch = options.branch;
  source = branch + ' ' + source + ' "' + name + '"';
  var gitCmd = 'git clone ' + shallow + ' -b' + source;
  console.log(chalk.green('Cloning branch: %s into destination folder:'), options.branch, name);
  console.log(chalk.green(gitCmd));

  // Running clone
  shell.exec(gitCmd , function(status, output) {
    if (status) {
      throw output;
    }
    console.log();
    done();

  });
}

function addGitRemote(name, done) {
  shell.exec('cd '+ name + ' && git remote rename origin upstream', function (status, error) {
    if (!status) {
      console.log('  Added the "remote" upstream origin');
      console.log();
      done();
    } else {
      throw error;
    }
  });
}

/*
function checkoutStableTag(name){
  utils.loadPackageJson('./' + name + '/package.json', function (err, data) {
    var stableTag = 'v'+ data.mean;
    console.log('xxx' ,data.mean);
    var tagCmd = 'cd ' + name + '&& git checkout  ' + stableTag;
    shell.exec(tagCmd);
    console.log(chalk.green('Checked out to stable Tag - '), stableTag);
  });

}
*/

function nextSteps(name) {
  // show the next steps after the person has seen the start/help text so sleep 1 second.
  setTimeout(function(){
    console.log('  Install node package dependencies:');
    console.log('    $ ' + chalk.green('cd %s && npm install'), name);
    console.log('  Bower install should be triggered for client side dependencies.');
    console.log('  If it did not run invoke it manually...');
    console.log('    $ ' + chalk.green('cd %s && bower install'), name);
    console.log();
    console.log('  Run the app by running:');
    console.log('    $ ' + chalk.green('cd %s and then run..'), name);
    console.log('    $ '+ chalk.green('gulp'));
    console.log();
  },1000);
}
<|endoftext|>


<commit_before>"use strict";

module.exports = exports = install;

exports.usage = 'Attempts to install pre-built binary for module';

var fs = require('fs');
var path = require('path');
var zlib = require('zlib');
var log = require('npmlog');
var existsAsync = fs.exists || path.exists;
var versioning = require('./util/versioning.js');
var testbinary = require('./testbinary.js');
var clean = require('./clean.js');

function download(uri,opts,callback) {
    log.http('GET', uri);

    var req = null;
    var requestOpts = {
        uri: uri.replace('+','%2B'),
        headers: {
          'User-Agent': 'node-pre-gyp (node ' + process.version + ')'
        }
    };

    var proxyUrl = opts.proxy ||
                    process.env.http_proxy ||
                    process.env.HTTP_PROXY ||
                    process.env.npm_config_proxy;
    if (proxyUrl) {
      if (/^https?:\/\//i.test(proxyUrl)) {
        log.verbose('download', 'using proxy url: "%s"', proxyUrl);
        requestOpts.proxy = proxyUrl;
      } else {
        log.warn('download', 'ignoring invalid "proxy" config setting: "%s"', proxyUrl);
      }
    }
    try {
        req = require('request')(requestOpts);
    } catch (e) {
        return callback(e);
    }
    if (req) {
      req.on('response', function (res) {
        log.http(res.statusCode, uri);
      });
    }
    return callback(null,req);
}

function place_binary(from,to,opts,callback) {
    download(from,opts,function(err,req) {
        if (err) return callback(err);
        if (!req) return callback(new Error("empty req"));
        var badDownload = false;
        var extractCount = 0;
        var gunzip = zlib.createGunzip();
        var extracter = require('tar').Extract({ path: to, strip: 1});

        function afterTarball(err) {
            if (err) return callback(err);
            if (badDownload) return callback(new Error("bad download"));
            if (extractCount === 0) {
                return callback(new Error('There was a fatal problem while downloading/extracting the tarball'));
            }
            log.info('tarball', 'done parsing tarball');
            callback();
        }

        function filter_func(entry) {
            // ensure directories are +x
            // https://github.com/mapnik/node-mapnik/issues/262
            entry.props.mode |= (entry.props.mode >>> 2) & parseInt('0111',8);
            log.info('install','unpacking ' + entry.path);
            extractCount++;
        }

        gunzip.on('error', callback);
        extracter.on('entry', filter_func);
        extracter.on('error', callback);
        extracter.on('end', afterTarball);

        req.on('error', function(err) {
            badDownload = true;
            return callback(err);
        });

        req.on('close', function () {
            if (extractCount === 0) {
                return callback(new Error('Connection closed while downloading tarball file'));
            }
        });

        req.on('response', function(res) {
            if (res.statusCode !== 200) {
                badDownload = true;
                if (res.statusCode == 404) {
                    return callback(new Error('Pre-built binary not available for your system, looked for ' + from));
                } else {
                    return callback(new Error(res.statusCode + ' status code downloading tarball ' + from));
                }
            }
            // start unzipping and untaring
            req.pipe(gunzip).pipe(extracter);
        });
    });
}

function do_build(gyp,argv,callback) {
  gyp.todo.push( { name: 'build', args: ['rebuild'] } );
  process.nextTick(callback);
}

function print_fallback_error(err,opts,package_json) {
    var fallback_message = ' (falling back to source compile with node-gyp)';
    var full_message = "Pre-built binaries not found for " + package_json.name + "@" + package_json.version;
    full_message += " and " + opts.runtime + "@" + (opts.target || process.versions.node) + " (" + opts.node_abi + " ABI)";
    full_message += fallback_message
    log.error("Tried to download: " + opts.hosted_tarball);
    log.error(full_message)
    log.http(err.message);
}

function install(gyp, argv, callback) {
    var package_json = JSON.parse(fs.readFileSync('./package.json'));
    var source_build = gyp.opts['build-from-source'] || gyp.opts.build_from_source;
    var update_binary = gyp.opts['update-binary'] || gyp.opts.update_binary;
    var should_do_source_build = source_build === package_json.name || (source_build === true || source_build === 'true');
    var no_rollback = gyp.opts.hasOwnProperty('rollback') && gyp.opts.rollback === false;
    if (should_do_source_build) {
        log.info('build','requesting source compile');
        return do_build(gyp,argv,callback);
    } else {
        var fallback_to_build = gyp.opts['fallback-to-build'] || gyp.opts.fallback_to_build;
        var should_do_fallback_build = fallback_to_build === package_json.name || (fallback_to_build === true || fallback_to_build === 'true');
        // but allow override from npm
        if (process.env.npm_config_argv) {
            var cooked = JSON.parse(process.env.npm_config_argv).cooked;
            var match = cooked.indexOf("--fallback-to-build");
            if (match > -1 && cooked.length > match && cooked[match+1] == "false") {
                should_do_fallback_build = false;
                log.info('install','Build fallback disabled via npm flag: --fallback-to-build=false');
            }
        }
        var opts;
        try {
            opts = versioning.evaluate(package_json, gyp.opts);
        } catch (err) {
            return callback(err);
        }
        var from = opts.hosted_tarball;
        var to = opts.module_path;
        var binary_module = path.join(to,opts.module_name + '.node');
        if (existsAsync(binary_module,function(found) {
            if (found && !update_binary) {
                testbinary(gyp, argv, function(err) {
                    if (err) {
                        console.error('['+package_json.name+'] ' + err.message);
                        log.error("Testing local pre-built binary failed, attempting to re-download");
                        place_binary(from,to,opts,function(err) {
                            if (err) {
                                if (should_do_fallback_build) {
                                    print_fallback_error(err,opts,package_json);
                                    return do_build(gyp,argv,callback);
                                } else {
                                    return callback(err);
                                }
                            } else {
                                console.log('['+package_json.name+'] Success: "' + binary_module + '" is reinstalled via remote');
                                return callback();
                            }
                        });
                    } else {
                        console.log('['+package_json.name+'] Success: "' + binary_module + '" already installed');
                        console.log('Pass --update-binary to reinstall or --build-from-source to recompile');
                        return callback();
                    }
                });
            } else {
                if (!update_binary) log.info('check','checked for "' + binary_module + '" (not found)');
                place_binary(from,to,opts,function(err) {
                    if (err && should_do_fallback_build) {
                        print_fallback_error(err,opts,package_json);
                        return do_build(gyp,argv,callback);
                    } else if (err) {
                        return callback(err);
                    } else {
                        testbinary(gyp, argv, function(err) {
                            if (err) {
                                if (no_rollback) {
                                    return callback(err);
                                }
                                gyp.opts.silent_clean = true;
                                clean(gyp, argv, function(error) {
                                    if (error) console.log(error);
                                    if (should_do_fallback_build) {
                                        console.error('['+package_json.name+'] ' + err.message);
                                        log.error("Testing pre-built binary failed, attempting to source compile");
                                        return do_build(gyp,argv,callback);
                                    } else {
                                        return callback(err);
                                    }
                                });
                            } else {
                                console.log('['+package_json.name+'] Success: "' + binary_module + '" is installed via remote');
                                return callback();
                            }
                        });
                    }
                });
            }
        }));
    }
}
<commit_msg>fix missing semicolon<commit_after>"use strict";

module.exports = exports = install;

exports.usage = 'Attempts to install pre-built binary for module';

var fs = require('fs');
var path = require('path');
var zlib = require('zlib');
var log = require('npmlog');
var existsAsync = fs.exists || path.exists;
var versioning = require('./util/versioning.js');
var testbinary = require('./testbinary.js');
var clean = require('./clean.js');

function download(uri,opts,callback) {
    log.http('GET', uri);

    var req = null;
    var requestOpts = {
        uri: uri.replace('+','%2B'),
        headers: {
          'User-Agent': 'node-pre-gyp (node ' + process.version + ')'
        }
    };

    var proxyUrl = opts.proxy ||
                    process.env.http_proxy ||
                    process.env.HTTP_PROXY ||
                    process.env.npm_config_proxy;
    if (proxyUrl) {
      if (/^https?:\/\//i.test(proxyUrl)) {
        log.verbose('download', 'using proxy url: "%s"', proxyUrl);
        requestOpts.proxy = proxyUrl;
      } else {
        log.warn('download', 'ignoring invalid "proxy" config setting: "%s"', proxyUrl);
      }
    }
    try {
        req = require('request')(requestOpts);
    } catch (e) {
        return callback(e);
    }
    if (req) {
      req.on('response', function (res) {
        log.http(res.statusCode, uri);
      });
    }
    return callback(null,req);
}

function place_binary(from,to,opts,callback) {
    download(from,opts,function(err,req) {
        if (err) return callback(err);
        if (!req) return callback(new Error("empty req"));
        var badDownload = false;
        var extractCount = 0;
        var gunzip = zlib.createGunzip();
        var extracter = require('tar').Extract({ path: to, strip: 1});

        function afterTarball(err) {
            if (err) return callback(err);
            if (badDownload) return callback(new Error("bad download"));
            if (extractCount === 0) {
                return callback(new Error('There was a fatal problem while downloading/extracting the tarball'));
            }
            log.info('tarball', 'done parsing tarball');
            callback();
        }

        function filter_func(entry) {
            // ensure directories are +x
            // https://github.com/mapnik/node-mapnik/issues/262
            entry.props.mode |= (entry.props.mode >>> 2) & parseInt('0111',8);
            log.info('install','unpacking ' + entry.path);
            extractCount++;
        }

        gunzip.on('error', callback);
        extracter.on('entry', filter_func);
        extracter.on('error', callback);
        extracter.on('end', afterTarball);

        req.on('error', function(err) {
            badDownload = true;
            return callback(err);
        });

        req.on('close', function () {
            if (extractCount === 0) {
                return callback(new Error('Connection closed while downloading tarball file'));
            }
        });

        req.on('response', function(res) {
            if (res.statusCode !== 200) {
                badDownload = true;
                if (res.statusCode == 404) {
                    return callback(new Error('Pre-built binary not available for your system, looked for ' + from));
                } else {
                    return callback(new Error(res.statusCode + ' status code downloading tarball ' + from));
                }
            }
            // start unzipping and untaring
            req.pipe(gunzip).pipe(extracter);
        });
    });
}

function do_build(gyp,argv,callback) {
  gyp.todo.push( { name: 'build', args: ['rebuild'] } );
  process.nextTick(callback);
}

function print_fallback_error(err,opts,package_json) {
    var fallback_message = ' (falling back to source compile with node-gyp)';
    var full_message = "Pre-built binaries not found for " + package_json.name + "@" + package_json.version;
    full_message += " and " + opts.runtime + "@" + (opts.target || process.versions.node) + " (" + opts.node_abi + " ABI)";
    full_message += fallback_message;
    log.error("Tried to download: " + opts.hosted_tarball);
    log.error(full_message);
    log.http(err.message);
}

function install(gyp, argv, callback) {
    var package_json = JSON.parse(fs.readFileSync('./package.json'));
    var source_build = gyp.opts['build-from-source'] || gyp.opts.build_from_source;
    var update_binary = gyp.opts['update-binary'] || gyp.opts.update_binary;
    var should_do_source_build = source_build === package_json.name || (source_build === true || source_build === 'true');
    var no_rollback = gyp.opts.hasOwnProperty('rollback') && gyp.opts.rollback === false;
    if (should_do_source_build) {
        log.info('build','requesting source compile');
        return do_build(gyp,argv,callback);
    } else {
        var fallback_to_build = gyp.opts['fallback-to-build'] || gyp.opts.fallback_to_build;
        var should_do_fallback_build = fallback_to_build === package_json.name || (fallback_to_build === true || fallback_to_build === 'true');
        // but allow override from npm
        if (process.env.npm_config_argv) {
            var cooked = JSON.parse(process.env.npm_config_argv).cooked;
            var match = cooked.indexOf("--fallback-to-build");
            if (match > -1 && cooked.length > match && cooked[match+1] == "false") {
                should_do_fallback_build = false;
                log.info('install','Build fallback disabled via npm flag: --fallback-to-build=false');
            }
        }
        var opts;
        try {
            opts = versioning.evaluate(package_json, gyp.opts);
        } catch (err) {
            return callback(err);
        }
        var from = opts.hosted_tarball;
        var to = opts.module_path;
        var binary_module = path.join(to,opts.module_name + '.node');
        if (existsAsync(binary_module,function(found) {
            if (found && !update_binary) {
                testbinary(gyp, argv, function(err) {
                    if (err) {
                        console.error('['+package_json.name+'] ' + err.message);
                        log.error("Testing local pre-built binary failed, attempting to re-download");
                        place_binary(from,to,opts,function(err) {
                            if (err) {
                                if (should_do_fallback_build) {
                                    print_fallback_error(err,opts,package_json);
                                    return do_build(gyp,argv,callback);
                                } else {
                                    return callback(err);
                                }
                            } else {
                                console.log('['+package_json.name+'] Success: "' + binary_module + '" is reinstalled via remote');
                                return callback();
                            }
                        });
                    } else {
                        console.log('['+package_json.name+'] Success: "' + binary_module + '" already installed');
                        console.log('Pass --update-binary to reinstall or --build-from-source to recompile');
                        return callback();
                    }
                });
            } else {
                if (!update_binary) log.info('check','checked for "' + binary_module + '" (not found)');
                place_binary(from,to,opts,function(err) {
                    if (err && should_do_fallback_build) {
                        print_fallback_error(err,opts,package_json);
                        return do_build(gyp,argv,callback);
                    } else if (err) {
                        return callback(err);
                    } else {
                        testbinary(gyp, argv, function(err) {
                            if (err) {
                                if (no_rollback) {
                                    return callback(err);
                                }
                                gyp.opts.silent_clean = true;
                                clean(gyp, argv, function(error) {
                                    if (error) console.log(error);
                                    if (should_do_fallback_build) {
                                        console.error('['+package_json.name+'] ' + err.message);
                                        log.error("Testing pre-built binary failed, attempting to source compile");
                                        return do_build(gyp,argv,callback);
                                    } else {
                                        return callback(err);
                                    }
                                });
                            } else {
                                console.log('['+package_json.name+'] Success: "' + binary_module + '" is installed via remote');
                                return callback();
                            }
                        });
                    }
                });
            }
        }));
    }
}
<|endoftext|>