<commit_before>var constants = require('constants')
var hypercore = require('hypercore')
var sublevel = require('subleveldown')
var fs = require('fs')
var mkdirp = require('mkdirp')
var bulk = require('bulk-write-stream')
var path = require('path')
var deltas = require('delta-list')
var from = require('from2')
var through = require('through2')
var pump = require('pump')
var pumpify = require('pumpify')
var octal = require('octal')
var util = require('util')
var events = require('events')
var storage
var rabin
if (process.browser) rabin = through
else rabin = require('rabin')
if (process.browser) storage = require('./lib/browser-storage')
else storage = require('./lib/storage')
var messages = require('./lib/messages')

var DMODE = octal(755)
var FMODE = octal(644)

module.exports = Hyperdrive

function Hyperdrive (db) {
  if (!(this instanceof Hyperdrive)) return new Hyperdrive(db)
  var self = this

  this.db = db
  this.drives = sublevel(db, 'drives', {valueEncoding: 'binary'})
  this.core = hypercore(db, {
    storage: function (feed) {
      return storage(self, feed)
    }
  })
}

// Hyperdrive.prototype.list = function () {
//   return this.drives.createKeyStream()
// }

Hyperdrive.prototype.createPeerStream = function () {
  return this.core.createPeerStream()
}

Hyperdrive.prototype.get = function (id, folder) {
  if (!folder || !id) throw new Error('id and folder required')
  return new Archive(this, folder, id)
}

Hyperdrive.prototype.add = function (folder) {
  if (!folder) throw new Error('folder required')
  return new Archive(this, folder, null)
}

function Progress () {
  this.bytesRead = 0
  this.bytesTotal = 0
  events.EventEmitter.call(this)
}

util.inherits(Progress, events.EventEmitter)

Progress.prototype.end = function (err) {
  if (!err) return this.emit('end')
  if (this.listeners('error').length) this.emit('error', err)
  return
}

function Archive (drive, folder, id) {
  events.EventEmitter.call(this)

  var self = this

  this.id = id
  this.directory = folder
  this.core = drive.core
  this.entries = 0
  this.metadata = {type: 'hyperdrive'}
  this.stats = {
    bytesRead: 0,
    bytesDownloaded: 0,
    filesRead: 0,
    filesDownloaded: 0
  }

  this._first = true

  if (!id) {
    this.feed = this.core.add({filename: null})
  } else {
    this.feed = this.core.get(id)
    this.feed.get(0, onmetadata)
  }

  this.feed.on('put', function (block, data) {
    self.stats.bytesDownloaded += data.length
    self.emit('download', data, block)
  })

  this.feed.ready(function (err) {
    if (err) return
    self.entries = self.feed.blocks - 1
    self.emit('ready')
  })

  function onmetadata (err, json) {
    if (err) self.emit('error', err)

    var doc

    try {
      doc = JSON.parse(json.toString())
    } catch (err) {
      // do nothing
    }

    if (!doc || doc.type !== 'hyperdrive') {
      self.emit('error', new Error('feed is not a hyperdrive'))
    } else {
      self.metadata = doc
      self.emit('metadata')
    }
  }
}

util.inherits(Archive, events.EventEmitter)

Archive.prototype.close = function (cb) {
  this.feed.close(cb)
}

Archive.prototype.ready = function (cb) {
  this.feed.ready(cb)
}

Archive.prototype.download = function (i, cb) {
  if (!cb) cb = noop

  var ptr = 0
  var self = this
  var stats = new Progress()

  if (typeof i === 'number') this.entry(i, onentry)
  else onentry(null, i)

  return stats

  function onentry (err, entry) {
    if (err) return cb(err)

    var feed = self._getFeed(entry)
    if (!feed) return cb(null)

    feed.on('put', kick)
    feed.open(kick)
    stats.bytesTotal = entry.size

    if (feed._storage) onstorage(feed._storage)
    else feed.on('_storage', onstorage)

    function onstorage (storage) {
      if (!storage._bytesWritten) return
      storage._bytesWritten(function (_, bytes) {
        if (bytes) stats.bytesRead = bytes
      })
    }

    function kick (block, data) {
      if (data) stats.bytesRead += data.length
      if (!feed.blocks) return
      for (; ptr < feed.blocks; ptr++) {
        if (!feed.has(ptr)) return
      }

      var dest = join(self.directory, entry.name)
      if (process.browser) return done()

      fs.stat(dest, function (_, st) {
        feed.removeListener('put', kick)

        // the size check probably isn't nessary here...
        if (st && st.size === entry.size) return done(null)

        // duplicate - just copy it in
        mkdirp(path.dirname(dest), function () {
          pump(self.createFileStream(entry), fs.createWriteStream(dest), done)
        })
      })
    }

    function done (err) {
      if (!err) self.stats.filesDownloaded++
      stats.end(err)
      cb(err)
    }
  }
}

Archive.prototype.select = function (i, cb) {
  if (!cb) cb = noop

  var self = this

  if (typeof i === 'number') this.entry(i, onentry)
  else onentry(null, i)

  function onentry (err, entry) {
    if (err) return cb(err)
    if (!entry || !entry.link) return cb(null, null)
    cb(null, self._getFeed(entry))
  }
}

Archive.prototype.deselect = function (i, cb) {
  throw new Error('not yet implemented')
}

Archive.prototype.entry = function (i, cb) {
  this.feed.get(i + 1, function (err, data) {
    if (err) return cb(err)
    if (!data) return cb(null, null)
    cb(null, messages.Entry.decode(data))
  })
}

Archive.prototype.finalize = function (cb) {
  if (!cb) cb = noop
  var self = this
  this.feed.finalize(function (err) {
    if (err) return cb(err)
    self.id = self.feed.id
    self.entries = self.feed.blocks - 1
    cb()
  })
}

Archive.prototype.createEntryStream = function (opts) {
  if (!opts) opts = {}
  var start = opts.start || 0
  var limit = opts.limit || Infinity
  var self = this
  return from.obj(read)

  function read (size, cb) {
    if (limit-- === 0) return cb(null, null)
    self.entry(start++, cb)
  }
}

Archive.prototype._getFeed = function (entry) {
  if (!entry.link) return null

  var self = this
  var done = false
  var contentBlocks = entry.link.blocks - entry.link.index.length
  var feed = this.core.get(entry.link.id, {
    filename: join(this.directory, entry.name),
    index: deltas.unpack(entry.link.index),
    contentBlocks: contentBlocks
  })

  var ptr = 0

  feed.on('put', function (block, data) {
    self.emit('file-download', entry, data, block)
    kick()
  })

  feed.open(function (err) {
    if (err) return
    kick()
    for (var i = 0; i < entry.link.index.length; i++) {
      if (!feed.has(i + contentBlocks)) feed.want.push({block: i + contentBlocks, callback: noop, critical: true})
    }
  })

  return feed

  function kick () {
    if (done || !feed.blocks) return
    for (; ptr < feed.blocks; ptr++) {
      if (!feed.has(ptr)) return
    }
    done = true
    self.emit('file-downloaded', entry)
  }
}

Archive.prototype.createFileCursor = function (i, opts) {
  throw new Error('not yet implemented')
}

Archive.prototype.createFileStream = function (i, opts) { // TODO: expose random access stuff
  if (!opts) opts = {}
  var start = opts.start || 0
  var limit = opts.limit || Infinity
  var self = this
  var feed = null
  return from.obj(read)

  function read (size, cb) {
    if (feed) {
      if (limit-- === 0) return cb(null, null)
      feed.get(start++, cb)
      return
    }

    if (typeof i === 'number') self.entry(i, onentry)
    else onentry(null, i)

    function onentry (err, entry) {
      if (err) return cb(err)
      if (!entry.link) return cb(null, null)
      feed = self._getFeed(entry)
      limit = Math.min(limit, entry.link.blocks - entry.link.index.length)
      read(0, cb)
    }
  }
}

Archive.prototype.append = function (entry, opts, cb) {
  if (typeof opts === 'function') return this.append(entry, null, opts)
  if (typeof entry === 'string') entry = {name: entry, type: 'file'}
  if (!entry.name) throw new Error('entry.name is required')
  if (!entry.type && entry.mode) entry.type = modeToType(entry.mode)
  if (entry.type && !entry.mode) entry.mode = entry.type === 'directory' ? DMODE : FMODE
  if (!opts) opts = {}

  var self = this

  if (entry.type !== 'file') {
    append(null, cb)
    return null
  }

  if (opts.filename === true) opts.filename = entry.name

  var size = 0
  var stats = opts.stats
  var feed = this.core.add({filename: opts.filename && path.resolve(this.directory, opts.filename)})
  var stream = pumpify(rabin(), bulk(write, end))

  if (cb) {
    stream.on('error', cb)
    stream.on('finish', forward)
  }

  return stream

  function forward () {
    cb(null, entry)
  }

  function append (link, cb) {
    if (link) self.stats.filesRead++
    entry.size = size
    entry.link = link
    if (self._first) {
      // first block is gonna be a human readable metadata block
      // this makes updating the protocol easier in the future
      self.feed.append(JSON.stringify(self.metadata))
      self._first = false
    }
    self.feed.append(messages.Entry.encode(entry), done)

    function done (err) {
      if (err) return cb(err)
      self.entries++
      cb(null)
    }
  }

  function write (buffers, cb) {
    for (var i = 0; i < buffers.length; i++) {
      size += buffers[i].length
      if (stats) stats.bytesRead += buffers[i].length
      self.stats.bytesRead += buffers[i].length
    }
    feed.append(buffers, cb)
  }

  function end (cb) {
    feed.finalize(function (err) {
      if (err) return cb(err)
      if (!feed.id) return append(null, cb)

      var link = {
        id: feed.id,
        blocks: feed.blocks,
        index: deltas.pack(feed._storage._index)
      }

      append(link, cb)
    })
  }
}

Archive.prototype.appendFile = function (filename, name, cb) {
  if (typeof name === 'function') return this.appendFile(filename, null, name)
  if (!cb) cb = noop
  if (!name) name = filename

  var self = this
  var stats = new Progress()

  fs.lstat(filename, function (err, st) {
    if (err) return done(err)

    var opts = {filename: filename, stats: stats}
    var ws = self.append({
      name: name,
      mode: st.mode,
      size: 0,
      link: null
    }, opts, done)

    stats.bytesTotal = st.size

    if (ws) {
      var readStream = fs.createReadStream(path.resolve(self.directory, filename))
      pump(readStream, ws)
    }
  })

  return stats

  function done (err) {
    stats.end(err)
    cb(err)
  }
}

function noop () {}

function modeToType (mode) { // from tar-stream
  switch (mode & constants.S_IFMT) {
    case constants.S_IFBLK: return 'block-device'
    case constants.S_IFCHR: return 'character-device'
    case constants.S_IFDIR: return 'directory'
    case constants.S_IFIFO: return 'fifo'
    case constants.S_IFLNK: return 'symlink'
  }

  return 'file'
}

function join (a, b) {
  return path.join(a, path.resolve('/', b))
}
<commit_msg>do not count metadata in stats<commit_after>var constants = require('constants')
var hypercore = require('hypercore')
var sublevel = require('subleveldown')
var fs = require('fs')
var mkdirp = require('mkdirp')
var bulk = require('bulk-write-stream')
var path = require('path')
var deltas = require('delta-list')
var from = require('from2')
var through = require('through2')
var pump = require('pump')
var pumpify = require('pumpify')
var octal = require('octal')
var util = require('util')
var events = require('events')
var storage
var rabin
if (process.browser) rabin = through
else rabin = require('rabin')
if (process.browser) storage = require('./lib/browser-storage')
else storage = require('./lib/storage')
var messages = require('./lib/messages')

var DMODE = octal(755)
var FMODE = octal(644)

module.exports = Hyperdrive

function Hyperdrive (db) {
  if (!(this instanceof Hyperdrive)) return new Hyperdrive(db)
  var self = this

  this.db = db
  this.drives = sublevel(db, 'drives', {valueEncoding: 'binary'})
  this.core = hypercore(db, {
    storage: function (feed) {
      return storage(self, feed)
    }
  })
}

// Hyperdrive.prototype.list = function () {
//   return this.drives.createKeyStream()
// }

Hyperdrive.prototype.createPeerStream = function () {
  return this.core.createPeerStream()
}

Hyperdrive.prototype.get = function (id, folder) {
  if (!folder || !id) throw new Error('id and folder required')
  return new Archive(this, folder, id)
}

Hyperdrive.prototype.add = function (folder) {
  if (!folder) throw new Error('folder required')
  return new Archive(this, folder, null)
}

function Progress () {
  this.bytesRead = 0
  this.bytesTotal = 0
  events.EventEmitter.call(this)
}

util.inherits(Progress, events.EventEmitter)

Progress.prototype.end = function (err) {
  if (!err) return this.emit('end')
  if (this.listeners('error').length) this.emit('error', err)
  return
}

function Archive (drive, folder, id) {
  events.EventEmitter.call(this)

  var self = this

  this.id = id
  this.directory = folder
  this.core = drive.core
  this.entries = 0
  this.metadata = {type: 'hyperdrive'}
  this.stats = {
    bytesRead: 0,
    bytesDownloaded: 0,
    filesRead: 0,
    filesDownloaded: 0
  }

  this._first = true

  if (!id) {
    this.feed = this.core.add({filename: null})
  } else {
    this.feed = this.core.get(id)
    this.feed.get(0, onmetadata)
  }

  this.feed.on('put', function (block, data) {
    self.stats.bytesDownloaded += data.length
    self.emit('download', data, block)
  })

  this.feed.ready(function (err) {
    if (err) return
    self.entries = self.feed.blocks - 1
    self.emit('ready')
  })

  function onmetadata (err, json) {
    if (err) self.emit('error', err)

    var doc

    try {
      doc = JSON.parse(json.toString())
    } catch (err) {
      // do nothing
    }

    if (!doc || doc.type !== 'hyperdrive') {
      self.emit('error', new Error('feed is not a hyperdrive'))
    } else {
      self.metadata = doc
      self.emit('metadata')
    }
  }
}

util.inherits(Archive, events.EventEmitter)

Archive.prototype.close = function (cb) {
  this.feed.close(cb)
}

Archive.prototype.ready = function (cb) {
  this.feed.ready(cb)
}

Archive.prototype.download = function (i, cb) {
  if (!cb) cb = noop

  var ptr = 0
  var self = this
  var stats = new Progress()

  if (typeof i === 'number') this.entry(i, onentry)
  else onentry(null, i)

  return stats

  function onentry (err, entry) {
    if (err) return cb(err)

    var feed = self._getFeed(entry)
    if (!feed) return cb(null)

    feed.on('put', kick)
    feed.open(kick)
    stats.bytesTotal = entry.size

    if (feed._storage) onstorage(feed._storage)
    else feed.on('_storage', onstorage)

    function onstorage (storage) {
      if (!storage._bytesWritten) return
      storage._bytesWritten(function (_, bytes) {
        if (bytes) stats.bytesRead = bytes
      })
    }

    function kick (block, data) {
      if (!feed.blocks) return

      if (data && this === feed && entry.link && block < feed.blocks - entry.link.index.length) {
        stats.bytesRead += data.length
      }
      for (; ptr < feed.blocks; ptr++) {
        if (!feed.has(ptr)) return
      }

      var dest = join(self.directory, entry.name)
      if (process.browser) return done()

      fs.stat(dest, function (_, st) {
        feed.removeListener('put', kick)

        // the size check probably isn't nessary here...
        if (st && st.size === entry.size) return done(null)

        // duplicate - just copy it in
        mkdirp(path.dirname(dest), function () {
          pump(self.createFileStream(entry), fs.createWriteStream(dest), done)
        })
      })
    }

    function done (err) {
      if (!err) self.stats.filesDownloaded++
      stats.bytesRead = stats.bytesTotal
      stats.end(err)
      cb(err)
    }
  }
}

Archive.prototype.select = function (i, cb) {
  if (!cb) cb = noop

  var self = this

  if (typeof i === 'number') this.entry(i, onentry)
  else onentry(null, i)

  function onentry (err, entry) {
    if (err) return cb(err)
    if (!entry || !entry.link) return cb(null, null)
    cb(null, self._getFeed(entry))
  }
}

Archive.prototype.deselect = function (i, cb) {
  throw new Error('not yet implemented')
}

Archive.prototype.entry = function (i, cb) {
  this.feed.get(i + 1, function (err, data) {
    if (err) return cb(err)
    if (!data) return cb(null, null)
    cb(null, messages.Entry.decode(data))
  })
}

Archive.prototype.finalize = function (cb) {
  if (!cb) cb = noop
  var self = this
  this.feed.finalize(function (err) {
    if (err) return cb(err)
    self.id = self.feed.id
    self.entries = self.feed.blocks - 1
    cb()
  })
}

Archive.prototype.createEntryStream = function (opts) {
  if (!opts) opts = {}
  var start = opts.start || 0
  var limit = opts.limit || Infinity
  var self = this
  return from.obj(read)

  function read (size, cb) {
    if (limit-- === 0) return cb(null, null)
    self.entry(start++, cb)
  }
}

Archive.prototype._getFeed = function (entry) {
  if (!entry.link) return null

  var self = this
  var done = false
  var contentBlocks = entry.link.blocks - entry.link.index.length
  var feed = this.core.get(entry.link.id, {
    filename: join(this.directory, entry.name),
    index: deltas.unpack(entry.link.index),
    contentBlocks: contentBlocks
  })

  var ptr = 0

  feed.on('put', function (block, data) {
    self.emit('file-download', entry, data, block)
    kick()
  })

  feed.open(function (err) {
    if (err) return
    kick()
    for (var i = 0; i < entry.link.index.length; i++) {
      if (!feed.has(i + contentBlocks)) feed.want.push({block: i + contentBlocks, callback: noop, critical: true})
    }
  })

  return feed

  function kick () {
    if (done || !feed.blocks) return
    for (; ptr < feed.blocks; ptr++) {
      if (!feed.has(ptr)) return
    }
    done = true
    self.emit('file-downloaded', entry)
  }
}

Archive.prototype.createFileCursor = function (i, opts) {
  throw new Error('not yet implemented')
}

Archive.prototype.createFileStream = function (i, opts) { // TODO: expose random access stuff
  if (!opts) opts = {}
  var start = opts.start || 0
  var limit = opts.limit || Infinity
  var self = this
  var feed = null
  return from.obj(read)

  function read (size, cb) {
    if (feed) {
      if (limit-- === 0) return cb(null, null)
      feed.get(start++, cb)
      return
    }

    if (typeof i === 'number') self.entry(i, onentry)
    else onentry(null, i)

    function onentry (err, entry) {
      if (err) return cb(err)
      if (!entry.link) return cb(null, null)
      feed = self._getFeed(entry)
      limit = Math.min(limit, entry.link.blocks - entry.link.index.length)
      read(0, cb)
    }
  }
}

Archive.prototype.append = function (entry, opts, cb) {
  if (typeof opts === 'function') return this.append(entry, null, opts)
  if (typeof entry === 'string') entry = {name: entry, type: 'file'}
  if (!entry.name) throw new Error('entry.name is required')
  if (!entry.type && entry.mode) entry.type = modeToType(entry.mode)
  if (entry.type && !entry.mode) entry.mode = entry.type === 'directory' ? DMODE : FMODE
  if (!opts) opts = {}

  var self = this

  if (entry.type !== 'file') {
    append(null, cb)
    return null
  }

  if (opts.filename === true) opts.filename = entry.name

  var size = 0
  var stats = opts.stats
  var feed = this.core.add({filename: opts.filename && path.resolve(this.directory, opts.filename)})
  var stream = pumpify(rabin(), bulk(write, end))

  if (cb) {
    stream.on('error', cb)
    stream.on('finish', forward)
  }

  return stream

  function forward () {
    cb(null, entry)
  }

  function append (link, cb) {
    if (link) self.stats.filesRead++
    entry.size = size
    entry.link = link
    if (self._first) {
      // first block is gonna be a human readable metadata block
      // this makes updating the protocol easier in the future
      self.feed.append(JSON.stringify(self.metadata))
      self._first = false
    }
    self.feed.append(messages.Entry.encode(entry), done)

    function done (err) {
      if (err) return cb(err)
      self.entries++
      cb(null)
    }
  }

  function write (buffers, cb) {
    for (var i = 0; i < buffers.length; i++) {
      size += buffers[i].length
      if (stats) stats.bytesRead += buffers[i].length
      self.stats.bytesRead += buffers[i].length
    }
    feed.append(buffers, cb)
  }

  function end (cb) {
    feed.finalize(function (err) {
      if (err) return cb(err)
      if (!feed.id) return append(null, cb)

      var link = {
        id: feed.id,
        blocks: feed.blocks,
        index: deltas.pack(feed._storage._index)
      }

      append(link, cb)
    })
  }
}

Archive.prototype.appendFile = function (filename, name, cb) {
  if (typeof name === 'function') return this.appendFile(filename, null, name)
  if (!cb) cb = noop
  if (!name) name = filename

  var self = this
  var stats = new Progress()

  fs.lstat(filename, function (err, st) {
    if (err) return done(err)

    var opts = {filename: filename, stats: stats}
    var ws = self.append({
      name: name,
      mode: st.mode,
      size: 0,
      link: null
    }, opts, done)

    stats.bytesTotal = st.size

    if (ws) {
      var readStream = fs.createReadStream(path.resolve(self.directory, filename))
      pump(readStream, ws)
    }
  })

  return stats

  function done (err) {
    stats.end(err)
    cb(err)
  }
}

function noop () {}

function modeToType (mode) { // from tar-stream
  switch (mode & constants.S_IFMT) {
    case constants.S_IFBLK: return 'block-device'
    case constants.S_IFCHR: return 'character-device'
    case constants.S_IFDIR: return 'directory'
    case constants.S_IFIFO: return 'fifo'
    case constants.S_IFLNK: return 'symlink'
  }

  return 'file'
}

function join (a, b) {
  return path.join(a, path.resolve('/', b))
}
<|endoftext|>


<commit_before>'use strict';

var boot = require('loopback-boot');
var compile = boot.compile;
var execute = boot.execute;

module.exports = function glueLoopBackApps(app, options, callback) {

	// Get the current app instructions first, as it takes
	// the top precendence
	var instructions = compile(options);

	//Checking if datasources configuration has any dynamic configuration
	// parameter
	getUpdatedDSConfig(instructions.config,instructions.dataSources,options);

	if(options.subapps && Array.isArray(options.subapps)) {
		options.subapps.forEach(function (subapp) {
			//console.log(subapp);

			try {
				var moduleName = Object.keys(subapp)[0];
				var moduleOption = subapp[moduleName];
				var moduleObj = require(moduleName);
				mergeInstructions(instructions, moduleObj.glue.instructions,moduleOption);
			} catch (e) {
				console.log('error=',e.stack);
			}
		});
	}

	options.env = options.env || app.get('env');

	var start = app.start;

	app.start = function(cb) {
		var instance;
		execute(app, instructions, function(err) {
			if(err) {
				console.log('Error with loading configuration');
				cb(err,null);
			}
			instance = start.call(this);
      cb(null,instance);
		});
	};

	return callback(null, instructions);
};

var mergeInstructions = exports.mergeInstructions = function(instructions, subAppInstructions, options) {

	if(options.loadModels)
		mergeModels(instructions, subAppInstructions, options);
	if(options.loadDatasources)
		mergeDataSources(instructions, subAppInstructions, options);

};

var mergeModels = exports.mergeModels = function(instructions, subAppInstructions, options) {

	var parentModelMap = {};
	instructions.models = instructions.models || [];

	instructions.models.forEach(function(model) {
		parentModelMap[model.name] = model;
	});

	subAppInstructions.models = subAppInstructions.models || [];

	subAppInstructions.models.forEach(function(model) {
		if(!parentModelMap.hasOwnProperty(model.name)){
			instructions.models.push(model);
			console.log('Adding model from sub app: ',model.name);
		} else if(parentModelMap.hasOwnProperty(model.name) && !parentModelMap[model.name].definition) {
			parentModelMap[model.name].definition = model.definition;
			parentModelMap[model.name].sourceFile = model.sourceFile;
			console.log('Merging model from sub app: ',model.name);
		}
	});
};

var mergeDataSources = exports.mergeDataSources = function(instructions, subAppInstructions, options) {

	instructions.dataSources = instructions.dataSources || {};

	subAppInstructions.dataSources = subAppInstructions.dataSources || {};

	Object.keys(subAppInstructions.dataSources).forEach(function(dsName) {
		if(!instructions.dataSources.hasOwnProperty(dsName)) {
			instructions.dataSources[dsName] = subAppInstructions.dataSources[dsName];
			console.log('Adding dataSources from sub app: ',dsName);
		}
	});
};

var getUpdatedDSConfig = exports.getUpdatedDSConfig = function(config, datasources,options) {
	var DYNAMIC_CONFIG_PARAM = /(.*)\$\{(\w+)\}(.*)/;

	datasources = datasources || [];

	Object.keys(datasources).forEach(function(dsName){

		Object.keys(datasources[dsName]).forEach(function(paramName){
			var paramValue = datasources[dsName][paramName];

			//console.log('paramValue==',typeof paramValue, paramValue);
			if(typeof paramValue === 'string') {
				var match = paramValue.match(DYNAMIC_CONFIG_PARAM);
				if (match) {

					//console.log(paramValue, match.length,'--',match[0],'--',match[1],'--',match[2],'--',match[3]);
					var appValue = config[(match[2])] || options[match[2]];

					if (appValue !== undefined) {
						datasources[dsName][paramName] = match[1] + appValue + match[3];
					} else {
						console.warn('%s does not resolve to a valid value. ' +
							'"%s" must be resolvable by app.get().', paramName, match[2]);
					}
				}
			}
		})
	});
};
<commit_msg>checking for callback function<commit_after>'use strict';

var boot = require('loopback-boot');
var compile = boot.compile;
var execute = boot.execute;

module.exports = function glueLoopBackApps(app, options, callback) {

	// Get the current app instructions first, as it takes
	// the top precendence
	var instructions = compile(options);

	//Checking if datasources configuration has any dynamic configuration
	// parameter
	getUpdatedDSConfig(instructions.config,instructions.dataSources,options);

	if(options.subapps && Array.isArray(options.subapps)) {
		options.subapps.forEach(function (subapp) {
			//console.log(subapp);

			try {
				var moduleName = Object.keys(subapp)[0];
				var moduleOption = subapp[moduleName];
				var moduleObj = require(moduleName);
				mergeInstructions(instructions, moduleObj.glue.instructions,moduleOption);
			} catch (e) {
				console.log('error=',e.stack);
			}
		});
	}

	options.env = options.env || app.get('env');

	var start = app.start;

	app.start = function(cb) {
		var instance;
		execute(app, instructions, function(err) {
			if(err) {
				console.log('Error with loading configuration');
				cb(err,null);
			}
			instance = start.call(this);
      		cb && cb(null,instance);
		});
	};

	return callback(null, instructions);
};

var mergeInstructions = exports.mergeInstructions = function(instructions, subAppInstructions, options) {

	if(options.loadModels)
		mergeModels(instructions, subAppInstructions, options);
	if(options.loadDatasources)
		mergeDataSources(instructions, subAppInstructions, options);

};

var mergeModels = exports.mergeModels = function(instructions, subAppInstructions, options) {

	var parentModelMap = {};
	instructions.models = instructions.models || [];

	instructions.models.forEach(function(model) {
		parentModelMap[model.name] = model;
	});

	subAppInstructions.models = subAppInstructions.models || [];

	subAppInstructions.models.forEach(function(model) {
		if(!parentModelMap.hasOwnProperty(model.name)){
			instructions.models.push(model);
			console.log('Adding model from sub app: ',model.name);
		} else if(parentModelMap.hasOwnProperty(model.name) && !parentModelMap[model.name].definition) {
			parentModelMap[model.name].definition = model.definition;
			parentModelMap[model.name].sourceFile = model.sourceFile;
			console.log('Merging model from sub app: ',model.name);
		}
	});
};

var mergeDataSources = exports.mergeDataSources = function(instructions, subAppInstructions, options) {

	instructions.dataSources = instructions.dataSources || {};

	subAppInstructions.dataSources = subAppInstructions.dataSources || {};

	Object.keys(subAppInstructions.dataSources).forEach(function(dsName) {
		if(!instructions.dataSources.hasOwnProperty(dsName)) {
			instructions.dataSources[dsName] = subAppInstructions.dataSources[dsName];
			console.log('Adding dataSources from sub app: ',dsName);
		}
	});
};

var getUpdatedDSConfig = exports.getUpdatedDSConfig = function(config, datasources,options) {
	var DYNAMIC_CONFIG_PARAM = /(.*)\$\{(\w+)\}(.*)/;

	datasources = datasources || [];

	Object.keys(datasources).forEach(function(dsName){

		Object.keys(datasources[dsName]).forEach(function(paramName){
			var paramValue = datasources[dsName][paramName];

			//console.log('paramValue==',typeof paramValue, paramValue);
			if(typeof paramValue === 'string') {
				var match = paramValue.match(DYNAMIC_CONFIG_PARAM);
				if (match) {

					//console.log(paramValue, match.length,'--',match[0],'--',match[1],'--',match[2],'--',match[3]);
					var appValue = config[(match[2])] || options[match[2]];

					if (appValue !== undefined) {
						datasources[dsName][paramName] = match[1] + appValue + match[3];
					} else {
						console.warn('%s does not resolve to a valid value. ' +
							'"%s" must be resolvable by app.get().', paramName, match[2]);
					}
				}
			}
		})
	});
};
<|endoftext|>


<commit_before>var env = process.env.NODE_ENV || 'dev'
var path = require('path')

if (env == 'dev') {
  require('dotenv').load()
  var twiMLUrl = "http://01c7d74d.ngrok.io/twilioVoice"
} else {
  // var twiMLUrl = path.join(__dirname, 'twilioVoice')
  var twiMLUrl = "https://callcenteranalytics.herokuapp.com/twilioVoice"
}

debugger
var express = require('express')
var app = express()
var bodyParser = require('body-parser')
var urlencoded = bodyParser.urlencoded({extended: false})
var mongoose = require('mongoose')
var async = require('async')

app.use(urlencoded)
app.use(express.static(path.join(__dirname, 'public')))
app.set('views', path.join(__dirname, 'views'))
app.set('view engine', 'ejs')

// app.locals({
//   makeTable  : function(obj) {
//     var template = fs.readFileSync('makeList.ejs', 'utf-8')
//     return ejs.render(template, list)
//   }
// })

var port = process.env.PORT || 5000
var uristring = process.env.MONGOLAB_URI || process.env.MONGOHQ_URL || 'mongodb://localhost/test'

var havenondemand = require('havenondemand')
var hodClient = new havenondemand.HODClient(process.env.HOD_APIKEY)

var twilioClient = require('twilio')(process.env.TWILIO_ACCOUNT_SID, process.env.TWILIO_AUTH_TOKEN)

var languageObj ={'en-US-tel': {'regular': 'US English', 'sentiment-analysis': 'eng'}, 'en-GB-tel': {'regular': 'British English', 'sentiment-analysis': 'eng'}, 'es-ES-tel': {'regular': 'European Spanish', 'sentiment-analysis': 'spa'}, 'fr-FR-tel': {'regular': 'French', 'sentiment-analysis': 'fre'}}

mongoose.connect(uristring, function (err, res) {
  if (err) {
  console.log ('ERROR connecting to: ' + uristring + '. ' + err);
  } else {
  console.log ('Succeeded connected to: ' + uristring);
  }
})

var schema = new mongoose.Schema({
  created: { type: Date, default: Date.now },
  processed: { type: Boolean, default: false },
  processing: { type: Boolean, default: false },
  error: { type: Boolean, default: false },
  indexed: { type: Boolean, default: false },
  audio: { type: Boolean, default: false },
  obtainingAudio: { type: Boolean, default: false },
  concepts: mongoose.Schema.Types.Mixed,
  sentiments: mongoose.Schema.Types.Mixed,
  entities: mongoose.Schema.Types.Mixed,
  entitiesSelected: [],
  confidence: Number,
  language: String,
  indexReference: String,
  ApiVersion: String,
  TranscriptionType: String,
  TranscriptionUrl: String,
  TranscriptionSid: String,
  Called: String,
  RecordingSid: String,
  CallStatus: String,
  RecordingUrl: String,
  From: String,
  Direction: String,
  url: String,
  AccountSid: String,
  TranscriptionText: String,
  Caller: String,
  TranscriptionStatus: String,
  CallSid: String,
  To: String
})

var Call = mongoose.model('Call', schema);

app.get('/', function(req, res) {
  var query = Call.find({})
  query.exec(function(err, calls) {
    if (err) {
      return handleError(err)
    } else {
      res.render('index', {
        calls: calls
      })
    }
  })
})

app.get('/about', function(req, res) {
  res.render('about', {

  })
})

app.post('/search', function(req, res) {
  debugger
  var searchText = req.body.search
  var data = {text: searchText, index: process.env.HOD_INDEX_NAME, print: 'all'}
  hodClient.call('querytextindex', data, function(err, resp, body) {
    if (resp) {
      if (resp.body) {
        if (resp.body.status == 'failed') {
          var documents = []
        } else {
          debugger
          var documents = resp.body.documents
        }
        res.render('search_results', {
          calls: documents
        })
      }
    }
  })
})

app.get('/findSimilar', function(req, res) {
  debugger
  var indexReference = req.query.indexReference
  var data = {index_reference: indexReference, indexes: process.env.HOD_INDEX_NAME, print: 'all'}
  hodClient.call('findsimilar', data, function(err, resp, body) {
    if (resp) {
      if (resp.body) {
        debugger
        var documents = resp.body.documents
        res.render('find_similar_results', {
          calls: documents
        })
      }
    }
  })
})

app.get('/callcenter', function(req, res) {
  res.render('callcenter', {})
})

app.get('/call/:CallSid', function(req, res) {
  var CallSid = req.params.CallSid
  var query = Call.findOne({'CallSid': CallSid})
  query.exec(function(err, call) {
    if (err) {
      return handleError(err)
    } else {
      var language = languageObj[call.language]['regular']
      res.render('call', {
        call: call,
        language: language
      })
    }
  })
})

function obtainAudio(CallSid) {
  console.log("Obtaining audio")
  console.log(CallSid)
  twilioClient.calls(CallSid).get(function(err1, call) {
    console.log(call)
    if (err1) {
      console.log(err1)
    } else {
      var dateCreated = call.date_created
      twilioClient.recordings.get({
        callSid: CallSid,
      }, function(err2, data) {
        if (err2) {
          console.log(err2)
        } else {
          if (data.recordings.length > 0) { //if there is a recording
            console.log(data.recordings)
            data.recordings.forEach(function(recording) {
              debugger
              var recordingUrl = "https://api.twilio.com"+recording.uri.split(".")[0]+".mp3"
              Call.update({'CallSid': CallSid}, {
                audio: true,
                RecordingUrl: recordingUrl,
                dateCreated: dateCreated
              }, function(err, numberAffected, rawResponse) {
                console.log("Audio file obtained for: " + CallSid)
              })
            })
          } else {
            obtainAudio(CallSid)
          }
        }
      })
    }
  })
}

app.get('/processCall', function(req, res) {
  debugger
  var CallSid = req.query.CallSid
  var language = req.query.language
  var entityType = req.query.entity_type
  console.log("Processing call: " + CallSid)
  Call.update({'CallSid': CallSid}, {
    processing: true,
    language: language,
    entitiesSelected: entityType
  }, function(error1, numberAffected, rawResponse) {
    //
    if (error1) {
      console.log(err)
    } else {
      Call.findOne({ CallSid: CallSid }, function (error2, doc){
        // doc.name = 'jason borne'
        // doc.visits.$inc()
        // doc.save()
        if (error2) {

        } else {
          var recordingUrl = doc.RecordingUrl
          var To = doc.To
          var From = doc.From
          var dateCreated = doc.dateCreated
          //
          var data1 = {url: recordingUrl, language: language, interval: 0}
          // debugger
          hodClient.call('recognizespeech', data1, true, function(err1, resp1, body1) {
            var jobID = resp1.body.jobID
            // debugger
            getAsyncResult(jobID, function(body) {
              // debugger
              if (body == 'failed') {
                createError(CallSid)
              } else {
                //continue
                // var text = body.actions[0].result.document[0].content
                var textSnippets = body.actions[0].result.document
                //
                processText(textSnippets, function(textObj) {
                  // debugger
                  var text = textObj.text
                  var confidenceAggregate = textObj.aggregate
                  //
                  console.log("Text: " + text)
                  if (text == "") {
                    createError(CallSid)
                  } else {
                    // HOD stuff
                    var sentimentAnalysisLanguage = languageObj[language]['sentiment-analysis']
                    var data2 = {text: text, language: sentimentAnalysisLanguage}
                    hodClient.call('analyzesentiment', data2, function(err2, resp2, body2) {
                      // debugger
                      console.log("Analyzed sentiment")
                      var sentimentResponse = body2
                      var aggregateSentiment = resp2.body.aggregate.sentiment
                      var aggregateScore = resp2.body.aggregate.score
                      hodClient.call('extractconcepts', data2, function(err3, resp3, body3) {
                        // debugger
                        console.log("Extracted concepts")
                        var conceptsResponse = body3
                        var data3 = {text: text, entity_type: entityType}
                        hodClient.call('extractentities', data3, function(err4, resp4, body4) {
                          console.log("Extracted entities")
                          var entityResponse = body4
                          //
                          //
                          var json = {
                            document: [
                              {
                                // title: counter.toString(),
                                // body: body
                                content: text,
                                CallSid: CallSid,
                                // sentiments: sentimentResponse,
                                aggregate_sentiment: aggregateSentiment,
                                aggregate_score: aggregateScore,
                                // concepts: conceptsResponse,
                                RecordingUrl: recordingUrl,
                                TranscriptionText: text,
                                confidence: confidenceAggregate,
                                From: From,
                                To: To,
                                date: dateCreated,
                                language: language
                              }
                            ]
                          }
                          var data4 = {
                            index: process.env.HOD_INDEX_NAME,
                            json: JSON.stringify(json)
                          }
                          // debugger
                          hodClient.call('addtotextindex', data4, function(err5, resp5, body5) {
                            // mongo
                            // debugger
                            var indexReference = resp5.body.references[0].reference
                            Call.update({'CallSid': CallSid}, {
                              text: text,
                              concepts: conceptsResponse,
                              sentiments: sentimentResponse,
                              entities: entityResponse,
                              RecordingUrl: recordingUrl,
                              TranscriptionText: text,
                              indexed: true,
                              processed: true,
                              confidence: confidenceAggregate,
                              indexReference: indexReference
                            }, function(err, numberAffected, rawResponse) {
                              console.log("Processed")
                            })
                            //
                          })
                          //
                        })
                      })
                    })
                  }
                  // HOD stuff
                  //
                })
              }
            })
          })
          //

        }
      })
    }
  })
})

function getAsyncResult(jobID, callback) {
  hodClient.getJobStatus(jobID, function(err, resp, body) {
    // debugger
    if (resp.body['status'] != 'finished') {
      if (resp.body.actions[0].status == 'failed') {
        callback('failed')
      } else {
        getAsyncResult(jobID, callback)
      }
    } else {
      callback(body)
    }
  })
}

function processText(textSnippetsArray, callback) {
  var oldAverage = 0
  var newAverage
  var counter = 1
  var text = ''
  async.each(textSnippetsArray, function(textInformation, c) {
    // debugger
    var confidence = textInformation['confidence']
    text += textInformation['content']
    newAverage = oldAverage * (counter-1)/counter + confidence/counter;   // New average = old average * (n-1)/n + new value /n
    oldAverage = newAverage; //set equal to new average for next go around of calling this function
    counter += 1
    console.log("Average: " + newAverage)
    console.log("confidence: " + confidence)
    if (counter > textSnippetsArray.length) {
      callback({aggregate: newAverage, text: text})
    }
    text += ' '
  }, function(err) {})
}

function createError(callSid) {
  Call.update({'CallSid': callSid}, {
    error: true
  }, function(err, numberAffected, rawResponse) {

  })
}

app.post('/makeCall', function(req, res) {
  // var query = req.query
  var phonenumber = req.body.phonenumber
  // var phoneNumber = query.phoneNumber
  twilioClient.calls.create({
      url: twiMLUrl,
      to: phonenumber,
      from: process.env.TWILIO_PHONE_NUMBER
  }, function(err, call) {
      if (err) {
        console.log(err)
      } else {
        console.log(call)
        process.stdout.write(call.sid)
        var callObj = {}
        callObj['CallSid'] = call.sid
        callObj['To'] = call.to
        callObj['From'] = call.from
        var callMongo = new Call (callObj)
        callMongo.save(function (err) {
          if (err)  {
            console.log ('Error on save!')
          } else { //loop to obtain audio
            obtainAudio(call.sid)
          }
        })
      }
  })
  res.redirect('/')
})

// Function for getting Twilio XML file
app.post('/twilioVoice', function(req, res) {
  var fileName = path.join(__dirname, '/twilioVoice.xml')
  res.sendFile(fileName, function (err) {
    if (err) {
      console.log(err)
    }
  })
})

app.listen(port, function() {
  console.log('Listening on port: ' + port)
})
<commit_msg>updated error handling for processing request twice<commit_after>var env = process.env.NODE_ENV || 'dev'
var path = require('path')

if (env == 'dev') {
  require('dotenv').load()
  var twiMLUrl = "http://01c7d74d.ngrok.io/twilioVoice"
} else {
  // var twiMLUrl = path.join(__dirname, 'twilioVoice')
  var twiMLUrl = "https://callcenteranalytics.herokuapp.com/twilioVoice"
}

debugger
var express = require('express')
var app = express()
var bodyParser = require('body-parser')
var urlencoded = bodyParser.urlencoded({extended: false})
var mongoose = require('mongoose')
var async = require('async')

app.use(urlencoded)
app.use(express.static(path.join(__dirname, 'public')))
app.set('views', path.join(__dirname, 'views'))
app.set('view engine', 'ejs')

// app.locals({
//   makeTable  : function(obj) {
//     var template = fs.readFileSync('makeList.ejs', 'utf-8')
//     return ejs.render(template, list)
//   }
// })

var port = process.env.PORT || 5000
var uristring = process.env.MONGOLAB_URI || process.env.MONGOHQ_URL || 'mongodb://localhost/test'

var havenondemand = require('havenondemand')
var hodClient = new havenondemand.HODClient(process.env.HOD_APIKEY)

var twilioClient = require('twilio')(process.env.TWILIO_ACCOUNT_SID, process.env.TWILIO_AUTH_TOKEN)

var languageObj ={'en-US-tel': {'regular': 'US English', 'sentiment-analysis': 'eng'}, 'en-GB-tel': {'regular': 'British English', 'sentiment-analysis': 'eng'}, 'es-ES-tel': {'regular': 'European Spanish', 'sentiment-analysis': 'spa'}, 'fr-FR-tel': {'regular': 'French', 'sentiment-analysis': 'fre'}}

mongoose.connect(uristring, function (err, res) {
  if (err) {
  console.log ('ERROR connecting to: ' + uristring + '. ' + err);
  } else {
  console.log ('Succeeded connected to: ' + uristring);
  }
})

var schema = new mongoose.Schema({
  created: { type: Date, default: Date.now },
  processed: { type: Boolean, default: false },
  processing: { type: Boolean, default: false },
  error: { type: Boolean, default: false },
  indexed: { type: Boolean, default: false },
  audio: { type: Boolean, default: false },
  obtainingAudio: { type: Boolean, default: false },
  concepts: mongoose.Schema.Types.Mixed,
  sentiments: mongoose.Schema.Types.Mixed,
  entities: mongoose.Schema.Types.Mixed,
  entitiesSelected: [],
  confidence: Number,
  language: String,
  indexReference: String,
  ApiVersion: String,
  TranscriptionType: String,
  TranscriptionUrl: String,
  TranscriptionSid: String,
  Called: String,
  RecordingSid: String,
  CallStatus: String,
  RecordingUrl: String,
  From: String,
  Direction: String,
  url: String,
  AccountSid: String,
  TranscriptionText: String,
  Caller: String,
  TranscriptionStatus: String,
  CallSid: String,
  To: String
})

var Call = mongoose.model('Call', schema);

app.get('/', function(req, res) {
  var query = Call.find({})
  query.exec(function(err, calls) {
    if (err) {
      return handleError(err)
    } else {
      res.render('index', {
        calls: calls
      })
    }
  })
})

app.get('/about', function(req, res) {
  res.render('about', {

  })
})

app.post('/search', function(req, res) {
  debugger
  var searchText = req.body.search
  var data = {text: searchText, index: process.env.HOD_INDEX_NAME, print: 'all'}
  hodClient.call('querytextindex', data, function(err, resp, body) {
    if (resp) {
      if (resp.body) {
        if (resp.body.status == 'failed') {
          var documents = []
        } else {
          debugger
          var documents = resp.body.documents
        }
        res.render('search_results', {
          calls: documents
        })
      }
    }
  })
})

app.get('/findSimilar', function(req, res) {
  debugger
  var indexReference = req.query.indexReference
  var data = {index_reference: indexReference, indexes: process.env.HOD_INDEX_NAME, print: 'all'}
  hodClient.call('findsimilar', data, function(err, resp, body) {
    if (resp) {
      if (resp.body) {
        debugger
        var documents = resp.body.documents
        res.render('find_similar_results', {
          calls: documents
        })
      }
    }
  })
})

app.get('/callcenter', function(req, res) {
  res.render('callcenter', {})
})

app.get('/call/:CallSid', function(req, res) {
  var CallSid = req.params.CallSid
  var query = Call.findOne({'CallSid': CallSid})
  query.exec(function(err, call) {
    if (err) {
      return handleError(err)
    } else {
      var language = languageObj[call.language]['regular']
      res.render('call', {
        call: call,
        language: language
      })
    }
  })
})

function obtainAudio(CallSid) {
  console.log("Obtaining audio")
  console.log(CallSid)
  twilioClient.calls(CallSid).get(function(err1, call) {
    console.log(call)
    if (err1) {
      console.log(err1)
    } else {
      var dateCreated = call.date_created
      twilioClient.recordings.get({
        callSid: CallSid,
      }, function(err2, data) {
        if (err2) {
          console.log(err2)
        } else {
          if (data.recordings.length > 0) { //if there is a recording
            console.log(data.recordings)
            data.recordings.forEach(function(recording) {
              debugger
              var recordingUrl = "https://api.twilio.com"+recording.uri.split(".")[0]+".mp3"
              Call.update({'CallSid': CallSid}, {
                audio: true,
                RecordingUrl: recordingUrl,
                dateCreated: dateCreated
              }, function(err, numberAffected, rawResponse) {
                console.log("Audio file obtained for: " + CallSid)
              })
            })
          } else {
            obtainAudio(CallSid)
          }
        }
      })
    }
  })
}

app.get('/processCall', function(req, res) {
  debugger
  var CallSid = req.query.CallSid
  var language = req.query.language
  var entityType = req.query.entity_type
  Call.findOne({ CallSid: CallSid }, function(error, doc1) {
    if (doc1.processing == false) {
      //
      console.log("Processing call: " + CallSid)
      Call.update({'CallSid': CallSid}, {
        processing: true,
        language: language,
        entitiesSelected: entityType
      }, function(error1, numberAffected, rawResponse) {
        //
        if (error1) {
          console.log(error1)
        } else {
          Call.findOne({ CallSid: CallSid }, function (error2, doc){
            // doc.name = 'jason borne'
            // doc.visits.$inc()
            // doc.save()
            if (error2) {

            } else {
              var recordingUrl = doc.RecordingUrl
              var To = doc.To
              var From = doc.From
              var dateCreated = doc.dateCreated
              //
              var data1 = {url: recordingUrl, language: language, interval: 0}
              // debugger
              hodClient.call('recognizespeech', data1, true, function(err1, resp1, body1) {
                var jobID = resp1.body.jobID
                // debugger
                getAsyncResult(jobID, function(body) {
                  // debugger
                  if (body == 'failed') {
                    createError(CallSid)
                  } else {
                    //continue
                    // var text = body.actions[0].result.document[0].content
                    var textSnippets = body.actions[0].result.document
                    //
                    processText(textSnippets, function(textObj) {
                      // debugger
                      var text = textObj.text
                      var confidenceAggregate = textObj.aggregate
                      //
                      console.log("Text: " + text)
                      if (text == "") {
                        createError(CallSid)
                      } else {
                        // HOD stuff
                        var sentimentAnalysisLanguage = languageObj[language]['sentiment-analysis']
                        var data2 = {text: text, language: sentimentAnalysisLanguage}
                        hodClient.call('analyzesentiment', data2, function(err2, resp2, body2) {
                          // debugger
                          console.log("Analyzed sentiment")
                          var sentimentResponse = body2
                          var aggregateSentiment = resp2.body.aggregate.sentiment
                          var aggregateScore = resp2.body.aggregate.score
                          hodClient.call('extractconcepts', data2, function(err3, resp3, body3) {
                            // debugger
                            console.log("Extracted concepts")
                            var conceptsResponse = body3
                            var data3 = {text: text, entity_type: entityType}
                            hodClient.call('extractentities', data3, function(err4, resp4, body4) {
                              console.log("Extracted entities")
                              var entityResponse = body4
                              //
                              //
                              var json = {
                                document: [
                                  {
                                    // title: counter.toString(),
                                    // body: body
                                    content: text,
                                    CallSid: CallSid,
                                    // sentiments: sentimentResponse,
                                    aggregate_sentiment: aggregateSentiment,
                                    aggregate_score: aggregateScore,
                                    // concepts: conceptsResponse,
                                    RecordingUrl: recordingUrl,
                                    TranscriptionText: text,
                                    confidence: confidenceAggregate,
                                    From: From,
                                    To: To,
                                    date: dateCreated,
                                    language: language
                                  }
                                ]
                              }
                              var data4 = {
                                index: process.env.HOD_INDEX_NAME,
                                json: JSON.stringify(json)
                              }
                              // debugger
                              hodClient.call('addtotextindex', data4, function(err5, resp5, body5) {
                                // mongo
                                // debugger
                                var indexReference = resp5.body.references[0].reference
                                Call.update({'CallSid': CallSid}, {
                                  text: text,
                                  concepts: conceptsResponse,
                                  sentiments: sentimentResponse,
                                  entities: entityResponse,
                                  RecordingUrl: recordingUrl,
                                  TranscriptionText: text,
                                  indexed: true,
                                  processed: true,
                                  confidence: confidenceAggregate,
                                  indexReference: indexReference
                                }, function(err, numberAffected, rawResponse) {
                                  console.log("Processed")
                                })
                                //
                              })
                              //
                            })
                          })
                        })
                      }
                      // HOD stuff
                      //
                    })
                  }
                })
              })
              //

            }
          })
        }
      })

      //
    } else {
      console.log("Call already processing! " + CallSid)
      res.redirect('/')
    }
  })
})

function getAsyncResult(jobID, callback) {
  hodClient.getJobStatus(jobID, function(err, resp, body) {
    // debugger
    if (resp.body['status'] != 'finished') {
      if (resp.body.actions[0].status == 'failed') {
        callback('failed')
      } else {
        getAsyncResult(jobID, callback)
      }
    } else {
      callback(body)
    }
  })
}

function processText(textSnippetsArray, callback) {
  var oldAverage = 0
  var newAverage
  var counter = 1
  var text = ''
  async.each(textSnippetsArray, function(textInformation, c) {
    // debugger
    var confidence = textInformation['confidence']
    text += textInformation['content']
    newAverage = oldAverage * (counter-1)/counter + confidence/counter;   // New average = old average * (n-1)/n + new value /n
    oldAverage = newAverage; //set equal to new average for next go around of calling this function
    counter += 1
    console.log("Average: " + newAverage)
    console.log("confidence: " + confidence)
    if (counter > textSnippetsArray.length) {
      callback({aggregate: newAverage, text: text})
    }
    text += ' '
  }, function(err) {})
}

function createError(callSid) {
  Call.update({'CallSid': callSid}, {
    error: true
  }, function(err, numberAffected, rawResponse) {

  })
}

app.post('/makeCall', function(req, res) {
  // var query = req.query
  var phonenumber = req.body.phonenumber
  // var phoneNumber = query.phoneNumber
  twilioClient.calls.create({
      url: twiMLUrl,
      to: phonenumber,
      from: process.env.TWILIO_PHONE_NUMBER
  }, function(err, call) {
      if (err) {
        console.log(err)
      } else {
        console.log(call)
        process.stdout.write(call.sid)
        var callObj = {}
        callObj['CallSid'] = call.sid
        callObj['To'] = call.to
        callObj['From'] = call.from
        var callMongo = new Call (callObj)
        callMongo.save(function (err) {
          if (err)  {
            console.log ('Error on save!')
          } else { //loop to obtain audio
            obtainAudio(call.sid)
          }
        })
      }
  })
  res.redirect('/')
})

// Function for getting Twilio XML file
app.post('/twilioVoice', function(req, res) {
  var fileName = path.join(__dirname, '/twilioVoice.xml')
  res.sendFile(fileName, function (err) {
    if (err) {
      console.log(err)
    }
  })
})

app.listen(port, function() {
  console.log('Listening on port: ' + port)
})
<|endoftext|>


<commit_before>'use strict';

var del = require('del');
var GlobalsFormatter = require('es6-module-transpiler-globals-formatter');
var gulp = require('gulp');
var gutil = require('gulp-util');
var jspm = require('jspm');
var jspmCore = require('jspm/lib/core');
var karma = require('karma').server;
var merge = require('merge');
var open = require('open');
var path = require('path');
var plugins = require('gulp-load-plugins')();
var renamer = require('gulp-es6-imports-renamer');
var runSequence = require('run-sequence');
var sourcemaps = require('gulp-sourcemaps');
var to5 = require('gulp-6to5');
var transpile = require('gulp-es6-module-transpiler');

function handleError(error) {
  console.error(error.toString());

  this.emit('end');
}

module.exports = function(options) {
  var bundleFileName = options.bundleFileName;

  gulp.task('build', function(done) {
    runSequence('clean', ['soy', 'copy'], ['build:globals', 'build:min'], done);
  });

  gulp.task('build:globals', ['jspm'], function() {
    return gulp.src('src/**/*.js')
      .pipe(sourcemaps.init())
      .pipe(renamer({
        basePath: process.cwd(),
        configPath: path.resolve('config.js')
      })).on('error', handleError)
      .pipe(transpile({
        basePath: process.cwd(),
        bundleFileName: bundleFileName,
        formatter: new GlobalsFormatter({
          globalName: 'aui'
        })
      }))
      .pipe(to5({
        blacklist: 'useStrict'
      })).on('error', handleError)
      .pipe(sourcemaps.write('./'))
      .pipe(gulp.dest('build'));
  });

  gulp.task('build:min', ['build:globals'], function() {
    return gulp.src(path.join('build/', bundleFileName))
      .pipe(plugins.rename(function(path) {
        path.basename += '-min';
      }))
      .pipe(plugins.uglify({
        preserveComments: 'some'
      }))
      .pipe(banner(options.pkg))
      .pipe(gulp.dest('build'));
  });

  gulp.task('clean', function(done) {
    del(['build'], done);
  });

  gulp.task('copy', function() {
    return gulp.src('src/**/*.css')
      .pipe(gulp.dest('build'));
  });

  gulp.task('jspm', function(done) {
    jspm.promptDefaults(true);
    jspm.install(true, {
      lock: true
    }).then(function() {
      return jspmCore.checkDlLoader();
    }).then(function() {
      return jspmCore.setMode('local');
    }).then(function() {
      gutil.log(gutil.colors.cyan('Install complete'));
      done();
    }, function(err) {
      gutil.log(gutil.colors.red('err', err.stack || err))
      gutil.log(gutil.colors.red('Installation changes not saved.'));
      done();
    });
  });

  gulp.task('lint', function() {
    return gulp.src(['src/**/*.js', 'test/**/*.js'])
      .pipe(plugins.jshint())
      .pipe(plugins.jshint.reporter(require('jshint-stylish')));
  });

  gulp.task('soy', function() {
    return gulp.src('src/*.soy')
      .pipe(plugins.soynode())
      .pipe(plugins.wrapper({
        header: '/* jshint ignore:start */',
        footer: 'export default templates;\n/* jshint ignore:end */'
      }))
      .pipe(plugins.ignore.exclude('*.soy'))
      .pipe(gulp.dest('src'));
  });

  gulp.task('test', function(done) {
    return runSequence('test:unit', 'test:complexity', done);
  });

  gulp.task('test:complexity', function() {
    return gulp.src(['src/**/*.js', '!src/**/*.soy.js', '!src/promise/Promise.js', 'test/**/*.js'])
      .pipe(to5())
      .pipe(plugins.complexity({
        halstead: [15, 15, 20]
      }));
  });

  gulp.task('test:unit', ['jspm'], function(done) {
    runKarma({}, done);
  });

  gulp.task('test:coverage', ['jspm'], function(done) {
    runKarma({}, function() {
      open(path.resolve('coverage/lcov/lcov-report/index.html'));
      done();
    });
  });

  gulp.task('test:browsers', ['jspm'], function(done) {
    runKarma({
      browsers: ['Chrome', 'Firefox', 'Safari', 'IE9 - Win7', 'IE10 - Win7', 'IE11 - Win7']
    }, done);
  });

  gulp.task('test:saucelabs', ['jspm'], function(done) {
    var launchers = {
      sl_chrome: {
            base: 'SauceLabs',
            browserName: 'chrome'
        },
        sl_firefox: {
            base: 'SauceLabs',
            browserName: 'firefox'
        },
        sl_ie_9: {
            base: 'SauceLabs',
            browserName: 'internet explorer',
            platform: 'Windows 7',
            version: '9'
        },
        sl_ie_10: {
            base: 'SauceLabs',
            browserName: 'internet explorer',
            platform: 'Windows 7',
            version: '10'
        },
        sl_ie_11: {
            base: 'SauceLabs',
            browserName: 'internet explorer',
            platform: 'Windows 8.1',
            version: '11'
        }
    };

    runKarma({
        browsers: Object.keys(launchers),

        browserDisconnectTimeout: 10000,
        browserDisconnectTolerance: 2,
        browserNoActivityTimeout: 240000,

        captureTimeout: 240000,
        customLaunchers: launchers,

        reporters: ['coverage', 'junit', 'progress', 'saucelabs'],

        sauceLabs: {
            testName: 'AlloyUI tests',
            recordScreenshots: false,
            startConnect: true,
            connectOptions: {
                port: 5757,
                'selenium-version': '2.41.0',
                logfile: 'sauce_connect.log'
            }
        }
    }, done);
  });

  gulp.task('test:watch', ['jspm'], function(done) {
    runKarma({
      singleRun: false
    }, done);
  });

  gulp.task('watch', ['build'], function() {
    gulp.watch('src/**/*', ['build']);
  });
};

// Private helpers
// ===============

function banner(pkg) {
  var stamp = [
    '/**',
    ' * <%= pkg.name %> - <%= pkg.description %>',
    ' * @version v<%= pkg.version %>',
    ' * @author <%= pkg.author.name %> <<%= pkg.author.email %>>',
    ' * @link http://liferay.com',
    ' * @license BSD',
    ' */',
    ''
  ].join('\n');

  return plugins.header(stamp, {
    pkg: pkg
  });
}

function runKarma(config, done) {
  config = merge({
    configFile: path.resolve('karma.conf.js'),
    singleRun: true
  }, config);
  karma.start(config, done);
}
<commit_msg>Remove JUnit reporter from SauceLabs test configuration<commit_after>'use strict';

var del = require('del');
var GlobalsFormatter = require('es6-module-transpiler-globals-formatter');
var gulp = require('gulp');
var gutil = require('gulp-util');
var jspm = require('jspm');
var jspmCore = require('jspm/lib/core');
var karma = require('karma').server;
var merge = require('merge');
var open = require('open');
var path = require('path');
var plugins = require('gulp-load-plugins')();
var renamer = require('gulp-es6-imports-renamer');
var runSequence = require('run-sequence');
var sourcemaps = require('gulp-sourcemaps');
var to5 = require('gulp-6to5');
var transpile = require('gulp-es6-module-transpiler');

function handleError(error) {
  console.error(error.toString());

  this.emit('end');
}

module.exports = function(options) {
  var bundleFileName = options.bundleFileName;

  gulp.task('build', function(done) {
    runSequence('clean', ['soy', 'copy'], ['build:globals', 'build:min'], done);
  });

  gulp.task('build:globals', ['jspm'], function() {
    return gulp.src('src/**/*.js')
      .pipe(sourcemaps.init())
      .pipe(renamer({
        basePath: process.cwd(),
        configPath: path.resolve('config.js')
      })).on('error', handleError)
      .pipe(transpile({
        basePath: process.cwd(),
        bundleFileName: bundleFileName,
        formatter: new GlobalsFormatter({
          globalName: 'aui'
        })
      }))
      .pipe(to5({
        blacklist: 'useStrict'
      })).on('error', handleError)
      .pipe(sourcemaps.write('./'))
      .pipe(gulp.dest('build'));
  });

  gulp.task('build:min', ['build:globals'], function() {
    return gulp.src(path.join('build/', bundleFileName))
      .pipe(plugins.rename(function(path) {
        path.basename += '-min';
      }))
      .pipe(plugins.uglify({
        preserveComments: 'some'
      }))
      .pipe(banner(options.pkg))
      .pipe(gulp.dest('build'));
  });

  gulp.task('clean', function(done) {
    del(['build'], done);
  });

  gulp.task('copy', function() {
    return gulp.src('src/**/*.css')
      .pipe(gulp.dest('build'));
  });

  gulp.task('jspm', function(done) {
    jspm.promptDefaults(true);
    jspm.install(true, {
      lock: true
    }).then(function() {
      return jspmCore.checkDlLoader();
    }).then(function() {
      return jspmCore.setMode('local');
    }).then(function() {
      gutil.log(gutil.colors.cyan('Install complete'));
      done();
    }, function(err) {
      gutil.log(gutil.colors.red('err', err.stack || err))
      gutil.log(gutil.colors.red('Installation changes not saved.'));
      done();
    });
  });

  gulp.task('lint', function() {
    return gulp.src(['src/**/*.js', 'test/**/*.js'])
      .pipe(plugins.jshint())
      .pipe(plugins.jshint.reporter(require('jshint-stylish')));
  });

  gulp.task('soy', function() {
    return gulp.src('src/*.soy')
      .pipe(plugins.soynode())
      .pipe(plugins.wrapper({
        header: '/* jshint ignore:start */',
        footer: 'export default templates;\n/* jshint ignore:end */'
      }))
      .pipe(plugins.ignore.exclude('*.soy'))
      .pipe(gulp.dest('src'));
  });

  gulp.task('test', function(done) {
    return runSequence('test:unit', 'test:complexity', done);
  });

  gulp.task('test:complexity', function() {
    return gulp.src(['src/**/*.js', '!src/**/*.soy.js', '!src/promise/Promise.js', 'test/**/*.js'])
      .pipe(to5())
      .pipe(plugins.complexity({
        halstead: [15, 15, 20]
      }));
  });

  gulp.task('test:unit', ['jspm'], function(done) {
    runKarma({}, done);
  });

  gulp.task('test:coverage', ['jspm'], function(done) {
    runKarma({}, function() {
      open(path.resolve('coverage/lcov/lcov-report/index.html'));
      done();
    });
  });

  gulp.task('test:browsers', ['jspm'], function(done) {
    runKarma({
      browsers: ['Chrome', 'Firefox', 'Safari', 'IE9 - Win7', 'IE10 - Win7', 'IE11 - Win7']
    }, done);
  });

  gulp.task('test:saucelabs', ['jspm'], function(done) {
    var launchers = {
      sl_chrome: {
            base: 'SauceLabs',
            browserName: 'chrome'
        },
        sl_firefox: {
            base: 'SauceLabs',
            browserName: 'firefox'
        },
        sl_ie_9: {
            base: 'SauceLabs',
            browserName: 'internet explorer',
            platform: 'Windows 7',
            version: '9'
        },
        sl_ie_10: {
            base: 'SauceLabs',
            browserName: 'internet explorer',
            platform: 'Windows 7',
            version: '10'
        },
        sl_ie_11: {
            base: 'SauceLabs',
            browserName: 'internet explorer',
            platform: 'Windows 8.1',
            version: '11'
        }
    };

    runKarma({
        browsers: Object.keys(launchers),

        browserDisconnectTimeout: 10000,
        browserDisconnectTolerance: 2,
        browserNoActivityTimeout: 240000,

        captureTimeout: 240000,
        customLaunchers: launchers,

        reporters: ['coverage', 'progress', 'saucelabs'],

        sauceLabs: {
            testName: 'AlloyUI tests',
            recordScreenshots: false,
            startConnect: true,
            connectOptions: {
                port: 5757,
                'selenium-version': '2.41.0',
                logfile: 'sauce_connect.log'
            }
        }
    }, done);
  });

  gulp.task('test:watch', ['jspm'], function(done) {
    runKarma({
      singleRun: false
    }, done);
  });

  gulp.task('watch', ['build'], function() {
    gulp.watch('src/**/*', ['build']);
  });
};

// Private helpers
// ===============

function banner(pkg) {
  var stamp = [
    '/**',
    ' * <%= pkg.name %> - <%= pkg.description %>',
    ' * @version v<%= pkg.version %>',
    ' * @author <%= pkg.author.name %> <<%= pkg.author.email %>>',
    ' * @link http://liferay.com',
    ' * @license BSD',
    ' */',
    ''
  ].join('\n');

  return plugins.header(stamp, {
    pkg: pkg
  });
}

function runKarma(config, done) {
  config = merge({
    configFile: path.resolve('karma.conf.js'),
    singleRun: true
  }, config);
  karma.start(config, done);
}
<|endoftext|>


<commit_before>/**
 * Module dependencies.
 */

var path = require('path'),
    Promise = require('bluebird'),
    fs = Promise.promisifyAll(require('fs')),
    moment = require('moment'),
    _ = require('underscore');

/**
 * Module defaults
 */
const DEFAULTS = {
    keepFiles: 10, // keep recent 10 files
    format: '<%= date %>_<%= ip %>'
};

/**
 * Convert header key string to Pascal-Case based string.
 *
 * e.g. `toPascalCase('content-type')` returns `'Content-Type'`.
 * @api private
 * @param key
 * @returns {string}
 */
function toPascalCase (key) {
    var chunks = key.split('-');
    return _.map(chunks, function (chunk) {
        var tmp = chunk.split('');
        tmp[0] = tmp[0].toUpperCase();
        return tmp.join('');
    }).join('-');
}

/**
 * Build raw HTTP Request Header from Request Object.
 *
 * @param req
 * @returns {string}
 */
function buildRequestHeader (req) {
    var head = [
            req.method.toUpperCase(),
            req.originalUrl,
            'HTTP/' + req.httpVersion
        ].join(' '), // GET /foo/bar?pretty=doge HTTP/1.1
        body = _.map(req.headers, function (val, key) {
            return toPascalCase(key) + ': ' + val;
        }).join('\n'); // Content-Length: 123 ...

    return head + '\n' + body;
}

/**
 * Cleanup old dump files
 *
 * @param max
 * @param dest
 */
function cleanup (max, dest) {
    var dmpExtRegexp = /\.dmp$/i;

    fs.readdirAsync(dest)
    .then(function (files) {
        var dumps = _.filter(files, function (path) {
            return path.match(dmpExtRegexp);
        });

        if (dumps.length <= max) {
            return;
        }

        Promise.map(dumps, function (filepath) {
            return fs.statAsync(path.join(dest, filepath))
                .then(function (stat) {
                    return Promise.resolve({
                        path: path.join(dest, filepath),
                        createdAt: stat.ctime.getTime()
                    });
                }).catch(function () {
                    return Promise.resolve({
                        path: path.join(dest, filepath),
                        createdAt: -1
                    });
                });
        }, {concurrent: 10})
        .then(function (dumps) {
            var willRemoved = _.sortBy(dumps, function (file) {
                return 0 - file.createdAt;
            }).slice(10);

            return Promise.map(willRemoved, function (file) {
                return fs.unlinkAsync(file.path)
                    .then(function () {
                        return Promise.resolve(true);
                    }).catch(function () {
                        // ignore
                        return Promise.resolve(true);
                    });
            }, {concurrent: 10});
        });
    })
    .catch(function (e) {
        // ignore
    })
}

module.exports = exports = function (req, opts, callback) {
    var ex,
        options = _.defaults(opts|| {}, DEFAULTS),
        filename = _.template(options.format)({
            date: moment().format('YYYYMMDDHHmmss'),
            ip: req.ip
        });

    if (typeof options.dest !== 'string') {
        ex = new Error('Destination path is unspecified, or not string.');
        if(typeof callback ==='function') {
            callback(ex);
            return;
        } else {
            throw ex;
        }
    }

    var stream = fs.createWriteStream(path.join(options.dest, filename + '.dmp'));
    stream.write(buildRequestHeader(req) + '\n\n', 'utf8');
    req.on('end', function () {
        if (typeof callback === 'function') callback(null);
        cleanup(options.keepFiles, options.dest);
    }).pipe(stream);
};<commit_msg>Close stream when request was ended.<commit_after>/**
 * Module dependencies.
 */

var path = require('path'),
    Promise = require('bluebird'),
    fs = Promise.promisifyAll(require('fs')),
    moment = require('moment'),
    _ = require('underscore');

/**
 * Module defaults
 */
const DEFAULTS = {
    keepFiles: 10, // keep recent 10 files
    format: '<%= date %>_<%= ip %>'
};

/**
 * Convert header key string to Pascal-Case based string.
 *
 * e.g. `toPascalCase('content-type')` returns `'Content-Type'`.
 * @api private
 * @param key
 * @returns {string}
 */
function toPascalCase (key) {
    var chunks = key.split('-');
    return _.map(chunks, function (chunk) {
        var tmp = chunk.split('');
        tmp[0] = tmp[0].toUpperCase();
        return tmp.join('');
    }).join('-');
}

/**
 * Build raw HTTP Request Header from Request Object.
 *
 * @param req
 * @returns {string}
 */
function buildRequestHeader (req) {
    var head = [
            req.method.toUpperCase(),
            req.originalUrl,
            'HTTP/' + req.httpVersion
        ].join(' '), // GET /foo/bar?pretty=doge HTTP/1.1
        body = _.map(req.headers, function (val, key) {
            return toPascalCase(key) + ': ' + val;
        }).join('\n'); // Content-Length: 123 ...

    return head + '\n' + body;
}

/**
 * Cleanup old dump files
 *
 * @param max
 * @param dest
 */
function cleanup (max, dest) {
    var dmpExtRegexp = /\.dmp$/i;

    fs.readdirAsync(dest)
    .then(function (files) {
        var dumps = _.filter(files, function (path) {
            return path.match(dmpExtRegexp);
        });

        if (dumps.length <= max) {
            return;
        }

        Promise.map(dumps, function (filepath) {
            return fs.statAsync(path.join(dest, filepath))
                .then(function (stat) {
                    return Promise.resolve({
                        path: path.join(dest, filepath),
                        createdAt: stat.ctime.getTime()
                    });
                }).catch(function () {
                    return Promise.resolve({
                        path: path.join(dest, filepath),
                        createdAt: -1
                    });
                });
        }, {concurrent: 10})
        .then(function (dumps) {
            var willRemoved = _.sortBy(dumps, function (file) {
                return 0 - file.createdAt;
            }).slice(10);

            return Promise.map(willRemoved, function (file) {
                return fs.unlinkAsync(file.path)
                    .then(function () {
                        return Promise.resolve(true);
                    }).catch(function () {
                        // ignore
                        return Promise.resolve(true);
                    });
            }, {concurrent: 10});
        });
    })
    .catch(function (e) {
        // ignore
    })
}

module.exports = exports = function (req, opts, callback) {
    var ex,
        options = _.defaults(opts|| {}, DEFAULTS),
        filename = _.template(options.format)({
            date: moment().format('YYYYMMDDHHmmss'),
            ip: req.ip
        });

    if (typeof options.dest !== 'string') {
        ex = new Error('Destination path is unspecified, or not string.');
        if(typeof callback ==='function') {
            callback(ex);
            return;
        } else {
            throw ex;
        }
    }

    var stream = fs.createWriteStream(path.join(options.dest, filename + '.dmp'));
    stream.write(buildRequestHeader(req) + '\n\n', 'utf8');
    req.on('end', function () {
        if (req.body) {
            stream.end(JSON.stringify(req.body), 'utf8');
        } else {
            stream.end();
        }

        if (typeof callback === 'function') callback(null);
        cleanup(options.keepFiles, options.dest);
    }).pipe(stream);
};<|endoftext|>