<commit_before>import qs from 'querystring'
import { browserHistory } from 'react-router'

import { _get } from 'common/helpers/super'

import { flattenFilters } from './query'

import {
  SEARCH_EXECUTE_PENDING,
  SEARCH_EXECUTE_SUCCESS,
  SEARCH_EXECUTE_FAILURE
} from './constants'

export const execute = (query, limit = 20) => dispatch => {
  dispatch({
    type: SEARCH_EXECUTE_PENDING
  })

  const filters = flattenFilters(query.filters)

  return _get(
    `https://inspire.data.gouv.fr/api/geogw/records?${qs.stringify({
      ...filters,
      q: query.textInput,
      offset: (query.page - 1) * limit,
      limit
    })}`
  )
  .then(data => {
    dispatch({
      type: SEARCH_EXECUTE_SUCCESS,
      payload: data
    })
  })
  .catch(err => {
    dispatch({
      type: SEARCH_EXECUTE_FAILURE,
      error: err
    })
  })
}

export const update = changes => {
  return (dispatch, getState) => {
    const location = { ...getState().location }
    const { q: iq, page: ipages, ...filters } = location.query

    const q = changes.q === undefined ? iq : changes.q
    const page = changes.page || ipages

    const query = {
      ...(q ? { q } : {}),

      ...(page && page > 1 ? { page } : {}),

      ...(changes.filters ? flattenFilters(changes.filters) : filters)
    }

    location.pathname = '/search'
    location.query = query

    browserHistory.push(location)
  }
}
<commit_msg>Remove hardcoded API url in search action<commit_after>import qs from 'querystring'
import { browserHistory } from 'react-router'

import { _get } from 'common/helpers/super'

import { flattenFilters } from './query'

import {
  SEARCH_EXECUTE_PENDING,
  SEARCH_EXECUTE_SUCCESS,
  SEARCH_EXECUTE_FAILURE
} from './constants'

const { INSPIRE_API_URL } = process.env

export const execute = (query, limit = 20) => dispatch => {
  dispatch({
    type: SEARCH_EXECUTE_PENDING
  })

  const filters = flattenFilters(query.filters)

  return _get(
    `${INSPIRE_API_URL}/records?${qs.stringify({
      ...filters,
      q: query.textInput,
      offset: (query.page - 1) * limit,
      limit
    })}`
  )
  .then(data => {
    dispatch({
      type: SEARCH_EXECUTE_SUCCESS,
      payload: data
    })
  })
  .catch(err => {
    dispatch({
      type: SEARCH_EXECUTE_FAILURE,
      error: err
    })
  })
}

export const update = changes => {
  return (dispatch, getState) => {
    const location = { ...getState().location }
    const { q: iq, page: ipages, ...filters } = location.query

    const q = changes.q === undefined ? iq : changes.q
    const page = changes.page || ipages

    const query = {
      ...(q ? { q } : {}),

      ...(page && page > 1 ? { page } : {}),

      ...(changes.filters ? flattenFilters(changes.filters) : filters)
    }

    location.pathname = '/search'
    location.query = query

    browserHistory.push(location)
  }
}
<|endoftext|>


<commit_before>(function(){
	const HOST = 'dev-hson-1';
	var ws = new WebSocket('ws://' + HOST + ':8082/jenkins'),
		output = JSON.parse(localStorage.getItem('messages'))||[],
		lines = {
			'fidodevelopment': 'FIDODEV',
			'fidotesting': 'FIDOTEST',
			'kerby-server': 'KERBY SVR'
		},
		render = function(data){
			if(!lines[data.project]) return;
			if(output.length === Board.rows.length){
				output.pop();
			}

			var projectNumber = ' #' + data.number,
				projectName = lines[data.project].substring(0, 14 - projectNumber.length);

			output.unshift(projectName.rpad(' ', 14 - projectNumber.length) + projectNumber + data.result[0]);

			Board.setMessage(output);
			localStorage.setItem('messages', JSON.stringify(output));
		};

	ws.onmessage = function(msg) {
		var data = JSON.parse(msg.data);

		render(data);
	};

	Board.bind('start', function(){
		Board.setMessage(output);
	});
})();<commit_msg>Adding zonza<commit_after>(function(){
	const HOST = 'dev-hson-1';
	const ZONZAHOST = 'dev-jen1';
	const CHARCOUNT = 16;
	var ws = new WebSocket('ws://' + HOST + ':8082/jenkins'),
	    ws2 = new WebSocket('ws://' + ZONZAHOST + ':8081/jenkins'),
		output = JSON.parse(localStorage.getItem('messages'))||[],
		lines = {
			'fidodevelopment': 'FIDODEV',
			'fidotesting': 'FIDOTEST',
			'kerby-server': 'KERBY SVR',
			'Zonza': 'ZONZA'
		},
		render = function(data){
			if(!lines[data.project]) return;
			if(output.length === Board.rows.length){
				output.pop();
			}

			var projectNumber = ' #' + data.number,
				projectName = lines[data.project].substring(0, CHARCOUNT - projectNumber.length);

			output.unshift(projectName.rpad(' ', CHARCOUNT - projectNumber.length) + projectNumber + data.result[0]);

			Board.setMessage(output);
			localStorage.setItem('messages', JSON.stringify(output));
		};

	ws.onmessage = ws2.onmessage = function(msg) {
		var data = JSON.parse(msg.data);

		render(data);
	};

	Board.bind('start', function(){
		Board.setMessage(output);
	});
})();<|endoftext|>


<commit_before>'use strict';

var restify = require('restify');
var session = require('client-sessions');
var async = require('async');
var mongoose = require('mongoose');
var url = require('url');
var kue = require('kue');
var http = require('http');
var https = require('https');
var Anyfetch = require('anyfetch');

var Token = require('./models/token.js');
var TempToken = require('./models/temp-token.js');

var connected = false;
var connectMongo = function(mongoUrl) {
  if(!connected) {
    // Connect mongoose
    mongoose.connect(mongoUrl || process.env.MONGO_URL || "mongodb://localhost/anyfetch-provider");
    connected = true;
  }
};

/**
* Simple wrapper around the token model.
* Only one token should match.
*
* retrieveData({accessToken: ...}) => data for this access token
* retrieveData({'data.grant': ...}) => data with this value of grant.
*/
module.exports.retrieveData = function(hash, cb) {
  Token.findOne(hash, function(err, token) {
    if(!token) {
      return cb(new Error("no data matches"));
    }
    cb(err, token.data);
  });
};

/**
 * Create a new provider server.
 * This server will use `config.task_generator` as its main function, to turn a file into metadata.
 *
 * @param {Object} config Configuration hash.
 *   Mandatory:
*       task_generator, to retrieve a list of tasks (a task is "a document to upload").
*       task_handler, the uploading function.
*    Optional:
*       concurrency, max number of simultaneous calls to your task_handler function (default is 1)
 */
module.exports.createServer = function createServer(connectFunctions, updateAccount, workers, config) {
  // Connect mongo
  connectMongo(config.mongoUrl);

  // Load endpoints generators
  var indexEndpoint = require('./handlers/index.js');
  var initConnectEndpoints = require('./handlers/init/connect.js');
  var initCallbackEndpoints = require('./handlers/init/callback.js');
  var updateEndpoint = require('./handlers/update.js');
  var statusEndpoint = require('./handlers/status.js');
  var resetEndpoint = require('./handlers/reset.js');

  http.globalAgent.maxSockets = config.maxSockets || 30;
  https.globalAgent.maxSockets = http.globalAgent.maxSockets;

  // Create server
  var server = restify.createServer();
  server.use(restify.acceptParser(server.acceptable));
  server.use(restify.queryParser());
  server.use(restify.bodyParser());
  server.use(session({
    cookieName: 'ANYFETCH_SESSION',
    secret: config.appSecret
  }));

  config.redisUrl = config.redisUrl || process.env.REDIS_URL || undefined;
  server.queue = kue.createQueue({
    prefix: config.providerUrl,
    redis: {
      port: (config.redisUrl) ? url.parse(config.redisUrl).port : 6379,
      host: (config.redisUrl) ? url.parse(config.redisUrl).hostname : "localhost",
      auth: (config.redisUrl) ? url.parse(config.redisUrl).auth.split(":")[1] : null,
    },
    disableSearch: true
  });

  //server.use('/queue', kue.app);

  Object.keys(workers).forEach(function createQueue(name) {
    server.queue.process(name, (workers[name].concurrency) ? workers[name].concurrency : 1, function(job, done) {
      job.task = job.data;

      job.anyfetchClient = new Anyfetch(job.task._anyfetchToken);
      job.anyfetchClient.setApiUrl(job.task._anyfetchApiUrl);

      Token.findOne({anyfetchToken: job.task._anyfetchToken}, function(err, token) {
        if(err) {
          return done(err);
        }

        if(!token) {
          return done(new Error("Can't retrieve Token"));
        }

        console.log("User `" + token.accountName + "` is executing task " + job.id + " onto `" + name + "`");

        job.serviceData = token.data;

        delete job.data;
        delete job.task._anyfetchToken;
        delete job.task._anyfetchApiUrl;

        workers[name](job, done);
      });
    });
  });

  // Load routes and generate endpoints using closures
  server.get('/', indexEndpoint);
  server.get('/init/connect', initConnectEndpoints(connectFunctions.redirectToService, config));
  server.get('/init/callback', initCallbackEndpoints(connectFunctions.retrieveTokens, config));

  server.post('/update', updateEndpoint(updateAccount, server.queue, workers));

  server.get('/status', statusEndpoint);
  server.del('/reset', resetEndpoint);

  // Expose the server
  return server;
};

module.exports.debug = {
  cleanTokens: function(cb) {
    connectMongo();
    async.parallel([
      function(cb) {
        Token.remove({}, cb);
      },
      function(cb) {
        TempToken.remove({}, cb);
      }
    ], cb);
  },
  createToken: function(hash, cb) {
    connectMongo();

    var token = new Token(hash);
    token.save(cb);
  },
  createTempToken: function(hash, cb) {
    connectMongo();

    var tempToken = new TempToken(hash);
    tempToken.save(cb);
  }
};
<commit_msg>Add a shut down handler for kue<commit_after>'use strict';

var restify = require('restify');
var session = require('client-sessions');
var async = require('async');
var mongoose = require('mongoose');
var url = require('url');
var kue = require('kue');
var http = require('http');
var https = require('https');
var Anyfetch = require('anyfetch');

var Token = require('./models/token.js');
var TempToken = require('./models/temp-token.js');

var connected = false;
var connectMongo = function(mongoUrl) {
  if(!connected) {
    // Connect mongoose
    mongoose.connect(mongoUrl || process.env.MONGO_URL || "mongodb://localhost/anyfetch-provider");
    connected = true;
  }
};

/**
* Simple wrapper around the token model.
* Only one token should match.
*
* retrieveData({accessToken: ...}) => data for this access token
* retrieveData({'data.grant': ...}) => data with this value of grant.
*/
module.exports.retrieveData = function(hash, cb) {
  Token.findOne(hash, function(err, token) {
    if(!token) {
      return cb(new Error("no data matches"));
    }
    cb(err, token.data);
  });
};

/**
 * Create a new provider server.
 * This server will use `config.task_generator` as its main function, to turn a file into metadata.
 *
 * @param {Object} config Configuration hash.
 *   Mandatory:
*       task_generator, to retrieve a list of tasks (a task is "a document to upload").
*       task_handler, the uploading function.
*    Optional:
*       concurrency, max number of simultaneous calls to your task_handler function (default is 1)
 */
module.exports.createServer = function createServer(connectFunctions, updateAccount, workers, config) {
  // Connect mongo
  connectMongo(config.mongoUrl);

  // Load endpoints generators
  var indexEndpoint = require('./handlers/index.js');
  var initConnectEndpoints = require('./handlers/init/connect.js');
  var initCallbackEndpoints = require('./handlers/init/callback.js');
  var updateEndpoint = require('./handlers/update.js');
  var statusEndpoint = require('./handlers/status.js');
  var resetEndpoint = require('./handlers/reset.js');

  http.globalAgent.maxSockets = config.maxSockets || 30;
  https.globalAgent.maxSockets = http.globalAgent.maxSockets;

  // Create server
  var server = restify.createServer();
  server.use(restify.acceptParser(server.acceptable));
  server.use(restify.queryParser());
  server.use(restify.bodyParser());
  server.use(session({
    cookieName: 'ANYFETCH_SESSION',
    secret: config.appSecret
  }));

  config.redisUrl = config.redisUrl || process.env.REDIS_URL || undefined;
  server.queue = kue.createQueue({
    prefix: config.providerUrl,
    redis: {
      port: (config.redisUrl) ? url.parse(config.redisUrl).port : 6379,
      host: (config.redisUrl) ? url.parse(config.redisUrl).hostname : "localhost",
      auth: (config.redisUrl) ? url.parse(config.redisUrl).auth.split(":")[1] : null,
    },
    disableSearch: true
  });

  function sigtermKue() {
    server.queue.shutdown(function(err) {
      console.log('Kue is shut down.', err || '');
      process.exit(0);
    }, 5000);
  }

  process.once('SIGTERM', sigtermKue);

  server.on('close', function() {
    process.removeListener('SIGTERM', sigtermKue);
  });

  Object.keys(workers).forEach(function createQueue(name) {
    server.queue.process(name, (workers[name].concurrency) ? workers[name].concurrency : 1, function(job, done) {
      job.task = job.data;

      job.anyfetchClient = new Anyfetch(job.task._anyfetchToken);
      job.anyfetchClient.setApiUrl(job.task._anyfetchApiUrl);

      Token.findOne({anyfetchToken: job.task._anyfetchToken}, function(err, token) {
        if(err) {
          return done(err);
        }

        if(!token) {
          return done(new Error("Can't retrieve Token"));
        }

        console.log("User `" + token.accountName + "` is executing task " + job.id + " onto `" + name + "`");

        job.serviceData = token.data;

        delete job.data;
        delete job.task._anyfetchToken;
        delete job.task._anyfetchApiUrl;

        workers[name](job, done);
      });
    });
  });

  // Load routes and generate endpoints using closures
  server.get('/', indexEndpoint);
  server.get('/init/connect', initConnectEndpoints(connectFunctions.redirectToService, config));
  server.get('/init/callback', initCallbackEndpoints(connectFunctions.retrieveTokens, config));

  server.post('/update', updateEndpoint(updateAccount, server.queue, workers));

  server.get('/status', statusEndpoint);
  server.del('/reset', resetEndpoint);

  // Expose the server
  return server;
};

module.exports.debug = {
  cleanTokens: function(cb) {
    connectMongo();
    async.parallel([
      function(cb) {
        Token.remove({}, cb);
      },
      function(cb) {
        TempToken.remove({}, cb);
      }
    ], cb);
  },
  createToken: function(hash, cb) {
    connectMongo();

    var token = new Token(hash);
    token.save(cb);
  },
  createTempToken: function(hash, cb) {
    connectMongo();

    var tempToken = new TempToken(hash);
    tempToken.save(cb);
  }
};
<|endoftext|>


<commit_before>/*
 * @copyright Copyright (c) Sematext Group, Inc. - All Rights Reserved
 *
 * @licence logparser-js is free-to-use, proprietary software.
 * THIS IS PROPRIETARY SOURCE CODE OF Sematext Group, Inc. (Sematext)
 * This source code may not be copied, reverse engineered, or altered for any purpose.
 * This source code is to be used exclusively by users and customers of Sematext.
 * Please see the full license (found in LICENSE in this distribution) for details on its license and the licenses of its dependencies.
 */
'use strict'
var yaml = require('js-yaml')
var fs = require('fs')
var moment = require('moment')
var crypto = require('crypto')
var MultiLine = require('./multiLine.js')
var geoip = null
var initJobs = []

function LogParser (fileName, options, readyCallback) {
  this.ready = false
  this.readyCallback = readyCallback
  this.options = options
  this.sources = {}
  this.load(fileName || require('path').join(__dirname, '../patterns.yml'), options)
}
function initGeoIp (err, fileName) {
  if (err) {
    this.cfg.geoIPEnabled = false
    if(this.readyCallback) {
      this.readyCallback()
      this.readyCallback = null
    }
    return
  }
  if (fs.existsSync(fileName))
  {
    geoip.init(fileName, {indexCache: true, checkForUpdates: true})
    this.cfg.geoIPEnabled = true
    if (this.cfg.globalTransform)
    {
      this.cfg.globalTransform = this.cfg.globalTransform.bind({
        moment: moment,
        geoip: geoip,
        enrichGeoIp: this.enrichGeoIp
      })
    }
  } else {
    this.cfg.geoIPEnabled = false
  }
  if(this.readyCallback) {
    this.readyCallback()
    this.readyCallback = null
  }
}
LogParser.prototype = {

  load: function (name, options) {
    try {
      this.cfg = yaml.load(fs.readFileSync(name, 'utf8'))
      if (this.cfg.autohash) {
        console.log('Hashing field content enabled for field names: ' + this.cfg.autohash)
      }
      // console.log(this.cfg)
      this.patterns = this.cfg.patterns
      if (options && options.whitelist) {
        this.whitelist(options.whitelist)
      }
      if (options && options.blacklist) {
        this.blacklist(options.blacklist)
      }
      // inject modules to global transform function

      if (this.cfg.geoIP && !(process.env.GEOIP_DISABLED=='true')) {
        var fileName = (process.env.MAXMIND_DB_DIR || this.cfg.maxmindDbDir) + 'GeoIPCity.dat'
        var cbInitGeoIp = null
        try {
          geoip = require('maxmind')
          if (fs.existsSync(fileName))
          {
            initGeoIp.bind(this)(null, fileName)
            cbInitGeoIp = null
          } else {
            cbInitGeoIp = initGeoIp.bind(this)
          }
          var maxmindUpdate=require('./maxmind-update')
          this.maxmindUpdate =  maxmindUpdate(this.cfg.debug,
            (process.env.MAXMIND_DB_DIR || this.cfg.maxmindDbDir),
            cbInitGeoIp)
        } catch (err) {
          console.log(err.stack)
          if(this.readyCallback) {
            setTimeout (this.readyCallback, 0)
          }
        }
      } else {
        if (this.readyCallback)
          this.readyCallback()
      }
    } catch (e) {
      if (e.reason && e.mark) {
        console.log('Error parsing file: ' + name + ' ' + e.reason + ' line:' + e.mark.line + ' column:' + e.mark.columns)
      } else {
        console.log('Error parsing file: ' + name + ' ' + e)
      }
      this.patterns = []
      process.exit()
    } finally {
      return this.patterns
    }
  },
  whitelist: function (whitelist) {
    this.patterns = this.patterns.filter(function (e) {
      return (e.sourceName && e.sourceName.match(whitelist))
    })
  },
  blacklist: function (blacklist) {
    this.patterns = this.patterns.filter(function (e) {
      return !(e.sourceName && e.sourceName.match(blacklist))
    })
  },
  enrichGeoIp: function (parsedObject, fieldName) {
    if (!this.cfg.geoIPEnabled || !fieldName)
        return null
    if (parsedObject[fieldName]) {
      var location = geoip.getLocation(parsedObject[fieldName])
      if (location) {
        parsedObject['geoip'] = {location: [location.longitude, location.latitude], info: location}
      }
    }
  },
  getPatternsForSource: function (sourceName) {
    if (!sourceName) {
      return this.patterns // try all
    }
    var src = this.sources[sourceName]
    if (src && src.patterns) {
      return this.sources[sourceName].patterns
    }
    var exclude = []
    var include = this.patterns.filter(function (p) {
      if (sourceName && sourceName.match && sourceName.match(p.sourceName)) {
        return true
      } else {
        exclude.push(p)
      }
    })
    var patternLists = include.concat(exclude)
    // console.log('Patterns for source ' + sourceName + ' : ex:' + exclude.length + ' incl:' + include.length)
    if (!src) {
      this.sources[sourceName] = {}
    }
    this.sources[sourceName].patterns = patternLists
    return patternLists
  },
  getMultiLineReader: function (sourceName, parser) {
    if (!sourceName) {
      return this.getMultiLineReader('unknown', parser)
    }
    var src = this.sources[sourceName]
    if (src && src.reader) {
      return this.sources[sourceName].reader
    } else {
      this.sources[sourceName] = {}
      var include = this.patterns.filter(function (p) {
        if (sourceName && sourceName.match && sourceName.match(p.sourceName)) {
          return true
        }
      })
      if (include.length > 0) {
        this.sources[sourceName].reader = new MultiLine(include[0].blockStart, parser)
        return this.sources[sourceName].reader
      } else {
        this.sources[sourceName].reader = new MultiLine(/^\S+/, parser)
        return this.sources[sourceName].reader
      }
    }
  },
  hash: function hash (input) {
    var sha1 = crypto.createHash('sha1')
    sha1.update(input.toString())
    return sha1.digest('hex')
  },
  parseDate: function (strDate, dateFormat) {
    var d = null
    if (dateFormat && strDate) {
      d = moment(String(strDate).trim().replace('  ', ' '), dateFormat.trim(), true) || moment(strDate.trim().replace('  ', ' '), this.cfg.dateFormats, true)
    }
    if (d && d.isValid()) {
      return d.toDate()
    } else {
      // console.log('DATE not matched' + strDate + ' ' + dateFormat)
      return null
    }
  },
  matchPatterns: function (p, parsed, line) {
    var match = line.match(p.regex)
    if (match) {
      parsed._type = p.type
      if (p.fields && (match.length > p.fields.length)) {
        for (var i = 0; i < p.fields.length; i++) {
          var value = match[i + 1]
          if (!isNaN(value) && value !== '') {
            value = Number(value)
          }
          if (this.cfg.autohash && this.cfg.autohash.test(p.fields[i])) {
            value = this.hash(value)
          }
          parsed[p.fields[i]] = value
          if(p.fields[i] == p.geoIP) {
            this.enrichGeoIp(parsed, p.geoIP)
          }
        }
        if (parsed['ts']) {
          var timestamp = this.parseDate(parsed['ts'], p.dateFormat)
          if (timestamp) {
            parsed['@timestamp'] = timestamp
          }
        }
        if (p.transform) {
          try {
            p.transform(parsed)
          } catch (ex) {
            console.log('Error in' + p.type + '.transform():' + ex)
          }
        }
        this.enrichGeoIp (p, p.geoIP)
        // remove ts field, because Elasticsearch
        // might have problems to index it
        // it could be recognized as string or data
        // depending on its content
        delete parsed.ts
        return 1
      }
    } else {
      return 0
    }
  },
  // optimize performance, by keeping last matched pattern
  // on top if the list,
  // assuming the next line will have the same format
  bubbleUp: function (plist, pos) {
    if (pos === 0) {
      return
    }
    // remove element on pos
    var tmp = plist[0]
    plist[0] = plist[pos]
    plist[pos] = tmp
  },
  parseLine: function (line, source, cbf) {
    var br = this.getMultiLineReader(source, function (data) {
        this._parseLine(data, source, cbf)
    }.bind(this))
    br.add(line)
  },
  globalTransform: function (source, parsed) {
    if (this.cfg.globalTransform) {
      try {
        this.cfg.globalTransform(source, parsed)
      } catch (ex) {
        console.error('Error in gloabalTransform():' + ex)
      }
    }
  },
  _parseLine: function (line, source, cbf) {
    if (line === null || line === '') {
      cbf('empty', null)
    }
    var parsed = {}
    if (this.cfg.originalLine === true)
    {
      parsed.originalLine = line
    }
    parsed['@source'] = source
    // JSON handling
    var trimedLine=line.trim()
    if (/^\[{0,1}\{.*\}\]{0,1}$/.test(trimedLine)) {
      try {
        parsed = JSON.parse(trimedLine)
        if (!(parsed['@timestamp'])) {
          if (parsed.time && parsed.time instanceof Date) {
            parsed['@timestamp'] = parsed.time
          } else if (parsed.t && parsed.t instanceof Date) {
            parsed['@timestamp'] = parsed.t
          } else if (parsed.timestamp && parsed.task_uuid) {
              // python eliot logs
              parsed['@timestamp'] = new Date(parsed.timestamp*1000)
          }else {
            parsed['@timestamp'] = new Date()
          }
        }
        if (!parsed.message && parsed.msg) {
          parsed.message = parsed.msg
        }
        // TODO JSON GeoIP enrichment
        this.globalTransform(source, parsed)
        return cbf(null, parsed)
      } catch (ex) {
        // ignore treat as text
      }
    }
    var patternList = this.getPatternsForSource(source) || this.patterns
    for (var k in patternList) {
      var patterns = patternList[k]
      for (var i = 0; i < patterns.match.length; i++) {
        if (this.matchPatterns(patterns.match[i], parsed, line)) {
          this.bubbleUp(patternList, k)
          if (!parsed._type) {
            parsed._type = patternList[k].type
          }
          if (this.gloabalTransform) {
            this.globalTransform(source, parsed)
          }
          return cbf(null, parsed)
        }
      }
    }
    return cbf('not found', {'@timestamp': new Date(), message: line, '@source': source}
    )
  }
}

module.exports = LogParser
<commit_msg>catch unknown errors in parseLine<commit_after>/*
 * @copyright Copyright (c) Sematext Group, Inc. - All Rights Reserved
 *
 * @licence logparser-js is free-to-use, proprietary software.
 * THIS IS PROPRIETARY SOURCE CODE OF Sematext Group, Inc. (Sematext)
 * This source code may not be copied, reverse engineered, or altered for any purpose.
 * This source code is to be used exclusively by users and customers of Sematext.
 * Please see the full license (found in LICENSE in this distribution) for details on its license and the licenses of its dependencies.
 */
'use strict'
var yaml = require('js-yaml')
var fs = require('fs')
var moment = require('moment')
var crypto = require('crypto')
var MultiLine = require('./multiLine.js')
var geoip = null
var initJobs = []

function LogParser (fileName, options, readyCallback) {
  this.ready = false
  this.readyCallback = readyCallback
  this.options = options
  this.sources = {}
  this.load(fileName || require('path').join(__dirname, '../patterns.yml'), options)
}
function initGeoIp (err, fileName) {
  if (err) {
    this.cfg.geoIPEnabled = false
    if(this.readyCallback) {
      this.readyCallback()
      this.readyCallback = null
    }
    return
  }
  if (fs.existsSync(fileName))
  {
    geoip.init(fileName, {indexCache: true, checkForUpdates: true})
    this.cfg.geoIPEnabled = true
    if (this.cfg.globalTransform)
    {
      this.cfg.globalTransform = this.cfg.globalTransform.bind({
        moment: moment,
        geoip: geoip,
        enrichGeoIp: this.enrichGeoIp
      })
    }
  } else {
    this.cfg.geoIPEnabled = false
  }
  if(this.readyCallback) {
    this.readyCallback()
    this.readyCallback = null
  }
}
LogParser.prototype = {

  load: function (name, options) {
    try {
      this.cfg = yaml.load(fs.readFileSync(name, 'utf8'))
      if (this.cfg.autohash) {
        console.log('Hashing field content enabled for field names: ' + this.cfg.autohash)
      }
      // console.log(this.cfg)
      this.patterns = this.cfg.patterns
      if (options && options.whitelist) {
        this.whitelist(options.whitelist)
      }
      if (options && options.blacklist) {
        this.blacklist(options.blacklist)
      }
      // inject modules to global transform function

      if (this.cfg.geoIP && !(process.env.GEOIP_DISABLED=='true')) {
        var fileName = (process.env.MAXMIND_DB_DIR || this.cfg.maxmindDbDir) + 'GeoIPCity.dat'
        var cbInitGeoIp = null
        try {
          geoip = require('maxmind')
          if (fs.existsSync(fileName))
          {
            initGeoIp.bind(this)(null, fileName)
            cbInitGeoIp = null
          } else {
            cbInitGeoIp = initGeoIp.bind(this)
          }
          var maxmindUpdate=require('./maxmind-update')
          this.maxmindUpdate =  maxmindUpdate(this.cfg.debug,
            (process.env.MAXMIND_DB_DIR || this.cfg.maxmindDbDir),
            cbInitGeoIp)
        } catch (err) {
          console.log(err.stack)
          if(this.readyCallback) {
            setTimeout (this.readyCallback, 0)
          }
        }
      } else {
        if (this.readyCallback)
          this.readyCallback()
      }
    } catch (e) {
      if (e.reason && e.mark) {
        console.log('Error parsing file: ' + name + ' ' + e.reason + ' line:' + e.mark.line + ' column:' + e.mark.columns)
      } else {
        console.log('Error parsing file: ' + name + ' ' + e)
      }
      this.patterns = []
      process.exit()
    } finally {
      return this.patterns
    }
  },
  whitelist: function (whitelist) {
    this.patterns = this.patterns.filter(function (e) {
      return (e.sourceName && e.sourceName.match(whitelist))
    })
  },
  blacklist: function (blacklist) {
    this.patterns = this.patterns.filter(function (e) {
      return !(e.sourceName && e.sourceName.match(blacklist))
    })
  },
  enrichGeoIp: function (parsedObject, fieldName) {
    if (!this.cfg.geoIPEnabled || !fieldName)
        return null
    if (parsedObject[fieldName]) {
      var location = geoip.getLocation(parsedObject[fieldName])
      if (location) {
        parsedObject['geoip'] = {location: [location.longitude, location.latitude], info: location}
      }
    }
  },
  getPatternsForSource: function (sourceName) {
    if (!sourceName) {
      return this.patterns // try all
    }
    var src = this.sources[sourceName]
    if (src && src.patterns) {
      return this.sources[sourceName].patterns
    }
    var exclude = []
    var include = this.patterns.filter(function (p) {
      if (sourceName && sourceName.match && sourceName.match(p.sourceName)) {
        return true
      } else {
        exclude.push(p)
      }
    })
    var patternLists = include.concat(exclude)
    // console.log('Patterns for source ' + sourceName + ' : ex:' + exclude.length + ' incl:' + include.length)
    if (!src) {
      this.sources[sourceName] = {}
    }
    this.sources[sourceName].patterns = patternLists
    return patternLists
  },
  getMultiLineReader: function (sourceName, parser) {
    if (!sourceName) {
      return this.getMultiLineReader('unknown', parser)
    }
    var src = this.sources[sourceName]
    if (src && src.reader) {
      return this.sources[sourceName].reader
    } else {
      this.sources[sourceName] = {}
      var include = this.patterns.filter(function (p) {
        if (sourceName && sourceName.match && sourceName.match(p.sourceName)) {
          return true
        }
      })
      if (include.length > 0) {
        this.sources[sourceName].reader = new MultiLine(include[0].blockStart, parser)
        return this.sources[sourceName].reader
      } else {
        this.sources[sourceName].reader = new MultiLine(/^\S+/, parser)
        return this.sources[sourceName].reader
      }
    }
  },
  hash: function hash (input) {
    var sha1 = crypto.createHash('sha1')
    sha1.update(input.toString())
    return sha1.digest('hex')
  },
  parseDate: function (strDate, dateFormat) {
    var d = null
    if (dateFormat && strDate) {
      d = moment(String(strDate).trim().replace('  ', ' '), dateFormat.trim(), true) || moment(strDate.trim().replace('  ', ' '), this.cfg.dateFormats, true)
    }
    if (d && d.isValid()) {
      return d.toDate()
    } else {
      // console.log('DATE not matched' + strDate + ' ' + dateFormat)
      return null
    }
  },
  matchPatterns: function (p, parsed, line) {
    var match = line.match(p.regex)
    if (match) {
      parsed._type = p.type
      if (p.fields && (match.length > p.fields.length)) {
        for (var i = 0; i < p.fields.length; i++) {
          var value = match[i + 1]
          if (!isNaN(value) && value !== '') {
            value = Number(value)
          }
          if (this.cfg.autohash && this.cfg.autohash.test(p.fields[i])) {
            value = this.hash(value)
          }
          parsed[p.fields[i]] = value
          if(p.fields[i] == p.geoIP) {
            this.enrichGeoIp(parsed, p.geoIP)
          }
        }
        if (parsed['ts']) {
          var timestamp = this.parseDate(parsed['ts'], p.dateFormat)
          if (timestamp) {
            parsed['@timestamp'] = timestamp
          }
        }
        if (p.transform) {
          try {
            p.transform(parsed)
          } catch (ex) {
            console.log('Error in' + p.type + '.transform():' + ex)
          }
        }
        this.enrichGeoIp (p, p.geoIP)
        // remove ts field, because Elasticsearch
        // might have problems to index it
        // it could be recognized as string or data
        // depending on its content
        delete parsed.ts
        return 1
      }
    } else {
      return 0
    }
  },
  // optimize performance, by keeping last matched pattern
  // on top if the list,
  // assuming the next line will have the same format
  bubbleUp: function (plist, pos) {
    if (pos === 0) {
      return
    }
    // remove element on pos
    var tmp = plist[0]
    plist[0] = plist[pos]
    plist[pos] = tmp
  },
  parseLine: function (line, source, cbf) {
    var br = this.getMultiLineReader(source, function (data) {
        try {
          this._parseLine(data, source, cbf)
        } catch (err) {
          // console.log('Error parsing logs from ' + source +  ': ' + err + ' originalLine: ' + line)
          cbf(err, data)
        }
    }.bind(this))
    br.add(line)
  },
  globalTransform: function (source, parsed) {
    if (this.cfg.globalTransform) {
      try {
        this.cfg.globalTransform(source, parsed)
      } catch (ex) {
        console.error('Error in gloabalTransform():' + ex)
      }
    }
  },
  _parseLine: function (line, source, cbf) {
    if (line === null || line === '') {
      cbf('empty', null)
    }
    var parsed = {}
    if (this.cfg.originalLine === true)
    {
      parsed.originalLine = line
    }
    parsed['@source'] = source
    // JSON handling
    var trimedLine=line.trim()
    if (/^\[{0,1}\{.*\}\]{0,1}$/.test(trimedLine)) {
      try {
        parsed = JSON.parse(trimedLine)
        if (!(parsed['@timestamp'])) {
          if (parsed.time && parsed.time instanceof Date) {
            parsed['@timestamp'] = parsed.time
          } else if (parsed.t && parsed.t instanceof Date) {
            parsed['@timestamp'] = parsed.t
          } else if (parsed.timestamp && parsed.task_uuid) {
              // python eliot logs
              parsed['@timestamp'] = new Date(parsed.timestamp*1000)
          }else {
            parsed['@timestamp'] = new Date()
          }
        }
        if (!parsed.message && parsed.msg) {
          parsed.message = parsed.msg
        }
        // TODO JSON GeoIP enrichment
        this.globalTransform(source, parsed)
        return cbf(null, parsed)
      } catch (ex) {
        // ignore treat as text
      }
    }
    var patternList = this.getPatternsForSource(source) || this.patterns
    for (var k in patternList) {
      var patterns = patternList[k]
      for (var i = 0; i < patterns.match.length; i++) {
        if (this.matchPatterns(patterns.match[i], parsed, line)) {
          this.bubbleUp(patternList, k)
          if (!parsed._type) {
            parsed._type = patternList[k].type
          }
          if (this.gloabalTransform) {
            this.globalTransform(source, parsed)
          }
          return cbf(null, parsed)
        }
      }
    }
    return cbf('not found', {'@timestamp': new Date(), message: line, '@source': source}
    )
  }
}

module.exports = LogParser
<|endoftext|>


<commit_before>var child_process = require('child_process');

var q = require('q');
function exec(command, options) {

    var deferred = q.defer();
    var childProcess;

    var args = Array.prototype.slice.call(arguments, 0);
    args.push(function(err, stdout, stderr) {
        if (err) {
            err.message += command + ' (exited with error code ' + err.code + ')';
            err.stdout = stdout;
            err.stderr = stderr;
            deferred.reject(err);
        }
        else {
            deferred.resolve({
                childProcess: childProcess,
                stdout: stdout,
                stderr: stderr
            });
        }
    });

    childProcess = child_process.exec.apply(child_process, args);
    process.nextTick(function() {
        deferred.notify(childProcess);
    });

    return deferred.promise;
}

function spawn(command, args, options) {
    var deferred = q.defer();

    var p = child_process.spawn(command, args, options);
    process.nextTick(function() { // Make sure the callee had a chance to add listeners
        deferred.notify(p);
    });

    p.on('close', function (code) {
        if (code !== 0) {
            var commandStr = command + (args.length ? (' ' + args.join(' ')) : '');
            deferred.reject('"' + commandStr + '" failed with code ' + code);
        }
        else {
            deferred.resolve();
        }
    });

    return deferred.promise;
}
exports.exec = exec;
exports.spawn = spawn;<commit_msg>Fixes spawn behavior for commands that cannot be found<commit_after>var child_process = require('child_process');

var q = require('q');
function exec(command, options) {

    var deferred = q.defer();
    var childProcess;

    var args = Array.prototype.slice.call(arguments, 0);
    args.push(function(err, stdout, stderr) {
        if (err) {
            err.message += command + ' (exited with error code ' + err.code + ')';
            err.stdout = stdout;
            err.stderr = stderr;
            deferred.reject(err);
        }
        else {
            deferred.resolve({
                childProcess: childProcess,
                stdout: stdout,
                stderr: stderr
            });
        }
    });

    childProcess = child_process.exec.apply(child_process, args);
    process.nextTick(function() {
        deferred.notify(childProcess);
    });

    return deferred.promise;
}

function spawn(command, args, options) {
    var deferred = q.defer();

    var p = child_process.spawn(command, args, options);
    process.nextTick(function() { // Make sure the callee had a chance to add listeners
        deferred.notify(p);
    });

    p.on('error', deferred.reject);
    p.on('close', function (code) {
        if (code !== 0) {
            var commandStr = command + (args.length ? (' ' + args.join(' ')) : '');
            deferred.reject('"' + commandStr + '" failed with code ' + code);
        }
        else {
            deferred.resolve();
        }
    });

    return deferred.promise;
}
exports.exec = exec;
exports.spawn = spawn;<|endoftext|>


<commit_before>//Supply the command minus the url (i.e. 'wget -qO - ') and supply 'this'
function setHeaderTable(command, element) {

    //Reset to defaults
    resetHeaderTable(element);

    //Change the table header
    $('#headerTable thead tr th.key').html('Command:');

    //Change the rows
    $('#headerTable > tbody > tr > td.key').each(function () {

        var rowKey = $(this).html();

        if (rowKey.match(/json/i)) {
            $(this).html(command + ' ip.swagger.pro/all');
        }
        else if (rowKey.match(/source-ip/i)) {
            $(this).html(command + ' ip.swagger.pro/');
        }
        else {
            $(this).html(command + ' ip.swagger.pro/' + rowKey.toLowerCase());
        }
    });
}

// Supply 'this'
function resetHeaderTable(element) {

    //Make the clicked nav link active
    $('#headerTableNav').find('li.active').removeClass('active');
    $(element).addClass('active');

    //Change the table header
    $('#headerTable thead tr th.key').html('Item:');

    //Set the rows to default
    $('#headerTable > tbody > tr > td.key').each(function () {

        if (($(this).html()).match(/(.*)[\/]$/i)) {
            $(this).html('source-ip');
        }
        else if (($(this).html()).match(/(.*)[\/]all$/i)) {
            $(this).html('JSON');
        }
        else {
            $(this).html(($(this).html()).replace(/(.*)[\/]/i, ''));
        }
    });
}

$(document).ready(function () {

    $('#browser').on('click', function (e) {
        resetHeaderTable(this);
        return false;
    });

    $('#wget').on('click', function (e) {
        setHeaderTable('wget -qO -', this);
        return false;
    });

    $('#curl').on('click', function (e) {
        setHeaderTable('curl', this);
        return false;
    });

    $('#fetch').on('click', function (e) {
        setHeaderTable('fetch -qo -', this);
        return false;
    });

    $('#powershell').on('click', function (e) {
        setHeaderTable('Invoke-RestMethod -URI http://', this);
        return false;
    });
});<commit_msg>Fixed a spacing issue<commit_after>//Supply the command minus the url (i.e. 'wget -qO - ') and supply 'this'
function setHeaderTable(command, element) {

    //Reset to defaults
    resetHeaderTable(element);

    //Change the table header
    $('#headerTable thead tr th.key').html('Command:');

    //Change the rows
    $('#headerTable > tbody > tr > td.key').each(function () {

        var rowKey = $(this).html();

        if (rowKey.match(/json/i)) {
            $(this).html(command + 'ip.swagger.pro/all');
        }
        else if (rowKey.match(/source-ip/i)) {
            $(this).html(command + 'ip.swagger.pro/');
        }
        else {
            $(this).html(command + 'ip.swagger.pro/' + rowKey.toLowerCase());
        }
    });
}

// Supply 'this'
function resetHeaderTable(element) {

    //Make the clicked nav link active
    $('#headerTableNav').find('li.active').removeClass('active');
    $(element).addClass('active');

    //Change the table header
    $('#headerTable thead tr th.key').html('Item:');

    //Set the rows to default
    $('#headerTable > tbody > tr > td.key').each(function () {

        if (($(this).html()).match(/(.*)[\/]$/i)) {
            $(this).html('source-ip');
        }
        else if (($(this).html()).match(/(.*)[\/]all$/i)) {
            $(this).html('JSON');
        }
        else {
            $(this).html(($(this).html()).replace(/(.*)[\/]/i, ''));
        }
    });
}

$(document).ready(function () {

    $('#browser').on('click', function (e) {
        resetHeaderTable(this);
        return false;
    });

    $('#wget').on('click', function (e) {
        setHeaderTable('wget -qO - ', this);
        return false;
    });

    $('#curl').on('click', function (e) {
        setHeaderTable('curl ', this);
        return false;
    });

    $('#fetch').on('click', function (e) {
        setHeaderTable('fetch -qo - ', this);
        return false;
    });

    $('#powershell').on('click', function (e) {
        setHeaderTable('Invoke-RestMethod -URI http://', this);
        return false;
    });
});<|endoftext|>


<commit_before>var express = require('express');
var url = require('url');
var net = require('net');
var path = require('path');
var fs = require('fs');
var EventEmitter = require('events').EventEmitter;
var exnted = require('util')._extend;
var util = require('../util');
var rules = require('./rules');
var https = require('./https');
var cert = require('./https/cert');
var rulesUtil = require('./rules/util');
var connect = require('./proxy/connect');
var config = require('../package.json');

function tunnelProxy(server, proxy) {

	server.on('connect', function(req, reqSocket, head) {//ws, wss, https proxy
		var tunnelUrl = req.url = util.setProtocol(req.url, true);
		var options = parseUrl();
		var resSocket, proxySocket;

		var port = options.port;
		if (rulesUtil.intercepts(options.hostname + (!port || port == 80 || port == 443 ? '' : ':' + port))) {
			cert.createCert(options.hostname, function(err) {
				if (err) {
					//TODO 通知到抓包页面
				}
				sendEstablished();
				https.dispatch(reqSocket, proxy);
			});
			return;
		}

		abortIfUnavailable(reqSocket);
		function emitError(err) {
			resSocket && resSocket.destroy();
			proxySocket && proxySocket.destroy();
			reqSocket.destroy();
		}

		function abortIfUnavailable(socket) {
			socket.on('error', emitError).on('close', emitError);
		}

		function parseUrl(_tunnelUrl, port) {
			_tunnelUrl = _tunnelUrl || tunnelUrl;
			options = req.options = url.parse(_tunnelUrl);
			options.port = options.port || port || 443;
			return options;
		}

		function sendEstablished() {
			reqSocket.write('HTTP/1.1 200 Connection Established\r\nProxy-Agent: ' + config.name + '\r\n\r\n');
			return reqSocket;
		}

		function tunnel(url) {
			var now = Date.now();
			rules.resolveHost(url || tunnelUrl, function(err, ip) {
				req.host = ip;
				req.clientHost  = reqSocket.remoteAddress;
				req.error = err;
				req.dnsTime = Date.now() - now;
				proxy.emit('tunnel', req);

				if (err) {
					emitError(err);
					return;
				}

				resSocket = net.connect(options.port, ip, function() {
					sendEstablished();
		            resSocket.pipe(reqSocket).pipe(resSocket);
		        });
				abortIfUnavailable(resSocket);
			});
		}

		var _rules = req.rules = rules.resolveRules(tunnelUrl);
		var proxyUrl = util.rule.getMatcher(_rules.proxy);

		if (proxyUrl) {
			var _tunnelUrl = 'https:' + util.removeProtocol(proxyUrl);
			var now = Date.now();
			rules.resolveHost(_tunnelUrl, function(err, ip) {
				req.host = ip;
				req.error = err;
				req.dnsTime = Date.now() - now;
				parseUrl(_tunnelUrl, config.port);
				if (!err && util.isLocalAddress(ip) && options.port == config.port) {
					parseUrl();
					return tunnel();
				}
				proxy.emit('tunnelProxy', req);
				if (err) {
					emitError(err);
					return;
				}
				options.host = ip;
				connect(req, function(err, _proxySocket) {
					if (err) {
						return emitError();
					}
					proxySocket = _proxySocket;
					abortIfUnavailable(_proxySocket);
					sendEstablished().pipe(_proxySocket).pipe(reqSocket);
				});
			});
		} else {
			if (_rules.rule) {
				var _url = util.setProtocol(util.rule.getMatcher(_rules.rule), true);
				if (/^https:/i.test(_url)) {
					parseUrl(_url);
				} else {
					port = util.getProtocol(_url);
					port = port.substring(0, port.length - 1);
					var localProtocol = port + 'sslport';
					if (port = config[localProtocol]) {
						proxy.emit(localProtocol, _url, tunnelUrl);
						parseUrl(_url);
						options.protocol = 'https:';
						options.hostname = '127.0.0.1';
						options.host = options.hostname + ':' + port;
						options.port = port;
						_url = 'ssl://' + options.host; //不解析_url的ip
					} else {
						_url = null;//如果规则无法处理，则直接访问
					}
				}
			}

			tunnel(_url);
		}
    });

	return server;
}

function preventThrowError(socket) {
	socket.removeListener('error', util.noop);
	socket.on('error', util.noop);
}

function proxy(conf) {
	if (conf) {
		['port', 'sockets', 'days', 'timeout',
		 'username', 'password', 'uipath', 'rules', 'debug']
			.forEach(function(name) {
				if (conf[name]) {
					config[name] = conf[name];
				}
			});

		if (Array.isArray(conf.ports)) {
			config.ports = config.ports.concat(conf.ports);
		}
	}

	if (!conf || typeof conf.middlewares != 'string') {
		if (!Array.isArray(conf.middlewares)) {
			config.middlewares = [];
		}
	} else {
		config.middlewares = conf.middlewares.trim().split(/\s*,\s*/g);
	}

	config.middlewares.map(util.resolvePath);

	var port = config.port;
	config.ports.forEach(function(name) {
		if (!/port$/.test(name) || name == 'port') {
			throw new Error('port name "' + name + '" must be end of "port", but not equals "port", like: ' + name + 'port');
		}
		config[name] = ++port;
	});

	var agentConfig = {
				maxSockets: config.sockets,
				keepAlive: config.keepAlive,
				keepAliveMsecs: config.keepAliveMsecs
			};
	config.httpAgent = config.debug ? false : new (require('http').Agent)(agentConfig).on('free', preventThrowError);
	config.httpsAgent = config.debug ? false : new (require('https').Agent)(agentConfig).on('free', preventThrowError);
	config.WEINRE_HOST = 'weinre.' + config.localUIHost;

	var app = express();
	var server = app.listen(config.port);
	var proxyEvents = new EventEmitter();
	var middlewares = ['./init',
	               '../biz']
				   .concat(require('./inspectors'))
				   .concat(config.middlewares)
	               .concat(require('./handlers'));

	proxyEvents.config = config;
	middlewares.forEach(function(mw) {
		mw && app.use((typeof mw == 'string' ? require(mw) : mw).bind(proxyEvents));
	});

	parseHosts(config);
	exportInterfaces(proxyEvents);
	tunnelProxy(server, proxyEvents);
	https.createServer(proxyEvents);
	require('./proxy')(proxyEvents);
	require('../biz/init')(proxyEvents);

	return proxyEvents;
}

function exportInterfaces(obj) {
	obj.rules = rules;
	obj.util = util;
	obj.rulesUtil = rulesUtil;
	return obj;
}

function parseHosts(config) {
	var rulesPath = config.rules;
	if (rulesPath) {
		try {
			rulesUtil.setPublicHosts(fs.readFileSync(path.resolve(rulesPath), {encoding: 'utf8'}));
			rulesUtil.enablePublicHosts();
		} catch(e) {}
	}

	rulesUtil.loadHosts(config);
}

process.on('uncaughtException', function(err){
	var stack = err.message;
	try {
		stack += '\r\n' + err.stack;
	} catch(e) {}
	fs.writeFileSync(path.join(process.cwd(), config.name + '.log'), '\r\n' + util.formatDate()  + '\r\n' +  stack + '\r\n', {flag: 'a'});
	console.error(stack);
	process.exit(1);
});

module.exports = exportInterfaces(proxy);<commit_msg>add crud for https rules<commit_after>var express = require('express');
var url = require('url');
var net = require('net');
var path = require('path');
var fs = require('fs');
var EventEmitter = require('events').EventEmitter;
var exnted = require('util')._extend;
var util = require('../util');
var rules = require('./rules');
var https = require('./https');
var cert = require('./https/cert');
var rulesUtil = require('./rules/util');
var connect = require('./proxy/connect');
var config = require('../package.json');

function tunnelProxy(server, proxy) {

	server.on('connect', function(req, reqSocket, head) {//ws, wss, https proxy
		var tunnelUrl = req.url = util.setProtocol(req.url, true);
		var options = parseUrl();
		var resSocket, proxySocket;

		if (rulesUtil.intercepts(options.hostname)) {
			cert.createCert(options.hostname, function(err) {
				if (err) {
					//TODO 通知到抓包页面
				}
				sendEstablished();
				https.dispatch(reqSocket, proxy);
			});
			return;
		}

		abortIfUnavailable(reqSocket);
		function emitError(err) {
			resSocket && resSocket.destroy();
			proxySocket && proxySocket.destroy();
			reqSocket.destroy();
		}

		function abortIfUnavailable(socket) {
			socket.on('error', emitError).on('close', emitError);
		}

		function parseUrl(_tunnelUrl, port) {
			_tunnelUrl = _tunnelUrl || tunnelUrl;
			options = req.options = url.parse(_tunnelUrl);
			options.port = options.port || port || 443;
			return options;
		}

		function sendEstablished() {
			reqSocket.write('HTTP/1.1 200 Connection Established\r\nProxy-Agent: ' + config.name + '\r\n\r\n');
			return reqSocket;
		}

		function tunnel(url) {
			var now = Date.now();
			rules.resolveHost(url || tunnelUrl, function(err, ip) {
				req.host = ip;
				req.clientHost  = reqSocket.remoteAddress;
				req.error = err;
				req.dnsTime = Date.now() - now;
				proxy.emit('tunnel', req);

				if (err) {
					emitError(err);
					return;
				}

				resSocket = net.connect(options.port, ip, function() {
					sendEstablished();
		            resSocket.pipe(reqSocket).pipe(resSocket);
		        });
				abortIfUnavailable(resSocket);
			});
		}

		var _rules = req.rules = rules.resolveRules(tunnelUrl);
		var proxyUrl = util.rule.getMatcher(_rules.proxy);

		if (proxyUrl) {
			var _tunnelUrl = 'https:' + util.removeProtocol(proxyUrl);
			var now = Date.now();
			rules.resolveHost(_tunnelUrl, function(err, ip) {
				req.host = ip;
				req.error = err;
				req.dnsTime = Date.now() - now;
				parseUrl(_tunnelUrl, config.port);
				if (!err && util.isLocalAddress(ip) && options.port == config.port) {
					parseUrl();
					return tunnel();
				}
				proxy.emit('tunnelProxy', req);
				if (err) {
					emitError(err);
					return;
				}
				options.host = ip;
				connect(req, function(err, _proxySocket) {
					if (err) {
						return emitError();
					}
					proxySocket = _proxySocket;
					abortIfUnavailable(_proxySocket);
					sendEstablished().pipe(_proxySocket).pipe(reqSocket);
				});
			});
		} else {
			if (_rules.rule) {
				var _url = util.setProtocol(util.rule.getMatcher(_rules.rule), true);
				if (/^https:/i.test(_url)) {
					parseUrl(_url);
				} else {
					var port = util.getProtocol(_url);
					port = port.substring(0, port.length - 1);
					var localProtocol = port + 'sslport';
					if (port = config[localProtocol]) {
						proxy.emit(localProtocol, _url, tunnelUrl);
						parseUrl(_url);
						options.protocol = 'https:';
						options.hostname = '127.0.0.1';
						options.host = options.hostname + ':' + port;
						options.port = port;
						_url = 'ssl://' + options.host; //不解析_url的ip
					} else {
						_url = null;//如果规则无法处理，则直接访问
					}
				}
			}

			tunnel(_url);
		}
    });

	return server;
}

function preventThrowError(socket) {
	socket.removeListener('error', util.noop);
	socket.on('error', util.noop);
}

function proxy(conf) {
	if (conf) {
		['port', 'sockets', 'days', 'timeout',
		 'username', 'password', 'uipath', 'rules', 'debug']
			.forEach(function(name) {
				if (conf[name]) {
					config[name] = conf[name];
				}
			});

		if (Array.isArray(conf.ports)) {
			config.ports = config.ports.concat(conf.ports);
		}
	}

	if (!conf || typeof conf.middlewares != 'string') {
		if (!Array.isArray(conf.middlewares)) {
			config.middlewares = [];
		}
	} else {
		config.middlewares = conf.middlewares.trim().split(/\s*,\s*/g);
	}

	config.middlewares.map(util.resolvePath);

	var port = config.port;
	config.ports.forEach(function(name) {
		if (!/port$/.test(name) || name == 'port') {
			throw new Error('port name "' + name + '" must be end of "port", but not equals "port", like: ' + name + 'port');
		}
		config[name] = ++port;
	});

	var agentConfig = {
				maxSockets: config.sockets,
				keepAlive: config.keepAlive,
				keepAliveMsecs: config.keepAliveMsecs
			};
	config.httpAgent = config.debug ? false : new (require('http').Agent)(agentConfig).on('free', preventThrowError);
	config.httpsAgent = config.debug ? false : new (require('https').Agent)(agentConfig).on('free', preventThrowError);
	config.WEINRE_HOST = 'weinre.' + config.localUIHost;

	var app = express();
	var server = app.listen(config.port);
	var proxyEvents = new EventEmitter();
	var middlewares = ['./init',
	               '../biz']
				   .concat(require('./inspectors'))
				   .concat(config.middlewares)
	               .concat(require('./handlers'));

	proxyEvents.config = config;
	middlewares.forEach(function(mw) {
		mw && app.use((typeof mw == 'string' ? require(mw) : mw).bind(proxyEvents));
	});

	parseHosts(config);
	exportInterfaces(proxyEvents);
	tunnelProxy(server, proxyEvents);
	https.createServer(proxyEvents);
	require('./proxy')(proxyEvents);
	require('../biz/init')(proxyEvents);

	return proxyEvents;
}

function exportInterfaces(obj) {
	obj.rules = rules;
	obj.util = util;
	obj.rulesUtil = rulesUtil;
	return obj;
}

function parseHosts(config) {
	var rulesPath = config.rules;
	if (rulesPath) {
		try {
			rulesUtil.setPublicHosts(fs.readFileSync(path.resolve(rulesPath), {encoding: 'utf8'}));
			rulesUtil.enablePublicHosts();
		} catch(e) {}
	}

	rulesUtil.loadHosts(config);
}

process.on('uncaughtException', function(err){
	var stack = err.message;
	try {
		stack += '\r\n' + err.stack;
	} catch(e) {}
	fs.writeFileSync(path.join(process.cwd(), config.name + '.log'), '\r\n' + util.formatDate()  + '\r\n' +  stack + '\r\n', {flag: 'a'});
	console.error(stack);
	process.exit(1);
});

module.exports = exportInterfaces(proxy);<|endoftext|>


<commit_before>/**
@module ember
@submodule ember-application
@private
*/

import EmberObject from "ember-runtime/system/object";
import run from "ember-metal/run_loop";

/**
  The `ApplicationInstance` encapsulates all of the stateful aspects of a
  running `Application`.

  At a high-level, we break application boot into two distinct phases:

  * Definition time, where all of the classes, templates, and other
    dependencies are loaded (typically in the browser).
  * Run time, where we begin executing the application once everything
    has loaded.

  Definition time can be expensive and only needs to happen once since it is
  an idempotent operation. For example, between test runs and FastBoot
  requests, the application stays the same. It is only the state that we want
  to reset.

  That state is what the `ApplicationInstance` manages: it is responsible for
  creating the container that contains all application state, and disposing of
  it once the particular test run or FastBoot request has finished.
*/

export default EmberObject.extend({
  /**
    The application instance's container. The container stores all of the
    instance-specific state for this application run.

    @property {Ember.Container} container
  */
  container: null,

  /**
    The application's registry. The registry contains the classes, templates,
    and other code that makes up the application.

    @property {Ember.Registry} registry
  */
  registry: null,

  /**
    The DOM events for which the event dispatcher should listen.

    By default, the application's `Ember.EventDispatcher` listens
    for a set of standard DOM events, such as `mousedown` and
    `keyup`, and delegates them to your application's `Ember.View`
    instances.

    @private
    @property {Object} customEvents
  */
  customEvents: null,

  /**
    The root DOM element of the Application as an element or a
    [jQuery-compatible selector
    string](http://api.jquery.com/category/selectors/).

    @private
    @property {String|DOMElement} rootElement
  */
  rootElement: null,

  init: function() {
    this._super();
    this.container = this.registry.container();
  },

  /**
    @private
  */
  startRouting: function(isModuleBasedResolver) {
    var router = this.container.lookup('router:main');
    if (!router) { return; }

    router.startRouting(isModuleBasedResolver);
  },

  /**
    @private
  */
  handleURL: function(url) {
    var router = this.container.lookup('router:main');

    return router.handleURL(url);
  },

  /**
    @private
  */
  setupEventDispatcher: function() {
    var dispatcher = this.container.lookup('event_dispatcher:main');

    dispatcher.setup(this.customEvents, this.rootElement);

    return dispatcher;
  },

  /**
    @private
  */
  willDestroy: function() {
    this._super.apply(this, arguments);
    run(this.container, 'destroy');
  }
});
<commit_msg>Pass arguments to _super<commit_after>/**
@module ember
@submodule ember-application
@private
*/

import EmberObject from "ember-runtime/system/object";
import run from "ember-metal/run_loop";

/**
  The `ApplicationInstance` encapsulates all of the stateful aspects of a
  running `Application`.

  At a high-level, we break application boot into two distinct phases:

  * Definition time, where all of the classes, templates, and other
    dependencies are loaded (typically in the browser).
  * Run time, where we begin executing the application once everything
    has loaded.

  Definition time can be expensive and only needs to happen once since it is
  an idempotent operation. For example, between test runs and FastBoot
  requests, the application stays the same. It is only the state that we want
  to reset.

  That state is what the `ApplicationInstance` manages: it is responsible for
  creating the container that contains all application state, and disposing of
  it once the particular test run or FastBoot request has finished.
*/

export default EmberObject.extend({
  /**
    The application instance's container. The container stores all of the
    instance-specific state for this application run.

    @property {Ember.Container} container
  */
  container: null,

  /**
    The application's registry. The registry contains the classes, templates,
    and other code that makes up the application.

    @property {Ember.Registry} registry
  */
  registry: null,

  /**
    The DOM events for which the event dispatcher should listen.

    By default, the application's `Ember.EventDispatcher` listens
    for a set of standard DOM events, such as `mousedown` and
    `keyup`, and delegates them to your application's `Ember.View`
    instances.

    @private
    @property {Object} customEvents
  */
  customEvents: null,

  /**
    The root DOM element of the Application as an element or a
    [jQuery-compatible selector
    string](http://api.jquery.com/category/selectors/).

    @private
    @property {String|DOMElement} rootElement
  */
  rootElement: null,

  init: function() {
    this._super.apply(this, arguments);
    this.container = this.registry.container();
  },

  /**
    @private
  */
  startRouting: function(isModuleBasedResolver) {
    var router = this.container.lookup('router:main');
    if (!router) { return; }

    router.startRouting(isModuleBasedResolver);
  },

  /**
    @private
  */
  handleURL: function(url) {
    var router = this.container.lookup('router:main');

    return router.handleURL(url);
  },

  /**
    @private
  */
  setupEventDispatcher: function() {
    var dispatcher = this.container.lookup('event_dispatcher:main');

    dispatcher.setup(this.customEvents, this.rootElement);

    return dispatcher;
  },

  /**
    @private
  */
  willDestroy: function() {
    this._super.apply(this, arguments);
    run(this.container, 'destroy');
  }
});
<|endoftext|>


<commit_before>/**
 * CbBuffer
 * Creates a new instance of `CbBuffer`.
 *
 * Example:
 *
 * ```js
 * var cb = new CbBuffer();
 * ```
 *
 * @name CbBuffer
 * @function
 * @return {Object} The `CbBuffer` instance containing:
 *
 *  - [`done` (Function)](#doneclb)
 *  - [`wait` (Function)](#wait)
 *  - [`callback` (Function)](#callback)
 *  - [`add` (Function)](#addfunc)
 *  - `buffer` (Array): An array with functions that will be
 *    called when the `callback` method is called.
 *  - `isWaiting` (Boolean): A flag that is `true` after
 *     calling the `wait` method.
 */
module.exports = function () {

    var self = this;

    // Attach the buffer array.
    self.buffer = [];

    // Set the `isWaiting` flat to `false`.
    self.isWaiting = false;

    /**
     * done
     * Calls the callback function provided as the first
     * parameter with cached arguments.
     *
     * @name done
     * @function
     * @param {Function} clb The callback function to be called.
     * @return {Object} The `CbBuffer` instance.
     */
    self.done = function (clb) {
        clb.apply(this, self._args);
    };

    /**
     * wait
     * Sets `isWaiting` flag to `true`.
     *
     * @name wait
     * @function
     * @return {Object} The `CbBuffer` instance.
     */
    self.wait = function () {
        self.isWaiting = true;
    };

    /**
     * callback
     * Fires all callbacks from the buffer end empties the it.
     *
     * @name callback
     * @function
     * @return {Object} The `CbBuffer` instance.
     */
    self.callback = function () {
        self._args = arguments;
        for (var i = 0; i < self.buffer.length; ++i) {
            self.done.call(this, self.buffer[i]);
        }
        self.buffer = [];
        self.isDone = true;
        self.isWaiting = false;
    };

    /**
     * add
     * Adds functions to the current buffer.
     *
     * @name add
     * @function
     * @param {Function} func The function that should be buffered.
     * @return {Object} The `CbBuffer` instance.
     */
    self.add = function (func) {
        self.buffer.push(func);
    };
};
<commit_msg>Fixed return values. Fixed #2.<commit_after>/**
 * CbBuffer
 * Creates a new instance of `CbBuffer`.
 *
 * Example:
 *
 * ```js
 * var cb = new CbBuffer();
 * ```
 *
 * @name CbBuffer
 * @function
 * @return {Object} The `CbBuffer` instance containing:
 *
 *  - [`done` (Function)](#doneclb)
 *  - [`wait` (Function)](#wait)
 *  - [`callback` (Function)](#callback)
 *  - [`add` (Function)](#addfunc)
 *  - `buffer` (Array): An array with functions that will be
 *    called when the `callback` method is called.
 *  - `isWaiting` (Boolean): A flag that is `true` after
 *     calling the `wait` method.
 */
module.exports = function () {

    var self = this;

    // Attach the buffer array.
    self.buffer = [];

    // Set the `isWaiting` flat to `false`.
    self.isWaiting = false;

    /**
     * done
     * Calls the callback function provided as the first
     * parameter with cached arguments.
     *
     * @name done
     * @function
     * @param {Function} clb The callback function to be called.
     * @return {Object} The `CbBuffer` instance.
     */
    self.done = function (clb) {
        clb.apply(this, self._args);
        return self;
    };

    /**
     * wait
     * Sets `isWaiting` flag to `true`.
     *
     * @name wait
     * @function
     * @return {Object} The `CbBuffer` instance.
     */
    self.wait = function () {
        self.isWaiting = true;
        return self;
    };

    /**
     * callback
     * Fires all callbacks from the buffer end empties the it.
     *
     * @name callback
     * @function
     * @return {Object} The `CbBuffer` instance.
     */
    self.callback = function () {
        self._args = arguments;
        for (var i = 0; i < self.buffer.length; ++i) {
            self.done.call(this, self.buffer[i]);
        }
        self.buffer = [];
        self.isDone = true;
        self.isWaiting = false;
        return self;
    };

    /**
     * add
     * Adds functions to the current buffer.
     *
     * @name add
     * @function
     * @param {Function} func The function that should be buffered.
     * @return {Object} The `CbBuffer` instance.
     */
    self.add = function (func) {
        self.buffer.push(func);
        return self;
    };
};
<|endoftext|>


<commit_before>/**
 * @title Move user_metadata attributes to profile root attributes.
 * @overview Moves select data from user_metadata to profile root attributes (family_name, given_name, name, nickname and picture).
 * @gallery false
 * @category enrich profile
 *
 * This rule moves select data from user_metadata to profile root attributes (family_name, given_name, name, nickname and picture).
 * Verify the field mapping before enabling this rule.
 * The rule will determine if there is a mapped field on the user_metadata before the update.
 * Important:
 *
 * 1- The rule updates the profile root attribute with the mapped field from user_metadata.
 * 2- The mapped fields from user_metadata will be removed following the update.
 * 3- This rule will be executed on each login event. Consider using it only if you use a custom signup form or Authentication Signup API.
 */
function (user, context, cb) {
  // Field Mapping, the property is the root attribute and the value is the field name on user_metadata.
  // You can change the value in case you don't have the same name on user_metadata.
  var fieldMapping = {
    family_name: 'family_name',
    given_name: 'given_name',
    name: 'name',
    nickname: 'nickname',
    picture: 'picture'
  };

  if (needMigration(user)) {
    var ManagementClient = require('auth0@2.9.1').ManagementClient;
    var management = new ManagementClient({
      domain: auth0.domain,
      token: auth0.accessToken
    });

    management.updateUser(
      { id: user.user_id }, generateUserPayload(user), function (err, updatedUser) {
        if ( err ) {
          cb(err);
        } else {
          updateRuleUser(user, updatedUser);
          cb(null, user, context);
        }
      }
    );
  } else {
    cb(null, user, context);
  }

  function needMigration(user) {
    if (user.user_metadata) {
      for (var key in fieldMapping) {
        if (typeof user.user_metadata[fieldMapping[key]] === 'string') {
          return true;
        }
      }
    }

    return false;
  }

  function generateUserPayload(user) {
    var payload = { user_metadata: {}};
    var userMetadata = user.user_metadata;

    for (var key in fieldMapping) {
      generateUserPayloadField(userMetadata, payload, key, fieldMapping[key]);
    }

    return payload;
  }

  function updateRuleUser(user, updatedUser) {
    for (var key in fieldMapping) {
      if (typeof user.user_metadata[fieldMapping[key]] === 'string') {
        user[key] = updatedUser[key];
        delete user.user_metadata[fieldMapping[key]];
      }
    }
  }

  function generateUserPayloadField(userMetadata, payload, rootField, metadataField) {
    if (typeof userMetadata[metadataField] === 'string') {
      payload[rootField] = userMetadata[metadataField];
      payload.user_metadata[metadataField] = null;
    }
  }
}
<commit_msg>IUM-415: Enable the root attribute migration script<commit_after>/**
 * @title Move user_metadata attributes to profile root attributes.
 * @overview Moves select data from user_metadata to profile root attributes (family_name, given_name, name, nickname and picture).
 * @gallery true
 * @category enrich profile
 *
 * This rule moves select data from user_metadata to profile root attributes (family_name, given_name, name, nickname and picture).
 * Verify the field mapping before enabling this rule.
 * The rule will determine if there is a mapped field on the user_metadata before the update.
 * Important:
 *
 * 1- The rule updates the profile root attribute with the mapped field from user_metadata.
 * 2- The mapped fields from user_metadata will be removed following the update.
 * 3- This rule will be executed on each login event. Consider using it only if you use a custom signup form or Authentication Signup API.
 */
function (user, context, cb) {
  // Field Mapping, the property is the root attribute and the value is the field name on user_metadata.
  // You can change the value in case you don't have the same name on user_metadata.
  var fieldMapping = {
    family_name: 'family_name',
    given_name: 'given_name',
    name: 'name',
    nickname: 'nickname',
    picture: 'picture'
  };

  if (needMigration(user)) {
    var ManagementClient = require('auth0@2.9.1').ManagementClient;
    var management = new ManagementClient({
      domain: auth0.domain,
      token: auth0.accessToken
    });

    management.updateUser(
      { id: user.user_id }, generateUserPayload(user), function (err, updatedUser) {
        if ( err ) {
          cb(err);
        } else {
          updateRuleUser(user, updatedUser);
          cb(null, user, context);
        }
      }
    );
  } else {
    cb(null, user, context);
  }

  function needMigration(user) {
    if (user.user_metadata) {
      for (var key in fieldMapping) {
        if (typeof user.user_metadata[fieldMapping[key]] === 'string') {
          return true;
        }
      }
    }

    return false;
  }

  function generateUserPayload(user) {
    var payload = { user_metadata: {}};
    var userMetadata = user.user_metadata;

    for (var key in fieldMapping) {
      generateUserPayloadField(userMetadata, payload, key, fieldMapping[key]);
    }

    return payload;
  }

  function updateRuleUser(user, updatedUser) {
    for (var key in fieldMapping) {
      if (typeof user.user_metadata[fieldMapping[key]] === 'string') {
        user[key] = updatedUser[key];
        delete user.user_metadata[fieldMapping[key]];
      }
    }
  }

  function generateUserPayloadField(userMetadata, payload, rootField, metadataField) {
    if (typeof userMetadata[metadataField] === 'string') {
      payload[rootField] = userMetadata[metadataField];
      payload.user_metadata[metadataField] = null;
    }
  }
}
<|endoftext|>


<commit_before>// @flow
/**
 * Part of GDL gdl-frontend.
 * Copyright (C) 2018 GDL
 *
 * See LICENSE
 */

import React, { type Node } from 'react';
import { Trans } from '@lingui/react';
import {
  Drawer,
  Divider,
  List,
  ListSubheader,
  ListItem,
  ListItemText,
  Typography
} from '@material-ui/core';

import { getBookLanguageCode } from '../../lib/storage';
import type { ReadingLevel } from '../../types';
import { fetchCategories } from '../../fetch';
import Link from '../BrowseLink';
import ReadingLevelTrans from '../ReadingLevelTrans';

type Props = {|
  children: (data: { onClick: () => void, loading: boolean }) => Node,
  onSelectCategory: () => void
|};

type CategoriesType = {
  classroom_books?: Array<ReadingLevel>,
  library_books?: Array<ReadingLevel>
};

export default class CategoriesMenu extends React.Component<
  Props,
  {
    showMenu: boolean,
    categories: ?'LOADING' | 'ERROR' | CategoriesType,
    languageCode: string
  }
> {
  state = {
    showMenu: false,
    categories: null,
    languageCode: getBookLanguageCode()
  };

  /**
   * We only load the categories if the menu is "open" and we're in an uninitialized state
   */
  componentDidUpdate() {
    if (this.state.showMenu && this.state.categories == null) {
      this.loadCategories();
    }
  }

  loadCategories = async () => {
    this.setState({ categories: 'LOADING' });
    const result = await fetchCategories(this.state.languageCode);

    this.setState({
      categories: result.isOk ? result.data : 'ERROR'
    });
  };

  handleShowMenu = () => this.setState({ showMenu: true });

  handleCloseMenu = () => this.setState({ showMenu: false });

  render() {
    const { children, onSelectCategory } = this.props;
    const { categories, showMenu, languageCode } = this.state;
    return (
      <>
        {children({
          onClick: this.handleShowMenu,
          loading: categories === 'LOADING'
        })}
        <Drawer
          open={showMenu && categories !== 'LOADING'}
          onClose={this.handleCloseMenu}
        >
          {categories === 'ERROR' && (
            <Typography component="span" color="error" css={{ margin: '1rem' }}>
              <Trans>Error loading data.</Trans>
            </Typography>
          )}

          {categories && categories !== 'LOADING' && categories !== 'ERROR' && (
            <Categories
              onSelectCategory={onSelectCategory}
              categories={categories}
              languageCode={languageCode}
            />
          )}
        </Drawer>
      </>
    );
  }
}

const Categories = ({ categories, onSelectCategory, languageCode }) => (
  <List component="nav">
    {categories.classroom_books && (
      <>
        <ListSubheader component="div">
          <Trans>Classroom books</Trans>
        </ListSubheader>
        {categories.classroom_books.map(level => (
          <Link
            key={level}
            lang={languageCode}
            readingLevel={level}
            category="classroom_books"
            passHref
          >
            <ListItem onClick={onSelectCategory} button component="a">
              <ListItemText inset>
                <ReadingLevelTrans readingLevel={level} />
              </ListItemText>
            </ListItem>
          </Link>
        ))}
        <Link
          category="classroom_books"
          lang={languageCode}
          sort="-arrivalDate"
          passHref
        >
          <ListItem onClick={onSelectCategory} button component="a">
            <ListItemText inset>
              <Trans>New arrivals</Trans>
            </ListItemText>
          </ListItem>
        </Link>
      </>
    )}

    {Boolean(categories.library_books) &&
      Boolean(categories.classroom_books) && <Divider />}

    {categories.library_books && (
      <>
        <ListSubheader component="div">
          <Trans>Library books</Trans>
        </ListSubheader>
        {categories.library_books.map(level => (
          <Link
            key={level}
            lang={languageCode}
            readingLevel={level}
            category="library_books"
          >
            <ListItem onClick={onSelectCategory} button component="a">
              <ListItemText inset>
                <ReadingLevelTrans readingLevel={level} />
              </ListItemText>
            </ListItem>
          </Link>
        ))}
        <Link category="library_books" lang={languageCode} sort="-arrivalDate">
          <ListItem button onClick={onSelectCategory}>
            <ListItemText inset>
              <Trans>New arrivals</Trans>
            </ListItemText>
          </ListItem>
        </Link>
      </>
    )}
  </List>
);
<commit_msg>Moved new arrivals to top of the global menu<commit_after>// @flow
/**
 * Part of GDL gdl-frontend.
 * Copyright (C) 2018 GDL
 *
 * See LICENSE
 */

import React, { type Node } from 'react';
import { Trans } from '@lingui/react';
import {
  Drawer,
  Divider,
  List,
  ListSubheader,
  ListItem,
  ListItemText,
  Typography
} from '@material-ui/core';

import { getBookLanguageCode } from '../../lib/storage';
import type { ReadingLevel } from '../../types';
import { fetchCategories } from '../../fetch';
import Link from '../BrowseLink';
import ReadingLevelTrans from '../ReadingLevelTrans';

type Props = {|
  children: (data: { onClick: () => void, loading: boolean }) => Node,
  onSelectCategory: () => void
|};

type CategoriesType = {
  classroom_books?: Array<ReadingLevel>,
  library_books?: Array<ReadingLevel>
};

export default class CategoriesMenu extends React.Component<
  Props,
  {
    showMenu: boolean,
    categories: ?'LOADING' | 'ERROR' | CategoriesType,
    languageCode: string
  }
> {
  state = {
    showMenu: false,
    categories: null,
    languageCode: getBookLanguageCode()
  };

  /**
   * We only load the categories if the menu is "open" and we're in an uninitialized state
   */
  componentDidUpdate() {
    if (this.state.showMenu && this.state.categories == null) {
      this.loadCategories();
    }
  }

  loadCategories = async () => {
    this.setState({ categories: 'LOADING' });
    const result = await fetchCategories(this.state.languageCode);

    this.setState({
      categories: result.isOk ? result.data : 'ERROR'
    });
  };

  handleShowMenu = () => this.setState({ showMenu: true });

  handleCloseMenu = () => this.setState({ showMenu: false });

  render() {
    const { children, onSelectCategory } = this.props;
    const { categories, showMenu, languageCode } = this.state;
    return (
      <>
        {children({
          onClick: this.handleShowMenu,
          loading: categories === 'LOADING'
        })}
        <Drawer
          open={showMenu && categories !== 'LOADING'}
          onClose={this.handleCloseMenu}
        >
          {categories === 'ERROR' && (
            <Typography component="span" color="error" css={{ margin: '1rem' }}>
              <Trans>Error loading data.</Trans>
            </Typography>
          )}

          {categories && categories !== 'LOADING' && categories !== 'ERROR' && (
            <Categories
              onSelectCategory={onSelectCategory}
              categories={categories}
              languageCode={languageCode}
            />
          )}
        </Drawer>
      </>
    );
  }
}

const Categories = ({ categories, onSelectCategory, languageCode }) => (
  <List component="nav">
    {categories.classroom_books && (
      <>
        <ListSubheader component="div">
          <Trans>Classroom books</Trans>
        </ListSubheader>
        {categories.classroom_books.map(level => (
          <Link
            key={level}
            lang={languageCode}
            readingLevel={level}
            category="classroom_books"
            passHref
          >
            <ListItem onClick={onSelectCategory} button component="a">
              <ListItemText inset>
                <ReadingLevelTrans readingLevel={level} />
              </ListItemText>
            </ListItem>
          </Link>
        ))}
        <Link
          category="classroom_books"
          lang={languageCode}
          sort="-arrivalDate"
          passHref
        >
          <ListItem onClick={onSelectCategory} button component="a">
            <ListItemText inset>
              <Trans>New arrivals</Trans>
            </ListItemText>
          </ListItem>
        </Link>
      </>
    )}

    {Boolean(categories.library_books) &&
      Boolean(categories.classroom_books) && <Divider />}

    {categories.library_books && (
      <>
        <ListSubheader component="div">
          <Trans>Library books</Trans>
        </ListSubheader>
        <Link category="library_books" lang={languageCode} sort="-arrivalDate">
          <ListItem button onClick={onSelectCategory}>
            <ListItemText inset>
              <Trans>New arrivals</Trans>
            </ListItemText>
          </ListItem>
        </Link>
        {categories.library_books.map(level => (
          <Link
            key={level}
            lang={languageCode}
            readingLevel={level}
            category="library_books"
          >
            <ListItem onClick={onSelectCategory} button component="a">
              <ListItemText inset>
                <ReadingLevelTrans readingLevel={level} />
              </ListItemText>
            </ListItem>
          </Link>
        ))}
      </>
    )}
  </List>
);
<|endoftext|>


<commit_before>'use strict';

const KeenQuery = require('./keen-query');
const parser = require('./parser');
const Aggregator = require('./aggregator');
const filters = require('./keen-query/filters');

function stringifyFunctionConfigs (functions, indenter) {
	return functions.map(func => {
		return `${indenter}->${func.func}(${func.params.join(',')})`
	});
}

function build (queryConf, name) {
	if (typeof queryConf === 'string') {
		queryConf = parser(queryConf);
	}

	if (queryConf.aggregator) {
		return Aggregator.factory(queryConf)
	}

	const keenQuery = new KeenQuery(queryConf.event);
	keenQuery.name = name;
	keenQuery._postProcessors = queryConf.postProcessing;
	return queryConf.functions.reduce((keenQuery, transform) => {
		if (transform.func === 'print') {
			keenQuery._printer = transform.params[0];
			return keenQuery;
		}
		return keenQuery[transform.func].apply(keenQuery, transform.params || []);
	}, keenQuery);
}

module.exports = KeenQuery;
module.exports.build = build;
module.exports.execute = str => build(str).print();
module.exports.parseFilter = filters.parse;

module.exports.Aggregator = Aggregator;
module.exports.aggregate = Aggregator.aggregate;

module.exports.defineAggregator = (name, constructor) => {
	return Aggregator.define(name, constructor);
}

module.exports.format = function format (str, indenter, isSingleLine) {

	let lineBreak = '\n';
	if (isSingleLine) {
		lineBreak = indenter = '';
	}
	const tree = typeof str === 'object' ? str : parser(str);

	if (tree.type === 'AggregateQuery') {
		return `@${tree.aggregator}(${lineBreak}\
${indenter}${tree.body.map(qs => format(qs, indenter, true)).join(`,${lineBreak}${indenter}`)}${lineBreak}\
)${lineBreak}\
${stringifyFunctionConfigs(tree.functions, indenter).join(lineBreak)}`
	} else {
		return `${tree.event}${lineBreak}${stringifyFunctionConfigs(tree.functions, indenter).join(lineBreak)}`
	}
}
<commit_msg>neater nested indentation (#129)<commit_after>'use strict';

const KeenQuery = require('./keen-query');
const parser = require('./parser');
const Aggregator = require('./aggregator');
const filters = require('./keen-query/filters');

function build (queryConf, name) {
	if (typeof queryConf === 'string') {
		queryConf = parser(queryConf);
	}

	if (queryConf.aggregator) {
		return Aggregator.factory(queryConf)
	}

	const keenQuery = new KeenQuery(queryConf.event);
	keenQuery.name = name;
	keenQuery._postProcessors = queryConf.postProcessing;
	return queryConf.functions.reduce((keenQuery, transform) => {
		if (transform.func === 'print') {
			keenQuery._printer = transform.params[0];
			return keenQuery;
		}
		return keenQuery[transform.func].apply(keenQuery, transform.params || []);
	}, keenQuery);
}

module.exports = KeenQuery;
module.exports.build = build;
module.exports.execute = str => build(str).print();
module.exports.parseFilter = filters.parse;

module.exports.Aggregator = Aggregator;
module.exports.aggregate = Aggregator.aggregate;

module.exports.defineAggregator = (name, constructor) => {
	return Aggregator.define(name, constructor);
}

function stringifyFunctionConfigs (functions, indenter) {
	return functions.map(func => {
		return `${indenter}->${func.func}(${func.params.join(',')})`
	});
}

module.exports.format = function format (str, indenter, isSingleLine, startingIndent) {
	// note down how far to indent each line by default, which allows deep nesting
	startingIndent = startingIndent || '';

	// If nested we line break and indent as our 'newline' character
	// If on single line we use an empty string
	let lineBreak = '\n' + startingIndent;
	if (isSingleLine) {
		lineBreak = indenter = '';
	}

	// handle ASTs or raw keen-query strings
	const tree = typeof str === 'object' ? str : parser(str);

	if (tree.type === 'AggregateQuery') {
		// recursively format each subquery in the aggregate's body, indent them, and start a new line for each one
		const stringifiedBody = tree.body.map(qs => format(qs, indenter, isSingleLine, startingIndent + indenter))
			.map(str => indenter + str)
			.join(`,${lineBreak}`);

		// if there are shared functions print them all on a new line each
		const sharedFunctions = tree.functions.length ? (lineBreak + stringifyFunctionConfigs(tree.functions, indenter).join(lineBreak)) : '';
		return `@${tree.aggregator}(${lineBreak}\
${stringifiedBody}${lineBreak}\
)${sharedFunctions}`
	} else {
		const functions = stringifyFunctionConfigs(tree.functions, indenter).join(lineBreak);
		return `${tree.event}${lineBreak}${functions}`
	}
}
<|endoftext|>


<commit_before>const CommandHandler = require('../util/msgUtil');
const toHex = require('colornames');
const Discordie = require('discordie');
const Events = Discordie.Events;

const { log } = require('../util/botUtil');
const { getCodePoint } = require('../util/stringUtil');

const config = require('../config');

/**
 *
 *
 * @param {IMessage} msg
 * @param {Discordie} client
 * @returns
 */
function respond (msg, client) {
	const { content: msgText, channel, guild, member: sender } = msg;
	if (!msgText.startsWith(config.prefix)) return;
	const botMember = client.User.memberOf(guild);

	/**
	 * Sends a message
	 * @param {string} s Text
	 * @param {object} e Embed object
	 * @returns {Promise<{}>}
	 */
	const sendMessage = (s, e) => channel.sendMessage(s, false, e);
	// const sendEmbed = (e) => sendMessage('', e);

	const command = msgText.slice(config.prefix.length);
	const handler = new CommandHandler(command);

	const addCommand = (c, f) => handler.addCommand(c, f);
	// const addCommandResponse = (c, r) => addCommand(c, () => sendMessage(r));
	/**
	 *
	 * @param {string} c
	 * @param {function(string)} f
	 */
	const addCommandSentence = (c, f) => handler.addCommandSentence(c, f);
	// const addCommandArgs = (c, f) => handler.addCommandArgs(c, f);

	const Emojis = {
		ONE: [0x31, 0x20E3],
		TWO: [0x32, 0x20E3],
		THREE: [0x33, 0x20E3],
		FOUR: [0x34, 0x20E3],
		FIVE: [0x35, 0x20E3]
	};

	addCommand('panel', () => {
		sendMessage([
			'1. -',
			'2. -',
			'3. -',
			'4. -',
			'5. -'
		].join('\n')).then((msg, err) => {
			if (err) return log(err);

			addSelectors(msg).then(() => {
				log('done');
				function respondToReact (e) {
					if (e.message.id !== msg.id) return client.Dispatcher.once(Events.MESSAGE_REACTION_ADD, respondToReact);
					log(e.emoji.id);
				}
				client.Dispatcher.once(Events.MESSAGE_REACTION_ADD, respondToReact);
			}, () => {
				log('\u{1F914}');
			});
		});
	});

	async function addSelectors (msg) {
		const p = [];
		p[0] = msg.addReaction(String.fromCodePoint(...Emojis.ONE)).catch(() => log('What?'));
		for (let i = 1; i < Object.keys(Emojis).length; i++) {
			const ind = Object.keys(Emojis)[i];
			log(ind);
			// Dfq so slow
			p[i] = msg.addReaction(String.fromCodePoint(...Emojis[ind])).catch(() => log('What..?'));
		}
		return Promise.all(...p);
	}

	addCommandSentence('unicode', a => {
		if (msgText.includes('<')) return;
		if (/^<:.+?:\d+>$/.test(msgText)) return;
		sendMessage(getCodePoint(a));
	});

	addCommandSentence('setColor', a => {
		const coloredRoles = sender.roles.filter(r => r.color > 0);

		for (const role in coloredRoles) {
			const membersWithRole = guild.members.filter(m => m.hasRole(role));

			if (membersWithRole.length > 1 || role.position > botMember.roles[0].position)
				continue;
			else {
				if (!toHex.get(a))
					return sendMessage(`I don't know of a color named \`${a}\``);
				else
					return role.commit(null, toHex(parseInt(a.slice(1), 16))).then(() =>
						sendMessage(`Color of \`${role.name}\` changed to ${a}`));
			}
		}
	});
}

exports.respond = respond;
<commit_msg>Support raw hex input for setting self role color<commit_after>const CommandHandler = require('../util/msgUtil');
const { log, tryRequire } = require('../util/botUtil');
const { getCodePoint } = require('../util/stringUtil');

const toHex = tryRequire('colornames');
const Discordie = require('discordie');
const Events = Discordie.Events;

const config = require('../config');

/**
 * @param {IMessage} msg
 * @param {Discordie} client
 * @returns
 */
function respond (msg, client) {
	const { content: msgText, channel, guild, member: sender } = msg;
	if (!msgText.startsWith(config.prefix)) return;
	const botMember = client.User.memberOf(guild);

	/**
	 * Sends a message
	 * @param {string} s Text
	 * @param {object} e Embed object
	 * @returns {Promise<{}>}
	 */
	const sendMessage = (s, e) => channel.sendMessage(s, false, e);
	// const sendEmbed = (e) => sendMessage('', e);

	const command = msgText.slice(config.prefix.length);
	const handler = new CommandHandler(command);

	const { addCommand, addCommandSentence } = handler;

	const Emojis = {
		ONE: [0x31, 0x20E3],
		TWO: [0x32, 0x20E3],
		THREE: [0x33, 0x20E3],
		FOUR: [0x34, 0x20E3],
		FIVE: [0x35, 0x20E3]
	};

	addCommand('panel', () => {
		sendMessage([
			'1. -',
			'2. -',
			'3. -',
			'4. -',
			'5. -'
		].join('\n')).then((msg, err) => {
			if (err) return log(err);

			addSelectors(msg).then(() => {
				log('done');
				function respondToReact (e) {
					if (e.message.id !== msg.id) return client.Dispatcher.once(Events.MESSAGE_REACTION_ADD, respondToReact);
					log(e.emoji.id);
				}
				client.Dispatcher.once(Events.MESSAGE_REACTION_ADD, respondToReact);
			}, () => {
				log('\u{1F914}');
			});
		});
	});

	async function addSelectors (msg) {
		const p = [];
		p[0] = msg.addReaction(String.fromCodePoint(...Emojis.ONE)).catch(() => log('What?'));
		for (let i = 1; i < Object.keys(Emojis).length; i++) {
			const ind = Object.keys(Emojis)[i];
			log(ind);
			// Dfq so slow
			p[i] = msg.addReaction(String.fromCodePoint(...Emojis[ind])).catch(() => log('What..?'));
		}
		return Promise.all(...p);
	}

	addCommandSentence('unicode', a => {
		if (msgText.includes('<')) return;
		if (/^<:.+?:\d+>$/.test(msgText)) return;
		sendMessage(getCodePoint(a));
	});

	addCommandSentence('setColor', a => {
		const coloredRoles = sender.roles.filter(r => r.color > 0);

		const topSelfRole = coloredRoles.find(r =>
			guild.members.filter(m => m.hasRole(r)).length > 1);

		if (!topSelfRole)
			return sendMessage('You don\'t have a self role.');

		if (!toHex) {
			let hexValue;

			if (a.length === 6)
				hexValue = parseInt(a, 16);
			else if (a.length === 7)
				hexValue = parseInt(a.slice(1), 16);

			if (!hexValue)
				return sendMessage('I\'m sorry, I didn\'t get that.');
			else
				return topSelfRole.commit(null, hexValue).then(() =>
					sendMessage(`Color of \`${topSelfRole.name}\` changed to ${a}`));
		} else {
			if (!toHex.get(a))
				return sendMessage(`I don't know of a color named \`${a}\``);
			else
				return topSelfRole.commit(null, parseInt(toHex(a).slice(1), 16)).then(() =>
					sendMessage(`Color of \`${topSelfRole.name}\` changed to ${a}`));
		}
	});
}

exports.respond = respond;
<|endoftext|>