<commit_before>'use strict';

const Fabric = require('../');

const path = require('path');

const Machine = require('./machine');
const Resource = require('./resource');
const Store = require('./store');
const Walker = require('./walker');
const Vector = require('./vector');

class Oracle extends Store {
  /**
   * Trusted point-of-reference for external services.
   * @param       {Object} initial - Initialization vector.
   * @constructor
   */
  constructor (init) {
    super(init);

    this.name = 'Oracle';
    this.config = Object.assign({
      path: './data/oracle'
    }, init);

    this.machine = new Machine();
    this.mempool = [];

    this.resources = new Set();
    this.keys = new Set();

    return this;
  }

  async define (name, definition) {
    let resource = Object.assign({
      name: name
    }, definition);

    this.resources[name] = new Resource(resource);
    return this.resources[name];
  }

  async start () {
    await super.start();

    await Promise.all([
      this.define('Asset', {
        attributes: {
          name: { type: 'String', required: true, max: 220 }
        }
      }),
      this.define('Hash', {
        attributes: {
          'sha256': { type: 'String', required: true, max: 32 },
          '@data': { type: 'String', required: true, max: 2048 }
        }
      })
    ]);

    // TODO: pre-populate
    // this.state = await this._GET('/');
    // console.log('state retrieved:', this.state);
    this.machine.on('changes', this._handleStateChange.bind(this));

    return this;
  }

  /**
   * Synchronously reads a local path into memory.
   * @param  {String} path - dir (path to read)
   * @return {Vector} Computed vector.
   */
  async _load (dir) {
    let self = this;
    let walker = new Walker();

    let map = await walker._define(dir, {});
    let list = Object.keys(map);

    for (var i = 0; i < list.length; i++) {
      let file = path.join('/', list[i]);
      let content = map[list[i]];
      let result = await self.storage.set(file, content);
      let vector = new Fabric.Vector(result)._sign();
    }

    let tree = list.map(function (x) {
      return x.replace(/^(.*)\/(.*)$/, '$2');
    });

    var response = [];

    try {
      let assets = await self._PUT('/assets', tree);
      self.tree = new Fabric.Vector(assets)._sign();
      response = self.tree;
    } catch (E) {
      console.error(E);
    }

    return response;
  }
}

Oracle.prototype._handleStateChange = function (changes) {
  this.mempool.push(changes);
  this.emit('changes', changes);
};

/**
 * Core messaging function for interacting with this object in system-time.
 * @param  {Message} msg Instance of a {@link module:Message} object, validated then transmitted verbatim.
 * @return {Boolean}     Returns `true` on success, `false` on failure.
 */
Oracle.prototype.broadcast = function (msg) {
  return this.emit('message', msg);
};

Oracle.prototype._sync = async function () {
  for (let name in this.machine.state) {
    let data = this.machine.state[name];
    let path = `/${name}`;
    await this._PUT(path, data);
  }
};

/**
 * Registers an object.  Necessary to store in a collection.
 * @param  {Object} obj Instance of the object to store.
 * @return {Vector}     Returned from `storage.set`
 */
Oracle.prototype._REGISTER = async function (obj) {
  let result = null;
  let vector = new Fabric.Vector(obj)._sign();

  try {
    result = await this.storage.set(vector.id, obj);
  } catch (E) {
    console.error('Error creating object:', E, obj);
  }

  return result;
};

Oracle.prototype._GET = async function (key, params) {
  let result = null;

  try {
    result = await this.storage.get(key);
  } catch (E) {
    console.error(E);
  }

  return result;
};

Oracle.prototype._PUT = async function (key, obj) {
  let output = null;
  // TODO: consider enforcing `/` as prefix?
  let op = { op: 'add', path: key, value: obj };

  try {
    this.machine.applyOperation(op);
  } catch (E) {
    console.log('error applying operation:', E);
  }

  this.machine.commit();

  try {
    let vector = new Vector(obj)._sign();
    let result = await this.storage.set(key, vector['@data']);
    if (result) {
      output = await this._GET(key);
    }
  } catch (E) {
    console.error('[ORACLE]', '_PUT', E);
  }

  return output;
};

/**
 * Handle a request from a client to `create` an object.
 * @param  {String} key [description]
 * @param  {Object} obj [description]
 * @return {Vector}     [description]
 */
Oracle.prototype._POST = async function (key, obj) {
  let result = null;
  let collection = await this._GET(key);
  let standard = new Vector(collection)._sign();
  let vector = new Vector(obj)._sign();

  // collection does not exist, or there was an error.
  if (!collection || !(collection instanceof Array)) collection = [];

  // TODO: canonicalize array storage
  collection.push(vector['@data']);
  // collection.push(vector.id);

  try {
    let id = key + '/' + vector['@id'];

    // update indexes
    this.keys.add(vector['@id']);
    this.keys.add(key);
    this.keys.add(id);

    let instance = await this._REGISTER(obj);
    let item = await this._PUT(id, vector['@data']);
    let index = await this._PUT(key, collection);

    result = item;
  } catch (E) {
    console.debug(E);
  }

  if (result) {
    this.emit(key, result);
    // TODO: standardize event names
    this.emit('collections:post', {
      path: key,
      data: obj
    });
  }

  return result;
};

Oracle.prototype._PATCH = async function (key, changes) {
  let result = null;

  try {
    let output = null;
    let start = await this._GET(key);
    let state = new Fabric.Vector(start)._sign();

    if (typeof state['@data'] === 'string') {
      // TODO: lookup of known id collection
      output = changes;
    } else {
      output = Object.assign({}, state['@data'] || {}, changes);
    }

    result = await this._PUT(key, output);
  } catch (E) {
    console.debug(E);
  }

  return result;
};

Oracle.prototype._DELETE = async function (key) {
  await this._PUT(key, null);
  return null;
};

Oracle.prototype._OPTIONS = async function (key) {
  var result = null;

  try {
    result = await this.storage.get(key);
  } catch (E) {
    console.debug(E);
  }

  try {
    result = new Vector(result)._sign();
  } catch (E) {
    console.debug(E);
  }

  return result;
};

Oracle.prototype.flush = async function () {
  this.log('[ORACLE]', 'flush requested:', this.keys);

  for (let item of this.keys) {
    this.log('...flushing:', item);
    try {
      await this._DELETE(item);
    } catch (E) {
      console.error(E);
    }
  }
};

module.exports = Oracle;
<commit_msg>Update Oracle definition<commit_after>'use strict';

const Fabric = require('../');

const path = require('path');

const Machine = require('./machine');
const Resource = require('./resource');
const Store = require('./store');
const Walker = require('./walker');
const Vector = require('./vector');

/**
 * An Oracle manages one or more collections, using a mempool for
 * transitive state.
 * @extends Store
 */
class Oracle extends Store {
  /**
   * Trusted point-of-reference for external services.
   * @param       {Object} initial - Initialization vector.
   */
  constructor (init) {
    super(init);

    this.name = 'Oracle';
    this.config = Object.assign({
      path: './data/oracle'
    }, init);

    this.machine = new Machine();
    this.mempool = [];

    this.resources = new Set();
    this.keys = new Set();

    return this;
  }

  async define (name, definition) {
    let resource = Object.assign({
      name: name
    }, definition);

    this.resources[name] = new Resource(resource);

    console.log('defined resource:', this.resources[name]);

    return this.resources[name];
  }

  async start () {
    await super.start();

    await Promise.all([
      this.define('Asset', {
        attributes: {
          name: { type: 'String', required: true, max: 220 }
        }
      }),
      this.define('Hash', {
        attributes: {
          'sha256': { type: 'String', required: true, max: 32 },
          '@data': { type: 'String', required: true, max: 2048 }
        }
      })
    ]);

    // TODO: pre-populate
    // this.state = await this._GET('/');
    // console.log('state retrieved:', this.state);
    this.machine.on('changes', this._handleStateChange.bind(this));

    return this;
  }

  /**
   * Synchronously reads a local path into memory.
   * @param  {String} path - dir (path to read)
   * @return {Vector} Computed vector.
   */
  async _load (dir) {
    let self = this;
    let walker = new Walker();

    let map = await walker._define(dir, {});
    let list = Object.keys(map);

    for (var i = 0; i < list.length; i++) {
      let file = path.join('/', list[i]);
      let content = map[list[i]];
      console.debug('[ORACLE]', 'saving:', content.length, 'bytes to', file);
      let result = await self.storage.set(file, content);
      let vector = new Fabric.Vector(result)._sign();
    }

    let tree = list.map(function (x) {
      return x.replace(/^(.*)\/(.*)$/, '$2');
    });

    var response = [];

    try {
      let assets = await self._PUT('/assets', tree);
      self.tree = new Fabric.Vector(assets)._sign();
      response = self.tree;
    } catch (E) {
      console.error(E);
    }

    return response;
  }

  async _sync () {
    for (let name in this.machine.state) {
      let data = this.machine.state[name];
      let path = `/${name}`;
      await this._PUT(path, data);
    }
  }

  /**
   * Registers an object.  Necessary to store in a collection.
   * @param  {Object} obj Instance of the object to store.
   * @return {Vector}     Returned from `storage.set`
   */
  async _REGISTER (obj) {
    let result = null;
    let vector = new Fabric.Vector(obj)._sign();

    try {
      result = await this._SET(`/actors/${vector.id}`, obj);
    } catch (E) {
      console.error('Error creating object:', E, obj);
    }

    return result;
  }

  /**
   * Handle a request from a client to `create` an object.
   * @param  {String} key [description]
   * @param  {Object} obj [description]
   * @return {Vector}     [description]
   */
  async _POST (key, obj) {
    let result = null;
    let collection = await this._GET(key);
    let standard = new Vector(collection)._sign();
    let vector = new Vector(obj)._sign();

    // collection does not exist, or there was an error.
    if (!collection || !(collection instanceof Array)) collection = [];

    // TODO: canonicalize array storage
    collection.push(vector['@data']);
    // collection.push(vector.id);

    try {
      let id = key + '/' + vector['@id'];

      // update indexes
      this.keys.add(vector['@id']);
      this.keys.add(key);
      this.keys.add(id);

      let instance = await this._REGISTER(obj);
      let item = await this._PUT(id, vector['@data']);
      let index = await this._PUT(key, collection);

      result = item;
    } catch (E) {
      console.debug(E);
    }

    if (result) {
      this.emit(key, result);
      // TODO: standardize event names
      this.emit('collections:post', {
        path: key,
        data: obj
      });
    }

    return result;
  }

  async _PATCH (key, changes) {
    let result = null;

    try {
      let output = null;
      let start = await this._GET(key);
      let state = new Fabric.Vector(start)._sign();

      if (typeof state['@data'] === 'string') {
        // TODO: lookup of known id collection
        output = changes;
      } else {
        output = Object.assign({}, state['@data'] || {}, changes);
      }

      result = await this._PUT(key, output);
    } catch (E) {
      console.debug(E);
    }

    return result;
  }

  async _DELETE (key) {
    await this._PUT(key, null);
    return null;
  }

  async flush () {
    this.log('[ORACLE]', 'flush requested:', this.keys);

    for (let item of this.keys) {
      this.log('...flushing:', item);
      try {
        await this._DELETE(item);
      } catch (E) {
        console.error(E);
      }
    }
  }

  _handleStateChange (changes) {
    this.mempool.push(changes);
    this.emit('changes', changes);
  }

  /**
   * Core messaging function for interacting with this object in system-time.
   * @param  {Message} msg Instance of a {@link module:Message} object, validated then transmitted verbatim.
   * @return {Boolean}     Returns `true` on success, `false` on failure.
   */
  broadcast (msg) {
    return this.emit('message', msg);
  }
}

module.exports = Oracle;
<|endoftext|>


<commit_before>'use strict';

const fetch = require('node-fetch');
const logger = require('winston');

const BlinkRetryError = require('../errors/BlinkRetryError');
const Worker = require('./Worker');

class TwilioSmsBroadcastGambitRelayWorker extends Worker {
  constructor(blink) {
    super(blink);
    this.blink = blink;

    this.baseURL = this.blink.config.gambit.converationsBaseUrl;
    this.apiKey = this.blink.config.gambit.converationsApiKey;

    // Bind process method to queue context
    this.consume = this.consume.bind(this);
  }

  setup() {
    this.queue = this.blink.queues.twilioSmsBroadcastGambitRelayQ;
  }

  async consume(message) {
    const body = JSON.stringify(message.getData());

    // Send only delivered messages to Gambit Conversations import.
    if (TwilioSmsBroadcastGambitRelayWorker.shouldSkip(message)) {
      const meta = {
        env: this.blink.config.app.env,
        code: 'success_gambit_broadcast_relay_expected_skip',
        worker: this.constructor.name,
        request_id: message ? message.getRequestId() : 'not_parsed',
      };
      logger.log('debug', body, meta);
      return true;
    }

    // Check that the message has broadcastId query string.
    // If it hasn't, something is wrong. We expect broadcastId to be
    // provided with all reciepts.
    const query = message.getMeta().query;
    if (!query || !query.broadcastId) {
      const meta = {
        env: this.blink.config.app.env,
        code: 'error_gambit_broadcast_relay_missing_broadcastId',
        worker: this.constructor.name,
        request_id: message ? message.getRequestId() : 'not_parsed',
      };
      logger.log('warning', body, meta);
      return true;
    }

    const headers = this.getRequestHeaders(message);
    const response = await fetch(
      `${this.baseURL}/import-message?broadcastId=${query.broadcastId}`,
      {
        method: 'POST',
        headers,
        body,
      },
    );

    if (response.status === 200) {
      this.log(
        'debug',
        message,
        response,
        'success_gambit_broadcast_relay_response_200',
      );
      return true;
    }

    if (this.checkRetrySuppress(response)) {
      this.log(
        'debug',
        message,
        response,
        'success_gambit_broadcast_relay_retry_suppress',
      );
      return true;
    }

    if (response.status === 422) {
      this.log(
        'warning',
        message,
        response,
        'error_gambit_broadcast_relay_response_422',
      );
      return false;
    }

    this.log(
      'warning',
      message,
      response,
      'error_gambit_broadcast_relay_response_not_200_retry',
    );

    throw new BlinkRetryError(
      `${response.status} ${response.statusText}`,
      message,
    );
  }

  async log(level, message, response, code = 'unexpected_code') {
    const cleanedBody = (await response.text()).replace(/\n/g, '\\n');

    const meta = {
      env: this.blink.config.app.env,
      code,
      worker: this.constructor.name,
      request_id: message ? message.getRequestId() : 'not_parsed',
      response_status: response.status,
      response_status_text: `"${response.statusText}"`,
    };
    // Todo: log error?
    logger.log(level, cleanedBody, meta);
  }

  getRequestHeaders(message) {
    const headers = {
      Authorization: `Basic ${this.apiKey}`,
      'X-Request-ID': message.getRequestId(),
      'Content-type': 'application/json',
    };

    if (message.getMeta().retry && message.getMeta().retry > 0) {
      headers['x-blink-retry-count'] = message.getMeta().retry;
    }

    return headers;
  }

  checkRetrySuppress(response) {
    // TODO: create common helper
    const headerResult = response.headers.get(this.blink.config.app.retrySuppressHeader);
    if (!headerResult) {
      return false;
    }
    return headerResult.toLowerCase() === 'true';
  }

  static shouldSkip(message) {
    return !message.isDelivered();
  }
}

module.exports = TwilioSmsBroadcastGambitRelayWorker;
<commit_msg>Typo fix<commit_after>'use strict';

const fetch = require('node-fetch');
const logger = require('winston');

const BlinkRetryError = require('../errors/BlinkRetryError');
const Worker = require('./Worker');

class TwilioSmsBroadcastGambitRelayWorker extends Worker {
  constructor(blink) {
    super(blink);
    this.blink = blink;

    this.baseURL = this.blink.config.gambit.converationsBaseUrl;
    this.apiKey = this.blink.config.gambit.converationsApiKey;

    // Bind process method to queue context
    this.consume = this.consume.bind(this);
  }

  setup() {
    this.queue = this.blink.queues.twilioSmsBroadcastGambitRelayQ;
  }

  async consume(message) {
    const body = JSON.stringify(message.getData());

    // Send only delivered messages to Gambit Conversations import.
    if (TwilioSmsBroadcastGambitRelayWorker.shouldSkip(message)) {
      const meta = {
        env: this.blink.config.app.env,
        code: 'success_gambit_broadcast_relay_expected_skip',
        worker: this.constructor.name,
        request_id: message ? message.getRequestId() : 'not_parsed',
      };
      logger.log('debug', body, meta);
      return true;
    }

    // Check that the message has broadcastId query string.
    // If it hasn't, something is wrong. We expect broadcastId to be
    // provided with all receipts.
    const query = message.getMeta().query;
    if (!query || !query.broadcastId) {
      const meta = {
        env: this.blink.config.app.env,
        code: 'error_gambit_broadcast_relay_missing_broadcastId',
        worker: this.constructor.name,
        request_id: message ? message.getRequestId() : 'not_parsed',
      };
      logger.log('warning', body, meta);
      return true;
    }

    const headers = this.getRequestHeaders(message);
    const response = await fetch(
      `${this.baseURL}/import-message?broadcastId=${query.broadcastId}`,
      {
        method: 'POST',
        headers,
        body,
      },
    );

    if (response.status === 200) {
      this.log(
        'debug',
        message,
        response,
        'success_gambit_broadcast_relay_response_200',
      );
      return true;
    }

    if (this.checkRetrySuppress(response)) {
      this.log(
        'debug',
        message,
        response,
        'success_gambit_broadcast_relay_retry_suppress',
      );
      return true;
    }

    if (response.status === 422) {
      this.log(
        'warning',
        message,
        response,
        'error_gambit_broadcast_relay_response_422',
      );
      return false;
    }

    this.log(
      'warning',
      message,
      response,
      'error_gambit_broadcast_relay_response_not_200_retry',
    );

    throw new BlinkRetryError(
      `${response.status} ${response.statusText}`,
      message,
    );
  }

  async log(level, message, response, code = 'unexpected_code') {
    const cleanedBody = (await response.text()).replace(/\n/g, '\\n');

    const meta = {
      env: this.blink.config.app.env,
      code,
      worker: this.constructor.name,
      request_id: message ? message.getRequestId() : 'not_parsed',
      response_status: response.status,
      response_status_text: `"${response.statusText}"`,
    };
    // Todo: log error?
    logger.log(level, cleanedBody, meta);
  }

  getRequestHeaders(message) {
    const headers = {
      Authorization: `Basic ${this.apiKey}`,
      'X-Request-ID': message.getRequestId(),
      'Content-type': 'application/json',
    };

    if (message.getMeta().retry && message.getMeta().retry > 0) {
      headers['x-blink-retry-count'] = message.getMeta().retry;
    }

    return headers;
  }

  checkRetrySuppress(response) {
    // TODO: create common helper
    const headerResult = response.headers.get(this.blink.config.app.retrySuppressHeader);
    if (!headerResult) {
      return false;
    }
    return headerResult.toLowerCase() === 'true';
  }

  static shouldSkip(message) {
    return !message.isDelivered();
  }
}

module.exports = TwilioSmsBroadcastGambitRelayWorker;
<|endoftext|>


<commit_before>'use strict';
var Promise = require('pinkie-promise');
var prettyBytes = require('pretty-bytes');
var sortOn = require('sort-on');
var humanizeUrl = require('humanize-url');
var Download = require('download');
var querystring = require('querystring');
var pify = require('pify');
var THRESHOLD = 70;
var RESOURCE_URL = 'https://developers.google.com/speed/pagespeed/insights/optimizeContents?';

function overview(url, strategy, scores) {
  var ret = [];

  ret.push({
    label: 'URL',
    value: url
  });

  ret.push({
    label: 'Strategy',
    value: strategy
  });

  ret.push({
    label: 'Speed',
    value: scores.SPEED.score
  });

  if (scores.USABILITY !== undefined) {
    ret.push({
      label: 'Usability',
      value: scores.USABILITY.score
    });
  }

  return ret;
}

function ruleSetResults(rulesets) {
  var ret = [];

  for (var title in rulesets) {
    if (rulesets.hasOwnProperty(title)) {
      ret.push({
        label: title,
        value: Math.ceil(rulesets[title].ruleImpact * 100) / 100
      });
    }
  }

  return sortOn(ret, 'label');
}

function statistics(stats) {
  var ret = [];

  for (var title in stats) {
    if (stats.hasOwnProperty(title)) {
      ret.push({
        label: title,
        value: title.indexOf('Bytes') === -1 ? stats[title] : prettyBytes(Number(stats[title]))
      });
    }
  }

  return sortOn(ret, 'label');
}

function getReporter(format) {
  format = ['cli', 'json', 'tap'].indexOf(format) === -1 ? 'cli' : format;
  return require('./formats/' + format);
}

module.exports = function (parameters, response) {
  return Promise.resolve().then(function () {
    var renderer = getReporter(parameters.format);
    var threshold = parameters.threshold || THRESHOLD;
    var optimizedResoruceURL = RESOURCE_URL + querystring.stringify({url: response.id, strategy: parameters.strategy});

    console.log(renderer(
      overview(humanizeUrl(response.id), parameters.strategy, response.ruleGroups),
      statistics(response.pageStats),
      ruleSetResults(response.formattedResults.ruleResults),
      threshold
    ));

    if (parameters.optimized) {
      console.log('\nHere are your optimized images: ', humanizeUrl(optimizedResoruceURL));
    }

    if (response.ruleGroups.SPEED.score < threshold) {
      var err = new Error('Threshold of ' + threshold + ' not met with score of ' + response.ruleGroups.SPEED.score);
      throw err;
    }

    if (parameters.download) {
      var download = new Download()
        .get(optimizedResoruceURL)
        .dest('.')
        .rename('./optimized.zip');

      return pify(download.run.bind(download), Promise)();
    }
  });
};
<commit_msg>optimizedResourceUrl spelling corrected in output.js.<commit_after>'use strict';
var Promise = require('pinkie-promise');
var prettyBytes = require('pretty-bytes');
var sortOn = require('sort-on');
var humanizeUrl = require('humanize-url');
var Download = require('download');
var querystring = require('querystring');
var pify = require('pify');
var THRESHOLD = 70;
var RESOURCE_URL = 'https://developers.google.com/speed/pagespeed/insights/optimizeContents?';

function overview(url, strategy, scores) {
  var ret = [];

  ret.push({
    label: 'URL',
    value: url
  });

  ret.push({
    label: 'Strategy',
    value: strategy
  });

  ret.push({
    label: 'Speed',
    value: scores.SPEED.score
  });

  if (scores.USABILITY !== undefined) {
    ret.push({
      label: 'Usability',
      value: scores.USABILITY.score
    });
  }

  return ret;
}

function ruleSetResults(rulesets) {
  var ret = [];

  for (var title in rulesets) {
    if (rulesets.hasOwnProperty(title)) {
      ret.push({
        label: title,
        value: Math.ceil(rulesets[title].ruleImpact * 100) / 100
      });
    }
  }

  return sortOn(ret, 'label');
}

function statistics(stats) {
  var ret = [];

  for (var title in stats) {
    if (stats.hasOwnProperty(title)) {
      ret.push({
        label: title,
        value: title.indexOf('Bytes') === -1 ? stats[title] : prettyBytes(Number(stats[title]))
      });
    }
  }

  return sortOn(ret, 'label');
}

function getReporter(format) {
  format = ['cli', 'json', 'tap'].indexOf(format) === -1 ? 'cli' : format;
  return require('./formats/' + format);
}

module.exports = function (parameters, response) {
  return Promise.resolve().then(function () {
    var renderer = getReporter(parameters.format);
    var threshold = parameters.threshold || THRESHOLD;
    var optimizedResourceURL = RESOURCE_URL + querystring.stringify({url: response.id, strategy: parameters.strategy});

    console.log(renderer(
      overview(humanizeUrl(response.id), parameters.strategy, response.ruleGroups),
      statistics(response.pageStats),
      ruleSetResults(response.formattedResults.ruleResults),
      threshold
    ));

    if (parameters.optimized) {
      console.log('\nHere are your optimized images: ', humanizeUrl(optimizedResourceURL));
    }

    if (response.ruleGroups.SPEED.score < threshold) {
      var err = new Error('Threshold of ' + threshold + ' not met with score of ' + response.ruleGroups.SPEED.score);
      throw err;
    }

    if (parameters.download) {
      var download = new Download()
        .get(optimizedResourceURL)
        .dest('.')
        .rename('./optimized.zip');

      return pify(download.run.bind(download), Promise)();
    }
  });
};
<|endoftext|>


<commit_before>/**
 * plugin.js
 *
 * Released under LGPL License.
 * Copyright (c) 2015 SIRAP SAS All rights reserved
 *
 * License: http://www.tinymce.com/license
 * Contributing: http://www.tinymce.com/contributing
 */

// browser globals
var MutationObserver = window.MutationObserver
var $ = window.$
var tinymce = window.tinymce

// register the createPlugin function
tinymce.PluginManager.add('menuscontroller', createPlugin)

function createPlugin (editor) {
  // expose the plugin API
  // example of usage:
  // var mceMenubar = editor.plugins.menuscontroller.getMenubar()
  this.getMenubar = getMenubar
  this.getMenuByName = getMenuByName
  this.getToolbars = getToolbars

  // trigger an event 'menusController:mceMenuItemRendered:<menu_item_id>' for each menu item when its parent menu is rendered
  $('body').on('menusController:mceMenuRendered', function (evt, menu) {
    $(menu).find('.mce-menu-item').each(function (i, menuItem) {
      $(menu).trigger('menusController:mceMenuItemRendered:' + $(menuItem).attr('id'))
    })
  })

  // start to observe menus rendering
  observeMenuRendering()

  function getMenubar () {
    var themeItems = editor.theme.panel.settings.items
    for (var i = 0; i < themeItems.length; i++) {
      var item = themeItems[i]
      if (item.type === 'menubar') return item.items
    }
  }

  function getMenuByName (name) {
    try {
      var menubar = getMenubar()
      for (var i = 0; i < menubar.length; i++) {
        var menu = menubar[i]
        if (menu.text.toLowerCase() === name.toLowerCase()) {
          return menu
        }
      }
    } catch (e) {
      throw new Error("Can't find the menu " + name + ' in menu bar')
    }
  }

  function getToolbars () {
    var themeItems = editor.theme.panel.settings.items
    for (var i = 0; i < themeItems.length; i++) {
      var item = themeItems[i]
      if (item.type === 'panel' && item.classes === 'toolbar-grp') return item.items
    }
  }
}

function observeMenuRendering () {
  // var targetDOMQuery = '#mce-plugin-headersfooters-remove-header'
  var menubarMenuDOMQuery = '.mce-container.mce-panel.mce-floatpanel.mce-menu'
  // select the target node
  var target = document.querySelector('body')
  // create an observer instance
  var observer = new MutationObserver(bodyMutationsCallback)
  // configuration of the observer:
  var config = { childList: true }

  // pass in the target node, as well as the observer options
  // later, we can stop observing with observer.disconnect()
  observer.observe(target, config)

  function bodyMutationsCallback (mutations) {
    mutations.forEach(function (mutation) {
      // search menu with menubarMenuDOMQuery
      var menus = $(mutation.addedNodes).filter(menubarMenuDOMQuery)
      if (menus.length) {
        // menus are renders once at time
        if (menus.length > 1) throw new Error('ERROR: there is many menus rendered ! This case is not implemented !')

        // trigger an 'menusController:mceMenuRendered' event with a 'evt.data.menu' property
        var menuRenderedEvent = $.Event('menusController:mceMenuRendered')
        $('body').trigger(menuRenderedEvent, menus[0])
      }
    })
  }
}
<commit_msg>document the plugin constructor<commit_after>/**
 * plugin.js
 *
 * Released under LGPL License.
 * Copyright (c) 2015 SIRAP SAS All rights reserved
 *
 * License: http://www.tinymce.com/license
 * Contributing: http://www.tinymce.com/contributing
 */

// browser globals
var MutationObserver = window.MutationObserver
var $ = window.$
var tinymce = window.tinymce

// register the createPlugin function
tinymce.PluginManager.add('menuscontroller', createPlugin)

/**
 * The plugin function definition
 * @constructor
 * @description The plugin expose an API and triggers two types of events
 * @param {tinymce.Editor}
 * @event menusController:mceMenuRendered triggered when a menu of the editor's menu bar is rendered. Fired once by menu.
 * @event menusController:mceMenuItemRendered triggered when a menu item of any of menubar's menu is rendered (just after the parent menu is rendered).
 */
function createPlugin (editor) {
  // expose the plugin API
  // example of usage:
  // var mceMenubar = editor.plugins.menuscontroller.getMenubar()
  this.getMenubar = getMenubar
  this.getMenuByName = getMenuByName
  this.getToolbars = getToolbars

  // trigger an event 'menusController:mceMenuItemRendered:<menu_item_id>' for each menu item when its parent menu is rendered
  $('body').on('menusController:mceMenuRendered', function (evt, menu) {
    $(menu).find('.mce-menu-item').each(function (i, menuItem) {
      $(menu).trigger('menusController:mceMenuItemRendered:' + $(menuItem).attr('id'))
    })
  })

  // start to observe menus rendering
  observeMenuRendering()

  function getMenubar () {
    var themeItems = editor.theme.panel.settings.items
    for (var i = 0; i < themeItems.length; i++) {
      var item = themeItems[i]
      if (item.type === 'menubar') return item.items
    }
  }

  function getMenuByName (name) {
    try {
      var menubar = getMenubar()
      for (var i = 0; i < menubar.length; i++) {
        var menu = menubar[i]
        if (menu.text.toLowerCase() === name.toLowerCase()) {
          return menu
        }
      }
    } catch (e) {
      throw new Error("Can't find the menu " + name + ' in menu bar')
    }
  }

  function getToolbars () {
    var themeItems = editor.theme.panel.settings.items
    for (var i = 0; i < themeItems.length; i++) {
      var item = themeItems[i]
      if (item.type === 'panel' && item.classes === 'toolbar-grp') return item.items
    }
  }
}

function observeMenuRendering () {
  // var targetDOMQuery = '#mce-plugin-headersfooters-remove-header'
  var menubarMenuDOMQuery = '.mce-container.mce-panel.mce-floatpanel.mce-menu'
  // select the target node
  var target = document.querySelector('body')
  // create an observer instance
  var observer = new MutationObserver(bodyMutationsCallback)
  // configuration of the observer:
  var config = { childList: true }

  // pass in the target node, as well as the observer options
  // later, we can stop observing with observer.disconnect()
  observer.observe(target, config)

  function bodyMutationsCallback (mutations) {
    mutations.forEach(function (mutation) {
      // search menu with menubarMenuDOMQuery
      var menus = $(mutation.addedNodes).filter(menubarMenuDOMQuery)
      if (menus.length) {
        // menus are renders once at time
        if (menus.length > 1) throw new Error('ERROR: there is many menus rendered ! This case is not implemented !')

        // trigger an 'menusController:mceMenuRendered' event with a 'evt.data.menu' property
        var menuRenderedEvent = $.Event('menusController:mceMenuRendered')
        $('body').trigger(menuRenderedEvent, menus[0])
      }
    })
  }
}
<|endoftext|>


<commit_before>/*    Copyright 2021-2022 Firewalla Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require('../net2/logger.js')(__filename);

const Sensor = require('./Sensor.js').Sensor;

const _ = require('lodash');

const validator = require('validator');

const IntelTool = require('../net2/IntelTool');
const intelTool = new IntelTool();

const rclient = require('../util/redis_manager.js').getRedisClient();
const sclient = require('../util/redis_manager.js').getSubscriptionClient();

const m = require('../extension/metrics/metrics.js');

const flowUtil = require('../net2/FlowUtil');
const f = require('../net2/Firewalla.js');

const userConfigFolder = f.getUserConfigFolder();
const dnsmasqConfigFolder = `${userConfigFolder}/dnsmasq`;

const cc = require('../extension/cloudcache/cloudcache.js');

const fs = require('fs');

const Promise = require('bluebird');
Promise.promisifyAll(fs);

const DNSMASQ = require('../extension/dnsmasq/dnsmasq.js');
const dnsmasq = new DNSMASQ();

const Alarm = require('../alarm/Alarm.js');
const AlarmManager2 = require('../alarm/AlarmManager2.js')
const am2 = new AlarmManager2();
const HostTool = require('../net2/HostTool.js')
const hostTool = new HostTool()
const BF_SERVER_MATCH = "bf_server_match"
const IdentityManager = require('../net2/IdentityManager.js');
const sem = require('../sensor/SensorEventManager.js').getInstance();
const extensionManager = require('./ExtensionManager.js')
const tm = require('../alarm/TrustManager.js');

const bf = require('../extension/bf/bf.js');

// slices a single byte into bits
// assuming only single bytes
const sliceBits = function (b, off, len) {
  const s = 7 - (off + len - 1);

  b = b >>> s;
  return b & ~(0xff << len);
};

const qnameToDomain = (qname) => {
  let domain = '';
  for (let i = 0; i < qname.length; i++) {
    if (qname[i] == 0) {
      //last char chop trailing .
      domain = domain.substring(0, domain.length - 1);
      break;
    }

    const tmpBuf = qname.slice(i + 1, i + qname[i] + 1);
    domain += tmpBuf.toString('binary', 0, tmpBuf.length);
    domain += '.';

    i = i + qname[i];
  }

  return domain;
};

const defaultExpireTime = 48 * 3600; // expire in two days, by default
const allowKey = "dns_proxy:allow_list"; //unused at this moment
const passthroughKey = "dns_proxy:passthrough_list";
const blockKey = "dns_proxy:block_list";
const featureName = "dns_proxy";

class DNSProxyPlugin extends Sensor {
  async run() {
    // invalidate cache keys when starting up
    await rclient.unlinkAsync(allowKey);
    await rclient.unlinkAsync(blockKey);
    await rclient.unlinkAsync(passthroughKey);

    extensionManager.registerExtension(featureName, this, {
      applyPolicy: this.applyDnsProxy
    });
    this.hookFeature(featureName);
  }

  getHashKeyName(item = {}, level = "default") {
    if (!item.count || !item.error || !item.prefix) {
      log.error("Invalid item:", item);
      return null;
    }

    const { count, error, prefix } = item;

    if (level) {
      return `bf:${level}_${prefix}:${count}:${error}`;
    }
    return `bf:${prefix}:${count}:${error}`;
  }

  getFilePath(item = {}) {
    if (!item.count || !item.error || !item.prefix) {
      log.error("Invalid item:", item);
      return null;
    }

    const { count, error, prefix, level } = item;

    if (level) {
      return `${f.getRuntimeInfoFolder()}/${featureName}.${level}_${prefix}.bf.data`;
    }
    return `${f.getRuntimeInfoFolder()}/${featureName}.${prefix}.bf.data`;
  }

  getDnsmasqConfigFile() {
    return `${dnsmasqConfigFolder}/${featureName}.conf`;
  }

  async enableDnsmasqConfig(data) {
    log.info("Enabling dnsmasq config file for dnsproxy...");
    let dnsmasqEntry = "mac-address-tag=%FF:FF:FF:FF:FF:FF$dns_proxy&1\n";
    for (const level in data) {
      const levelData = data[level];
      for (const item of levelData) {
        item.level = level;
        const fp = this.getFilePath(item);
        if (!fp) {
          continue;
        }
        const entry = `server-bf-high=<${fp},${item.count},${item.error}><${allowKey}><${blockKey}><${passthroughKey}>127.0.0.153#59953$dns_proxy\n`;
        dnsmasqEntry += entry;
      }
    }

    await fs.writeFileAsync(this.getDnsmasqConfigFile(), dnsmasqEntry);
    dnsmasq.scheduleRestartDNSService();
  }

  async disableDnsmasqConfig() {
    log.info("Disabling dnsmasq config file for dnsproxy...");
    await fs.unlinkAsync(this.getDnsmasqConfigFile()).catch(() => undefined); // ignore error
    dnsmasq.scheduleRestartDNSService();
  }

  async applyDnsProxy(host, ip, policy) {
    log.info("Applying dns_proxy", ip, policy);
    if (policy) {
      this.dnsProxyData = policy;
    }
    if (!this.dnsProxyData) {
      this.dnsProxyData = {};
      this.dnsProxyData["default"] = this.config.data;
    }

    if (!this.state) {
      log.info("dns_proxy feature is disabled, skip applying policy");
      return;
    }

    // level: strict, default... usually just one level at the same time, but the code supports multiple anyway
    for (const level in this.dnsProxyData) {
      const levelData = this.dnsProxyData[level];
      // item: data, new... each one is a bloom data
      for (const item of levelData) {
        item.level = level;
        const hashKeyName = bf.getHashKeyName(item);
        if (!hashKeyName) continue;

        log.info("Processing data file:", hashKeyName);
        const outputFilePath = this.getFilePath(item);
        await cc.enableCache(hashKeyName, async (data) => {
          if (data) {
            await bf.updateBFData(item, data, outputFilePath).catch((err) => {
              log.error("Failed to process data file, err:", err);
            });
          } else {
            log.error(`no dns_proxy data ${hashKeyName}. delete data file ${outputFilePath}`);
            await bf.deleteBFData(outputFilePath);
          }
          // always reschedule dnsmasq restarts when bf data is updated
          dnsmasq.scheduleRestartDNSService();
        });
      }
    }

    await this.enableDnsmasqConfig(this.dnsProxyData).catch((err) => {
      log.error("Failed to enable dnsmasq config, err", err);
    });
  }

  async globalOn() {
    this.state = true;
    sclient.subscribe(BF_SERVER_MATCH);
    sclient.on("message", async (channel, message) => {

      switch (channel) {
        case BF_SERVER_MATCH:
          let msgObj;
          try {
            msgObj = JSON.parse(message)
          } catch (err) {
            log.error("parse msg failed", err, message)
          }
          if (msgObj && msgObj.mac) {
            const MAC = msgObj.mac.toUpperCase();
            const ip = msgObj.ip4 || msgObj.ip6;

            // skip category bf match result
            if (msgObj.bf_path.startsWith("/home/pi/.firewalla/run/category_data/filters")) {
              return;
            }

            await this.processRequest(msgObj.domain, ip, MAC);
          }

          await m.incr("dns_proxy_request_cnt");
          break;
      }
    });

    sem.on("FastDNSPolicyComplete", async (event) => {
      await rclient.zaddAsync(passthroughKey, Math.floor(new Date() / 1000), event.domain);
    });

    await this.applyDnsProxy();
  }

  async globalOff() {
    this.state = false;

    sclient.unsubscribe(BF_SERVER_MATCH);

    if (!_.isEmpty(this.dnsProxyData)) {
      for (const level in this.dnsProxyData) {
        const levelData = this.dnsProxyData[level];
        for (const item of levelData) {
          item.level = level;
          const hashKeyName = this.getHashKeyName(item);
          if (!hashKeyName) continue;
          await cc.disableCache(hashKeyName).catch((err) => {
            log.error("Failed to disable cache, err:", err);
          });
        }
      }
    }

    await this.disableDnsmasqConfig();
  }

  async checkCache(domain) { // only return if there is exact-match intel in redis
    return intelTool.getDomainIntel(domain);
  }

  async _genSecurityAlarm(ip, mac, dn, item) {
    let macEntry;
    let realLocal;
    let guid;
    if (mac) macEntry = await hostTool.getMACEntry(mac);
    if (macEntry) {
      ip = macEntry.ipv4 || macEntry.ipv6 || ip;
    } else {
      const identity = ip && IdentityManager.getIdentityByIP(ip);
      if (identity) {
        guid = IdentityManager.getGUID(identity);
        realLocal = IdentityManager.getEndpointByIP(ip);
      }
      if (!guid) return;
      mac = guid;
    }
    const alarm = new Alarm.IntelAlarm(new Date() / 1000, ip, "major", {
      "p.device.ip": ip,
      "p.dest.name": dn,
      "p.dest.category": "intel",
      "p.security.reason": item.msg,
      "p.device.mac": mac,
      "p.action.block": true,
      "p.blockby": "fastdns",
      "p.local_is_client": "1"
    });
    if (realLocal) {
      alarm["p.device.real.ip"] = realLocal;
    }
    if (guid) {
      const identity = IdentityManager.getIdentityByGUID(guid);
      if (identity)
        alarm[identity.constructor.getKeyOfUIDInAlarm()] = identity.getUniqueId();
      alarm["p.device.guid"] = guid;
    }
    await am2.enrichDeviceInfo(alarm);
    am2.enqueueAlarm(alarm); // use enqueue to ensure no dup alarms
  }

  async processRequest(qname, ip, mac) {
    const domain = qname;
    log.info("dns request is", domain);
    const begin = new Date() / 1;
    const cache = await this.checkCache(domain);
    if (cache) {
      log.info(`inteldns:${domain} is already cached locally, updating redis cache keys directly...`);
      if (cache.c === "intel") {

        const isTrusted = await tm.matchDomain(domain);

        if(isTrusted) {
          await rclient.zaddAsync(passthroughKey, Math.floor(new Date() / 1000), domain);
        } else {
          await this._genSecurityAlarm(ip, mac, domain, cache)
        }

      } else {
        await rclient.zaddAsync(passthroughKey, Math.floor(new Date() / 1000), domain);
      }
      return; // do need to do anything
    }

    // if no cache, will trigger cloud inquiry and store the result in cache
    // assume client will send dns query again when cache is ready
    const result = await intelTool.checkIntelFromCloud(undefined, domain); // parameter ip is undefined since it's a dns request only
    await this.updateCache(domain, result, ip, mac);
    const end = new Date() / 1;
    log.info("dns intel result is", result, "took", Math.floor(end - begin), "ms");
  }

  redisfy(item) {
    for (const k in item) { // to suppress redis error
      const v = item[k];
      if (_.isBoolean(v) || _.isNumber(v) || _.isString(v)) {
        continue;
      }
      item[k] = JSON.stringify(v);
    }
  }

  getSortedItems(items) {
    // sort by length of originIP
    return items.sort((a, b) => {
      const oia = a.originIP;
      const oib = b.originIP;
      if (!oia && !oib) {
        return 0;
      }
      if (oia && !oib) {
        return -1;
      }
      if (!oia && oib) {
        return 1;
      }

      if (oia.length > oib.length) {
        return -1;
      } else if (oia.length < oib.length) {
        return 1;
      }

      return 0;
    });
  }

  async updateCache(domain, result, ip, mac) {
    if (_.isEmpty(result)) { // empty intel, means the domain is good
      const domains = flowUtil.getSubDomains(domain);
      if (!domains) {
        log.warn("Invalid Domain", domain, "skipped.");
        return;
      }

      // since result is empty, it means all sub domains of this domain are good
      const placeholder = { c: 'x', a: '1' };
      for (const dn of domains) {
        await intelTool.addDomainIntel(dn, placeholder, defaultExpireTime);
      }

      // only the exact dn is added to the passthrough and block list
      // generic domains can't be added, because another sub domain may be malicous
      // example:
      //   domain1.blogspot.com may be good
      //   and domain2.blogspot.com may be malicous
      // when domain1 is checked to be good, we should NOT add blogspot.com to passthrough_list
      await rclient.zaddAsync(passthroughKey, Math.floor(new Date() / 1000), domain);

    } else {
      // sort by length of originIP
      const validItems = result.filter((item) => item.originIP && validator.isFQDN(item.originIP));
      const sortedItems = this.getSortedItems(validItems);

      if (_.isEmpty(sortedItems)) {
        return;
      }

      for (const item of sortedItems) {
        this.redisfy(item);
        const dn = item.originIP; // originIP is the domain
        await intelTool.addDomainIntel(dn, item, item.e || defaultExpireTime);
        if (item.c !== "intel") {
          await rclient.zaddAsync(passthroughKey, Math.floor(new Date() / 1000), domain);
        }
      }

      for (const item of sortedItems) {
        if (item.c === "intel") {
          const dn = item.originIP; // originIP is the domain
          await this._genSecurityAlarm(ip, mac, dn, item);
          break;
        }
      }
    }
  }

}

module.exports = DNSProxyPlugin;
<commit_msg>use uhigh options in dnsproxy<commit_after>/*    Copyright 2021-2022 Firewalla Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require('../net2/logger.js')(__filename);

const Sensor = require('./Sensor.js').Sensor;

const _ = require('lodash');

const validator = require('validator');

const IntelTool = require('../net2/IntelTool');
const intelTool = new IntelTool();

const rclient = require('../util/redis_manager.js').getRedisClient();
const sclient = require('../util/redis_manager.js').getSubscriptionClient();

const m = require('../extension/metrics/metrics.js');

const flowUtil = require('../net2/FlowUtil');
const f = require('../net2/Firewalla.js');

const userConfigFolder = f.getUserConfigFolder();
const dnsmasqConfigFolder = `${userConfigFolder}/dnsmasq`;

const cc = require('../extension/cloudcache/cloudcache.js');

const fs = require('fs');

const Promise = require('bluebird');
Promise.promisifyAll(fs);

const DNSMASQ = require('../extension/dnsmasq/dnsmasq.js');
const dnsmasq = new DNSMASQ();

const Alarm = require('../alarm/Alarm.js');
const AlarmManager2 = require('../alarm/AlarmManager2.js')
const am2 = new AlarmManager2();
const HostTool = require('../net2/HostTool.js')
const hostTool = new HostTool()
const BF_SERVER_MATCH = "bf_server_match"
const IdentityManager = require('../net2/IdentityManager.js');
const sem = require('../sensor/SensorEventManager.js').getInstance();
const extensionManager = require('./ExtensionManager.js')
const tm = require('../alarm/TrustManager.js');

const bf = require('../extension/bf/bf.js');

// slices a single byte into bits
// assuming only single bytes
const sliceBits = function (b, off, len) {
  const s = 7 - (off + len - 1);

  b = b >>> s;
  return b & ~(0xff << len);
};

const qnameToDomain = (qname) => {
  let domain = '';
  for (let i = 0; i < qname.length; i++) {
    if (qname[i] == 0) {
      //last char chop trailing .
      domain = domain.substring(0, domain.length - 1);
      break;
    }

    const tmpBuf = qname.slice(i + 1, i + qname[i] + 1);
    domain += tmpBuf.toString('binary', 0, tmpBuf.length);
    domain += '.';

    i = i + qname[i];
  }

  return domain;
};

const defaultExpireTime = 48 * 3600; // expire in two days, by default
const allowKey = "dns_proxy:allow_list"; //unused at this moment
const passthroughKey = "dns_proxy:passthrough_list";
const blockKey = "dns_proxy:block_list";
const featureName = "dns_proxy";

class DNSProxyPlugin extends Sensor {
  async run() {
    // invalidate cache keys when starting up
    await rclient.unlinkAsync(allowKey);
    await rclient.unlinkAsync(blockKey);
    await rclient.unlinkAsync(passthroughKey);

    extensionManager.registerExtension(featureName, this, {
      applyPolicy: this.applyDnsProxy
    });
    this.hookFeature(featureName);
  }

  getHashKeyName(item = {}, level = "default") {
    if (!item.count || !item.error || !item.prefix) {
      log.error("Invalid item:", item);
      return null;
    }

    const { count, error, prefix } = item;

    if (level) {
      return `bf:${level}_${prefix}:${count}:${error}`;
    }
    return `bf:${prefix}:${count}:${error}`;
  }

  getFilePath(item = {}) {
    if (!item.count || !item.error || !item.prefix) {
      log.error("Invalid item:", item);
      return null;
    }

    const { count, error, prefix, level } = item;

    if (level) {
      return `${f.getRuntimeInfoFolder()}/${featureName}.${level}_${prefix}.bf.data`;
    }
    return `${f.getRuntimeInfoFolder()}/${featureName}.${prefix}.bf.data`;
  }

  getDnsmasqConfigFile() {
    return `${dnsmasqConfigFolder}/${featureName}.conf`;
  }

  async enableDnsmasqConfig(data) {
    log.info("Enabling dnsmasq config file for dnsproxy...");
    let dnsmasqEntry = "mac-address-tag=%FF:FF:FF:FF:FF:FF$dns_proxy&1\n";
    for (const level in data) {
      const levelData = data[level];
      for (const item of levelData) {
        item.level = level;
        const fp = this.getFilePath(item);
        if (!fp) {
          continue;
        }
        const entry = `server-bf-uhigh=<${fp},${item.count},${item.error}><${allowKey}><${blockKey}><${passthroughKey}>127.0.0.153#59953$dns_proxy\n`;
        dnsmasqEntry += entry;
      }
    }

    await fs.writeFileAsync(this.getDnsmasqConfigFile(), dnsmasqEntry);
    dnsmasq.scheduleRestartDNSService();
  }

  async disableDnsmasqConfig() {
    log.info("Disabling dnsmasq config file for dnsproxy...");
    await fs.unlinkAsync(this.getDnsmasqConfigFile()).catch(() => undefined); // ignore error
    dnsmasq.scheduleRestartDNSService();
  }

  async applyDnsProxy(host, ip, policy) {
    log.info("Applying dns_proxy", ip, policy);
    if (policy) {
      this.dnsProxyData = policy;
    }
    if (!this.dnsProxyData) {
      this.dnsProxyData = {};
      this.dnsProxyData["default"] = this.config.data;
    }

    if (!this.state) {
      log.info("dns_proxy feature is disabled, skip applying policy");
      return;
    }

    // level: strict, default... usually just one level at the same time, but the code supports multiple anyway
    for (const level in this.dnsProxyData) {
      const levelData = this.dnsProxyData[level];
      // item: data, new... each one is a bloom data
      for (const item of levelData) {
        item.level = level;
        const hashKeyName = bf.getHashKeyName(item);
        if (!hashKeyName) continue;

        log.info("Processing data file:", hashKeyName);
        const outputFilePath = this.getFilePath(item);
        await cc.enableCache(hashKeyName, async (data) => {
          if (data) {
            await bf.updateBFData(item, data, outputFilePath).catch((err) => {
              log.error("Failed to process data file, err:", err);
            });
          } else {
            log.error(`no dns_proxy data ${hashKeyName}. delete data file ${outputFilePath}`);
            await bf.deleteBFData(outputFilePath);
          }
          // always reschedule dnsmasq restarts when bf data is updated
          dnsmasq.scheduleRestartDNSService();
        });
      }
    }

    await this.enableDnsmasqConfig(this.dnsProxyData).catch((err) => {
      log.error("Failed to enable dnsmasq config, err", err);
    });
  }

  async globalOn() {
    this.state = true;
    sclient.subscribe(BF_SERVER_MATCH);
    sclient.on("message", async (channel, message) => {

      switch (channel) {
        case BF_SERVER_MATCH:
          let msgObj;
          try {
            msgObj = JSON.parse(message)
          } catch (err) {
            log.error("parse msg failed", err, message)
          }
          if (msgObj && msgObj.mac) {
            const MAC = msgObj.mac.toUpperCase();
            const ip = msgObj.ip4 || msgObj.ip6;

            // skip category bf match result
            if (msgObj.bf_path.startsWith("/home/pi/.firewalla/run/category_data/filters")) {
              return;
            }

            await this.processRequest(msgObj.domain, ip, MAC);
          }

          await m.incr("dns_proxy_request_cnt");
          break;
      }
    });

    sem.on("FastDNSPolicyComplete", async (event) => {
      await rclient.zaddAsync(passthroughKey, Math.floor(new Date() / 1000), event.domain);
    });

    await this.applyDnsProxy();
  }

  async globalOff() {
    this.state = false;

    sclient.unsubscribe(BF_SERVER_MATCH);

    if (!_.isEmpty(this.dnsProxyData)) {
      for (const level in this.dnsProxyData) {
        const levelData = this.dnsProxyData[level];
        for (const item of levelData) {
          item.level = level;
          const hashKeyName = this.getHashKeyName(item);
          if (!hashKeyName) continue;
          await cc.disableCache(hashKeyName).catch((err) => {
            log.error("Failed to disable cache, err:", err);
          });
        }
      }
    }

    await this.disableDnsmasqConfig();
  }

  async checkCache(domain) { // only return if there is exact-match intel in redis
    return intelTool.getDomainIntel(domain);
  }

  async _genSecurityAlarm(ip, mac, dn, item) {
    let macEntry;
    let realLocal;
    let guid;
    if (mac) macEntry = await hostTool.getMACEntry(mac);
    if (macEntry) {
      ip = macEntry.ipv4 || macEntry.ipv6 || ip;
    } else {
      const identity = ip && IdentityManager.getIdentityByIP(ip);
      if (identity) {
        guid = IdentityManager.getGUID(identity);
        realLocal = IdentityManager.getEndpointByIP(ip);
      }
      if (!guid) return;
      mac = guid;
    }
    const alarm = new Alarm.IntelAlarm(new Date() / 1000, ip, "major", {
      "p.device.ip": ip,
      "p.dest.name": dn,
      "p.dest.category": "intel",
      "p.security.reason": item.msg,
      "p.device.mac": mac,
      "p.action.block": true,
      "p.blockby": "fastdns",
      "p.local_is_client": "1"
    });
    if (realLocal) {
      alarm["p.device.real.ip"] = realLocal;
    }
    if (guid) {
      const identity = IdentityManager.getIdentityByGUID(guid);
      if (identity)
        alarm[identity.constructor.getKeyOfUIDInAlarm()] = identity.getUniqueId();
      alarm["p.device.guid"] = guid;
    }
    await am2.enrichDeviceInfo(alarm);
    am2.enqueueAlarm(alarm); // use enqueue to ensure no dup alarms
  }

  async processRequest(qname, ip, mac) {
    const domain = qname;
    log.info("dns request is", domain);
    const begin = new Date() / 1;
    const cache = await this.checkCache(domain);
    if (cache) {
      log.info(`inteldns:${domain} is already cached locally, updating redis cache keys directly...`);
      if (cache.c === "intel") {

        const isTrusted = await tm.matchDomain(domain);

        if(isTrusted) {
          await rclient.zaddAsync(passthroughKey, Math.floor(new Date() / 1000), domain);
        } else {
          await this._genSecurityAlarm(ip, mac, domain, cache)
        }

      } else {
        await rclient.zaddAsync(passthroughKey, Math.floor(new Date() / 1000), domain);
      }
      return; // do need to do anything
    }

    // if no cache, will trigger cloud inquiry and store the result in cache
    // assume client will send dns query again when cache is ready
    const result = await intelTool.checkIntelFromCloud(undefined, domain); // parameter ip is undefined since it's a dns request only
    await this.updateCache(domain, result, ip, mac);
    const end = new Date() / 1;
    log.info("dns intel result is", result, "took", Math.floor(end - begin), "ms");
  }

  redisfy(item) {
    for (const k in item) { // to suppress redis error
      const v = item[k];
      if (_.isBoolean(v) || _.isNumber(v) || _.isString(v)) {
        continue;
      }
      item[k] = JSON.stringify(v);
    }
  }

  getSortedItems(items) {
    // sort by length of originIP
    return items.sort((a, b) => {
      const oia = a.originIP;
      const oib = b.originIP;
      if (!oia && !oib) {
        return 0;
      }
      if (oia && !oib) {
        return -1;
      }
      if (!oia && oib) {
        return 1;
      }

      if (oia.length > oib.length) {
        return -1;
      } else if (oia.length < oib.length) {
        return 1;
      }

      return 0;
    });
  }

  async updateCache(domain, result, ip, mac) {
    if (_.isEmpty(result)) { // empty intel, means the domain is good
      const domains = flowUtil.getSubDomains(domain);
      if (!domains) {
        log.warn("Invalid Domain", domain, "skipped.");
        return;
      }

      // since result is empty, it means all sub domains of this domain are good
      const placeholder = { c: 'x', a: '1' };
      for (const dn of domains) {
        await intelTool.addDomainIntel(dn, placeholder, defaultExpireTime);
      }

      // only the exact dn is added to the passthrough and block list
      // generic domains can't be added, because another sub domain may be malicous
      // example:
      //   domain1.blogspot.com may be good
      //   and domain2.blogspot.com may be malicous
      // when domain1 is checked to be good, we should NOT add blogspot.com to passthrough_list
      await rclient.zaddAsync(passthroughKey, Math.floor(new Date() / 1000), domain);

    } else {
      // sort by length of originIP
      const validItems = result.filter((item) => item.originIP && validator.isFQDN(item.originIP));
      const sortedItems = this.getSortedItems(validItems);

      if (_.isEmpty(sortedItems)) {
        return;
      }

      for (const item of sortedItems) {
        this.redisfy(item);
        const dn = item.originIP; // originIP is the domain
        await intelTool.addDomainIntel(dn, item, item.e || defaultExpireTime);
        if (item.c !== "intel") {
          await rclient.zaddAsync(passthroughKey, Math.floor(new Date() / 1000), domain);
        }
      }

      for (const item of sortedItems) {
        if (item.c === "intel") {
          const dn = item.originIP; // originIP is the domain
          await this._genSecurityAlarm(ip, mac, dn, item);
          break;
        }
      }
    }
  }

}

module.exports = DNSProxyPlugin;
<|endoftext|>


<commit_before>/**
 * Get scene name
 * @param  {Element} scene Scene element
 * @return {string}       Scene ID or slugify from the current path
 */
export function getSceneName (scene) {
  return scene.id || slugify(window.location.host + window.location.pathname);
}

/**
 * Slugify the string removing non-word chars and spaces
 * @param  {string} text String to slugify
 * @return {string}      Slugified string
 */
function slugify (text) {
  return text.toString().toLowerCase()
    .replace(/\s+/g, '-')           // Replace spaces with -
    .replace(/[^\w\-]+/g, '-')      // Replace all non-word chars with -
    .replace(/\-\-+/g, '-')         // Replace multiple - with single -
    .replace(/^-+/, '')             // Trim - from start of text
    .replace(/-+$/, '');            // Trim - from end of text
}

/**
 * Generate a filtered stringify HTML from the current page
 * @return {string} String that contains the filtered HTML of the current page
 */
export function generateHtml () {
  var parser = new window.DOMParser();
  var xmlDoc = parser.parseFromString(document.documentElement.innerHTML, 'text/html');

  // Remove all the components that are being injected by aframe-inspector or aframe
  // @todo Use custom class to prevent this hack
  var elementsToRemove = xmlDoc.querySelectorAll([
    // Injected by the inspector
    '[data-aframe-inspector]',
    'script[src$="aframe-inspector.js"]',
    'style[type="text/css"]',
    // Injected by aframe
    '[aframe-injected]',
    'style[data-href$="aframe.css"]',
    // Injected by stats
    '.rs-base',
    'style[data-href$="rStats.css"]'
  ].join(','));
  for (var i = 0; i < elementsToRemove.length; i++) {
    var el = elementsToRemove[i];
    el.parentNode.removeChild(el);
  }

  return xmlToString(xmlDoc);
}

/**
 * Returns the string representation of a XML
 * @param  {Document} xmlData XML to stringify
 * @return {string}         String representation of the XML document
 */
function xmlToString (xmlData) {
  var xmlString;
  // IE
  if (window.ActiveXObject) {
    xmlString = xmlData.xml;
  } else {
    // Mozilla, Firefox, Opera, etc.
    xmlString = (new window.XMLSerializer()).serializeToString(xmlData);
  }
  return xmlString;
}
<commit_msg>flushToDOM() before generate HTML<commit_after>/**
 * Get scene name
 * @param  {Element} scene Scene element
 * @return {string}       Scene ID or slugify from the current path
 */
export function getSceneName (scene) {
  return scene.id || slugify(window.location.host + window.location.pathname);
}

/**
 * Slugify the string removing non-word chars and spaces
 * @param  {string} text String to slugify
 * @return {string}      Slugified string
 */
function slugify (text) {
  return text.toString().toLowerCase()
    .replace(/\s+/g, '-')           // Replace spaces with -
    .replace(/[^\w\-]+/g, '-')      // Replace all non-word chars with -
    .replace(/\-\-+/g, '-')         // Replace multiple - with single -
    .replace(/^-+/, '')             // Trim - from start of text
    .replace(/-+$/, '');            // Trim - from end of text
}

/**
 * Generate a filtered stringify HTML from the current page
 * @return {string} String that contains the filtered HTML of the current page
 */
export function generateHtml () {
  // flishToDOM first because the elements are posibilly modified by user in Inspector.
  document.querySelector('a-scene').flushToDOM(true);

  var parser = new window.DOMParser();
  var xmlDoc = parser.parseFromString(document.documentElement.innerHTML, 'text/html');

  // Remove all the components that are being injected by aframe-inspector or aframe
  // @todo Use custom class to prevent this hack
  var elementsToRemove = xmlDoc.querySelectorAll([
    // Injected by the inspector
    '[data-aframe-inspector]',
    'script[src$="aframe-inspector.js"]',
    'style[type="text/css"]',
    // Injected by aframe
    '[aframe-injected]',
    'style[data-href$="aframe.css"]',
    // Injected by stats
    '.rs-base',
    'style[data-href$="rStats.css"]'
  ].join(','));
  for (var i = 0; i < elementsToRemove.length; i++) {
    var el = elementsToRemove[i];
    el.parentNode.removeChild(el);
  }

  return xmlToString(xmlDoc);
}

/**
 * Returns the string representation of a XML
 * @param  {Document} xmlData XML to stringify
 * @return {string}         String representation of the XML document
 */
function xmlToString (xmlData) {
  var xmlString;
  // IE
  if (window.ActiveXObject) {
    xmlString = xmlData.xml;
  } else {
    // Mozilla, Firefox, Opera, etc.
    xmlString = (new window.XMLSerializer()).serializeToString(xmlData);
  }
  return xmlString;
}
<|endoftext|>


<commit_before>const fs = require('fs');
const assert = require('assert');
const common = require('../prelude/common.js');
const log = require('./log.js').log;
const pkgVersion = require('../package.json').version;

const STORE_CODE = common.STORE_CODE;
const STORE_CONTENT = common.STORE_CONTENT;
const STORE_LINKS = common.STORE_LINKS;
const STORE_STAT = common.STORE_STAT;

const isDotJS = common.isDotJS;
const isDotJSON = common.isDotJSON;
const snapshotify = common.snapshotify;

const bootstrapText = fs.readFileSync(
  require.resolve('../prelude/bootstrap.js'), 'utf8'
).replace('%PKG_VERSION%', pkgVersion);

const commonText = fs.readFileSync(
  require.resolve('../prelude/common.js'), 'utf8'
);

function itemsToText (items) {
  const len = items.length;
  return len.toString() +
    (len % 10 === 1 ? ' item' : ' items');
}

function reduceRecords (records) {
  assert(Array.isArray(records), 'packer: bad records to reduce');
  const result = {};

  records.some(function (record) {
    if (record.discard) return;
    const file = record.file;
    if (!result[file]) result[file] = {};
    result[file][record.store] = record.body;
  });

  return result;
}

function packer (opts, cb) {
  const stripe = [];

  function write (x) {
    assert(typeof x === 'string', 'packer: can write only strings');
    stripe.push(x);
  }

  const records = reduceRecords(opts.records);

  write('(function(REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT) {');
  write(bootstrapText);
  write('})(function(exports) {');
  write(commonText);
  write('}, {\n');

  Object.keys(records).some(function (file, fileIndex) {
    if (fileIndex) {
      write(',');
    }

    write(JSON.stringify(snapshotify(file, opts.slash)));
    write(':[\n');

    const record = records[file];
    assert(record[STORE_STAT], 'packer: no STORE_STAT');

    if ((typeof record[STORE_CODE] !== 'undefined') &&
        (typeof record[STORE_CONTENT] !== 'undefined')) {
      delete record[STORE_CODE];
    }

    [ STORE_CODE, STORE_CONTENT, STORE_LINKS, STORE_STAT
    ].some(function (store, storeIndex) {
      assert(store === storeIndex, 'packer: stores misordered');
      if (storeIndex) {
        write(',');
      }

      const value = record[store];

      if (typeof value === 'undefined') {
        write('null');
        return;
      }

      if (store === STORE_CODE) {
        assert(typeof value === 'string', 'packer: bad STORE_CODE');

        write('function(exports, require, module, __filename, __dirname) {\n');
        write(value);
        write('\n}'); // dont remove \n, otherwise last comment will cover right brace

        log.debug('The file was included as compiled code (no sources)', file);
      } else
      if (store === STORE_CONTENT) {
        if (Buffer.isBuffer(value)) {
          write('new Buffer(\'');
          write(value.toString('base64'));
          write('\',\'base64\')');
        } else
        if (typeof value === 'string') {
          write('new Buffer(\'');
          write((new Buffer(value)).toString('base64'));
          write('\',\'base64\')');
        } else {
          assert(false, 'packer: bad STORE_CONTENT');
        }

        const disclosed = isDotJS(file) || isDotJSON(file);
        log.debug(disclosed ? 'The file was included as DISCLOSED code (with sources)'
                            : 'The file was included as asset content', file);
      } else
      if (store === STORE_LINKS) {
        assert(Array.isArray(value), 'packer: bad STORE_LINKS');
        write(JSON.stringify(value));
        log.debug('The directory files list was included (' + itemsToText(value) + ')', file);
      } else
      if (store === STORE_STAT) {
        assert(typeof value === 'object', 'packer: bad STORE_STAT');
        const newValue = Object.assign({}, value);
        newValue.atime = value.atime.getTime();
        newValue.mtime = value.mtime.getTime();
        newValue.ctime = value.ctime.getTime();
        newValue.birthtime = value.birthtime.getTime();
        newValue.isFileValue = value.isFile();
        newValue.isDirectoryValue = value.isDirectory();
        write(JSON.stringify(newValue));
      } else {
        assert(false, 'packer: unknown store');
      }
    });

    write('\n]');
  });

  write('\n},');

  opts.records.some(function (record) {
    if (record.entrypoint) {
      write(JSON.stringify(snapshotify(record.file, opts.slash)));
      return true;
    }
  });

  write('\n)');

  cb(null, stripe);
}

module.exports = function (opts) {
  return new Promise((resolve, reject) => {
    packer(opts, (error, stripe) => {
      if (error) return reject(error);
      resolve(stripe);
    });
  });
};
<commit_msg>minor styling after PR<commit_after>const fs = require('fs');
const assert = require('assert');
const common = require('../prelude/common.js');
const log = require('./log.js').log;
const pkgVersion = require('../package.json').version;

const STORE_CODE = common.STORE_CODE;
const STORE_CONTENT = common.STORE_CONTENT;
const STORE_LINKS = common.STORE_LINKS;
const STORE_STAT = common.STORE_STAT;

const isDotJS = common.isDotJS;
const isDotJSON = common.isDotJSON;
const snapshotify = common.snapshotify;

const bootstrapText = fs.readFileSync(
  require.resolve('../prelude/bootstrap.js'), 'utf8'
).replace('%PKG_VERSION%', pkgVersion);

const commonText = fs.readFileSync(
  require.resolve('../prelude/common.js'), 'utf8'
);

function itemsToText (items) {
  const len = items.length;
  return len.toString() +
    (len % 10 === 1 ? ' item' : ' items');
}

function reduceRecords (records) {
  assert(Array.isArray(records), 'packer: bad records to reduce');
  const result = {};

  records.some(function (record) {
    if (record.discard) return;
    const file = record.file;
    if (!result[file]) result[file] = {};
    result[file][record.store] = record.body;
  });

  return result;
}

function packer (opts, cb) {
  const stripe = [];

  function write (x) {
    assert(typeof x === 'string', 'packer: can write only strings');
    stripe.push(x);
  }

  const records = reduceRecords(opts.records);

  write('(function(REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT) {');
  write(bootstrapText);
  write('})(function(exports) {');
  write(commonText);
  write('}, {\n');

  Object.keys(records).some(function (file, indexOfFile) {
    if (indexOfFile > 0) write(',');
    write(JSON.stringify(snapshotify(file, opts.slash)));
    write(':[\n');

    const record = records[file];
    assert(record[STORE_STAT], 'packer: no STORE_STAT');

    if ((typeof record[STORE_CODE] !== 'undefined') &&
        (typeof record[STORE_CONTENT] !== 'undefined')) {
      delete record[STORE_CODE];
    }

    [ STORE_CODE, STORE_CONTENT, STORE_LINKS, STORE_STAT
    ].some(function (store, indexOfStore) {
      assert(store === indexOfStore, 'packer: stores misordered');
      if (indexOfStore > 0) write(',');

      const value = record[store];

      if (typeof value === 'undefined') {
        write('null');
        return;
      }

      if (store === STORE_CODE) {
        assert(typeof value === 'string', 'packer: bad STORE_CODE');

        write('function(exports, require, module, __filename, __dirname) {\n');
        write(value);
        write('\n}'); // dont remove \n, otherwise last comment will cover right brace

        log.debug('The file was included as compiled code (no sources)', file);
      } else
      if (store === STORE_CONTENT) {
        if (Buffer.isBuffer(value)) {
          write('new Buffer(\'');
          write(value.toString('base64'));
          write('\',\'base64\')');
        } else
        if (typeof value === 'string') {
          write('new Buffer(\'');
          write((new Buffer(value)).toString('base64'));
          write('\',\'base64\')');
        } else {
          assert(false, 'packer: bad STORE_CONTENT');
        }

        const disclosed = isDotJS(file) || isDotJSON(file);
        log.debug(disclosed ? 'The file was included as DISCLOSED code (with sources)'
                            : 'The file was included as asset content', file);
      } else
      if (store === STORE_LINKS) {
        assert(Array.isArray(value), 'packer: bad STORE_LINKS');
        write(JSON.stringify(value));
        log.debug('The directory files list was included (' + itemsToText(value) + ')', file);
      } else
      if (store === STORE_STAT) {
        assert(typeof value === 'object', 'packer: bad STORE_STAT');
        const newValue = Object.assign({}, value);
        newValue.atime = value.atime.getTime();
        newValue.mtime = value.mtime.getTime();
        newValue.ctime = value.ctime.getTime();
        newValue.birthtime = value.birthtime.getTime();
        newValue.isFileValue = value.isFile();
        newValue.isDirectoryValue = value.isDirectory();
        write(JSON.stringify(newValue));
      } else {
        assert(false, 'packer: unknown store');
      }
    });

    write('\n]');
  });

  write('\n},');

  opts.records.some(function (record) {
    if (record.entrypoint) {
      write(JSON.stringify(snapshotify(record.file, opts.slash)));
      return true;
    }
  });

  write('\n)');

  cb(null, stripe);
}

module.exports = function (opts) {
  return new Promise((resolve, reject) => {
    packer(opts, (error, stripe) => {
      if (error) return reject(error);
      resolve(stripe);
    });
  });
};
<|endoftext|>


<commit_before>/*    Copyright 2021 Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require('../net2/logger.js')(__filename);

const Sensor = require('./Sensor.js').Sensor;

const _ = require('lodash');

const validator = require('validator');

const IntelTool = require('../net2/IntelTool');
const intelTool = new IntelTool();

const rclient = require('../util/redis_manager.js').getRedisClient();

const flowUtil = require('../net2/FlowUtil');

const sys = require('sys'),
      Buffer = require('buffer').Buffer,
      dgram = require('dgram');

// slices a single byte into bits
// assuming only single bytes
const sliceBits = function(b, off, len) {
    var s = 7 - (off + len - 1);

    b = b >>> s;
    return b & ~(0xff << len);
};

const qnameToDomain = (qname) => {
  var domain= '';
  for(var i=0;i<qname.length;i++) {
    if (qname[i] == 0) {
      //last char chop trailing .
      domain = domain.substring(0, domain.length - 1);
      break;
    }

    var tmpBuf = qname.slice(i+1, i+qname[i]+1);
    domain += tmpBuf.toString('binary', 0, tmpBuf.length);
    domain += '.';

    i = i + qname[i];
  }

  return domain;
}

const expireTime = 48 * 3600; // expire in two days, by default
const allowKey = "fastdns:allow_list";
const blockKey = "fastdns:block_list";
const defaultListenPort = 9963;

class DNSProxyPlugin extends Sensor {
  async run() {
    this.launchServer();
  }

  launchServer() {
    this.server = dgram.createSocket('udp4');

    this.server.on('message', async (msg, info) => {
      let req = this.parseRequest(msg);
      await this.processRequest(req);
      // never need to reply back to client as this is not a true dns server
    });

    let port = this.config.listenPort || defaultListenPort;
    if(!_.isNumber(port)) {
      log.warn("Invalid port", port, ", reverting back to default port.");
      port = defaultListenPort;
    }

    this.server.bind(port, '127.0.0.1');
    log.info("DNS proxy server is now running at port:", port);
  }

  parseRequest(req) {
    //see rfc1035 for more details
    //http://tools.ietf.org/html/rfc1035#section-4.1.1

    var query = {};
    query.header = {};
    //TODO write code to break questions up into an array
    query.question = {};

    var tmpSlice;
    var tmpByte;

    //transaction id
    // 2 bytes
    query.header.id = req.slice(0,2);

    //slice out a byte for the next section to dice into binary.
    tmpSlice = req.slice(2,3);
    //convert the binary buf into a string and then pull the char code
    //for the byte
    tmpByte = tmpSlice.toString('binary', 0, 1).charCodeAt(0);

    //qr
    // 1 bit
    query.header.qr = sliceBits(tmpByte, 0,1);
    //opcode
    // 0 = standard, 1 = inverse, 2 = server status, 3-15 reserved
    // 4 bits
    query.header.opcode = sliceBits(tmpByte, 1,4);
    //authorative answer
    // 1 bit
    query.header.aa = sliceBits(tmpByte, 5,1);
    //truncated
    // 1 bit
    query.header.tc = sliceBits(tmpByte, 6,1);
    //recursion desired
    // 1 bit
    query.header.rd = sliceBits(tmpByte, 7,1);

    //slice out a byte to dice into binary
    tmpSlice = req.slice(3,4);
    //convert the binary buf into a string and then pull the char code
    //for the byte
    tmpByte = tmpSlice.toString('binary', 0, 1).charCodeAt(0);

    //recursion available
    // 1 bit
    query.header.ra = sliceBits(tmpByte, 0,1);

    //reserved 3 bits
    // rfc says always 0
    query.header.z = sliceBits(tmpByte, 1,3);

    //response code
    // 0 = no error, 1 = format error, 2 = server failure
    // 3 = name error, 4 = not implemented, 5 = refused
    // 6-15 reserved
    // 4 bits
    query.header.rcode = sliceBits(tmpByte, 4,4);

    //question count
    // 2 bytes
    query.header.qdcount = req.slice(4,6);
    //answer count
    // 2 bytes
    query.header.ancount = req.slice(6,8);
    //ns count
    // 2 bytes
    query.header.nscount = req.slice(8,10);
    //addition resources count
    // 2 bytes
    query.header.arcount = req.slice(10, 12);

    //assuming one question
    //qname is the sequence of domain labels
    //qname length is not fixed however it is 4
    //octets from the end of the buffer
    query.question.qname = req.slice(12, req.length - 4);
    //qtype
    query.question.qtype = req.slice(req.length - 4, req.length - 2);
    //qclass
    query.question.qclass = req.slice(req.length - 2, req.length);

    return query;
  }

  getKey(domain) {
    return `intel:dns:${domain}`;
  }

  async checkCache(domain) {
    const domains = flowUtil.getSubDomains(domain);
    if(!domains) {
      log.warn("Invalid Domain", domain, "skipped.");
      return;
    }

    for(const dn of domains) {
      const key = this.getKey(dn);
      const info = await rclient.hgetallAsync(key);
      if(!_.isEmpty(info)) {
        return info;
      }
    }

    return null;
  }

  async processRequest(req) {
    const qname = req.question.qname;
    const domain = qnameToDomain(qname);
    log.info("dns request is", domain);
    const begin = new Date() / 1;
    const cache = await this.checkCache(domain);
    if(cache) {
      log.info("domain info is already cached locally:", cache);
      return; // do need to do anything
    }
    const result = await intelTool.checkIntelFromCloud(undefined, domain); // parameter ip is undefined since it's a dns request only
    await this.updateCache(domain, result);
    const end = new Date() / 1;
    log.info("dns intel result is", result, "took", Math.floor(end - begin), "ms");
  }

  async updateCache(domain, result) {
    if(_.isEmpty(result)) { // empty intel, means the domain is good
      const domains = flowUtil.getSubDomains(domain);
      if(!domains) {
        log.warn("Invalid Domain", domain, "skipped.");
        return;
      }

      const lastDN = domains[domains.length - 1];
      const key = this.getKey(lastDN);
      await rclient.hmsetAsync(key, {c: 'x', a: '1'});
      await rclient.saddAsync(allowKey, lastDN);
      await rclient.expire(key, expireTime);
    } else {
      for(const item of result) {
        const dn = item.originIP; // originIP is the domain
        const isDomain = validator.isFQDN(dn);
        if(!isDomain) {
          continue;
        }

        const key = this.getKey(dn);
        await rclient.hmsetAsync(key, item);
        if(item.c === 'intel') { // need to decide the criteria better
          item.a = '0';
          await rclient.saddAsync(blockKey, dn);
        } else {
          item.a = '1';
          await rclient.saddAsync(allowKey, dn);
        }
        const expire = item.e || expireTime;
        await rclient.expire(key, expire);
      }
    }
  }
}

module.exports = DNSProxyPlugin;
<commit_msg>bug fix<commit_after>/*    Copyright 2021 Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require('../net2/logger.js')(__filename);

const Sensor = require('./Sensor.js').Sensor;

const _ = require('lodash');

const validator = require('validator');

const IntelTool = require('../net2/IntelTool');
const intelTool = new IntelTool();

const rclient = require('../util/redis_manager.js').getRedisClient();

const flowUtil = require('../net2/FlowUtil');

const sys = require('sys'),
      Buffer = require('buffer').Buffer,
      dgram = require('dgram');

// slices a single byte into bits
// assuming only single bytes
const sliceBits = function(b, off, len) {
    var s = 7 - (off + len - 1);

    b = b >>> s;
    return b & ~(0xff << len);
};

const qnameToDomain = (qname) => {
  var domain= '';
  for(var i=0;i<qname.length;i++) {
    if (qname[i] == 0) {
      //last char chop trailing .
      domain = domain.substring(0, domain.length - 1);
      break;
    }

    var tmpBuf = qname.slice(i+1, i+qname[i]+1);
    domain += tmpBuf.toString('binary', 0, tmpBuf.length);
    domain += '.';

    i = i + qname[i];
  }

  return domain;
}

const expireTime = 48 * 3600; // expire in two days, by default
const allowKey = "fastdns:allow_list";
const blockKey = "fastdns:block_list";
const defaultListenPort = 9963;

class DNSProxyPlugin extends Sensor {
  async run() {
    this.launchServer();
  }

  launchServer() {
    this.server = dgram.createSocket('udp4');

    this.server.on('message', async (msg, info) => {
      let req = this.parseRequest(msg);
      await this.processRequest(req);
      // never need to reply back to client as this is not a true dns server
    });

    let port = this.config.listenPort || defaultListenPort;
    if(!_.isNumber(port)) {
      log.warn("Invalid port", port, ", reverting back to default port.");
      port = defaultListenPort;
    }

    this.server.bind(port, '127.0.0.1');
    log.info("DNS proxy server is now running at port:", port);
  }

  parseRequest(req) {
    //see rfc1035 for more details
    //http://tools.ietf.org/html/rfc1035#section-4.1.1

    var query = {};
    query.header = {};
    //TODO write code to break questions up into an array
    query.question = {};

    var tmpSlice;
    var tmpByte;

    //transaction id
    // 2 bytes
    query.header.id = req.slice(0,2);

    //slice out a byte for the next section to dice into binary.
    tmpSlice = req.slice(2,3);
    //convert the binary buf into a string and then pull the char code
    //for the byte
    tmpByte = tmpSlice.toString('binary', 0, 1).charCodeAt(0);

    //qr
    // 1 bit
    query.header.qr = sliceBits(tmpByte, 0,1);
    //opcode
    // 0 = standard, 1 = inverse, 2 = server status, 3-15 reserved
    // 4 bits
    query.header.opcode = sliceBits(tmpByte, 1,4);
    //authorative answer
    // 1 bit
    query.header.aa = sliceBits(tmpByte, 5,1);
    //truncated
    // 1 bit
    query.header.tc = sliceBits(tmpByte, 6,1);
    //recursion desired
    // 1 bit
    query.header.rd = sliceBits(tmpByte, 7,1);

    //slice out a byte to dice into binary
    tmpSlice = req.slice(3,4);
    //convert the binary buf into a string and then pull the char code
    //for the byte
    tmpByte = tmpSlice.toString('binary', 0, 1).charCodeAt(0);

    //recursion available
    // 1 bit
    query.header.ra = sliceBits(tmpByte, 0,1);

    //reserved 3 bits
    // rfc says always 0
    query.header.z = sliceBits(tmpByte, 1,3);

    //response code
    // 0 = no error, 1 = format error, 2 = server failure
    // 3 = name error, 4 = not implemented, 5 = refused
    // 6-15 reserved
    // 4 bits
    query.header.rcode = sliceBits(tmpByte, 4,4);

    //question count
    // 2 bytes
    query.header.qdcount = req.slice(4,6);
    //answer count
    // 2 bytes
    query.header.ancount = req.slice(6,8);
    //ns count
    // 2 bytes
    query.header.nscount = req.slice(8,10);
    //addition resources count
    // 2 bytes
    query.header.arcount = req.slice(10, 12);

    //assuming one question
    //qname is the sequence of domain labels
    //qname length is not fixed however it is 4
    //octets from the end of the buffer
    query.question.qname = req.slice(12, req.length - 4);
    //qtype
    query.question.qtype = req.slice(req.length - 4, req.length - 2);
    //qclass
    query.question.qclass = req.slice(req.length - 2, req.length);

    return query;
  }

  getKey(domain) {
    return `intel:dns:${domain}`;
  }

  async checkCache(domain) {
    const domains = flowUtil.getSubDomains(domain);
    if(!domains) {
      log.warn("Invalid Domain", domain, "skipped.");
      return;
    }

    for(const dn of domains) {
      const key = this.getKey(dn);
      const info = await rclient.hgetallAsync(key);
      if(!_.isEmpty(info)) {
        return info;
      }
    }

    return null;
  }

  async processRequest(req) {
    const qname = req.question.qname;
    const domain = qnameToDomain(qname);
    log.info("dns request is", domain);
    const begin = new Date() / 1;
    const cache = await this.checkCache(domain);
    if(cache) {
      log.info("domain info is already cached locally:", cache);
      return; // do need to do anything
    }
    const result = await intelTool.checkIntelFromCloud(undefined, domain); // parameter ip is undefined since it's a dns request only
    await this.updateCache(domain, result);
    const end = new Date() / 1;
    log.info("dns intel result is", result, "took", Math.floor(end - begin), "ms");
  }

  async updateCache(domain, result) {
    if(_.isEmpty(result)) { // empty intel, means the domain is good
      const domains = flowUtil.getSubDomains(domain);
      if(!domains) {
        log.warn("Invalid Domain", domain, "skipped.");
        return;
      }

      const lastDN = domains[domains.length - 1];
      const key = this.getKey(lastDN);
      await rclient.hmsetAsync(key, {c: 'x', a: '1'});
      await rclient.saddAsync(allowKey, lastDN);
      await rclient.expire(key, expireTime);
    } else {
      for(const item of result) {
        const dn = item.originIP; // originIP is the domain
        const isDomain = validator.isFQDN(dn);
        if(!isDomain) {
          continue;
        }

        const key = this.getKey(dn);
        for(const k in item) { // to suppress redis error
          const v = item[k];
          if(_.isBoolean(v) || _.isNumber(v) || _.isString(v)) {
            continue;
          }
          item[k] = JSON.stringify(v);
        }
        await rclient.hmsetAsync(key, item);
        if(item.c === 'intel') { // need to decide the criteria better
          item.a = '0';
          await rclient.saddAsync(blockKey, dn);
        } else {
          item.a = '1';
          await rclient.saddAsync(allowKey, dn);
        }
        const expire = item.e || expireTime;
        await rclient.expire(key, expire);
      }
    }
  }
}

module.exports = DNSProxyPlugin;
<|endoftext|>


<commit_before>import emojis from 'emojilib';

function getEmoji(keyword) {
	let puncs = '!"#$%&\'()*+,-./:;<=>?@[]^_`{|}~';
	keyword = keyword.trim().toLowerCase();

	if (!keyword || keyword === '' || keyword === 'it')
		return	'';

	let maybeSingular = '';
	if (keyword.length > 2 && keyword[keyword.length - 1] == 's')
		maybeSingular = keyword.slice(0, keyword.length - 1);

	let maybePlural = (keyword.length == 1) ? '' : keyword + 's';

	let startPuncs='', endPuncs='';
	while(puncs.indexOf(keyword[0]) != -1)
	{
		startPuncs+=keyword[0];
		keyword = keyword.slice(1, keyword.length);
	}

	while(puncs.indexOf(keyword[keyword.length-1]) != -1)
	{
		endPuncs+=keyword[keyword.length-1];
		keyword = keyword.slice(0, keyword.length-1);
	}

	for (let emoji in emojis) {
		let keywords = emojis[emoji].keywords;
		if (emoji == keyword || emoji == maybeSingular || emoji == maybePlural ||
				(keywords && keywords.indexOf(keyword) >= 0) ||
				(keywords && keywords.indexOf(maybeSingular) >= 0) ||
        (keywords && keywords.indexOf(maybePlural) >= 0))
		{

			if(emojis[emoji].char)
				return startPuncs+emojis[emoji].char+endPuncs;

			else
				return '';
		}

	}
	return '';
}

export default getEmoji;
<commit_msg>Update getEmoji.js<commit_after>import emojis from 'emojilib';

function getEmoji(keyword) {
	let puncs = '!"#$%&\'()*+,-./:;<=>?@[]^_`{|}~';
	keyword = keyword.trim().toLowerCase();

	if (!keyword || keyword === '' || keyword === 'it')
		return	'';

	let maybeSingular = '';
	if (keyword.length > 2 && keyword[keyword.length - 1] == 's')
		maybeSingular = keyword.slice(0, keyword.length - 1);

	let maybePlural = (keyword.length == 1) ? '' : keyword + 's';

	let startPuncs='', endPuncs='';
	while(puncs.indexOf(keyword[0]) != -1)
	{
		startPuncs+=keyword[0];
		keyword = keyword.slice(1, keyword.length);
	}

	while(puncs.indexOf(keyword[keyword.length-1]) != -1)
	{
		endPuncs+=keyword[keyword.length-1];
		keyword = keyword.slice(0, keyword.length-1);
	}

	for (let emoji in emojis) {
		let keywords = emojis[emoji].keywords;
		if (emoji == keyword || emoji == maybeSingular || emoji == maybePlural ||
				(keywords && keywords.indexOf(keyword) >= 0) ||
				(keywords && keywords.indexOf(maybeSingular) >= 0) ||
        (keywords && keywords.indexOf(maybePlural) >= 0))
		{
			if(emojis[emoji].char)
				return startPuncs+emojis[emoji].char+endPuncs;
		}

	}
	return '';
}

export default getEmoji;
<|endoftext|>