<commit_before>/**
 * @license
 * Copyright 2018 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// This file create the main interface of the app.
'use strict';

(function() {

var loadTests = function(testType) {
  // We have to make it compatible to the legacy url format.
  var testName = testType.substr(0, testType.indexOf('-'));
  testName = util.MakeCapitalName(testName) + 'Test';
  return window[testName]();
};

var parseParam = function(param, defaultValue) {
  var regex = new RegExp('(\\?|\\&)' + param + '=([-:,\\w]+)', 'g');
  var value = regex.exec(document.URL);
  return value ? value[2] : defaultValue;
};

var parseParams = function(testSuiteConfig) {
  var config = {};
  config.testType = parseParam('test_type', testSuiteConfig.defaultTestSuite);
  config.command = parseParam('command', '');
  config.timeout = Number(parseParam('timeout', TestBase.timeout));
  config.logging = !util.stringToBoolean(parseParam('disable_log', false));
  config.loop = util.stringToBoolean(parseParam('loop', false));
  config.stoponfailure = util.stringToBoolean(
      parseParam('stoponfailure', false));
  config.enablewebm = util.stringToBoolean(
      parseParam('enablewebm', testSuiteConfig.enablewebm));
  config.novp9 = util.stringToBoolean(parseParam('novp9', false));
  config.tests = parseParam('tests');
  config.exclude = parseParam('exclude');
  config.testsMask = parseParam('tests_mask', '');
  config.testid = parseParam('testid', '');

  config.is_cobalt = util.isCobalt();
  config.support_hdr = util.supportHdr();
  config.support_webgl = util.supportWebGL();
  config.support_webspeech = util.supportWebSpeech();

  // Overloaded run command to support browsers that have limitations on extra
  // parameters. Example usage: command=run:1,2,3
  if (config.command.indexOf(':') != -1) {
    var commandSplit = config.command.split(':');
    config.command = commandSplit[0];
    config.tests = commandSplit[1];
  }
  return config;
};

var configureHarness = function(testSuiteConfig) {
  harnessConfig.controlMediaFormatSelection =
      testSuiteConfig.controlMediaFormatSelection;
  harnessConfig.recycleVideoTag = true;
  TestBase.timeout = harnessConfig.timeout;

  if (harnessConfig.testsMask) {
    harnessConfig.testsMask += '0';
  } else if (harnessConfig.tests) {
    harnessConfig.tests = harnessConfig.tests.split(',').
        map(function(x) {return parseInt(x);}).
        sort(function(a, b) {return a - b;});
    for (var i = 0; i < harnessConfig.tests.length; ++i) {
      var index = harnessConfig.tests[i] * 1 - 1;
      if (index < 0)
        continue;
      harnessConfig.testsMask = util.resize(
          harnessConfig.testsMask, index, '0');
      harnessConfig.testsMask += '1';
    }
    harnessConfig.testsMask += '0';
  } else if (harnessConfig.exclude) {
    harnessConfig.exclude = harnessConfig.exclude.split(',').
        map(function(x) {return parseInt(x);}).
        sort(function(a, b) {return a - b;});
    for (var i = 0; i < harnessConfig.exclude.length; ++i) {
      var index = harnessConfig.exclude[i] * 1 - 1;
      if (index < 0)
        continue;
      harnessConfig.testsMask = util.resize(
          harnessConfig.testsMask, index, '1');
      harnessConfig.testsMask += '0';
    }
    harnessConfig.testsMask += '1';
  }

  if (!harnessConfig.testsMask) {
    harnessConfig.testsMask = '1';
  }
};

var createLogger = function() {
  window.LOG = function() {
    if (!harnessConfig.logging)
      return;
    var output = document.getElementById('output');
    var text = '';

    for (var i = 0; i < arguments.length; ++i)
      text += arguments[i].toString() + ' ';

    console.log(text);
    output.innerHTML = text + '\n' + output.innerHTML;
  };
};

window.globalRunner = null;

var createRunner = function(testSuite, testSuiteVer, testsMask) {
  var runner = new TestExecutor(testSuite, testsMask, testSuiteVer);

  // Expose the runner so outside/injected scripts can read it.
  window.globalRunner = runner;

  var id = 0;
  runner.getNewVideoTag = function() {
    var testarea = document.getElementById('testarea');
    var vid = 'v' + id;
    if (harnessConfig.recycleVideoTag)
      ++id;
    if (!document.getElementById(vid)) {
      testarea.innerHTML = '';
      testarea.appendChild(util.createElement('video', vid, 'box-right'));
      document.getElementById(vid).controls = true;
    }
    return document.getElementById(vid);
  };

  runner.getControlContainer = function() {
    return document.getElementById('control');
  };

  runner.initialize();
  return runner;
};

var addTimestampHash = function() {
  var newTimeStamp = (new Date()).getTime();
  window.location.hash = newTimeStamp;
};

window.startMseTest = function(testSuiteVer) {
  setupMsePortability(testSuiteVer);
  var testSuiteVersion = testSuiteVersions[testSuiteVer];
  window.harnessConfig = parseParams(testSuiteVersion.config);
  window.harnessConfig.testSuite = testSuiteVer;

  addTimestampHash();

  if (!testSuiteVersion.testSuites.indexOf(harnessConfig.testType) === -1) {
    alert('Cannot find test type ' + harnessConfig.testType);
    throw 'Cannot find test type ' + harnessConfig.testType;
  }
  // This is a hack that will not work in all cases.
  if (!!window.Media && harnessConfig.novp9) {
    Media.VP9 = Media.H264;
  }

  configureHarness(testSuiteVersion.config);
  createLogger();

  var testSuite = loadTests(harnessConfig.testType);
  var runner = createRunner(testSuite, testSuiteVer, harnessConfig.testsMask);
  if (harnessConfig.command === 'run')
    runner.startTest(0, runner.testList.length);
};

})();
<commit_msg>Test Selection functionality<commit_after>/**
 * @license
 * Copyright 2018 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// This file create the main interface of the app.
'use strict';

(function() {

var loadTests = function(testType) {
  // We have to make it compatible to the legacy url format.
  var testName = testType.substr(0, testType.indexOf('-'));
  testName = util.MakeCapitalName(testName) + 'Test';
  return window[testName]();
};

var parseParam = function(param, defaultValue) {
  var regex = new RegExp('(\\?|\\&)' + param + '=([-:,\\w]+)', 'g');
  var value = regex.exec(document.URL);
  return value ? value[2] : defaultValue;
};

var parseParams = function(testSuiteConfig) {
  var config = {};
  config.testType = parseParam('test_type', testSuiteConfig.defaultTestSuite);
  config.command = parseParam('command', '');
  config.timeout = Number(parseParam('timeout', TestBase.timeout));
  config.logging = !util.stringToBoolean(parseParam('disable_log', false));
  config.loop = util.stringToBoolean(parseParam('loop', false));
  config.stoponfailure = util.stringToBoolean(
      parseParam('stoponfailure', false));
  config.enablewebm = util.stringToBoolean(
      parseParam('enablewebm', testSuiteConfig.enablewebm));
  config.novp9 = util.stringToBoolean(parseParam('novp9', false));
  config.tests = parseParam('tests');
  config.exclude = parseParam('exclude');
  config.testsMask = parseParam('tests_mask', '');
  config.testid = parseParam('testid', '');

  config.is_cobalt = util.isCobalt();
  config.support_hdr = util.supportHdr();
  config.support_webgl = util.supportWebGL();
  config.support_webspeech = util.supportWebSpeech();

  // Overloaded run command to support browsers that have limitations on extra
  // parameters. Example usage: command=run:1,2,3
  if (config.command.indexOf(':') != -1) {
    var commandSplit = config.command.split(':');
    config.command = commandSplit[0];
    config.tests = commandSplit[1];
  }
  return config;
};

var configureHarness = function(testSuiteConfig) {
  harnessConfig.controlMediaFormatSelection =
      testSuiteConfig.controlMediaFormatSelection;
  harnessConfig.recycleVideoTag = true;
  TestBase.timeout = harnessConfig.timeout;

  if (harnessConfig.testsMask) {
    harnessConfig.testsMask += '0';
  } else if (harnessConfig.tests) {
    harnessConfig.tests =
        harnessConfig.tests.split(',')
            .map(function(x) {
              if (x.indexOf('-') != -1) {
                var [start, end] = x.split('-');
                var arr = [];
                for (var i = parseInt(start); i <= parseInt(end); i++) {
                  arr.push(i);
                }
                return arr;
              } else {
                return parseInt(x);
              }
            })
            .reduce((acc, val) => acc.concat(val), [])
            .sort(function(a, b) {
              return a - b;
            });
    harnessConfig.tests = [...new Set(harnessConfig.tests)];
    for (var i = 0; i < harnessConfig.tests.length; ++i) {
      var index = harnessConfig.tests[i] * 1 - 1;
      if (index < 0)
        continue;
      harnessConfig.testsMask =
          util.resize(harnessConfig.testsMask, index, '0');
      harnessConfig.testsMask += '1';
    }
    harnessConfig.testsMask += '0';
  } else if (harnessConfig.exclude) {
    harnessConfig.exclude = harnessConfig.exclude.split(',').
        map(function(x) {return parseInt(x);}).
        sort(function(a, b) {return a - b;});
    for (var i = 0; i < harnessConfig.exclude.length; ++i) {
      var index = harnessConfig.exclude[i] * 1 - 1;
      if (index < 0)
        continue;
      harnessConfig.testsMask =
          util.resize(harnessConfig.testsMask, index, '1');
      harnessConfig.testsMask += '0';
    }
    harnessConfig.testsMask += '1';
  }

  if (!harnessConfig.testsMask) {
    harnessConfig.testsMask = '1';
  }
};

var createLogger = function() {
  window.LOG = function() {
    if (!harnessConfig.logging)
      return;
    var output = document.getElementById('output');
    var text = '';

    for (var i = 0; i < arguments.length; ++i)
      text += arguments[i].toString() + ' ';

    console.log(text);
    output.innerHTML = text + '\n' + output.innerHTML;
  };
};

window.globalRunner = null;

var createRunner = function(testSuite, testSuiteVer, testsMask) {
  var runner = new TestExecutor(testSuite, testsMask, testSuiteVer);

  // Expose the runner so outside/injected scripts can read it.
  window.globalRunner = runner;

  var id = 0;
  runner.getNewVideoTag = function() {
    var testarea = document.getElementById('testarea');
    var vid = 'v' + id;
    if (harnessConfig.recycleVideoTag)
      ++id;
    if (!document.getElementById(vid)) {
      testarea.innerHTML = '';
      testarea.appendChild(util.createElement('video', vid, 'box-right'));
      document.getElementById(vid).controls = true;
    }
    return document.getElementById(vid);
  };

  runner.getControlContainer = function() {
    return document.getElementById('control');
  };

  runner.initialize();
  return runner;
};

var addTimestampHash = function() {
  var newTimeStamp = (new Date()).getTime();
  window.location.hash = newTimeStamp;
};

window.startMseTest = function(testSuiteVer) {
  setupMsePortability(testSuiteVer);
  var testSuiteVersion = testSuiteVersions[testSuiteVer];
  window.harnessConfig = parseParams(testSuiteVersion.config);
  window.harnessConfig.testSuite = testSuiteVer;

  addTimestampHash();

  if (!testSuiteVersion.testSuites.indexOf(harnessConfig.testType) === -1) {
    alert('Cannot find test type ' + harnessConfig.testType);
    throw 'Cannot find test type ' + harnessConfig.testType;
  }
  // This is a hack that will not work in all cases.
  if (!!window.Media && harnessConfig.novp9) {
    Media.VP9 = Media.H264;
  }

  configureHarness(testSuiteVersion.config);
  createLogger();

  var testSuite = loadTests(harnessConfig.testType);
  var runner = createRunner(testSuite, testSuiteVer, harnessConfig.testsMask);
  if (harnessConfig.command === 'run')
    runner.startTest(0, runner.testList.length);
};

})();
<|endoftext|>


<commit_before>function Metronome() {
  this.bpm = 60;
  this.beepPattern = [{
    gain: 0.5,
    frequency: 440,
    duration: 0.02,
  }];

  this._context = null;
  this._gain = null;
  this._osc = null;
  this._queuedTime = null;
  this._patternIndex = 0;
}

Metronome.prototype.start = function() {
  if (this._startTime != null) return;

  this._context = new AudioContext();
  this._gain = this._context.createGain();
  this._osc = this._context.createOscillator();
  this._osc.type = 'triangle';

  this._osc.connect(this._gain);
  this._gain.connect(this._context.destination);

  this._gain.gain.value = 0;
  this._osc.start();
  this._queuedTime = this._context.currentTime;
  this._patternIndex = 0;
  this._queueEvents();
}

Metronome.prototype.stop = function() {
  this._osc.stop();
  this._gain.gain.cancelScheduledValues(this._context.currentTime);
  this._startTime = null;
  this._gain = null;
  this._osc = null;
  this._context.close();
}

Metronome.prototype._queueEvents = function() {
  if (this._queuedTime == null) return;
  var kTimeoutMs = 100;
  var kMinDuration = 0.1;
  var kMaxBeepDuration = 0.8 * kMinDuration;

  var beatDuration = Math.max(60.0 / this.bpm, kMinDuration);
  var target = this._context.currentTime + 2*kTimeoutMs/1000.0;
  while (this._queuedTime < target) {
    this._queuedTime += beatDuration;

    var pattern = this.beepPattern[this._patternIndex];
    this._patternIndex = (this._patternIndex + 1) % this.beepPattern.length;

    this._osc.frequency.setValueAtTime(pattern.frequency, this._queuedTime);

    var duration = Math.min(pattern.duration, kMaxBeepDuration);
    var tau = duration / 5.0;
    this._gain.gain.setTargetAtTime(pattern.gain, this._queuedTime, tau);
    this._gain.gain.setTargetAtTime(0, this._queuedTime + duration, tau);
  }

  var that = this;
  window.setTimeout(function() { that._queueEvents(); }, kTimeoutMs);
}
<commit_msg>Change the way beeps are specified.<commit_after>function Metronome() {
  this.bpm = 60;
  this.beepParams = {
    gain: 0.5,
    frequency: 440,
    duration: 0.04,
  };

  // A list of modifier functions that operate on a copy of this.beepParams.
  this.beepModifiers = [function(beep) {}];

  this._context = null;
  this._gain = null;
  this._osc = null;
  this._queuedTime = null;
  this._modifierIndex = 0;
}

Metronome.prototype.start = function() {
  if (this._startTime != null) return;

  this._context = new AudioContext();
  this._gain = this._context.createGain();
  this._osc = this._context.createOscillator();
  this._osc.type = 'triangle';

  this._osc.connect(this._gain);
  this._gain.connect(this._context.destination);

  this._gain.gain.value = 0;
  this._osc.start();
  this._queuedTime = this._context.currentTime;
  this._modifierIndex = -1;
  this._queueEvents();
}

Metronome.prototype.stop = function() {
  this._osc.stop();
  this._gain.gain.cancelScheduledValues(this._context.currentTime);
  this._startTime = null;
  this._gain = null;
  this._osc = null;
  this._context.close();
}

Metronome.prototype._queueEvents = function() {
  if (this._queuedTime == null) return;
  var kTimeoutMs = 100;
  var kMinDuration = 0.1;
  var kMaxBeepDuration = 0.8 * kMinDuration;

  var beatDuration = Math.max(60.0 / this.bpm, kMinDuration);
  var target = this._context.currentTime + 2*kTimeoutMs/1000.0;
  while (this._queuedTime < target) {
    this._queuedTime += beatDuration;

    var beep = Object.create(this.beepParams);
    this._modifierIndex = (this._modifierIndex + 1) % this.beepModifiers.length;
    this.beepModifiers[this._modifierIndex](beep);

    this._osc.frequency.setValueAtTime(beep.frequency, this._queuedTime);

    var duration = Math.min(beep.duration, kMaxBeepDuration);
    var tau = duration / 5.0;
    this._gain.gain.setTargetAtTime(beep.gain, this._queuedTime, tau);
    this._gain.gain.setTargetAtTime(0, this._queuedTime + duration, tau);
  }

  var that = this;
  window.setTimeout(function() { that._queueEvents(); }, kTimeoutMs);
}
<|endoftext|>


<commit_before>export { NeuralNet_TestParams_Base as Base };

import * as Pool from "../../util/Pool.js";
import * as Recyclable from "../../util/Recyclable.js";
import * as RandTools from "../../util/RandTools.js";
import * as ValueDesc from "../../Unpacker/ValueDesc.js";
import * as NeuralNet from "../../Conv/NeuralNet.js";
import * as TestParams from "./TestParams.js";
import * as Embedding_TestParams from "./Embedding_TestParams.js";
import * as Stage_TestParams from "./Stage_TestParams.js";

/**
 *
 * This is an object { id, in, out } which has one number and two sub-objects.
 *
 *
 * @see TestParams.Base
 */
class NeuralNet_TestParams_Base extends TestParams.Base {

  /**
   * Used as default NeuralNet_TestParams.Base provider for conforming to Recyclable interface.
   */
  static Pool = new Pool.Root( "NeuralNet_TestParams.Base.Pool",
    NeuralNet_TestParams_Base, NeuralNet_TestParams_Base.setAsConstructor );

  /**
   */
  constructor( id ) {
    super( id );
    NeuralNet_TestParams_Base.setAsConstructor_self.call( this );
  }

  /** @override */
  static setAsConstructor( id ) {
    super.setAsConstructor( id );
    NeuralNet_TestParams_Base.setAsConstructor_self.call( this );
    return this;
  }

  /** @override */
  static setAsConstructor_self() {
    this.stageArray = Recyclable.OwnerArray.Pool.get_or_create_by();
    this.out = NeuralNet.ParamsBase.Pool.get_or_create_by();
  }

  /** @override */
  disposeResources() {
    this.out?.disposeResources_and_recycleToPool();
    this.out = null;

    this.stageArray?.disposeResources_and_recycleToPool();
    this.stageArray = null;

    this.embedding?.disposeResources_and_recycleToPool();
    this.embedding = null;

    super.disposeResources();
  }

  /**
   * Use scattered parameters to fills the following proterties:
   *   - this.in
   *   - this.in_weights
   *   - this.out
   *
   * @return {Base}
   *   Return this object self.
   */
  set_byParamsScattered(
    input_height, input_width, input_channelCount,
    vocabularyChannelCount, vocabularyCountPerInputChannel,
    nConvStageTypeId, stageCountRequested,
    blockCountRequested,
    bKeepInputTensor
  ) {

    if ( this.out ) {
      this.out.disposeResources_and_recycleToPool();
      this.out = null;
    }

    this.out = NeuralNet.ParamsBase.Pool.get_or_create_by(
      input_height, input_width, input_channelCount,
      vocabularyChannelCount, vocabularyCountPerInputChannel,
      nConvStageTypeId, stageCountRequested,
      blockCountRequested,
      bKeepInputTensor
    );

    Object.assign( this.in, this.out ); // So that all parameters are by specified (none is by evolution).

    let weightElementOffsetBegin = 0;
    return this.set_byParamsNumberArrayObject_ParamsOut( weightElementOffsetBegin );
  }

  /**
   * Fills the following proterties:
   *   - this.in_weights
   *   - this.out.inferencedParams
   *
   * @param {object} this.in.paramsNumberArrayObject
   *   Pass in an object. The result will be put into this object. It is a map from a string name (e.g. parameter name) to a number array.
   * The name should be one of Base.paramsNameOrderArray[] elements.
   *
   * @param {NeuralNet.ParamsBase} this.out
   *   An object which will be the final result of NeuralNet.Params.
   *
   * @param {number} weightElementOffsetBegin
   *   Offset how many elements (4 bytes per element) at the beginning of the result inputWeightArray.
   * The this.in.byteOffsetBegin will be ( 4 * weightElementOffsetBegin ).
   *
   * @return {NeuralNet_TestParams_Base}
   *   Return this object self.
   */
  set_byParamsNumberArrayObject_ParamsOut( weightElementOffsetBegin = 0 ) {
    let neuralNetParams = this.out;

    this.generate_out_inferencedParams();

    this.in.paramsNumberArrayObject.length = 0;

    // 1. Embedding
    {
      this.embedding?.disposeResources_and_recycleToPool();
      this.embedding = Embedding_TestParams.Pool.get_or_create_by( this.id );

      this.embedding.set_byParamsScattered(
        neuralNetParams.input_height,
        neuralNetParams.input_width,
        neuralNetParams.input_channelCount,
        neuralNetParams.vocabularyChannelCount,
        neuralNetParams.vocabularyCountPerInputChannel,
        neuralNetParams.inferencedParams.bEmbedVocabularyId,
        neuralNetParams.bKeepInputTensor
      );

      this.in.paramsNumberArrayObject.push( this.embedding.in_weights.weightArray );
    }

    // 2. Stages
    let stageParamsArray = neuralNetParams.inferencedParams.stageParamsArray;

    this.stageArray.clear();
    this.stageArray.length = stageParamsArray.length;

    for ( let i = 0; i < stageParamsArray.length; ++i ) { // Stage0, 1, 2, 3, ..., StageLast.
      let stageParams = stageParamsArray[ i ];

      let stageTestParams = Stage_TestParams.Base.Pool.get_or_create_by( this.id );
      blockTestParams.set_byParamsScattered(
        stageParams.input_height, stageParams.input_width, stageParams.input_channelCount,
        stageParams.nConvStageTypeId,
        stageParams.blockCountRequested,
        stageParams.bPointwise1,
        stageParams.depthwiseFilterHeight, stageParams.depthwiseFilterWidth,
        stageParams.bPointwise2ActivatedAtStageEnd,
        stageParams.nSqueezeExcitationChannelCountDivisor,
        stageParams.nActivationId,
        stageParams.bKeepInputTensor
      );

      this.stageArray[ i ] = stageTestParams;
      this.in.paramsNumberArrayObject.push( stageTestParams.in_weights.weightArray ); // Place every stage's parameters in sequence.
    }

    // Pack all parameters, look-up tables weights into a (pre-allocated and re-used) NumberArray.
    this.in_weights.set_byConcat(
      NeuralNet_TestParams_Base.paramsNameOrderArray_Basic, this.in.paramsNumberArrayObject, weightElementOffsetBegin );

    return this;
  }

  /** Fill this.out.inferencedParams according to this.out */
  generate_out_inferencedParams() {
    this.out.inferencedParams_create();
  }

  /**
   * @override
   */
  onYield_isLegal() {
    return true;
  }

  /**
   * @override
   */
  onYield_before() {

    // For testing not start at the offset 0.
    let weightElementOffsetBegin = RandTools.getRandomIntInclusive( 0, 3 ); // Skip a random un-used element count.

    this.set_byParamsNumberArrayObject_ParamsOut( weightElementOffsetBegin );
  }

  /**
   * @override
   */
  onYield_after() {
    this.stageArray.clear();

    this.embedding?.disposeResources_and_recycleToPool();
    this.embedding = null;
  }

  /**
   * Responsible for generating testing paramters combinations.
   *
   * @yield {Base}
   *   Yield this object itself. The returned object (it is this object itself) should not be modified because it will be re-used.
   */
  * ParamsGenerator() {

    // Restrict some parameter's large kinds. Otherwise, too many combination will be generated.
    let valueOutMinMax = this.valueOutMinMax = {
      input_height: [ 1, 5 ],
      input_width: [ 1, 5 ],

      input_channelCount: [
        NeuralNet.Params.input_channelCount.valueDesc.range.min,
        4
      ],

      vocabularyChannelCount: [
        NeuralNet.Params.vocabularyChannelCount.valueDesc.range.min,
        4
      ],

      vocabularyCountPerInputChannel: [
        256, //NeuralNet.Params.vocabularyCountPerInputChannel.valueDesc.range.min,
        257
      ],

      nConvStageTypeId: [
        NeuralNet.Params.nConvStageTypeId.valueDesc.range.min,
        NeuralNet.Params.nConvStageTypeId.valueDesc.range.max
      ],

      stageCountRequested: [
        NeuralNet.Params.stageCountRequested.valueDesc.range.min,
        10 //NeuralNet.Params.stageCountRequested.valueDesc.range.max
      ],

      blockCountRequested: [
        NeuralNet.Params.blockCountRequested.valueDesc.range.min,
        4 //NeuralNet.Params.blockCountRequested.valueDesc.range.max
      ],

      // bKeepInputTensor: undefined,
      bKeepInputTensor: [
        NeuralNet.Params.bKeepInputTensor.valueDesc.range.min,
        NeuralNet.Params.bKeepInputTensor.valueDesc.range.max
      ],
    };

    // All the parameters to be tried.
    //
    // Note: The order of these element could be adjusted to change testing order. The last element will be tested (changed) first.
    let paramDescConfigArray = [
      new TestParams.ParamDescConfig( NeuralNet.Params.input_height,                   valueOutMinMax.input_height ),
      new TestParams.ParamDescConfig( NeuralNet.Params.input_width,                    valueOutMinMax.input_width ),
      new TestParams.ParamDescConfig( NeuralNet.Params.input_channelCount,             valueOutMinMax.input_channelCount ),

      new TestParams.ParamDescConfig( NeuralNet.Params.vocabularyChannelCount,         valueOutMinMax.vocabularyChannelCount ),
      new TestParams.ParamDescConfig( NeuralNet.Params.vocabularyCountPerInputChannel, valueOutMinMax.vocabularyCountPerInputChannel ),

      new TestParams.ParamDescConfig( NeuralNet.Params.nConvStageTypeId,               valueOutMinMax.nConvStageTypeId ),
      new TestParams.ParamDescConfig( NeuralNet.Params.stageCountRequested,            valueOutMinMax.stageCountRequested ),
      new TestParams.ParamDescConfig( NeuralNet.Params.blockCountRequested,            valueOutMinMax.blockCountRequested ),

      new TestParams.ParamDescConfig( NeuralNet.Params.bKeepInputTensor,               valueOutMinMax.bKeepInputTensor ),
    ];

    yield *NeuralNet_TestParams_Base.ParamsGenerator.call( this, paramDescConfigArray );
  }

  /**
   * Fill an object's property as a number array.
   *
   * Similar to Base.ensure_object_property_numberArray_length_filled(). But the
   * property will be a shared number array. Its value may be shared with other caller.
   *
   * This may have better performance because of number array re-using (instead
   * of re-generating).
   *
   *
   * @param {object} io_object            The object to be checked and modified.
   * @param {string|numner} propertyName  The property io_object[ propertyName ] will be ensured as a number array.
   * @param {number} elementCount         The property io_object[ propertyName ].length will be ensured as elementCount.
   *
   */
  fill_object_property_numberArray( io_object, propertyName, elementCount ) {
    super.ensure_object_property_numberArray_length_existed( io_object, propertyName,
      elementCount,
      TestParams.Base.weightsRandomOffset.min, TestParams.Base.weightsRandomOffset.max,
      TestParams.Base.weightsDivisorForRemainder
    );
  }

}

/**
 * The order when generate inputWeightArray[].
 *
 * This order could not be changed arbitrarily. It must be the same as the parameter extracting order of NeuralNet.initer().
 */
NeuralNet_TestParams_Base.paramsNameOrderArray_Basic = [
  NeuralNet.Params.input_height.paramName,
  NeuralNet.Params.input_width.paramName,
  NeuralNet.Params.input_channelCount.paramName,
  NeuralNet.Params.vocabularyChannelCount.paramName,
  NeuralNet.Params.vocabularyCountPerInputChannel.paramName,
  NeuralNet.Params.nConvStageTypeId.paramName,
  NeuralNet.Params.stageCountRequested.paramName,
  NeuralNet.Params.blockCountRequested.paramName,
  NeuralNet.Params.bKeepInputTensor.paramName,
];
<commit_msg>Embedding_TestParams.Base<commit_after>export { NeuralNet_TestParams_Base as Base };

import * as Pool from "../../util/Pool.js";
import * as Recyclable from "../../util/Recyclable.js";
import * as RandTools from "../../util/RandTools.js";
import * as ValueDesc from "../../Unpacker/ValueDesc.js";
import * as NeuralNet from "../../Conv/NeuralNet.js";
import * as TestParams from "./TestParams.js";
import * as Embedding_TestParams from "./Embedding_TestParams.js";
import * as Stage_TestParams from "./Stage_TestParams.js";

/**
 *
 * This is an object { id, in, out } which has one number and two sub-objects.
 *
 *
 * @see TestParams.Base
 */
class NeuralNet_TestParams_Base extends TestParams.Base {

  /**
   * Used as default NeuralNet_TestParams.Base provider for conforming to Recyclable interface.
   */
  static Pool = new Pool.Root( "NeuralNet_TestParams.Base.Pool",
    NeuralNet_TestParams_Base, NeuralNet_TestParams_Base.setAsConstructor );

  /**
   */
  constructor( id ) {
    super( id );
    NeuralNet_TestParams_Base.setAsConstructor_self.call( this );
  }

  /** @override */
  static setAsConstructor( id ) {
    super.setAsConstructor( id );
    NeuralNet_TestParams_Base.setAsConstructor_self.call( this );
    return this;
  }

  /** @override */
  static setAsConstructor_self() {
    this.stageArray = Recyclable.OwnerArray.Pool.get_or_create_by();
    this.out = NeuralNet.ParamsBase.Pool.get_or_create_by();
  }

  /** @override */
  disposeResources() {
    this.out?.disposeResources_and_recycleToPool();
    this.out = null;

    this.stageArray?.disposeResources_and_recycleToPool();
    this.stageArray = null;

    this.embedding?.disposeResources_and_recycleToPool();
    this.embedding = null;

    super.disposeResources();
  }

  /**
   * Use scattered parameters to fills the following proterties:
   *   - this.in
   *   - this.in_weights
   *   - this.out
   *
   * @return {Base}
   *   Return this object self.
   */
  set_byParamsScattered(
    input_height, input_width, input_channelCount,
    vocabularyChannelCount, vocabularyCountPerInputChannel,
    nConvStageTypeId, stageCountRequested,
    blockCountRequested,
    bKeepInputTensor
  ) {

    if ( this.out ) {
      this.out.disposeResources_and_recycleToPool();
      this.out = null;
    }

    this.out = NeuralNet.ParamsBase.Pool.get_or_create_by(
      input_height, input_width, input_channelCount,
      vocabularyChannelCount, vocabularyCountPerInputChannel,
      nConvStageTypeId, stageCountRequested,
      blockCountRequested,
      bKeepInputTensor
    );

    Object.assign( this.in, this.out ); // So that all parameters are by specified (none is by evolution).

    let weightElementOffsetBegin = 0;
    return this.set_byParamsNumberArrayObject_ParamsOut( weightElementOffsetBegin );
  }

  /**
   * Fills the following proterties:
   *   - this.in_weights
   *   - this.out.inferencedParams
   *
   * @param {object} this.in.paramsNumberArrayObject
   *   Pass in an object. The result will be put into this object. It is a map from a string name (e.g. parameter name) to a number array.
   * The name should be one of Base.paramsNameOrderArray[] elements.
   *
   * @param {NeuralNet.ParamsBase} this.out
   *   An object which will be the final result of NeuralNet.Params.
   *
   * @param {number} weightElementOffsetBegin
   *   Offset how many elements (4 bytes per element) at the beginning of the result inputWeightArray.
   * The this.in.byteOffsetBegin will be ( 4 * weightElementOffsetBegin ).
   *
   * @return {NeuralNet_TestParams_Base}
   *   Return this object self.
   */
  set_byParamsNumberArrayObject_ParamsOut( weightElementOffsetBegin = 0 ) {
    let neuralNetParams = this.out;

    this.generate_out_inferencedParams();

    this.in.paramsNumberArrayObject.length = 0;

    // 1. Embedding
    {
      this.embedding?.disposeResources_and_recycleToPool();
      this.embedding = Embedding_TestParams.Base.Pool.get_or_create_by( this.id );

      this.embedding.set_byParamsScattered(
        neuralNetParams.input_height,
        neuralNetParams.input_width,
        neuralNetParams.input_channelCount,
        neuralNetParams.vocabularyChannelCount,
        neuralNetParams.vocabularyCountPerInputChannel,
        neuralNetParams.inferencedParams.bEmbedVocabularyId,
        neuralNetParams.bKeepInputTensor
      );

      this.in.paramsNumberArrayObject.push( this.embedding.in_weights.weightArray );
    }

    // 2. Stages
    let stageParamsArray = neuralNetParams.inferencedParams.stageParamsArray;

    this.stageArray.clear();
    this.stageArray.length = stageParamsArray.length;

    for ( let i = 0; i < stageParamsArray.length; ++i ) { // Stage0, 1, 2, 3, ..., StageLast.
      let stageParams = stageParamsArray[ i ];

      let stageTestParams = Stage_TestParams.Base.Pool.get_or_create_by( this.id );
      blockTestParams.set_byParamsScattered(
        stageParams.input_height, stageParams.input_width, stageParams.input_channelCount,
        stageParams.nConvStageTypeId,
        stageParams.blockCountRequested,
        stageParams.bPointwise1,
        stageParams.depthwiseFilterHeight, stageParams.depthwiseFilterWidth,
        stageParams.bPointwise2ActivatedAtStageEnd,
        stageParams.nSqueezeExcitationChannelCountDivisor,
        stageParams.nActivationId,
        stageParams.bKeepInputTensor
      );

      this.stageArray[ i ] = stageTestParams;
      this.in.paramsNumberArrayObject.push( stageTestParams.in_weights.weightArray ); // Place every stage's parameters in sequence.
    }

    // Pack all parameters, look-up tables weights into a (pre-allocated and re-used) NumberArray.
    this.in_weights.set_byConcat(
      NeuralNet_TestParams_Base.paramsNameOrderArray_Basic, this.in.paramsNumberArrayObject, weightElementOffsetBegin );

    return this;
  }

  /** Fill this.out.inferencedParams according to this.out */
  generate_out_inferencedParams() {
    this.out.inferencedParams_create();
  }

  /**
   * @override
   */
  onYield_isLegal() {
    return true;
  }

  /**
   * @override
   */
  onYield_before() {

    // For testing not start at the offset 0.
    let weightElementOffsetBegin = RandTools.getRandomIntInclusive( 0, 3 ); // Skip a random un-used element count.

    this.set_byParamsNumberArrayObject_ParamsOut( weightElementOffsetBegin );
  }

  /**
   * @override
   */
  onYield_after() {
    this.stageArray.clear();

    this.embedding?.disposeResources_and_recycleToPool();
    this.embedding = null;
  }

  /**
   * Responsible for generating testing paramters combinations.
   *
   * @yield {Base}
   *   Yield this object itself. The returned object (it is this object itself) should not be modified because it will be re-used.
   */
  * ParamsGenerator() {

    // Restrict some parameter's large kinds. Otherwise, too many combination will be generated.
    let valueOutMinMax = this.valueOutMinMax = {
      input_height: [ 1, 5 ],
      input_width: [ 1, 5 ],

      input_channelCount: [
        NeuralNet.Params.input_channelCount.valueDesc.range.min,
        4
      ],

      vocabularyChannelCount: [
        NeuralNet.Params.vocabularyChannelCount.valueDesc.range.min,
        4
      ],

      vocabularyCountPerInputChannel: [
        256, //NeuralNet.Params.vocabularyCountPerInputChannel.valueDesc.range.min,
        257
      ],

      nConvStageTypeId: [
        NeuralNet.Params.nConvStageTypeId.valueDesc.range.min,
        NeuralNet.Params.nConvStageTypeId.valueDesc.range.max
      ],

      stageCountRequested: [
        NeuralNet.Params.stageCountRequested.valueDesc.range.min,
        10 //NeuralNet.Params.stageCountRequested.valueDesc.range.max
      ],

      blockCountRequested: [
        NeuralNet.Params.blockCountRequested.valueDesc.range.min,
        4 //NeuralNet.Params.blockCountRequested.valueDesc.range.max
      ],

      // bKeepInputTensor: undefined,
      bKeepInputTensor: [
        NeuralNet.Params.bKeepInputTensor.valueDesc.range.min,
        NeuralNet.Params.bKeepInputTensor.valueDesc.range.max
      ],
    };

    // All the parameters to be tried.
    //
    // Note: The order of these element could be adjusted to change testing order. The last element will be tested (changed) first.
    let paramDescConfigArray = [
      new TestParams.ParamDescConfig( NeuralNet.Params.input_height,                   valueOutMinMax.input_height ),
      new TestParams.ParamDescConfig( NeuralNet.Params.input_width,                    valueOutMinMax.input_width ),
      new TestParams.ParamDescConfig( NeuralNet.Params.input_channelCount,             valueOutMinMax.input_channelCount ),

      new TestParams.ParamDescConfig( NeuralNet.Params.vocabularyChannelCount,         valueOutMinMax.vocabularyChannelCount ),
      new TestParams.ParamDescConfig( NeuralNet.Params.vocabularyCountPerInputChannel, valueOutMinMax.vocabularyCountPerInputChannel ),

      new TestParams.ParamDescConfig( NeuralNet.Params.nConvStageTypeId,               valueOutMinMax.nConvStageTypeId ),
      new TestParams.ParamDescConfig( NeuralNet.Params.stageCountRequested,            valueOutMinMax.stageCountRequested ),
      new TestParams.ParamDescConfig( NeuralNet.Params.blockCountRequested,            valueOutMinMax.blockCountRequested ),

      new TestParams.ParamDescConfig( NeuralNet.Params.bKeepInputTensor,               valueOutMinMax.bKeepInputTensor ),
    ];

    yield *NeuralNet_TestParams_Base.ParamsGenerator.call( this, paramDescConfigArray );
  }

  /**
   * Fill an object's property as a number array.
   *
   * Similar to Base.ensure_object_property_numberArray_length_filled(). But the
   * property will be a shared number array. Its value may be shared with other caller.
   *
   * This may have better performance because of number array re-using (instead
   * of re-generating).
   *
   *
   * @param {object} io_object            The object to be checked and modified.
   * @param {string|numner} propertyName  The property io_object[ propertyName ] will be ensured as a number array.
   * @param {number} elementCount         The property io_object[ propertyName ].length will be ensured as elementCount.
   *
   */
  fill_object_property_numberArray( io_object, propertyName, elementCount ) {
    super.ensure_object_property_numberArray_length_existed( io_object, propertyName,
      elementCount,
      TestParams.Base.weightsRandomOffset.min, TestParams.Base.weightsRandomOffset.max,
      TestParams.Base.weightsDivisorForRemainder
    );
  }

}

/**
 * The order when generate inputWeightArray[].
 *
 * This order could not be changed arbitrarily. It must be the same as the parameter extracting order of NeuralNet.initer().
 */
NeuralNet_TestParams_Base.paramsNameOrderArray_Basic = [
  NeuralNet.Params.input_height.paramName,
  NeuralNet.Params.input_width.paramName,
  NeuralNet.Params.input_channelCount.paramName,
  NeuralNet.Params.vocabularyChannelCount.paramName,
  NeuralNet.Params.vocabularyCountPerInputChannel.paramName,
  NeuralNet.Params.nConvStageTypeId.paramName,
  NeuralNet.Params.stageCountRequested.paramName,
  NeuralNet.Params.blockCountRequested.paramName,
  NeuralNet.Params.bKeepInputTensor.paramName,
];
<|endoftext|>


<commit_before>openerp.base_kanban = function (openerp) {
QWeb.add_template('/base_kanban/static/src/xml/base_kanban.xml');
openerp.base.views.add('kanban', 'openerp.base_kanban.KanbanView');
openerp.base_kanban.KanbanView = openerp.base.View.extend({

    init: function(parent, element_id, dataset, view_id) {
        this._super(parent, element_id);
        this.view_manager = parent;
        this.dataset = dataset;
        this.domain = dataset.domain;
        this.context = dataset.context;
        this.model = this.dataset.model;
        this.view_id = view_id;
        this.group_by_field = false;
        this.source_index = {};
        this.all_display_data = false;
        this.groups = [];
    },

    start: function() {
        this.rpc("/base_kanban/kanbanview/load",
        {"model": this.model, "view_id": this.view_id}, this.on_loaded);
    },

    on_loaded: function(data) {
        var self = this;
        this.template_xml = '';
        this.columns = data.all_fields;
        _.each(data.fields_view.arch.children, function(child) {
            if (child.tag == "template"){
                self.template_xml = openerp.base.json_node_to_xml(child, true)
            }
        });
        if(this.template_xml){
            self.dataset.read_slice({
                context: self.dataset.get_context(),
                domain: self.dataset.get_domain()}, function (records) {
                self.all_display_data = [{'records': records, 'value':false, 'header': false, 'ids': self.dataset.ids}];
                self.on_show_data(self.all_display_data);
            });
        }
    },

    on_show_data: function(datas) {
        var self = this;
        var new_qweb = new QWeb2.Engine();
        self.$element.html(QWeb.render("KanbanBiew", {"datas" :datas}));

        this.on_reload_kanban();
        this.$element.find(".oe_column").sortable({
            connectWith: ".oe_column",
            start: function(event, ui) {
                self.source_index['index'] = ui.item.index();
                self.source_index['column'] = ui.item.parent().attr('id');
            },
            stop: self.on_recieve_record,
        });
        this.$element.find(".record").addClass("ui-widget ui-widget-content ui-helper-clearfix ui-corner-all")
            .find(".record-header")
                .addClass("ui-widget-header ui-corner-all")
                .prepend( "")
                .end()
            .find( ".record-content" );

        this.$element.find(".record-header .ui-icon").click(function() {
            $(this).toggleClass("ui-icon-minusthick").toggleClass("ui-icon-plusthick");
            $(this).parents(".record:first").find(".record-content").toggle();
        });
        this.$element.find('.record .ui-icon-closethick').click(this.on_close_action);
        this.$element.find(".oe_column").disableSelection();
        this.$element.find('button.oe_kanban_button_new').click(this.do_add_record);
    },

    on_button_click: function(button_attrs, record_id){
        var self = this;
        if(this.groups.length){
            _.each(this.groups, function (group) {
                group.list([],
                    function (groups) {},
                    function (dataset) {
                        dataset.read_slice({}, function(records) {
                            var index = parseInt(_.indexOf(dataset.ids, record_id));
                            if(index >= 0) {
                                self.on_confirm_click(dataset, button_attrs, index, record_id);
                            }
                        });
                    }
                );
            });
        } else{
            var index = parseInt(_.indexOf(self.dataset.ids, record_id));
            if(index >= 0) {
                _.extend(self.dataset, {domain: self.domain, context: self.context})
                self.on_confirm_click(self.dataset, button_attrs, index, record_id)
            }
        }
    },

    on_confirm_click: function(dataset, button_attrs, index, record_id){
        if(button_attrs.type == 'edit') {
            this.do_edit_record(dataset, index);
        } else{
            this.on_execute_button_click(dataset, button_attrs, record_id)
        }
    },

    do_add_record: function () {
        this.do_edit_record(this.dataset, null);
    },

    do_edit_record: function (dataset, index) {
        var self = this;
        _.extend(this.dataset, {
            domain: dataset.domain,
            context: dataset.get_context()
        }).read_slice({}, function () {
            self.dataset.index = index;
            self.do_switch_view('form');
        });
    },

    do_delete: function (id) {
        var self = this;
        return $.when(this.dataset.unlink([id])).then(function () {
            self.drop_records(id);
        });
    },

    drop_records: function (id) {
        var self = this;
        _.each(self.all_display_data, function(data, index) {
            _.each(data.records, function(record, index_row) {
                if(parseInt(record.id) == id) {
                    self.all_display_data[index]['records'].splice(index_row, 1)
                    self.all_display_data[index]['ids'].splice(index_row, 1)
                    return false;
                }
            });
        });
        self.$element.find("#main_" + id).remove()
    },

    on_execute_button_click: function(dataset, button_attrs, record_id) {
        var self = this;
        this.execute_action(
            button_attrs, dataset,
            record_id, function () {
                var count = 1;
                _.each(self.all_display_data, function(data, index) {
                    self.dataset.read_ids( data.ids, [], function(records){
                        self.all_display_data[index].records = records;
                        if(self.all_display_data.length == count) {
                            self.on_reload_kanban();
                        }
                        count++;
                    });
                });
            }
        );
    },

    on_close_action: function(e) {
        var record_id = $(e.currentTarget).parents('.record:first').attr("id")
        if(record_id) {
            record_id = parseInt(record_id.split("_")[1])
            if(record_id) {
                this.do_delete(record_id);
            }
        }
    },

    on_recieve_record: function(event, ui) {
        var self = this;
        var from = ui.item.index();
        var search_action = false;
        var to = ui.item.prev().index() || 0;
        if(!ui.item.attr("id")){
            return false;
        }
        if(self.columns.sequence && (self.source_index.index >= 0 && self.source_index.index != from) ||
                (self.source_index.column && self.source_index.column != ui.item.parent().attr('id'))) {
            var child_record = ui.item.parent().children();
            var data, sequence = 1, index = to;
            child_record.splice(0, to);
            if(to >= 0 && child_record) {
                var record_id = parseInt($(child_record).attr("id").split("_")[1]);
                if(record_id) {
                    var flag = false;
                    _.each(self.all_display_data, function(data, index) {
                        _.each(data.records, function(record, index_row) {
	                        if(record_id == record.id && record.sequence) {
	                            sequence = record.sequence;
	                            flag = true;
	                            return false;
	                        }
                        });
                        if(flag) {return false;}
                    });
                }
            }
            _.each(child_record, function (child) {
                var child_id = parseInt($(child).attr("id").split("_")[1]);
                if(child_id) {
                    var flag = false;
                    _.each(self.all_display_data, function(data, index) {
                        _.each(data.records, function(record, index_row) {
                            if(parseInt(record.id) == child_id) {
                                self.all_display_data[index]['records'][index_row]['sequence'] = sequence;
                                flag = true;
                                return false;
                            }
                        });
                        if(flag) {return false;}
                    });
                    self.dataset.write(child_id, {sequence: sequence});
                    sequence++;
                    search_action = true;
                }
            });
        }
        if(self.group_by_field && self.source_index.column && self.source_index.column != ui.item.parent().attr('id')) {
            var value = ui.item.closest("td").attr("id")
            if(value) {
                var data_val = {};
                var wirte_id = parseInt(ui.item.attr("id").split("_")[1]);
                value = value.split("_")[1];
                if(value == 'false') {
                    value = false;
                }
                var update_record = false;
                _.each(self.all_display_data, function(data, index) {
                    _.each(data.records, function(record, index_row) {
                        if(parseInt(record.id) == wirte_id) {
                            self.all_display_data[index]['records'][index_row][self.group_by_field] = value;
                            update_record = self.all_display_data[index]['records'].splice(index_row,1)
                            return false;
                        }
                    });
                    if(update_record) {return false;}
                });
                _.each(self.all_display_data, function(data, index) {
                    if (data.value == value || (data.value == 'false' && value == false)) {
                        self.all_display_data[index]['records'].push(update_record[0]);
                    }
                });
                data_val[self.group_by_field] = value;
                self.dataset.write(wirte_id, data_val);
                search_action = true;
            }
        }
        if(search_action) {
            self.on_reload_kanban();
        }
        this.source_index = {};
    },

    on_reload_kanban: function(){
        var self = this;
        var new_qweb = new QWeb2.Engine();
        new_qweb.add_template('<templates><t t-name="custom_template">' + this.template_xml + '</t></templates>')
        _.each(self.all_display_data, function(data, index) {
            if(data.records.length > 0){
                _.each(data.records, function(record) {
                    self.$element.find("#data_" + record.id).children().remove()
                    self.$element.find("#data_" + record.id).append(new_qweb.render('custom_template', record));
                });
            }
            else{
                self.$element.find("#column_" + data.value).remove();
                self.all_display_data.splice(index, 1);
            }
        });
        this.$element.find( ".oe_table_column " ).css("width", 99 / self.all_display_data.length +"%");
        this.$element.find('button').click(function(){
            var record_id = $(this).closest(".record").attr("id");
            if(record_id) {
                record_id = parseInt(record_id.split("_")[1])
                if(record_id) {
                    if($(this).data("type") == "delete") {
                        self.do_delete(record_id);
                    }
                    else{
                        var button_attrs = $(this).data()
                        self.on_button_click(button_attrs, record_id);
                    }
                }
            }
        });
    },

    do_search: function (domains, contexts, group_by) {
        var self = this;
        this.rpc('/base/session/eval_domain_and_context', {
            domains: domains,
            contexts: contexts,
            group_by_seq: group_by
        }, function (results) {
            self.group_by_field = false;
            self.datagroup = new openerp.base.DataGroup(self, self.model, results.domain, results.context, results.group_by || []);
            self.dataset.context = results.context;
            self.dataset.domain = results.domain;
            self.datagroup.list([],
                function (groups) {
                    self.groups = groups;
                    if (group_by.length >= 1) {
                        self.group_by_field = group_by[0].group_by;
                        self.do_render_group(groups);
                    }
                },
                function (dataset) {
                    self.domain = dataset.domain;
                    self.context = dataset.context;
                    self.groups = [];
                    self.dataset.read_slice({}, function(records) {
                        self.all_display_data = [{'records': records, 'value':false, 'header' : false, 'ids': self.dataset.ids}];
                        self.$element.find("#kanbanview").remove();
                        self.on_show_data(self.all_display_data);
                    });
                }
            );

        });
    },

    do_render_group : function(datagroups){
        this.all_display_data = [];
        var self = this;
        _.each(datagroups, function (group) {
            self.dataset.context = group.context;
            self.dataset.domain = group.domain;
            var group_name = group.value;
            var group_value = group.value;
            if(!group.value) {
                group_name = "Undefined";
                group_value = 'false';
            }
            else if(group.value instanceof Array) {
                group_name = group.value[1]
                group_value = group.value[0]
            }
            self.dataset.read_slice({}, function(records) {
                self.all_display_data.push({"value" : group_value, "records": records, 'header':group_name, 'ids': self.dataset.ids});
                if (datagroups.length == self.all_display_data.length) {
                    self.$element.find("#kanbanview").remove();
                    self.on_show_data(self.all_display_data);
                }
            });
        });
    },

    do_show: function () {
        this.$element.show();
    },

    do_hide: function () {
        this.$element.hide();
    },

});
};

// vim:et fdc=0 fdl=0 foldnestmax=3 fdm=syntax:
<commit_msg>[IMP] Improve code.<commit_after>openerp.base_kanban = function (openerp) {
QWeb.add_template('/base_kanban/static/src/xml/base_kanban.xml');
openerp.base.views.add('kanban', 'openerp.base_kanban.KanbanView');
openerp.base_kanban.KanbanView = openerp.base.View.extend({

    init: function (parent, element_id, dataset, view_id) {
        this._super(parent, element_id);
        this.view_manager = parent;
        this.dataset = dataset;
        this.domain = dataset.domain;
        this.context = dataset.context;
        this.model = this.dataset.model;
        this.view_id = view_id;
        this.group_by_field = false;
        this.source_index = {};
        this.all_display_data = false;
        this.groups = [];
    },
    start: function () {
        this.rpc("/base_kanban/kanbanview/load",
        {"model": this.model, "view_id": this.view_id}, this.on_loaded);
    },
    on_loaded: function (data) {
        var self = this;
        this.template_xml = '';
        this.columns = data.all_fields;
        _.each(data.fields_view.arch.children, function(child) {
            if (child.tag == "template"){
                self.template_xml = openerp.base.json_node_to_xml(child, true);
            }
        });
        if (this.template_xml) {
            self.dataset.read_slice({
                context: self.dataset.get_context(),
                domain: self.dataset.get_domain()}, function (records) {
                self.all_display_data = [{'records': records, 'value':false, 'header': false, 'ids': self.dataset.ids}];
                self.on_show_data(self.all_display_data);
            });
        }
    },
    on_show_data: function (datas) {
        var self = this;
        var new_qweb = new QWeb2.Engine();
        self.$element.html(QWeb.render("KanbanBiew", {"datas" :datas}));

        this.on_reload_kanban();
        this.$element.find(".oe_column").sortable({
            connectWith: ".oe_column",
            start: function(event, ui) {
                self.source_index['index'] = ui.item.index();
                self.source_index['column'] = ui.item.parent().attr('id');
            },
            stop: self.on_recieve_record,
        });
        this.$element.find(".record").addClass("ui-widget ui-widget-content ui-helper-clearfix ui-corner-all")
            .find(".record-header")
                .addClass("ui-widget-header ui-corner-all")
                .prepend( "")
                .end()
            .find( ".record-content" );

        this.$element.find(".record-header .ui-icon").click(function() {
            $(this).toggleClass("ui-icon-minusthick").toggleClass("ui-icon-plusthick");
            $(this).parents(".record:first").find(".record-content").toggle();
        });
        this.$element.find('.record .ui-icon-closethick').click(this.on_close_action);
        this.$element.find(".oe_column").disableSelection();
        this.$element.find('button.oe_kanban_button_new').click(this.do_add_record);
    },
    on_button_click: function (button_attrs, record_id) {
        var self = this;
        if (this.groups.length) {
            _.each(this.groups, function (group) {
                group.list([],
                    function (groups) {},
                    function (dataset) {
                        dataset.read_slice({}, function(records) {
                            var index = parseInt(_.indexOf(dataset.ids, record_id));
                            if(index >= 0) {
                                self.on_confirm_click(dataset, button_attrs, index, record_id);
                            }
                        });
                    }
                );
            });
        } else {
            var index = parseInt(_.indexOf(self.dataset.ids, record_id));
            if (index >= 0) {
                _.extend(self.dataset, {domain: self.domain, context: self.context});
                self.on_confirm_click(self.dataset, button_attrs, index, record_id);
            }
        }
    },
    on_confirm_click: function (dataset, button_attrs, index, record_id) {
        if (button_attrs.type == 'edit') {
            this.do_edit_record(dataset, index);
        } else {
            this.on_execute_button_click(dataset, button_attrs, record_id);
        }
    },
    do_add_record: function () {
        this.do_edit_record(this.dataset, null);
    },
    do_edit_record: function (dataset, index) {
        var self = this;
        _.extend(this.dataset, {
            domain: dataset.domain,
            context: dataset.get_context()
        }).read_slice({}, function () {
            self.dataset.index = index;
            self.do_switch_view('form');
        });
    },
    do_delete: function (id) {
        var self = this;
        return $.when(this.dataset.unlink([id])).then(function () {
            self.drop_records(id);
        });
    },
    drop_records: function (id) {
        var self = this;
        _.each(self.all_display_data, function(data, index) {
            _.each(data.records, function(record, index_row) {
                if (parseInt(record.id) == id) {
                    self.all_display_data[index]['records'].splice(index_row, 1);
                    self.all_display_data[index]['ids'].splice(index_row, 1);
                    return false;
                }
            });
        });
        self.$element.find("#main_" + id).remove();
    },
    on_execute_button_click: function (dataset, button_attrs, record_id) {
        var self = this;
        this.execute_action(
            button_attrs, dataset,
            record_id, function () {
                var count = 1;
                _.each(self.all_display_data, function(data, index) {
                    self.dataset.read_ids( data.ids, [], function(records){
                        self.all_display_data[index].records = records;
                        if(self.all_display_data.length == count) {
                            self.on_reload_kanban();
                        }
                        count++;
                    });
                });
            }
        );
    },
    on_close_action: function (e) {
        var record_id = $(e.currentTarget).parents('.record:first').attr("id");
        if (record_id) {
            record_id = parseInt(record_id.split("_")[1]);
            if (record_id) {
                this.do_delete(record_id);
            }
        }
    },
    on_recieve_record: function (event, ui) {
        var self = this;
        var from = ui.item.index();
        var search_action = false;
        var to = ui.item.prev().index() || 0;
        if (!ui.item.attr("id")) {
            return false;
        }
        if (self.columns.sequence && (self.source_index.index >= 0 && self.source_index.index != from) ||
                (self.source_index.column && self.source_index.column != ui.item.parent().attr('id'))) {
            var child_record = ui.item.parent().children();
            var data, sequence = 1, index = to;
            child_record.splice(0, to);
            var flag = false;
            if (to >= 0 && child_record) {
                var record_id = parseInt($(child_record).attr("id").split("_")[1]);
                if (record_id) {
                    _.each(self.all_display_data, function(data, index) {
                        _.each(data.records, function(record, index_row) {
	                        if(record_id == record.id && record.sequence) {
	                            sequence = record.sequence;
	                            flag = true;
	                            return false;
	                        }
                        });
                        if(flag) {return false;}
                    });
                }
            }
            _.each(child_record, function (child) {
                var child_id = parseInt($(child).attr("id").split("_")[1]);
                if (child_id) {
                    flag = false;
                    _.each(self.all_display_data, function(data, index) {
                        _.each(data.records, function(record, index_row) {
                            if(parseInt(record.id) == child_id) {
                                self.all_display_data[index]['records'][index_row]['sequence'] = sequence;
                                flag = true;
                                return false;
                            }
                        });
                        if (flag) {return false;}
                    });
                    self.dataset.write(child_id, {sequence: sequence});
                    sequence++;
                    search_action = true;
                }
            });
        }
        if (self.group_by_field && self.source_index.column && self.source_index.column != ui.item.parent().attr('id')) {
            var value = ui.item.closest("td").attr("id");
            if (value) {
                var data_val = {};
                var wirte_id = parseInt(ui.item.attr("id").split("_")[1]);
                value = value.split("_")[1];
                if (value == 'false') {
                    value = false;
                }
                var update_record = false;
                _.each(self.all_display_data, function(data, index) {
                    _.each(data.records, function(record, index_row) {
                        if(parseInt(record.id) == wirte_id) {
                            self.all_display_data[index]['records'][index_row][self.group_by_field] = value;
                            update_record = self.all_display_data[index]['records'].splice(index_row,1)
                            return false;
                        }
                    });
                    if (update_record) {return false;}
                });
                _.each(self.all_display_data, function(data, index) {
                    if (data.value == value || (data.value == 'false' && value == false)) {
                        self.all_display_data[index]['records'].push(update_record[0]);
                    }
                });
                data_val[self.group_by_field] = value;
                self.dataset.write(wirte_id, data_val);
                search_action = true;
            }
        }
        if (search_action) {
            self.on_reload_kanban();
        }
        this.source_index = {};
    },
    on_reload_kanban: function (){
        var self = this;
        var new_qweb = new QWeb2.Engine();
        new_qweb.add_template('<templates><t t-name="custom_template">' + this.template_xml + '</t></templates>');
        _.each(self.all_display_data, function(data, index) {
            if (data.records.length > 0){
                _.each(data.records, function(record) {
                    self.$element.find("#data_" + record.id).children().remove();
                    self.$element.find("#data_" + record.id).append(new_qweb.render('custom_template', record));
                });
            } else {
                self.$element.find("#column_" + data.value).remove();
                self.all_display_data.splice(index, 1);
            }
        });
        this.$element.find( ".oe_table_column " ).css("width", 99 / self.all_display_data.length +"%");
        this.$element.find('button').click(function() {
            var record_id = $(this).closest(".record").attr("id");
            if (record_id) {
                record_id = parseInt(record_id.split("_")[1])
                if (record_id) {
                    if ($(this).data("type") == "delete") {
                        self.do_delete(record_id);
                    } else {
                        var button_attrs = $(this).data()
                        self.on_button_click(button_attrs, record_id);
                    }
                }
            }
        });
    },

    do_search: function (domains, contexts, group_by) {
        var self = this;
        this.rpc('/base/session/eval_domain_and_context', {
            domains: domains,
            contexts: contexts,
            group_by_seq: group_by
        }, function (results) {
            self.group_by_field = false;
            self.datagroup = new openerp.base.DataGroup(self, self.model, results.domain, results.context, results.group_by || []);
            self.dataset.context = results.context;
            self.dataset.domain = results.domain;
            self.datagroup.list([],
                function (groups) {
                    self.groups = groups;
                    if (group_by.length >= 1) {
                        self.group_by_field = group_by[0].group_by;
                        self.do_render_group(groups);
                    }
                },
                function (dataset) {
                    self.domain = dataset.domain;
                    self.context = dataset.context;
                    self.groups = [];
                    self.dataset.read_slice({}, function(records) {
                        self.all_display_data = [{'records': records, 'value':false, 'header' : false, 'ids': self.dataset.ids}];
                        self.$element.find("#kanbanview").remove();
                        self.on_show_data(self.all_display_data);
                    });
                }
            );

        });
    },
    do_render_group : function (datagroups) {
        this.all_display_data = [];
        var self = this;
        _.each(datagroups, function (group) {
            self.dataset.context = group.context;
            self.dataset.domain = group.domain;
            var group_name = group.value;
            var group_value = group.value;
            if (!group.value) {
                group_name = "Undefined";
                group_value = 'false';
            } else if (group.value instanceof Array) {
                group_name = group.value[1];
                group_value = group.value[0];
            }
            self.dataset.read_slice({}, function(records) {
                self.all_display_data.push({"value" : group_value, "records": records, 'header':group_name, 'ids': self.dataset.ids});
                if (datagroups.length == self.all_display_data.length) {
                    self.$element.find("#kanbanview").remove();
                    self.on_show_data(self.all_display_data);
                }
            });
        });
    },

    do_show: function () {
        this.$element.show();
    },

    do_hide: function () {
        this.$element.hide();
    },

});
};

// vim:et fdc=0 fdl=0 foldnestmax=3 fdm=syntax:
<|endoftext|>


<commit_before>/*    Copyright 2020 Firewalla Inc
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require("../net2/logger.js")(__filename);
const rclient = require('../util/redis_manager.js').getRedisClient();
const sem = require('../sensor/SensorEventManager.js').getInstance();
const Block = require('./Block.js');
const DNSTool = require('../net2/DNSTool.js')
const dnsTool = new DNSTool()
const IntelTool = require('../net2/IntelTool');
const intelTool = new IntelTool();
const _ = require('lodash');
const fc = require('../net2/config.js');
const featureName = 'smart_block';
let instance = null;
const expiring = 24 * 60 * 60 * 3;  // three days

class BlockManager {
    constructor() {
        if (instance == null) {
            instance = this;
            const refreshInterval = 15 * 60; // 15 mins
            sem.once('IPTABLES_READY', async () => {
                if (fc.isFeatureOn(featureName)) {
                    this.scheduleId = setInterval(() => {
                        this.scheduleRefreshBlockLevel();
                    }, refreshInterval * 1000)
                }
                fc.onFeature(featureName, async (feature, status) => {
                    if (feature !== featureName) {
                        return
                    }
                    await this.reenforcePolicies();
                    if (status) {
                        this.scheduleId = setInterval(() => {
                            this.scheduleRefreshBlockLevel();
                        }, refreshInterval * 1000)
                    } else {
                        clearInterval(this.scheduleId);
                    }
                })
            })
        }
        return instance
    }
    async reenforcePolicies() {
        const PolicyManager2 = require('../alarm/PolicyManager2.js');
        const pm2 = new PolicyManager2();
        log.info('smart block enabled, re-enforce policies');
        const policies = await pm2.loadActivePoliciesAsync();
        policies.map((policy) => {
            pm2.tryPolicyEnforcement(policy, 'reenforce', policy);
        })
    }
    ipBlockInfoKey(ip) {
        return `ip:block:info:${ip}`
    }
    domainBlockInfoKey(domain) {
        return `domain:block:info:${domain}`
    }
    categoryDomainBlockInfoKey(domain) {
        return `category:block:info:${domain}`
    }
    getCategoryIpMapping(category) {
        return `rdns:category:${category}`
    }
    async getPureCategoryIps(category, categoryIps, originDomain) {
        if (!fc.isFeatureOn(featureName)) {
            return categoryIps;
        }
        const pureCategoryIps = [], mixupCategoryIps = [], mixupIpInfos = [];
        try {
            for (const categoryIp of categoryIps) {
                let pure = true;
                let mixupDomain;
                let mixupCategory;
                const domains = await dnsTool.getAllDns(categoryIp);
                for (const domain of domains) {
                    mixupDomain = domain;
                    const ips = await dnsTool.getIPsByDomain(domain);
                    for (const ip of ips) {
                        const intel = (await intelTool.getIntel(ip)) || {};
                        mixupCategory = intel.category;
                        pure = pure && intel.category == category;
                        if (!pure) break;
                    }
                    if (!pure) break;
                }
                if (pure) {
                    pureCategoryIps.push(categoryIp)
                } else {
                    mixupCategoryIps.push(categoryIp);
                    mixupIpInfos.push({
                        ip: categoryIp,
                        mixupDomain: mixupDomain,
                        mixupCategory: mixupCategory
                    })
                }
            }
            const categoryIpMappingKey = this.getCategoryIpMapping(category);
            mixupCategoryIps.length > 0 && await rclient.sremAsync(categoryIpMappingKey, mixupCategoryIps);
            pureCategoryIps.length > 0 && await rclient.saddAsync(categoryIpMappingKey, pureCategoryIps);
            const categoryDomainBlockInfoKey = this.categoryDomainBlockInfoKey(originDomain);
            await rclient.setAsync(categoryDomainBlockInfoKey, JSON.stringify({
                pureCategoryIps: pureCategoryIps,
                mixupIpInfos: mixupIpInfos
            }))
            rclient.expireat(categoryDomainBlockInfoKey, parseInt((+new Date) / 1000) + expiring);
        } catch (e) {
            log.info("get pure category ips failed", e)
        }
        return pureCategoryIps;
    }
    async applyNewDomain(ip, domain) {
        await this.updateIpBlockInfo(ip, domain, 'newDomain')
    }
    async scheduleRefreshBlockLevel() {
        const ipBlockKeys = await rclient.keysAsync("ip:block:info:*");
        log.info('schedule refresh block level for these ips:', ipBlockKeys)
        ipBlockKeys.map(async (key) => {
            try {
                let ipBlockInfo = JSON.parse(await rclient.getAsync(key));
                if (!ipBlockInfo) continue;
                const { targetDomains, ip, blockLevel, blockSet } = ipBlockInfo;
                const allDomains = await dnsTool.getAllDns(ip);
                const sharedDomains = _.differenceWith(allDomains, targetDomains, (a, b) => {
                    return this.domainCovered(b, a);
                });
                if (sharedDomains.length == 0 && blockLevel == 'domain') {
                    Block.block(ip, blockSet)
                }
                if (sharedDomains.length > 0 && blockLevel == 'ip') {
                    Block.unblock(ip, blockSet);
                }
                ipBlockInfo.ts = new Date() / 1000;
                ipBlockInfo.sharedDomains = sharedDomains;
                ipBlockInfo.allDomains = allDomains;
                await rclient.setAsync(key, JSON.stringify(ipBlockInfo));
            } catch (err) {
                log.warn('parse error', err);
            }
        })
    }
    async updateIpBlockInfo(ip, domain, action, blockSet = 'block_domain_set') {
        let ipBlockInfo = {
            blockSet: blockSet,
            ip: ip,
            targetDomains: [],
            sharedDomains: [],
            allDomains: [],
            ts: new Date() / 1000
        }
        let exist;
        if (!fc.isFeatureOn(featureName)) {
            ipBlockInfo.blockLevel = 'ip';
        } else {
            const key = this.ipBlockInfoKey(ip);
            exist = (await rclient.existsAsync(key) == 1);
            if (exist) {
                ipBlockInfo = JSON.parse(await rclient.getAsync(key));
            }
            switch (action) {
                case 'block': {
                    // if the ip shared with other domain, should not apply ip level block
                    // if a.com and b.com share ip and one of them block, it should be domain level
                    // if both block, should update to ip level
                    !ipBlockInfo.targetDomains.includes(domain) && ipBlockInfo.targetDomains.push(domain);
                    const allDomains = await dnsTool.getAllDns(ip);
                    const sharedDomains = _.differenceWith(allDomains, ipBlockInfo.targetDomains, (a, b) => {
                        return this.domainCovered(b, a);
                    });
                    sharedDomains.length > 0 && log.info(`${ipBlockInfo.targetDomains.join(',')} ip ${ip} shared with domains ${sharedDomains.join(',')}`)
                    if (sharedDomains.length == 0) {
                        ipBlockInfo.blockLevel = 'ip';
                    } else {
                        ipBlockInfo.blockLevel = 'domain';
                    }
                    ipBlockInfo.sharedDomains = sharedDomains;
                    ipBlockInfo.allDomains = allDomains;
                    ipBlockInfo.ts = new Date() / 1000;
                    await rclient.setAsync(key, JSON.stringify(ipBlockInfo));
                    break;
                }
                case 'unblock': {
                    !ipBlockInfo.sharedDomains.includes(domain) && ipBlockInfo.sharedDomains.push(domain);
                    ipBlockInfo.targetDomains = _.filter(ipBlockInfo.targetDomains, (a) => {
                        return a != domain;
                    })
                    if (ipBlockInfo.targetDomains.length == 0) {
                        await rclient.delAsync(key);
                    } else {
                        ipBlockInfo.ts = new Date() / 1000;
                        ipBlockInfo.blockLevel = 'domain';
                        await rclient.setAsync(key, JSON.stringify(ipBlockInfo));
                    }
                    break;
                }
                case 'newDomain': {
                    if (exist) {
                        // it is old ip and new domain
                        const { blockSet, targetDomains } = ipBlockInfo;
                        const alreayExistInTargetDomains = _.find(targetDomains, (targetDomain) => {
                            return this.domainCovered(targetDomain, domain);
                        })
                        if (!alreayExistInTargetDomains) {
                            if (ipBlockInfo.blockLevel == 'ip') {
                                log.info('ip block level change when new doamin comming', ip, domain)
                                ipBlockInfo.blockLevel = 'domain';
                                Block.unblock(ip, blockSet);
                            }
                            ipBlockInfo.sharedDomains.push(domain);
                            ipBlockInfo.allDomains = await dnsTool.getAllDns(ip);
                            ipBlockInfo.ts = new Date() / 1000;
                            await rclient.setAsync(key, JSON.stringify(ipBlockInfo));
                        }
                    }
                    break;
                }
            }
        }
        if (action == 'block' || action == 'unblock' || (action == 'newDomain' && exist)) {
            await this.updateDomainBlockInfo(domain, ipBlockInfo);
        }
        return ipBlockInfo;
    }
    async updateDomainBlockInfo(domain, ipBlockInfo) {
        const key = this.domainBlockInfoKey(domain);
        let domainBlockInfo = await rclient.getAsync(key);
        try {
            domainBlockInfo = JSON.parse(domainBlockInfo) || {};
        } catch (err) {
            domainBlockInfo = {};
        }
        domainBlockInfo[ipBlockInfo.ip] = ipBlockInfo;
        await rclient.setAsync(key, JSON.stringify(domainBlockInfo));
        rclient.expireat(key, parseInt((+new Date) / 1000) + expiring);
    }
    domainCovered(blockDomain, otherDomain) {
        // a.b.com covred x.a.b.com
        if (!otherDomain) return true;
        if (blockDomain.startsWith('*.')) blockDomain = blockDomain.substring(2)
        if (otherDomain.startsWith('*.')) otherDomain = otherDomain.substring(2)
        const h1Sections = blockDomain.split('.').reverse();
        const h2Sections = otherDomain.split('.').reverse();
        for (let i = 0; i < h1Sections.length; i++) {
            if (h1Sections[i] !== h2Sections[i])
                return false;
        }
        return true;
    }
}

module.exports = BlockManager
<commit_msg>update<commit_after>/*    Copyright 2020 Firewalla Inc
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require("../net2/logger.js")(__filename);
const rclient = require('../util/redis_manager.js').getRedisClient();
const sem = require('../sensor/SensorEventManager.js').getInstance();
const Block = require('./Block.js');
const DNSTool = require('../net2/DNSTool.js')
const dnsTool = new DNSTool()
const IntelTool = require('../net2/IntelTool');
const intelTool = new IntelTool();
const _ = require('lodash');
const fc = require('../net2/config.js');
const featureName = 'smart_block';
let instance = null;
const expiring = 24 * 60 * 60 * 3;  // three days

class BlockManager {
    constructor() {
        if (instance == null) {
            instance = this;
            const refreshInterval = 15 * 60; // 15 mins
            sem.once('IPTABLES_READY', async () => {
                if (fc.isFeatureOn(featureName)) {
                    this.scheduleId = setInterval(() => {
                        this.scheduleRefreshBlockLevel();
                    }, refreshInterval * 1000)
                }
                fc.onFeature(featureName, async (feature, status) => {
                    if (feature !== featureName) {
                        return
                    }
                    await this.reenforcePolicies();
                    if (status) {
                        this.scheduleId = setInterval(() => {
                            this.scheduleRefreshBlockLevel();
                        }, refreshInterval * 1000)
                    } else {
                        clearInterval(this.scheduleId);
                    }
                })
            })
        }
        return instance
    }
    async reenforcePolicies() {
        const PolicyManager2 = require('../alarm/PolicyManager2.js');
        const pm2 = new PolicyManager2();
        log.info('smart block enabled, re-enforce policies');
        const policies = await pm2.loadActivePoliciesAsync();
        policies.map((policy) => {
            pm2.tryPolicyEnforcement(policy, 'reenforce', policy);
        })
    }
    ipBlockInfoKey(ip) {
        return `ip:block:info:${ip}`
    }
    domainBlockInfoKey(domain) {
        return `domain:block:info:${domain}`
    }
    categoryDomainBlockInfoKey(domain) {
        return `category:block:info:${domain}`
    }
    getCategoryIpMapping(category) {
        return `rdns:category:${category}`
    }
    async getPureCategoryIps(category, categoryIps, originDomain) {
        if (!fc.isFeatureOn(featureName)) {
            return categoryIps;
        }
        const pureCategoryIps = [], mixupCategoryIps = [], mixupIpInfos = [];
        try {
            for (const categoryIp of categoryIps) {
                let pure = true;
                let mixupDomain;
                let mixupCategory;
                const domains = await dnsTool.getAllDns(categoryIp);
                for (const domain of domains) {
                    mixupDomain = domain;
                    const ips = await dnsTool.getIPsByDomain(domain);
                    for (const ip of ips) {
                        const intel = (await intelTool.getIntel(ip)) || {};
                        mixupCategory = intel.category;
                        pure = pure && intel.category == category;
                        if (!pure) break;
                    }
                    if (!pure) break;
                }
                if (pure) {
                    pureCategoryIps.push(categoryIp)
                } else {
                    mixupCategoryIps.push(categoryIp);
                    mixupIpInfos.push({
                        ip: categoryIp,
                        mixupDomain: mixupDomain,
                        mixupCategory: mixupCategory
                    })
                }
            }
            const categoryIpMappingKey = this.getCategoryIpMapping(category);
            mixupCategoryIps.length > 0 && await rclient.sremAsync(categoryIpMappingKey, mixupCategoryIps);
            pureCategoryIps.length > 0 && await rclient.saddAsync(categoryIpMappingKey, pureCategoryIps);
            const categoryDomainBlockInfoKey = this.categoryDomainBlockInfoKey(originDomain);
            await rclient.setAsync(categoryDomainBlockInfoKey, JSON.stringify({
                pureCategoryIps: pureCategoryIps,
                mixupIpInfos: mixupIpInfos
            }))
            rclient.expireat(categoryDomainBlockInfoKey, parseInt((+new Date) / 1000) + expiring);
        } catch (e) {
            log.info("get pure category ips failed", e)
        }
        return pureCategoryIps;
    }
    async applyNewDomain(ip, domain) {
        await this.updateIpBlockInfo(ip, domain, 'newDomain')
    }
    async scheduleRefreshBlockLevel() {
        const ipBlockKeys = await rclient.keysAsync("ip:block:info:*");
        log.info('schedule refresh block level for these ips:', ipBlockKeys)
        ipBlockKeys.map(async (key) => {
            try {
                let ipBlockInfo = JSON.parse(await rclient.getAsync(key));
                if (!ipBlockInfo) return;
                const { targetDomains, ip, blockLevel, blockSet } = ipBlockInfo;
                const allDomains = await dnsTool.getAllDns(ip);
                const sharedDomains = _.differenceWith(allDomains, targetDomains, (a, b) => {
                    return this.domainCovered(b, a);
                });
                if (sharedDomains.length == 0 && blockLevel == 'domain') {
                    Block.block(ip, blockSet)
                }
                if (sharedDomains.length > 0 && blockLevel == 'ip') {
                    Block.unblock(ip, blockSet);
                }
                ipBlockInfo.ts = new Date() / 1000;
                ipBlockInfo.sharedDomains = sharedDomains;
                ipBlockInfo.allDomains = allDomains;
                await rclient.setAsync(key, JSON.stringify(ipBlockInfo));
            } catch (err) {
                log.warn('parse error', err);
            }
        })
    }
    async updateIpBlockInfo(ip, domain, action, blockSet = 'block_domain_set') {
        let ipBlockInfo = {
            blockSet: blockSet,
            ip: ip,
            targetDomains: [],
            sharedDomains: [],
            allDomains: [],
            ts: new Date() / 1000
        }
        let exist;
        if (!fc.isFeatureOn(featureName)) {
            ipBlockInfo.blockLevel = 'ip';
        } else {
            const key = this.ipBlockInfoKey(ip);
            exist = (await rclient.existsAsync(key) == 1);
            if (exist) {
                ipBlockInfo = JSON.parse(await rclient.getAsync(key));
            }
            switch (action) {
                case 'block': {
                    // if the ip shared with other domain, should not apply ip level block
                    // if a.com and b.com share ip and one of them block, it should be domain level
                    // if both block, should update to ip level
                    !ipBlockInfo.targetDomains.includes(domain) && ipBlockInfo.targetDomains.push(domain);
                    const allDomains = await dnsTool.getAllDns(ip);
                    const sharedDomains = _.differenceWith(allDomains, ipBlockInfo.targetDomains, (a, b) => {
                        return this.domainCovered(b, a);
                    });
                    sharedDomains.length > 0 && log.info(`${ipBlockInfo.targetDomains.join(',')} ip ${ip} shared with domains ${sharedDomains.join(',')}`)
                    if (sharedDomains.length == 0) {
                        ipBlockInfo.blockLevel = 'ip';
                    } else {
                        ipBlockInfo.blockLevel = 'domain';
                    }
                    ipBlockInfo.sharedDomains = sharedDomains;
                    ipBlockInfo.allDomains = allDomains;
                    ipBlockInfo.ts = new Date() / 1000;
                    await rclient.setAsync(key, JSON.stringify(ipBlockInfo));
                    break;
                }
                case 'unblock': {
                    !ipBlockInfo.sharedDomains.includes(domain) && ipBlockInfo.sharedDomains.push(domain);
                    ipBlockInfo.targetDomains = _.filter(ipBlockInfo.targetDomains, (a) => {
                        return a != domain;
                    })
                    if (ipBlockInfo.targetDomains.length == 0) {
                        await rclient.delAsync(key);
                    } else {
                        ipBlockInfo.ts = new Date() / 1000;
                        ipBlockInfo.blockLevel = 'domain';
                        await rclient.setAsync(key, JSON.stringify(ipBlockInfo));
                    }
                    break;
                }
                case 'newDomain': {
                    if (exist) {
                        // it is old ip and new domain
                        const { blockSet, targetDomains } = ipBlockInfo;
                        const alreayExistInTargetDomains = _.find(targetDomains, (targetDomain) => {
                            return this.domainCovered(targetDomain, domain);
                        })
                        if (!alreayExistInTargetDomains) {
                            if (ipBlockInfo.blockLevel == 'ip') {
                                log.info('ip block level change when new doamin comming', ip, domain)
                                ipBlockInfo.blockLevel = 'domain';
                                Block.unblock(ip, blockSet);
                            }
                            ipBlockInfo.sharedDomains.push(domain);
                            ipBlockInfo.allDomains = await dnsTool.getAllDns(ip);
                            ipBlockInfo.ts = new Date() / 1000;
                            await rclient.setAsync(key, JSON.stringify(ipBlockInfo));
                        }
                    }
                    break;
                }
            }
        }
        if (action == 'block' || action == 'unblock' || (action == 'newDomain' && exist)) {
            await this.updateDomainBlockInfo(domain, ipBlockInfo);
        }
        return ipBlockInfo;
    }
    async updateDomainBlockInfo(domain, ipBlockInfo) {
        const key = this.domainBlockInfoKey(domain);
        let domainBlockInfo = await rclient.getAsync(key);
        try {
            domainBlockInfo = JSON.parse(domainBlockInfo) || {};
        } catch (err) {
            domainBlockInfo = {};
        }
        domainBlockInfo[ipBlockInfo.ip] = ipBlockInfo;
        await rclient.setAsync(key, JSON.stringify(domainBlockInfo));
        rclient.expireat(key, parseInt((+new Date) / 1000) + expiring);
    }
    domainCovered(blockDomain, otherDomain) {
        // a.b.com covred x.a.b.com
        if (!otherDomain) return true;
        if (blockDomain.startsWith('*.')) blockDomain = blockDomain.substring(2)
        if (otherDomain.startsWith('*.')) otherDomain = otherDomain.substring(2)
        const h1Sections = blockDomain.split('.').reverse();
        const h2Sections = otherDomain.split('.').reverse();
        for (let i = 0; i < h1Sections.length; i++) {
            if (h1Sections[i] !== h2Sections[i])
                return false;
        }
        return true;
    }
}

module.exports = BlockManager
<|endoftext|>


<commit_before>if (document.getElementById("speedy-delete") !== null) {
	var path = mw.config.get('wgArticlePath');
	var node = document.createElement("span");
	node.id = "delete-requester";
	node.style = "margin-left: 5px;";
	node.innerHTML = '提刪者：未取得';
	document.getElementsByClassName("mw-indicators mw-body-content")[0].appendChild(node);
	$.ajax({
		type: "GET",
		url: "https://zh.wikipedia.org/w/api.php",
		data: {
			action: "query",
			format: "json",
			prop: "revisions",
			titles: mw.config.get('wgPageName'),
			rvprop: "user|tags",
			rvlimit: "50"
		},
		success: function success(data) {
			var message = "找不到提刪者";
			var path = mw.config.get('wgArticlePath');
			for (var id in data.query.pages) {
				var page = data.query.pages[id];
				var requester = [];
				for (var i = 0; i < page.revisions.length; i++) {
					if (page.revisions[i].tags.indexOf("添加刪除模板") !== -1) {
						var user = page.revisions[i].user;
						requester.push('' + user + '');
					}
				}
				if (requester.length != 0) {
					message = requester.join("、");
				}
			}
			delrequester.innerHTML = message;
		},
		error: function error(e) {
			delrequester.innerHTML = "抓取錯誤";
		}
	});
}
if (mw.config.get('wgNamespaceNumber') !== -1 && (mw.config.get('wgAction') === "view" || mw.config.get('wgAction') === "edit")) {
	var path = mw.config.get('wgArticlePath');
	var node = document.createElement("span");
	node.id = "delete-log";
	node.style = "margin-left: 5px;";
	node.innerHTML = '刪除紀錄：未取得';
	document.getElementsByClassName("mw-indicators mw-body-content")[0].appendChild(node);
	$.ajax({
		type: "GET",
		url: "https://zh.wikipedia.org/w/api.php",
		data: {
			action: "query",
			format: "json",
			list: "logevents",
			leprop: "comment|type",
			letype: "delete",
			letitle: mw.config.get('wgPageName')
		},
		success: function success(data) {
			var message = "沒有";
			var path = mw.config.get('wgArticlePath');
			var log = [];
			for (var i = 0; i < data.query.logevents.length; i++) {
				var comment = data.query.logevents[i].comment;
				if (data.query.logevents[i].action === "restore") {
					comment = "還原";
				} else if (data.query.logevents[i].action === "revision") {
					continue;
				} else if (comment === "") {
					comment = "空";
				} else if (comment.match("删除以便移动") !== null) {
					continue;
				} else {
					comment = comment.replace(/.*\[\[:?(?:WP|Wikipedia)\:CSD\#([^|\]]+).*/g, "$1");
					comment = comment.replace(/.*?(Wikipedia:(頁面|檔案)存廢討論\/記錄\/\d{4}\/\d{2}\/\d{2}).*/g, '存廢');
					comment = comment.replace(/根據投票結果刪除.*/, "存廢");
					comment = comment.replace("列入[[WP:CV|侵权验证页面]]超过七日", "侵權");
					comment = comment.replace(/侵犯版权.*/, "侵權");
					comment = comment.replace(/侵犯版權.*/, "侵權");
					comment = comment.replace(/存廢討論通過.*/, "存廢");
					comment = comment.replace(/.*\[\[WP:CV.*/, "侵權");
					comment = comment.replace(/^内容为：.+/, "空");
					comment = comment.replace(/^content was.+/, "空");
					comment = comment.replace(/^make space.*/, "G8");
					comment = comment.replace(/^大量删除\[\[Special:Contributions\/.+/, "批刪");
					comment = comment.replace(/.*?([AGOF]\d{1,2}).*/i, "$1");
				}
				log.push(comment);
			}
			if (log.length != 0) {
				message = log.join("、");
			}
			dellog.innerHTML = message;
		},
		error: function error(e) {
			dellog.innerHTML = "抓取錯誤";
		}
	});
}
<commit_msg>delete-status add CSD R<commit_after>if (document.getElementById("speedy-delete") !== null) {
	var path = mw.config.get('wgArticlePath');
	var node = document.createElement("span");
	node.id = "delete-requester";
	node.style = "margin-left: 5px;";
	node.innerHTML = '提刪者：未取得';
	document.getElementsByClassName("mw-indicators mw-body-content")[0].appendChild(node);
	$.ajax({
		type: "GET",
		url: "https://zh.wikipedia.org/w/api.php",
		data: {
			action: "query",
			format: "json",
			prop: "revisions",
			titles: mw.config.get('wgPageName'),
			rvprop: "user|tags",
			rvlimit: "50"
		},
		success: function success(data) {
			var message = "找不到提刪者";
			var path = mw.config.get('wgArticlePath');
			for (var id in data.query.pages) {
				var page = data.query.pages[id];
				var requester = [];
				for (var i = 0; i < page.revisions.length; i++) {
					if (page.revisions[i].tags.indexOf("添加刪除模板") !== -1) {
						var user = page.revisions[i].user;
						requester.push('' + user + '');
					}
				}
				if (requester.length != 0) {
					message = requester.join("、");
				}
			}
			delrequester.innerHTML = message;
		},
		error: function error(e) {
			delrequester.innerHTML = "抓取錯誤";
		}
	});
}
if (mw.config.get('wgNamespaceNumber') !== -1 && (mw.config.get('wgAction') === "view" || mw.config.get('wgAction') === "edit")) {
	var path = mw.config.get('wgArticlePath');
	var node = document.createElement("span");
	node.id = "delete-log";
	node.style = "margin-left: 5px;";
	node.innerHTML = '刪除紀錄：未取得';
	document.getElementsByClassName("mw-indicators mw-body-content")[0].appendChild(node);
	$.ajax({
		type: "GET",
		url: "https://zh.wikipedia.org/w/api.php",
		data: {
			action: "query",
			format: "json",
			list: "logevents",
			leprop: "comment|type",
			letype: "delete",
			letitle: mw.config.get('wgPageName')
		},
		success: function success(data) {
			var message = "沒有";
			var path = mw.config.get('wgArticlePath');
			var log = [];
			for (var i = 0; i < data.query.logevents.length; i++) {
				var comment = data.query.logevents[i].comment;
				if (data.query.logevents[i].action === "restore") {
					comment = "還原";
				} else if (data.query.logevents[i].action === "revision") {
					continue;
				} else if (comment === "") {
					comment = "空";
				} else if (comment.match("删除以便移动") !== null) {
					continue;
				} else {
					comment = comment.replace(/.*\[\[:?(?:WP|Wikipedia)\:CSD\#([^|\]]+).*/g, "$1");
					comment = comment.replace(/.*?(Wikipedia:(頁面|檔案)存廢討論\/記錄\/\d{4}\/\d{2}\/\d{2}).*/g, '存廢');
					comment = comment.replace(/根據投票結果刪除.*/, "存廢");
					comment = comment.replace("列入[[WP:CV|侵权验证页面]]超过七日", "侵權");
					comment = comment.replace(/侵犯版权.*/, "侵權");
					comment = comment.replace(/侵犯版權.*/, "侵權");
					comment = comment.replace(/存廢討論通過.*/, "存廢");
					comment = comment.replace(/.*\[\[WP:CV.*/, "侵權");
					comment = comment.replace(/^内容为：.+/, "空");
					comment = comment.replace(/^content was.+/, "空");
					comment = comment.replace(/^make space.*/, "G8");
					comment = comment.replace(/^大量删除\[\[Special:Contributions\/.+/, "批刪");
					comment = comment.replace(/.*?([AGOFR]\d{1,2}).*/i, "$1");
				}
				log.push(comment);
			}
			if (log.length != 0) {
				message = log.join("、");
			}
			dellog.innerHTML = message;
		},
		error: function error(e) {
			dellog.innerHTML = "抓取錯誤";
		}
	});
}
<|endoftext|>