<commit_before>// Common settings and helpers for setting up Karma config.
//
// To run all the tests in a suite and print results to the console:
//
//   karma start <karma_config_for_suite_path>
//   E.g. karma start lms/static/karma_lms.conf.js
//
//
// To run the tests for debugging: Debugging can be done in any browser
// but Chrome's developer console debugging experience is best.
//
//   karma start <karma_config_for_suite_path> --browsers=BROWSER --single-run=false
//
//
// To run the tests with coverage and junit reports:
//
//   karma start <karma_config_for_suite_path> --browsers=BROWSER
// --coverage --junitreportpath=<xunit_report_path> --coveragereportpath=<report_path>
//
// where `BROWSER` could be Chrome or Firefox.
//

/* jshint node: true */
/*jshint -W079 */
'use strict';

var path = require('path');
var _ = require('underscore');
var appRoot = path.join(__dirname, '../../../../');

/**
 * Customize the name attribute in xml testcase element
 * @param {Object} browser
 * @param {Object} result
 * @return {String}
 */
function junitNameFormatter(browser, result) {
    return result.suite[0] + ': ' + result.description;
}

/**
 * Customize the classname attribute in xml testcase element
 * @param {Object} browser
 * @return {String}
 */
function junitClassNameFormatter(browser) {
    return "Javascript." + browser.name.split(" ")[0];
}

/**
 * Return array containing default and user supplied reporters
 * @param {Object} config
 * @return {Array}
 */
function reporters(config) {
    var defaultReporters = ['spec', 'junit', 'kjhtml'];
    if (config.coverage) {
        defaultReporters.push('coverage');
    }
    return defaultReporters;
}

/**
 * Split a filepath into basepath and filename
 * @param {String} filepath
 * @return {Object}
 */
function getBasepathAndFilename(filepath) {
    if (!filepath) {
        // these will configure the reporters to create report files relative to this karma config file
        return {
            dir: undefined,
            file: undefined
        };
    }

    var file = filepath.replace(/^.*[\\\/]/, ''),
        dir = filepath.replace(file, '');

    return {
        dir: dir,
        file: file
    };
}

/**
 * Return coverage reporter settings
 * @param {String} config
 * @return {Object}
 */
function coverageSettings(config) {
    var path = getBasepathAndFilename(config.coveragereportpath);
    return {
        dir: path.dir,
        subdir: '.',
        includeAllSources: true,
        reporters: [
            {type: 'cobertura', file: path.file},
            {type: 'text-summary'}
        ]
    };
}

/**
 * Return junit reporter settings
 * @param {String} config
 * @return {Object}
 */
function junitSettings(config) {
    var path = getBasepathAndFilename(config.junitreportpath);
    return {
        outputDir: path.dir,
        outputFile: path.file,
        suite: 'javascript',
        useBrowserName: false,
        nameFormatter: junitNameFormatter,
        classNameFormatter: junitClassNameFormatter
    };
}

var normalizePathsForCoverage = function(files, normalize_func) {
    var filesForCoverage = {};

    files.forEach(function (file) {
        if (_.isObject(file) && file.ignoreCoverage) { return; }

        var pattern = _.isObject(file) ? file.pattern : file;
        if (normalize_func) {
            pattern = normalize_func(appRoot, pattern);
        } else {
            if (pattern.match(/^common\/js/)) {
                pattern = path.join(appRoot, '/common/static/' + pattern);
            } else if (pattern.match(/^xmodule_js\/common_static/)) {
                pattern = path.join(appRoot, '/common/static/' +
                  pattern.replace(/^xmodule_js\/common_static\//, ''));
            }
        }
        filesForCoverage[pattern] = ['coverage'];
    });

    return filesForCoverage;
};

/**
 * Sets nocache on each file in the list.
 * @param {Object} files
 * @param {Bool} enable
 * @return {Object}
 */
var setNocache = function (files, enable) {
    files.forEach(function (f) {
        if (_.isObject(f)) {
            f.nocache = enable;
        }
    });
    return files;
};

/**
 * Sets defaults for each file pattern.
 * @param {Object} files
 * @return {Object}
 */
var setDefaults = function (files) {
    return files.map(function (f) {
        var file = _.isObject(f) ? f : {pattern: f};
        if (!file.included) {
            f.included = false;
        }
        return file;
    });
};

var getBaseConfig = function (config, useRequireJs) {

    var getFrameworkFiles = function () {
        var files = [
            'node_modules/jquery/dist/jquery.js',
            'node_modules/jasmine-core/lib/jasmine-core/jasmine.js',
            'node_modules/karma-jasmine/lib/boot.js',
            'node_modules/karma-jasmine/lib/adapter.js',
            'node_modules/jasmine-jquery/lib/jasmine-jquery.js'
        ];

        if (useRequireJs) {
            files = files.concat([
                'node_modules/requirejs/require.js',
                'node_modules/karma-requirejs/lib/adapter.js'
            ]);
        }

        return files;
    };

    // Manually prepends the framework files to the karma files array
    // bypassing the karma's framework config. This is necessary if you want
    // to add a library or framework that isn't a karma plugin. e.g. we add jasmine-jquery
    // which isn't a karma plugin. Though a karma framework for jasmine-jquery is available
    // but it's not actively maintained. In future we also wanna add jQuery at the top when
    // we upgrade to jQuery 2
    var initFrameworks = function (files) {
        getFrameworkFiles().reverse().forEach(function (f) {
            files.unshift({
                pattern: path.join(appRoot, f),
                included: true,
                served: true,
                watch: false
            });
        });
    };

    initFrameworks.$inject = ['config.files'];

    var customPlugin = {
        'framework:custom': ['factory', initFrameworks]
    };

    return {
        // base path that will be used to resolve all patterns (eg. files, exclude)
        basePath: '',

        // frameworks to use
        // available frameworks: https://npmjs.org/browse/keyword/karma-adapter
        frameworks: ['custom'],

        plugins: [
            'karma-jasmine',
            'karma-jasmine-html-reporter',
            'karma-requirejs',
            'karma-junit-reporter',
            'karma-coverage',
            'karma-chrome-launcher',
            'karma-firefox-launcher',
            'karma-spec-reporter',
            customPlugin
        ],

        // list of files to exclude
        exclude: [],

        // karma-reporter
        reporters: reporters(config),

        coverageReporter: coverageSettings(config),

        junitReporter: junitSettings(config),

        // web server port
        port: 9876,

        // enable / disable colors in the output (reporters and logs)
        colors: true,

        // level of logging
        /* possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN
         || config.LOG_INFO || config.LOG_DEBUG */
        logLevel: config.LOG_INFO,

        // enable / disable watching file and executing tests whenever any file changes
        autoWatch: false,

        // start these browsers
        // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
        browsers: ['Firefox'],

        // Continuous Integration mode
        // if true, Karma captures browsers, runs the tests and exits
        singleRun: config.singleRun,

        // Concurrency level
        // how many browser should be started simultaneous
        concurrency: Infinity,

        browserNoActivityTimeout: 50000
    };
};

var configure = function(data) {
    var useRequireJs = data.useRequireJs === undefined ? true : useRequireJs,
        baseConfig = getBaseConfig(data.config, useRequireJs);

    var files = _.flatten(
        _.map(
            ['libraryFilesToInclude', 'libraryFiles', 'sourceFiles', 'specFiles', 'fixtureFiles', 'runFiles'],
            function(item) { return data.files[item] || []; }
        )
    );

    files.unshift(
        {pattern: path.join(appRoot, 'common/static/common/js/jasmine.common.conf.js'), included: true}
    );

    if (useRequireJs) {
      files.unshift({pattern: 'common/js/utils/require-serial.js', included: true});
    }

    files = setDefaults(files);

    // With nocache=true, Karma always serves the latest files from disk.
    // However, that prevents coverage tracking from working.
    // So we only set it if coverage tracking is off.
    setNocache(files, !data.config.coverage);

    var filesForCoverage = _.flatten(
        _.map(
            ['sourceFiles', 'specFiles'],
            function(item) { return data.files[item]; }
        )
    );

    var preprocessors = _.extend(
      {},
      data.preprocessors,
      normalizePathsForCoverage(filesForCoverage, data.normalizePathsForCoverageFunc)
    );

    data.config.set(_.extend(baseConfig, {
        files: files,
        preprocessors: preprocessors
    }));
};

module.exports = {
    configure: configure,
    appRoot: appRoot
};
<commit_msg>Simplified normalizePathsForCoverage.<commit_after>// Common settings and helpers for setting up Karma config.
//
// To run all the tests in a suite and print results to the console:
//
//   karma start <karma_config_for_suite_path>
//   E.g. karma start lms/static/karma_lms.conf.js
//
//
// To run the tests for debugging: Debugging can be done in any browser
// but Chrome's developer console debugging experience is best.
//
//   karma start <karma_config_for_suite_path> --browsers=BROWSER --single-run=false
//
//
// To run the tests with coverage and junit reports:
//
//   karma start <karma_config_for_suite_path> --browsers=BROWSER
// --coverage --junitreportpath=<xunit_report_path> --coveragereportpath=<report_path>
//
// where `BROWSER` could be Chrome or Firefox.
//

/* jshint node: true */
/*jshint -W079 */
'use strict';

var path = require('path');
var _ = require('underscore');
var appRoot = path.join(__dirname, '../../../../');

/**
 * Customize the name attribute in xml testcase element
 * @param {Object} browser
 * @param {Object} result
 * @return {String}
 */
function junitNameFormatter(browser, result) {
    return result.suite[0] + ': ' + result.description;
}

/**
 * Customize the classname attribute in xml testcase element
 * @param {Object} browser
 * @return {String}
 */
function junitClassNameFormatter(browser) {
    return "Javascript." + browser.name.split(" ")[0];
}

/**
 * Return array containing default and user supplied reporters
 * @param {Object} config
 * @return {Array}
 */
function reporters(config) {
    var defaultReporters = ['spec', 'junit', 'kjhtml'];
    if (config.coverage) {
        defaultReporters.push('coverage');
    }
    return defaultReporters;
}

/**
 * Split a filepath into basepath and filename
 * @param {String} filepath
 * @return {Object}
 */
function getBasepathAndFilename(filepath) {
    if (!filepath) {
        // these will configure the reporters to create report files relative to this karma config file
        return {
            dir: undefined,
            file: undefined
        };
    }

    var file = filepath.replace(/^.*[\\\/]/, ''),
        dir = filepath.replace(file, '');

    return {
        dir: dir,
        file: file
    };
}

/**
 * Return coverage reporter settings
 * @param {String} config
 * @return {Object}
 */
function coverageSettings(config) {
    var path = getBasepathAndFilename(config.coveragereportpath);
    return {
        dir: path.dir,
        subdir: '.',
        includeAllSources: true,
        reporters: [
            {type: 'cobertura', file: path.file},
            {type: 'text-summary'}
        ]
    };
}

/**
 * Return junit reporter settings
 * @param {String} config
 * @return {Object}
 */
function junitSettings(config) {
    var path = getBasepathAndFilename(config.junitreportpath);
    return {
        outputDir: path.dir,
        outputFile: path.file,
        suite: 'javascript',
        useBrowserName: false,
        nameFormatter: junitNameFormatter,
        classNameFormatter: junitClassNameFormatter
    };
}

var defaultNormalizeFunc = function (appRoot, pattern) {
    if (pattern.match(/^common\/js/)) {
        pattern = path.join(appRoot, '/common/static/' + pattern);
    } else if (pattern.match(/^xmodule_js\/common_static/)) {
        pattern = path.join(appRoot, '/common/static/' +
          pattern.replace(/^xmodule_js\/common_static\//, ''));
    }
    return pattern;
};

var normalizePathsForCoverage = function(files, normalizeFunc) {
    var normalizeFn = normalizeFunc || defaultNormalizeFunc,
        filesForCoverage = {};

    files.forEach(function (file) {
        if (!file.ignoreCoverage) {
          filesForCoverage[normalizeFn(appRoot, file.pattern)] = ['coverage'];
        }
    });

    return filesForCoverage;
};

/**
 * Sets nocache on each file in the list.
 * @param {Object} files
 * @param {Bool} enable
 * @return {Object}
 */
var setNocache = function (files, enable) {
    files.forEach(function (f) {
        if (_.isObject(f)) {
            f.nocache = enable;
        }
    });
    return files;
};

/**
 * Sets defaults for each file pattern.
 * @param {Object} files
 * @return {Object}
 */
var setDefaults = function (files) {
    return files.map(function (f) {
        var file = _.isObject(f) ? f : {pattern: f};
        if (!file.included) {
            f.included = false;
        }
        return file;
    });
};

var getBaseConfig = function (config, useRequireJs) {

    var getFrameworkFiles = function () {
        var files = [
            'node_modules/jquery/dist/jquery.js',
            'node_modules/jasmine-core/lib/jasmine-core/jasmine.js',
            'node_modules/karma-jasmine/lib/boot.js',
            'node_modules/karma-jasmine/lib/adapter.js',
            'node_modules/jasmine-jquery/lib/jasmine-jquery.js'
        ];

        if (useRequireJs) {
            files = files.concat([
                'node_modules/requirejs/require.js',
                'node_modules/karma-requirejs/lib/adapter.js'
            ]);
        }

        return files;
    };

    // Manually prepends the framework files to the karma files array
    // bypassing the karma's framework config. This is necessary if you want
    // to add a library or framework that isn't a karma plugin. e.g. we add jasmine-jquery
    // which isn't a karma plugin. Though a karma framework for jasmine-jquery is available
    // but it's not actively maintained. In future we also wanna add jQuery at the top when
    // we upgrade to jQuery 2
    var initFrameworks = function (files) {
        getFrameworkFiles().reverse().forEach(function (f) {
            files.unshift({
                pattern: path.join(appRoot, f),
                included: true,
                served: true,
                watch: false
            });
        });
    };

    initFrameworks.$inject = ['config.files'];

    var customPlugin = {
        'framework:custom': ['factory', initFrameworks]
    };

    return {
        // base path that will be used to resolve all patterns (eg. files, exclude)
        basePath: '',

        // frameworks to use
        // available frameworks: https://npmjs.org/browse/keyword/karma-adapter
        frameworks: ['custom'],

        plugins: [
            'karma-jasmine',
            'karma-jasmine-html-reporter',
            'karma-requirejs',
            'karma-junit-reporter',
            'karma-coverage',
            'karma-chrome-launcher',
            'karma-firefox-launcher',
            'karma-spec-reporter',
            customPlugin
        ],

        // list of files to exclude
        exclude: [],

        // karma-reporter
        reporters: reporters(config),

        coverageReporter: coverageSettings(config),

        junitReporter: junitSettings(config),

        // web server port
        port: 9876,

        // enable / disable colors in the output (reporters and logs)
        colors: true,

        // level of logging
        /* possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN
         || config.LOG_INFO || config.LOG_DEBUG */
        logLevel: config.LOG_INFO,

        // enable / disable watching file and executing tests whenever any file changes
        autoWatch: false,

        // start these browsers
        // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
        browsers: ['Firefox'],

        // Continuous Integration mode
        // if true, Karma captures browsers, runs the tests and exits
        singleRun: config.singleRun,

        // Concurrency level
        // how many browser should be started simultaneous
        concurrency: Infinity,

        browserNoActivityTimeout: 50000
    };
};

var configure = function(data) {
    var useRequireJs = data.useRequireJs === undefined ? true : useRequireJs,
        baseConfig = getBaseConfig(data.config, useRequireJs);

    var files = _.flatten(
        _.map(
            ['libraryFilesToInclude', 'libraryFiles', 'sourceFiles', 'specFiles', 'fixtureFiles', 'runFiles'],
            function(item) { return data.files[item] || []; }
        )
    );

    files.unshift(
        {pattern: path.join(appRoot, 'common/static/common/js/jasmine.common.conf.js'), included: true}
    );

    if (useRequireJs) {
      files.unshift({pattern: 'common/js/utils/require-serial.js', included: true});
    }

    files = setDefaults(files);

    // With nocache=true, Karma always serves the latest files from disk.
    // However, that prevents coverage tracking from working.
    // So we only set it if coverage tracking is off.
    setNocache(files, !data.config.coverage);

    var filesForCoverage = _.flatten(
        _.map(
            ['sourceFiles', 'specFiles'],
            function(item) { return data.files[item]; }
        )
    );

    var preprocessors = _.extend(
      {},
      data.preprocessors,
      normalizePathsForCoverage(filesForCoverage, data.normalizePathsForCoverageFunc)
    );

    data.config.set(_.extend(baseConfig, {
        files: files,
        preprocessors: preprocessors
    }));
};

module.exports = {
    configure: configure,
    appRoot: appRoot
};
<|endoftext|>


<commit_before>export { ChannelPartInfo };

/**
 * Half channels information. Describe channel index range of lower half or higher half.
 *
 * @member {number} inChannelBegin
 *   The beginning channel index of this half input channels.
 *
 * @member {number} inChannelEnd
 *   The ending channel index of this half input channels.
 *
 */
class ChannelPartInfo {

  /**
   *
   */
  constructor( inChannelBegin, inChannelEnd ) {
    this.inChannelBegin = inChannelBegin;
    this.inChannelEnd = inChannelEnd;
  }

}
<commit_msg>Update ChannelPartInfo.js<commit_after>// (2022/02/24) Deprecated.

export { ChannelPartInfo };

/**
 * Half channels information. Describe channel index range of lower half or higher half.
 *
 * @member {number} inChannelBegin
 *   The beginning channel index of this half input channels.
 *
 * @member {number} inChannelEnd
 *   The ending channel index of this half input channels.
 *
 */
class ChannelPartInfo {

  /**
   *
   */
  constructor( inChannelBegin, inChannelEnd ) {
    this.inChannelBegin = inChannelBegin;
    this.inChannelEnd = inChannelEnd;
  }

}
<|endoftext|>


<commit_before>export { BoundsArraySet };

import * as ConvBiasActivation from "../ConvBiasActivation.js";
import * as FloatValue from "../../Unpacker/FloatValue.js";
import * as ValueDesc from "../../Unpacker/ValueDesc.js";
import * as Weights from "../../Unpacker/Weights.js";
import { ChannelPartInfo } from  "./Depthwise_ChannelPartInfo.js";

/**
 * The element value bounds set for depthwise convolution-bias-activation.
 *
 */
class BoundsArraySet extends ConvBiasActivation.BoundsArraySet {

  /**
   */
  constructor( inputChannelCount, outputChannelCount ) {
    super( inputChannelCount, outputChannelCount );
  }

//!!! ...unfinished... (2022/01/07) What about ( bDepthwise == false )?

//!!! ...unfinished... (2022/01/09)
// Even if ( this.AvgMax_Or_ChannelMultiplier == ValueDesc.AvgMax_Or_ChannelMultiplier.Singleton.Ids.NONE ),
// this function should work correctly and BoundsArraySet should result in pass-through input to output.

  /**
   *
   * @param {ConvBiasActivation.BoundsArraySet} previous_ConvBiasActivation_BoundsArraySet
   *   The value bounds set of the previous convolution-bias-activation.
   *
   * @param {Depthwise.ChannelPartInfo[]} inChannelPartInfoArray
   *   The input channel range array which describe lower/higher half channels index range.
   *
   * @param {number} channelMultiplier
   *   The channel multiplier of this depthwise operation.
   *
   * @param {number} nActivationId
   *   The activation function id (ValueDesc.ActivationFunction.Singleton.Ids.Xxx) of this depthwise convolution.
   *
   * @param {number[]} filtersArray
   *   The weights of the depthwise convolution. Its content will not be used. Only be tested against null. If ( filtersArray != null ),
   * this is a depthwise convolution. If ( filtersArray == null ), this is an avg/max pooling (i.e. no filters weights).
   *
   * @param {number[]} biasesArray
   *   The weights of the bias. Its content will be used. If ( biasesArray != null ), there is bias operation.
   * If ( biasesArray == null ), there is no bias operation.
   */
  set_all_by_inChannelPartInfoArray(
    previous_ConvBiasActivation_BoundsArraySet, inChannelPartInfoArray,
    channelMultiplier, nActivationId, filtersArray, biasesArray
  ) {

     // Because they are extracted from Weights which should have been regulated by Weights.Base.ValueBounds.Float32Array_RestrictedClone().
     const filtersValueBounds = Weights.Base.ValueBounds;
//!!! (2022/01/09 Remarked) Since we know what the filter and bias value is, the real value (i.e. not the supposed bounds range) could be used directly.
//     const biasesValueBounds = Weights.Base.ValueBounds;

    let theActivationFunctionInfo = ValueDesc.ActivationFunction.Singleton.getInfoById( nActivationId );

    // 1. Determine .input
    //
    // Note: Even if avg/max pooling, input value bounds is the same as the previous ooutput value bounds
    this.input.set_all_byBoundsArray( previous_ConvBiasActivation_BoundsArraySet.output );

    //let filterIndex = 0;
    //let biasIndex = 0;

    // ( inChannelPartIndex == 0 ), lower half channels. (or, all channels)
    // ( inChannelPartIndex == 1 ), higher half channels.
    for ( let inChannelPartIndex = 0; inChannelPartIndex < inChannelPartInfoArray.length; ++inChannelPartIndex ) {
      let inChannelPartInfo = inChannelPartInfoArray[ inChannelPartIndex ];
      let inChannelBegin = inChannelPartInfo.beginIndex;
      let inChannelEnd = inChannelPartInfo.endIndex;

      let outChannel = inChannelBegin * channelMultiplier;

      for ( let inChannel = inChannelBegin; inChannel < inChannelEnd; ++inChannel ) {

        let undoPreviousEscapingScale = previous_ConvBiasActivation_BoundsArraySet.activationEscaping_ScaleArraySet.undo.scales[ inChannel ];

        // 2. Determine .afterUndoPreviousActivationEscaping
        this.afterUndoPreviousActivationEscaping.set_one_byBoundsArray( inChannel, this.input, inChannel );
        this.afterUndoPreviousActivationEscaping.multiply_one_byN( inChannel, undoPreviousEscapingScale );

        for ( let outChannelSub = 0; outChannelSub < channelMultiplier; ++outChannelSub, ++outChannel ) {

          // 3. Determine .afterFilter
          this.afterFilter.set_one_byBoundsArray( outChannel, this.afterUndoPreviousActivationEscaping, inChannel );
          if ( filtersArray ) {

            if ( inChannelPartInfo.bPassThrough ) { // For pass-through half channels.
              // Do nothing. The value bounds does not change at all because it is just be past through.

            } else { // Non pass-through half channels.
              this.afterFilter
                .multiply_one_byBounds( outChannel, filtersValueBounds )
                .multiply_one_byN( outChannel, this.filterSize );
            }

          } else { // ( !filtersArray ). No filters array to be extracted. (i.e. avg/max pooling)
            // Do nothing. The value bounds does not change for avg/max pooling.
          }

          // 4. Determine .afterBias
          this.afterBias.set_one_byBoundsArray( outChannel, this.afterFilter, outChannel );
          if ( biasesArray ) {

            if ( inChannelPartInfo.bPassThrough ) { // For pass-through half channels.
              // Do nothing. The value bounds does not change at all because it is just be past through.

            } else { // Non pass-through half channels.

//!!! (2022/01/09 Remarked) Since we know what the filter and bias value is, the real value (i.e. not the supposed bounds range) could be used directly.
//              this.afterBias.add_one_byBounds( outChannel, biasesValueBounds ); // Shift the value bounds by the bias bounds.
              this.afterBias.add_one_byN( outChannel, biasesArray[ outChannel ] ); // Shift the value bounds by the real bias value.
            }

          } else { // ( !biasesArray ). No biases array to be extracted.
            // Do nothing. The value bounds does not change since no bias.
          }

          // 6. Determine .afterActivationEscaping
          {
            // 6.1 Determine .activationEscaping_ScaleArraySet
            {
              // 6.1.1 Determine .do

              if ( nActivationId == ValueDesc.ActivationFunction.Singleton.Ids.NONE ) {

                // Since no activation function, no need to escape. (i.e. scale = 1 for no scale)
                this.activationEscaping_ScaleArraySet.do.set_one_byN( outChannel, 1 );

              } else {

                if ( inChannelPartInfo.bPassThrough ) { // For pass-through half channels.

                  // Calculate the scale for escaping bias result from activation function's non-linear domain into linear domain.
                  //
                  // Note: This does not work for avg/max pooling.
                  this.activationEscaping_ScaleArraySet.do.set_one_by_fromLowerUpper_toLowerUpper( outChannel,
                    this.afterBias.lowers[ outChannel ], this.afterBias.uppers[ outChannel ],
                    theActivationFunctionInfo.inputDomainLinear.lower, theActivationFunctionInfo.inputDomainLinear.upper
                  );

                  let doEscapingScale = this.activationEscaping_ScaleArraySet.do.scales[ outChannel ];
                  tf.util.assert( ( Number.isNaN( doEscapingScale ) == false ),
                    `Depthwise.FiltersArray_BiasesArray.set_boundsArraySet_by_inChannelPartInfoArray(): `
                      + `this.activationEscaping_ScaleArraySet.do.scales[ ${outChannel} ] ( ${doEscapingScale} ) `
                      + `should not be NaN. `
                      + `Please use activation function (e.g. tanh()) which has both negative and positive parts near origin point.`
                  );

                } else { // Non pass-through half channels.
                  // Since non-pass-through, no need to escape. (i.e. scale = 1 for no scale)
                  this.activationEscaping_ScaleArraySet.do.set_one_byN( outChannel, 1 );
                }
              }

              // 6.1.2 Determine .undo (Prepared for the next convolution-bias-activation. Not for this.)
              this.activationEscaping_ScaleArraySet.undo.set_one_byUndo_ScaleArray(
                outChannel, this.activationEscaping_ScaleArraySet.do, outChannel );
            }

            // 6.2 Determine .afterActivationEscaping
            this.afterActivationEscaping
              .set_one_byBoundsArray( outChannel, this.afterBias, outChannel )
              .multiply_one_byNs( outChannel, this.activationEscaping_ScaleArraySet.do.scales, outChannel );
          }

          // 7. Determine .afterActivation
          {
            // If no activation function, the output range is determined by .afterActivationEscaping.
            if ( this.nActivationId == ValueDesc.ActivationFunction.Singleton.Ids.NONE ) {
              this.afterActivation.set_one_byBoundsArray( outChannel, this.afterActivationEscaping, outChannel )

            // Otherwise, the activation function dominates the output range.
            } else {
              this.afterActivation.set_one_byBounds( outChannel, theActivationFunctionInfo.outputRange );
            }
          }

        } // outChannelSub, outChannel
      } // inChannel
    } // inChannelPartIndex

  }

//!!! (2022/01/07 Remarked) seems not used. because should be channel by channel.
//
// //!!! ...unfinished... (2021/12/27) should become BoundsArray_byChannelIndex.
//
//   /**
//    *
//    * @param {ConvBiasActivation.BoundsArraySet} previous_ConvBiasActivation_BoundsArraySet
//    *   The previous convolution-bias-activation value bounds set of this depthwise convolution.
//    *
//    * @param {boolean} bDepthwise
//    *   If true, the depthwise convolution (with/without bias, with/without activation) exists.
//    *
//    * @param {number} filterHeight
//    *   The filter height of the depthwise convolution.
//    *
//    * @param {number} filterWidth
//    *   The filter width of the depthwise convolution.
//    *
//    * @param {boolean} bBias
//    *   If true, the bias operation exists.
//    *
//    * @param {number} nActivationId
//    *   The activation function id (ValueDesc.ActivationFunction.Singleton.Ids.Xxx) after the bias operation.
//    */
//    set_by( previous_ConvBiasActivation_BoundsArraySet, bDepthwise, filterHeight, filterWidth, bBias, nActivationId ) {
//
//     // 0. Default as BoundsArraySet.output of previous convolution-bias-activation.
//     this.reset_byBounds( previous_ConvBiasActivation_BoundsArraySet.output );
//
//     // 1. No operation at all.
//     if ( !bDepthwise )
//       return;
//
//     // 2. Before activation function.
//     {
//       // Because they are extracted from Weights which should have been regulated by Weights.Base.ValueBounds.Float32Array_RestrictedClone().
//       const filtersValueBounds = Weights.Base.ValueBounds;
//       const biasesValueBounds = Weights.Base.ValueBounds;
//
//       // Note: For maximum pooling, the multiply_Bounds is a little bit overestimated (but should be acceptable).
//       let filterSize = filterHeight * filterWidth;
//       this.beforeActivation.multiply_all_byBounds( filtersValueBounds ).multiply_all_byN( filterSize );
//
//       if ( bBias )
//         this.beforeActivation.add_all_byBounds( biasesValueBounds );
//
//
// //!!! ...unfinished... (2021/12/26)
// // The .undo should also be applied to the real filter value and bias value of this convolution-bias (i.e. not just applied here ScaleTranslate).
// //
// // Problem: What if this convolution-bias-activation could only undo partially (e.g. this convolution does not have bias)?
// //   How should the .undo of this convolution-bias-activation be calculated?
//       {
//         this.beforeActivation.multiply_all_byN(
//           previous_ConvBiasActivation_BoundsArraySet.activationEscaping_ScaleTranslateArraySet.undo.scale );
//
//         if ( this.bBias )
//           this.beforeActivation.add_all_byN(
//             previous_ConvBiasActivation_BoundsArraySet.activationEscaping_ScaleTranslateArraySet.undo.translate );
//       }
//
//     }
//
//     // 3. Output.
//     this.set_output_by_beforeActivation_ActivationId( nActivationId );
//
//     // 4. ActivationEscaping.ScaleTranslateArraySet.
//     this.activationEscaping_ScaleTranslateArraySet.set_by_currentBoundsArraySet_previousActivationEscaping(
//       this, previous_ConvBiasActivation_BoundsArraySet.activationEscaping_ScaleTranslateArraySet );
//   }

}

<commit_msg>Update Depthwise_BoundsArraySet.js<commit_after>export { BoundsArraySet };

import * as ConvBiasActivation from "../ConvBiasActivation.js";
import * as FloatValue from "../../Unpacker/FloatValue.js";
import * as ValueDesc from "../../Unpacker/ValueDesc.js";
import * as Weights from "../../Unpacker/Weights.js";
import { ChannelPartInfo } from  "./Depthwise_ChannelPartInfo.js";

/**
 * The element value bounds set for depthwise convolution-bias-activation.
 *
 */
class BoundsArraySet extends ConvBiasActivation.BoundsArraySet {

  /**
   */
  constructor( inputChannelCount, outputChannelCount ) {
    super( inputChannelCount, outputChannelCount );
  }

//!!! ...unfinished... (2022/01/07) What about ( bDepthwise == false )?

//!!! ...unfinished... (2022/01/09)
// Even if ( this.AvgMax_Or_ChannelMultiplier == ValueDesc.AvgMax_Or_ChannelMultiplier.Singleton.Ids.NONE ),
// this function should work correctly and BoundsArraySet should result in pass-through input to output.

  /**
   *
   * @param {ConvBiasActivation.BoundsArraySet} previous_ConvBiasActivation_BoundsArraySet
   *   The value bounds set of the previous convolution-bias-activation.
   *
   * @param {Depthwise.ChannelPartInfo[]} inChannelPartInfoArray
   *   The input channel range array which describe lower/higher half channels index range.
   *
   * @param {number} channelMultiplier
   *   The channel multiplier of this depthwise operation.
   *
   * @param {number} nActivationId
   *   The activation function id (ValueDesc.ActivationFunction.Singleton.Ids.Xxx) of this depthwise convolution.
   *
   * @param {number[]} filtersArray
   *   The weights of the depthwise convolution. Its content will not be used. Only be tested against null. If ( filtersArray != null ),
   * this is a depthwise convolution. If ( filtersArray == null ), this is an avg/max pooling (i.e. no filters weights).
   *
   * @param {number[]} biasesArray
   *   The weights of the bias. Its content will not be used. Only be tested against null. If ( biasesArray != null ), there is
   * bias operation. If ( biasesArray == null ), there is no bias operation.
   */
  set_all_by_inChannelPartInfoArray(
    previous_ConvBiasActivation_BoundsArraySet, inChannelPartInfoArray,
    channelMultiplier, nActivationId, filtersArray, biasesArray
  ) {

     // Because they are extracted from Weights which should have been regulated by Weights.Base.ValueBounds.Float32Array_RestrictedClone().
     //
     // Q: Why not use the real filter and bias value (i.e. not the supposed bounds range) directly.
     // A: Their values are unknown when determine value bounds. Even if they are known, their values are pre-scaled (i.e. altered; not original) value.
     const filtersValueBounds = Weights.Base.ValueBounds;
     const biasesValueBounds = Weights.Base.ValueBounds;

    let theActivationFunctionInfo = ValueDesc.ActivationFunction.Singleton.getInfoById( nActivationId );

    // 1. Determine .input
    //
    // Note: Even if avg/max pooling, input value bounds is the same as the previous ooutput value bounds
    this.input.set_all_byBoundsArray( previous_ConvBiasActivation_BoundsArraySet.output );

    // ( inChannelPartIndex == 0 ), lower half channels. (or, all channels)
    // ( inChannelPartIndex == 1 ), higher half channels.
    for ( let inChannelPartIndex = 0; inChannelPartIndex < inChannelPartInfoArray.length; ++inChannelPartIndex ) {
      let inChannelPartInfo = inChannelPartInfoArray[ inChannelPartIndex ];
      let inChannelBegin = inChannelPartInfo.beginIndex;
      let inChannelEnd = inChannelPartInfo.endIndex;

      let outChannel = inChannelBegin * channelMultiplier;

      for ( let inChannel = inChannelBegin; inChannel < inChannelEnd; ++inChannel ) {

        let undoPreviousEscapingScale = previous_ConvBiasActivation_BoundsArraySet.activationEscaping_ScaleArraySet.undo.scales[ inChannel ];

        // 2. Determine .afterUndoPreviousActivationEscaping
        this.afterUndoPreviousActivationEscaping.set_one_byBoundsArray( inChannel, this.input, inChannel );
        this.afterUndoPreviousActivationEscaping.multiply_one_byN( inChannel, undoPreviousEscapingScale );

        for ( let outChannelSub = 0; outChannelSub < channelMultiplier; ++outChannelSub, ++outChannel ) {

          // 3. Determine .afterFilter
          this.afterFilter.set_one_byBoundsArray( outChannel, this.afterUndoPreviousActivationEscaping, inChannel );
          if ( filtersArray ) {

            if ( inChannelPartInfo.bPassThrough ) { // For pass-through half channels.
              // Do nothing. The value bounds does not change at all because it is just be past through.

            } else { // Non pass-through half channels.
              this.afterFilter
                .multiply_one_byBounds( outChannel, filtersValueBounds )
                .multiply_one_byN( outChannel, this.filterSize );
            }

          } else { // ( !filtersArray ). No filters array to be extracted. (i.e. avg/max pooling)
            // Do nothing. The value bounds does not change for avg/max pooling.
          }

          // 4. Determine .afterBias
          this.afterBias.set_one_byBoundsArray( outChannel, this.afterFilter, outChannel );
          if ( biasesArray ) {

            if ( inChannelPartInfo.bPassThrough ) { // For pass-through half channels.
              // Do nothing. The value bounds does not change at all because it is just be past through.

            } else { // Non pass-through half channels.
              this.afterBias.add_one_byBounds( outChannel, biasesValueBounds ); // Shift the value bounds by the bias bounds.
            }

          } else { // ( !biasesArray ). No biases array to be extracted.
            // Do nothing. The value bounds does not change since no bias.
          }

          // 6. Determine .afterActivationEscaping
          {
            // 6.1 Determine .activationEscaping_ScaleArraySet
            {
              // 6.1.1 Determine .do

              if ( nActivationId == ValueDesc.ActivationFunction.Singleton.Ids.NONE ) {

                // Since no activation function, no need to escape. (i.e. scale = 1 for no scale)
                this.activationEscaping_ScaleArraySet.do.set_one_byN( outChannel, 1 );

              } else {

                if ( inChannelPartInfo.bPassThrough ) { // For pass-through half channels.

                  // Calculate the scale for escaping bias result from activation function's non-linear domain into linear domain.
                  //
                  // Note: This does not work for avg/max pooling.
                  this.activationEscaping_ScaleArraySet.do.set_one_by_fromLowerUpper_toLowerUpper( outChannel,
                    this.afterBias.lowers[ outChannel ], this.afterBias.uppers[ outChannel ],
                    theActivationFunctionInfo.inputDomainLinear.lower, theActivationFunctionInfo.inputDomainLinear.upper
                  );

                  let doEscapingScale = this.activationEscaping_ScaleArraySet.do.scales[ outChannel ];
                  tf.util.assert( ( Number.isNaN( doEscapingScale ) == false ),
                    `Depthwise.FiltersArray_BiasesArray.set_boundsArraySet_by_inChannelPartInfoArray(): `
                      + `this.activationEscaping_ScaleArraySet.do.scales[ ${outChannel} ] ( ${doEscapingScale} ) `
                      + `should not be NaN. `
                      + `Please use activation function (e.g. tanh()) which has both negative and positive parts near origin point.`
                  );

                } else { // Non pass-through half channels.
                  // Since non-pass-through, no need to escape. (i.e. scale = 1 for no scale)
                  this.activationEscaping_ScaleArraySet.do.set_one_byN( outChannel, 1 );
                }
              }

              // 6.1.2 Determine .undo (Prepared for the next convolution-bias-activation. Not for this.)
              this.activationEscaping_ScaleArraySet.undo.set_one_byUndo_ScaleArray(
                outChannel, this.activationEscaping_ScaleArraySet.do, outChannel );
            }

            // 6.2 Determine .afterActivationEscaping
            this.afterActivationEscaping
              .set_one_byBoundsArray( outChannel, this.afterBias, outChannel )
              .multiply_one_byNs( outChannel, this.activationEscaping_ScaleArraySet.do.scales, outChannel );
          }

          // 7. Determine .afterActivation
          {
            // If no activation function, the output range is determined by .afterActivationEscaping.
            if ( this.nActivationId == ValueDesc.ActivationFunction.Singleton.Ids.NONE ) {
              this.afterActivation.set_one_byBoundsArray( outChannel, this.afterActivationEscaping, outChannel )

            // Otherwise, the activation function dominates the output range.
            } else {
              this.afterActivation.set_one_byBounds( outChannel, theActivationFunctionInfo.outputRange );
            }
          }

        } // outChannelSub, outChannel
      } // inChannel
    } // inChannelPartIndex

  }

//!!! (2022/01/07 Remarked) seems not used. because should be channel by channel.
//
// //!!! ...unfinished... (2021/12/27) should become BoundsArray_byChannelIndex.
//
//   /**
//    *
//    * @param {ConvBiasActivation.BoundsArraySet} previous_ConvBiasActivation_BoundsArraySet
//    *   The previous convolution-bias-activation value bounds set of this depthwise convolution.
//    *
//    * @param {boolean} bDepthwise
//    *   If true, the depthwise convolution (with/without bias, with/without activation) exists.
//    *
//    * @param {number} filterHeight
//    *   The filter height of the depthwise convolution.
//    *
//    * @param {number} filterWidth
//    *   The filter width of the depthwise convolution.
//    *
//    * @param {boolean} bBias
//    *   If true, the bias operation exists.
//    *
//    * @param {number} nActivationId
//    *   The activation function id (ValueDesc.ActivationFunction.Singleton.Ids.Xxx) after the bias operation.
//    */
//    set_by( previous_ConvBiasActivation_BoundsArraySet, bDepthwise, filterHeight, filterWidth, bBias, nActivationId ) {
//
//     // 0. Default as BoundsArraySet.output of previous convolution-bias-activation.
//     this.reset_byBounds( previous_ConvBiasActivation_BoundsArraySet.output );
//
//     // 1. No operation at all.
//     if ( !bDepthwise )
//       return;
//
//     // 2. Before activation function.
//     {
//       // Because they are extracted from Weights which should have been regulated by Weights.Base.ValueBounds.Float32Array_RestrictedClone().
//       const filtersValueBounds = Weights.Base.ValueBounds;
//       const biasesValueBounds = Weights.Base.ValueBounds;
//
//       // Note: For maximum pooling, the multiply_Bounds is a little bit overestimated (but should be acceptable).
//       let filterSize = filterHeight * filterWidth;
//       this.beforeActivation.multiply_all_byBounds( filtersValueBounds ).multiply_all_byN( filterSize );
//
//       if ( bBias )
//         this.beforeActivation.add_all_byBounds( biasesValueBounds );
//
//
// //!!! ...unfinished... (2021/12/26)
// // The .undo should also be applied to the real filter value and bias value of this convolution-bias (i.e. not just applied here ScaleTranslate).
// //
// // Problem: What if this convolution-bias-activation could only undo partially (e.g. this convolution does not have bias)?
// //   How should the .undo of this convolution-bias-activation be calculated?
//       {
//         this.beforeActivation.multiply_all_byN(
//           previous_ConvBiasActivation_BoundsArraySet.activationEscaping_ScaleTranslateArraySet.undo.scale );
//
//         if ( this.bBias )
//           this.beforeActivation.add_all_byN(
//             previous_ConvBiasActivation_BoundsArraySet.activationEscaping_ScaleTranslateArraySet.undo.translate );
//       }
//
//     }
//
//     // 3. Output.
//     this.set_output_by_beforeActivation_ActivationId( nActivationId );
//
//     // 4. ActivationEscaping.ScaleTranslateArraySet.
//     this.activationEscaping_ScaleTranslateArraySet.set_by_currentBoundsArraySet_previousActivationEscaping(
//       this, previous_ConvBiasActivation_BoundsArraySet.activationEscaping_ScaleTranslateArraySet );
//   }

}

<|endoftext|>


<commit_before>/***
 This will reset the preferences that have been removed completely from the arkenfox user.js.

 Last updated: 02-Nov-2020

 For instructions see:
 https://github.com/arkenfox/user.js/wiki/3.1-Resetting-Inactive-Prefs-[Scripts]
***/

(function() {
  let ops = [
    /* removed in arkenfox user.js v52-57 */
    /* 52-alpha */
    'browser.search.reset.enabled',
    'browser.search.reset.whitelist',
    /* 54-alpha */
    'browser.migrate.automigrate.enabled',
    'services.sync.enabled',
    'webextensions.storage.sync.enabled',
    'webextensions.storage.sync.serverURL',
    /* 55-alpha */
    'dom.keyboardevent.dispatch_during_composition', // default is false anyway
    'dom.vr.oculus.enabled', // covered by dom.vr.enabled
    'dom.vr.openvr.enabled', // ditto
    'dom.vr.osvr.enabled', // ditto
    'extensions.pocket.api', // covered by extensions.pocket.enabled
    'extensions.pocket.oAuthConsumerKey', // ditto
    'extensions.pocket.site', // ditto
    /* 56-alpha: none */
    /* 57-alpha */
    'geo.wifi.xhr.timeout', // covered by geo.enabled
    'browser.search.geoip.timeout', // ditto
    'media.webspeech.recognition.enable', // default is false anyway
    'gfx.layerscope.enabled', // default is false anyway
    /* 58-alpha */
    //  excluding these e10 settings
       // 'browser.tabs.remote.autostart',
       // 'browser.tabs.remote.autostart.2',
       // 'browser.tabs.remote.force-enable',
       // 'browser.tabs.remote.separateFileUriProcess',
       // 'extensions.e10sBlocksEnabling',
       // 'extensions.webextensions.remote',
       // 'dom.ipc.processCount',
       // 'dom.ipc.shims.enabledWarnings',
       // 'dom.ipc.processCount.extension',
       // 'dom.ipc.processCount.file',
       // 'security.sandbox.content.level',
       // 'dom.ipc.plugins.sandbox-level.default',
       // 'dom.ipc.plugins.sandbox-level.flash',
       // 'security.sandbox.logging.enabled',
    'dom.presentation.controller.enabled',
    'dom.presentation.discoverable',
    'dom.presentation.discovery.enabled',
    'dom.presentation.enabled',
    'dom.presentation.receiver.enabled',
    'dom.presentation.session_transport.data_channel.enable',
    /* 59-alpha */
    'browser.stopReloadAnimation.enabled',
    'browser.tabs.insertRelatedAfterCurrent',
    'browser.tabs.loadDivertedInBackground',
    'browser.tabs.loadInBackground',
    'browser.tabs.selectOwnerOnClose',
    'browser.urlbar.clickSelectsAll',
    'browser.urlbar.doubleClickSelectsAll',
    'media.flac.enabled',
    'media.mediasource.enabled',
    'media.mediasource.mp4.enabled',
    'media.mediasource.webm.audio.enabled',
    'media.mediasource.webm.enabled',
    'media.mp4.enabled',
    'media.ogg.enabled',
    'media.ogg.flac.enabled',
    'media.opus.enabled',
    'media.raw.enabled',
    'media.wave.enabled',
    'media.webm.enabled',
    'media.wmf.amd.vp9.enabled',
    'media.wmf.enabled',
    'media.wmf.vp9.enabled',
    'ui.submenuDelay',
    /* 60-beta - these were all at default anyway */
    'device.storage.enabled',
    'general.useragent.compatMode.firefox',
    'network.dns.blockDotOnion',
    'network.stricttransportsecurity.preloadlist',
    'security.block_script_with_wrong_mime',
    'security.fileuri.strict_origin_policy',
    'security.sri.enable',
    /* 61-beta */
    'browser.laterrun.enabled',
    'browser.offline-apps.notify',
    'browser.rights.3.shown',
    'browser.slowStartup.maxSamples',
    'browser.slowStartup.notificationDisabled',
    'browser.slowStartup.samples',
    'browser.storageManager.enabled',
    'dom.allow_scripts_to_close_windows',
    'dom.disable_window_flip',
    'network.http.fast-fallback-to-IPv4',
    'offline-apps.quota.warn',
    'services.blocklist.signing.enforced',
    /* 62-beta */
    'browser.urlbar.autoFill.typed',
    'security.tls.version.fallback-limit',
    /* 63-beta */
    'extensions.webextensions.keepStorageOnUninstall',
    'extensions.webextensions.keepUuidOnUninstall',
    'privacy.trackingprotection.ui.enabled',
    /* 64-beta */
    'browser.eme.ui.enabled',
    'browser.sessionstore.max_windows_undo',
    'network.auth.subresource-img-cross-origin-http-auth-allow',
    'media.peerconnection.ice.tcp',
    'media.peerconnection.identity.enabled',
    'media.peerconnection.identity.timeout',
    'media.peerconnection.turn.disable',
    'media.peerconnection.use_document_iceservers',
    'media.peerconnection.video.enabled',
    'media.navigator.video.enabled',
    /* 65-beta */
    'browser.contentblocking.enabled',
    'browser.urlbar.maxHistoricalSearchSuggestions',
    /* 67-beta */
    'app.update.service.enabled',
    'app.update.silent',
    'app.update.staging.enabled',
    'browser.cache.disk.capacity',
    'browser.cache.disk.smart_size.enabled',
    'browser.cache.disk.smart_size.first_run',
    'browser.cache.offline.insecure.enable',
    'browser.safebrowsing.downloads.remote.url',
    'browser.safebrowsing.provider.google.reportMalwareMistakeURL',
    'browser.safebrowsing.provider.google.reportPhishMistakeURL',
    'browser.safebrowsing.provider.google.reportURL',
    'browser.safebrowsing.provider.google4.dataSharing.enabled',
    'browser.safebrowsing.provider.google4.dataSharingURL',
    'browser.safebrowsing.provider.google4.reportMalwareMistakeURL',
    'browser.safebrowsing.provider.google4.reportPhishMistakeURL',
    'browser.safebrowsing.provider.google4.reportURL',
    'browser.safebrowsing.reportPhishURL',
    'browser.sessionhistory.max_total_viewers',
    'browser.urlbar.filter.javascript',
    'canvas.capturestream.enabled',
    'dom.imagecapture.enabled',
    'dom.popup_maximum',
    'gfx.offscreencanvas.enabled',
    'javascript.options.shared_memory',
    'media.gmp-gmpopenh264.autoupdate',
    'media.gmp-gmpopenh264.enabled',
    'media.gmp-manager.updateEnabled',
    'media.gmp-manager.url',
    'media.gmp-manager.url.override',
    'media.gmp.trial-create.enabled',
    'media.gmp-widevinecdm.autoupdate',
    'network.cookie.leave-secure-alone',
    'network.cookie.same-site.enabled',
    'network.dnsCacheEntries',
    'network.dnsCacheExpiration',
    'network.proxy.autoconfig_url.include_path',
    'pdfjs.enableWebGL',
    'plugin.default.state',
    'plugin.defaultXpi.state',
    'plugin.scan.plid.all',
    'security.data_uri.block_toplevel_data_uri_navigations',
    'security.insecure_field_warning.contextual.enabled',
    'security.insecure_password.ui.enabled',
    'signon.autofillForms.http',
    'signon.storeWhenAutocompleteOff',
    'xpinstall.whitelist.required',
    /* 67-beta: Blocklist, SB & TP cleanup: these were all inactive */
    'browser.safebrowsing.downloads.remote.block_dangerous',
    'browser.safebrowsing.downloads.remote.block_dangerous_host',
    'browser.safebrowsing.blockedURIs.enabled',
    'browser.safebrowsing.provider.google.gethashURL',
    'browser.safebrowsing.provider.google.updateURL',
    'browser.safebrowsing.provider.google4.gethashURL',
    'browser.safebrowsing.provider.google4.updateURL',
    'browser.safebrowsing.provider.mozilla.gethashURL',
    'browser.safebrowsing.provider.mozilla.updateURL',
    'browser.urlbar.userMadeSearchSuggestionsChoice',
    'privacy.trackingprotection.annotate_channels',
    'privacy.trackingprotection.enabled',
    'privacy.trackingprotection.lower_network_priority',
    'privacy.trackingprotection.pbmode.enabled',
    'services.blocklist.addons.collection',
    'services.blocklist.gfx.collection',
    'services.blocklist.onecrl.collection',
    'services.blocklist.plugins.collection',
    'services.blocklist.update_enabled',
    'urlclassifier.trackingTable',
    /* 68-beta */
    'dom.forms.datetime',
    'font.blacklist.underline_offset',
    'font.name.monospace.x-unicode',
    'font.name.monospace.x-western',
    'font.name.sans-serif.x-unicode',
    'font.name.sans-serif.x-western',
    'font.name.serif.x-unicode',
    'font.name.serif.x-western',
    'layout.css.font-loading-api.enabled',
    'toolkit.telemetry.cachedClientID',
    /* 69-beta */
    'plugin.sessionPermissionNow.intervalInMinutes',
    /* 70-beta */
    'browser.cache.disk_cache_ssl',
    'browser.sessionhistory.max_entries',
    'dom.push.connection.enabled',
    'dom.push.serverURL',
    'extensions.getAddons.discovery.api_url',
    'extensions.htmlaboutaddons.discover.enabled',
    'extensions.webservice.discoverURL',
    'intl.locale.requested',
    'intl.regional_prefs.use_os_locales',
    'privacy.usercontext.about_newtab_segregation.enabled',
    'security.insecure_connection_icon.pbmode.enabled',
    'security.insecure_connection_text.pbmode.enabled',
    'webgl.dxgl.enabled',
    /* 71-beta */
    'media.block-autoplay-until-in-foreground',
    'middlemouse.paste',
    /* 75-beta */
    'browser.search.geoip.url',
    'browser.search.region',
    /* 79-beta */
    'browser.urlbar.usepreloadedtopurls.enabled',
    /* 80 */
    'dom.IntersectionObserver.enabled',
    /* 82-beta */
    'extensions.screenshots.upload-disabled',
    'privacy.partition.network_state',
    'security.ssl3.dhe_rsa_aes_128_sha',
    'security.ssl3.dhe_rsa_aes_256_sha',
    /* reset parrot: check your open about:config after running the script */
    '_user.js.parrot'
  ]

  if("undefined" === typeof(Services)) {
    alert("about:config needs to be the active tab!");
    return;
  }

  let c = 0;
  for (let i = 0, len = ops.length; i < len; i++) {
    if (Services.prefs.prefHasUserValue(ops[i])) {
      Services.prefs.clearUserPref(ops[i]);
      if (!Services.prefs.prefHasUserValue(ops[i])) {
        console.log("reset", ops[i]);
        c++;
      } else { console.log("failed to reset", ops[i]); }
    }
  }

  focus();

  let d = (c==1) ? " pref" : " prefs";
  if (c > 0) {
    alert("successfully reset " + c + d + "\n\nfor details check the Browser Console (Ctrl+Shift+J)");
  } else { alert("nothing to reset"); }

})();
<commit_msg>domIntersectionObserver<commit_after>/***
 This will reset the preferences that have been removed completely from the arkenfox user.js.

 Last updated: 02-Nov-2020

 For instructions see:
 https://github.com/arkenfox/user.js/wiki/3.1-Resetting-Inactive-Prefs-[Scripts]
***/

(function() {
  let ops = [
    /* removed in arkenfox user.js v52-57 */
    /* 52-alpha */
    'browser.search.reset.enabled',
    'browser.search.reset.whitelist',
    /* 54-alpha */
    'browser.migrate.automigrate.enabled',
    'services.sync.enabled',
    'webextensions.storage.sync.enabled',
    'webextensions.storage.sync.serverURL',
    /* 55-alpha */
    'dom.keyboardevent.dispatch_during_composition', // default is false anyway
    'dom.vr.oculus.enabled', // covered by dom.vr.enabled
    'dom.vr.openvr.enabled', // ditto
    'dom.vr.osvr.enabled', // ditto
    'extensions.pocket.api', // covered by extensions.pocket.enabled
    'extensions.pocket.oAuthConsumerKey', // ditto
    'extensions.pocket.site', // ditto
    /* 56-alpha: none */
    /* 57-alpha */
    'geo.wifi.xhr.timeout', // covered by geo.enabled
    'browser.search.geoip.timeout', // ditto
    'media.webspeech.recognition.enable', // default is false anyway
    'gfx.layerscope.enabled', // default is false anyway
    /* 58-alpha */
    //  excluding these e10 settings
       // 'browser.tabs.remote.autostart',
       // 'browser.tabs.remote.autostart.2',
       // 'browser.tabs.remote.force-enable',
       // 'browser.tabs.remote.separateFileUriProcess',
       // 'extensions.e10sBlocksEnabling',
       // 'extensions.webextensions.remote',
       // 'dom.ipc.processCount',
       // 'dom.ipc.shims.enabledWarnings',
       // 'dom.ipc.processCount.extension',
       // 'dom.ipc.processCount.file',
       // 'security.sandbox.content.level',
       // 'dom.ipc.plugins.sandbox-level.default',
       // 'dom.ipc.plugins.sandbox-level.flash',
       // 'security.sandbox.logging.enabled',
    'dom.presentation.controller.enabled',
    'dom.presentation.discoverable',
    'dom.presentation.discovery.enabled',
    'dom.presentation.enabled',
    'dom.presentation.receiver.enabled',
    'dom.presentation.session_transport.data_channel.enable',
    /* 59-alpha */
    'browser.stopReloadAnimation.enabled',
    'browser.tabs.insertRelatedAfterCurrent',
    'browser.tabs.loadDivertedInBackground',
    'browser.tabs.loadInBackground',
    'browser.tabs.selectOwnerOnClose',
    'browser.urlbar.clickSelectsAll',
    'browser.urlbar.doubleClickSelectsAll',
    'media.flac.enabled',
    'media.mediasource.enabled',
    'media.mediasource.mp4.enabled',
    'media.mediasource.webm.audio.enabled',
    'media.mediasource.webm.enabled',
    'media.mp4.enabled',
    'media.ogg.enabled',
    'media.ogg.flac.enabled',
    'media.opus.enabled',
    'media.raw.enabled',
    'media.wave.enabled',
    'media.webm.enabled',
    'media.wmf.amd.vp9.enabled',
    'media.wmf.enabled',
    'media.wmf.vp9.enabled',
    'ui.submenuDelay',
    /* 60-beta - these were all at default anyway */
    'device.storage.enabled',
    'general.useragent.compatMode.firefox',
    'network.dns.blockDotOnion',
    'network.stricttransportsecurity.preloadlist',
    'security.block_script_with_wrong_mime',
    'security.fileuri.strict_origin_policy',
    'security.sri.enable',
    /* 61-beta */
    'browser.laterrun.enabled',
    'browser.offline-apps.notify',
    'browser.rights.3.shown',
    'browser.slowStartup.maxSamples',
    'browser.slowStartup.notificationDisabled',
    'browser.slowStartup.samples',
    'browser.storageManager.enabled',
    'dom.allow_scripts_to_close_windows',
    'dom.disable_window_flip',
    'network.http.fast-fallback-to-IPv4',
    'offline-apps.quota.warn',
    'services.blocklist.signing.enforced',
    /* 62-beta */
    'browser.urlbar.autoFill.typed',
    'security.tls.version.fallback-limit',
    /* 63-beta */
    'extensions.webextensions.keepStorageOnUninstall',
    'extensions.webextensions.keepUuidOnUninstall',
    'privacy.trackingprotection.ui.enabled',
    /* 64-beta */
    'browser.eme.ui.enabled',
    'browser.sessionstore.max_windows_undo',
    'network.auth.subresource-img-cross-origin-http-auth-allow',
    'media.peerconnection.ice.tcp',
    'media.peerconnection.identity.enabled',
    'media.peerconnection.identity.timeout',
    'media.peerconnection.turn.disable',
    'media.peerconnection.use_document_iceservers',
    'media.peerconnection.video.enabled',
    'media.navigator.video.enabled',
    /* 65-beta */
    'browser.contentblocking.enabled',
    'browser.urlbar.maxHistoricalSearchSuggestions',
    /* 67-beta */
    'app.update.service.enabled',
    'app.update.silent',
    'app.update.staging.enabled',
    'browser.cache.disk.capacity',
    'browser.cache.disk.smart_size.enabled',
    'browser.cache.disk.smart_size.first_run',
    'browser.cache.offline.insecure.enable',
    'browser.safebrowsing.downloads.remote.url',
    'browser.safebrowsing.provider.google.reportMalwareMistakeURL',
    'browser.safebrowsing.provider.google.reportPhishMistakeURL',
    'browser.safebrowsing.provider.google.reportURL',
    'browser.safebrowsing.provider.google4.dataSharing.enabled',
    'browser.safebrowsing.provider.google4.dataSharingURL',
    'browser.safebrowsing.provider.google4.reportMalwareMistakeURL',
    'browser.safebrowsing.provider.google4.reportPhishMistakeURL',
    'browser.safebrowsing.provider.google4.reportURL',
    'browser.safebrowsing.reportPhishURL',
    'browser.sessionhistory.max_total_viewers',
    'browser.urlbar.filter.javascript',
    'canvas.capturestream.enabled',
    'dom.imagecapture.enabled',
    'dom.popup_maximum',
    'gfx.offscreencanvas.enabled',
    'javascript.options.shared_memory',
    'media.gmp-gmpopenh264.autoupdate',
    'media.gmp-gmpopenh264.enabled',
    'media.gmp-manager.updateEnabled',
    'media.gmp-manager.url',
    'media.gmp-manager.url.override',
    'media.gmp.trial-create.enabled',
    'media.gmp-widevinecdm.autoupdate',
    'network.cookie.leave-secure-alone',
    'network.cookie.same-site.enabled',
    'network.dnsCacheEntries',
    'network.dnsCacheExpiration',
    'network.proxy.autoconfig_url.include_path',
    'pdfjs.enableWebGL',
    'plugin.default.state',
    'plugin.defaultXpi.state',
    'plugin.scan.plid.all',
    'security.data_uri.block_toplevel_data_uri_navigations',
    'security.insecure_field_warning.contextual.enabled',
    'security.insecure_password.ui.enabled',
    'signon.autofillForms.http',
    'signon.storeWhenAutocompleteOff',
    'xpinstall.whitelist.required',
    /* 67-beta: Blocklist, SB & TP cleanup: these were all inactive */
    'browser.safebrowsing.downloads.remote.block_dangerous',
    'browser.safebrowsing.downloads.remote.block_dangerous_host',
    'browser.safebrowsing.blockedURIs.enabled',
    'browser.safebrowsing.provider.google.gethashURL',
    'browser.safebrowsing.provider.google.updateURL',
    'browser.safebrowsing.provider.google4.gethashURL',
    'browser.safebrowsing.provider.google4.updateURL',
    'browser.safebrowsing.provider.mozilla.gethashURL',
    'browser.safebrowsing.provider.mozilla.updateURL',
    'browser.urlbar.userMadeSearchSuggestionsChoice',
    'privacy.trackingprotection.annotate_channels',
    'privacy.trackingprotection.enabled',
    'privacy.trackingprotection.lower_network_priority',
    'privacy.trackingprotection.pbmode.enabled',
    'services.blocklist.addons.collection',
    'services.blocklist.gfx.collection',
    'services.blocklist.onecrl.collection',
    'services.blocklist.plugins.collection',
    'services.blocklist.update_enabled',
    'urlclassifier.trackingTable',
    /* 68-beta */
    'dom.forms.datetime',
    'font.blacklist.underline_offset',
    'font.name.monospace.x-unicode',
    'font.name.monospace.x-western',
    'font.name.sans-serif.x-unicode',
    'font.name.sans-serif.x-western',
    'font.name.serif.x-unicode',
    'font.name.serif.x-western',
    'layout.css.font-loading-api.enabled',
    'toolkit.telemetry.cachedClientID',
    /* 69-beta */
    'plugin.sessionPermissionNow.intervalInMinutes',
    /* 70-beta */
    'browser.cache.disk_cache_ssl',
    'browser.sessionhistory.max_entries',
    'dom.push.connection.enabled',
    'dom.push.serverURL',
    'extensions.getAddons.discovery.api_url',
    'extensions.htmlaboutaddons.discover.enabled',
    'extensions.webservice.discoverURL',
    'intl.locale.requested',
    'intl.regional_prefs.use_os_locales',
    'privacy.usercontext.about_newtab_segregation.enabled',
    'security.insecure_connection_icon.pbmode.enabled',
    'security.insecure_connection_text.pbmode.enabled',
    'webgl.dxgl.enabled',
    /* 71-beta */
    'media.block-autoplay-until-in-foreground',
    'middlemouse.paste',
    /* 75-beta */
    'browser.search.geoip.url',
    'browser.search.region',
    /* 79-beta */
    'browser.urlbar.usepreloadedtopurls.enabled',
    /* 82-beta */
    'dom.IntersectionObserver.enabled',
    'extensions.screenshots.upload-disabled',
    'privacy.partition.network_state',
    'security.ssl3.dhe_rsa_aes_128_sha',
    'security.ssl3.dhe_rsa_aes_256_sha',
    /* reset parrot: check your open about:config after running the script */
    '_user.js.parrot'
  ]

  if("undefined" === typeof(Services)) {
    alert("about:config needs to be the active tab!");
    return;
  }

  let c = 0;
  for (let i = 0, len = ops.length; i < len; i++) {
    if (Services.prefs.prefHasUserValue(ops[i])) {
      Services.prefs.clearUserPref(ops[i]);
      if (!Services.prefs.prefHasUserValue(ops[i])) {
        console.log("reset", ops[i]);
        c++;
      } else { console.log("failed to reset", ops[i]); }
    }
  }

  focus();

  let d = (c==1) ? " pref" : " prefs";
  if (c > 0) {
    alert("successfully reset " + c + d + "\n\nfor details check the Browser Console (Ctrl+Shift+J)");
  } else { alert("nothing to reset"); }

})();
<|endoftext|>


<commit_before>import fs from 'fs';
import Throttle from 'throttle';
import ffmpeg from 'fluent-ffmpeg';

import settings from './config.json';
import mimeTypes from './mimeTypes';

let ffmpegKeyGen = 0;
const ffmpegHash = {};
const dataHash = {};

const spiderStreamer = (data, query, rangeString, res) => {
  let stream;
  const info = {};
  let ext;
  let range;
  let i;
  let timerId;

  ext = data.name.match(/.*(\..+?)$/);

  if (ext === null || ext.length !== 2 || (info.mime = mimeTypes[ext[1].toLowerCase()]) === undefined) {
    console.error('spiderStreamer Error:'.red, 'Invalid mime type:', data.name);
    res.json({ error: 'badMime' });
    return false;
  }

  console.error('spiderStreamer Notice: Mime type', info.mime, 'found for file:', data.name);

  info.file = data.name;
  info.path = data.path;
  info.size = data.length;
  info.modified = data.date;

  new Promise(((fulfill, reject) => {
    /* ONLY DO THE FOLLOWING IF NOT MP4, WEBM, OR OGG */
    if (info.mime !== 'video/mp4' && info.mime !== 'video/webm' && info.mime !== 'video/ogg') {
      console.log('spiderStreamer Notice: Needs to be converted to video/mp4:', info.path);
      const old_path = info.path;
      const converted_path = `${info.path}.converted.mp4`;
      const converted_file = `${info.file}.converted.mp4`;
      const key = ++ffmpegKeyGen;
      if (ffmpegHash[old_path] === undefined) {
        console.log('fluent-ffmpeg Notice:', `${key}:`, 'Movie not yet converted, competing for key...');
        ffmpegHash[old_path] = key;
      }
      if (ffmpegHash[old_path] === key) {
        console.log('fluent-ffmpeg Notice:', `${key}:`, 'Chosen for conversion');
        console.log('spiderStreamer Notice: Converting to video/mp4');
        let fails = 0;
        let busy = false;
        const intervalId = setInterval(() => {
          if (!busy) {
            busy = true;
            try {
              // var format = ext[1].slice(1);
              // if (format === 'mkv') format = 'matroska';
              // ffmpeg().input(stream)
              ffmpeg().input(old_path)
                .on('error', (err, stdout, stderr) => {
                  console.error('spiderStreamer Error:'.red, 'Could not convert file:', old_path);
                  console.log('fluent-ffmpeg Error:'.red, '\nErr:', err, '\nStdOut:', stdout, '\nStdErr:', stderr);
                  /* Handle error */
                  ++fails;
                  busy = false;
                  // console.log('spiderStreamer Notice: Giving up: Piping raw stream');
                  // stream.pipe(res);
                })
                .on('start', (cmd) => {
                  console.log('fluent-ffmpeg Notice: Started:', cmd);
                })
                .on('codecData', (data) => {
                  console.log('fluent-ffmpeg Notice: CodecData:', data);
                  clearInterval(intervalId);
                  fulfill(data);
                  dataHash[old_path] = data;
                })
              // .on('progress', function(progress) {
              // 	console.log('fluent-ffmpeg Notice: Progress:', progress.timemark, 'converted');
              // })
              // .inputFormat(format)
                .audioCodec('aac')
                .videoCodec('libx264')
                .output(converted_path)
                .outputFormat('mp4')
                .outputOptions('-movflags frag_keyframe+empty_moov')
                .run();
              // .pipe(res);
            } catch (exception) {
              console.error('spiderStreamer Error:'.red, 'Could not convert file:', old_path);
              console.error('fluent-ffmpeg Error:'.red, exception);
              /* Handle error */
              ++fails;
              busy = false;
              // console.log('spiderStreamer Notice: Giving up: Piping raw stream');
              // stream.pipe(res);
            }
          } else {
            console.log('fluent-ffmpeg is busy');
          }
          if (fails > 30 && busy === false) {
            clearInterval(intervalId);
            reject('fluent-ffmpeg never launched without error');
          }
        }, 3000);
      } else {
        console.log('fluent-ffmpeg Notice:', `${key}:`, 'Movie already converted');
        fulfill(dataHash[old_path]);
      }

      info.file = converted_file;
      info.path = converted_path;
      info.mime = 'video/mp4';
      // info.modified = startup_date;
      info.modified = new Date();
      try {
        info.size = fs.statSync(info.path).size;
      } catch (exception) {
        console.log('spiderStreamer Error:'.red, 'Converted movie size not found');
        info.size = 0;
      }
    } else {
      console.log('spiderStreamer Notice: No conversion needed:', info.mime);
      fulfill(false);
    }
    /* ONLY DO THE ABOVE IF NOT MP4, WEBM, OR OGG */
  })).then(
    (success) => {
      new Promise(((fulfill, reject) => {
        let fails = 0;
        var intervalId = setInterval(() => {
          try {
            info.size = fs.statSync(info.path).size;
            console.log('spiderStreamer Notice:', info.path, ' size:', info.size);
            if (info.size > 5000000) {
              clearInterval(intervalId);
              fulfill(info.size);
              return;
            }
            console.log('spiderStreamer Notice: Movie file not yet big enough; fails:', fails);
          } catch (exception) {
            console.error('spiderStreamer Error:'.red, exception);
          }
          ++fails;
          if (fails > 30) {
            clearInterval(intervalId);
            reject('Movie file never grew to at least 5mb');
          }
        }, 2000);
      })).then(
        (success) => {
          info.rangeRequest = false;
          info.start = 0;
          info.end = info.size - 1;
          if (rangeString && (range = rangeString.match(/bytes=(.+)-(.+)?/)) !== null) {
            info.start = isNumber(range[1]) && range[1] >= 0 && range[1] < info.end ? range[1] - 0 : info.start;
            info.end = isNumber(range[2]) && range[2] > info.start && range[2] <= info.end ? range[2] - 0 : info.end;
            info.rangeRequest = true;
          } else if (query.start || query.end) {
            // This is a range request, but doesn't get range headers. So there.
            info.start = isNumber(query.start) && query.start >= 0 && query.start < info.end ? query.start - 0 : info.start;
            info.end = isNumber(query.end) && query.end > info.start && query.end <= info.end ? query.end - 0 : info.end;
          }

          info.length = info.end - info.start + 1;

          console.log('spiderStreamer Notice: Header Info:', info);

          console.log('spiderStreamer Notice: Sending header');
          downloadHeader(res, info);

          // // Flash vids seem to need this on the front, even if they start part way through. (JW Player does anyway.)
          // if (info.start > 0 && info.mime === "video/x-flv") {
          // 	res.write("FLV" + pack("CCNN", 1, 5, 9, 9));
          // }
          try {
            stream = fs.createReadStream(info.path, { flags: 'r', start: info.start, end: info.end });
            if (settings.throttle) {
              stream = stream.pipe(new Throttle(settings.throttle));
            }
            console.log('spiderStreamer Notice: Piping stream...');
            stream.pipe(res);
            console.log('spiderStreamer Notice: Pipe set');
          } catch (exception) {
            stream = null;
            i = 0;
            console.log('spiderStreamer Error:'.red, exception);
            console.log('spiderStreamer Notice: Retrying... i:', i);
            timerId = setInterval(() => {
              ++i;
              if (stream === null) {
                if (i === 5) {
                  clearInterval(timerId);
                  console.error('spiderStreamer Error:'.red, 'Could not stream file:', info.path);
                  /* Can't set headers after they are sent. */
                  // handler.emit("badFile", res);
                  return;
                }

                try {
                  stream = fs.createReadStream(info.path, { flags: 'r', start: info.start, end: info.end });
                } catch (exception) {
                  console.log('spiderStreamer Error:'.red, exception);
                  console.log('spiderStreamer Notice: Retrying in 3 seconds... i:', i);
                  stream = null;
                }
                if (stream !== null) {
                  clearInterval(timerId);
                  if (settings.throttle) {
                    stream = stream.pipe(new Throttle(settings.throttle));
                  }
                  console.log('spiderStreamer Notice: Piping stream...');
                  stream.pipe(res);
                  console.log('spiderStreamer Notice: Pipe set');
                }
              } else if (stream !== null) {
                clearInterval(timerId);
              }
            }, 3000);
          }
        },
        (failure) => {
          console.log('spiderStreamer Error:'.red, failure);
        }
      );
    },
    (failure) => {
      console.log('spiderStreamer Error:'.red, failure);
    }
  );
};

spiderStreamer.settings = function (s) {
  for (const prop in s) { settings[prop] = s[prop]; }
  return spiderStreamer;
};

var downloadHeader = function (res, info) {
  let code = 200;
  let header;

  // 'Connection':'close',
  // 'Cache-Control':'private',
  // 'Transfer-Encoding':'chunked'

  if (settings.forceDownload) {
    header = {
      Expires: 0,
      'Cache-Control': 'must-revalidate, post-check=0, pre-check=0',
      // "Cache-Control": "private",
      'Content-Type': info.mime,
      'Content-Disposition': `attachment; filename=${info.file};`
    };
  } else {
    header = {
      'Cache-Control': `public; max-age=${settings.maxAge}`,
      Connection: 'keep-alive',
      'Content-Type': info.mime,
      'Content-Disposition': `inline; filename=${info.file};`,
      'Accept-Ranges': 'bytes'
    };

    if (info.rangeRequest) {
      // Partial http response
      code = 206;
      header.Status = '206 Partial Content';
      header['Content-Range'] = `bytes ${info.start}-${info.end}/${info.size}`;
    }
  }

  header.Pragma = 'public';
  header['Last-Modified'] = info.modified.toUTCString();
  header['Content-Transfer-Encoding'] = 'binary';
  header['Content-Length'] = info.length;
  if (settings.cors) {
    header['Access-Control-Allow-Origin'] = '*';
    header['Access-Control-Allow-Headers'] = 'Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept';
  }
  header.Server = settings.server;

  res.writeHead(code, header);
};

// A tiny subset of http://phpjs.org/functions/pack:880
const pack = (format) => {
  let result = '';

  for (let pos = 1, len = arguments.length; pos < len; pos++) {
    if (format[pos - 1] == 'N') {
      result += String.fromCharCode(arguments[pos] >> 24 & 0xFF);
      result += String.fromCharCode(arguments[pos] >> 16 & 0xFF);
      result += String.fromCharCode(arguments[pos] >> 8 & 0xFF);
      result += String.fromCharCode(arguments[pos] & 0xFF);
    } else {
      result += String.fromCharCode(arguments[pos]);
    }
  }

  return result;
};

const isNumber = (n) => (
  !isNaN(parseFloat(n)) && isFinite(n)
);

export default spiderStreamer;
<commit_msg>torrent stream<commit_after>import fs from 'fs';
import Throttle from 'throttle';
import ffmpeg from 'fluent-ffmpeg';

import settings from './config.json';
import mimeTypes from './mimeTypes';

let ffmpegKeyGen = 0;
const ffmpegHash = {};
const dataHash = {};

const getFileExtension = (name) => {
  if (typeof name !== typeof '') return '';

  const extension = name.match(/\..+?$/);

  if (extension === null) return '';
  return extension[0].toLowerCase();
};

// const movie_data = {
//   name: movieFile.name,
//   length: movieFile.length,
//   date: movie.released,
//   stream: movieFile.createReadStream({ flags: "r", start: 0, end: movieFile.length - 1 })
//   path: `${engine.path}/${movieFile.path}`
// };
//
// info.name = file.name;
// info.path = data.path;
// info.size = data.length;
// info.modified = data.date;

const spiderStreamer = (file, query, rangeString, res) => {
  let stream;
  const info = {};
  let range;
  let i;
  let timerId;

  const extension = getFileExtension(file.name)
  if (!extension) {
    console.error('spiderStreamer Error:', 'No extension', file.name);
    return res.json({ error: 'No extension' });
  }

  file.mime = mimeTypes[extension]
  if (!file.mime) {
    console.error('spiderStreamer Error:', 'File extension not supported.', extension);
    return res.json({ error: 'Unsupported file extension' });
  }

  console.error('spiderStreamer Notice: Mime type', file.mime, 'found for file:', file.name);

  new Promise(((fulfill, reject) => {

    // media conversion if not mp4, nor webm, nor ogg
    if (file.mime !== 'video/mp4' && file.mime !== 'video/webm' && file.mime !== 'video/ogg') {
      const oldPath = file.path;
      const convertedPath = `${file.path}.converted.mp4`;
      const convertedFile = `${file.name}.converted.mp4`;
      const key = ++ffmpegKeyGen;
      if (ffmpegHash[oldPath] === undefined) {
        console.log('fluent-ffmpeg Notice:', `${key}:`, 'Movie not yet converted, competing for key...');
        ffmpegHash[oldPath] = key;
      }
      if (ffmpegHash[oldPath] === key) {
        console.log('fluent-ffmpeg Notice:', `${key}:`, 'Chosen for conversion');
        console.log('spiderStreamer Notice: Converting to video/mp4');
        let fails = 0;
        let busy = false;
        const intervalId = setInterval(() => {
          if (!busy) {
            busy = true;
            try {
              // var format = ext[1].slice(1);
              // if (format === 'mkv') format = 'matroska';
              // ffmpeg().input(stream)
              ffmpeg().input(oldPath)
                .on('error', (err, stdout, stderr) => {
                  console.error('spiderStreamer Error:'.red, 'Could not convert file:', oldPath);
                  console.log('fluent-ffmpeg Error:'.red, '\nErr:', err, '\nStdOut:', stdout, '\nStdErr:', stderr);
                  /* Handle error */
                  ++fails;
                  busy = false;
                  // console.log('spiderStreamer Notice: Giving up: Piping raw stream');
                  // stream.pipe(res);
                })
                .on('start', (cmd) => {
                  console.log('fluent-ffmpeg Notice: Started:', cmd);
                })
                .on('codecData', (data) => {
                  console.log('fluent-ffmpeg Notice: CodecData:', data);
                  clearInterval(intervalId);
                  fulfill(data);
                  dataHash[oldPath] = data;
                })
              // .on('progress', function(progress) {
              // 	console.log('fluent-ffmpeg Notice: Progress:', progress.timemark, 'converted');
              // })
              // .inputFormat(format)
                .audioCodec('aac')
                .videoCodec('libx264')
                .output(converted_path)
                .outputFormat('mp4')
                .outputOptions('-movflags frag_keyframe+empty_moov')
                .run();
              // .pipe(res);
            } catch (exception) {
              console.error('spiderStreamer Error:'.red, 'Could not convert file:', oldPath);
              console.error('fluent-ffmpeg Error:'.red, exception);
              /* Handle error */
              ++fails;
              busy = false;
              // console.log('spiderStreamer Notice: Giving up: Piping raw stream');
              // stream.pipe(res);
            }
          } else {
            console.log('fluent-ffmpeg is busy');
          }
          if (fails > 30 && busy === false) {
            clearInterval(intervalId);
            reject('fluent-ffmpeg never launched without error');
          }
        }, 3000);
      } else {
        console.log('fluent-ffmpeg Notice:', `${key}:`, 'Movie already converted');
        fulfill(dataHash[oldPath]);
      }

      info.file = converted_file;
      info.path = converted_path;
      info.mime = 'video/mp4';
      // info.modified = startup_date;
      info.modified = new Date();
      try {
        info.size = fs.statSync(info.path).size;
      } catch (exception) {
        console.log('spiderStreamer Error:'.red, 'Converted movie size not found');
        info.size = 0;
      }
    } else {
      console.log('spiderStreamer Notice: No conversion needed:', info.mime);
      fulfill(false);
    }
    /* ONLY DO THE ABOVE IF NOT MP4, WEBM, OR OGG */
  })).then(
    (success) => {
      new Promise(((fulfill, reject) => {
        let fails = 0;
        var intervalId = setInterval(() => {
          try {
            info.size = fs.statSync(info.path).size;
            console.log('spiderStreamer Notice:', info.path, ' size:', info.size);
            if (info.size > 5000000) {
              clearInterval(intervalId);
              fulfill(info.size);
              return;
            }
            console.log('spiderStreamer Notice: Movie file not yet big enough; fails:', fails);
          } catch (exception) {
            console.error('spiderStreamer Error:'.red, exception);
          }
          ++fails;
          if (fails > 30) {
            clearInterval(intervalId);
            reject('Movie file never grew to at least 5mb');
          }
        }, 2000);
      })).then(
        (success) => {
          info.rangeRequest = false;
          info.start = 0;
          info.end = info.size - 1;
          if (rangeString && (range = rangeString.match(/bytes=(.+)-(.+)?/)) !== null) {
            info.start = isNumber(range[1]) && range[1] >= 0 && range[1] < info.end ? range[1] - 0 : info.start;
            info.end = isNumber(range[2]) && range[2] > info.start && range[2] <= info.end ? range[2] - 0 : info.end;
            info.rangeRequest = true;
          } else if (query.start || query.end) {
            // This is a range request, but doesn't get range headers. So there.
            info.start = isNumber(query.start) && query.start >= 0 && query.start < info.end ? query.start - 0 : info.start;
            info.end = isNumber(query.end) && query.end > info.start && query.end <= info.end ? query.end - 0 : info.end;
          }

          info.length = info.end - info.start + 1;

          console.log('spiderStreamer Notice: Header Info:', info);

          console.log('spiderStreamer Notice: Sending header');
          downloadHeader(res, info);

          // // Flash vids seem to need this on the front, even if they start part way through. (JW Player does anyway.)
          // if (info.start > 0 && info.mime === "video/x-flv") {
          // 	res.write("FLV" + pack("CCNN", 1, 5, 9, 9));
          // }
          try {
            stream = fs.createReadStream(info.path, { flags: 'r', start: info.start, end: info.end });
            if (settings.throttle) {
              stream = stream.pipe(new Throttle(settings.throttle));
            }
            console.log('spiderStreamer Notice: Piping stream...');
            stream.pipe(res);
            console.log('spiderStreamer Notice: Pipe set');
          } catch (exception) {
            stream = null;
            i = 0;
            console.log('spiderStreamer Error:'.red, exception);
            console.log('spiderStreamer Notice: Retrying... i:', i);
            timerId = setInterval(() => {
              ++i;
              if (stream === null) {
                if (i === 5) {
                  clearInterval(timerId);
                  console.error('spiderStreamer Error:'.red, 'Could not stream file:', info.path);
                  /* Can't set headers after they are sent. */
                  // handler.emit("badFile", res);
                  return;
                }

                try {
                  stream = fs.createReadStream(info.path, { flags: 'r', start: info.start, end: info.end });
                } catch (exception) {
                  console.log('spiderStreamer Error:'.red, exception);
                  console.log('spiderStreamer Notice: Retrying in 3 seconds... i:', i);
                  stream = null;
                }
                if (stream !== null) {
                  clearInterval(timerId);
                  if (settings.throttle) {
                    stream = stream.pipe(new Throttle(settings.throttle));
                  }
                  console.log('spiderStreamer Notice: Piping stream...');
                  stream.pipe(res);
                  console.log('spiderStreamer Notice: Pipe set');
                }
              } else if (stream !== null) {
                clearInterval(timerId);
              }
            }, 3000);
          }
        },
        (failure) => {
          console.log('spiderStreamer Error:'.red, failure);
        }
      );
    },
    (failure) => {
      console.log('spiderStreamer Error:'.red, failure);
    }
  );
};

spiderStreamer.settings = function (s) {
  for (const prop in s) { settings[prop] = s[prop]; }
  return spiderStreamer;
};

var downloadHeader = function (res, info) {
  let code = 200;
  let header;

  // 'Connection':'close',
  // 'Cache-Control':'private',
  // 'Transfer-Encoding':'chunked'

  if (settings.forceDownload) {
    header = {
      Expires: 0,
      'Cache-Control': 'must-revalidate, post-check=0, pre-check=0',
      // "Cache-Control": "private",
      'Content-Type': info.mime,
      'Content-Disposition': `attachment; filename=${info.file};`
    };
  } else {
    header = {
      'Cache-Control': `public; max-age=${settings.maxAge}`,
      Connection: 'keep-alive',
      'Content-Type': info.mime,
      'Content-Disposition': `inline; filename=${info.file};`,
      'Accept-Ranges': 'bytes'
    };

    if (info.rangeRequest) {
      // Partial http response
      code = 206;
      header.Status = '206 Partial Content';
      header['Content-Range'] = `bytes ${info.start}-${info.end}/${info.size}`;
    }
  }

  header.Pragma = 'public';
  header['Last-Modified'] = info.modified.toUTCString();
  header['Content-Transfer-Encoding'] = 'binary';
  header['Content-Length'] = info.length;
  if (settings.cors) {
    header['Access-Control-Allow-Origin'] = '*';
    header['Access-Control-Allow-Headers'] = 'Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept';
  }
  header.Server = settings.server;

  res.writeHead(code, header);
};

// A tiny subset of http://phpjs.org/functions/pack:880
const pack = (format) => {
  let result = '';

  for (let pos = 1, len = arguments.length; pos < len; pos++) {
    if (format[pos - 1] == 'N') {
      result += String.fromCharCode(arguments[pos] >> 24 & 0xFF);
      result += String.fromCharCode(arguments[pos] >> 16 & 0xFF);
      result += String.fromCharCode(arguments[pos] >> 8 & 0xFF);
      result += String.fromCharCode(arguments[pos] & 0xFF);
    } else {
      result += String.fromCharCode(arguments[pos]);
    }
  }

  return result;
};

const isNumber = (n) => (
  !isNaN(parseFloat(n)) && isFinite(n)
);

export default spiderStreamer;
<|endoftext|>