<commit_before>//
// OpenRASP 官方插件只是抛砖引玉，具体检测能力请根据业务来定制
//

'use strict'
var plugin  = new RASP('offical')

const clean = {
    action:     'ignore',
    message:    '无风险',
    confidence: 0
}

var forcefulBrowsing = {
    dotFiles: /\.(7z|tar|gz|bz2|xz|rar|zip|sql|db)$/,
    nonUserDirectory: /^\/(proc|sys|root)/,
    unwantedFilenames: [
        // user files
        '.DS_Store',
        'id_rsa', 'id_rsa.pub', 'known_hosts', 'authorized_keys',
        '.bash_history', '.csh_history', '.zsh_history', '.mysql_history',

        // project files
        '.htaccess', '.user.ini',

        'web.config', 'web.xml', 'build.property.xml', 'bower.json',
        'Gemfile', 'Gemfile.lock',
        '.gitignore',
        'error_log', 'error.log', 'nohup.out',
    ],
    unwantedDirectory: [
        '/',
        '/home',
        '/var/log',
        '/proc',
        '/sys'
    ],
    absolutePaths: [
        '/etc/shadow',
        '/etc/hosts',
        '/etc/apache2/apache2.conf',
        '/root/.bash_history',
        '/root/.bash_profile',
    ]
}

var scriptFileRegex = /\.(jspx?|php[345]?|phtml)\.?$/i
var ntfsRegex       = /::\$(DATA|INDEX)$/i // 其他的stream都没啥用

String.prototype.replaceAll = function(token, tokenValue) {
    var index  = 0;
    var string = this;

    do {
        string = string.replace(token, tokenValue);
    } while((index = string.indexOf(token, index + 1)) > -1);

    return string
}

function canonicalPath (path) {
    return path.replaceAll('/./', '/').replaceAll('//', '/').replaceAll('//', '/')
}

function basename (path) {
    var idx = path.lastIndexOf('/')
    return path.substr(idx + 1)
}

// 主要用于识别webshell里的文件管理器
// 通常程序不会主动列目录或者查看敏感目录，e.g /home /etc /var/log 等等
//
// 若有特例可调整
// 可结合业务定制: e.g 不能超出应用根目录
plugin.register('directory', function (params, context) {
    var realpath = params.realpath

    for (var i = 0; i < forcefulBrowsing.unwantedDirectory.length; i ++) {
        if (realpath == forcefulBrowsing.unwantedDirectory[i]) {
            return {
                action:     'block',
                message:    '疑似WebShell文件管理器 - 读取敏感根目录',
                confidence: 100
            }
        }
    }

    return clean
})

plugin.register('readFile', function (params, context) {

    // 算法1: 检查是否为成功的目录扫描
    var filename_1 = basename(context.url)
    var filename_2 = basename(params.realpath)

    if (filename_1 == filename_2) {
        var matched = false

        // 尝试下载压缩包、SQL文件等等
        if (forcefulBrowsing.dotFiles.test(filename_1)) {
            matched = true
        } else {
            // 尝试访问敏感文件
            for (var i = 0; i < forcefulBrowsing.unwantedFilenames; i ++) {
                if (forcefulBrowsing.unwantedFilenames[i] == filename_1) {
                    matched = true
                }
            }
        }

        if (matched) {
            return {
                action:     'block',
                message:    '尝试下载敏感文件(' + context.method.toUpperCase() + ' 方式): ' + params.realpath,

                // 如果是HEAD方式下载敏感文件，100% 扫描器攻击
                confidence: context.method == 'head' ? 100 : 90
            }
        }
    }

    // 算法2: 如果使用绝对路径访问敏感文件，判定为 webshell
    if (params.realpath == params.path) {
        for (var j = 0; j < forcefulBrowsing.absolutePaths.length; j ++) {
            if (forcefulBrowsing.absolutePaths[j] == params.realpath) {
                return {
                    action:     'block',
                    message:    'WebShell/文件管理器 - 尝试读取系统文件: ' + params.realpath,
                    confidence: 90
                }
            }
        }
    }

    // 算法3: 检查文件遍历，看是否超出web目录范围
    var path        = canonicalPath(params.path)
    var appBasePath = context.appBasePath

    if (params.realpath.indexOf(appBasePath) == -1 && (path.indexOf('/../') !== -1 || path.indexOf('\\..\\') !== -1)) {
        return {
            action:     'block',
            message:    '目录遍历攻击，跳出web目录范围 (' + appBasePath + ')',
            confidence: 90
        }
    }

    return clean
})

plugin.register('webdav', function (params, context) {

    // 源文件不是脚本 && 目标文件是脚本，判定为MOVE方式写后门
    if (! scriptFileRegex.test(params.source) && scriptFileRegex.test(params.dest)) {
        return {
            action:    'block',
            message:   '尝试通过 ' + context.method + ' 方式上传脚本文件: ' + params.dest,
            confidence: 100
        }
    }

    return clean
})

plugin.register('include', function (params, context) {
    var items = params.url.split('://')

    // 必须有协议的情况下才能利用漏洞
    if (items.length != 2) {
        return clean
    }

    // http 方式 SSRF
    if (items[0].toLowerCase() == 'http') {
        return {
            action:     'block',
            message:    'SSRF漏洞: ' + params.function + ' 方式',
            confidence: 70
        }
    }

    // file 协议
    if (items[0].toLowerCase() == 'file') {
        var basename = items[1].split('/').pop()

        if (items[1].endsWith('/')) {
            return {
                action:     'block',
                message:    '敏感目录访问: ' + params.function + ' 方式',
                confidence: 100
            }
        }

        for (var i = 0; i < forcefulBrowsing.unwantedFilenames.length; i ++) {
            if (basename == forcefulBrowsing.unwantedFilenames[i]) {
                return {
                    action:     'block',
                    message:    '敏感文件下载: ' + params.function + ' 方式',
                    confidence: 100
                }
            }
        }
    }

    return clean
})

plugin.register('writeFile', function (params, context) {
    if (ntfsRegex.test(params.realpath)) {
        return {
            action:     'block',
            message:    '尝试利用NTFS流上传后门: ' + params.realpath,
            confidence: 90
        }
    }

    if (scriptFileRegex.test(params.realpath)) {
        return {
            action:     'log',
            message:    '尝试写入脚本文件: ' + params.realpath,
            confidence: 90
        }
    }
    return clean
})

plugin.register('fileUpload', function (params, context) {
    if (scriptFileRegex.test(params.filename) || ntfsRegex.test(params.filename)) {
        return {
            action:     'block',
            message:    '尝试上传脚本文件: ' + params.filename,
            confidence: 90
        }
    }
    return clean
})

plugin.register('sql', function (params, context) {
    var reason     = false
    var parameters = context.parameter
    var tokens     = RASP.sql_tokenize(params.query, params.server)

    // console.log(tokens)

    // 算法1: 匹配用户输入
    // 1. 简单识别逻辑是否发生改变
    // 2. 识别数据库管理器
    if (1) {
        Object.keys(parameters).some(function (name) {
            var value = parameters[name][0]

            // 请求参数长度超过10才考虑
            if (value.length <= 10) {
                return
            }

            // 判断是否为数据库管理器
            if (value.length == params.query.length && value == params.query) {
                reason = '算法2: WebShell - 数据库管理器'
                return true
            }

            // 简单识别用户输入
            if (params.query.indexOf(value) == -1) {
                return
            }

            // 去掉用户输入再次匹配
            var tokens2 = RASP.sql_tokenize(params.query.replace(value, ''), params.server)
            if (tokens.length - tokens2.length > 2) {
                reason = '算法1: 数据库查询逻辑发生改变'
                return true
            }
        })
        if (reason !== false) {
            return {
                'action':     'block',
                'confidence': 90,
                'message':    reason
            }
        }
    }

    // 算法2: SQL语句策略检查（模拟SQL防火墙功能）
    if (1) {
        var func_list = {
            'load_file': true,
            'benchmark': true,
            'sleep':     true,
            'pg_sleep':  true
        }
        var tokens_lc = tokens.map(v => v.toLowerCase())

        for (var i = 0; i < tokens_lc.length; i ++) {
            if (tokens_lc[i] == ';') {
                reason = '禁止多语句查询'
                break
            } else if (tokens_lc[i][0] === '0' && tokens_lc[i][1] === 'x') {
                reason = '禁止16进制字符串'
                break
            } else if (tokens_lc[i][0] === '/' && tokens_lc[i][1] === '*' && tokens_lc[i][2] === '!') {
                reason = '禁止MySQL版本号注释'
                break
            } else if (i > 0 && i < tokens_lc.length - 1 &&
                (tokens_lc[i] === 'xor'
                    || tokens_lc[i][0] === '<'
                    || tokens_lc[i][0] === '>'
                    || tokens_lc[i][0] === '=')) {
                // @FIXME: 可绕过，暂时不更新
                // 简单识别 NUMBER (>|<|>=|<=|xor) NUMBER
                //          i-1         i          i+2

                var op1    = tokens_lc[i - 1]
                var op2    = tokens_lc[i + 1]

                if (! isNaN(op1) && ! isNaN(op2)) {
                    reason = '禁止常量比较操作'
                    break
                }
            } else if (i > 0 && tokens_lc[i].indexOf('(') == 0) {
                // @FIXME: 可绕过，暂时不更新
                if (func_list[tokens_lc[i - 1]]) {
                    reason = '禁止执行敏感函数: ' + tokens_lc[i - 1]
                    break
                }
            }
        }

        if (reason !== false) {
            return {
                action:     'block',
                message:    '数据库语句异常: ' + reason + '（算法3）',
                confidence: 100
            }
        }
    }

    // 算法3: 简单正则匹配（即将移除）
    if (0) {
        var sqlRegex = /\bupdatexml\s*\(|\bextractvalue\s*\(|\bunion.*select.*(from|into|benchmark).*\b/i

        if (sqlRegex.test(params.query)) {
            return {
                action:     'block',
                message:    'SQL 注入攻击（算法4）',
                confidence: 100
            }
        }
    }

    return clean
})

plugin.register('command', function (params, context) {
    // 默认禁止命令执行
    return {
        action:    'block',
        message:   '尝试执行命令',
        confidence: 90
    }
})

plugin.register('xxe', function (params, context) {
    var items = params.entity.split('://')

    if (items.length >= 2) {
        var protocol = items[0]

        if (protocol === 'gopher' || protocol === 'dict' || protocol === 'expect') {
            return {
                action:     'block',
                message:    'SSRF 攻击 (' + protocol + ' 协议)',
                confidence: 100
            }
        }

        if (protocol === 'file') {
            return {
                action:     'log',
                message:    '尝试读取外部实体 (file 协议)',
                confidence: 90
            }
        }
    }
    return clean
})

plugin.register('ognl', function (params, context) {
    // 常见 struts payload 语句特征
    var ognlPayloads = [
        'ognl.OgnlContext',
        'ognl.TypeConverter',
        'ognl.MemberAccess',
        '_memberAccess',
        'ognl.ClassResolver',
        'java.lang.Runtime',
        'java.lang.Class',
        'java.lang.ClassLoader',
        'java.lang.System',
        'java.lang.ProcessBuilder',
        'java.lang.Object',
        'java.lang.Shutdown',
        'java.io.File',
        'javax.script.ScriptEngineManager',
        'com.opensymphony.xwork2.ActionContext'
    ]

    var ognlExpression = params.expression
    for (var index in ognlPayloads) {
        if (ognlExpression.indexOf(ognlPayloads[index]) > -1) {
            return {
                action:     'block',
                message:    '尝试ognl远程命令执行',
                confidence: 100
            }
        }

    }
    return clean
})

// [[ 近期调整~ ]]
plugin.register('deserialization', function (params, context) {
    var deserializationInvalidClazz = [
        'org.apache.commons.collections.functors.InvokerTransformer',
        'org.apache.commons.collections.functors.InstantiateTransformer',
        'org.apache.commons.collections4.functors.InvokerTransformer',
        'org.apache.commons.collections4.functors.InstantiateTransformer',
        'org.codehaus.groovy.runtime.ConvertedClosure',
        'org.codehaus.groovy.runtime.MethodClosure',
        'org.springframework.beans.factory.ObjectFactory',
        'xalan.internal.xsltc.trax.TemplatesImpl'
    ]

    var clazz = params.clazz
    for (var index in deserializationInvalidClazz) {
        if (clazz === deserializationInvalidClazz[index]) {
            return {
                action:     'block',
                message:    '尝试反序列化攻击',
                confidence: 100
            }
        }
    }
    return clean
})

plugin.register('reflection', function(params, context) {
    var title = '异常的执行流'
    var known = {
        'com.thoughtworks.xstream.XStream.unmarshal': 'xstream 反序列化攻击',
        'org.apache.commons.collections4.functors.InvokerTransformer.transform': 'transformer 反序列化攻击'
    }

    // 是否为已知类型的反序列化攻击？
    params.stack.some(function (method) {
        if (known[method]) {
            title = known[method]
            return true;
        }
    });

    return {
        action:     'block',
        message:    title + ': ' + params.clazz + '.' + params.method,
        confidence: 100
    }
})

plugin.log('初始化成功')
<commit_msg>SSRF 检测第一版<commit_after>//
// OpenRASP 官方插件只是抛砖引玉，具体检测能力请根据业务来定制
//

'use strict'
var plugin  = new RASP('offical')

const clean = {
    action:     'ignore',
    message:    '无风险',
    confidence: 0
}

var forcefulBrowsing = {
    dotFiles: /\.(7z|tar|gz|bz2|xz|rar|zip|sql|db)$/,
    nonUserDirectory: /^\/(proc|sys|root)/,
    unwantedFilenames: [
        // user files
        '.DS_Store',
        'id_rsa', 'id_rsa.pub', 'known_hosts', 'authorized_keys',
        '.bash_history', '.csh_history', '.zsh_history', '.mysql_history',

        // project files
        '.htaccess', '.user.ini',

        'web.config', 'web.xml', 'build.property.xml', 'bower.json',
        'Gemfile', 'Gemfile.lock',
        '.gitignore',
        'error_log', 'error.log', 'nohup.out',
    ],
    unwantedDirectory: [
        '/',
        '/home',
        '/var/log',
        '/proc',
        '/sys'
    ],
    absolutePaths: [
        '/etc/shadow',
        '/etc/hosts',
        '/etc/apache2/apache2.conf',
        '/root/.bash_history',
        '/root/.bash_profile',
    ]
}

var scriptFileRegex = /\.(jspx?|php[345]?|phtml)\.?$/i
var ntfsRegex       = /::\$(DATA|INDEX)$/i // 其他的stream都没啥用

String.prototype.replaceAll = function(token, tokenValue) {
    var index  = 0;
    var string = this;

    do {
        string = string.replace(token, tokenValue);
    } while((index = string.indexOf(token, index + 1)) > -1);

    return string
}

function canonicalPath (path) {
    return path.replaceAll('/./', '/').replaceAll('//', '/').replaceAll('//', '/')
}

function basename (path) {
    var idx = path.lastIndexOf('/')
    return path.substr(idx + 1)
}

function ip2long(ipstr) {
    var items  = ipstr.split('.')
    var result = 0

    for (var i = 0; i < items.length; i ++) {
        result = (result << 8) + parseInt(items[i], 10)
    }
    return result >>> 0
    return ip.split('.').reduce(function(ipInt, octet) { return (ipInt<<8) + parseInt(octet, 10)}, 0) >>> 0;
}

// 主要用于识别webshell里的文件管理器
// 通常程序不会主动列目录或者查看敏感目录，e.g /home /etc /var/log 等等
//
// 若有特例可调整
// 可结合业务定制: e.g 不能超出应用根目录
plugin.register('directory', function (params, context) {
    var realpath = params.realpath

    for (var i = 0; i < forcefulBrowsing.unwantedDirectory.length; i ++) {
        if (realpath == forcefulBrowsing.unwantedDirectory[i]) {
            return {
                action:     'block',
                message:    '疑似WebShell文件管理器 - 读取敏感根目录',
                confidence: 100
            }
        }
    }

    return clean
})

plugin.register('readFile', function (params, context) {

    // 算法1: 检查是否为成功的目录扫描
    var filename_1 = basename(context.url)
    var filename_2 = basename(params.realpath)

    if (filename_1 == filename_2) {
        var matched = false

        // 尝试下载压缩包、SQL文件等等
        if (forcefulBrowsing.dotFiles.test(filename_1)) {
            matched = true
        } else {
            // 尝试访问敏感文件
            for (var i = 0; i < forcefulBrowsing.unwantedFilenames; i ++) {
                if (forcefulBrowsing.unwantedFilenames[i] == filename_1) {
                    matched = true
                }
            }
        }

        if (matched) {
            return {
                action:     'block',
                message:    '尝试下载敏感文件(' + context.method.toUpperCase() + ' 方式): ' + params.realpath,

                // 如果是HEAD方式下载敏感文件，100% 扫描器攻击
                confidence: context.method == 'head' ? 100 : 90
            }
        }
    }

    // 算法2: 如果使用绝对路径访问敏感文件，判定为 webshell
    if (params.realpath == params.path) {
        for (var j = 0; j < forcefulBrowsing.absolutePaths.length; j ++) {
            if (forcefulBrowsing.absolutePaths[j] == params.realpath) {
                return {
                    action:     'block',
                    message:    'WebShell/文件管理器 - 尝试读取系统文件: ' + params.realpath,
                    confidence: 90
                }
            }
        }
    }

    // 算法3: 检查文件遍历，看是否超出web目录范围
    var path        = canonicalPath(params.path)
    var appBasePath = context.appBasePath

    if (params.realpath.indexOf(appBasePath) == -1 && (path.indexOf('/../') !== -1 || path.indexOf('\\..\\') !== -1)) {
        return {
            action:     'block',
            message:    '目录遍历攻击，跳出web目录范围 (' + appBasePath + ')',
            confidence: 90
        }
    }

    return clean
})

plugin.register('ssrf', function (params, context) {
    var hostname = params.hostname
    var reason   = false

    // 检查常见探测域名
    if (hostname.endsWith('.xip.io') || hostname.endsWith('.burpcollaborator.net')) {
        reason = '访问已知的内网探测域名'
    }
    // 检测AWS私有地址，如有误报可注释掉
    else if (1 && hostname == '169.254.169.254') {
        reason = '尝试读取 AWS metadata'
    }
    // 检查混淆:
    // http://2130706433
    //
    // 以下混淆方式没有检测
    // http://0x7f.0x0.0x0.0x1
    else if (! isNaN(hostname)) {
        reason = '尝试使用纯数字IP'
    }
    // 如果明确服务器不会访问内网地址，可以打开这个
    else if (0) {

    }

    if (reason) {
        return {
            action:    'block',
            message:   'SSRF攻击: ' + reason,
            confidence: 100
        }
    }
    return clean
})

plugin.register('webdav', function (params, context) {

    // 源文件不是脚本 && 目标文件是脚本，判定为MOVE方式写后门
    if (! scriptFileRegex.test(params.source) && scriptFileRegex.test(params.dest)) {
        return {
            action:    'block',
            message:   '尝试通过 ' + context.method + ' 方式上传脚本文件: ' + params.dest,
            confidence: 100
        }
    }

    return clean
})

plugin.register('include', function (params, context) {
    var items = params.url.split('://')

    // 必须有协议的情况下才能利用漏洞
    if (items.length != 2) {
        return clean
    }

    // http 方式 SSRF
    if (items[0].toLowerCase() == 'http') {
        return {
            action:     'block',
            message:    'SSRF漏洞: ' + params.function + ' 方式',
            confidence: 70
        }
    }

    // file 协议
    if (items[0].toLowerCase() == 'file') {
        var basename = items[1].split('/').pop()

        if (items[1].endsWith('/')) {
            return {
                action:     'block',
                message:    '敏感目录访问: ' + params.function + ' 方式',
                confidence: 100
            }
        }

        for (var i = 0; i < forcefulBrowsing.unwantedFilenames.length; i ++) {
            if (basename == forcefulBrowsing.unwantedFilenames[i]) {
                return {
                    action:     'block',
                    message:    '敏感文件下载: ' + params.function + ' 方式',
                    confidence: 100
                }
            }
        }
    }

    return clean
})

plugin.register('writeFile', function (params, context) {
    if (ntfsRegex.test(params.realpath)) {
        return {
            action:     'block',
            message:    '尝试利用NTFS流上传后门: ' + params.realpath,
            confidence: 90
        }
    }

    if (scriptFileRegex.test(params.realpath)) {
        return {
            action:     'log',
            message:    '尝试写入脚本文件: ' + params.realpath,
            confidence: 90
        }
    }
    return clean
})

plugin.register('fileUpload', function (params, context) {
    if (scriptFileRegex.test(params.filename) || ntfsRegex.test(params.filename)) {
        return {
            action:     'block',
            message:    '尝试上传脚本文件: ' + params.filename,
            confidence: 90
        }
    }
    return clean
})

plugin.register('sql', function (params, context) {
    var reason     = false
    var parameters = context.parameter
    var tokens     = RASP.sql_tokenize(params.query, params.server)

    // console.log(tokens)

    // 算法1: 匹配用户输入
    // 1. 简单识别逻辑是否发生改变
    // 2. 识别数据库管理器
    if (1) {
        Object.keys(parameters).some(function (name) {
            var value = parameters[name][0]

            // 请求参数长度超过10才考虑
            if (value.length <= 10) {
                return
            }

            // 判断是否为数据库管理器
            if (value.length == params.query.length && value == params.query) {
                reason = '算法2: WebShell - 数据库管理器'
                return true
            }

            // 简单识别用户输入
            if (params.query.indexOf(value) == -1) {
                return
            }

            // 去掉用户输入再次匹配
            var tokens2 = RASP.sql_tokenize(params.query.replace(value, ''), params.server)
            if (tokens.length - tokens2.length > 2) {
                reason = '算法1: 数据库查询逻辑发生改变'
                return true
            }
        })
        if (reason !== false) {
            return {
                'action':     'block',
                'confidence': 90,
                'message':    reason
            }
        }
    }

    // 算法2: SQL语句策略检查（模拟SQL防火墙功能）
    if (1) {
        var func_list = {
            'load_file': true,
            'benchmark': true,
            'sleep':     true,
            'pg_sleep':  true
        }
        var tokens_lc = tokens.map(v => v.toLowerCase())

        for (var i = 0; i < tokens_lc.length; i ++) {
            if (tokens_lc[i] == ';') {
                reason = '禁止多语句查询'
                break
            } else if (tokens_lc[i][0] === '0' && tokens_lc[i][1] === 'x') {
                reason = '禁止16进制字符串'
                break
            } else if (tokens_lc[i][0] === '/' && tokens_lc[i][1] === '*' && tokens_lc[i][2] === '!') {
                reason = '禁止MySQL版本号注释'
                break
            } else if (i > 0 && i < tokens_lc.length - 1 &&
                (tokens_lc[i] === 'xor'
                    || tokens_lc[i][0] === '<'
                    || tokens_lc[i][0] === '>'
                    || tokens_lc[i][0] === '=')) {
                // @FIXME: 可绕过，暂时不更新
                // 简单识别 NUMBER (>|<|>=|<=|xor) NUMBER
                //          i-1         i          i+2

                var op1    = tokens_lc[i - 1]
                var op2    = tokens_lc[i + 1]

                if (! isNaN(op1) && ! isNaN(op2)) {
                    reason = '禁止常量比较操作'
                    break
                }
            } else if (i > 0 && tokens_lc[i].indexOf('(') == 0) {
                // @FIXME: 可绕过，暂时不更新
                if (func_list[tokens_lc[i - 1]]) {
                    reason = '禁止执行敏感函数: ' + tokens_lc[i - 1]
                    break
                }
            }
        }

        if (reason !== false) {
            return {
                action:     'block',
                message:    '数据库语句异常: ' + reason + '（算法3）',
                confidence: 100
            }
        }
    }

    // 算法3: 简单正则匹配（即将移除）
    if (0) {
        var sqlRegex = /\bupdatexml\s*\(|\bextractvalue\s*\(|\bunion.*select.*(from|into|benchmark).*\b/i

        if (sqlRegex.test(params.query)) {
            return {
                action:     'block',
                message:    'SQL 注入攻击（算法4）',
                confidence: 100
            }
        }
    }

    return clean
})

plugin.register('command', function (params, context) {
    // 默认禁止命令执行
    return {
        action:    'block',
        message:   '尝试执行命令',
        confidence: 90
    }
})

plugin.register('xxe', function (params, context) {
    var items = params.entity.split('://')

    if (items.length >= 2) {
        var protocol = items[0]

        if (protocol === 'gopher' || protocol === 'dict' || protocol === 'expect') {
            return {
                action:     'block',
                message:    'SSRF 攻击 (' + protocol + ' 协议)',
                confidence: 100
            }
        }

        if (protocol === 'file') {
            return {
                action:     'log',
                message:    '尝试读取外部实体 (file 协议)',
                confidence: 90
            }
        }
    }
    return clean
})

plugin.register('ognl', function (params, context) {
    // 常见 struts payload 语句特征
    var ognlPayloads = [
        'ognl.OgnlContext',
        'ognl.TypeConverter',
        'ognl.MemberAccess',
        '_memberAccess',
        'ognl.ClassResolver',
        'java.lang.Runtime',
        'java.lang.Class',
        'java.lang.ClassLoader',
        'java.lang.System',
        'java.lang.ProcessBuilder',
        'java.lang.Object',
        'java.lang.Shutdown',
        'java.io.File',
        'javax.script.ScriptEngineManager',
        'com.opensymphony.xwork2.ActionContext'
    ]

    var ognlExpression = params.expression
    for (var index in ognlPayloads) {
        if (ognlExpression.indexOf(ognlPayloads[index]) > -1) {
            return {
                action:     'block',
                message:    '尝试ognl远程命令执行',
                confidence: 100
            }
        }

    }
    return clean
})

// [[ 近期调整~ ]]
plugin.register('deserialization', function (params, context) {
    var deserializationInvalidClazz = [
        'org.apache.commons.collections.functors.InvokerTransformer',
        'org.apache.commons.collections.functors.InstantiateTransformer',
        'org.apache.commons.collections4.functors.InvokerTransformer',
        'org.apache.commons.collections4.functors.InstantiateTransformer',
        'org.codehaus.groovy.runtime.ConvertedClosure',
        'org.codehaus.groovy.runtime.MethodClosure',
        'org.springframework.beans.factory.ObjectFactory',
        'xalan.internal.xsltc.trax.TemplatesImpl'
    ]

    var clazz = params.clazz
    for (var index in deserializationInvalidClazz) {
        if (clazz === deserializationInvalidClazz[index]) {
            return {
                action:     'block',
                message:    '尝试反序列化攻击',
                confidence: 100
            }
        }
    }
    return clean
})

plugin.register('reflection', function(params, context) {
    var title = '异常的执行流'
    var known = {
        'com.thoughtworks.xstream.XStream.unmarshal': 'xstream 反序列化攻击',
        'org.apache.commons.collections4.functors.InvokerTransformer.transform': 'transformer 反序列化攻击'
    }

    // 是否为已知类型的反序列化攻击？
    params.stack.some(function (method) {
        if (known[method]) {
            title = known[method]
            return true;
        }
    });

    return {
        action:     'block',
        message:    title + ': ' + params.clazz + '.' + params.method,
        confidence: 100
    }
})

plugin.log('初始化成功')
<|endoftext|>


<commit_before>////////////////////////////////////////////////////////////////////////////////
/// @brief test the collection interface
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2010-2012 triagens GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is triAGENS GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2012, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

var jsunity = require("jsunity");

var arangodb = require("org/arangodb");
var internal = require("internal");
var db = arangodb.db;

// -----------------------------------------------------------------------------
// --SECTION--                                                               any
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief test suite
////////////////////////////////////////////////////////////////////////////////

function AnySuite () {
  var cn = "example";
  var c;

  var threshold = 15; // maximum tolerated stddev for distribution

  var stddev = function (dist) {
    var v;
    var sum = 0;
    var count = 0;

    for (v in dist) {
      if (dist.hasOwnProperty(v)) {
        sum += dist[v];
        count++;
      }
    }

    var avg = sum / count;
    var sum2 = 0;

    for (v in dist) {
      if (dist.hasOwnProperty(v)) {
        var d = dist[v] - avg;
        sum2 += d * d;
      }
    }

    return Math.sqrt(sum2 / count);
  };

  var getDistribution = function (n, rng) {
    var dist = { };
    var i;

    for (i = 0; i < n; ++i) {
      var pick = rng();

      if (dist.hasOwnProperty(pick)) {
        dist[pick]++;
      }
      else {
        dist[pick] = 1;
      }
    }

    return dist;
  };

  return {

////////////////////////////////////////////////////////////////////////////////
/// @brief set up
////////////////////////////////////////////////////////////////////////////////

    setUp : function () {
      db._drop(cn);
      c = db._create(cn);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief tear down
////////////////////////////////////////////////////////////////////////////////

    tearDown : function () {
      db._drop(cn);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of Math.random()
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyNative : function () {
      var i, n;

      n = 100;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 100, function () {
        return parseInt(Math.random() * 100, 10);
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), just one document
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionOne : function () {
      c.save({ value: 1 });

      var dist = getDistribution(100, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) <= 0.01);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), few picks
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionFew1 : function () {
      var i, n;

      n = 3;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 200, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold * 3.0);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), few picks
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionFew2 : function () {
      var i, n;

      n = 10;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 100, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), more picks
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionMore : function () {
      var i, n;

      n = 500;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 100, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), with many documents deleted
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionHalf : function () {
      var i, n;

      n = 500;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      // remove 50 % of entries
      var d = Math.round(n * 0.5);
      for (i = 0; i < d; ++i) {
        c.remove(c.any());
      }

      var dist = getDistribution(n * 50, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), with most documents deleted
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionSparse : function () {
      var i, n;

      n = 500;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      // remove 50 % of entries
      var d = Math.round(n * 0.95);
      for (i = 0; i < d; ++i) {
        c.remove(c.any());
      }

      var dist = getDistribution(n * 5, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    }

  };
}

// -----------------------------------------------------------------------------
// --SECTION--                                                              main
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief executes the test suite
////////////////////////////////////////////////////////////////////////////////

jsunity.run(AnySuite);

return jsunity.done();

// Local Variables:
// mode: outline-minor
// outline-regexp: "^\\(/// @brief\\|/// @addtogroup\\|// --SECTION--\\|/// @page\\|/// @}\\)"
// End:

<commit_msg>increased test threshold<commit_after>////////////////////////////////////////////////////////////////////////////////
/// @brief test the collection interface
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2010-2012 triagens GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is triAGENS GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2012, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

var jsunity = require("jsunity");

var arangodb = require("org/arangodb");
var internal = require("internal");
var db = arangodb.db;

// -----------------------------------------------------------------------------
// --SECTION--                                                               any
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief test suite
////////////////////////////////////////////////////////////////////////////////

function AnySuite () {
  var cn = "example";
  var c;

  var threshold = 15; // maximum tolerated stddev for distribution

  var stddev = function (dist) {
    var v;
    var sum = 0;
    var count = 0;

    for (v in dist) {
      if (dist.hasOwnProperty(v)) {
        sum += dist[v];
        count++;
      }
    }

    var avg = sum / count;
    var sum2 = 0;

    for (v in dist) {
      if (dist.hasOwnProperty(v)) {
        var d = dist[v] - avg;
        sum2 += d * d;
      }
    }

    return Math.sqrt(sum2 / count);
  };

  var getDistribution = function (n, rng) {
    var dist = { };
    var i;

    for (i = 0; i < n; ++i) {
      var pick = rng();

      if (dist.hasOwnProperty(pick)) {
        dist[pick]++;
      }
      else {
        dist[pick] = 1;
      }
    }

    return dist;
  };

  return {

////////////////////////////////////////////////////////////////////////////////
/// @brief set up
////////////////////////////////////////////////////////////////////////////////

    setUp : function () {
      db._drop(cn);
      c = db._create(cn);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief tear down
////////////////////////////////////////////////////////////////////////////////

    tearDown : function () {
      db._drop(cn);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of Math.random()
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyNative : function () {
      var i, n;

      n = 100;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 100, function () {
        return parseInt(Math.random() * 100, 10);
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), just one document
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionOne : function () {
      c.save({ value: 1 });

      var dist = getDistribution(100, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) <= 0.01);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), few picks
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionFew1 : function () {
      var i, n;

      n = 3;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 200, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold * 3.0);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), few picks
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionFew2 : function () {
      var i, n;

      n = 10;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 100, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold * 1.5);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), more picks
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionMore : function () {
      var i, n;

      n = 500;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 100, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), with many documents deleted
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionHalf : function () {
      var i, n;

      n = 500;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      // remove 50 % of entries
      var d = Math.round(n * 0.5);
      for (i = 0; i < d; ++i) {
        c.remove(c.any());
      }

      var dist = getDistribution(n * 50, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), with most documents deleted
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionSparse : function () {
      var i, n;

      n = 500;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      // remove 50 % of entries
      var d = Math.round(n * 0.95);
      for (i = 0; i < d; ++i) {
        c.remove(c.any());
      }

      var dist = getDistribution(n * 5, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    }

  };
}

// -----------------------------------------------------------------------------
// --SECTION--                                                              main
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief executes the test suite
////////////////////////////////////////////////////////////////////////////////

jsunity.run(AnySuite);

return jsunity.done();

// Local Variables:
// mode: outline-minor
// outline-regexp: "^\\(/// @brief\\|/// @addtogroup\\|// --SECTION--\\|/// @page\\|/// @}\\)"
// End:

<|endoftext|>


<commit_before>/**
 * Markdown Plugin based on PPL(Page Plugin Loader).
 *
 * @author luics<luics.king@gmail.com>
 * @date 2012-10-20
 * @license MIT
 *
 * Feature:
 * Add contents for page changed from markdown formation.
 *
 * Usage:
 * You can define an option:
 * __markdown_ppl_opt = {
 *   contents: ['h2', 'h3', 'h4'],
 *   top: ['h2', 'h3', 'h4', 'h5', 'h6']
 * }
 */
;//safe in min js
(function () {
  console = window.console || {};
  console.log = console.log || function () {
  };
  function fm() {
    var len = arguments.length;
    if (len == 0) {
      return '';
    } else if (len == 1) {
      return arguments[0];
    }

    var res = arguments[0], i;
    for (i = 1; i < len; ++i) {
      var re = new RegExp('\\{' + (i - 1) + '\\}', 'g');
      res = res.replace(re, arguments[i]);
    }
    return res;
  }

  console.log('Markdown Plugin loaded');
  /**
   * PPL(Page Plugin Loader) entry
   * @param $ js lib reference, default is window.jQuery
   */
  window.__ppl_main = function ($) {
    console.log('Markdown Plugin', $);
    var opt = window.__markdown_ppl_opt || {},
      contents = opt.contents || ['h2', 'h3', 'h4'],
      top = opt.top || ['h2', 'h3', 'h4', 'h5', 'h6'];

    if (contents.length > 0) addContents($, contents);
    if (top.length > 0) addTop($, top);
  }

  /**
   * PPL(Page Plugin Loader) option
   */
  window.__ppl_main_opt = {
    lib:'jquery'
  }

  /**
   * Add Contents for page
   */
  function addContents($, contents) {
    console.log('addContents', contents);
    var body = document.getElementsByTagName('BODY');
    if (!body) return;

    var $body = $(body[0]),
      contentsStyle = [
        'position:fixed;right:1em;top:1em;',
        'padding:0.5em;min-width:120px;',
        'font-size:90%;line-height:18px;',
        'border:1px solid #aaa;background: #F9F9F9;'
      ].join(''),
      html = [],
      order = [];

    for (var i = 0; i < contents.length; ++i) {
      order[i] = 0;
    }

    function indexOf(tag) {
      for (var i = 0; i < contents.length && contents[i].toLowerCase() !== tag; ++i);
      return i;
    }

    $(contents.join(',')).each(function (i, obj) {
      var tag = obj.tagName.toLowerCase(),
        index = indexOf(tag),
        anchor = '__id_' + tag + Math.floor(9999999 * Math.random());

      // Clear low level order
      order[index]++;
      for (var j = index + 1; j < contents.length; ++j) {
        order[j] = 0;
      }

      // Add anchor
      $(obj).append(fm('', anchor));
      // Add contents item
      html.push(fm('
{1}
',
        index * 1.5, order.slice(0, index + 1).join('.') + ' ' + $(obj).text(), anchor));
    });

    var $contentsWrap = $(fm(['
',
        '
',
        'Contents hide',
        '
',
        '
{1}
',
        '
'].join(''), contentsStyle, html.join(''))).prependTo($body),
      $toggle = $contentsWrap.find('> :first').find('> :last'),
      $contents = $contentsWrap.find('> :last');
    console.log($contentsWrap, $toggle, $contents);

    $toggle.click(function () {
      $contents.slideToggle();
      $toggle.html($toggle.html() === 'show' ? 'hide' : 'show');
    });
  }

  /**
   * Add "to Top" link for mark h
   */
  function addTop($, top) {
    console.log('addTop', top);

    $(top.join(',')).each(function (i, obj) {
      //$(obj).append(' Top');
      $(obj).prepend(['
',
        '',
        '✦',
        '
'].join(''));

      var $prefix = $(this).find(':first').find(':first');
      //var $top = $(this).find('a:last');
      //console.log($prefix, $top);
      var rawCol = $(obj).css('background-color');
      $(obj).mouseover(
        function () {
          $prefix.css('height', $(this).css('height'));
          $prefix.css('line-height', $(this).css('line-height'));
          $prefix.css('visibility', 'visible');
          //$top.show();
          $(this).css('background-color', '#FFF8D7');
        }).mouseout(function () {
          $prefix.css('visibility', 'hidden');
          //$top.hide();
          $(this).css('background-color', rawCol);
        });
    });
  }

  /**
   * Load ppl.js
   */
  document.write(unescape("%3Cscript src='https://raw.github.com/luics/markdown-util/master/lib/tpl/ppl.js' type='text/javascript'%3E%3C/script%3E"));
})();
<commit_msg>+real hash name<commit_after>/**
 * Markdown Plugin based on PPL(Page Plugin Loader).
 *
 * @author luics<luics.king@gmail.com>
 * @date 2012-10-20
 * @license MIT
 *
 * Feature:
 * Add contents for page changed from markdown formation.
 *
 * Usage:
 * You can define an option:
 * __markdown_ppl_opt = {
 *   contents: ['h2', 'h3', 'h4'],
 *   top: ['h2', 'h3', 'h4', 'h5', 'h6']
 * }
 */
;//safe in min js
(function () {
  console = window.console || {};
  console.log = console.log || function () {
  };
  function fm() {
    var len = arguments.length;
    if (len == 0) {
      return '';
    } else if (len == 1) {
      return arguments[0];
    }

    var res = arguments[0], i;
    for (i = 1; i < len; ++i) {
      var re = new RegExp('\\{' + (i - 1) + '\\}', 'g');
      res = res.replace(re, arguments[i]);
    }
    return res;
  }

  console.log('Markdown Plugin loaded');
  /**
   * PPL(Page Plugin Loader) entry
   * @param $ js lib reference, default is window.jQuery
   */
  window.__ppl_main = function ($) {
    console.log('Markdown Plugin', $);
    var opt = window.__markdown_ppl_opt || {},
      contents = opt.contents || ['h2', 'h3', 'h4'],
      top = opt.top || ['h2', 'h3', 'h4', 'h5', 'h6'];

    if (contents.length > 0) addContents($, contents);
    if (top.length > 0) addTop($, top);
  }

  /**
   * PPL(Page Plugin Loader) option
   */
  window.__ppl_main_opt = {
    lib:'jquery'
  }

  /**
   * Add Contents for page
   */
  function addContents($, contents) {
    console.log('addContents', contents);
    var body = document.getElementsByTagName('BODY');
    if (!body) return;

    var $body = $(body[0]),
      contentsStyle = [
        'position:fixed;right:1em;top:1em;',
        'padding:0.5em;min-width:120px;',
        'font-size:90%;line-height:18px;',
        'border:1px solid #aaa;background: #F9F9F9;'
      ].join(''),
      html = [],
      order = [],
      hash = [];

    for (var i = 0; i < contents.length; ++i) {
      order[i] = 0;
      hash[i] = '';
    }

    function indexOf(tag) {
      for (var i = 0; i < contents.length && contents[i].toLowerCase() !== tag; ++i);
      return i;
    }

    $(contents.join(',')).each(function (i, obj) {
      var index = indexOf(obj.tagName.toLowerCase());
      order[index]++;
      hash[index] = $(obj).text();
      for (var j = index + 1; j < contents.length; ++j) {
        // Clear low level order
        order[j] = 0;
        hash[j] = '';
      }
      var anchor = hash.slice(0, index + 1).join('-');
      //anchor = '__id_' + tag + Math.floor(9999999 * Math.random());

      // Add anchor
      $(obj).append(fm('', anchor));
      // Add contents item
      html.push(fm('
{1}
',
        index * 1.5, order.slice(0, index + 1).join('.') + ' ' + hash[index], anchor));
    });

    var $contentsWrap = $(fm(['
',
        '
',
        'Contents hide',
        '
',
        '
{1}
',
        '
'].join(''), contentsStyle, html.join(''))).prependTo($body),
      $toggle = $contentsWrap.find('> :first').find('> :last'),
      $contents = $contentsWrap.find('> :last');
    console.log($contentsWrap, $toggle, $contents);

    $toggle.click(function () {
      $contents.slideToggle();
      $toggle.html($toggle.html() === 'show' ? 'hide' : 'show');
    });
  }

  /**
   * Add "to Top" link for mark h
   */
  function addTop($, top) {
    console.log('addTop', top);

    $(top.join(',')).each(function (i, obj) {
      //$(obj).append(' Top');
      $(obj).prepend(['
',
        '',
        '✦',
        '
'].join(''));

      var $prefix = $(this).find(':first').find(':first');
      //var $top = $(this).find('a:last');
      //console.log($prefix, $top);
      var rawCol = $(obj).css('background-color');
      $(obj).mouseover(
        function () {
          $prefix.css('height', $(this).css('height'));
          $prefix.css('line-height', $(this).css('line-height'));
          $prefix.css('visibility', 'visible');
          $(this).css('background-color', '#FFF8D7');
        }).mouseout(function () {
          $prefix.css('visibility', 'hidden');
          $(this).css('background-color', rawCol);
        });
    });
  }

  /**
   * Load ppl.js
   */
  document.write(unescape("%3Cscript src='http://raw.github.com/luics/markdown-util/master/lib/tpl/ppl.js' type='text/javascript'%3E%3C/script%3E"));
})();
<|endoftext|>


<commit_before>////////////////////////////////////////////////////////////////////////////////
/// @brief test the collection interface
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2010-2012 triagens GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is triAGENS GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2012, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

var jsunity = require("jsunity");

var arangodb = require("org/arangodb");
var internal = require("internal");
var db = arangodb.db;

// -----------------------------------------------------------------------------
// --SECTION--                                                               any
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief test suite
////////////////////////////////////////////////////////////////////////////////

function AnySuite () {
  var cn = "example";
  var c;

  var threshold = 15; // maximum tolerated stddev for distribution

  var stddev = function (dist) {
    var v;
    var sum = 0;
    var count = 0;

    for (v in dist) {
      if (dist.hasOwnProperty(v)) {
        sum += dist[v];
        count++;
      }
    }

    var avg = sum / count;
    var sum2 = 0;

    for (v in dist) {
      if (dist.hasOwnProperty(v)) {
        var d = dist[v] - avg;
        sum2 += d * d;
      }
    }

    return Math.sqrt(sum2 / count);
  };

  var getDistribution = function (n, rng) {
    var dist = { };
    var i;

    for (i = 0; i < n; ++i) {
      var pick = rng();

      if (dist.hasOwnProperty(pick)) {
        dist[pick]++;
      }
      else {
        dist[pick] = 1;
      }
    }

    return dist;
  };

  return {

////////////////////////////////////////////////////////////////////////////////
/// @brief set up
////////////////////////////////////////////////////////////////////////////////

    setUp : function () {
      db._drop(cn);
      c = db._create(cn);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief tear down
////////////////////////////////////////////////////////////////////////////////

    tearDown : function () {
      db._drop(cn);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of Math.random()
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyNative : function () {
      var i, n;

      n = 100;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 100, function () {
        return parseInt(Math.random() * 100, 10);
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), just one document
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionOne : function () {
      c.save({ value: 1 });

      var dist = getDistribution(100, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) <= 0.01);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), few picks
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionFew1 : function () {
      var i, n;

      n = 3;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 200, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold * 1.5);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), few picks
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionFew2 : function () {
      var i, n;

      n = 10;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 100, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), more picks
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionMore : function () {
      var i, n;

      n = 500;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 100, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), with many documents deleted
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionHalf : function () {
      var i, n;

      n = 500;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      // remove 50 % of entries
      var d = Math.round(n * 0.5);
      for (i = 0; i < d; ++i) {
        c.remove(c.any());
      }

      var dist = getDistribution(n * 50, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), with most documents deleted
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionSparse : function () {
      var i, n;

      n = 500;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      // remove 50 % of entries
      var d = Math.round(n * 0.95);
      for (i = 0; i < d; ++i) {
        c.remove(c.any());
      }

      var dist = getDistribution(n * 5, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    }

  };
}

// -----------------------------------------------------------------------------
// --SECTION--                                                              main
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief executes the test suite
////////////////////////////////////////////////////////////////////////////////

jsunity.run(AnySuite);

return jsunity.done();

// Local Variables:
// mode: outline-minor
// outline-regexp: "^\\(/// @brief\\|/// @addtogroup\\|// --SECTION--\\|/// @page\\|/// @}\\)"
// End:

<commit_msg>greater tolerance for stddev dist<commit_after>////////////////////////////////////////////////////////////////////////////////
/// @brief test the collection interface
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2010-2012 triagens GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is triAGENS GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2012, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

var jsunity = require("jsunity");

var arangodb = require("org/arangodb");
var internal = require("internal");
var db = arangodb.db;

// -----------------------------------------------------------------------------
// --SECTION--                                                               any
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief test suite
////////////////////////////////////////////////////////////////////////////////

function AnySuite () {
  var cn = "example";
  var c;

  var threshold = 15; // maximum tolerated stddev for distribution

  var stddev = function (dist) {
    var v;
    var sum = 0;
    var count = 0;

    for (v in dist) {
      if (dist.hasOwnProperty(v)) {
        sum += dist[v];
        count++;
      }
    }

    var avg = sum / count;
    var sum2 = 0;

    for (v in dist) {
      if (dist.hasOwnProperty(v)) {
        var d = dist[v] - avg;
        sum2 += d * d;
      }
    }

    return Math.sqrt(sum2 / count);
  };

  var getDistribution = function (n, rng) {
    var dist = { };
    var i;

    for (i = 0; i < n; ++i) {
      var pick = rng();

      if (dist.hasOwnProperty(pick)) {
        dist[pick]++;
      }
      else {
        dist[pick] = 1;
      }
    }

    return dist;
  };

  return {

////////////////////////////////////////////////////////////////////////////////
/// @brief set up
////////////////////////////////////////////////////////////////////////////////

    setUp : function () {
      db._drop(cn);
      c = db._create(cn);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief tear down
////////////////////////////////////////////////////////////////////////////////

    tearDown : function () {
      db._drop(cn);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of Math.random()
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyNative : function () {
      var i, n;

      n = 100;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 100, function () {
        return parseInt(Math.random() * 100, 10);
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), just one document
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionOne : function () {
      c.save({ value: 1 });

      var dist = getDistribution(100, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) <= 0.01);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), few picks
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionFew1 : function () {
      var i, n;

      n = 3;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 200, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold * 2.0);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), few picks
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionFew2 : function () {
      var i, n;

      n = 10;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 100, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), more picks
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionMore : function () {
      var i, n;

      n = 500;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      var dist = getDistribution(n * 100, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), with many documents deleted
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionHalf : function () {
      var i, n;

      n = 500;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      // remove 50 % of entries
      var d = Math.round(n * 0.5);
      for (i = 0; i < d; ++i) {
        c.remove(c.any());
      }

      var dist = getDistribution(n * 50, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief check entropy of any(), with most documents deleted
////////////////////////////////////////////////////////////////////////////////

    testCheckEntropyCollectionSparse : function () {
      var i, n;

      n = 500;

      for (i = 0; i < n; ++i) {
        c.save({ value: i });
      }

      // remove 50 % of entries
      var d = Math.round(n * 0.95);
      for (i = 0; i < d; ++i) {
        c.remove(c.any());
      }

      var dist = getDistribution(n * 5, function () {
        return c.any().value;
      });

      assertTrue(stddev(dist) < threshold);
    }

  };
}

// -----------------------------------------------------------------------------
// --SECTION--                                                              main
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief executes the test suite
////////////////////////////////////////////////////////////////////////////////

jsunity.run(AnySuite);

return jsunity.done();

// Local Variables:
// mode: outline-minor
// outline-regexp: "^\\(/// @brief\\|/// @addtogroup\\|// --SECTION--\\|/// @page\\|/// @}\\)"
// End:

<|endoftext|>


<commit_before>"use strict";

/**
 * Uses a SOAP endpoint in the cloud API to continuously poll for
 * available updates.
 *
 * The SOAP endpoint (`/ethComet.asmx`) is undocumented, but is the
 * one used by the [client web application](http://wirelesstag.net/media/mytaglist.com/apidoc.html).
 * The difference to the client web application is that this
 * implementation uses a proper SOAP API interface (in contrast to
 * hand-building the XML to be transmitted, and to parsing the
 * returned XML with a regex), and that it calls a different method at
 * that endpoint.
 *
 * This updater can operate in two modes. In the default mode, only those
 * tag objects previously registered are updated, and all other data
 * updates returned by the polling API are ignored. In discovery mode,
 * data updates for which no matching tag object is registered will
 * result in new tag objects to be created on the fly, and for each of
 * these a `data` event is emitted by the updater (instead of by the
 * registered tag).
 *
 * This updater should receive updates resulting from armed sensors
 * going below or above their configured thresholds, or detecting
 * motion. The one caveat is that there is a short wait time (see
 * [UPDATE_LOOP_WAIT]{@link module:plugins/polling-updater~UPDATE_LOOP_WAIT}),
 * and an exponentially increasing wait time after errors (see
 * [WAIT_AFTER_ERROR]{@link module:plugins/polling-updater~WAIT_AFTER_ERROR})
 * until the next poll is issued. Updates falling into this time
 * period will only be caught at the next regular update interval
 * configured for a tag.
 *
 * @module
 */

var request = require('request'),
    soap = require('soap'),
    util = require('util'),
    EventEmitter = require('events');

/**
 * @const {string} - the path (relative to `API_BASE_URI`) of the WSDL
 *                   endpoint description for polling
 * @default
 */
const WSDL_URL_PATH = "/ethComet.asmx?WSDL";
/**
 * @const {string} - the base URI of the polling API endpoint
 * @default
 */
const API_BASE_URI = "https://www.mytaglist.com";
/**
 * @const {number} - the time to wait between subsequent calls of the
 *                   polling endpoint (in milliseconds)
 * @default
 */
const UPDATE_LOOP_WAIT = 10;
/**
 * @const {number} - the minimum time to wait between subsequent calls of the
 *                   polling endpoint after an error occurred (in milliseconds)
 * @default
 */
const WAIT_AFTER_ERROR = 1000;
/**
 * @const {number} - the maximum time to wait between subsequent calls of the
 *                   polling endpoint (in milliseconds)
 * @default
 */
const MAX_UPDATE_LOOP_WAIT = 30 * 60 * 1000;

/**
 * Creates the updater instance.
 *
 * @param {WirelessTagPlatform} [platform] - Platform object through
 *               which tags to be updated were (or are going to be)
 *               found (or created in discovery mode). In normal mode,
 *               this is only used to possibly override configuration
 *               options, specifically the base URI for the cloud API
 *               endpoints. For discovery mode to work, either this or
 *               `options.factory` must be provided.
 * @param {object} [options] - overriding configuration options
 * @param {string} [options.wsdl_url] - the full path for obtaining
 *               the WSDL for the SOAP service to be polled (default:
 *               [WSDL_URL_PATH]{@link module:plugins/polling-updater~WSDL_URL_PATH})
 * @param {string} [options.apiBaseURI] - the base URI to use for API
 *               endpoints (default: [API_BASE_URI]{@link
 *               module:plugins/polling-updater~API_BASE_URI})
 * @param {boolean} [options.discoveryMode] - whether to run in
 *               discovery mode or not. Defaults to `false`.
 * @param {WirelessTagPlatform~factory} [options.factory] - the tag
 *               and tag manager object factory to use in discovery
 *               mode. Either this, or the `platform` parameter must
 *               be provided for discovery mode to work.
 *
 * @constructor
 */
function PollingTagUpdater(platform, options) {
    EventEmitter.call(this);
    this.tagsByUUID = {};
    this.options = Object.assign({}, options);
    if (! this.options.wsdl_url) {
        let apiBaseURI;
        if (platform) apiBaseURI = platform.apiBaseURI;
        if (options && options.apiBaseURI) apiBaseURI = options.apiBaseURI;
        if (! apiBaseURI) apiBaseURI = API_BASE_URI;
        this.options.wsdl_url = apiBaseURI + WSDL_URL_PATH;
    }
    /**
     * @name discoveryMode
     * @type {boolean}
     * @memberof module:plugins/polling-updater~PollingTagUpdater#
     */
    Object.defineProperty(this, "discoveryMode", {
        enumerable: true,
        get: function() { return this.options.discoveryMode },
        set: function(v) { this.options.discoveryMode = v }
    });
    /** @member {WirelessTagPlatform} */
    this.platform = platform;
    /** @member {WirelessTagPlatform~factory} */
    this.factory = options.factory;
}
util.inherits(PollingTagUpdater, EventEmitter);

/**
 * Adds the given tag object(s) to the ones to be updated by this updater.
 *
 * Adding the same (determined by identity) object again has no
 * effect. However, an object that represents the same tag as one
 * already added (i.e., has the same `uuid` property value) will be
 * registered for updates, too.
 *
 * @param {(WirelessTag|WirelessTag[])} tags - the tag object(s) to
 *                                           be updated
 *
 * @return {module:plugins/polling-updater~PollingTagUpdater}
 */
PollingTagUpdater.prototype.addTags = function(tags) {
    if (!Array.isArray(tags)) tags = [tags];
    for (let tag of tags) {
        if (this.tagsByUUID[tag.uuid]) {
            this.tagsByUUID[tag.uuid].add(tag);
        } else {
            this.tagsByUUID[tag.uuid] = new Set([tag]);
        }
    }
    return this;
};

/**
 * Removes the given tags from the ones to be updated by this updater.
 *
 * Note that only the given object(s) will be removed. Specifically,
 * other tag objects with the same `uuid` property value, if
 * previously added, remain registered.
 *
 * @param {(WirelessTag|WirelessTag[])} tags - the tag object(s) to
 *                                           be removed from updating
 *
 * @return {module:plugins/polling-updater~PollingTagUpdater}
 */
PollingTagUpdater.prototype.removeTags = function(tags) {
    if (tags && !Array.isArray(tags)) tags = [tags];
    for (let tag of tags) {
        if (this.tagsByUUID[tag.uuid]) {
            this.tagsByUUID[tag.uuid].delete(tag);
        }
    }
    return this;
};

/**
 * Starts the continuous update loop. Registered tags will get updated
 * until they are removed, or [stopUpdateLoop()]{@link module:plugins/polling-updater~PollingTagUpdater#stopUpdateLoop}
 * is called.
 *
 * Has no effect if a continuous update loop is already running.
 *
 * @param {number} [waitTime] - the time to wait until scheduling the
 *                 next update; defaults to [UPDATE_LOOP_WAIT]{@link
 *                 module:plugins/polling-updater~UPDATE_LOOP_WAIT}.
 * @param {module:wirelesstags~apiCallback} [callback] - result.value
 *                 will be array of tag data objects returned by the
 *                 polling API
 */
PollingTagUpdater.prototype.startUpdateLoop = function(waitTime, callback) {
    if ('function' === typeof waitTime) {
        callback = waitTime;
        waitTime = undefined;
    }
    if (waitTime === undefined) waitTime = UPDATE_LOOP_WAIT;
    if (waitTime > MAX_UPDATE_LOOP_WAIT) waitTime = MAX_UPDATE_LOOP_WAIT;

    if (this._updateTimer) return this._updateTimer;
    this._updateTimer = true; // placeholder to avoid race conditions

    let action = () => {
        this._updateTimer = true; // timer is done but action not yet
        this.apiClient().then((client) => {
            // if all tags are associated with a single tag manager,
            // limit updates to that tag manager
            let mgrs = this.discoveryMode ? [] : this.uniqueTagManagers();
            return pollForNextUpdate(client,
                                     mgrs.length === 1 ? mgrs[0] : undefined,
                                     callback);
        }).then((tagDataList) => {
            const EMPTY = { size: () => 0 };
            let newTagProms = [];
            tagDataList.forEach((tagData) => {
                let tagObjs = this.tagsByUUID[tagData.uuid] || EMPTY;
                if (tagObjs.size > 0) {
                    tagObjs.forEach( (tag) => updateTag(tag, tagData) );
                } else if (this.discoveryMode) {
                    newTagProms.push(createTag(tagData,
                                               this.platform, this.factory));
                }
            });
            return Promise.all(newTagProms);
        }).then((newTagList) => {
            newTagList.forEach( (tag) => this.emit('data', tag) );
            // reset wait time upon success
            waitTime = undefined;
        }).catch((err) => {
            console.error(err.stack ? err.stack : err);
            waitTime = waitTime < WAIT_AFTER_ERROR ?
                WAIT_AFTER_ERROR : waitTime * 2;
        }).then(() => {
            // with the preceding catch() this is in essence a finally()
            if (this._updateTimer) {
                this._updateTimer = null;
                this.startUpdateLoop(waitTime, callback);
            }
            // otherwise we have been cancelled while running the update
        });
    };
    // ensure that updates weren't cancelled since we entered here
    if (this._updateTimer === true) {
        this._updateTimer = setTimeout(action, waitTime);
    }
    return this._updateTimer;
};

/**
 * Stops the continuous update loop. Has no effect if an update loop
 * is not currently active.
 */
PollingTagUpdater.prototype.stopUpdateLoop = function() {
    let timer = this._updateTimer;
    this._updateTimer = null;   // avoid race conditions
    if (timer && timer !== true) {
        clearTimeout(timer);
    }
};

/**
 * If necessary creates, and otherwise obtains from cache the SOAP
 * client instance for the WSDL endpoint.
 *
 * @returns {Promise} Resolves to the SOAP client object.
 */
PollingTagUpdater.prototype.apiClient = function() {
    if (this._client) return Promise.resolve(this._client);
    return createSoapClient(this.options).then((client) => {
        this._client = client;
        return client;
    });
};

/**
 * Determines the list of tag managers with unique MACs used by the
 * tag objects registered with this updater, and returns it.
 *
 * @returns {WirelessTagManager[]}
 */
PollingTagUpdater.prototype.uniqueTagManagers = function() {
    let mgrs = Object.keys(this.tagsByUUID).map((uuid) => {
        let tags = Array.from(this.tagsByUUID[uuid]);
        return tags.length > 0 ? tags[0].wirelessTagManager : undefined;
    });
    let mgrByMAC = new Map();
    mgrs.forEach( (mgr) => mgrByMAC.set(mgr.mac, mgr) );
    return Array.from(mgrByMAC.values());
};

const managerProps = ['managerName','mac','dbid','mirrors'];

/**
 * Updates the tag corresponding to the given tag data. Does nothing
 * if the respective tag is undefined or null.
 *
 * @param {WirelessTag} tag - the tag object to be updated
 * @param {object} tagData - the data to update the tag object with;
 *                 this is normally returned from the API endpoint
 *
 * @private
 */
function updateTag(tag, tagData) {
    // if not a valid object for receiving updates, we are done
    if (! tag) return;
    // check that this is the current tag manager
    if (tagData.mac && (tag.wirelessTagManager.mac !== tagData.mac)) {
        throw new Error("expected tag " + tag.uuid
                        + " to be with tag manager " + tag.mac
                        + " but is reported to be with " + tagData.mac);
    }
    // we don't currently have anything more to do for the extra properties
    // identifying the tag manager, so simply get rid of them
    managerProps.forEach( (k) => { delete tagData[k] } );
    // almost done
    tag.data = tagData;
}

/**
 * Creates a tag object from the given attribute data object, and returns it.
 *
 * @param {object} tagData - the attribute data object as returned by
 *             the polling API endpoint
 * @param {WirelessTagPlatform} platform - the platform instance for
 *             creating tag and tag manager objects. If the value does
 *             not provide a factory, `factory` must be provided.
 * @param {WirelessTagPlatform~factory} [factory] - the tag and tag
 *             manager object factory to use
 * @returns {WirelessTag}
 * @private
 */
function createTag(tagData, platform, factory) {
    let mgrData = {};
    managerProps.forEach((k) => {
        let mk = k.replace(/^manager([A-Z])/,'$1');
        if (mk !== k) mk = mk.toLowerCase();
        mgrData[mk] = tagData[k];
        delete tagData[k];
    });
    if (! platform) platform = {};
    if (! factory) factory = platform.factory;
    if (! factory) throw new TypeError("must have valid platform object or "
                                       + "object factory in discovery mode");
    let mgrProm = platform.findTagManager ?
        platform.findTagManager(mgrData.mac) :
        Promise.resolve(factory.createTagManager(mgrData));
    return mgrProm.then((mgr) => {
        if (! mgr) throw new Error("no such tag manager: " + mgrData.mac);
        return factory.createTag(mgr, tagData);
    });
}

/**
 * Creates the SOAP client, using the supplied options for locating
 * the WSDL document for the endpoint.
 *
 * @param {object} [opts] - WSDL and SOAP endpoint options
 * @param {string} [opts.wsdl_url] - the URL from which to fetch the
 *                 WSDL document; defaults to the concatenation of
 *                 [API_BASE_URI]{@link
 *                 module:plugins/polling-updater~API_BASE_URI} and
 *                 [WSDL_URL_PATH]{@link
 *                 module:plugins/polling-updater~WSDL_URL_PATH}
 *
 * @returns {Promise} On success, resolves to the created SOAP client object
 * @private
 */
function createSoapClient(opts) {
    let wsdl = opts && opts.wsdl_url ?
        opts.wsdl_url : API_BASE_URI + WSDL_URL_PATH;
    let clientOpts = { request: request.defaults({ jar: true, gzip: true }) };
    return new Promise((resolve, reject) => {
        soap.createClient(wsdl, clientOpts, (err, client) => {
            if (err) return reject(err);
            resolve(client);
        });
    });
}

/**
 * Polls the API endpoint for available updates and returns them.
 *
 * @param {object} client - the SOAP client object
 * @param {WirelessTagManager} [tagManager] - the tag manager to which
 *                             to restrict updates
 * @param {module:wirelesstags~apiCallback} [callback] - if provided,
 *                             the `tagManager` parameter must be
 *                             provided too (even if as undefined or
 *                             null)
 *
 * @returns {Promise} On success, resolves to an array of tag data objects
 * @private
 */
function pollForNextUpdate(client, tagManager, callback) {
    let req = new Promise((resolve, reject) => {
        let methodName = tagManager ?
            "GetNextUpdateForAllManagersOnDB" :
            "GetNextUpdateForAllManagers";
        let soapMethod = client[methodName];
        let args = {};
        if (tagManager) args.dbid = tagManager.dbid;
        soapMethod(args, function(err, result) {
            if (err) return reject(err);
            let tagDataList = JSON.parse(result[methodName + "Result"]);
            try {
                if (callback) callback(null, { object: tagManager,
                                               value: tagDataList});
            }
            catch(err) {
                console.error("error in callback:");
                console.error(err.stack ? err.stack : err);
                // no good reason to escalate an error thrown by callback
            }
            resolve(tagDataList);
        });
    });
    if (callback) req = req.catch((err) => { callback(err); throw err; });
    return req;
}

module.exports = PollingTagUpdater;
<commit_msg>Fixes bug in assigning factory property<commit_after>"use strict";

/**
 * Uses a SOAP endpoint in the cloud API to continuously poll for
 * available updates.
 *
 * The SOAP endpoint (`/ethComet.asmx`) is undocumented, but is the
 * one used by the [client web application](http://wirelesstag.net/media/mytaglist.com/apidoc.html).
 * The difference to the client web application is that this
 * implementation uses a proper SOAP API interface (in contrast to
 * hand-building the XML to be transmitted, and to parsing the
 * returned XML with a regex), and that it calls a different method at
 * that endpoint.
 *
 * This updater can operate in two modes. In the default mode, only those
 * tag objects previously registered are updated, and all other data
 * updates returned by the polling API are ignored. In discovery mode,
 * data updates for which no matching tag object is registered will
 * result in new tag objects to be created on the fly, and for each of
 * these a `data` event is emitted by the updater (instead of by the
 * registered tag).
 *
 * This updater should receive updates resulting from armed sensors
 * going below or above their configured thresholds, or detecting
 * motion. The one caveat is that there is a short wait time (see
 * [UPDATE_LOOP_WAIT]{@link module:plugins/polling-updater~UPDATE_LOOP_WAIT}),
 * and an exponentially increasing wait time after errors (see
 * [WAIT_AFTER_ERROR]{@link module:plugins/polling-updater~WAIT_AFTER_ERROR})
 * until the next poll is issued. Updates falling into this time
 * period will only be caught at the next regular update interval
 * configured for a tag.
 *
 * @module
 */

var request = require('request'),
    soap = require('soap'),
    util = require('util'),
    EventEmitter = require('events');

/**
 * @const {string} - the path (relative to `API_BASE_URI`) of the WSDL
 *                   endpoint description for polling
 * @default
 */
const WSDL_URL_PATH = "/ethComet.asmx?WSDL";
/**
 * @const {string} - the base URI of the polling API endpoint
 * @default
 */
const API_BASE_URI = "https://www.mytaglist.com";
/**
 * @const {number} - the time to wait between subsequent calls of the
 *                   polling endpoint (in milliseconds)
 * @default
 */
const UPDATE_LOOP_WAIT = 10;
/**
 * @const {number} - the minimum time to wait between subsequent calls of the
 *                   polling endpoint after an error occurred (in milliseconds)
 * @default
 */
const WAIT_AFTER_ERROR = 1000;
/**
 * @const {number} - the maximum time to wait between subsequent calls of the
 *                   polling endpoint (in milliseconds)
 * @default
 */
const MAX_UPDATE_LOOP_WAIT = 30 * 60 * 1000;

/**
 * Creates the updater instance.
 *
 * @param {WirelessTagPlatform} [platform] - Platform object through
 *               which tags to be updated were (or are going to be)
 *               found (or created in discovery mode). In normal mode,
 *               this is only used to possibly override configuration
 *               options, specifically the base URI for the cloud API
 *               endpoints. For discovery mode to work, either this or
 *               `options.factory` must be provided.
 * @param {object} [options] - overriding configuration options
 * @param {string} [options.wsdl_url] - the full path for obtaining
 *               the WSDL for the SOAP service to be polled (default:
 *               [WSDL_URL_PATH]{@link module:plugins/polling-updater~WSDL_URL_PATH})
 * @param {string} [options.apiBaseURI] - the base URI to use for API
 *               endpoints (default: [API_BASE_URI]{@link
 *               module:plugins/polling-updater~API_BASE_URI})
 * @param {boolean} [options.discoveryMode] - whether to run in
 *               discovery mode or not. Defaults to `false`.
 * @param {WirelessTagPlatform~factory} [options.factory] - the tag
 *               and tag manager object factory to use in discovery
 *               mode. Either this, or the `platform` parameter must
 *               be provided for discovery mode to work.
 *
 * @constructor
 */
function PollingTagUpdater(platform, options) {
    EventEmitter.call(this);
    this.tagsByUUID = {};
    this.options = Object.assign({}, options);
    if (! this.options.wsdl_url) {
        let apiBaseURI;
        if (platform) apiBaseURI = platform.apiBaseURI;
        if (options && options.apiBaseURI) apiBaseURI = options.apiBaseURI;
        if (! apiBaseURI) apiBaseURI = API_BASE_URI;
        this.options.wsdl_url = apiBaseURI + WSDL_URL_PATH;
    }
    /**
     * @name discoveryMode
     * @type {boolean}
     * @memberof module:plugins/polling-updater~PollingTagUpdater#
     */
    Object.defineProperty(this, "discoveryMode", {
        enumerable: true,
        get: function() { return this.options.discoveryMode },
        set: function(v) { this.options.discoveryMode = v }
    });
    /** @member {WirelessTagPlatform} */
    this.platform = platform;
    /** @member {WirelessTagPlatform~factory} */
    this.factory = this.options.factory;
}
util.inherits(PollingTagUpdater, EventEmitter);

/**
 * Adds the given tag object(s) to the ones to be updated by this updater.
 *
 * Adding the same (determined by identity) object again has no
 * effect. However, an object that represents the same tag as one
 * already added (i.e., has the same `uuid` property value) will be
 * registered for updates, too.
 *
 * @param {(WirelessTag|WirelessTag[])} tags - the tag object(s) to
 *                                           be updated
 *
 * @return {module:plugins/polling-updater~PollingTagUpdater}
 */
PollingTagUpdater.prototype.addTags = function(tags) {
    if (!Array.isArray(tags)) tags = [tags];
    for (let tag of tags) {
        if (this.tagsByUUID[tag.uuid]) {
            this.tagsByUUID[tag.uuid].add(tag);
        } else {
            this.tagsByUUID[tag.uuid] = new Set([tag]);
        }
    }
    return this;
};

/**
 * Removes the given tags from the ones to be updated by this updater.
 *
 * Note that only the given object(s) will be removed. Specifically,
 * other tag objects with the same `uuid` property value, if
 * previously added, remain registered.
 *
 * @param {(WirelessTag|WirelessTag[])} tags - the tag object(s) to
 *                                           be removed from updating
 *
 * @return {module:plugins/polling-updater~PollingTagUpdater}
 */
PollingTagUpdater.prototype.removeTags = function(tags) {
    if (tags && !Array.isArray(tags)) tags = [tags];
    for (let tag of tags) {
        if (this.tagsByUUID[tag.uuid]) {
            this.tagsByUUID[tag.uuid].delete(tag);
        }
    }
    return this;
};

/**
 * Starts the continuous update loop. Registered tags will get updated
 * until they are removed, or [stopUpdateLoop()]{@link module:plugins/polling-updater~PollingTagUpdater#stopUpdateLoop}
 * is called.
 *
 * Has no effect if a continuous update loop is already running.
 *
 * @param {number} [waitTime] - the time to wait until scheduling the
 *                 next update; defaults to [UPDATE_LOOP_WAIT]{@link
 *                 module:plugins/polling-updater~UPDATE_LOOP_WAIT}.
 * @param {module:wirelesstags~apiCallback} [callback] - result.value
 *                 will be array of tag data objects returned by the
 *                 polling API
 */
PollingTagUpdater.prototype.startUpdateLoop = function(waitTime, callback) {
    if ('function' === typeof waitTime) {
        callback = waitTime;
        waitTime = undefined;
    }
    if (waitTime === undefined) waitTime = UPDATE_LOOP_WAIT;
    if (waitTime > MAX_UPDATE_LOOP_WAIT) waitTime = MAX_UPDATE_LOOP_WAIT;

    if (this._updateTimer) return this._updateTimer;
    this._updateTimer = true; // placeholder to avoid race conditions

    let action = () => {
        this._updateTimer = true; // timer is done but action not yet
        this.apiClient().then((client) => {
            // if all tags are associated with a single tag manager,
            // limit updates to that tag manager
            let mgrs = this.discoveryMode ? [] : this.uniqueTagManagers();
            return pollForNextUpdate(client,
                                     mgrs.length === 1 ? mgrs[0] : undefined,
                                     callback);
        }).then((tagDataList) => {
            const EMPTY = { size: () => 0 };
            let newTagProms = [];
            tagDataList.forEach((tagData) => {
                let tagObjs = this.tagsByUUID[tagData.uuid] || EMPTY;
                if (tagObjs.size > 0) {
                    tagObjs.forEach( (tag) => updateTag(tag, tagData) );
                } else if (this.discoveryMode) {
                    newTagProms.push(createTag(tagData,
                                               this.platform, this.factory));
                }
            });
            return Promise.all(newTagProms);
        }).then((newTagList) => {
            newTagList.forEach( (tag) => this.emit('data', tag) );
            // reset wait time upon success
            waitTime = undefined;
        }).catch((err) => {
            console.error(err.stack ? err.stack : err);
            waitTime = waitTime < WAIT_AFTER_ERROR ?
                WAIT_AFTER_ERROR : waitTime * 2;
        }).then(() => {
            // with the preceding catch() this is in essence a finally()
            if (this._updateTimer) {
                this._updateTimer = null;
                this.startUpdateLoop(waitTime, callback);
            }
            // otherwise we have been cancelled while running the update
        });
    };
    // ensure that updates weren't cancelled since we entered here
    if (this._updateTimer === true) {
        this._updateTimer = setTimeout(action, waitTime);
    }
    return this._updateTimer;
};

/**
 * Stops the continuous update loop. Has no effect if an update loop
 * is not currently active.
 */
PollingTagUpdater.prototype.stopUpdateLoop = function() {
    let timer = this._updateTimer;
    this._updateTimer = null;   // avoid race conditions
    if (timer && timer !== true) {
        clearTimeout(timer);
    }
};

/**
 * If necessary creates, and otherwise obtains from cache the SOAP
 * client instance for the WSDL endpoint.
 *
 * @returns {Promise} Resolves to the SOAP client object.
 */
PollingTagUpdater.prototype.apiClient = function() {
    if (this._client) return Promise.resolve(this._client);
    return createSoapClient(this.options).then((client) => {
        this._client = client;
        return client;
    });
};

/**
 * Determines the list of tag managers with unique MACs used by the
 * tag objects registered with this updater, and returns it.
 *
 * @returns {WirelessTagManager[]}
 */
PollingTagUpdater.prototype.uniqueTagManagers = function() {
    let mgrs = Object.keys(this.tagsByUUID).map((uuid) => {
        let tags = Array.from(this.tagsByUUID[uuid]);
        return tags.length > 0 ? tags[0].wirelessTagManager : undefined;
    });
    let mgrByMAC = new Map();
    mgrs.forEach( (mgr) => mgrByMAC.set(mgr.mac, mgr) );
    return Array.from(mgrByMAC.values());
};

const managerProps = ['managerName','mac','dbid','mirrors'];

/**
 * Updates the tag corresponding to the given tag data. Does nothing
 * if the respective tag is undefined or null.
 *
 * @param {WirelessTag} tag - the tag object to be updated
 * @param {object} tagData - the data to update the tag object with;
 *                 this is normally returned from the API endpoint
 *
 * @private
 */
function updateTag(tag, tagData) {
    // if not a valid object for receiving updates, we are done
    if (! tag) return;
    // check that this is the current tag manager
    if (tagData.mac && (tag.wirelessTagManager.mac !== tagData.mac)) {
        throw new Error("expected tag " + tag.uuid
                        + " to be with tag manager " + tag.mac
                        + " but is reported to be with " + tagData.mac);
    }
    // we don't currently have anything more to do for the extra properties
    // identifying the tag manager, so simply get rid of them
    managerProps.forEach( (k) => { delete tagData[k] } );
    // almost done
    tag.data = tagData;
}

/**
 * Creates a tag object from the given attribute data object, and returns it.
 *
 * @param {object} tagData - the attribute data object as returned by
 *             the polling API endpoint
 * @param {WirelessTagPlatform} platform - the platform instance for
 *             creating tag and tag manager objects. If the value does
 *             not provide a factory, `factory` must be provided.
 * @param {WirelessTagPlatform~factory} [factory] - the tag and tag
 *             manager object factory to use
 * @returns {WirelessTag}
 * @private
 */
function createTag(tagData, platform, factory) {
    let mgrData = {};
    managerProps.forEach((k) => {
        let mk = k.replace(/^manager([A-Z])/,'$1');
        if (mk !== k) mk = mk.toLowerCase();
        mgrData[mk] = tagData[k];
        delete tagData[k];
    });
    if (! platform) platform = {};
    if (! factory) factory = platform.factory;
    if (! factory) throw new TypeError("must have valid platform object or "
                                       + "object factory in discovery mode");
    let mgrProm = platform.findTagManager ?
        platform.findTagManager(mgrData.mac) :
        Promise.resolve(factory.createTagManager(mgrData));
    return mgrProm.then((mgr) => {
        if (! mgr) throw new Error("no such tag manager: " + mgrData.mac);
        return factory.createTag(mgr, tagData);
    });
}

/**
 * Creates the SOAP client, using the supplied options for locating
 * the WSDL document for the endpoint.
 *
 * @param {object} [opts] - WSDL and SOAP endpoint options
 * @param {string} [opts.wsdl_url] - the URL from which to fetch the
 *                 WSDL document; defaults to the concatenation of
 *                 [API_BASE_URI]{@link
 *                 module:plugins/polling-updater~API_BASE_URI} and
 *                 [WSDL_URL_PATH]{@link
 *                 module:plugins/polling-updater~WSDL_URL_PATH}
 *
 * @returns {Promise} On success, resolves to the created SOAP client object
 * @private
 */
function createSoapClient(opts) {
    let wsdl = opts && opts.wsdl_url ?
        opts.wsdl_url : API_BASE_URI + WSDL_URL_PATH;
    let clientOpts = { request: request.defaults({ jar: true, gzip: true }) };
    return new Promise((resolve, reject) => {
        soap.createClient(wsdl, clientOpts, (err, client) => {
            if (err) return reject(err);
            resolve(client);
        });
    });
}

/**
 * Polls the API endpoint for available updates and returns them.
 *
 * @param {object} client - the SOAP client object
 * @param {WirelessTagManager} [tagManager] - the tag manager to which
 *                             to restrict updates
 * @param {module:wirelesstags~apiCallback} [callback] - if provided,
 *                             the `tagManager` parameter must be
 *                             provided too (even if as undefined or
 *                             null)
 *
 * @returns {Promise} On success, resolves to an array of tag data objects
 * @private
 */
function pollForNextUpdate(client, tagManager, callback) {
    let req = new Promise((resolve, reject) => {
        let methodName = tagManager ?
            "GetNextUpdateForAllManagersOnDB" :
            "GetNextUpdateForAllManagers";
        let soapMethod = client[methodName];
        let args = {};
        if (tagManager) args.dbid = tagManager.dbid;
        soapMethod(args, function(err, result) {
            if (err) return reject(err);
            let tagDataList = JSON.parse(result[methodName + "Result"]);
            try {
                if (callback) callback(null, { object: tagManager,
                                               value: tagDataList});
            }
            catch(err) {
                console.error("error in callback:");
                console.error(err.stack ? err.stack : err);
                // no good reason to escalate an error thrown by callback
            }
            resolve(tagDataList);
        });
    });
    if (callback) req = req.catch((err) => { callback(err); throw err; });
    return req;
}

module.exports = PollingTagUpdater;
<|endoftext|>