<commit_before>// takes an IR function object and returns a list of Scratch blocks

module.exports.ffi = {};

module.exports.generateFunctionHat = function(functionContext, func) {
	var spec = func.funcName;
	var inputs = [];
	var defaults = [];

	functionContext.params = [];

	for(var i = 0; i < func.paramList.length; ++i) {
		var pName = "param" + i;

		if(func.paramList[i][1])
			pName = func.paramList[i][1];

		inputs.push(pName);
		functionContext.params.push(pName);

		defaults.push(defaultForType(func.paramList[i][0]));
		spec += " "+specifierForType(func.paramList[i][0]);
	}

	return ["procDef", spec, inputs, defaults, false];

}

module.exports.compileFunction = function(func) {
	console.log("Compiling "+JSON.stringify(func)+"...");

	var functionContext = {
		locals: {},
		localDepth: 0,
		params: [],
		gotoInit: false
	}

	var blockList = [module.exports.generateFunctionHat(functionContext, func)]
					.concat(initLocal());

	if(func.inGotoComplex) {
		blockList = blockList.concat(initGotoComplex());
	}

	for(var i = 0; i < func.code.length; ++i) {
		console.log(func.code[i]);

		var hasGotoComplex = functionContext.gotoComplex && functionContext.gotoComplex.okToUse && functionContext.gotoComplex.active; // this MUST be before compileInstruction for branching to work
		var instruction = compileInstruction(functionContext, func.code[i]);

		if(!functionContext.gotoInit && functionContext.gotoComplex && functionContext.gotoComplex.okToUse) {
			blockList = blockList.concat([functionContext.gotoComplex.forever]);
			functionContext.gotoInit = true;
		}
		if(hasGotoComplex) {
			if(functionContext.gotoComplex.currentContext[2]) {
				functionContext.gotoComplex.currentContext[2] =
					functionContext.gotoComplex.currentContext[2].concat(instruction);
			} else {
				functionContext.gotoComplex.currentContext[2] = instruction;
			}
		} else {
			blockList = blockList.concat(instruction);
		}
	}

	//blockList = blockList.concat(returnBlock());

	return blockList;
}

function compileInstruction(ctx, block) {
	if(block.type == "call") {
		// calling a (potentially foreign) function
		return callBlock(ctx, block);
	} else if(block.type == "ffi") {
		// FFI block
		// load the code from the options
		return module.exports.ffi[block.ffiBlock];
	} else if(block.type == "set") {
		var val = 0;

		console.log("SET: "+JSON.stringify(block));

		if(block.val.type == "return value") {
			val = ["readVariable", "return value"];
		} else if(block.val.type == "variable") {
			val = fetchByName(ctx, block.val.name);
		} else if(block.val.type == "arithmetic") {
			val = [block.val.operation, fetchByName(block.val.operand1), fetchByName(ctx, block.val.operand2)];
		} else if(block.val.type == "comparison") {
			val = castToNumber([specForComparison(block.val.operation), fetchByName(ctx, block.val.left), fetchByName(ctx, block.val.right)]);
		}

		return compileInstruction(ctx, block.computation)
				.concat(allocateLocal(ctx, val, block.name));
	} else if(block.type == "ret") {
		return returnBlock(ctx, block.value);
	} else if(block.type == "store") {
		return dereferenceAndSet(ctx, block.destination.value, block.src.value);
	} else if(block.type == "gotoComplex") {
		ctx.gotoComplex = {
			context: [],
			okToUse: false,
			forever: ["doForever", []],
			active: true
		}

		//return [ctx.gotoComplex.forever];
	} else if(block.type == "label") {
		var chunk = ["doIfElse", ["=", getCurrentLabel(), block.label], [], []];

		ctx.gotoComplex.okToUse = true;
		ctx.gotoComplex.active = true;

		if(ctx.gotoComplex.currentContext) {
			ctx.gotoComplex.currentContext[3] = [chunk];
			ctx.gotoComplex.currentContext = ctx.gotoComplex.currentContext[3][0];
		} else {
			ctx.gotoComplex.currentContext = chunk;
			ctx.gotoComplex.context = ctx.gotoComplex.currentContext;
			ctx.gotoComplex.forever[1] = [ctx.gotoComplex.context];
		}

	} else if(block.type == "branch") {
		ctx.gotoComplex.active = false;

		if(block.conditional) {
			return [
				["doIfElse", ["=", fetchByName(ctx, block.condition), 1], absoluteBranch(block.dest.slice(1)), absoluteBranch(block.falseDest.slice(1))]
			];
		} else {
			return absoluteBranch(block.dest);
		}
	}

	return [];
}

// fixme: stub
function defaultForType(type) {
	console.log(type);
	return 0;
}

// fixme: stub
function specifierForType(type) {
	return "%s";
}

// fixme: stub
function formatValue(ctx, type, value) {
	console.log("FORMAT: "+type+","+value);

	if(value[0] == '%') {
		return fetchByName(ctx, value);
	}

	return value;
}

function getOffset(ctx, value) {
	return ctx.localDepth - ctx.locals[value];
}

function stackPosFromOffset(offset) {
	return ["-", ["lineCountOfList:", "Stack"], offset];
}

// higher-level code generation
function initLocal() {
	return [
		["append:toList:", "0", "# of locals"]
	]
}

function allocateLocal(ctx, val, name) {
	if(name) {
		console.log(name+","+val);
		ctx.locals[name] = ++ctx.localDepth;
	}

	return [
		["setLine:ofList:to:", "last", "# of locals", ["+", ["getLine:ofList:", "last", "# of locals"], 1]],
		["append:toList:", val, "Stack"]
	];
}

function freeLocals() {
	return [
		["doRepeat", ["getLine:ofList:", "last", "# of locals"], [["deleteLine:ofList:", "last", "Stack"]]],
		["deleteLine:ofList:", "last", "# of locals"]
	];
}

function fetchByName(ctx, n) {
	if(ctx.locals[n])
		return ["getLine:ofList:", stackPosFromOffset(getOffset(ctx, n)), "Stack"];
	else if(ctx.params.indexOf(n) > -1)
		return ["getParam", n, "r"];
	else if( (n * 1) == n)
		return n
	else
		return 0;
}

function returnBlock(ctx, val) {
	var proc = freeLocals();

	if(ctx.gotoComplex) {
		proc = proc.concat(cleanGotoComplex());
	}

	if(val) {
		proc.push(["setVar:to:", "return value", formatValue(ctx, val[0], val[1])]);
	}

	proc.push(["stopScripts", "this script"]);

	return proc;
}

function callBlock(ctx, block) {
	var spec = block.funcName;
	var args = [];

	for(var a = 0; a < block.paramList.length; ++a) {
		args.push(formatValue(ctx, block.paramList[a][0], block.paramList[a][1]));
		spec += " "+specifierForType(block.paramList[a][0]);
	}

	return [
		["call", spec].concat(args)
	];
}

// TODO: more robust implementation to support heap

function dereferenceAndSet(ctx, ptr, content) {
	return [
		[
			"setLine:ofList:to:",
			stackPosFromOffset(getOffset(ctx, ptr)),
			"Stack",
			fetchByName(ctx, content)
		]
	];
}

function specForComparison(comp) {
	if(comp == "eq") {
		return "=";
	}
	return "undefined";
}

function initGotoComplex() {
	return [
		["append:toList:", 0, "Label Stack"]
	];
}

function getCurrentLabel() {
	return ["getLine:ofList:", "last", "Label Stack"];
}

function cleanGotoComplex() {
	return [
		["deleteLine:ofList:", "last", "Label Stack"]
	];
}

function absoluteBranch(dest) {
	return [
		["setLine:ofList:to:", "last", "Label Stack", dest]
	];
}

function castToNumber(b) {
	return ["*", b, 1];
}<commit_msg>Negation of comparisons<commit_after>// takes an IR function object and returns a list of Scratch blocks

module.exports.ffi = {};

module.exports.generateFunctionHat = function(functionContext, func) {
	var spec = func.funcName;
	var inputs = [];
	var defaults = [];

	functionContext.params = [];

	for(var i = 0; i < func.paramList.length; ++i) {
		var pName = "param" + i;

		if(func.paramList[i][1])
			pName = func.paramList[i][1];

		inputs.push(pName);
		functionContext.params.push(pName);

		defaults.push(defaultForType(func.paramList[i][0]));
		spec += " "+specifierForType(func.paramList[i][0]);
	}

	return ["procDef", spec, inputs, defaults, false];

}

module.exports.compileFunction = function(func) {
	console.log("Compiling "+JSON.stringify(func)+"...");

	var functionContext = {
		locals: {},
		localDepth: 0,
		params: [],
		gotoInit: false
	}

	var blockList = [module.exports.generateFunctionHat(functionContext, func)]
					.concat(initLocal());

	if(func.inGotoComplex) {
		blockList = blockList.concat(initGotoComplex());
	}

	for(var i = 0; i < func.code.length; ++i) {
		console.log(func.code[i]);

		var hasGotoComplex = functionContext.gotoComplex && functionContext.gotoComplex.okToUse && functionContext.gotoComplex.active; // this MUST be before compileInstruction for branching to work
		var instruction = compileInstruction(functionContext, func.code[i]);

		if(!functionContext.gotoInit && functionContext.gotoComplex && functionContext.gotoComplex.okToUse) {
			blockList = blockList.concat([functionContext.gotoComplex.forever]);
			functionContext.gotoInit = true;
		}
		if(hasGotoComplex) {
			if(functionContext.gotoComplex.currentContext[2]) {
				functionContext.gotoComplex.currentContext[2] =
					functionContext.gotoComplex.currentContext[2].concat(instruction);
			} else {
				functionContext.gotoComplex.currentContext[2] = instruction;
			}
		} else {
			blockList = blockList.concat(instruction);
		}
	}

	//blockList = blockList.concat(returnBlock());

	return blockList;
}

function compileInstruction(ctx, block) {
	if(block.type == "call") {
		// calling a (potentially foreign) function
		return callBlock(ctx, block);
	} else if(block.type == "ffi") {
		// FFI block
		// load the code from the options
		return module.exports.ffi[block.ffiBlock];
	} else if(block.type == "set") {
		var val = 0;

		console.log("SET: "+JSON.stringify(block));

		if(block.val.type == "return value") {
			val = ["readVariable", "return value"];
		} else if(block.val.type == "variable") {
			val = fetchByName(ctx, block.val.name);
		} else if(block.val.type == "arithmetic") {
			val = [block.val.operation, fetchByName(block.val.operand1), fetchByName(ctx, block.val.operand2)];
		} else if(block.val.type == "comparison") {
			var spec = specForComparison(block.val.operation);
			var negate = false;

			if(spec[0] == "!") {
				negate = true;
				spec = spec.slice(1);
			}

			var b = [spec, fetchByName(ctx, block.val.left), fetchByName(ctx, block.val.right)];

			if(negate) {
				b = ["not", b];
			}

			val = castToNumber(b);
		}

		return compileInstruction(ctx, block.computation)
				.concat(allocateLocal(ctx, val, block.name));
	} else if(block.type == "ret") {
		return returnBlock(ctx, block.value);
	} else if(block.type == "store") {
		return dereferenceAndSet(ctx, block.destination.value, block.src.value);
	} else if(block.type == "gotoComplex") {
		ctx.gotoComplex = {
			context: [],
			okToUse: false,
			forever: ["doForever", []],
			active: true
		}

		//return [ctx.gotoComplex.forever];
	} else if(block.type == "label") {
		var chunk = ["doIfElse", ["=", getCurrentLabel(), block.label], [], []];

		ctx.gotoComplex.okToUse = true;
		ctx.gotoComplex.active = true;

		if(ctx.gotoComplex.currentContext) {
			ctx.gotoComplex.currentContext[3] = [chunk];
			ctx.gotoComplex.currentContext = ctx.gotoComplex.currentContext[3][0];
		} else {
			ctx.gotoComplex.currentContext = chunk;
			ctx.gotoComplex.context = ctx.gotoComplex.currentContext;
			ctx.gotoComplex.forever[1] = [ctx.gotoComplex.context];
		}

	} else if(block.type == "branch") {
		ctx.gotoComplex.active = false;

		if(block.conditional) {
			return [
				["doIfElse", ["=", fetchByName(ctx, block.condition), 1], absoluteBranch(block.dest.slice(1)), absoluteBranch(block.falseDest.slice(1))]
			];
		} else {
			return absoluteBranch(block.dest);
		}
	}

	return [];
}

// fixme: stub
function defaultForType(type) {
	console.log(type);
	return 0;
}

// fixme: stub
function specifierForType(type) {
	return "%s";
}

// fixme: stub
function formatValue(ctx, type, value) {
	console.log("FORMAT: "+type+","+value);

	if(value[0] == '%') {
		return fetchByName(ctx, value);
	}

	return value;
}

function getOffset(ctx, value) {
	return ctx.localDepth - ctx.locals[value];
}

function stackPosFromOffset(offset) {
	return ["-", ["lineCountOfList:", "Stack"], offset];
}

// higher-level code generation
function initLocal() {
	return [
		["append:toList:", "0", "# of locals"]
	]
}

function allocateLocal(ctx, val, name) {
	if(name) {
		console.log(name+","+val);
		ctx.locals[name] = ++ctx.localDepth;
	}

	return [
		["setLine:ofList:to:", "last", "# of locals", ["+", ["getLine:ofList:", "last", "# of locals"], 1]],
		["append:toList:", val, "Stack"]
	];
}

function freeLocals() {
	return [
		["doRepeat", ["getLine:ofList:", "last", "# of locals"], [["deleteLine:ofList:", "last", "Stack"]]],
		["deleteLine:ofList:", "last", "# of locals"]
	];
}

function fetchByName(ctx, n) {
	if(ctx.locals[n])
		return ["getLine:ofList:", stackPosFromOffset(getOffset(ctx, n)), "Stack"];
	else if(ctx.params.indexOf(n) > -1)
		return ["getParam", n, "r"];
	else if( (n * 1) == n)
		return n
	else
		return 0;
}

function returnBlock(ctx, val) {
	var proc = freeLocals();

	if(ctx.gotoComplex) {
		proc = proc.concat(cleanGotoComplex());
	}

	if(val) {
		proc.push(["setVar:to:", "return value", formatValue(ctx, val[0], val[1])]);
	}

	proc.push(["stopScripts", "this script"]);

	return proc;
}

function callBlock(ctx, block) {
	var spec = block.funcName;
	var args = [];

	for(var a = 0; a < block.paramList.length; ++a) {
		args.push(formatValue(ctx, block.paramList[a][0], block.paramList[a][1]));
		spec += " "+specifierForType(block.paramList[a][0]);
	}

	return [
		["call", spec].concat(args)
	];
}

// TODO: more robust implementation to support heap

function dereferenceAndSet(ctx, ptr, content) {
	return [
		[
			"setLine:ofList:to:",
			stackPosFromOffset(getOffset(ctx, ptr)),
			"Stack",
			fetchByName(ctx, content)
		]
	];
}

function specForComparison(comp) {
	if(comp == "eq") {
		return "=";
	} else if(comp == "neq") {
		return "!=";
	}
	return "undefined";
}

function initGotoComplex() {
	return [
		["append:toList:", 0, "Label Stack"]
	];
}

function getCurrentLabel() {
	return ["getLine:ofList:", "last", "Label Stack"];
}

function cleanGotoComplex() {
	return [
		["deleteLine:ofList:", "last", "Label Stack"]
	];
}

function absoluteBranch(dest) {
	return [
		["setLine:ofList:to:", "last", "Label Stack", dest]
	];
}

function castToNumber(b) {
	return ["*", b, 1];
}<|endoftext|>


<commit_before>'use strict';

module.exports = {fileRenameMap, dirRenameMap, renameFiles};

const fs = require ('graceful-fs');
const exifParser = require('exif-parser');
const moment = require('moment');

const util = require ('./util');

function listFiles (dir, filterFunc) {
    return fs.readdirSync(dir).filter(filterFunc);
}
// checks if the file is a directory OR a symlink to a directory
function isDirectory (fn) {
    return fs.statSync(fn).isDirectory();
}

// reads `date taken` from fileName (full path) and passes it to callback
// in the form of Date object, null if failed.
// Uses UTC since no timezone is available in DateTimeOriginal tag.
function getExifDate (fileName, callback) {
    if(!callback) throw 'Empty callback';

    fs.stat(fileName, processFile);

    function processFile (error, stats) {
        if (error) {
            console.log('File ', fileName, ' inaccesible');
            callback(null);
            return;
        }
        fs.open(fileName, 'r', processExifBlock.bind(undefined, stats));
    }

    function processExifBlock (stats, error, fileDescriptor) {
        if (error) {
            console.log('File ', fileName, ' inaccesible');
            callback(null);
            return;
        }
        var exifEnd = 65635; // the maximum position within the JPEG file where EXIF metadata block could reach
        var exifBuffer = Buffer.allocUnsafe(stats.size < exifEnd ? stats.size : exifEnd);
        fs.read(fileDescriptor, exifBuffer, 0, exifBuffer.length, 0, parseExif.bind(undefined, fileDescriptor));
    }

    function parseExif (fileDescriptor, error, bytesRead, buffer) {
        if (error) {
            console.log('Unable to read file: ', fileName);
            callback(null);
            return;
        }
        var parser = exifParser.create(buffer);
        parser.enableImageSize(false);
        parser.enableSimpleValues(true); // returns UNIX timestamp w. this flag (raw string otherwise)
        var result;
        try {
            result = parser.parse();
        }
        catch (err) {
            console.warn('Error parsing EXIF in ' + fileName);
        }
        var exifDate = result && result.tags.DateTimeOriginal ?
            new Date(result.tags.DateTimeOriginal * 1000) :
            null;
        var msg = null;
        if (!exifDate) {
            msg = 'EXIF date not present or EXIF unreadable in file: ';
        }
        function logFileErr (err) {
            return function (closeErr) {
                if(err || closeErr) {
                    console.log(err ? err : '', fileName, closeErr ? ' (error on closing)' : '');
                }
            };
        }
        fs.close(fileDescriptor, logFileErr(msg));
        callback(exifDate);
        return;
    }
}

/* builds a file->date map object which maps filenames
from fileArray located in dir to their EXIF DateCreated dates
and calls callback with that object
*/
function fileDateMap (dir, fileArray, callback) {
    if(!callback) throw 'Empty callback';
    var result = {};
    result.filesPending = fileArray.length;
    Object.defineProperty(result, 'baseDir', {value: dir, enumerable: false});
    fileArray.forEach(gatherDates);

    function gatherDates (fileName) {
        var filePath = result.baseDir + '/' + fileName;
        getExifDate(filePath, collectDate.bind(undefined, fileName));
    }

    function collectDate (fileName, date) {
        result[fileName] = date;
        if (--result.filesPending === 0) {
            delete result.filesPending;
            return callback(result);
        }
    }
}

/* Composes a new file name from the old one, a date object and options object.
options with defaults:
dateSeparator: '.'  - separator of date parts
timeSeparator: '.'  - separator of time parts
dateTimeSeparator: '_'  - separates date from time
*/
function makeNewFileName (oldFileName, fileDate, options) {
    if(fileDate === null) return null;

    var compose = (src => [src.y, src.ds, src.m, src.ds, src.d, src.dts, src.h, src.ts, src.M, src.ts, src.s].join(''));

    return makeNewName(oldFileName, fileDate, options, compose);
}

/* Composes a new file/dir name based on old name, date (or date range in an array) and options.
Also takes a composition function, which should compose the date part of a new name
based on given object consisting of values with keys:
y - year,
m - month,
d - day,
h - hour,
M - minute,
s - second,
ds - date separator,
ts - time separator,
dts - date-time separator (separates date from time),
rs - range separator (separates a date from another one or its part),
y2 - year of the last contained file,
m2 - month of the last contained file,
d2 - day of the last contained file.
*/
function makeNewName (oldName, dates, options, composeFunc) {
    if(! oldName) throw('Missing oldName argument');
    if(! dates) throw('Missing dates argument');
    if(! options || util.getType(options) !== 'object') throw('Missing or invalid options argument: should be an object.');
    if(! composeFunc || !(composeFunc instanceof Function)) throw('Missing or invalid composeFunc argument: should be a function.');

    var src = {};
    src.ds = options.hasOwnProperty('dateSeparator') ? options.dateSeparator : '.';
    src.ts = options.hasOwnProperty('timeSeparator') ? options.timeSeparator : '.';
    src.dts = options.hasOwnProperty('dateTimeSeparator') ? options.dateTimeSeparator : '_';
    src.rs = options.hasOwnProperty('rangeSeparator') ? options.rangeSeparator : '-';
    var dayStart = options.hasOwnProperty('dayStart') ? options.dayStart : '0';
    if(isNaN(dayStart)) dayStart = 0;

    function padTo2 (nr) {
        nr += '';
        if(nr.length < 2) nr = '0' + nr;
        return nr;
    }

    var date1;
    var date2;
    if(util.getType(dates) === 'array') {
        date1 = dates[0].getUTCHours() >= dayStart ? dates[0] : moment(dates[0]).subtract(1, 'day').toDate();
        date2 = dates[1].getUTCHours() >= dayStart ? dates[1] : moment(dates[1]).subtract(1, 'day').toDate();
        src.y2 = date2.getUTCFullYear();
        src.m2 = padTo2(date2.getUTCMonth() + 1);  // strange JS month handling (0-indexed)
        src.d2 = padTo2(date2.getUTCDate());
    } else {
        date1 = dates;
    }
    src.y = date1.getUTCFullYear();
    src.m = padTo2(date1.getUTCMonth() + 1);
    src.d = padTo2(date1.getUTCDate());
    src.h = padTo2(date1.getUTCHours());
    src.M = padTo2(date1.getUTCMinutes());
    src.s = padTo2(date1.getUTCSeconds());
    var datePart = composeFunc(src);
    var title = extractTitle(oldName);
    // if the title starts with a letter or number, direct appending to the date would be unpleasant
    // (this method is Unicode-safe)
    if(title && (title[0].toLowerCase() !== title[0].toUpperCase() || /\d/.test(title[0]))) {
        title = ' ' + title;
    }
    return datePart + title;
}

/* Extracts and returns file/dir title from a given file/dir name, if there is one,
empty string otherwise. The title is everything after: the standard filename
(without extension) given by the DCF compliant camera, a date or date range.
*/
function extractTitle (fileName) {
    /* from Wikipedia on DCF specification:
    Subdirectory names (such as "123ABCDE") consist of a unique directory number (in the range 100…999)
    and five alphanumeric characters, which may be freely chosen.
    These directories contain files with names such as "ABCD1234.JPG"
    that consist of four alphanumeric characters (often "DSC_", "DSC0", "DSCF", "IMG_"/"MOV_", or "P000"),
    followed by a number.
    The file extension is "JPG" for Exif files and "THM" for Exif files that represent thumbnails
    of other files than "JPG".

    The ultimate online tool for analysing that terrible thing below: https://regex101.com/
    The regex consists of 4 alternative subpatterns: 1st for dates in filenames,
    2nd for DCF filenames, 3rd for DCF directory names, and 4th for dates and date ranges in dirnames
    */
    var baseNamePattern = /\s?\d{4}\D?\d\d\D?\d\d((\D?\d\d){2,3})?|^\w{4}\d+|^\d{3}\w{5}|\d{4}\W?\d\d\W?\d\d(-(((\d{4}\W?)?\d\d\W?)?\d\d))?/g;
    return fileName.replace(baseNamePattern, '');
}

// Generates an object which maps filenames to their new names and calls callback with it.
// Takes a directory name, filename filter function and options object for makeNewFileName.
function fileRenameMap (dir, filterFunc, options, callback) {
    if(!callback) throw 'Empty callback';

    var fileList = listFiles(dir, filterFunc).filter(f => ! isDirectory(dir + '/' + f));
    fileDateMap(dir, fileList, generateRenameMap);

    function generateRenameMap (dateMap) {
        var result = {};
        Object.keys(dateMap).forEach(fileName => {
            if(dateMap[fileName]) {
                result[fileName] = makeNewFileName(fileName, dateMap[fileName], options);
            } else {
                result[fileName] = null;
            }
        });
        callback(util.filterObject(result, fn => fn !== result[fn]));
    }
}

/* Calls callback with a date range array: [start_date end_date]
extracted from files in dir filtered by filterFunc.
*/
function extractDirDateRange (dir, filterFunc, callback) {
    var fileList = null;
    try {
        fileList = listFiles(dir, filterFunc);
    } catch (err) {
        console.log('Error opening directory ', dir, ' - ', err.message);
    }
    if(! fileList) {
        callback(null);
        return;
    } else {
        if(fileList.length === 0) handleDateList(null);
        var dateList = [];
        var filesPending = fileList.length;
        fileList.forEach(collectDate);
    }

    function collectDate (fileName) {
        getExifDate(dir + '/' + fileName, date => {
            if(date !== null) dateList.push(date);
            if(--filesPending === 0) handleDateList(dateList);
        });
    }
    function handleDateList (dateList) {
        if(! dateList || dateList.length === 0) {
            console.log(dir + ' does not contain any file with readable EXIF.');
            callback(null);
            return;
        }
        var dateListSorted;
        if(dateList.length === 1) {
            dateListSorted = dateList.concat(dateList);
        } else {
            dateListSorted = dateList.slice().sort((date1, date2) => date1 - date2);
        }
        var out;
        if(dateList.length > 2) {
            out = dateListSorted.splice(1, dateList.length - 2);
        } else {
            out = dateListSorted;
        }
        callback(out);
    }
}

/* Returns a new name for a directory based on old name, date range (array of 2 dates) and options.
Options with defaults:
dateSeparator: '.'  - separator of date parts
rangeSeparator: '-'  - separates the first date from (a part of) the second one
dayStart: 0  -  hour which starts a new day (any date with hour < dayStart will be converted to the preceding day)
*/
function makeNewDirName (oldName, dateRange, options) {
    if(dateRange === null) return null;
    if(util.getType(dateRange) !== 'array' || util.getType(dateRange[0]) !== 'date' || util.getType(dateRange[1]) !== 'date') {
        throw(new Error('dateRange should be an array of 2 dates.'));
    }
    var compose = function (src) {
        var y2s;  // second date's year with appended separator - and so on below
        if(src.y === src.y2) {
            y2s = '';
        } else {
            y2s = src.y2 + src.ds;
        }
        var m2s;
        if(src.y === src.y2 && src.m === src.m2) {
            m2s = '';
        } else {
            m2s = src.m2 + src.ds;
        }
        var d2;
        var rs;
        if(src.y === src.y2 && src.m === src.m2 && src.d === src.d2) {
            d2 = '';
            rs = '';
        } else {
            d2 = src.d2;
            rs = src.rs;
        }
        return [src.y, src.ds, src.m, src.ds, src.d, rs, y2s, m2s, d2].join('');
    };
    return makeNewName(oldName, dateRange, options, compose);
}

// Generates an object mapping current dir names to their new names and calls callback with it.
// Passes options object to makeNewDirName.
function dirRenameMap (outerDir, options, callback) {
    var dirList = fs.readdirSync(outerDir).filter(fn => isDirectory(outerDir + '/' + fn));
    var dirDateMap = {'pending': dirList.length};
    Object.defineProperty(dirDateMap, 'baseDir', {value: outerDir, enumerable: false});
    dirList.forEach(dirName => {
        var dirPath = outerDir + '/' + dirName;
        extractDirDateRange(dirPath, fn => /\.jpe?g$/i.test(fn), buildDirDateMap.bind(this, dirName));
    });
    function buildDirDateMap (dirName, dateRange) {
        dirDateMap[dirName] = dateRange;
        if(--dirDateMap.pending === 0) {
            delete dirDateMap.pending;
            buildDirRenameMap(dirDateMap);
        }
    }
    function buildDirRenameMap (dirDateMap) {
        var dirRenameMap = {};
        Object.defineProperty(dirRenameMap, 'baseDir', {value: outerDir, enumerable: false});

        Object.keys(dirDateMap).forEach(dirName => {
            dirRenameMap[dirName] = makeNewDirName(dirName, dirDateMap[dirName], options);
        });
        callback(util.filterObject(dirRenameMap, fn => fn !== dirRenameMap[fn]));
    }
}

// Executes renaming of files or directories passed as a map object: {old: new, ...}.
// Refuses to replace an existing file.
// Calls callback with an object mapping filenames to failure messages (if any).
function renameFiles (renameMap, callback) {
    var failures = {};
    var pending = Object.keys(renameMap).length;
    Object.keys(renameMap).forEach(fn => renameIfExistsNot(fn));
    function renameIfExistsNot (fn) {
        fs.access(renameMap[fn], fs.constants.F_OK, err => {
            renameFile(err ? null : new Error('target exists'), fn, renameMap[fn]);
        });
    }
    function renameFile (err, oldName, newName) {
        if (err) {
            buildFailureList(err, oldName);
        }
        else {
            fs.rename(oldName, newName, err => buildFailureList(err, oldName));
        }
    }
    function buildFailureList(err, fn) {
        if (err) {
            failures[fn] = err.message;
        }
        if (--pending === 0) {
            return callback(failures);
        }
    }
}

// TODO: what about *.thm files and associated objects, especially videos?
// Also, videos without *.thm files, having their dates only in fs attributes...

// TODO: suspicious cases (very old dates, subdirs containing both photos and another subdirs)

// TODO: option to ignore existing titles in file names (important)<commit_msg>backend.js: fileRenameMap passes baseDir the same way as dirRenameMap<commit_after>'use strict';

module.exports = {fileRenameMap, dirRenameMap, renameFiles};

const fs = require ('graceful-fs');
const exifParser = require('exif-parser');
const moment = require('moment');

const util = require ('./util');

function listFiles (dir, filterFunc) {
    return fs.readdirSync(dir).filter(filterFunc);
}
// checks if the file is a directory OR a symlink to a directory
function isDirectory (fn) {
    return fs.statSync(fn).isDirectory();
}

// reads `date taken` from fileName (full path) and passes it to callback
// in the form of Date object, null if failed.
// Uses UTC since no timezone is available in DateTimeOriginal tag.
function getExifDate (fileName, callback) {
    if(!callback) throw 'Empty callback';

    fs.stat(fileName, processFile);

    function processFile (error, stats) {
        if (error) {
            console.log('File ', fileName, ' inaccesible');
            callback(null);
            return;
        }
        fs.open(fileName, 'r', processExifBlock.bind(undefined, stats));
    }

    function processExifBlock (stats, error, fileDescriptor) {
        if (error) {
            console.log('File ', fileName, ' inaccesible');
            callback(null);
            return;
        }
        var exifEnd = 65635; // the maximum position within the JPEG file where EXIF metadata block could reach
        var exifBuffer = Buffer.allocUnsafe(stats.size < exifEnd ? stats.size : exifEnd);
        fs.read(fileDescriptor, exifBuffer, 0, exifBuffer.length, 0, parseExif.bind(undefined, fileDescriptor));
    }

    function parseExif (fileDescriptor, error, bytesRead, buffer) {
        if (error) {
            console.log('Unable to read file: ', fileName);
            callback(null);
            return;
        }
        var parser = exifParser.create(buffer);
        parser.enableImageSize(false);
        parser.enableSimpleValues(true); // returns UNIX timestamp w. this flag (raw string otherwise)
        var result;
        try {
            result = parser.parse();
        }
        catch (err) {
            console.warn('Error parsing EXIF in ' + fileName);
        }
        var exifDate = result && result.tags.DateTimeOriginal ?
            new Date(result.tags.DateTimeOriginal * 1000) :
            null;
        var msg = null;
        if (!exifDate) {
            msg = 'EXIF date not present or EXIF unreadable in file: ';
        }
        function logFileErr (err) {
            return function (closeErr) {
                if(err || closeErr) {
                    console.log(err ? err : '', fileName, closeErr ? ' (error on closing)' : '');
                }
            };
        }
        fs.close(fileDescriptor, logFileErr(msg));
        callback(exifDate);
        return;
    }
}

/* builds a file->date map object which maps filenames
from fileArray located in dir to their EXIF DateCreated dates
and calls callback with that object
*/
function fileDateMap (dir, fileArray, callback) {
    if(!callback) throw 'Empty callback';
    var result = {};
    result.filesPending = fileArray.length;
    Object.defineProperty(result, 'baseDir', {value: dir, enumerable: false});
    fileArray.forEach(gatherDates);

    function gatherDates (fileName) {
        var filePath = result.baseDir + '/' + fileName;
        getExifDate(filePath, collectDate.bind(undefined, fileName));
    }

    function collectDate (fileName, date) {
        result[fileName] = date;
        if (--result.filesPending === 0) {
            delete result.filesPending;
            return callback(result);
        }
    }
}

/* Composes a new file name from the old one, a date object and options object.
options with defaults:
dateSeparator: '.'  - separator of date parts
timeSeparator: '.'  - separator of time parts
dateTimeSeparator: '_'  - separates date from time
*/
function makeNewFileName (oldFileName, fileDate, options) {
    if(fileDate === null) return null;

    var compose = (src => [src.y, src.ds, src.m, src.ds, src.d, src.dts, src.h, src.ts, src.M, src.ts, src.s].join(''));

    return makeNewName(oldFileName, fileDate, options, compose);
}

/* Composes a new file/dir name based on old name, date (or date range in an array) and options.
Also takes a composition function, which should compose the date part of a new name
based on given object consisting of values with keys:
y - year,
m - month,
d - day,
h - hour,
M - minute,
s - second,
ds - date separator,
ts - time separator,
dts - date-time separator (separates date from time),
rs - range separator (separates a date from another one or its part),
y2 - year of the last contained file,
m2 - month of the last contained file,
d2 - day of the last contained file.
*/
function makeNewName (oldName, dates, options, composeFunc) {
    if(! oldName) throw('Missing oldName argument');
    if(! dates) throw('Missing dates argument');
    if(! options || util.getType(options) !== 'object') throw('Missing or invalid options argument: should be an object.');
    if(! composeFunc || !(composeFunc instanceof Function)) throw('Missing or invalid composeFunc argument: should be a function.');

    var src = {};
    src.ds = options.hasOwnProperty('dateSeparator') ? options.dateSeparator : '.';
    src.ts = options.hasOwnProperty('timeSeparator') ? options.timeSeparator : '.';
    src.dts = options.hasOwnProperty('dateTimeSeparator') ? options.dateTimeSeparator : '_';
    src.rs = options.hasOwnProperty('rangeSeparator') ? options.rangeSeparator : '-';
    var dayStart = options.hasOwnProperty('dayStart') ? options.dayStart : '0';
    if(isNaN(dayStart)) dayStart = 0;

    function padTo2 (nr) {
        nr += '';
        if(nr.length < 2) nr = '0' + nr;
        return nr;
    }

    var date1;
    var date2;
    if(util.getType(dates) === 'array') {
        date1 = dates[0].getUTCHours() >= dayStart ? dates[0] : moment(dates[0]).subtract(1, 'day').toDate();
        date2 = dates[1].getUTCHours() >= dayStart ? dates[1] : moment(dates[1]).subtract(1, 'day').toDate();
        src.y2 = date2.getUTCFullYear();
        src.m2 = padTo2(date2.getUTCMonth() + 1);  // strange JS month handling (0-indexed)
        src.d2 = padTo2(date2.getUTCDate());
    } else {
        date1 = dates;
    }
    src.y = date1.getUTCFullYear();
    src.m = padTo2(date1.getUTCMonth() + 1);
    src.d = padTo2(date1.getUTCDate());
    src.h = padTo2(date1.getUTCHours());
    src.M = padTo2(date1.getUTCMinutes());
    src.s = padTo2(date1.getUTCSeconds());
    var datePart = composeFunc(src);
    var title = extractTitle(oldName);
    // if the title starts with a letter or number, direct appending to the date would be unpleasant
    // (this method is Unicode-safe)
    if(title && (title[0].toLowerCase() !== title[0].toUpperCase() || /\d/.test(title[0]))) {
        title = ' ' + title;
    }
    return datePart + title;
}

/* Extracts and returns file/dir title from a given file/dir name, if there is one,
empty string otherwise. The title is everything after: the standard filename
(without extension) given by the DCF compliant camera, a date or date range.
*/
function extractTitle (fileName) {
    /* from Wikipedia on DCF specification:
    Subdirectory names (such as "123ABCDE") consist of a unique directory number (in the range 100…999)
    and five alphanumeric characters, which may be freely chosen.
    These directories contain files with names such as "ABCD1234.JPG"
    that consist of four alphanumeric characters (often "DSC_", "DSC0", "DSCF", "IMG_"/"MOV_", or "P000"),
    followed by a number.
    The file extension is "JPG" for Exif files and "THM" for Exif files that represent thumbnails
    of other files than "JPG".

    The ultimate online tool for analysing that terrible thing below: https://regex101.com/
    The regex consists of 4 alternative subpatterns: 1st for dates in filenames,
    2nd for DCF filenames, 3rd for DCF directory names, and 4th for dates and date ranges in dirnames
    */
    var baseNamePattern = /\s?\d{4}\D?\d\d\D?\d\d((\D?\d\d){2,3})?|^\w{4}\d+|^\d{3}\w{5}|\d{4}\W?\d\d\W?\d\d(-(((\d{4}\W?)?\d\d\W?)?\d\d))?/g;
    return fileName.replace(baseNamePattern, '');
}

// Generates an object which maps filenames to their new names and calls callback with it.
// Takes a directory name, filename filter function and options object for makeNewFileName.
function fileRenameMap (dir, filterFunc, options, callback) {
    if(!callback) throw 'Empty callback';

    var fileList = listFiles(dir, filterFunc).filter(f => ! isDirectory(dir + '/' + f));
    fileDateMap(dir, fileList, generateRenameMap);

    function generateRenameMap (dateMap) {
        var result = {};
        Object.keys(dateMap).forEach(fileName => {
            if(dateMap[fileName]) {
                result[fileName] = makeNewFileName(fileName, dateMap[fileName], options);
            } else {
                result[fileName] = null;
            }
        });
        var filteredResult = util.filterObject(result, fn => fn !== result[fn]);
        Object.defineProperty(filteredResult, 'baseDir', {value: dir, enumerable: false});
        callback(filteredResult);
    }
}

/* Calls callback with a date range array: [start_date end_date]
extracted from files in dir filtered by filterFunc.
*/
function extractDirDateRange (dir, filterFunc, callback) {
    var fileList = null;
    try {
        fileList = listFiles(dir, filterFunc);
    } catch (err) {
        console.log('Error opening directory ', dir, ' - ', err.message);
    }
    if(! fileList) {
        callback(null);
        return;
    } else {
        if(fileList.length === 0) handleDateList(null);
        var dateList = [];
        var filesPending = fileList.length;
        fileList.forEach(collectDate);
    }

    function collectDate (fileName) {
        getExifDate(dir + '/' + fileName, date => {
            if(date !== null) dateList.push(date);
            if(--filesPending === 0) handleDateList(dateList);
        });
    }
    function handleDateList (dateList) {
        if(! dateList || dateList.length === 0) {
            console.log(dir + ' does not contain any file with readable EXIF.');
            callback(null);
            return;
        }
        var dateListSorted;
        if(dateList.length === 1) {
            dateListSorted = dateList.concat(dateList);
        } else {
            dateListSorted = dateList.slice().sort((date1, date2) => date1 - date2);
        }
        var out;
        if(dateList.length > 2) {
            out = dateListSorted.splice(1, dateList.length - 2);
        } else {
            out = dateListSorted;
        }
        callback(out);
    }
}

/* Returns a new name for a directory based on old name, date range (array of 2 dates) and options.
Options with defaults:
dateSeparator: '.'  - separator of date parts
rangeSeparator: '-'  - separates the first date from (a part of) the second one
dayStart: 0  -  hour which starts a new day (any date with hour < dayStart will be converted to the preceding day)
*/
function makeNewDirName (oldName, dateRange, options) {
    if(dateRange === null) return null;
    if(util.getType(dateRange) !== 'array' || util.getType(dateRange[0]) !== 'date' || util.getType(dateRange[1]) !== 'date') {
        throw(new Error('dateRange should be an array of 2 dates.'));
    }
    var compose = function (src) {
        var y2s;  // second date's year with appended separator - and so on below
        if(src.y === src.y2) {
            y2s = '';
        } else {
            y2s = src.y2 + src.ds;
        }
        var m2s;
        if(src.y === src.y2 && src.m === src.m2) {
            m2s = '';
        } else {
            m2s = src.m2 + src.ds;
        }
        var d2;
        var rs;
        if(src.y === src.y2 && src.m === src.m2 && src.d === src.d2) {
            d2 = '';
            rs = '';
        } else {
            d2 = src.d2;
            rs = src.rs;
        }
        return [src.y, src.ds, src.m, src.ds, src.d, rs, y2s, m2s, d2].join('');
    };
    return makeNewName(oldName, dateRange, options, compose);
}

// Generates an object mapping current dir names to their new names and calls callback with it.
// Passes options object to makeNewDirName.
function dirRenameMap (outerDir, options, callback) {
    var dirList = fs.readdirSync(outerDir).filter(fn => isDirectory(outerDir + '/' + fn));
    var dirDateMap = {'pending': dirList.length};
    Object.defineProperty(dirDateMap, 'baseDir', {value: outerDir, enumerable: false});
    dirList.forEach(dirName => {
        var dirPath = outerDir + '/' + dirName;
        extractDirDateRange(dirPath, fn => /\.jpe?g$/i.test(fn), buildDirDateMap.bind(this, dirName));
    });
    function buildDirDateMap (dirName, dateRange) {
        dirDateMap[dirName] = dateRange;
        if(--dirDateMap.pending === 0) {
            delete dirDateMap.pending;
            buildDirRenameMap(dirDateMap);
        }
    }
    function buildDirRenameMap (dirDateMap) {
        var dirRenameMap = {};
        Object.defineProperty(dirRenameMap, 'baseDir', {value: outerDir, enumerable: false});

        Object.keys(dirDateMap).forEach(dirName => {
            dirRenameMap[dirName] = makeNewDirName(dirName, dirDateMap[dirName], options);
        });
        callback(util.filterObject(dirRenameMap, fn => fn !== dirRenameMap[fn]));
    }
}

// Executes renaming of files or directories passed as a map object: {old: new, ...}.
// Refuses to replace an existing file.
// Calls callback with an object mapping filenames to failure messages (if any).
function renameFiles (renameMap, callback) {
    var failures = {};
    var pending = Object.keys(renameMap).length;
    Object.keys(renameMap).forEach(fn => renameIfExistsNot(fn));
    function renameIfExistsNot (fn) {
        fs.access(renameMap[fn], fs.constants.F_OK, err => {
            renameFile(err ? null : new Error('target exists'), fn, renameMap[fn]);
        });
    }
    function renameFile (err, oldName, newName) {
        if (err) {
            buildFailureList(err, oldName);
        }
        else {
            fs.rename(oldName, newName, err => buildFailureList(err, oldName));
        }
    }
    function buildFailureList(err, fn) {
        if (err) {
            failures[fn] = err.message;
        }
        if (--pending === 0) {
            return callback(failures);
        }
    }
}

// TODO: what about *.thm files and associated objects, especially videos?
// Also, videos without *.thm files, having their dates only in fs attributes...

// TODO: suspicious cases (very old dates, subdirs containing both photos and another subdirs)

// TODO: option to ignore existing titles in file names (important)<|endoftext|>


<commit_before>var async = require("async"),
    express = require('express'),
    auth = require('http-auth'),
    request = require('request'),
    utility = require('./utility'),
    matches = utility.matches,
    players = utility.players,
    cheerio = require('cheerio'),
    kue = require('kue'),
    winston = require('winston');
var jobs = kue.createQueue();
var memwatch = require('memwatch');
var request = require("request");
var aq = async.queue(apiRequest, 1)
var api_url = "https://api.steampowered.com/IDOTA2Match_570"
var summaries_url = "http://api.steampowered.com/ISteamUser"
var remote = "http://dotabuff.com"
var queuedMatches = {}
var trackedPlayers = {}
var jobTimeout = 60 * 1000 // Job timeout for kue
var next_seq;
var logger = new(winston.Logger)({
    transports: [
        new(winston.transports.File)({
            filename: 'backend.log',
            level: 'info'
        })
    ]
});
var basic = auth.basic({
    realm: "Kue"
}, function(username, password, callback) { // Custom authentication method.
    callback(username === process.env.KUE_USER && password === process.env.KUE_PASS);
});
var app = express();
app.use(auth.connect(basic));
app.use(kue.app);
app.listen(process.env.KUE_PORT || 5001);

setInterval(findStuckJobs, jobTimeout)

memwatch.on('leak', function(info) {
    logger.info('[LEAK]' + info);
});
updateConstants(function(err) {});
async.series([
    function(cb) {
        //scrape players full match history
        players.find({
            full_history: 1
        }, function(err, docs) {
            async.mapSeries(docs, function(player, cb2) {
                var account_id = player.account_id
                var player_url = remote + "/players/" + account_id + "/matches"
                getMatchPage(player_url, function(err) {
                    //done scraping player
                    players.update({
                        account_id: account_id
                    }, {
                        $set: {
                            full_history: 0
                        }
                    })
                    cb2(null)
                })
            }, function(err) {
                //done scraping all players
                cb(null)
            })
        })
    },
    function(cb) {
        //check most recent 100 matches for tracked players
        players.find({
            track: 1
        }, function(err, docs) {
            aq.push(docs, function(err) {})
        })
        //parse unparsed matches
        matches.find({
            parse_status: 0
        }, function(err, docs) {
            docs.forEach(function(match) {
                requestParse(match)
            })
        })

        cb(null)
    },
    function(cb) {
        //determine sequence number to begin scan at
        if(process.env.SAVE_ALL_MATCHES) {
            matches.findOne({}, {
                sort: {
                    match_seq_num: -1
                }
            }, function(err, doc) {
                next_seq = doc ? doc.match_seq_num + 1 : 0
                cb(null)
            })
        } else {
            getData(api_url + "/GetMatchHistory/V001/?key=" + process.env.STEAM_API_KEY, function(err, data) {
                next_seq = data.result.matches[0].match_seq_num
                cb(null)
            })
        }
    }
], function(err) {
    getMatches()
    aq.empty = function() {
        getMatches();
    }
})

function updateConstants(cb) {
    var constants = require('./constants.json')
    async.map(Object.keys(constants), function(key, cb) {
        var val = constants[key]
        if(typeof(val) == "string" && val.slice(0, 4) == "http") {
            getData(val, function(err, result) {
                constants[key] = result
                cb(null)
            })
        } else {
            cb(null)
        }
    }, function(err) {
        var heroes = constants.heroes
        heroes.forEach(function(hero) {
            hero.name = hero.name.replace("npc_dota_hero_", "")
            hero.img = "http://cdn.dota2.com/apps/dota2/images/heroes/" + hero.name + "_sb.png"
        })
        constants.heroes = buildLookup(heroes)
        constants.hero_names = {}
        for(var i = 0; i < heroes.length; i++) {
            constants.hero_names[heroes[i].name] = heroes[i]
        }
        var items = constants.items.itemdata
        constants.item_ids = {}
        for(var key in items) {
            constants.item_ids[items[key].id] = key
            items[key].img = "http://cdn.dota2.com/apps/dota2/images/items/" + items[key].img
        }
        constants.items = items
        var lookup = {}
        var ability_ids = constants.ability_ids.abilities
        for(var i = 0; i < ability_ids.length; i++) {
            lookup[ability_ids[i].id] = ability_ids[i].name
        }
        constants.ability_ids = lookup
        constants.ability_ids["5601"] = "techies_suicide"
        constants.ability_ids["5088"] = "skeleton_king_mortal_strike"
        constants.ability_ids["5060"] = "nevermore_shadowraze1"
        constants.ability_ids["5061"] = "nevermore_shadowraze1"
        var abilities = constants.abilities.abilitydata
        for(var key in abilities) {
            abilities[key].img = "http://cdn.dota2.com/apps/dota2/images/abilities/" + key + "_md.png"
        }
        abilities["stats"] = {
            dname: "Stats",
            img: '../../public/images/Stats.png'
        }
        constants.abilities = abilities
        var lookup = {}
        var regions = constants.regions.regions
        for(var i = 0; i < regions.length; i++) {
            lookup[regions[i].id] = regions[i].name
        }
        constants.regions = lookup
        constants.regions["251"] = "Peru"
        utility.constants.update({}, constants, {
            upsert: true
        }, function(err) {
            logger.info("[CONSTANTS] updated constants")
            cb(null)
        })
    })
}

function buildLookup(array) {
    var lookup = {}
    for(var i = 0; i < array.length; i++) {
        lookup[array[i].id] = array[i]
    }
    return lookup
}

function getMatches() {
    players.find({
        track: 1
    }, function(err, docs) {
        //rebuild set of tracked players before every check
        trackedPlayers = {}
        docs.forEach(function(player) {
            trackedPlayers[player.account_id] = true
        })
        aq.push({}, function(err) {})
    })
}

function requestParse(match) {
    request.get('http://localhost:' + (process.env.KUE_PORT || 5001) + '/job/search?q=' + match.match_id, {
        'auth': {
            'user': process.env.KUE_USER,
            'pass': process.env.KUE_PASS
        }
    }, function(err, res, body) {
        if(!err && res.statusCode === 200) {
            if(JSON.parse(body).length === 0) {
                jobs.create('parse', {
                    title: match.match_id,
                    match: {
                        match_id: match.match_id,
                        start_time: match.start_time
                    }
                }).priority('high').attempts(5).backoff({
                    delay: 30000,
                    type: 'exponential'
                }).searchKeys(['title']).save(function(err) {
                    if(!err) logger.info('[KUE] Parse added for ' + match.match_id)
                        });
            } else {
                logger.info('[KUE] ' + match.match_id + ' already queued.')
            }
        } else {
            logger.info('[KUE] Could not connect to Kue server.')
        }
    })
}

function findStuckJobs() {
    logger.info('[KUE] Looking for stuck jobs.')

    kue.Job.rangeByType('parse', 'active', 0, 10, 'ASC', function(err, ids) {
        if(!err) {
            ids.forEach(function(job){
                if (Date.now() - job.updated_at > jobTimeout) {
                    job.state('inactive', function(err){
                        if (err) logger.info('[KUE] Failed to move from active to inactive.')
                        else logger.info('[KUE] Moved ' + job.data.match.match_id + ' back to queue.')
                    })
                }
            })
        } else {
            logger.info('[KUE] Could not connect to Kue server.')
        }
    })
}

function requestDetails(match, cb) {
    matches.findOne({
        match_id: match.match_id
    }, function(err, doc) {
        if(!doc && !(match.match_id in queuedMatches)) {
            queuedMatches[match.match_id] = true
            aq.push(match, function(err) {})
        }
        cb(null)
    })
}

function getMatchPage(url, cb) {
    request(url, function(err, resp, body) {
        logger.info("[REMOTE] %s", url)
        var parsedHTML = cheerio.load(body);
        var matchCells = parsedHTML('td[class=cell-xlarge]')
        matchCells.each(function(i, matchCell) {
            var match_url = remote + cheerio(matchCell).children().first().attr('href');
            var match = {}
            match.match_id = Number(match_url.split(/[/]+/).pop());
            requestDetails(match, function(err) {})
        })
        var nextPath = parsedHTML('a[rel=next]').first().attr('href')
        if(nextPath) {
            getMatchPage(remote + nextPath, cb);
        } else {
            cb(null)
        }
    })
}
/*
 * Processes a request to an api
 */

function apiRequest(req, cb) {
    setTimeout(function() {
        logger.info("[QUEUE] api requests: %s", aq.length())
        var url;
        if(req.match_id) {
            url = api_url + "/GetMatchDetails/V001/?key=" + process.env.STEAM_API_KEY + "&match_id=" + req.match_id;
        } else if(req.summaries_id) {
            url = summaries_url + "/GetPlayerSummaries/v0002/?key=" + process.env.STEAM_API_KEY + "&steamids=" + req.query
        } else if(req.account_id) {
            url = api_url + "/GetMatchHistory/V001/?key=" + process.env.STEAM_API_KEY + "&account_id=" + req.account_id
        } else {
            url = api_url + "/GetMatchHistoryBySequenceNum/V001/?key=" + process.env.STEAM_API_KEY + "&start_at_match_seq_num=" + next_seq
        }
        getData(url, function(err, data) {
            if(data.response) {
                async.map(data.response.players, insertPlayer, function(err) {
                    cb(null)
                })
            } else if(data.result.error || data.result.status == 2) {
                logger.info(data)
                return cb(null)
            } else if(req.match_id) {
                var match = data.result
                insertMatch(match, function(err) {
                    delete queuedMatches[match.match_id]
                    cb(null)
                })
            } else {
                var resp = data.result.matches
                if(req.account_id) {
                    async.map(resp, function(match, cb) {
                        requestDetails(match, function(err) {
                            cb(null)
                        })
                    }, function(err) {
                        cb(null)
                    })
                } else {
                    logger.info("[API] seq_num: %s, found %s matches", next_seq, resp.length)
                    async.mapSeries(resp, insertMatch, function(err) {
                        if(resp.length > 0) {
                            next_seq = resp[resp.length - 1].match_seq_num + 1
                        }
                        cb(null)
                    })
                }
            }
        })
    }, 1000)
}

function insertMatch(match, cb) {
    var track = match.players.some(function(element) {
        return(element.account_id in trackedPlayers)
    })
    match.parse_status = (track ? 0 : 3)
    if(process.env.SAVE_ALL_MATCHES || track) {
        matches.insert(match);
    }
    if(track) {
        //todo get player summaries separately
        summaries = {}
        summaries.summaries_id = 1
        var steamids = []
        match.players.forEach(function(player) {
            steamids.push(utility.convert32to64(player.account_id).toString())
        })
        summaries.query = steamids.join()
        aq.unshift(summaries, function(err) {})
        requestParse(match)
    }
    cb(null)
}
/*
 * Inserts/updates a player in the database
 */

function insertPlayer(player, cb) {
    var account_id = Number(utility.convert64to32(player.steamid))
    players.update({
        account_id: account_id
    }, {
        $set: player
    }, {
        upsert: true
    }, function(err) {
        cb(err)
    })
}

function getData(url, cb) {
    request(url, function(err, res, body) {
        logger.info("[API] %s", url)
        if(err || res.statusCode != 200 || !body) {
            logger.info("[API] error getting data, retrying")
            return getData(url, cb)
        } else {
            cb(null, JSON.parse(body))
        }
    })
}<commit_msg>increase timeout interval<commit_after>var async = require("async"),
    express = require('express'),
    auth = require('http-auth'),
    request = require('request'),
    utility = require('./utility'),
    matches = utility.matches,
    players = utility.players,
    cheerio = require('cheerio'),
    kue = require('kue'),
    winston = require('winston');
var jobs = kue.createQueue();
var memwatch = require('memwatch');
var request = require("request");
var aq = async.queue(apiRequest, 1)
var api_url = "https://api.steampowered.com/IDOTA2Match_570"
var summaries_url = "http://api.steampowered.com/ISteamUser"
var remote = "http://dotabuff.com"
var queuedMatches = {}
var trackedPlayers = {}
var jobTimeout = 5 * 60 * 1000 // Job timeout for kue
var next_seq;
var logger = new(winston.Logger)({
    transports: [
        new(winston.transports.File)({
            filename: 'backend.log',
            level: 'info'
        })
    ]
});
var basic = auth.basic({
    realm: "Kue"
}, function(username, password, callback) { // Custom authentication method.
    callback(username === process.env.KUE_USER && password === process.env.KUE_PASS);
});
var app = express();
app.use(auth.connect(basic));
app.use(kue.app);
app.listen(process.env.KUE_PORT || 5001);

setInterval(findStuckJobs, jobTimeout)

memwatch.on('leak', function(info) {
    logger.info('[LEAK]' + info);
});
updateConstants(function(err) {});
async.series([
    function(cb) {
        //scrape players full match history
        players.find({
            full_history: 1
        }, function(err, docs) {
            async.mapSeries(docs, function(player, cb2) {
                var account_id = player.account_id
                var player_url = remote + "/players/" + account_id + "/matches"
                getMatchPage(player_url, function(err) {
                    //done scraping player
                    players.update({
                        account_id: account_id
                    }, {
                        $set: {
                            full_history: 0
                        }
                    })
                    cb2(null)
                })
            }, function(err) {
                //done scraping all players
                cb(null)
            })
        })
    },
    function(cb) {
        //check most recent 100 matches for tracked players
        players.find({
            track: 1
        }, function(err, docs) {
            aq.push(docs, function(err) {})
        })
        //parse unparsed matches
        matches.find({
            parse_status: 0
        }, function(err, docs) {
            docs.forEach(function(match) {
                requestParse(match)
            })
        })

        cb(null)
    },
    function(cb) {
        //determine sequence number to begin scan at
        if(process.env.SAVE_ALL_MATCHES) {
            matches.findOne({}, {
                sort: {
                    match_seq_num: -1
                }
            }, function(err, doc) {
                next_seq = doc ? doc.match_seq_num + 1 : 0
                cb(null)
            })
        } else {
            getData(api_url + "/GetMatchHistory/V001/?key=" + process.env.STEAM_API_KEY, function(err, data) {
                next_seq = data.result.matches[0].match_seq_num
                cb(null)
            })
        }
    }
], function(err) {
    getMatches()
    aq.empty = function() {
        getMatches();
    }
})

function updateConstants(cb) {
    var constants = require('./constants.json')
    async.map(Object.keys(constants), function(key, cb) {
        var val = constants[key]
        if(typeof(val) == "string" && val.slice(0, 4) == "http") {
            getData(val, function(err, result) {
                constants[key] = result
                cb(null)
            })
        } else {
            cb(null)
        }
    }, function(err) {
        var heroes = constants.heroes
        heroes.forEach(function(hero) {
            hero.name = hero.name.replace("npc_dota_hero_", "")
            hero.img = "http://cdn.dota2.com/apps/dota2/images/heroes/" + hero.name + "_sb.png"
        })
        constants.heroes = buildLookup(heroes)
        constants.hero_names = {}
        for(var i = 0; i < heroes.length; i++) {
            constants.hero_names[heroes[i].name] = heroes[i]
        }
        var items = constants.items.itemdata
        constants.item_ids = {}
        for(var key in items) {
            constants.item_ids[items[key].id] = key
            items[key].img = "http://cdn.dota2.com/apps/dota2/images/items/" + items[key].img
        }
        constants.items = items
        var lookup = {}
        var ability_ids = constants.ability_ids.abilities
        for(var i = 0; i < ability_ids.length; i++) {
            lookup[ability_ids[i].id] = ability_ids[i].name
        }
        constants.ability_ids = lookup
        constants.ability_ids["5601"] = "techies_suicide"
        constants.ability_ids["5088"] = "skeleton_king_mortal_strike"
        constants.ability_ids["5060"] = "nevermore_shadowraze1"
        constants.ability_ids["5061"] = "nevermore_shadowraze1"
        var abilities = constants.abilities.abilitydata
        for(var key in abilities) {
            abilities[key].img = "http://cdn.dota2.com/apps/dota2/images/abilities/" + key + "_md.png"
        }
        abilities["stats"] = {
            dname: "Stats",
            img: '../../public/images/Stats.png'
        }
        constants.abilities = abilities
        var lookup = {}
        var regions = constants.regions.regions
        for(var i = 0; i < regions.length; i++) {
            lookup[regions[i].id] = regions[i].name
        }
        constants.regions = lookup
        constants.regions["251"] = "Peru"
        utility.constants.update({}, constants, {
            upsert: true
        }, function(err) {
            logger.info("[CONSTANTS] updated constants")
            cb(null)
        })
    })
}

function buildLookup(array) {
    var lookup = {}
    for(var i = 0; i < array.length; i++) {
        lookup[array[i].id] = array[i]
    }
    return lookup
}

function getMatches() {
    players.find({
        track: 1
    }, function(err, docs) {
        //rebuild set of tracked players before every check
        trackedPlayers = {}
        docs.forEach(function(player) {
            trackedPlayers[player.account_id] = true
        })
        aq.push({}, function(err) {})
    })
}

function requestParse(match) {
    request.get('http://localhost:' + (process.env.KUE_PORT || 5001) + '/job/search?q=' + match.match_id, {
        'auth': {
            'user': process.env.KUE_USER,
            'pass': process.env.KUE_PASS
        }
    }, function(err, res, body) {
        if(!err && res.statusCode === 200) {
            if(JSON.parse(body).length === 0) {
                jobs.create('parse', {
                    title: match.match_id,
                    match: {
                        match_id: match.match_id,
                        start_time: match.start_time
                    }
                }).priority('high').attempts(5).backoff({
                    delay: 30000,
                    type: 'exponential'
                }).searchKeys(['title']).save(function(err) {
                    if(!err) logger.info('[KUE] Parse added for ' + match.match_id)
                        });
            } else {
                logger.info('[KUE] ' + match.match_id + ' already queued.')
            }
        } else {
            logger.info('[KUE] Could not connect to Kue server.')
        }
    })
}

function findStuckJobs() {
    logger.info('[KUE] Looking for stuck jobs.')

    kue.Job.rangeByType('parse', 'active', 0, 20, 'ASC', function(err, ids) {
        if(!err) {
            ids.forEach(function(job){
                if (Date.now() - job.updated_at > jobTimeout) {
                    job.state('inactive', function(err){
                        if (err) logger.info('[KUE] Failed to move from active to inactive.')
                        else logger.info('[KUE] Moved ' + job.data.match.match_id + ' back to queue.')
                    })
                }
            })
        } else {
            logger.info('[KUE] Could not connect to Kue server.')
        }
    })
}

function requestDetails(match, cb) {
    matches.findOne({
        match_id: match.match_id
    }, function(err, doc) {
        if(!doc && !(match.match_id in queuedMatches)) {
            queuedMatches[match.match_id] = true
            aq.push(match, function(err) {})
        }
        cb(null)
    })
}

function getMatchPage(url, cb) {
    request(url, function(err, resp, body) {
        logger.info("[REMOTE] %s", url)
        var parsedHTML = cheerio.load(body);
        var matchCells = parsedHTML('td[class=cell-xlarge]')
        matchCells.each(function(i, matchCell) {
            var match_url = remote + cheerio(matchCell).children().first().attr('href');
            var match = {}
            match.match_id = Number(match_url.split(/[/]+/).pop());
            requestDetails(match, function(err) {})
        })
        var nextPath = parsedHTML('a[rel=next]').first().attr('href')
        if(nextPath) {
            getMatchPage(remote + nextPath, cb);
        } else {
            cb(null)
        }
    })
}
/*
 * Processes a request to an api
 */

function apiRequest(req, cb) {
    setTimeout(function() {
        logger.info("[QUEUE] api requests: %s", aq.length())
        var url;
        if(req.match_id) {
            url = api_url + "/GetMatchDetails/V001/?key=" + process.env.STEAM_API_KEY + "&match_id=" + req.match_id;
        } else if(req.summaries_id) {
            url = summaries_url + "/GetPlayerSummaries/v0002/?key=" + process.env.STEAM_API_KEY + "&steamids=" + req.query
        } else if(req.account_id) {
            url = api_url + "/GetMatchHistory/V001/?key=" + process.env.STEAM_API_KEY + "&account_id=" + req.account_id
        } else {
            url = api_url + "/GetMatchHistoryBySequenceNum/V001/?key=" + process.env.STEAM_API_KEY + "&start_at_match_seq_num=" + next_seq
        }
        getData(url, function(err, data) {
            if(data.response) {
                async.map(data.response.players, insertPlayer, function(err) {
                    cb(null)
                })
            } else if(data.result.error || data.result.status == 2) {
                logger.info(data)
                return cb(null)
            } else if(req.match_id) {
                var match = data.result
                insertMatch(match, function(err) {
                    delete queuedMatches[match.match_id]
                    cb(null)
                })
            } else {
                var resp = data.result.matches
                if(req.account_id) {
                    async.map(resp, function(match, cb) {
                        requestDetails(match, function(err) {
                            cb(null)
                        })
                    }, function(err) {
                        cb(null)
                    })
                } else {
                    logger.info("[API] seq_num: %s, found %s matches", next_seq, resp.length)
                    async.mapSeries(resp, insertMatch, function(err) {
                        if(resp.length > 0) {
                            next_seq = resp[resp.length - 1].match_seq_num + 1
                        }
                        cb(null)
                    })
                }
            }
        })
    }, 1000)
}

function insertMatch(match, cb) {
    var track = match.players.some(function(element) {
        return(element.account_id in trackedPlayers)
    })
    match.parse_status = (track ? 0 : 3)
    if(process.env.SAVE_ALL_MATCHES || track) {
        matches.insert(match);
    }
    if(track) {
        //todo get player summaries separately
        summaries = {}
        summaries.summaries_id = 1
        var steamids = []
        match.players.forEach(function(player) {
            steamids.push(utility.convert32to64(player.account_id).toString())
        })
        summaries.query = steamids.join()
        aq.unshift(summaries, function(err) {})
        requestParse(match)
    }
    cb(null)
}
/*
 * Inserts/updates a player in the database
 */

function insertPlayer(player, cb) {
    var account_id = Number(utility.convert64to32(player.steamid))
    players.update({
        account_id: account_id
    }, {
        $set: player
    }, {
        upsert: true
    }, function(err) {
        cb(err)
    })
}

function getData(url, cb) {
    request(url, function(err, res, body) {
        logger.info("[API] %s", url)
        if(err || res.statusCode != 200 || !body) {
            logger.info("[API] error getting data, retrying")
            return getData(url, cb)
        } else {
            cb(null, JSON.parse(body))
        }
    })
}<|endoftext|>


<commit_before>/**
 * @namespace {FORGEProgram} FORGE.ShaderLib
 */

//jscs:disable

FORGE.ShaderLib = {

    screenToWorld:
    {
        rectilinear:
        {
            /** @type {FORGEUniform} */
            uniforms:
            {
                tOpacity: { type: "f", value: 1.0 },
                tTexture: { type: "t", value: null },
                tViewportResolution: { type: "v2", value: new THREE.Vector2() },
                tViewportResolutionRatio: { type: "f", value: 1.0 },
                tModelViewMatrixInverse: { type: "m4", value: new THREE.Matrix4() },
                tProjectionScale: { type: "f", value: 1.0 }
            },

            vertexShader: FORGE.ShaderChunk.stw_vert_proj,
            fragmentShader: FORGE.ShaderChunk.stw_frag_proj_rectilinear
        },

        gopro:
        {
            /** @type {FORGEUniform} */
            uniforms:
            {
                tOpacity: { type: "f", value: 1.0 },
                tTexture: { type: "t", value: null },
                tViewportResolution: { type: "v2", value: new THREE.Vector2() },
                tViewportResolutionRatio: { type: "f", value: 1.0 },
                tModelViewMatrixInverse: { type: "m4", value: new THREE.Matrix4() },
                tProjectionDistance: { type: "f", value: 0.0 },
                tProjectionScale: { type: "f", value: 1.0 }
            },

            vertexShader: FORGE.ShaderChunk.stw_vert_proj,
            fragmentShader: FORGE.ShaderChunk.stw_frag_proj_gopro
        }
    },

    worldToScreen:
    {
        rectilinear:
        {
            mapping:
            {
                /** @type {FORGEUniform} */
                uniforms:
                {
                    tOpacity: { type: "f", value: 1.0 },
                    tTexture: { type: "t", value: null },
                    tProjectionScale: { type: "f", value: 1 },
                    tModelViewMatrixInverse: { type: "m4", value: null }
                },
                vertexShader: FORGE.ShaderChunk.wts_vert_rectilinear,
                fragmentShader: FORGE.ShaderChunk.wts_frag
            },

            equirectangular:
            {
                /** @type {FORGEUniform} */
                uniforms:
                {
                    tOpacity: { type: "f", value: 1.0 },
                    tTexture: { type: "t", value: null },
                    tProjectionScale: { type: "f", value: 1 },
                    tViewportResolutionRatio: { type: "f", value: 1.0 },
                    tModelViewMatrixInverse: { type: "m4", value: null }
                },
                vertexShader: FORGE.ShaderChunk.wts_vert_rectilinear_equirectangular,
                fragmentShader: FORGE.ShaderChunk.wts_frag_equirectangular
            }
        },

        littlePlanet:
        {
            equirectangular:
            {
                /** @type {FORGEUniform} */
                uniforms:
                {
                    tOpacity: { type: "f", value: 1.0 },
                    tTexture: { type: "t", value: null },
                    tProjectionScale: { type: "f", value: 1 },
                    tViewportResolutionRatio: { type: "f", value: 1.0 },
                    tModelViewMatrixInverse: { type: "m4", value: null }
                },
                vertexShader: FORGE.ShaderChunk.wts_vert_littleplanet_equirectangular,
                fragmentShader: FORGE.ShaderChunk.wts_frag_equirectangular
            }
        },

        gopro:
        {
            mapping:
            {
                /** @type {FORGEUniform} */
                uniforms:
                {
                    tOpacity: { type: "f", value: 1.0 },
                    tTexture: { type: "t", value: null },
                    tProjectionScale: { type: "f", value: 1 },
                    tProjectionDistance: { type: "f", value: 1 },
                    tModelViewMatrixInverse: { type: "m4", value: null }
                },
                vertexShader: FORGE.ShaderChunk.wts_vert_gopro,
                fragmentShader: FORGE.ShaderChunk.wts_frag
            },

            equirectangular:
            {
                /** @type {FORGEUniform} */
                uniforms:
                {
                    tOpacity: { type: "f", value: 1.0 },
                    tTexture: { type: "t", value: null },
                    tProjectionScale: { type: "f", value: 1 },
                    tProjectionDistance: { type: "f", value: 1 },
                    tViewportResolutionRatio: { type: "f", value: 1.0 },
                    tModelViewMatrixInverse: { type: "m4", value: null }
                },
                vertexShader: FORGE.ShaderChunk.wts_vert_gopro_equirectangular,
                fragmentShader: FORGE.ShaderChunk.wts_frag_equirectangular
            }
        }
    }
};

//jscs:enable

FORGE.ShaderLib.parseIncludes = function(string)
{
    var pattern = /#include +<([\w\d.]+)>/g;

    function replace( match, include )
    {
        var r = FORGE.ShaderChunk[include];

        if (typeof r === "undefined")
        {
            r = THREE.ShaderChunk[include];

            if (typeof r === "undefined")
            {
                throw new Error( "Can not resolve #include <" + include + ">" );
            }
        }

        return FORGE.ShaderLib.parseIncludes(r);
    }

    return string.replace( pattern, replace );
};
<commit_msg>ShaderLib - add wireframe wts shader definitions<commit_after>/**
 * @namespace {FORGEProgram} FORGE.ShaderLib
 */

//jscs:disable

FORGE.ShaderLib = {

    screenToWorld:
    {
        rectilinear:
        {
            /** @type {FORGEUniform} */
            uniforms:
            {
                tOpacity: { type: "f", value: 1.0 },
                tTexture: { type: "t", value: null },
                tViewportResolution: { type: "v2", value: new THREE.Vector2() },
                tViewportResolutionRatio: { type: "f", value: 1.0 },
                tModelViewMatrixInverse: { type: "m4", value: new THREE.Matrix4() },
                tProjectionScale: { type: "f", value: 1.0 }
            },

            vertexShader: FORGE.ShaderChunk.stw_vert_proj,
            fragmentShader: FORGE.ShaderChunk.stw_frag_proj_rectilinear
        },

        gopro:
        {
            /** @type {FORGEUniform} */
            uniforms:
            {
                tOpacity: { type: "f", value: 1.0 },
                tTexture: { type: "t", value: null },
                tViewportResolution: { type: "v2", value: new THREE.Vector2() },
                tViewportResolutionRatio: { type: "f", value: 1.0 },
                tModelViewMatrixInverse: { type: "m4", value: new THREE.Matrix4() },
                tProjectionDistance: { type: "f", value: 0.0 },
                tProjectionScale: { type: "f", value: 1.0 }
            },

            vertexShader: FORGE.ShaderChunk.stw_vert_proj,
            fragmentShader: FORGE.ShaderChunk.stw_frag_proj_gopro
        }
    },

    worldToScreen:
    {
        rectilinear:
        {
            mapping:
            {
                /** @type {FORGEUniform} */
                uniforms:
                {
                    tOpacity: { type: "f", value: 1.0 },
                    tTexture: { type: "t", value: null },
                    tProjectionScale: { type: "f", value: 1 },
                    tModelViewMatrixInverse: { type: "m4", value: null }
                },
                vertexShader: FORGE.ShaderChunk.wts_vert_rectilinear,
                fragmentShader: FORGE.ShaderChunk.wts_frag
            },

            wireframe:
            {
                /** @type {FORGEUniform} */
                uniforms:
                {
                    tColor: { type: "c", value: null },
                    tProjectionScale: { type: "f", value: 1 },
                    tModelViewMatrixInverse: { type: "m4", value: null }
                },
                vertexShader: FORGE.ShaderChunk.wts_vert_rectilinear_wireframe,
                fragmentShader: FORGE.ShaderChunk.wts_frag_wireframe
            },

            equirectangular:
            {
                /** @type {FORGEUniform} */
                uniforms:
                {
                    tOpacity: { type: "f", value: 1.0 },
                    tTexture: { type: "t", value: null },
                    tProjectionScale: { type: "f", value: 1 },
                    tViewportResolutionRatio: { type: "f", value: 1.0 },
                    tModelViewMatrixInverse: { type: "m4", value: null }
                },
                vertexShader: FORGE.ShaderChunk.wts_vert_rectilinear_equirectangular,
                fragmentShader: FORGE.ShaderChunk.wts_frag_equirectangular
            }
        },

        littlePlanet:
        {
            equirectangular:
            {
                /** @type {FORGEUniform} */
                uniforms:
                {
                    tOpacity: { type: "f", value: 1.0 },
                    tTexture: { type: "t", value: null },
                    tProjectionScale: { type: "f", value: 1 },
                    tViewportResolutionRatio: { type: "f", value: 1.0 },
                    tModelViewMatrixInverse: { type: "m4", value: null }
                },
                vertexShader: FORGE.ShaderChunk.wts_vert_littleplanet_equirectangular,
                fragmentShader: FORGE.ShaderChunk.wts_frag_equirectangular
            }
        },

        gopro:
        {
            mapping:
            {
                /** @type {FORGEUniform} */
                uniforms:
                {
                    tOpacity: { type: "f", value: 1.0 },
                    tTexture: { type: "t", value: null },
                    tProjectionScale: { type: "f", value: 1 },
                    tProjectionDistance: { type: "f", value: 1 },
                    tModelViewMatrixInverse: { type: "m4", value: null }
                },
                vertexShader: FORGE.ShaderChunk.wts_vert_gopro,
                fragmentShader: FORGE.ShaderChunk.wts_frag
            },

            wireframe:
            {
                /** @type {FORGEUniform} */
                uniforms:
                {
                    tColor: { type: "c", value: null },
                    tProjectionScale: { type: "f", value: 1 },
                    tProjectionDistance: { type: "f", value: 1 },
                    tModelViewMatrixInverse: { type: "m4", value: null }
                },
                vertexShader: FORGE.ShaderChunk.wts_vert_gopro_wireframe,
                fragmentShader: FORGE.ShaderChunk.wts_frag_wireframe
            },

            equirectangular:
            {
                /** @type {FORGEUniform} */
                uniforms:
                {
                    tOpacity: { type: "f", value: 1.0 },
                    tTexture: { type: "t", value: null },
                    tProjectionScale: { type: "f", value: 1 },
                    tProjectionDistance: { type: "f", value: 1 },
                    tViewportResolutionRatio: { type: "f", value: 1.0 },
                    tModelViewMatrixInverse: { type: "m4", value: null }
                },
                vertexShader: FORGE.ShaderChunk.wts_vert_gopro_equirectangular,
                fragmentShader: FORGE.ShaderChunk.wts_frag_equirectangular
            }
        }
    }
};

//jscs:enable

FORGE.ShaderLib.parseIncludes = function(string)
{
    var pattern = /#include +<([\w\d.]+)>/g;

    function replace( match, include )
    {
        var r = FORGE.ShaderChunk[include];

        if (typeof r === "undefined")
        {
            r = THREE.ShaderChunk[include];

            if (typeof r === "undefined")
            {
                throw new Error( "Can not resolve #include <" + include + ">" );
            }
        }

        return FORGE.ShaderLib.parseIncludes(r);
    }

    return string.replace( pattern, replace );
};
<|endoftext|>


<commit_before>import Datas from 'src/main/datas';
import API from 'src/util/api';
import UI from 'src/util/ui';

import Roc from '../rest-on-couch/Roc';

import ExpandableMolecule from './ExpandableMolecule';
import Nmr1dManager from './Nmr1dManager';
import MF from './MF';
import { createVar } from './jpaths';
import elnPlugin from './libs/elnPlugin';
import CCE from './libs/CCE';

const DataObject = Datas.DataObject;

var defaultOptions = {
  varName: 'sample',
  track: false,
  bindChange: true
};

class Sample {
  constructor(couchDB, uuid, options) {
    this.options = Object.assign({}, defaultOptions, options);

    var roc = API.cache('roc');
    if (!roc) {
      roc = new Roc({
        url: couchDB.url,
        database: couchDB.database,
        processor: elnPlugin,
        kind: couchDB.kind
      });
      API.cache('roc', roc);
    }
    this.roc = roc;

    if (options.onSync) {
      var emitter = this.roc.getDocumentEventEmitter(uuid);
      emitter.on('sync', () => options.onSync(true));
      emitter.on('unsync', () => options.onSync(false));
    }

    this.uuid = uuid;
    if (!this.uuid) {
      UI.showNotification('Cannot create an editable sample without an uuid', 'error');
      return;
    }
    this._loadInstanceInVisualizer();
  }

  async _loadInstanceInVisualizer() {
    this.sample = await this.roc.document(this.uuid, this.options);

    if (!this.sample.$content.general) {
      this.sample.$content.general = {};
    }

    var sampleVar = API.getVar(this.options.varName);

    createVar(sampleVar, 'sampleCode');
    createVar(sampleVar, 'batchCode');
    createVar(sampleVar, 'creationDate');
    createVar(sampleVar, 'modificationDate');
    createVar(sampleVar, 'content');
    createVar(sampleVar, 'general');
    createVar(sampleVar, 'molfile');
    createVar(sampleVar, 'sequence');
    createVar(sampleVar, 'mf');
    createVar(sampleVar, 'mw');
    createVar(sampleVar, 'em');
    createVar(sampleVar, 'title');
    createVar(sampleVar, 'description');
    createVar(sampleVar, 'keyword');
    createVar(sampleVar, 'name');
    createVar(sampleVar, 'physical');
    createVar(sampleVar, 'bp');
    createVar(sampleVar, 'nd');
    createVar(sampleVar, 'mp');
    createVar(sampleVar, 'density');
    createVar(sampleVar, 'ir');
    createVar(sampleVar, 'mass');
    createVar(sampleVar, 'nmr');
    createVar(sampleVar, 'xray');
    createVar(sampleVar, 'chromatogram');
    createVar(sampleVar, 'thermogravimetricAnalysis');
    createVar(sampleVar, 'differentialScanningCalorimetry');
    createVar(sampleVar, 'image');
    createVar(sampleVar, 'sampleCode');
    createVar(sampleVar, 'attachments');
    createVar(sampleVar, 'nucleic');

    this._initializeObjects();

    this.onChange = (event) => {
      var jpathStr = event.jpath.join('.');
      if (jpathStr.match(/\$content.spectra.nmr.[0-9]+.range/)) {
        this.nmr1dManager.rangesHasChanged();
      }

      switch (event.jpath.join('.')) {
        case '$content.general.molfile':
          this.mf.fromMolfile();
          this.nmr1dManager.handleAction({ name: 'clearAllAssignments' });
          break;
        case '$content.general.mf':
          try {
            this.mf.fromMF();
            this.nmr1dManager.updateIntegralOptionsFromMF();
          } catch (e) {
            console.log(e); // eslint-disable-line no-console
          }
          break;
        case '$content.general.sequence':
          try {
            var sequenceOriginal = `${this.sample.getChildSync(['$content', 'general', 'sequence']) || ''}`;
            var sequence = CCE.convertAASequence(sequenceOriginal);
            this.sample.setChildSync(['$content', 'general', 'mf'], sequence);
          } catch (e) {
            console.log(e); // eslint-disable-line no-console
          }
          break;
        default:
          break; // ignore
      }
    };

    this.bindChange();
  }

  _initializeObjects() {
    this.expandableMolecule = new ExpandableMolecule(this.sample, this.options);
    this.nmr1dManager = new Nmr1dManager(this.sample);

    this.mf = new MF(this.sample);
    this.mf.fromMF();
  }

  bindChange() {
    if (this.options.bindChange) {
      this.sample.unbindChange(this.onChange);
      this.sample.onChange(this.onChange);
    }
  }

  unbindChange() {
    if (this.options.bindChange) this.sample.unbindChange(this.onChange);
  }

  /** An image with a special name that is used to display on the
     * first page of a sample
     */
  async handleOverview(variableName) {
    let data = API.getData(variableName);
    if (data && data.file && data.file[0]) {
      let file = data.file[0];
      // we only accepts 3 mimetype
      switch (file.mimetype) {
        case 'image/png':
          file.filename = 'overview.png';
          break;
        case 'image/jpeg':
          file.filename = 'overview.jpg';
          break;
        case 'image/svg+xml':
          file.filename = 'overview.svg';
          break;
        default:
          UI.showNotification('For overview only the following formats are allowed: png, jpg and svg.', 'error');
          return undefined;
      }
      return this.handleDrop(variableName, false);
    }
    return undefined;
  }

  async handleDrop(name, askType) {
    var type;
    if (!name) {
      throw new Error('handleDrop expects a variable name');
    }
    name = String(name);
    if (!askType) {
      // maps name of variable to type of data
      var types = {
        droppedNmr: 'nmr',
        droppedIR: 'ir',
        droppedMS: 'mass',
        droppedXray: 'xray',
        droppedOverview: 'image',
        droppedImage: 'image',
        droppedGenbank: 'genbank'
      };
      if (!types[name]) {
        throw new Error('Unexpected variable name');
      }
      type = types[name];
    } else {
      type = await UI.choose({
        nmr: 'NMR (jcamp, pdf)',
        mass: 'Mass (jcamp, pdf, netcdf, xml)',
        ir: 'Infrared (jcamp, pdf)',
        chromatogram: 'Chromatogram LC, GC, LC/MS, GC/MS (jcamp, pdf, netcdf, xml)',
        thermogravimetricAnalysis: 'Thermogravimetric Analysis (txt)',
        differentialScanningCalorimetry: 'Differential Scanning Calorimetry (txt)',
        xray: 'Xray (cif, pdb)',
        image: 'Images (jpg, png or tiff)',
        other: 'Other'
      }, {
        noConfirmation: true,
        columns: [
          {
            id: 'description',
            name: 'description',
            field: 'description'
          }
        ]
      });
      if (!type) return;
    }

    // Dropped data can be an array
    // Expecting format as from drag and drop module
    var droppedDatas = API.getData(name);
    droppedDatas = droppedDatas.file || droppedDatas.str;

    if (type === 'other') {
      await this.roc.addAttachment(this.sample, droppedDatas);
    } else {
      await this.attachFiles(droppedDatas, type);
    }
  }

  async handleAction(action) {
    if (!action) return;

    if (this.expandableMolecule && this.expandableMolecule.handleAction(action)) return;
    if (this.nmr1dManager && this.nmr1dManager.handleAction(action)) return;

    switch (action.name) {
      case 'save':
        await this.roc.update(this.sample);
        break;
      case 'createOptions':
        var advancedOptions1H = API.cache('nmr1hAdvancedOptions');
        if (advancedOptions1H) {
          API.createData('nmr1hOndeTemplate', API.cache('nmr1hOndeTemplates').full);
        } else {
          API.createData('nmr1hOndeTemplate', API.cache('nmr1hOndeTemplates').short);
        }
        break;
      case 'deleteAttachment':
        var attachment = action.value.name;
        await this.roc.deleteAttachment(this.sample, attachment);
        break;
      case 'deleteNmr': // Deprecated. Use unattach. Leave this for backward compatibility
      case 'unattach':
        await this.roc.unattach(this.sample, action.value);
        break;
      case 'attachNMR':
      case 'attachIR':
      case 'attachMass': {
        var tempType = action.name.replace('attach', '');
        var type = tempType.charAt(0).toLowerCase() + tempType.slice(1);
        var droppedDatas = action.value;
        droppedDatas = droppedDatas.file || droppedDatas.str;
        await this.attachFiles(droppedDatas, type);
        break;
      }
      case 'refresh': {
        const ok = await UI.confirm('Are you sure you want to refresh? This will discard your local modifications.');
        if (!ok) return;
        this.unbindChange();
        this.expandableMolecule.unbindChange();
        await this.roc.discardLocal(this.sample);
        this._initializeObjects();
        this.bindChange();
        this.nmr1dManager.handleAction({ name: 'nmrChanged' });
        break;
      }
      default:
        break;
    }
  }

  async attachFiles(files, type) {
    if (!files || !type) return;

    if (!Array.isArray(files)) {
      files = [files];
    }
    for (let i = 0; i < files.length; i++) {
      const data = DataObject.resurrect(files[i]);
      await this.roc.attach(type, this.sample, data);
    }
  }
}

module.exports = Sample;
<commit_msg>explode nucleic sequence<commit_after>import Datas from 'src/main/datas';
import API from 'src/util/api';
import UI from 'src/util/ui';

import Roc from '../rest-on-couch/Roc';

import ExpandableMolecule from './ExpandableMolecule';
import Nmr1dManager from './Nmr1dManager';
import MF from './MF';
import { createVar } from './jpaths';
import elnPlugin from './libs/elnPlugin';
import CCE from './libs/CCE';
import { explodeNucleic } from './nucleic/util';

const DataObject = Datas.DataObject;

var defaultOptions = {
  varName: 'sample',
  track: false,
  bindChange: true
};

class Sample {
  constructor(couchDB, uuid, options) {
    this.options = Object.assign({}, defaultOptions, options);

    var roc = API.cache('roc');
    if (!roc) {
      roc = new Roc({
        url: couchDB.url,
        database: couchDB.database,
        processor: elnPlugin,
        kind: couchDB.kind
      });
      API.cache('roc', roc);
    }
    this.roc = roc;

    if (options.onSync) {
      var emitter = this.roc.getDocumentEventEmitter(uuid);
      emitter.on('sync', () => options.onSync(true));
      emitter.on('unsync', () => options.onSync(false));
    }

    this.uuid = uuid;
    if (!this.uuid) {
      UI.showNotification('Cannot create an editable sample without an uuid', 'error');
      return;
    }
    this._loadInstanceInVisualizer();
  }

  async _loadInstanceInVisualizer() {
    this.sample = await this.roc.document(this.uuid, this.options);

    if (!this.sample.$content.general) {
      this.sample.$content.general = {};
    }

    var sampleVar = API.getVar(this.options.varName);

    createVar(sampleVar, 'sampleCode');
    createVar(sampleVar, 'batchCode');
    createVar(sampleVar, 'creationDate');
    createVar(sampleVar, 'modificationDate');
    createVar(sampleVar, 'content');
    createVar(sampleVar, 'general');
    createVar(sampleVar, 'molfile');
    createVar(sampleVar, 'sequence');
    createVar(sampleVar, 'mf');
    createVar(sampleVar, 'mw');
    createVar(sampleVar, 'em');
    createVar(sampleVar, 'title');
    createVar(sampleVar, 'description');
    createVar(sampleVar, 'keyword');
    createVar(sampleVar, 'name');
    createVar(sampleVar, 'physical');
    createVar(sampleVar, 'bp');
    createVar(sampleVar, 'nd');
    createVar(sampleVar, 'mp');
    createVar(sampleVar, 'density');
    createVar(sampleVar, 'ir');
    createVar(sampleVar, 'mass');
    createVar(sampleVar, 'nmr');
    createVar(sampleVar, 'xray');
    createVar(sampleVar, 'chromatogram');
    createVar(sampleVar, 'thermogravimetricAnalysis');
    createVar(sampleVar, 'differentialScanningCalorimetry');
    createVar(sampleVar, 'image');
    createVar(sampleVar, 'sampleCode');
    createVar(sampleVar, 'attachments');
    createVar(sampleVar, 'nucleic');

    this._initializeObjects();

    this.onChange = (event) => {
      var jpathStr = event.jpath.join('.');
      if (jpathStr.match(/\$content.spectra.nmr.[0-9]+.range/)) {
        this.nmr1dManager.rangesHasChanged();
      }

      switch (event.jpath.join('.')) {
        case '$content.biology.nucleic':
          API.createData('explodedNucleic', explodeNucleic(this.sample.$content.biology.nucleic))
          break;
        case '$content.general.molfile':
          this.mf.fromMolfile();
          this.nmr1dManager.handleAction({ name: 'clearAllAssignments' });
          break;
        case '$content.general.mf':
          try {
            this.mf.fromMF();
            this.nmr1dManager.updateIntegralOptionsFromMF();
          } catch (e) {
            console.log(e); // eslint-disable-line no-console
          }
          break;
        case '$content.general.sequence':
          try {
            var sequenceOriginal = `${this.sample.getChildSync(['$content', 'general', 'sequence']) || ''}`;
            var sequence = CCE.convertAASequence(sequenceOriginal);
            this.sample.setChildSync(['$content', 'general', 'mf'], sequence);
          } catch (e) {
            console.log(e); // eslint-disable-line no-console
          }
          break;
        default:
          break; // ignore
      }
    };

    this.bindChange();
  }

  _initializeObjects() {
    this.expandableMolecule = new ExpandableMolecule(this.sample, this.options);
    this.nmr1dManager = new Nmr1dManager(this.sample);

    this.mf = new MF(this.sample);
    this.mf.fromMF();
  }

  bindChange() {
    if (this.options.bindChange) {
      this.sample.unbindChange(this.onChange);
      this.sample.onChange(this.onChange);
    }
  }

  unbindChange() {
    if (this.options.bindChange) this.sample.unbindChange(this.onChange);
  }

  /** An image with a special name that is used to display on the
     * first page of a sample
     */
  async handleOverview(variableName) {
    let data = API.getData(variableName);
    if (data && data.file && data.file[0]) {
      let file = data.file[0];
      // we only accepts 3 mimetype
      switch (file.mimetype) {
        case 'image/png':
          file.filename = 'overview.png';
          break;
        case 'image/jpeg':
          file.filename = 'overview.jpg';
          break;
        case 'image/svg+xml':
          file.filename = 'overview.svg';
          break;
        default:
          UI.showNotification('For overview only the following formats are allowed: png, jpg and svg.', 'error');
          return undefined;
      }
      return this.handleDrop(variableName, false);
    }
    return undefined;
  }

  async handleDrop(name, askType) {
    var type;
    if (!name) {
      throw new Error('handleDrop expects a variable name');
    }
    name = String(name);
    if (!askType) {
      // maps name of variable to type of data
      var types = {
        droppedNmr: 'nmr',
        droppedIR: 'ir',
        droppedMS: 'mass',
        droppedXray: 'xray',
        droppedOverview: 'image',
        droppedImage: 'image',
        droppedGenbank: 'genbank'
      };
      if (!types[name]) {
        throw new Error('Unexpected variable name');
      }
      type = types[name];
    } else {
      type = await UI.choose({
        nmr: 'NMR (jcamp, pdf)',
        mass: 'Mass (jcamp, pdf, netcdf, xml)',
        ir: 'Infrared (jcamp, pdf)',
        chromatogram: 'Chromatogram LC, GC, LC/MS, GC/MS (jcamp, pdf, netcdf, xml)',
        thermogravimetricAnalysis: 'Thermogravimetric Analysis (txt)',
        differentialScanningCalorimetry: 'Differential Scanning Calorimetry (txt)',
        xray: 'Xray (cif, pdb)',
        image: 'Images (jpg, png or tiff)',
        other: 'Other'
      }, {
        noConfirmation: true,
        columns: [
          {
            id: 'description',
            name: 'description',
            field: 'description'
          }
        ]
      });
      if (!type) return;
    }

    // Dropped data can be an array
    // Expecting format as from drag and drop module
    var droppedDatas = API.getData(name);
    droppedDatas = droppedDatas.file || droppedDatas.str;

    if (type === 'other') {
      await this.roc.addAttachment(this.sample, droppedDatas);
    } else {
      await this.attachFiles(droppedDatas, type);
    }
  }

  async handleAction(action) {
    if (!action) return;

    if (this.expandableMolecule && this.expandableMolecule.handleAction(action)) return;
    if (this.nmr1dManager && this.nmr1dManager.handleAction(action)) return;

    switch (action.name) {
      case 'save':
        await this.roc.update(this.sample);
        break;
      case 'createOptions':
        var advancedOptions1H = API.cache('nmr1hAdvancedOptions');
        if (advancedOptions1H) {
          API.createData('nmr1hOndeTemplate', API.cache('nmr1hOndeTemplates').full);
        } else {
          API.createData('nmr1hOndeTemplate', API.cache('nmr1hOndeTemplates').short);
        }
        break;
      case 'deleteAttachment':
        var attachment = action.value.name;
        await this.roc.deleteAttachment(this.sample, attachment);
        break;
      case 'deleteNmr': // Deprecated. Use unattach. Leave this for backward compatibility
      case 'unattach':
        await this.roc.unattach(this.sample, action.value);
        break;
      case 'attachNMR':
      case 'attachIR':
      case 'attachMass': {
        var tempType = action.name.replace('attach', '');
        var type = tempType.charAt(0).toLowerCase() + tempType.slice(1);
        var droppedDatas = action.value;
        droppedDatas = droppedDatas.file || droppedDatas.str;
        await this.attachFiles(droppedDatas, type);
        break;
      }
      case 'refresh': {
        const ok = await UI.confirm('Are you sure you want to refresh? This will discard your local modifications.');
        if (!ok) return;
        this.unbindChange();
        this.expandableMolecule.unbindChange();
        await this.roc.discardLocal(this.sample);
        this._initializeObjects();
        this.bindChange();
        this.nmr1dManager.handleAction({ name: 'nmrChanged' });
        break;
      }
      default:
        break;
    }
  }

  async attachFiles(files, type) {
    if (!files || !type) return;

    if (!Array.isArray(files)) {
      files = [files];
    }
    for (let i = 0; i < files.length; i++) {
      const data = DataObject.resurrect(files[i]);
      await this.roc.attach(type, this.sample, data);
    }
  }
}

module.exports = Sample;
<|endoftext|>