<commit_before>var Xeit = (function () {
	var cls = function (attachment) {
		this.init = function (attachment) {
			this.attachment = attachment || '';

			if ($('#XEIViewer').length !== -1) {
				this.blob = {
					vendor: 'xe',
					smime_header: attachment.find('param[name="smime_header"]').val(),
					smime_body: attachment.find('param[name="smime_body"]').val(),
					ui_desc: attachment.find('param[name="ui_desc"]').val()
				};
			} else if ($('#IniMasPluginObj').length !== -1) {
				//TODO: use '#InitechSMMsgToReplace'?
				this.blob = {
					vendor: 'it',
					IniSMContents: attachment.find('param[name="IniSMContents"]').val(),
					AttachedFile: attachment.find('param[name="AttachedFile"]').val()
				};
			} else {
				this.blob = {
					vendor: 'none'
				};
				parent.postMessage('fallback', '*');
			}

			//TODO: updateBlob(blob)
		};

		this.updateAuthDialog = function () {
			var senders = {
				TRUEFRIEND: { name: '한국투자증권', support: true },
				'보안메일': { name: 'KB카드', support: true },
				'HyundaiCard': { name: '현대카드', support: true }
			};
			var sender = senders[this.blob.ui_desc];
			var name = sender ? sender['name'] : '?';
			var support = sender ? sender['support'] : false;

			$('#auth-sender-name').val(name);
			if (support) {
				$('#auth-sender-support').addClass('label-success').text('지원');
			} else {
				$('#auth-sender-support').addClass('label-important').text('미지원');
				$('#auth-password').prop('disabled', true);
				$('#auth-submit').addClass('disabled');
			}
		};

		this.decrypt = function (password) {
			if (this.blob.vendor == 'xe') {
				var headerBase64 = this.blob.smime_header.replace(/\n/g, '');
				var headerWords = CryptoJS.enc.Base64.parse(headerBase64);
				var header = CryptoJS.enc.CP949.stringify(headerWords);

				var contentType = header.match(/Content-Type: \s*([a-zA-Z/-]*);*/i)[1];
				if (contentType == 'application/pkcs') {
					return this.decrypt_XE_PKCS(password);
				} else if (contentType == 'application/x-pwd') {
					alert("TODO: x-pwd");
				}
			}
		};

		this.decrypt_XE_PKCS = function (password) {
			var der = Base64.unarmor(this.blob.smime_body);
			var asn1 = ASN1.decode(der);

			/*
			var p = document.createElement('P')
			p.appendChild(asn1.toDOM())
			document.body.appendChild(p)
			*/

			// pkcs7-envelopedData = 1.2.840.113549.1.7.3
			var envelopedDataContentType = asn1.sub[0].content();
	console.log(oids[envelopedDataContentType].d); //== 'envelopedData'
			var envelopedData = asn1.sub[1].sub[0];
			var recipientInfos = envelopedData.sub[1];
			var keyTransportRecipientInfo = recipientInfos.sub[0];
			// des-cbc = 1.3.14.3.2.7
			var keyEncryptionAlgorithm = keyTransportRecipientInfo.sub[2].sub[0].content();
	console.log(oids[keyEncryptionAlgorithm].d); //== 'desCBC'
			var encryptedKey = keyTransportRecipientInfo.sub[3].content();

			var c = keyTransportRecipientInfo.sub[3].posContent();
			var l = keyTransportRecipientInfo.sub[3].length;
			var w = keyTransportRecipientInfo.sub[3].stream.parseStringISO(c, c+l);
			var W = CryptoJS.enc.Latin1.parse(w);

			//var m = /\([\w\s]*\) (\w*)/.exec(encryptedKey);
			//var w = CryptoJS.enc.Hex.parse(m[1]);

			var ciphers = {
				desCBC: CryptoJS.DES,
				seedCBC: CryptoJS.SEED
			}

			var P = CryptoJS.SHA1(password);
			var IV = CryptoJS.enc.Hex.parse("0");
			//var K = CryptoJS.DES.decrypt({ciphertext: W}, P, {iv: IV});
			//var K = CryptoJS.SEED.decrypt({ciphertext: W}, P, {iv: IV});
			var K = ciphers[oids[keyEncryptionAlgorithm].d].decrypt({ciphertext: W}, P, {iv: IV});

			console.log(K);
			console.log(K.words);

			var encryptedContentInfo = envelopedData.sub[2];
			// pkcs7-data = 1.2.840.113549.1.7.1
			var dataContentType = encryptedContentInfo.sub[0].content();
	console.log(oids[dataContentType].d); //== 'data'
			var contentEncryptionAlgorithm = encryptedContentInfo.sub[1].sub[0].content();
	console.log(oids[contentEncryptionAlgorithm].d);
			var encryptedContent = encryptedContentInfo.sub[2];

			var c = encryptedContent.posContent();
			var l = encryptedContent.length;
			var s = encryptedContent.stream.parseStringISO(c, c+l);
			var S = CryptoJS.enc.Latin1.parse(s);

			//var R = CryptoJS.DES.decrypt({ciphertext: S}, K, {iv: IV});
			//var R = CryptoJS.SEED.decrypt({ciphertext: S}, K, {iv: IV});
			var R = ciphers[oids[contentEncryptionAlgorithm].d].decrypt({ciphertext: S}, K, {iv: IV});

			var message = CryptoJS.enc.Latin1.stringify(R);
			if (message.match(/utf-8/i)) {
				message = CryptoJS.enc.Utf8.stringify(R);
			} else {
				message = CryptoJS.enc.CP949.stringify(R);
	        	message = message.replace(/euc-kr/ig, 'utf-8');
			}

			//HACK: DOCTYPE 없는 메일도 있으니 헤더와 본문 사이의 줄바꿈으로 인식.
			//var message = M.replace(/[\s\S]*(<!DOCTYPE)/i, "$1")
			var offset = /[\n\r]{3,}/.exec(message);
			if (offset) {
				message = message.slice(offset.index);
			}
			return message;
		};

		this.init(attachment);
	};

	return cls;
})();<commit_msg>PKCS 복호화 알고리즘 코드 정리<commit_after>var Xeit = (function () {
	var cls = function (attachment) {
		this.init = function (attachment) {
			this.attachment = attachment || '';

			if ($('#XEIViewer').length !== -1) {
				this.blob = {
					vendor: 'xe',
					smime_header: attachment.find('param[name="smime_header"]').val(),
					smime_body: attachment.find('param[name="smime_body"]').val(),
					ui_desc: attachment.find('param[name="ui_desc"]').val()
				};
			} else if ($('#IniMasPluginObj').length !== -1) {
				//TODO: use '#InitechSMMsgToReplace'?
				this.blob = {
					vendor: 'it',
					IniSMContents: attachment.find('param[name="IniSMContents"]').val(),
					AttachedFile: attachment.find('param[name="AttachedFile"]').val()
				};
			} else {
				this.blob = {
					vendor: 'none'
				};
				parent.postMessage('fallback', '*');
			}

			//TODO: updateBlob(blob)
		};

		this.updateAuthDialog = function () {
			var senders = {
				TRUEFRIEND: { name: '한국투자증권', support: true },
				'보안메일': { name: 'KB카드', support: true },
				'HyundaiCard': { name: '현대카드', support: true }
			};
			var sender = senders[this.blob.ui_desc];
			var name = sender ? sender['name'] : '?';
			var support = sender ? sender['support'] : false;

			$('#auth-sender-name').val(name);
			if (support) {
				$('#auth-sender-support').addClass('label-success').text('지원');
			} else {
				$('#auth-sender-support').addClass('label-important').text('미지원');
				$('#auth-password').prop('disabled', true);
				$('#auth-submit').addClass('disabled');
			}
		};

		this.decrypt = function (password) {
			if (this.blob.vendor == 'xe') {
				var headerBase64 = this.blob.smime_header.replace(/\n/g, '');
				var headerWords = CryptoJS.enc.Base64.parse(headerBase64);
				var header = CryptoJS.enc.CP949.stringify(headerWords);

				var contentType = header.match(/Content-Type: \s*([a-zA-Z/-]*);*/i)[1];
				if (contentType == 'application/pkcs') {
					return this.decrypt_XE_PKCS(password);
				} else if (contentType == 'application/x-pwd') {
					alert("TODO: x-pwd");
				}
			}
		};

		this.decrypt_XE_PKCS = function (password) {
			var ciphers = {
				desCBC: CryptoJS.DES,	// 1.3.14.3.2.7,
				seedCBC: CryptoJS.SEED	// 1.2.410.200004.1.4
			};

			ASN1.prototype.contentRaw = function () {
				var offset = this.posContent();
				var length = this.length;
				return this.stream.parseStringISO(offset, offset + length);
			};

			var der = Base64.unarmor(this.blob.smime_body);
			var asn1 = ASN1.decode(der);
			var envelopedData = asn1.sub[1].sub[0];

			// 주민등록번호로 암호화된 대칭키 복호화.
			var recipientInfos = envelopedData.sub[1];
			var keyTransportRecipientInfo = recipientInfos.sub[0];
			var keyEncryptionAlgorithm = oids[keyTransportRecipientInfo.sub[2].sub[0].content()].d;
			var encryptedKey = CryptoJS.enc.Latin1.parse(keyTransportRecipientInfo.sub[3].contentRaw());
			var passwordKey = CryptoJS.SHA1(password);
			var iv = CryptoJS.enc.Hex.parse("0");
			var decryptedKey = ciphers[keyEncryptionAlgorithm].decrypt(
				{ ciphertext: encryptedKey },
				passwordKey,
				{ iv: iv }
			);

			// 대칭키로 암호화된 메일 본문 복호화.
			var encryptedContentInfo = envelopedData.sub[2];
			var contentEncryptionAlgorithm = oids[encryptedContentInfo.sub[1].sub[0].content()].d;
			var encryptedContent = CryptoJS.enc.Latin1.parse(encryptedContentInfo.sub[2].contentRaw());
			var decryptedContent = ciphers[contentEncryptionAlgorithm].decrypt(
				{ ciphertext: encryptedContent },
				decryptedKey,
				{ iv: iv }
			);

			// 메일 본문 인코딩 변환.
			var message = CryptoJS.enc.Latin1.stringify(decryptedContent);
			if (message.match(/utf-8/i)) {
				message = CryptoJS.enc.Utf8.stringify(decryptedContent);
			} else {
				message = CryptoJS.enc.CP949.stringify(decryptedContent);
	        	message = message.replace(/euc-kr/ig, 'utf-8');
			}

			//HACK: DOCTYPE 없는 메일도 있으니 헤더와 본문 사이의 줄바꿈으로 인식.
			// message = message.replace(/[\s\S]*(<!DOCTYPE)/i, "$1")
			var offset = /[\n\r]{3,}/.exec(message);
			if (offset) {
				message = message.slice(offset.index);
			}
			return message;
		};

		this.init(attachment);
	};

	return cls;
})();
<|endoftext|>


<commit_before>const BagItFile = require('./bagit_file');
const async = require('async');
const crypto = require('crypto');
const fs = require('fs');
const os = require('os');
const path = require('path');
const stream = require('stream');
const tar = require('tar-stream');

const WRITE_AS_DIR = 'dir';
const WRITE_AS_TAR = 'tar';

// writeIntoTarArchive is the function that the async queue
// will manage. When writing to a tar archive, we must add
// files one at a time.
function writeIntoTarArchive(data, done) {
    console.log(data);
    var writer = data.tar.entry(data.header, done);
    for (var h of data.hashes) {
        data.reader.pipe(h)
    }
    data.reader.pipe(writer);
}

class Bagger {

    constructor(job) {
        this.job = job;
        this.writeAs = WRITE_AS_DIR;
        this.bagPath = path.join(job.baggingDirectory, job.bagName);
        if (job.bagItProfile.mustBeTarred()) {
            this.bagPath += ".tar";
            this.writeAs = WRITE_AS_TAR;
        }
        this.files = [];
        this.errors = [];

        // Curses!
        this.asyncQueue = async.queue(writeIntoTarArchive, 1);

        // preValidationResult: we validate the job before running it
        // and store the result here. This is a ValidationResult object.
        // Check .isValid() to see if it's valid.
        // Check the .errors hash for info about what went wrong.
        this.preValidationResult = null;

        // Private
        this._tarPacker = null;
        this._tarOutputWriter = null;
        this._outputDirInitialized = false;
    }

    initOutputDir() {
        if (!this._outputDirInitialized) {
            if (path.extname(this.bagPath) == '' && !fs.existsSync(this.bagPath)) {
                fs.mkdirSync(this.bagPath, 0o755);
            } else if (!fs.existsSync(path.dirname(this.bagPath))) {
                fs.mkdirSync(path.dirname(this.bagPath), 0o755);
            }
            this._outputDirInitialized = true;
        }
    }

    getTarPacker() {
        if (this._tarPacker == null) {
            this._tarPacker = tar.pack();
            this._tarPacker.pipe(this.getTarOutputWriter());
        }
        return this._tarPacker;
    }

    getTarOutputWriter() {
        if (this._tarOutputWriter == null) {
            if (!this.bagPath.endsWith(".tar")) {
                var msg = `bagPath '${this.bagPath}' must have .tar extension`;
                this.errors.push(msg);
                throw msg;
            }
            var options = {
                mode: 0o644,
                autoClose: false
            };
            this._tarOutputWriter = fs.createWriteStream(this.bagPath, options);
        }
        return this._tarOutputWriter;
    }

    create() {
        this.preValidationResult = this.job.validate();
        if (!this.preValidationResult.isValid()) {
            this.errors.push("See job validation errors");
            return false;
        }
        this.initOutputDir();
        for (var f of this.job.filesToPackage()) {
            this.copyFile(f, 'data' + f.absPath);
        }
        if (this.writeAs == WRITE_AS_TAR) {
            this.asyncQueue.drain = function () {
                console.log("Done adding payload files");
                //this.writeTagFiles();
            }
        }
        // write tag files
        // write manifests
        // write tag manifests
        // validate bag
        // copy tag data to database
        // copy manifest data to database

        // TODO: Call this when all writing is done.
        if (this.writeAs == WRITE_AS_TAR) {
        //     this.getTarPacker().finalize();
        }
    }

    // copyFile copies file at f.absSourcePath into relDestPath of the bag.
    // Param f is a hash with keys absSourcePath (absolute path to file)
    // and stats (Node fs.Stats object).
    copyFile(f, relDestPath) {
        // Copy file from src to dest
        // stat file and save srcPath, destPath, size, and checksums
        // as a BagItFile and push that into the files array.
        // Preserve owner, group, permissions and timestamps on copy.
        // if (!fs.existsSync(absSourcePath)) {
        //     var msg = `File does not exist: ${absSourcePath}`;
        //     this.errors.push(msg);
        //     throw msg;
        // }
        // var stats = fs.statSync(absSourcePath);
        var bagItFile = new BagItFile(f.absPath, relDestPath, f.stats);
        this.files.push(bagItFile);
        console.log(`Copying ${bagItFile.absSourcePath} to ${bagItFile.relDestPath}`);
        if (this.writeAs == WRITE_AS_DIR) {
            this._copyIntoDir(bagItFile);
        } else if (this.writeAs == WRITE_AS_TAR) {
            this._copyIntoTar(bagItFile);
        } else {
            throw `Unknown writeAs value: '${this.writeAs}'`
        }
    }

    // Copies a file into a directory (unserialized bag)
    _copyIntoDir(bagItFile) {
        var absDestPath = path.join(this.bagPath, bagItFile.relDestPath);
        var writer = fs.createWriteStream(absDestPath);
        var reader = fs.createReadStream(bagItFile.absSourcePath);
        var hashes = this._getCryptoHashes(f);
        console.log(`Setting up pipes for ${hashes.length} digests + file`);
        for (var h of hashes) {
            reader.pipe(h)
        }
        reader.pipe(writer);
    }

    // Copies a file into a tarred bag.
    // These calls need to be synchronized, because the tar library
    // can write only one entry at a time.
    _copyIntoTar(bagItFile) {
        var bagger = this;
        var header = {
            name: bagItFile.relDestPath,
            size: bagItFile.stats.size,
            mode: bagItFile.stats.mode,
            uid: bagItFile.stats.uid,
            gid: bagItFile.stats.gid,
            mtime: bagItFile.stats.mtimeMs
        };
        var reader = fs.createReadStream(bagItFile.absSourcePath);
        var data = {
            reader: reader,
            header: header,
            tar: this.getTarPacker(),
            hashes: this._getCryptoHashes(bagItFile)
        };
        // Write files one at a time.
        this.asyncQueue.push(data);
    }

    _getCryptoHashes(bagItFile) {
        var hashes = [];
        var profile = this.job.bagItProfile;
        for (var algorithm of profile.manifestsRequired) {
            console.log(`Adding ${algorithm} for ${bagItFile.relDestPath}`);
            var hash = crypto.createHash(algorithm);
            hash.setEncoding('hex');
            hash.on('finish', function() {
                hash.end();
                bagItFile.checksums[algorithm] = hash.read();
            });
            hashes.push(hash);
        }
        return hashes;
    }

    payloadFileCount() {
        return this.files.length;
    }

    payloadByteCount() {
        var byteCount = 0;
        for (var f of this.files) {
            byteCount += f.stats.size;
        }
        return byteCount;
    }

    payloadOxum() {
        return `${this.payloadByteCount()}.${this.payloadFileCount()}`
    }

    writeManifest(algorithm) {
        // Write payload manifest with data from files.
    }

    writeTagFile(relDestPath) {
        // Write tag file with data from profile.
        // Tag file data has to go through the hashing algorithms,
        // so we can put the checksums in the tag manifests.
    }
}

module.exports.Bagger = Bagger;
module.exports.WRITE_AS_DIR = WRITE_AS_DIR;
module.exports.WRITE_AS_TAR = WRITE_AS_TAR;
<commit_msg>Added some notes/pseudo-code for bagger<commit_after>const BagItFile = require('./bagit_file');
const async = require('async');
const crypto = require('crypto');
const fs = require('fs');
const os = require('os');
const path = require('path');
const stream = require('stream');
const tar = require('tar-stream');

const WRITE_AS_DIR = 'dir';
const WRITE_AS_TAR = 'tar';

// writeIntoTarArchive is the function that the async queue
// will manage. When writing to a tar archive, we must add
// files one at a time.
function writeIntoTarArchive(data, done) {
    console.log(data);
    var writer = data.tar.entry(data.header, done);
    for (var h of data.hashes) {
        data.reader.pipe(h)
    }
    data.reader.pipe(writer);
}

class Bagger {

    constructor(job) {
        this.job = job;
        this.writeAs = WRITE_AS_DIR;
        this.bagPath = path.join(job.baggingDirectory, job.bagName);
        if (job.bagItProfile.mustBeTarred()) {
            this.bagPath += ".tar";
            this.writeAs = WRITE_AS_TAR;
        }
        this.files = [];
        this.errors = [];

        // Curses!
        this.asyncQueue = async.queue(writeIntoTarArchive, 1);

        // preValidationResult: we validate the job before running it
        // and store the result here. This is a ValidationResult object.
        // Check .isValid() to see if it's valid.
        // Check the .errors hash for info about what went wrong.
        this.preValidationResult = null;

        // Private
        this._tarPacker = null;
        this._tarOutputWriter = null;
        this._outputDirInitialized = false;
    }

    initOutputDir() {
        if (!this._outputDirInitialized) {
            if (path.extname(this.bagPath) == '' && !fs.existsSync(this.bagPath)) {
                fs.mkdirSync(this.bagPath, 0o755);
            } else if (!fs.existsSync(path.dirname(this.bagPath))) {
                fs.mkdirSync(path.dirname(this.bagPath), 0o755);
            }
            this._outputDirInitialized = true;
        }
    }

    getTarPacker() {
        if (this._tarPacker == null) {
            this._tarPacker = tar.pack();
            this._tarPacker.pipe(this.getTarOutputWriter());
        }
        return this._tarPacker;
    }

    getTarOutputWriter() {
        if (this._tarOutputWriter == null) {
            if (!this.bagPath.endsWith(".tar")) {
                var msg = `bagPath '${this.bagPath}' must have .tar extension`;
                this.errors.push(msg);
                throw msg;
            }
            var options = {
                mode: 0o644,
                autoClose: false
            };
            this._tarOutputWriter = fs.createWriteStream(this.bagPath, options);
        }
        return this._tarOutputWriter;
    }

    create() {
        this.preValidationResult = this.job.validate();
        if (!this.preValidationResult.isValid()) {
            this.errors.push("See job validation errors");
            return false;
        }
        this.initOutputDir();
        for (var f of this.job.filesToPackage()) {
            this.copyFile(f, 'data' + f.absPath);
        }
        if (this.writeAs == WRITE_AS_TAR) {
            this.asyncQueue.drain = function () {
                console.log("Done adding payload files");
                //this.writeTagFiles();
            }
        }
        // write tag files
        //    - use job.bagItProfile.requiredTagFileNames()
        //      to get list of tag files
        //    - use job.bagItProfile.getTagFileContents('tag-file-name.txt')
        //      to get the contents to write
        //    - use copyFile, so we get checksums
        //    - what about non-parsable custom tag files
        //      and tag files in special directories?
        // write manifests
        //    - use copyFile, so we get checksums
        //    - loop through this.files & write out checksums
        //      for anything that's a payload file
        //    - call BagItFile.getManifestEntry(algorithm) to get the
        //      manifest entry
        // write tag manifests
        //    - use copyFile, so we get checksums
        //    - loop through this.files & write out checksums
        //      for anything that's a tag file
        //    - call BagItFile.getManifestEntry(algorithm) to get the
        //      manifest entry
        // validate bag
        // copy tag data to database
        // copy manifest data to database

        // TODO: Call this when all writing is done.
        if (this.writeAs == WRITE_AS_TAR) {
        //     this.getTarPacker().finalize();
        }
    }

    // copyFile copies file at f.absSourcePath into relDestPath of the bag.
    // Param f is a hash with keys absSourcePath (absolute path to file)
    // and stats (Node fs.Stats object).
    copyFile(f, relDestPath) {
        // Copy file from src to dest
        // stat file and save srcPath, destPath, size, and checksums
        // as a BagItFile and push that into the files array.
        // Preserve owner, group, permissions and timestamps on copy.
        // if (!fs.existsSync(absSourcePath)) {
        //     var msg = `File does not exist: ${absSourcePath}`;
        //     this.errors.push(msg);
        //     throw msg;
        // }
        // var stats = fs.statSync(absSourcePath);
        var bagItFile = new BagItFile(f.absPath, relDestPath, f.stats);
        this.files.push(bagItFile);
        console.log(`Copying ${bagItFile.absSourcePath} to ${bagItFile.relDestPath}`);
        if (this.writeAs == WRITE_AS_DIR) {
            this._copyIntoDir(bagItFile);
        } else if (this.writeAs == WRITE_AS_TAR) {
            this._copyIntoTar(bagItFile);
        } else {
            throw `Unknown writeAs value: '${this.writeAs}'`
        }
    }

    // Copies a file into a directory (unserialized bag)
    _copyIntoDir(bagItFile) {
        var absDestPath = path.join(this.bagPath, bagItFile.relDestPath);
        var writer = fs.createWriteStream(absDestPath);
        var reader = fs.createReadStream(bagItFile.absSourcePath);
        var hashes = this._getCryptoHashes(f);
        console.log(`Setting up pipes for ${hashes.length} digests + file`);
        for (var h of hashes) {
            reader.pipe(h)
        }
        reader.pipe(writer);
    }

    // Copies a file into a tarred bag.
    // These calls need to be synchronized, because the tar library
    // can write only one entry at a time.
    _copyIntoTar(bagItFile) {
        var bagger = this;
        var header = {
            name: bagItFile.relDestPath,
            size: bagItFile.stats.size,
            mode: bagItFile.stats.mode,
            uid: bagItFile.stats.uid,
            gid: bagItFile.stats.gid,
            mtime: bagItFile.stats.mtimeMs
        };
        var reader = fs.createReadStream(bagItFile.absSourcePath);
        var data = {
            reader: reader,
            header: header,
            tar: this.getTarPacker(),
            hashes: this._getCryptoHashes(bagItFile)
        };
        // Write files one at a time.
        this.asyncQueue.push(data);
    }

    _getCryptoHashes(bagItFile) {
        var hashes = [];
        var profile = this.job.bagItProfile;
        for (var algorithm of profile.manifestsRequired) {
            console.log(`Adding ${algorithm} for ${bagItFile.relDestPath}`);
            var hash = crypto.createHash(algorithm);
            hash.setEncoding('hex');
            hash.on('finish', function() {
                hash.end();
                bagItFile.checksums[algorithm] = hash.read();
            });
            hashes.push(hash);
        }
        return hashes;
    }

    payloadFileCount() {
        return this.files.length;
    }

    payloadByteCount() {
        var byteCount = 0;
        for (var f of this.files) {
            byteCount += f.stats.size;
        }
        return byteCount;
    }

    payloadOxum() {
        return `${this.payloadByteCount()}.${this.payloadFileCount()}`
    }

    writeManifest(algorithm) {
        // Write payload manifest with data from files.
    }

    writeTagFile(relDestPath) {
        // Write tag file with data from profile.
        // Tag file data has to go through the hashing algorithms,
        // so we can put the checksums in the tag manifests.
    }
}

module.exports.Bagger = Bagger;
module.exports.WRITE_AS_DIR = WRITE_AS_DIR;
module.exports.WRITE_AS_TAR = WRITE_AS_TAR;
<|endoftext|>


<commit_before>/*    Copyright 2016 Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

'use strict'

let log = require('../../net2/logger.js')(__filename);

let express = require('express');
let router = express.Router();
let bodyParser = require('body-parser')

let Promise = require('bluebird');

let async = require('asyncawait/async');
let await = require('asyncawait/await');

router.get('/:ip',
           (req, res, next) => {
             let ip = req.params.ip

             let DestIPFoundHook = require('../../hook/DestIPFoundHook');
             let destIPFoundHook = new DestIPFoundHook();

             destIPFoundHook.processIP(ip, true) // do not store result in redis
             .then((json) => {
                res.json(json);
             }).catch((err) => {
               res.status(500).json({error: err});
             });
           });

module.exports = router;
<commit_msg>bug fix<commit_after>/*    Copyright 2016 Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

'use strict'

let log = require('../../net2/logger.js')(__filename);

let express = require('express');
let router = express.Router();
let bodyParser = require('body-parser')

let Promise = require('bluebird');

let async = require('asyncawait/async');
let await = require('asyncawait/await');

router.get('/:ip',
           (req, res, next) => {
             let ip = req.params.ip
             let force = req.query.force

             let options = {skipUpdate: true};

             if(force) {
               options = {forceUpdate: true};
             }

             let DestIPFoundHook = require('../../hook/DestIPFoundHook');
             let destIPFoundHook = new DestIPFoundHook();

             destIPFoundHook.processIP(ip, options) // do not store result in redis
             .then((json) => {
                res.json(json);
             }).catch((err) => {
               res.status(500).json({error: err});
             });
           });

module.exports = router;
<|endoftext|>


<commit_before>import Ember from 'ember';
import RouteLayers from 'ember-route-layers/services/route-layers';

const EMBER_MAJOR_VERSION = Ember.VERSION.split('.')[0];

export function initialize () {
  let application = arguments[1] || arguments[0];

  application.register('service:route-layers', RouteLayers);
  application.inject('route', 'routeLayers', 'service:route-layers');

  const additionalRouteAttributes = {
    routeLayer: 'default',

    afterModel: function (model, transition) {
      this._super(...arguments);

      if (!model || !transition) { return; }

      // Leaf route only
      var leafRouteName = Ember.A(transition.handlerInfos).get('lastObject.handler.routeName');
      if (this.routeName !== leafRouteName) { return; }

      transition.promise.then(() => {
        if (transition.isAborted) { return; }
        this.routeLayers.push(transition);
      });
    },
  };

  const actionHashName = EMBER_MAJOR_VERSION === '1' ? '_actions' : 'actions';
  additionalRouteAttributes[actionHashName] = {
    exitRouteLayer: function () {
      var exitPoint = this.routeLayers.pop();
      if (exitPoint) {
        exitPoint.transition.retry();
      } else {
        if (this.get('router.router.currentHandlerInfos') === null) {
          // Initial transition must complete before we can send an action.
          this.get('router.router.activeTransition').then(() => {
            this.send('exitRouteLayerFallback');
          });
        } else {
          this.send('exitRouteLayerFallback');
        }
      }
    },

    exitRouteLayerFallback: function () {
      this.transitionTo('index');
    }
  };

  Ember.Route.reopen(additionalRouteAttributes);
}

export default {
  name: 'route-layers',
  initialize: initialize
};
<commit_msg>guard against nil transition only. (#22)<commit_after>import Ember from 'ember';
import RouteLayers from 'ember-route-layers/services/route-layers';

const EMBER_MAJOR_VERSION = Ember.VERSION.split('.')[0];

export function initialize () {
  let application = arguments[1] || arguments[0];

  application.register('service:route-layers', RouteLayers);
  application.inject('route', 'routeLayers', 'service:route-layers');

  const additionalRouteAttributes = {
    routeLayer: 'default',

    afterModel: function (model, transition) {
      this._super(...arguments);

      if (!transition) { return; }

      // Leaf route only
      var leafRouteName = Ember.A(transition.handlerInfos).get('lastObject.handler.routeName');
      if (this.routeName !== leafRouteName) { return; }

      transition.promise.then(() => {
        if (transition.isAborted) { return; }
        this.routeLayers.push(transition);
      });
    },
  };

  const actionHashName = EMBER_MAJOR_VERSION === '1' ? '_actions' : 'actions';
  additionalRouteAttributes[actionHashName] = {
    exitRouteLayer: function () {
      var exitPoint = this.routeLayers.pop();
      if (exitPoint) {
        exitPoint.transition.retry();
      } else {
        if (this.get('router.router.currentHandlerInfos') === null) {
          // Initial transition must complete before we can send an action.
          this.get('router.router.activeTransition').then(() => {
            this.send('exitRouteLayerFallback');
          });
        } else {
          this.send('exitRouteLayerFallback');
        }
      }
    },

    exitRouteLayerFallback: function () {
      this.transitionTo('index');
    }
  };

  Ember.Route.reopen(additionalRouteAttributes);
}

export default {
  name: 'route-layers',
  initialize: initialize
};
<|endoftext|>


<commit_before>import Ember from 'ember';
import Util from 'ember-cli-pagination/util';
import LockToRange from 'ember-cli-pagination/watch/lock-to-range';
import { QueryParamsForBackend, ChangeMeta } from './mapping';
import PageMixin from '../page-mixin';

var ArrayProxyPromiseMixin = Ember.Mixin.create(Ember.PromiseProxyMixin, {
  then: function(success,failure) {
    var promise = this.get('promise');
    var me = this;

    return promise.then(function() {
      success(me);
    }, failure);
  }
});

export default Ember.ArrayProxy.extend(PageMixin, Ember.Evented, ArrayProxyPromiseMixin, {
  page: 1,
  paramMapping: Ember.computed(() => {
    return {};
  }),

  init: function() {
    var initCallback = this.get('initCallback');
    if (initCallback) {
      initCallback(this);
    }

    try {
      this.get('promise');
    }
    catch (e) {
      this.set('promise', this.fetchContent());
    }
  },

  addParamMapping: function(key,mappedKey,mappingFunc) {
    var paramMapping = this.get('paramMapping') || {};
    if (mappingFunc) {
      paramMapping[key] = [mappedKey,mappingFunc];
    }
    else {
      paramMapping[key] = mappedKey;
    }
    this.set('paramMapping',paramMapping);
    this.incrementProperty('paramsForBackendCounter');
    //this.pageChanged();
  },

  addQueryParamMapping: function(key,mappedKey,mappingFunc) {
    return this.addParamMapping(key,mappedKey,mappingFunc);
  },

  addMetaResponseMapping: function(key,mappedKey,mappingFunc) {
    return this.addParamMapping(key,mappedKey,mappingFunc);
  },

  paramsForBackend: Ember.computed('page','perPage','paramMapping','paramsForBackendCounter', function() {
    var paramsObj = QueryParamsForBackend.create({page: this.getPage(),
                                                  perPage: this.getPerPage(),
                                                  paramMapping: this.get('paramMapping')});
    var ops = paramsObj.make();

    // take the otherParams hash and add the values at the same level as page/perPage
    ops = Util.mergeHashes(ops,this.get('otherParams')||{});

    return ops;
  }),

  rawFindFromStore: function() {
    var store = this.get('store');
    var modelName = this.get('modelName');

    var ops = this.get('paramsForBackend');
    var res = store.query(modelName, ops);

    return res;
  },

  fetchContent: function() {
    this.set("loading",true);
    var res = this.rawFindFromStore();
    this.incrementProperty("numRemoteCalls");
    var me = this;

    res.then(function(rows) {
      var metaObj = ChangeMeta.create({paramMapping: me.get('paramMapping'),
                                       meta: rows.meta,
                                       page: me.getPage(),
                                       perPage: me.getPerPage()});

      me.set("loading",false);
      return me.set("meta", metaObj.make());

    }, function(error) {
      Util.log("PagedRemoteArray#fetchContent error " + error);
      me.set("loading",false);
    });

    return res;
  },

  totalPages: Ember.computed.alias("meta.total_pages"),

  lastPage: null,

  pageChanged: Ember.observer("page", "perPage", function() {
    var page = this.get('page');
    var lastPage = this.get('lastPage');
    if (lastPage != page) {
      this.set('lastPage', page);
      this.set("promise", this.fetchContent());
    }
  }),

  lockToRange: function() {
    LockToRange.watch(this);
  },

  watchPage: Ember.observer('page','totalPages', function() {
    var page = this.get('page');
    var totalPages = this.get('totalPages');
    if (parseInt(totalPages) <= 0) {
      return;
    }

    this.trigger('pageChanged',page);

    if (page < 1 || page > totalPages) {
      this.trigger('invalidPage',{page: page, totalPages: totalPages, array: this});
    }
  }),

  setOtherParam: function(k,v) {
    if (!this.get('otherParams')) {
      this.set('otherParams',{});
    }

    this.get('otherParams')[k] = v;
    this.incrementProperty('paramsForBackendCounter');
    Ember.run.once(this,"pageChanged");
  }
});
<commit_msg>Add updates counter and update events triggers<commit_after>import Ember from 'ember';
import Util from 'ember-cli-pagination/util';
import LockToRange from 'ember-cli-pagination/watch/lock-to-range';
import { QueryParamsForBackend, ChangeMeta } from './mapping';
import PageMixin from '../page-mixin';

var ArrayProxyPromiseMixin = Ember.Mixin.create(Ember.PromiseProxyMixin, {
  then: function(success,failure) {
    var promise = this.get('promise');
    var me = this;

    return promise.then(function() {
      success(me);
    }, failure);
  }
});

export default Ember.ArrayProxy.extend(PageMixin, Ember.Evented, ArrayProxyPromiseMixin, {
  page: 1,
  paramMapping: Ember.computed(() => {
    return {};
  }),
  contentUpdated: 0,

  init: function() {
    var initCallback = this.get('initCallback');
    if (initCallback) {
      initCallback(this);
    }

    this.addArrayObserver({
      arrayWillChange(me) {
        me.trigger('contentWillChange');
      },
      arrayDidChange(me) {
        me.incrementProperty('contentUpdated');
        me.trigger('contentUpdated');
      },
    });

    try {
      this.get('promise');
    }
    catch (e) {
      this.set('promise', this.fetchContent());
    }
  },

  addParamMapping: function(key,mappedKey,mappingFunc) {
    var paramMapping = this.get('paramMapping') || {};
    if (mappingFunc) {
      paramMapping[key] = [mappedKey,mappingFunc];
    }
    else {
      paramMapping[key] = mappedKey;
    }
    this.set('paramMapping',paramMapping);
    this.incrementProperty('paramsForBackendCounter');
    //this.pageChanged();
  },

  addQueryParamMapping: function(key,mappedKey,mappingFunc) {
    return this.addParamMapping(key,mappedKey,mappingFunc);
  },

  addMetaResponseMapping: function(key,mappedKey,mappingFunc) {
    return this.addParamMapping(key,mappedKey,mappingFunc);
  },

  paramsForBackend: Ember.computed('page','perPage','paramMapping','paramsForBackendCounter', function() {
    var paramsObj = QueryParamsForBackend.create({page: this.getPage(),
                                                  perPage: this.getPerPage(),
                                                  paramMapping: this.get('paramMapping')});
    var ops = paramsObj.make();

    // take the otherParams hash and add the values at the same level as page/perPage
    ops = Util.mergeHashes(ops,this.get('otherParams')||{});

    return ops;
  }),

  rawFindFromStore: function() {
    var store = this.get('store');
    var modelName = this.get('modelName');

    var ops = this.get('paramsForBackend');
    var res = store.query(modelName, ops);

    return res;
  },

  fetchContent: function() {
    this.set("loading",true);
    var res = this.rawFindFromStore();
    this.incrementProperty("numRemoteCalls");
    var me = this;

    res.then(function(rows) {
      var metaObj = ChangeMeta.create({paramMapping: me.get('paramMapping'),
                                       meta: rows.meta,
                                       page: me.getPage(),
                                       perPage: me.getPerPage()});

      me.set("loading",false);
      return me.set("meta", metaObj.make());

    }, function(error) {
      Util.log("PagedRemoteArray#fetchContent error " + error);
      me.set("loading",false);
    });

    return res;
  },

  totalPages: Ember.computed.alias("meta.total_pages"),

  lastPage: null,

  pageChanged: Ember.observer("page", "perPage", function() {
    var page = this.get('page');
    var lastPage = this.get('lastPage');
    if (lastPage != page) {
      this.set('lastPage', page);
      this.set("promise", this.fetchContent());
    }
  }),

  lockToRange: function() {
    LockToRange.watch(this);
  },

  watchPage: Ember.observer('page','totalPages', function() {
    var page = this.get('page');
    var totalPages = this.get('totalPages');
    if (parseInt(totalPages) <= 0) {
      return;
    }

    this.trigger('pageChanged',page);

    if (page < 1 || page > totalPages) {
      this.trigger('invalidPage',{page: page, totalPages: totalPages, array: this});
    }
  }),

  setOtherParam: function(k,v) {
    if (!this.get('otherParams')) {
      this.set('otherParams',{});
    }

    this.get('otherParams')[k] = v;
    this.incrementProperty('paramsForBackendCounter');
    Ember.run.once(this,"pageChanged");
  }
});
<|endoftext|>


<commit_before>import Ember from 'ember'
import A from 'ember-frost-navigation/utils/asserts'

export default Ember.Service.extend({
  routing: Ember.inject.service('-routing'),
  _controller: null,
  _activeCategory: null,
  categories: Ember.A(),
  _registerCategory (config = {}) {
    Ember.assert(A.categoryName, config.name)
    let exists = this.categories.some(e => e.name === config.name)
    Ember.assert(A.categoryExists, !exists)
    let c
    this.categories.push(c = {
      name: config.name,
      icon: config.icon,
      pack: config.pack,
      columns: config.columns || []
    })
    return c
  },
  transitionTo (route) {
    this.get('routing').transitionTo(route)
    this.set('_activeCategory', null)
  },
  performAction (item) {
    let controller = this.get('_controller')
    let _actionHandler
    Ember.assert(
      `Action[${item.action}] does not exist on controller: ${controller.toString()}`,
      _actionHandler = controller.get(item.action)
    )
    if (item.dismiss) {
      this.set('_activeCategory', null)
    }
    _actionHandler(item)
  }
})
<commit_msg>Force absolute truth on truthey / falsey assert.<commit_after>import Ember from 'ember'
import A from 'ember-frost-navigation/utils/asserts'

export default Ember.Service.extend({
  routing: Ember.inject.service('-routing'),
  _controller: null,
  _activeCategory: null,
  categories: Ember.A(),
  _registerCategory (config = {}) {
    Ember.assert(A.categoryName, config.name)
    let exists = this.categories.some(e => e.name === config.name)
    Ember.assert(A.categoryExists, !exists)
    let c
    this.categories.push(c = {
      name: config.name,
      icon: config.icon,
      pack: config.pack,
      columns: config.columns || []
    })
    return c
  },
  transitionTo (route) {
    this.get('routing').transitionTo(route)
    this.set('_activeCategory', null)
  },
  performAction (item) {
    let controller = this.get('_controller')
    let _actionHandler
    Ember.assert(
      `Action[${item.action}] does not exist on controller: ${controller.toString()}`,
      !!(_actionHandler = controller.get(item.action))
    )
    if (item.dismiss) {
      this.set('_activeCategory', null)
    }
    _actionHandler(item)
  }
})
<|endoftext|>


<commit_before>'use strict';

var Path = require('path');

exports.register = function(server, options, next) {
	server.route({
		method: 'GET',
		path: '/static/{path*}',
		handler: function(request, reply) {
			var file = request.params.path.split('.'),
				path = [process.cwd(), 'public'];

			if (['js', 'css'].indexOf(file[file.length - 1]) !== -1) {
				path.push(file[file.length - 1]);
			}

			path.push(request.params.path);

			return reply.file(Path.join(path.join('/')));
		},

		config: {
			cache: {
				expiresIn: 60 * 60 * 1000 * 24,
				privacy: 'private'
			},
			cors: {
				origin: [
					'https://vanham.io'
				]
			}
		}
	});

	next();
};

exports.register.attributes = require('./package.json');
<commit_msg>set caching max-age to one week<commit_after>'use strict';

var Path = require('path');

exports.register = function(server, options, next) {
	server.route({
		method: 'GET',
		path: '/static/{path*}',
		handler: function(request, reply) {
			var file = request.params.path.split('.'),
				path = [process.cwd(), 'public'];

			if (['js', 'css'].indexOf(file[file.length - 1]) !== -1) {
				path.push(file[file.length - 1]);
			}

			path.push(request.params.path);

			return reply.file(Path.join(path.join('/')));
		},

		config: {
			cache: {
				expiresIn: 60 * 60 * 1000 * 24 * 7,
				privacy: 'private'
			},
			cors: {
				origin: [
					'https://vanham.io'
				]
			}
		}
	});

	next();
};

exports.register.attributes = require('./package.json');
<|endoftext|>


<commit_before>/*
  Harvester.
  ==========

  Load CSV tab separated data into a proper neo4j db, v.2.2.

  `options.source` locate the tsv resource on fs:

    > node .\scripts\activities.js --source=contents\activities.tsv

  The `options.source` file MUST have as column names:

    - position        string "description"
    - slug            string "person slug" already existing person, identified by this slug
    - field           string "academic|ambassador"
    - institution     string "institution name"
    - language        lang   'en|fr'
    - d.pos_0_start   year   YYYY
    - d.pos_0_end     year   YYYY
    - ...
    - d.pos_5_start   year   YYYY
    - d.pos_5_end     year   YYYY

*/
var fs         = require('fs'),
    csv        = require('csv'),

    settings   = require('../settings'),
    helpers    = require('../helpers'),

    neo4j      = require('seraph')(settings.neo4j.host),
    activity   = require('../models/activity'),

    async      = require('async'),
    _          = require('lodash'),
    clc        = require('cli-color'),

    options    = require('minimist')(process.argv.slice(2));

console.log('\n\n                                      __^__');
console.log('                                     /(o o)\\');
console.log('==================================oOO==(_)==OOo=======================\n');

if(!options.source) {
  console.log('Please specify', clc.redBright('--source=/path/to/activities.tsv'));
  return;
}

var COUNTRY_CODES = require('../ISO_3166-1.js'),
    disambiguated_positions = {
      en: [],
      fr: []
    };

async.waterfall([
  function importDisambiguatedPositions(next) {
    csv.parse(''+fs.readFileSync('./contents/positions.tsv'), {columns : true, delimiter: '\t', quote: '´'}, function (err, data) {
      if(err)
        throw err;
      // clean data here
      disambiguated_positions.en = data.map(function (d) {
        return helpers.extract.smartSlug(d['Forme anglaise']);
      });
      disambiguated_positions.fr = data.map(function (d) {
        return helpers.extract.smartSlug(d['Forme française']);
      });
      next();
    });
  },

  /**

  Step 1A
  -------

  Import and check for duplicates in file
  */
  function importActivitiesFromSource(next) {
    if(!options.source) { // skip this step is no source has been specified
      console.log(clc.blackBright('source file not specified,', clc.yellowBright('skipping'), 'import people from source file'))
      next();
      return;
    }
    csv.parse(''+fs.readFileSync(options.source), {columns : true, delimiter: '\t', quote: '´'}, function (err, data) {
      if(err)
        throw err;

      // clean data here
      var people = {},
          toberealigned = [],
          aligned       = [];

      data.forEach(function (d, i) {
        if(!people[d.slug])
          people[d.slug] = {
            slug: d.slug,
            duties_fr: [],
            duties_en: [],
            activities: {}
          };

        if(!d.pos_0_start.trim().length)
          return;

        // add duties
        for(var j = 0; j < 10; j++) {
          if(!d['pos_' + j + '_start'])
            break;
          var start_date = d['pos_' + j + '_start'].replace(/[^0-9]/g, ''),
              end_date = d['pos_' + j + '_end'].replace(/[^0-9]/g, ''),
              country = '',
              start,
              end;

          if(start_date.length != 4) {
            console.log(d)
            throw 'pos_' + j + '_start date, line ' + i + ' is not valid, found "' + start_date +'"';
          }
          if(end_date.length && end_date.length!= 4) {
            console.log(d)
            throw 'pos_' + j + '_end date, line ' + i + ' is not valid, found "' + end_date +'"';
          }
          if(!end_date.length) {
            end_date = '' + start_date; // clone
          }
          start = helpers.extract.dates(start_date + '-01-01', 'YYYY-MM-DD', true);
          end   = helpers.extract.dates(end_date + '-12-31', 'YYYY-MM-DD', true);

          if(d.country.trim().length) {
            country = _.result(_.find(COUNTRY_CODES, {'value': d.country}), 'code');
            if(!country)
              throw 'country "' + d.country + '" does not have a proper ISO code, at line ' + i + ', please check';

          }

          people[d.slug]['duties_' + d.language].push({
            position: d.position,
            country: country,
            start_date: start_date,
            start_time: start.time, // 1980-01-01
            end_date: end_date,
            end_time: end.time, // 1980-01-01
          });
        };
      });

      // get statistics
      for(var slug in people) {
        var years_fr = _.indexBy(people[slug].duties_fr, function(d) {
          return d.start_date + '-' + d.end_date;
        });
        var years_en = _.indexBy(people[slug].duties_en, function(d) {
          return d.start_date + '-' + d.end_date;
        });
        var difference_en = _.difference(_.keys(years_en), _.keys(years_fr));
        var difference_fr = _.difference(_.keys(years_fr), _.keys(years_en));

        if(difference_en.length) {
          console.log(clc.red('beware EN'), clc.yellowBright(slug))
          //console.log(difference_en);
          // for(var i in difference_en)
          //   console.log(years_en[difference_en[i]], 'vs', years_fr[difference_en[i]])
          // //throw 'there are differences between en and fr version... "'
          toberealigned.push({
            slug: slug,
            language: 'fr',
            difference: difference_en
          })
        } else if(difference_fr.length) {
          console.log(clc.red('beware FR'), clc.yellowBright(slug))
          //console.log(difference_fr);
          // for(var i in difference_fr)
          //   console.log(years_en[difference_fr[i]], 'vs', years_fr[difference_fr[i]])
          // //throw 'there are differences between en and fr version... "'
          toberealigned.push({
            slug: slug,
            language: 'en',
            difference: difference_fr
          })
        } else {
          console.log('perfect overlapping for', clc.yellowBright(slug))//, _.keys(years_en), _.keys(years_fr));
          for(var year in years_en) {
            if(!people[slug].activities[year])
              people[slug].activities[year] = {
                country: years_en[year].country,
                start_date: years_en[year].start_date,
                start_time: years_en[year].start_time, // 1980-01-01
                end_date: years_en[year].end_date,
                end_time: years_en[year].end_time
              }
            people[slug].activities[year].description_en = years_en[year].position;
            people[slug].activities[year].description_fr = years_fr[year].position;
          }

          aligned.push({
            slug: slug,
            language: 'en',
            difference: difference_fr
          })
        }
      };

      console.log(toberealigned)
      console.log('aligned:    ', aligned.length)
      console.log('NOT aligned:', toberealigned.length);
      var notyetaperson = [];

      var q = async.queue(function (slug, nextSlug) {
          neo4j.query('MATCH (n:person) WHERE n.original_slug = {slug} RETURN n', {
              slug: slug
            }, function (err, nodes) {
              if(err)
                throw err;
              if(_.values(people[slug].activities).length == 0) {
                nextSlug();
                return;
              }
              if(!nodes.length) {
                notyetaperson.push(slug)
                console.log(clc.redBright('is not yet a person'), clc.yellowBright(slug), clc.blackBright('with', clc.magenta(_.values(people[slug].activities).length), 'activities'));
                setTimeout(function(){
                  nextSlug();
                }, 100);
                return;
              };
              if(!nodes.length > 0) {
                console.log(nodes)
                throw 'duplicate person slug!'
              };
              //console.log(people[slug].activities)
              console.log(clc.blackBright('saving activities for'), clc.yellowBright(slug), nodes[0].original_slug, nodes[0].slug)

              var qi = async.queue(function (act, nextActivity) {
                activity.merge({
                  person: nodes[0],
                  description_en: act.description_en,
                  description_fr: act.description_fr,
                  start_date: act.start_date,
                  start_time: act.start_time, // 1980-01-01
                  end_date: act.end_date,
                  end_time: act.end_time, // 1980-01-01
                  country: act.country
                }, function(err) {
                  if(err)
                    throw err;
                  nextActivity();
                });
              }, 1);

              qi.push(_.values(people[slug].activities));
              qi.drain = nextSlug;
            });
          }, 1);

      //console.log(people)
      q.push(_.unique(_.map(aligned, 'slug')));
      q.drain = function() {
        fs.writeFileSync('script.activities.report.json', JSON.stringify(notyetaperson.map(function(slug) { return people[slug] }), null, 2));
        console.log(clc.blackBright('missing people'), notyetaperson.length)
        console.log(clc.cyanBright('completed'))
      }

    })
  }
], function() {

})
<commit_msg>save names and activities of unkwnown persons (without doi)<commit_after>/*
  Harvester.
  ==========

  Load CSV tab separated data into a proper neo4j db, v.2.2.

  `options.source` locate the tsv resource on fs:

    > node .\scripts\activities.js --source=contents\activities.tsv

  The `options.source` file MUST have as column names:

    - position        string "description"
    - slug            string "person slug" already existing person, identified by this slug
    - field           string "academic|ambassador"
    - institution     string "institution name"
    - language        lang   'en|fr'
    - d.pos_0_start   year   YYYY
    - d.pos_0_end     year   YYYY
    - ...
    - d.pos_5_start   year   YYYY
    - d.pos_5_end     year   YYYY

*/
var fs         = require('fs'),
    csv        = require('csv'),

    settings   = require('../settings'),
    helpers    = require('../helpers'),

    neo4j      = require('seraph')(settings.neo4j.host),
    activity   = require('../models/activity'),

    async      = require('async'),
    _          = require('lodash'),
    clc        = require('cli-color'),

    options    = require('minimist')(process.argv.slice(2));

console.log('\n\n                                      __^__');
console.log('                                     /(o o)\\');
console.log('==================================oOO==(_)==OOo=======================\n');

if(!options.source) {
  console.log('Please specify', clc.redBright('--source=/path/to/activities.tsv'));
  return;
}

var COUNTRY_CODES = require('../ISO_3166-1.js'),
    disambiguated_positions = {
      en: [],
      fr: []
    };

async.waterfall([
  function importDisambiguatedPositions(next) {
    csv.parse(''+fs.readFileSync('./contents/positions.tsv'), {columns : true, delimiter: '\t', quote: '´'}, function (err, data) {
      if(err)
        throw err;
      // clean data here
      disambiguated_positions.en = data.map(function (d) {
        return helpers.extract.smartSlug(d['Forme anglaise']);
      });
      disambiguated_positions.fr = data.map(function (d) {
        return helpers.extract.smartSlug(d['Forme française']);
      });
      next();
    });
  },

  /**

  Step 1A
  -------

  Import and check for duplicates in file
  */
  function importActivitiesFromSource(next) {
    if(!options.source) { // skip this step is no source has been specified
      console.log(clc.blackBright('source file not specified,', clc.yellowBright('skipping'), 'import people from source file'))
      next();
      return;
    }
    csv.parse(''+fs.readFileSync(options.source), {columns : true, delimiter: '\t', quote: '´'}, function (err, data) {
      if(err)
        throw err;

      // clean data here
      var people = {},
          toberealigned = [],
          aligned       = [];

      data.forEach(function (d, i) {
        if(!people[d.slug])
          people[d.slug] = {
            slug: d.slug,
            duties_fr: [],
            duties_en: [],
            activities: {}
          };
        console.log(d)
        if(!d.pos_0_start.trim().length)
          return;

        // add duties
        for(var j = 0; j < 10; j++) {
          if(!d['pos_' + j + '_start'])
            continue;
          var start_date = d['pos_' + j + '_start'].replace(/[^0-9]/g, ''),
              end_date = d['pos_' + j + '_end'].replace(/[^0-9]/g, ''),
              country = '',
              start,
              end;

          if(start_date.length != 4) {
            console.log(d)
            throw 'pos_' + j + '_start date, line ' + i + ' is not valid, found "' + start_date +'"';
          }
          if(end_date.length && end_date.length!= 4) {
            console.log(d)
            throw 'pos_' + j + '_end date, line ' + i + ' is not valid, found "' + end_date +'"';
          }
          if(!end_date.length) {
            end_date = '' + start_date; // clone
          }
          start = helpers.extract.dates(start_date + '-01-01', 'YYYY-MM-DD', true);
          end   = helpers.extract.dates(end_date + '-12-31', 'YYYY-MM-DD', true);

          if(d.country.trim().length) {
            country = _.result(_.find(COUNTRY_CODES, {'value': d.country}), 'code');
            if(!country)
              throw 'country "' + d.country + '" does not have a proper ISO code, at line ' + i + ', please check';

          }

          people[d.slug]['duties_' + d.language].push({
            position: d.position,
            country: country,
            start_date: start_date,
            start_time: start.time, // 1980-01-01
            end_date: end_date,
            end_time: end.time, // 1980-01-01
          });
        };
      });

      // get statistics
      for(var slug in people) {
        var years_fr = _.indexBy(people[slug].duties_fr, function(d) {
          return d.start_date + '-' + d.end_date;
        });
        var years_en = _.indexBy(people[slug].duties_en, function(d) {
          return d.start_date + '-' + d.end_date;
        });
        var difference_en = _.difference(_.keys(years_en), _.keys(years_fr));
        var difference_fr = _.difference(_.keys(years_fr), _.keys(years_en));

        if(difference_en.length) {
          console.log(clc.red('beware EN'), clc.yellowBright(slug))
          //console.log(difference_en);
          // for(var i in difference_en)
          //   console.log(years_en[difference_en[i]], 'vs', years_fr[difference_en[i]])
          // //throw 'there are differences between en and fr version... "'
          toberealigned.push({
            slug: slug,
            language: 'fr',
            difference: difference_en
          })
        } else if(difference_fr.length) {
          console.log(clc.red('beware FR'), clc.yellowBright(slug))
          //console.log(difference_fr);
          // for(var i in difference_fr)
          //   console.log(years_en[difference_fr[i]], 'vs', years_fr[difference_fr[i]])
          // //throw 'there are differences between en and fr version... "'
          toberealigned.push({
            slug: slug,
            language: 'en',
            difference: difference_fr
          })
        } else {
          console.log('perfect overlapping for', clc.yellowBright(slug))//, _.keys(years_en), _.keys(years_fr));
          for(var year in years_en) {
            if(!people[slug].activities[year])
              people[slug].activities[year] = {
                country: years_en[year].country,
                start_date: years_en[year].start_date,
                start_time: years_en[year].start_time, // 1980-01-01
                end_date: years_en[year].end_date,
                end_time: years_en[year].end_time
              }
            people[slug].activities[year].description_en = years_en[year].position;
            people[slug].activities[year].description_fr = years_fr[year].position;
          }

          aligned.push({
            slug: slug,
            language: 'en',
            difference: difference_fr
          })
        }
      };

      console.log(toberealigned)
      console.log('aligned:    ', aligned.length)
      console.log('NOT aligned:', toberealigned.length);
      var notyetaperson = [],
          tobecompleted = [];

      var q = async.queue(function (slug, nextSlug) {
          neo4j.query('MATCH (n:person) WHERE n.original_slug = {slug} RETURN n', {
              slug: slug
            }, function (err, nodes) {
              if(err)
                throw err;
              if(_.values(people[slug].activities).length == 0) {
                nextSlug();
                return;
              }
              if(!nodes.length) {
                notyetaperson.push(slug)
                console.log(clc.redBright('is not yet a person'), clc.yellowBright(slug), clc.blackBright('with', clc.magenta(_.values(people[slug].activities).length), 'activities'));
                setTimeout(function(){
                  nextSlug();
                }, 100);
                return;
              };
              if(!nodes.length > 0) {
                console.log(nodes)
                throw 'duplicate person slug!'
              };
              //console.log(people[slug].activities)
              console.log(clc.blackBright('saving activities for'), clc.yellowBright(slug), nodes[0].original_slug, nodes[0].slug)

              var qi = async.queue(function (act, nextActivity) {
                activity.merge({
                  person: nodes[0],
                  description_en: act.description_en,
                  description_fr: act.description_fr,
                  start_date: act.start_date,
                  start_time: act.start_time, // 1980-01-01
                  end_date: act.end_date,
                  end_time: act.end_time, // 1980-01-01
                  country: act.country
                }, function(err) {
                  if(err) {
                    tobecompleted.push(nodes[0])
                  }//  throw err;
                  nextActivity();
                });
              }, 1);

              qi.push(_.values(people[slug].activities));
              qi.drain = nextSlug;
            });
          }, 1);

      //console.log(people)
      q.push(_.unique(_.map(aligned, 'slug')));
      q.drain = function() {
        fs.writeFileSync('script.activities.missing-people.json', JSON.stringify(notyetaperson.map(function(slug) { return people[slug] }), null, 2));
        fs.writeFileSync('script.activities.incompleted-people.json', JSON.stringify(tobecompleted, null, 2));
        console.log(clc.blackBright('missing people'), notyetaperson.length)
        console.log(clc.blackBright('incomplete people'), tobecompleted.length)
        console.log(clc.cyanBright('completed'))
      }

    })
  }
], function() {

})
<|endoftext|>


<commit_before>const TAG = 'CodeUtil'

/**
 * @param name
 * @param reqObj
 * @return
 */
function parseJava(name, reqObj) {
  if (reqObj == null || reqObj == '') {
    log(TAG, 'parseJava  reqObj == null || reqObj.isEmpty() >> return null;');
    return null;
  }
  if (typeof reqObj != 'object') {
    log(TAG, 'parseJava  typeof reqObj != object >> return null;');
    return null;
  }
  log(TAG, '\n\n\n parseJava  name = ' + name + '; reqObj = \n' + format(JSON.stringify(reqObj)));
  let parentKey = isArrayKey(name) ? getItemKey(name) : getTableKey(name);

  let response = '\n' + 'JSONRequest ' + parentKey + ' = new JSONRequest();';

  let value;
  for (var key in reqObj) {
    log(TAG, 'parseJava  for  key = ' + key);
    value = reqObj[key];
    if (value == null) {
      continue;
    }

    if (value instanceof Object && (value instanceof Array) == false) {//APIJSON Array转为常规JSONArray
      log(TAG, 'parseJava  for typeof value === "object" >>  ' );

      if (isArrayKey(key)) { // APIJSON Array转为常规JSONArray
        log(TAG, 'parseJava  for isArrayKey(key) >>  ' );

        response += '\n' + '\n' + '//' + key + '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<';

        const count = value.count || 0;
        const page = value.page || 0;

        log(TAG, 'parseJava  for  count = ' + count + '; page = ' + page);

        delete value.count;
        delete value.page;

        response += parseJava(key, value);

        log(TAG, 'parseJava  for delete >> count = ' + count + '; page = ' + page);

        let prefix = key.substring(0, key.length - 2);
        response += '\n' + '\n'
          + parentKey + '.putAll(' +  getItemKey(key) + '.toArray('
          + count  + ', ' + page + (prefix.length <= 0 ? '' : ', "' + prefix + '"') + '));';

        response += '\n' + '//' + key + '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>' + '\n';
      }
      else { // 常规JSONObject，往下一级提取
        log(TAG, 'parseJava  for isArrayKey(key) == false >>  ' );

        response += '\n' + '\n' + '//' + key + '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<';

        response += parseJava(key, value);

        response += '\n' + '\n' + parentKey + '.put("' + key + '", ' + getTableKey(key) + ');';
        response += '\n' + '//' + key + '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>' + '\n';
      }
    }
    else { // 其它Object，直接填充
      if (typeof value === 'string') {
        log(TAG, 'parseJava  for typeof value === "string" >>  ' );

        value = '"' + value + '"';
      }
      else if (value instanceof Array) {
        log(TAG, 'parseJava  for typeof value === "array" >>  ' );

        value = 'new Object[]{' + value.join() + '}';
      }

      response += '\n' + parentKey + '.put("' + key + '", ' + value + ');';
    }
  }

  log(TAG, 'parseJava  return response = \n' + response + '\n\n\n');
  return response;
}

/**获取Table变量名
 * @param key
 * @return empty ? 'reqObj' : key + 'Request' 且首字母小写
 */
function getTableKey(key) {
  return addSuffix(key, 'Request');
}
/**获取Table变量名
 * @param key
 * @return empty ? 'reqObj' : key + 'Request' 且首字母小写
 */
function getItemKey(key) {
  return addSuffix(key.substring(0, key.length - 2), 'Item');
}
/**判断key是否为表名
 * @param key
 * @return
 */
function isTableKey(key) {
  log(TAG, 'isTableKey  typeof key = ' + (typeof key));
  if (key == null) {
    return false;
  }
  return /^[A-Z][A-Za-z0-9_]*$/.test(key);
}
/**判断key是否为数组名
 * @param key
 * @return
 */
function isArrayKey(key) {
  log(TAG, 'isArrayKey  typeof key = ' + (typeof key));
  if (key == null) {
    return false;
  }
  return key.endsWith('[]');
}<commit_msg>CodeUtil中parseJava分拆为新parseJava和parseCode<commit_after>const TAG = 'CodeUtil'

function parseJava(name, reqObj) {
  return parseCode(name, reqObj, {

    onParseParent: function (parentKey) {
      return '\nJSONRequest ' + parentKey + ' = new JSONRequest();';
    },

    onParseArray: function (parentKey, key, value) {

      let s = '\n\n//' + key + '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<';

      const count = value.count || 0;
      const page = value.page || 0;

      log(TAG, 'parseJava  for  count = ' + count + '; page = ' + page);

      delete value.count;
      delete value.page;

      s += parseJava(key, value);

      log(TAG, 'parseJava  for delete >> count = ' + count + '; page = ' + page);

      let prefix = key.substring(0, key.length - 2);

      s += '\n\n'
        + parentKey + '.putAll(' +  getItemKey(key) + '.toArray('
        + count  + ', ' + page + (prefix.length <= 0 ? '' : ', "' + prefix + '"') + '));';

      s += '\n//' + key + '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n';

      return s;
    },

    onParseObject: function (parentKey, key, value) {
      let s = '\n\n//' + key + '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<';

      s += parseJava(key, value);
      s += '\n\n' + parentKey + '.put("' + key + '", ' + getTableKey(key) + ');';

      s += '\n//' + key + '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n';

      return s;
    },

    onParseOther: function (parentKey, key, value) {

      if (typeof value === 'string') {
        log(TAG, 'parseJava  for typeof value === "string" >>  ' );

        value = '"' + value + '"';
      }
      else if (value instanceof Array) {
        log(TAG, 'parseJava  for typeof value === "array" >>  ' );

        value = 'new Object[]{' + value.join() + '}';
      }

      return '\n' + parentKey + '.put("' + key + '", ' + value + ');';
    }
  })

}

/**
 * @param name
 * @param reqObj
 * @param callback Object，带以下回调函数function：
 *                 解析父对象Parent的onParseParent,
 *                 解析APIJSON数组Object的onParseArray,
 *                 解析普通Object的onParseObject,
 *                 解析其它键值对的onParseOther
 * @return
 */
function parseCode(name, reqObj, callback) {
  if (reqObj == null || reqObj == '') {
    log(TAG, 'parseCode  reqObj == null || reqObj.isEmpty() >> return null;');
    return null;
  }
  if (typeof reqObj != 'object') {
    log(TAG, 'parseCode  typeof reqObj != object >> return null;');
    return null;
  }
  log(TAG, '\n\n\n parseCode  name = ' + name + '; reqObj = \n' + format(JSON.stringify(reqObj)));
  let parentKey = isArrayKey(name) ? getItemKey(name) : getTableKey(name);

  let response = callback.onParseParent(parentKey);

  let value;
  for (var key in reqObj) {
    log(TAG, 'parseCode  for  key = ' + key);
    value = reqObj[key];
    if (value == null) {
      continue;
    }

    if (value instanceof Object && (value instanceof Array) == false) {//APIJSON Array转为常规JSONArray
      log(TAG, 'parseCode  for typeof value === "object" >>  ' );

      if (isArrayKey(key)) { // APIJSON Array转为常规JSONArray
        log(TAG, 'parseCode  for isArrayKey(key) >>  ' );

        response += callback.onParseArray(parentKey, key, value);
      }
      else { // 常规JSONObject，往下一级提取
        log(TAG, 'parseCode  for isArrayKey(key) == false >>  ' );

        response += callback.onParseObject(parentKey, key, value);
      }
    }
    else { // 其它Object，直接填充

      response += callback.onParseOther(parentKey, key, value);
    }
  }

  log(TAG, 'parseCode  return response = \n' + response + '\n\n\n');
  return response;
}

/**获取Table变量名
 * @param key
 * @return empty ? 'reqObj' : key + 'Request' 且首字母小写
 */
function getTableKey(key) {
  return addSuffix(key, 'Request');
}
/**获取数组内Object变量名
 * @param key
 * @return empty ? 'reqObj' : key + 'Request' 且首字母小写
 */
function getItemKey(key) {
  return addSuffix(key.substring(0, key.length - 2), 'Item');
}
/**判断key是否为表名
 * @param key
 * @return
 */
function isTableKey(key) {
  log(TAG, 'isTableKey  typeof key = ' + (typeof key));
  return key != null && /^[A-Z][A-Za-z0-9_]*$/.test(key);
}
/**判断key是否为数组名
 * @param key
 * @return
 */
function isArrayKey(key) {
  log(TAG, 'isArrayKey  typeof key = ' + (typeof key));
  return key != null && key.endsWith('[]');
}<|endoftext|>


<commit_before>var dbWorker = require('./db.js');
var path = require('path');
var appDir = path.resolve('./');
var bodyParser = require('body-parser');
var fs = require("fs");
var morgan = require('morgan');
var jwt = require('jwt-simple');
var app = require("./app").app;
var express = require("./app").express;
var compression = require('compression');
var gzipStatic = require('connect-gzip-static');
app.use(gzipStatic(appDir + '/build'));
app.set('jwtTokenSecret', 'Big Beardy Beary Gnome');

app.use(bodyParser.json()); // to support JSON-encoded bodies
app.use(bodyParser.urlencoded({ // to support URL-encoded bodies
    extended: true
}));
//app.use(compression());
const accessLogStream = fs.createWriteStream(__dirname + '/server_logs.log', {
    flags: 'a'
});
app.use(require('skipper')());

var port = 8080;
var server = app.listen(port, function () {
    console.log(this.address());
});

const getRawToken = function (req) {
    var token = (req.body && req.body.access_token) || (req.query && req.query.access_token) || req.headers['x-access-token'];
    return token;
}

const getToken = function (req) {
    var token = getRawToken(req);
    if (token) {
        try {
            var decoded = jwt.decode(token, app.get('jwtTokenSecret'));
        } catch (err) {
            return "invalid token"
        }
    } else {
        return "no_token";
    }
    return decoded;
}

const checkToken = function (req, res, callback) {
    var token = getToken(req);
    if (token == "no_token" || token == "invalid token") {
        res.send(token);
        return;
    } else {
        if (token.exp <= Date.now()) {
            res.end('Access token has expired', 400);
            return;
        }
        if (token.userid == 1) {
            callback(req, res);
        }
    }
}

app.use(express.static('build'));
app.use(express.static(appDir));

app.use(morgan(':method :url :response-time :remote-addr :date', {
    stream: accessLogStream
}));
var browser = require('file-manager-js');
app.all('/browse_url', browser.browse);
app.post('/upload_url', browser.upload);

app.get('/database/drop', function (req, res) {
    //dbWorker.schemaOps.drop();
    res.send("Don't even try")
});

app.get('/googlee*.html', function (req, res) {
    res.sendFile(appDir + "/node/googlee0b3bf85105efcd6.html");
})

app.get('/dbprepare',
    function (req, res) {
        dbWorker.schemaOps.createCrypto();
    });

var userSessions = {};

app.post("/checkSession", function (req, res) {
    req.setTimeout(0) // no timeout
    const token = getToken(req);
    if (token == "no_token" || token == "invalid token") {
        res.send("not logged in");
    } else {
        var intervalId = setInterval(() => {
            if (token.exp <= Date.now()) {
                res.send("session over");
                clearInterval(intervalId);
            }
        }, 1000);
        userSessions[JSON.stringify(token)] = ({
            endSession: () => {
                res.send("session over");
                clearInterval(intervalId);
            }
        });
    }
});

app.post("/endSession", function (req, res) {
    const token = getToken(req);
    const stringToken = JSON.stringify(token);
    if (token && userSessions[stringToken] && userSessions[stringToken].endSession) {
        userSessions[stringToken].endSession();
        userSessions[stringToken] = null;
        res.send("session ended");
    }
});

app.get('/database/create', function (req, res) {
    //dbWorker.schemaOps.createDb();
    res.send('Db created successfully(20 years ago)');
})

app.post('/admin/login', function (req, res) {
    dbWorker.entities['admin'].checkPassword(req.body, res);
});

app.post('/structure/update', function (req, res) {
    const callback = (req, res) => {
        var name = req.body;
        dbWorker.entities['structure'].updateStructure(req.body, res);
    };
    checkToken(req, res, callback);
});
app.get('/structure/get', function (req, res) {
    var result = dbWorker.entities['structure'].getStructure(res);
});
app.get('/data/viewsList', function (req, res) {
    dbWorker.entities['structure'].getViewsList(res);
});

app.post('/data/addArticle', function (req, res) {
    const callback = (req, res) => {
        var name = req.body;
        dbWorker.entities['articles'].create(req.body, res);
    };
    checkToken(req, res, callback);
});
app.post('/data/updateArticle', function (req, res) {
    const callback = (req, res) => {
        var name = req.body;
        dbWorker.entities['articles'].update(req.body, res);
    };
    checkToken(req, res, callback);
});
app.post('/data/updateArticleGrid', function (req, res) {
    const callback = (req, res) => {
        dbWorker.entities['articles'].updateGrid(req.body, res);
    };
    checkToken(req, res, callback);
});
app.get('/data/articles', function (req, res) {
    var result = dbWorker.entities['articles'].getAll(res);
});
app.get('/data/articles/:alias', function (req, res) {
    var result = dbWorker.entities['articles'].getSingleByAlias(req.params.alias, res);
});
app.get('/data/articles/byViewId/:id', function (req, res) {
    var result = dbWorker.entities['articles'].getSingleByViewId(req.params.id, res);
});
app.get('/data/articles/byViewAlias/:alias', function (req, res) {
    var result = dbWorker.entities['articles'].getSingleByViewAlias(req.params.alias, res);
});

app.use(function (err, req, res, next) {
    console.error(err.stack);
    res.status(500).send('Something broke!');
});

module.exports = app;
<commit_msg>removed node map<commit_after>var dbWorker = require('./db.js');
var path = require('path');
var appDir = path.resolve('./');
var bodyParser = require('body-parser');
var fs = require("fs");
var morgan = require('morgan');
var jwt = require('jwt-simple');
var app = require("./app").app;
var express = require("./app").express;
app.set('jwtTokenSecret', 'Big Beardy Beary Gnome');

app.use(bodyParser.json()); // to support JSON-encoded bodies
app.use(bodyParser.urlencoded({ // to support URL-encoded bodies
    extended: true
}));
//app.use(compression());
const accessLogStream = fs.createWriteStream(__dirname + '/server_logs.log', {
    flags: 'a'
});
app.use(require('skipper')());

var port = 8080;
var server = app.listen(port, function () {
    console.log(this.address());
});

const getRawToken = function (req) {
    var token = (req.body && req.body.access_token) || (req.query && req.query.access_token) || req.headers['x-access-token'];
    return token;
}

const getToken = function (req) {
    var token = getRawToken(req);
    if (token) {
        try {
            var decoded = jwt.decode(token, app.get('jwtTokenSecret'));
        } catch (err) {
            return "invalid token"
        }
    } else {
        return "no_token";
    }
    return decoded;
}

const checkToken = function (req, res, callback) {
    var token = getToken(req);
    if (token == "no_token" || token == "invalid token") {
        res.send(token);
        return;
    } else {
        if (token.exp <= Date.now()) {
            res.end('Access token has expired', 400);
            return;
        }
        if (token.userid == 1) {
            callback(req, res);
        }
    }
}

app.use(morgan(':method :url :response-time :remote-addr :date', {
    stream: accessLogStream
}));
var browser = require('file-manager-js');
app.all('/browse_url', browser.browse);
app.post('/upload_url', browser.upload);

app.get('/database/drop', function (req, res) {
    //dbWorker.schemaOps.drop();
    res.send("Don't even try")
});

app.get('/googlee*.html', function (req, res) {
    res.sendFile(appDir + "/node/googlee0b3bf85105efcd6.html");
})

app.get('/dbprepare',
    function (req, res) {
        dbWorker.schemaOps.createCrypto();
    });

var userSessions = {};

app.post("/checkSession", function (req, res) {
    req.setTimeout(0) // no timeout
    const token = getToken(req);
    if (token == "no_token" || token == "invalid token") {
        res.send("not logged in");
    } else {
        var intervalId = setInterval(() => {
            if (token.exp <= Date.now()) {
                res.send("session over");
                clearInterval(intervalId);
            }
        }, 1000);
        userSessions[JSON.stringify(token)] = ({
            endSession: () => {
                res.send("session over");
                clearInterval(intervalId);
            }
        });
    }
});

app.post("/endSession", function (req, res) {
    const token = getToken(req);
    const stringToken = JSON.stringify(token);
    if (token && userSessions[stringToken] && userSessions[stringToken].endSession) {
        userSessions[stringToken].endSession();
        userSessions[stringToken] = null;
        res.send("session ended");
    }
});

app.get('/database/create', function (req, res) {
    //dbWorker.schemaOps.createDb();
    res.send('Db created successfully(20 years ago)');
})

app.post('/admin/login', function (req, res) {
    dbWorker.entities['admin'].checkPassword(req.body, res);
});

app.post('/structure/update', function (req, res) {
    const callback = (req, res) => {
        var name = req.body;
        dbWorker.entities['structure'].updateStructure(req.body, res);
    };
    checkToken(req, res, callback);
});
app.get('/structure/get', function (req, res) {
    var result = dbWorker.entities['structure'].getStructure(res);
});
app.get('/data/viewsList', function (req, res) {
    dbWorker.entities['structure'].getViewsList(res);
});

app.post('/data/addArticle', function (req, res) {
    const callback = (req, res) => {
        var name = req.body;
        dbWorker.entities['articles'].create(req.body, res);
    };
    checkToken(req, res, callback);
});
app.post('/data/updateArticle', function (req, res) {
    const callback = (req, res) => {
        var name = req.body;
        dbWorker.entities['articles'].update(req.body, res);
    };
    checkToken(req, res, callback);
});
app.post('/data/updateArticleGrid', function (req, res) {
    const callback = (req, res) => {
        dbWorker.entities['articles'].updateGrid(req.body, res);
    };
    checkToken(req, res, callback);
});
app.get('/data/articles', function (req, res) {
    var result = dbWorker.entities['articles'].getAll(res);
});
app.get('/data/articles/:alias', function (req, res) {
    var result = dbWorker.entities['articles'].getSingleByAlias(req.params.alias, res);
});
app.get('/data/articles/byViewId/:id', function (req, res) {
    var result = dbWorker.entities['articles'].getSingleByViewId(req.params.id, res);
});
app.get('/data/articles/byViewAlias/:alias', function (req, res) {
    var result = dbWorker.entities['articles'].getSingleByViewAlias(req.params.alias, res);
});

app.use(function (err, req, res, next) {
    console.error(err.stack);
    res.status(500).send('Something broke!');
});

module.exports = app;
<|endoftext|>


<commit_before>'use strict';

const path = require('path');
const sinon = require('sinon');
const editor = require('..');
const memFs = require('mem-fs');
const escape = require('escape-regexp');

describe('#readJSON()', () => {
  let store;
  let fs;

  beforeEach(() => {
    store = memFs.create();
    fs = editor.create(store);
  });

  it('read the content of a file', () => {
    const obj = fs.readJSON(path.join(__dirname, 'fixtures/file.json'));
    expect(obj.foo).toBe('bar');
  });

  it('calls read() with path', () => {
    const read = sinon.spy(fs, 'read');
    const file = path.join(__dirname, 'fixtures/file.json');
    fs.readJSON(file);
    sinon.assert.calledOnce(read);
    sinon.assert.calledWith(read, file);
    read.restore();
  });

  it('return defaults if file does not exist and defaults is provided', () => {
    const obj = fs.readJSON(path.join(__dirname, 'no-such-file.json'), {foo: 'bar'});
    expect(obj.foo).toBe('bar');
  });

  it('throw error if file could not be parsed as JSON, even if defaults is provided', () => {
    expect(fs.readJSON.bind(fs, path.join(__dirname, 'fixtures/file-tpl.txt'), {foo: 'bar'})).toThrow();
  });

  it('throw error with file path info', () => {
    var filePath = path.join(__dirname, 'fixtures/file-tpl.txt');
    expect(fs.readJSON.bind(fs, filePath), new RegExp(escape(filePath))).toThrow();
  });
});
<commit_msg>Fix broken test<commit_after>'use strict';

const path = require('path');
const sinon = require('sinon');
const editor = require('..');
const memFs = require('mem-fs');
const escape = require('escape-regexp');

describe('#readJSON()', () => {
  let store;
  let fs;

  beforeEach(() => {
    store = memFs.create();
    fs = editor.create(store);
  });

  it('read the content of a file', () => {
    const obj = fs.readJSON(path.join(__dirname, 'fixtures/file.json'));
    expect(obj.foo).toBe('bar');
  });

  it('calls read() with path', () => {
    const read = sinon.spy(fs, 'read');
    const file = path.join(__dirname, 'fixtures/file.json');
    fs.readJSON(file);
    sinon.assert.calledOnce(read);
    sinon.assert.calledWith(read, file);
    read.restore();
  });

  it('return defaults if file does not exist and defaults is provided', () => {
    const obj = fs.readJSON(path.join(__dirname, 'no-such-file.json'), {foo: 'bar'});
    expect(obj.foo).toBe('bar');
  });

  it('throw error if file could not be parsed as JSON, even if defaults is provided', () => {
    expect(fs.readJSON.bind(fs, path.join(__dirname, 'fixtures/file-tpl.txt'), {foo: 'bar'})).toThrow();
  });

  it('throw error with file path info', () => {
    var filePath = path.join(__dirname, 'fixtures/file-tpl.txt');
    expect(fs.readJSON.bind(fs, new RegExp(escape(filePath)))).toThrow();
  });
});
<|endoftext|>


<commit_before>import Vue from 'vue'

import VueMaterial from 'vue-material'
import 'vue-material/dist/vue-material.css'

import axios from 'axios'
import VueAxios from 'vue-axios'

// import lodash from 'lodash'
// import VueLodash from 'vue-lodash/dist/vue-lodash.min'

import io from 'socket.io-client'
import VueSocketIO from 'vue-socket.io'

import App from './App'
import router from './router'

Vue.config.productionTip = false

Vue.use(VueMaterial)
Vue.use(VueAxios, axios)
Vue.use(VueSocketIO, io('localhost:5000'))
// Vue.use(VueLodash, lodash)

Vue.material.registerTheme({
  main: {
    primary: 'blue',
    accent: 'black'
  },
  login: {
    primary: 'blue',
    accent: {
      color: 'light-green',
      hue: 'A700'
    },
    warn: {
      color: 'red',
      hue: 'A700'
    }
  },
  success: {
    primary: {
      color: 'light-green',
      hue: 'A700'
    },
    accent: {
      color: 'light-green',
      hue: 'A700'
    },
    warn: {
      color: 'light-green',
      hue: 'A700'
    }
  },
  warning: {
    primary: {
      color: 'red',
      hue: 'A700'
    },
    accent: {
      color: 'red',
      hue: 'A700'
    },
    warn: {
      color: 'red',
      hue: 'A700'
    }
  }
})

/* eslint-disable no-new */
new Vue({
  el: '#app',
  router,
  render: h => h(App)
})
<commit_msg>fix development connection with socket<commit_after>import Vue from 'vue'

import VueMaterial from 'vue-material'
import 'vue-material/dist/vue-material.css'

import axios from 'axios'
import VueAxios from 'vue-axios'

import io from 'socket.io-client'
import VueSocketIO from 'vue-socket.io'

import App from './App'
import router from './router'

Vue.config.productionTip = false

Vue.use(VueMaterial)
Vue.use(VueAxios, axios)
Vue.use(VueSocketIO, process.env.NODE_ENV === 'development' ? io('localhost:5000') : io())

Vue.material.registerTheme({
  main: {
    primary: 'blue',
    accent: 'black'
  },
  login: {
    primary: 'blue',
    accent: {
      color: 'light-green',
      hue: 'A700'
    },
    warn: {
      color: 'red',
      hue: 'A700'
    }
  },
  success: {
    primary: {
      color: 'light-green',
      hue: 'A700'
    },
    accent: {
      color: 'light-green',
      hue: 'A700'
    },
    warn: {
      color: 'light-green',
      hue: 'A700'
    }
  },
  warning: {
    primary: {
      color: 'red',
      hue: 'A700'
    },
    accent: {
      color: 'red',
      hue: 'A700'
    },
    warn: {
      color: 'red',
      hue: 'A700'
    }
  }
})

/* eslint-disable no-new */
new Vue({
  el: '#app',
  router,
  render: h => h(App)
})
<|endoftext|>