<commit_before>var app = require('express')();
var http = require('http').Server(app);
var fs = require('fs');
var Cookies = require('cookies');
var io = require('socket.io')(http);
var SocketIOFileUpload = require('socketio-file-upload');
var ssh2 = require('ssh2');

var MathServer = function (overrideOptions) {
    var options = {
        port: 8002, // default port number to use
        PRUNE_CLIENT_INTERVAL: 1000 * 60 * 10, // 10 minutes
        MAX_AGE: 1000 * 60 * 60 * 24 * 7, // 1 week
        CONTAINERS: './dummy_containers.js',
        MATH_PROGRAM: "Macaulay2",
        MATH_PROGRAM_COMMAND: 'M2'
    };

    var logExceptOnTest = function(string) {
        if (process.env.NODE_ENV !== 'test') {
            console.log(string);
        }
    };

    var overrideDefaultOptions = function (overrideOptions) {
        for (var opt in overrideOptions) {
            if (options.hasOwnProperty(opt)) {
                options[opt] = overrideOptions[opt];
                logExceptOnTest("server option: " + opt + " set to " + options[opt]);
            }
        }
    };

    overrideDefaultOptions(overrideOptions);

    var cookieName = "try" + options.MATH_PROGRAM;

    var totalUsers = 0; //only used for stats: total # users since server started

    // Global array of all Client objects.  Each has a math program process.
    var clients = {};
    if (!options.CONTAINERS) {
        console.error("error, no container management given.");
        throw ("No CONTAINERS!");
    }

    var credentialManager = require(options.CONTAINERS).manager();

    // preamble every log with the client ID
    var logClient = function (clientID, str) {
        if (process.env.NODE_ENV !== 'test') {
            logExceptOnTest(clientID + ": " + str);
        }
    };

    var deleteClient = function (clientID) {
        credentialManager.removeCredentials(clients[clientID].credentials);
        delete clients[clientID];
    };

    var timeBeforeInterval = function (timeInterval) {
        var now = Date.now();
        logExceptOnTest("It is currently " + now + " milliseconds.");
        return now - timeInterval;
    };

    var removeOldClients = function (minimalLastActiveTimeForClient) {
        for (var clientID in clients) {
            if (clients.hasOwnProperty(clientID)) {
                if (clients[clientID].lastActiveTime < minimalLastActiveTimeForClient) {
                    deleteClient(clientID);
                }
            }
        }
    };

    var logCurrentlyActiveClients = function () {
        for (var clientID in clients) {
            if (clients.hasOwnProperty(clientID)) {
                logExceptOnTest(clientID);
            }
        }
    };

    var pruneClients = function () {
        logExceptOnTest("Pruning clients...");
        var minimalLastActiveTimeForClient = timeBeforeInterval(options.MAX_AGE);
        removeOldClients(minimalLastActiveTimeForClient);
        logExceptOnTest("Done pruning clients...  Continuing clients:");
        logCurrentlyActiveClients();
    };

    var Client = function () {
        this.recentlyRestarted = false;
        this.lastActiveTime = Date.now(); // milliseconds when client was last active
        this.credentials = 0;
    };

    var clientIDExists = function (clientID) {
        if (clients[clientID] == null) {
            return false;
        }
        logClient(clientID, "Client already exists");
        return true;
    };

    var getNewClientID = function () {
        totalUsers += 1;
        do {
            var clientID = Math.random() * 1000000;
            clientID = Math.floor(clientID);
        } while (clientIDExists(clientID));
        clientID = "user" + clientID.toString(10);
        logExceptOnTest("New Client ID " + clientID);
        return clientID;
    };

    var setCookie = function (cookies, clientID) {
        cookies.set(cookieName, clientID, {
            httpOnly: false
        });
    };

    var getCredentials = function (clientID, next) {
        if (clients[clientID].credentials) {
            next(clients[clientID].credentials);
        } else {
            credentialManager.getNewCredentials(next);
        }
    };

    /*var getSSHOptions = function (credentials) {
     // ssh -t fs "stty isig intr ^N -echoctl ; trap '/bin/true' SIGINT; sleep 1000; echo f" > foo
     var sshOptions = ["-q", "-o", "StrictHostKeyChecking=no"];
     if (credentials.sshKey) {
     sshOptions.push("-i");
     sshOptions.push(credentials.sshKey);
     }
     if (credentials.port){
     sshOptions.push("-p");
     sshOptions.push(credentials.port);
     }
     if (credentials.username) {
     sshOptions.push(credentials.username + "@" + credentials.host);
     } else {
     sshOptions.push(credentials.host);
     }
     //sshOptions.push("stty isig intr ^N; trap '/bin/true' SIGINT; " + options.MATH_PROGRAM_COMMAND);
     return sshOptions;
     };*/

    var spawnMathProgramInSecureContainer = function (clientID, next) {
        getCredentials(clientID, function (credentials) {
            clients[clientID].credentials = credentials;
            var connection = new ssh2.Client();
            connection.on('ready', function () {
                connection.exec(options.MATH_PROGRAM_COMMAND, {pty: true}, function (err, stream) {
                    if (err) throw err;
                    stream.on('close', function () {
                        connection.end();
                    });
                    stream.on('end', function () {
                        stream.close();
                        logExceptOnTest('I ended.');
                        //connection.end();
                    });
                    next(stream);
                    /*connection.on('end', function(){
                        delete stream;
                        delete connection;
                    });*/
                });
            }).connect({
                host: credentials.host,
                port: credentials.port,
                username: credentials.username,
                privateKey: fs.readFileSync(credentials.sshKey)
            });

        });
    };

    var mathProgramStart = function (clientID, next) {
        logClient(clientID, "Spawning new MathProgram process...");
        spawnMathProgramInSecureContainer(clientID, function (stream) {
            stream.setEncoding("utf8");
            clients[clientID].mathProgramInstance = stream;
            attachListenersToOutput(clientID);
            if(next){
                next(clientID);
            }
        });
    };

    var captureSpecialEvent = function (data) {
        var eventData = data.match(/>>SPECIAL_EVENT_START>>(.*)<<SPECIAL_EVENT_END/);
        if (eventData) {
            // logExceptOnTest("Have special event: " + eventData[1]);
            return eventData[1];
        }
    };

    var emitUrlForUserGeneratedFileToClient = function (clientId, path) {
        var partAfterLastSlash = /([^\/]*)$/;
        var fileName = path.match(partAfterLastSlash);
        if (fileName) {
            fileName = fileName[0];
        } else {
            return;
        }
        var sshConnection = ssh2();

        sshConnection.on('end', function () {
            logExceptOnTest("Image action ended.");
        });

        sshConnection.on('ready', function () {
            sshConnection.sftp(function (err, sftp) {
                var folderName = "/files/";
                fs.mkdir("./public" + folderName, function(err){
                    if(err){
                        logExceptOnTest("Folder exists, but we proceed anyway");
                    }
                    sftp.fastGet(path, "./public" + folderName + fileName, function (error) {
                        if (error) {
                            logExceptOnTest("Error while downloading image. PATH: " + path + ", ERROR: " + error);
                        } else {
                            clients[clientId].socket.emit("image", folderName + fileName);
                        }
                    });
                });
            });
        });

        sshConnection.connect(
            {
                host: clients[clientId].credentials.host,
                port: clients[clientId].credentials.port,
                username: clients[clientId].credentials.username,
                privateKey: fs.readFileSync(clients[clientId].credentials.sshKey)
            }
        );

    };

    var emitHelpUrlToClient = function (clientID, viewHelp) {
        logExceptOnTest("Look at " + viewHelp);
        var helpPath = viewHelp.match(/(\/Macaulay2Doc.*)$/);
        if (helpPath) {
            helpPath = helpPath[0];
        } else {
            return;
        }
        helpPath = "http://www.math.uiuc.edu/Macaulay2/doc/Macaulay2-1.7/share/doc/Macaulay2" + helpPath;
        logExceptOnTest(helpPath);
        clients[clientID].socket.emit("viewHelp", helpPath);
    };

    var isViewHelpEvent = function (eventData) {
        return eventData.match(/^file:.*/);
    };

    var emitEventUrlToClient = function(clientID, eventType, data){
        if (isViewHelpEvent(eventType)) {
            console.log(data);
            emitHelpUrlToClient(clientID, eventType);
            return;
        } else {
            emitUrlForUserGeneratedFileToClient(clientID, eventType);
        }
        var outputData = data.replace(/>>SPECIAL_EVENT_START>>/,"opening ");
        outputData = outputData.replace(/<<SPECIAL_EVENT_END<</,"");
        clients[clientID].socket.emit('result', outputData);
    };

    var sendDataToClient = function (clientID) {
        return function (dataObject) {
            var data = dataObject.toString();
            var socket = clients[clientID].socket;
            if (!socket) {
                logClient(clientID, "Error, no socket for client.");
                return;
            }
            updateLastActiveTime(clientID);
            var specialEvent = captureSpecialEvent(data);
            if (specialEvent) {
                emitEventUrlToClient(clientID, specialEvent, data);
                return;
            }
            socket.emit('result', data);
        };
    };

    var attachListenersToOutput = function (clientID) {
        var client = clients[clientID];
        if (!client) {
            return;
        }
        if (client.mathProgramInstance) {
            clients[clientID].mathProgramInstance
                .removeAllListeners('data')
                .on('data', sendDataToClient(clientID));
                //.stderr('data', sendDataToClient(clientID));
        }
    };

    var stats = function (request, response) {
        // to do: authorization
        response.writeHead(200, {
            "Content-Type": "text/html"
        });
        var currentUsers = 0;
        for (var c in clients) {
            if (clients.hasOwnProperty(c))
                currentUsers = currentUsers + 1;
        }
        response.write(
            '<link rel="stylesheet" href="mathProgram.css" type="text/css" media="screen">');
        response.write('
' + options.MATH_PROGRAM + ' User Statistics
');
        response.write('There are currently ' + currentUsers +
        ' users using ' + options.MATH_PROGRAM + '.
');
        response.write('In total, there were ' + totalUsers +
        ' users since the server started.
');
        response.write('Enjoy ' + options.MATH_PROGRAM + '!');
        response.end();
    };

    var updateLastActiveTime = function (clientID) {
        clients[clientID].lastActiveTime = Date.now();
    };

    var killMathProgram = function (stream, clientID) {
        logClient(clientID, "killMathProgramClient.");
        stream.close();
    };

    var resetRecentlyRestarted = function (client) {
        client.recentlyRestarted = true;
        setTimeout(function () {
            client.recentlyRestarted = false;
        }, 1000);
    };

    var checkCookie = function (request, response, next) {
        var cookies = new Cookies(request, response);
        var clientID = cookies.get(cookieName);
        if (!clientID) {
            logExceptOnTest('New client without a cookie set came along');
            logExceptOnTest('Set new cookie!');
            clientID = getNewClientID();
        }
        setCookie(cookies, clientID);

        if (!clients[clientID]) {
            clients[clientID] = new Client();
        }
        next();
    };

    var unhandled = function (request, response) {
        var url = require('url').parse(request.url).pathname;
        logExceptOnTest("Request for something we don't serve: " + request.url);
        response.writeHead(404, "Request for something we don't serve.");
        response.write("404");
        response.end();
    };

    var initializeServer = function () {
        var favicon = require('serve-favicon');
        var serveStatic = require('serve-static');
        var winston = require('winston'),
            expressWinston = require('express-winston');

        var loggerSettings = {
            transports: [
                new winston.transports.Console({
                    level: 'error',
                    json: true,
                    colorize: true
                })
            ]
        };
        var prefix = "public-" + options.MATH_PROGRAM + "/";
        var tutorialReader = require('./tutorialReader.js')(prefix, fs);
        app.use(favicon(__dirname + '/../public-' + options.MATH_PROGRAM + '/favicon.ico'));
        app.use(SocketIOFileUpload.router);
        app.use(checkCookie);
        app.use(serveStatic(__dirname + '/../public-' + options.MATH_PROGRAM));
        app.use(serveStatic(__dirname + '/../public'));
        app.use(expressWinston.logger(loggerSettings));
        app.use('/admin', stats)
            .use('/getListOfTutorials', tutorialReader.getList)
            .use(unhandled);
    };

    var attachUploadListenerToSocket = function (clientId, socket) {
        var uploader = new SocketIOFileUpload();
        uploader.listen(socket);

        uploader.on("error", function (event) {
            console.error("Error in upload " + event);
        });

        uploader.on("start", function (event) {
            clients[clientId].fileUploadBuffer = "";
            logExceptOnTest('File upload ' + event.file.name);
        });

        uploader.on("progress", function (event) {
            // TODO: Limit size.
            clients[clientId].fileUploadBuffer += event.buffer;
        });

        uploader.on("complete", completeFileUpload(clientId));
    };

    var completeFileUpload = function (clientId) {
        return function (event) {
            var connection = ssh2();

            connection.on('end', function () {
            });

            connection.on('ready', function () {
                connection.sftp(function (err, sftp) {
                    var stream = sftp.createWriteStream(event.file.name);
                    stream.write(clients[clientId].fileUploadBuffer.toString());
                    stream.end(function () {
                        connection.end();
                    });
                    clients[clientId].fileUploadBuffer = "";
                });
            });

            connection.connect(
                {
                    host: clients[clientId].credentials.host,
                    port: clients[clientId].credentials.port,
                    username: clients[clientId].credentials.username,
                    privateKey: fs.readFileSync(clients[clientId].credentials.sshKey)
                }
            );

        };
    };

    var socketSanityCheck = function (clientId, socket) {
        if (!clients[clientId]) {
            clients[clientId] = new Client();
            clients[clientId].clientID = clientId;
        }
        clients[clientId].socket = socket;

        if (!clients[clientId].mathProgramInstance) {
            mathProgramStart(clientId);
        }
    };

    var listen = function () {
        var cookieParser = require('socket.io-cookie');
        io.use(cookieParser);
        io.on('connection', function (socket) {
            var cookies = socket.request.headers.cookie;
            var clientId = cookies[cookieName];
            socketSanityCheck(clientId, socket);
            attachUploadListenerToSocket(clientId, socket);
            socket.on('input', socketInputAction(clientId));
            socket.on('reset', socketResetAction(clientId));
        });
        return http.listen(options.port);
    };

    var checkStdinAndWrite = function(clientId, msg){
            var stdinIsDown = clients[clientId].mathProgramInstance._writableState.ended;
            if(stdinIsDown){
                mathProgramStart(clientId, function(){
                    checkStdinAndWrite(clientId, msg);
                });
            } else {
                clients[clientId].mathProgramInstance.stdin.write(msg, function (err) {
                    if (err) {
                        logClient(clientId, "write failed: " + err);
                    }
                });
            }
    };

    var socketInputAction = function (clientId) {
        return function (msg) {
            updateLastActiveTime(clientId);
            checkStdinAndWrite(clientId, msg);
        };
    };

    var socketResetAction = function (clientId) {
        return function () {
            logExceptOnTest('Received reset.');
            var client = clients[clientId];
            if (client.recentlyRestarted) {
                logExceptOnTest("Ignore repeated restart request");
                return;
            }
            if (client.mathProgramInstance) {
                killMathProgram(client.mathProgramInstance, clientId);
            }
            resetRecentlyRestarted(client);
            mathProgramStart(clientId);
        };
    };

    process.on('uncaughtException', function (err) {
        console.error('Caught exception in global process object: ' + err);
    });

    setInterval(pruneClients, options.PRUNE_CLIENT_INTERVAL);
    initializeServer();

    // These are the methods available from the outside:
    return {
        listen: listen,
        close: function () {
            http.close();
        }
    };
};

exports.MathServer = MathServer;
<commit_msg>Delete user generate files from Server (not from Docker container)<commit_after>var app = require('express')();
var http = require('http').Server(app);
var fs = require('fs');
var Cookies = require('cookies');
var io = require('socket.io')(http);
var SocketIOFileUpload = require('socketio-file-upload');
var ssh2 = require('ssh2');

var MathServer = function (overrideOptions) {
    var staticFolder = __dirname + '/../public';

    var options = {
        port: 8002, // default port number to use
        PRUNE_CLIENT_INTERVAL: 1000 * 60 * 10, // 10 minutes
        MAX_AGE: 1000 * 60 * 60 * 24 * 7, // 1 week
        CONTAINERS: './dummy_containers.js',
        MATH_PROGRAM: "Macaulay2",
        MATH_PROGRAM_COMMAND: 'M2'
    };

    var logExceptOnTest = function (string) {
        if (process.env.NODE_ENV !== 'test') {
            console.log(string);
        }
    };

    var overrideDefaultOptions = function (overrideOptions) {
        for (var opt in overrideOptions) {
            if (options.hasOwnProperty(opt)) {
                options[opt] = overrideOptions[opt];
                logExceptOnTest("server option: " + opt + " set to " + options[opt]);
            }
        }
    };

    overrideDefaultOptions(overrideOptions);

    var cookieName = "try" + options.MATH_PROGRAM;

    var totalUsers = 0; //only used for stats: total # users since server started

    // Global array of all Client objects.  Each has a math program process.
    var clients = {};
    if (!options.CONTAINERS) {
        console.error("error, no container management given.");
        throw ("No CONTAINERS!");
    }

    var credentialManager = require(options.CONTAINERS).manager();

    // preamble every log with the client ID
    var logClient = function (clientID, str) {
        if (process.env.NODE_ENV !== 'test') {
            logExceptOnTest(clientID + ": " + str);
        }
    };

    var userSpecificPath = function (clientId) {
        return "/" + clientId + "-files/";
    };

    var deleteClient = function (clientID) {
        credentialManager.removeCredentials(clients[clientID].credentials);
        logExceptOnTest("deleting folder " + staticFolder + userSpecificPath(clientID));
        fs.rmdir(staticFolder + userSpecificPath(clientID), function(error) {
            if(error) {
                console.error('Error deleting user folder: ' + error);
            }
        });
        delete clients[clientID];
    };

    var timeBeforeInterval = function (timeInterval) {
        var now = Date.now();
        logExceptOnTest("It is currently " + now + " milliseconds.");
        return now - timeInterval;
    };

    var removeOldClients = function (minimalLastActiveTimeForClient) {
        for (var clientID in clients) {
            if (clients.hasOwnProperty(clientID)) {
                if (clients[clientID].lastActiveTime < minimalLastActiveTimeForClient) {
                    deleteClient(clientID);
                }
            }
        }
    };

    var logCurrentlyActiveClients = function () {
        for (var clientID in clients) {
            if (clients.hasOwnProperty(clientID)) {
                logExceptOnTest(clientID);
            }
        }
    };

    var pruneClients = function () {
        logExceptOnTest("Pruning clients...");
        var minimalLastActiveTimeForClient = timeBeforeInterval(options.MAX_AGE);
        removeOldClients(minimalLastActiveTimeForClient);
        logExceptOnTest("Done pruning clients...  Continuing clients:");
        logCurrentlyActiveClients();
    };

    var Client = function () {
        this.recentlyRestarted = false;
        this.lastActiveTime = Date.now(); // milliseconds when client was last active
        this.credentials = 0;
    };

    var clientIDExists = function (clientID) {
        if (clients[clientID] == null) {
            return false;
        }
        logClient(clientID, "Client already exists");
        return true;
    };

    var getNewClientID = function () {
        totalUsers += 1;
        do {
            var clientID = Math.random() * 1000000;
            clientID = Math.floor(clientID);
        } while (clientIDExists(clientID));
        clientID = "user" + clientID.toString(10);
        logExceptOnTest("New Client ID " + clientID);
        return clientID;
    };

    var setCookie = function (cookies, clientID) {
        cookies.set(cookieName, clientID, {
            httpOnly: false
        });
    };

    var getCredentials = function (clientID, next) {
        if (clients[clientID].credentials) {
            next(clients[clientID].credentials);
        } else {
            credentialManager.getNewCredentials(next);
        }
    };

    var spawnMathProgramInSecureContainer = function (clientID, next) {
        getCredentials(clientID, function (credentials) {
            clients[clientID].credentials = credentials;
            var connection = new ssh2.Client();
            connection.on('ready', function () {
                connection.exec(options.MATH_PROGRAM_COMMAND, {pty: true}, function (err, stream) {
                    if (err) throw err;
                    stream.on('close', function () {
                        connection.end();
                    });
                    stream.on('end', function () {
                        stream.close();
                        logExceptOnTest('I ended.');
                        //connection.end();
                    });
                    next(stream);
                    /*connection.on('end', function(){
                     delete stream;
                     delete connection;
                     });*/
                });
            }).connect({
                host: credentials.host,
                port: credentials.port,
                username: credentials.username,
                privateKey: fs.readFileSync(credentials.sshKey)
            });

        });
    };

    var mathProgramStart = function (clientID, next) {
        logClient(clientID, "Spawning new MathProgram process...");
        spawnMathProgramInSecureContainer(clientID, function (stream) {
            stream.setEncoding("utf8");
            clients[clientID].mathProgramInstance = stream;
            attachListenersToOutput(clientID);
            if (next) {
                next(clientID);
            }
        });
    };

    var captureSpecialEvent = function (data) {
        var eventData = data.match(/>>SPECIAL_EVENT_START>>(.*)<<SPECIAL_EVENT_END/);
        if (eventData) {
            // logExceptOnTest("Have special event: " + eventData[1]);
            return eventData[1];
        }
    };

    var emitUrlForUserGeneratedFileToClient = function (clientId, path) {
        var partAfterLastSlash = /([^\/]*)$/;
        var fileName = path.match(partAfterLastSlash);
        if (fileName) {
            fileName = fileName[0];
        } else {
            return;
        }
        var sshConnection = ssh2();

        sshConnection.on('end', function () {
            logExceptOnTest("Image action ended.");
        });

        sshConnection.on('ready', function () {
            sshConnection.sftp(function (err, sftp) {
                fs.mkdir(staticFolder + userSpecificPath(clientId), function (err) {
                    if (err) {
                        logExceptOnTest("Folder exists, but we proceed anyway");
                    }
                    var completePath = staticFolder + userSpecificPath(clientId) + fileName;
                    sftp.fastGet(path, completePath, function (error) {
                        if (error) {
                            console.error("Error while downloading image. PATH: " + path + ", ERROR: " + error);
                        } else {
                            setTimeout(function () {
                                fs.unlink(completePath, function (err) {
                                    if (err) {
                                        console.error("Error unlinking user generated file " + completePath);
                                        console.error(err);
                                    }
                                })
                            }, 1000 * 60 * 10);
                            clients[clientId].socket.emit(
                                "image", userSpecificPath(clientId) + fileName
                            );
                        }
                    });
                });
            });
        });

        sshConnection.connect(
            {
                host: clients[clientId].credentials.host,
                port: clients[clientId].credentials.port,
                username: clients[clientId].credentials.username,
                privateKey: fs.readFileSync(clients[clientId].credentials.sshKey)
            }
        );

    };

    var emitHelpUrlToClient = function (clientID, viewHelp) {
        logExceptOnTest("Look at " + viewHelp);
        var helpPath = viewHelp.match(/(\/Macaulay2Doc.*)$/);
        if (helpPath) {
            helpPath = helpPath[0];
        } else {
            return;
        }
        helpPath = "http://www.math.uiuc.edu/Macaulay2/doc/Macaulay2-1.7/share/doc/Macaulay2" + helpPath;
        logExceptOnTest(helpPath);
        clients[clientID].socket.emit("viewHelp", helpPath);
    };

    var isViewHelpEvent = function (eventData) {
        return eventData.match(/^file:.*/);
    };

    var emitEventUrlToClient = function (clientID, eventType, data) {
        if (isViewHelpEvent(eventType)) {
            logCurrentlyActiveClients(data);
            emitHelpUrlToClient(clientID, eventType);
            return;
        } else {
            emitUrlForUserGeneratedFileToClient(clientID, eventType);
        }
        var outputData = data.replace(/>>SPECIAL_EVENT_START>>/, "opening ");
        outputData = outputData.replace(/<<SPECIAL_EVENT_END<</, "");
        clients[clientID].socket.emit('result', outputData);
    };

    var sendDataToClient = function (clientID) {
        return function (dataObject) {
            var data = dataObject.toString();
            var socket = clients[clientID].socket;
            if (!socket) {
                logClient(clientID, "Error, no socket for client.");
                return;
            }
            updateLastActiveTime(clientID);
            var specialEvent = captureSpecialEvent(data);
            if (specialEvent) {
                emitEventUrlToClient(clientID, specialEvent, data);
                return;
            }
            socket.emit('result', data);
        };
    };

    var attachListenersToOutput = function (clientID) {
        var client = clients[clientID];
        if (!client) {
            return;
        }
        if (client.mathProgramInstance) {
            clients[clientID].mathProgramInstance
                .removeAllListeners('data')
                .on('data', sendDataToClient(clientID));
            //.stderr('data', sendDataToClient(clientID));
        }
    };

    var stats = function (request, response) {
        // to do: authorization
        response.writeHead(200, {
            "Content-Type": "text/html"
        });
        var currentUsers = 0;
        for (var c in clients) {
            if (clients.hasOwnProperty(c))
                currentUsers = currentUsers + 1;
        }
        response.write(
            '<link rel="stylesheet" href="mathProgram.css" type="text/css" media="screen">');
        response.write('
' + options.MATH_PROGRAM + ' User Statistics
');
        response.write('There are currently ' + currentUsers +
        ' users using ' + options.MATH_PROGRAM + '.
');
        response.write('In total, there were ' + totalUsers +
        ' users since the server started.
');
        response.write('Enjoy ' + options.MATH_PROGRAM + '!');
        response.end();
    };

    var updateLastActiveTime = function (clientID) {
        clients[clientID].lastActiveTime = Date.now();
    };

    var killMathProgram = function (stream, clientID) {
        logClient(clientID, "killMathProgramClient.");
        stream.close();
    };

    var resetRecentlyRestarted = function (client) {
        client.recentlyRestarted = true;
        setTimeout(function () {
            client.recentlyRestarted = false;
        }, 1000);
    };

    var checkCookie = function (request, response, next) {
        var cookies = new Cookies(request, response);
        var clientID = cookies.get(cookieName);
        if (!clientID) {
            logExceptOnTest('New client without a cookie set came along');
            logExceptOnTest('Set new cookie!');
            clientID = getNewClientID();
        }
        setCookie(cookies, clientID);

        if (!clients[clientID]) {
            clients[clientID] = new Client();
        }
        next();
    };

    var unhandled = function (request, response) {
        var url = require('url').parse(request.url).pathname;
        logExceptOnTest("Request for something we don't serve: " + request.url);
        response.writeHead(404, "Request for something we don't serve.");
        response.write("404");
        response.end();
    };

    var initializeServer = function () {
        var favicon = require('serve-favicon');
        var serveStatic = require('serve-static');
        var winston = require('winston'),
            expressWinston = require('express-winston');

        var loggerSettings = {
            transports: [
                new winston.transports.Console({
                    level: 'error',
                    json: true,
                    colorize: true
                })
            ]
        };
        var prefix = staticFolder + "-" + options.MATH_PROGRAM + "/";
        var tutorialReader = require('./tutorialReader.js')(prefix, fs);
        app.use(favicon(staticFolder + '-' + options.MATH_PROGRAM + '/favicon.ico'));
        app.use(SocketIOFileUpload.router);
        app.use(checkCookie);
        app.use(serveStatic(staticFolder + '-' + options.MATH_PROGRAM));
        app.use(serveStatic(staticFolder));
        app.use(expressWinston.logger(loggerSettings));
        app.use('/admin', stats)
            .use('/getListOfTutorials', tutorialReader.getList)
            .use(unhandled);
    };

    var attachUploadListenerToSocket = function (clientId, socket) {
        var uploader = new SocketIOFileUpload();
        uploader.listen(socket);

        uploader.on("error", function (event) {
            console.error("Error in upload " + event);
        });

        uploader.on("start", function (event) {
            clients[clientId].fileUploadBuffer = "";
            logExceptOnTest('File upload ' + event.file.name);
        });

        uploader.on("progress", function (event) {
            // TODO: Limit size.
            clients[clientId].fileUploadBuffer += event.buffer;
        });

        uploader.on("complete", completeFileUpload(clientId));
    };

    var completeFileUpload = function (clientId) {
        return function (event) {
            var connection = ssh2();

            connection.on('end', function () {
            });

            connection.on('ready', function () {
                connection.sftp(function (err, sftp) {
                    var stream = sftp.createWriteStream(event.file.name);
                    stream.write(clients[clientId].fileUploadBuffer.toString());
                    stream.end(function () {
                        connection.end();
                    });
                    clients[clientId].fileUploadBuffer = "";
                });
            });

            connection.connect(
                {
                    host: clients[clientId].credentials.host,
                    port: clients[clientId].credentials.port,
                    username: clients[clientId].credentials.username,
                    privateKey: fs.readFileSync(clients[clientId].credentials.sshKey)
                }
            );

        };
    };

    var socketSanityCheck = function (clientId, socket) {
        if (!clients[clientId]) {
            clients[clientId] = new Client();
            clients[clientId].clientID = clientId;
        }
        clients[clientId].socket = socket;

        if (!clients[clientId].mathProgramInstance) {
            mathProgramStart(clientId);
        }
    };

    var listen = function () {
        var cookieParser = require('socket.io-cookie');
        io.use(cookieParser);
        io.on('connection', function (socket) {
            var cookies = socket.request.headers.cookie;
            var clientId = cookies[cookieName];
            socketSanityCheck(clientId, socket);
            attachUploadListenerToSocket(clientId, socket);
            socket.on('input', socketInputAction(clientId));
            socket.on('reset', socketResetAction(clientId));
        });
        return http.listen(options.port);
    };

    var checkStdinAndWrite = function (clientId, msg) {
        var stdinIsDown = clients[clientId].mathProgramInstance._writableState.ended;
        if (stdinIsDown) {
            mathProgramStart(clientId, function () {
                checkStdinAndWrite(clientId, msg);
            });
        } else {
            clients[clientId].mathProgramInstance.stdin.write(msg, function (err) {
                if (err) {
                    logClient(clientId, "write failed: " + err);
                }
            });
        }
    };

    var socketInputAction = function (clientId) {
        return function (msg) {
            updateLastActiveTime(clientId);
            checkStdinAndWrite(clientId, msg);
        };
    };

    var socketResetAction = function (clientId) {
        return function () {
            logExceptOnTest('Received reset.');
            var client = clients[clientId];
            if (client.recentlyRestarted) {
                logExceptOnTest("Ignore repeated restart request");
                return;
            }
            if (client.mathProgramInstance) {
                killMathProgram(client.mathProgramInstance, clientId);
            }
            resetRecentlyRestarted(client);
            mathProgramStart(clientId);
        };
    };

    process.on('uncaughtException', function (err) {
        console.error('Caught exception in global process object: ' + err);
    });

    setInterval(pruneClients, options.PRUNE_CLIENT_INTERVAL);
    initializeServer();

    // These are the methods available from the outside:
    return {
        listen: listen,
        close: function () {
            http.close();
        }
    };
};

exports.MathServer = MathServer;
<|endoftext|>


<commit_before>"use strict";
var fs = require('fs-extra'),
    glob = require('glob'),
    crypto = require('crypto'),
    async = require('async'),
    path = require('path'),
    zlib = require('zlib'),
    mime = require('mime');

mime.define({
    'application/xml': ['plist']
});

function isGlob(s){
    return s.indexOf('/*') > -1;
}

function globToRegex(s){
    return new RegExp(s.replace(/\*?\*/, '(.*)'));
}

// Temp.
function compressData(data, done){
    zlib.gzip(data, done);
}

// @todo (lucas) Could context hold open resources that can be flushed to
// disk after a step is completed?
function Context(metadata){
    this.ready = false;
    this.keys = [];
    if(metadata){
        this.extend(metadata);
    }
    this.baseDir = process.cwd();

    this.filesTouched = {};
}

Context.prototype.toKey = function(src){
    return src.replace(this.path('build/'), '');
};

Context.prototype.path = function(src){
    var args = Array.prototype.slice.call(arguments, 0);
    // return path.resolve(this.baseDir, path.join(args));
    return path.resolve(this.baseDir, src);
};

Context.prototype.dest = function(key, src){
    return this[key][src].dest;
};

Context.prototype.writeFile = function(path, buf, done){
    var self = this;

    fs.writeFile(path, buf, function(err){
        self.filesTouched[path] = buf;
        done(err);
    });
    return this;
};

Context.prototype.readFile = function(src, done){
    var self = this;

    // if(self.filesTouched[path]){
    //     return done(undefined, self.filesTouched[path]);
    // }
    fs.readFile(src, function(err, buf){
        done(err, buf);
    });
    return this;
};

Context.prototype.copyFile = function(src, dest, done){
    var self = this;
    fs.mkdirs(path.dirname(dest), function(err){
        if(err){
            return done(err);
        }
        fs.copy(src, dest, function(err){
            self.filesTouched[dest] = true;
            done(err);
        });
    });
    return this;
};

// Return paths to build files this context produced.
Context.prototype.files = function(){
    return Object.keys(this.filesTouched);
};

Context.prototype.getFileInfo = function(src, done){
    fs.stat(src, function(err, stats){
        if(err) {
            return done(err);
        }

        stats.src = src;
        if(!stats.isFile()){
            return done(null, stats);
        }

        fs.readFile(src, function(err, data){
            if(err) {
                return done(err);
            }
            var hash = crypto.createHash('md5').update(data);

            stats.mime = mime.lookup(src);
            stats.body = data;
            stats.md5 =  crypto.createHash('md5').update(data).digest('hex');
            stats.contentMD5 = crypto.createHash('md5').update(data).digest('base64');

            return done(null, stats);
        });
    });
};

Context.prototype.extend = function(o){
    for(var key in o){
        this[key] = o[key];
        this.keys.push(key);
    }
    return this;
};

Context.prototype.getConfig = function(){
    var o = {},
        self = this;

    (this.config.export || Object.keys(this.config)).map(function(key){
        o[key] = self.config[key];
    });
    o.environment = this.environment;
    return o;
};

Context.prototype.prepare = function(done){
    var self = this,
        tasks = [];

    tasks = this.keys.map(function(key){
        if(self[key] !== Object(self[key])){
            return null;
        }
        return function(callback){

            var v = Object.keys(self[key]);
            async.parallel(v.map(function(src){
                return function(cb){
                    var dest = self.path('build/' + (self[key][src].dest || self[key][src]));
                    if(isGlob(src)){
                        new glob.Glob(self.path(src), {'mark': true}, function(err, matches){
                            if(err){
                                return cb(err);
                            }
                            var re = globToRegex(self.path(src));

                            matches.map(function(match){
                                self[key][match] = {};
                                // Now for fun...
                                // Handle pages/*.jade: pages/$1.html
                                if(dest.indexOf('$') > -1){
                                    self[key][match].dest = match.replace(re, dest);
                                }
                                else{
                                    self[key][match].dest = dest;
                                }
                            });
                            delete self[key][src];
                            cb();
                        });
                    }
                    else {
                        if(self[key][src] !== Object(self[key][src])){
                            self[key][self.path(src)] = {};
                        }
                        else {
                            self[key][self.path(src)] = self[key][src];
                        }

                        self[key][self.path(src)].dest = dest;
                        delete self[key][src];
                        cb();
                    }
                };
            }), callback);
        };
    }).filter(function(r){
        return r !== null;
    });
    async.parallel(tasks, function(err){
        if(err){
            return done(err);
        }
        self.ready = true;
        done();
    });
};

Context.prototype.runTask = function(name, done){
    return this.cookbook.exec(name, {}, done);
};

module.exports = Context;
<commit_msg>Don't export all config keys by default yo<commit_after>"use strict";
var fs = require('fs-extra'),
    glob = require('glob'),
    crypto = require('crypto'),
    async = require('async'),
    path = require('path'),
    zlib = require('zlib'),
    mime = require('mime');

mime.define({
    'application/xml': ['plist']
});

function isGlob(s){
    return s.indexOf('/*') > -1;
}

function globToRegex(s){
    return new RegExp(s.replace(/\*?\*/, '(.*)'));
}

// Temp.
function compressData(data, done){
    zlib.gzip(data, done);
}

// @todo (lucas) Could context hold open resources that can be flushed to
// disk after a step is completed?
function Context(metadata){
    this.ready = false;
    this.keys = [];
    if(metadata){
        this.extend(metadata);
    }
    this.baseDir = process.cwd();

    this.filesTouched = {};
}

Context.prototype.toKey = function(src){
    return src.replace(this.path('build/'), '');
};

Context.prototype.path = function(src){
    var args = Array.prototype.slice.call(arguments, 0);
    // return path.resolve(this.baseDir, path.join(args));
    return path.resolve(this.baseDir, src);
};

Context.prototype.dest = function(key, src){
    return this[key][src].dest;
};

Context.prototype.writeFile = function(path, buf, done){
    var self = this;

    fs.writeFile(path, buf, function(err){
        self.filesTouched[path] = buf;
        done(err);
    });
    return this;
};

Context.prototype.readFile = function(src, done){
    var self = this;

    // if(self.filesTouched[path]){
    //     return done(undefined, self.filesTouched[path]);
    // }
    fs.readFile(src, function(err, buf){
        done(err, buf);
    });
    return this;
};

Context.prototype.copyFile = function(src, dest, done){
    var self = this;
    fs.mkdirs(path.dirname(dest), function(err){
        if(err){
            return done(err);
        }
        fs.copy(src, dest, function(err){
            self.filesTouched[dest] = true;
            done(err);
        });
    });
    return this;
};

// Return paths to build files this context produced.
Context.prototype.files = function(){
    return Object.keys(this.filesTouched);
};

Context.prototype.getFileInfo = function(src, done){
    fs.stat(src, function(err, stats){
        if(err) {
            return done(err);
        }

        stats.src = src;
        if(!stats.isFile()){
            return done(null, stats);
        }

        fs.readFile(src, function(err, data){
            if(err) {
                return done(err);
            }
            var hash = crypto.createHash('md5').update(data);

            stats.mime = mime.lookup(src);
            stats.body = data;
            stats.md5 =  crypto.createHash('md5').update(data).digest('hex');
            stats.contentMD5 = crypto.createHash('md5').update(data).digest('base64');

            return done(null, stats);
        });
    });
};

Context.prototype.extend = function(o){
    for(var key in o){
        this[key] = o[key];
        this.keys.push(key);
    }
    return this;
};

Context.prototype.getConfig = function(){
    var o = {},
        self = this;

    (this.config.export || []).map(function(key){
        o[key] = self.config[key];
    });
    o.environment = this.environment;
    return o;
};

Context.prototype.prepare = function(done){
    var self = this,
        tasks = [];

    tasks = this.keys.map(function(key){
        if(self[key] !== Object(self[key])){
            return null;
        }
        return function(callback){

            var v = Object.keys(self[key]);
            async.parallel(v.map(function(src){
                return function(cb){
                    var dest = self.path('build/' + (self[key][src].dest || self[key][src]));
                    if(isGlob(src)){
                        new glob.Glob(self.path(src), {'mark': true}, function(err, matches){
                            if(err){
                                return cb(err);
                            }
                            var re = globToRegex(self.path(src));

                            matches.map(function(match){
                                self[key][match] = {};
                                // Now for fun...
                                // Handle pages/*.jade: pages/$1.html
                                if(dest.indexOf('$') > -1){
                                    self[key][match].dest = match.replace(re, dest);
                                }
                                else{
                                    self[key][match].dest = dest;
                                }
                            });
                            delete self[key][src];
                            cb();
                        });
                    }
                    else {
                        if(self[key][src] !== Object(self[key][src])){
                            self[key][self.path(src)] = {};
                        }
                        else {
                            self[key][self.path(src)] = self[key][src];
                        }

                        self[key][self.path(src)].dest = dest;
                        delete self[key][src];
                        cb();
                    }
                };
            }), callback);
        };
    }).filter(function(r){
        return r !== null;
    });
    async.parallel(tasks, function(err){
        if(err){
            return done(err);
        }
        self.ready = true;
        done();
    });
};

Context.prototype.runTask = function(name, done){
    return this.cookbook.exec(name, {}, done);
};

module.exports = Context;
<|endoftext|>


<commit_before>/**
 * Created by barakedry on 6/19/15.
 */
'use strict';
var _ = require('lodash');
var util = require('util');
var utils = require('./utils');
var DiffTracker = require('./diff-tracker');
var EventEmitter = require('events').EventEmitter;
var debug = require('debuglog')('merge-diff');

function MergeDiff(object, options) {
    this.options = _.defaults(options || {}, {
        emitEvents: true, // settings this to false should allow faster merging but it is not implemented yet
        undefinedKeyword: '__$$U',
        deleteKeyword: '__$$D',
        protoKeyword: '__$$P',
        mergeDeletions: true,
        mergeAdditions: true,
        emitAdditions: true,
        emitUpdates: true,
        emitDifferences: true,
        maxKeysInLevel: 1000,
        maxLevels: 50,
        maxListeners: 1000000
    });

    this.object = object || {};
    //this.prototypes = []; // prototypes are stored in a special collection

    this.setMaxListeners(this.options.maxListeners);
}

util.inherits(MergeDiff, EventEmitter);

MergeDiff.prototype.merge = function (delta, path, options) {

    options = _.defaults(options || {}, this.options);

    if (!_.isObject(delta) && !path) {
        debug('invalid merge, source and delta must be objects');
        return;
    }

    this._mergeObject(this.object, utils.wrapByPath(delta, path), "", options, 0);
};

MergeDiff.prototype.override = function (delta, path, options) {

    options = _.defaults(options || {}, this.options);

    if (!_.isObject(delta) && !path) {
        debug('invalid merge, source and delta must be objects');
        return;
    }

    this._mergeObject(this.object, utils.wrapByPath(delta, path), "", options, 0, path);
};

MergeDiff.prototype.delete = function (path) {

    if (!(path && _.isString(path))) {
        debug('invalid path, cannot delete');
        return;
    }

    this._mergeObject(this.object, utils.wrapByPath(this.options.deleteKeyword, path), "", this.options, 0);
};

MergeDiff.prototype.get = function (path) {

    if (!(path && _.isString(path))) {
        debug('invalid path, cannot get');
        return;
    }

    return _.get(this.object, path);
};

/************************************************************************************
 * The basic merging recursion implementation:
 * ._mergeObject() -> ._mergeAtKey() -> ._mergeObject() -> ._mergeAtKey() -> ...
 *
 * ._mergeObject() iterate keys at level and calls ._mergeAtKey() for each key,
 * after iteration ends it emits and returns level changes to the caller.
 *
 * ._mergeAtKey() assigns/delete primitives and calls _.mergeObject() for objects
 ************************************************************************************/
MergeDiff.prototype._mergeObject = function (source, delta, path, options, level, override) {
    var keys,
        key,
        i,
        length,
        levelDiffs,
        isSourceArray;

    if (!(_.isObject(source) && _.isObject(delta))) {
        debug('invalid merge, source and delta must be objects');
        this.emit('error', new Error('invalid merge, source and delta must be objects'));
        return;
    }

    if (level > options.maxLevels) {
        debug('Trying to merge too deep, stopping at level %d', level);
        this.emit('error', new Error('Trying to merge too deep, stopping at level ' + level));
        return;
    }

    keys = _.keys(delta);
    length = keys.length;
    isSourceArray = _.isArray(source);

    if (options.emitEvents) {
        levelDiffs = DiffTracker.create();
        levelDiffs.path = path;
    }

    if (isSourceArray) {
        levelDiffs = levelDiffs || {};
        levelDiffs.arrayOffset = 0;
    }

    if (length > options.maxKeysInLevel) {
        debug('Stopping Merge, Too many keys in object - %d out of %d allowed keys.', length, options.maxKeysInLevel);
        this.emit('error', new Error('Stopping Merge, Too many keys in object - ' + length + ' out of ' + options.maxKeysInLevel + ' allowed keys.'));
        return levelDiffs;
    }

    // override is either undefined, a path or true
    if (!_.isUndefined(override) && (override === true || override === path)) {
        override = true;
    }

    for (i = 0; i < length; i++) {
        key = keys[i];

        if (utils.isValid(delta[key]) && delta[key] !== source[key]) {
            levelDiffs = this._mergeAtKey(source, delta, path, key, levelDiffs, options, isSourceArray, override);
        }

    }

    if (override) {
        // find keys at this level that exists at the source object and delete them
        levelDiffs = this._detectDeletionsAtLevel(source, key, delta, levelDiffs, path, options, isSourceArray);
    }

    if (options.emitEvents) {
        this.emit(path || "*", levelDiffs);
    }

    return levelDiffs;
};

MergeDiff.prototype._mergeAtKey = function (source, delta, path, key, levelDiffs, options, isSourceArray, override) {

    var childDiffs,
        mergingValue,
        existingValue,
        srcKey,
        mergedValue, // this value is what goes out to the tracker its not allways the same as mergingValue
        isExistingValueArray,
        isObjectMerge = false;

    mergingValue = delta[key];

    if (isSourceArray) {
        srcKey = Number(key) + levelDiffs.arrayOffset;
    } else {
        srcKey = key;
    }

    if (_.isFunction(mergingValue)) {
        mergedValue = utils.SERIALIZABLE_FUNCTION;
    } else {
        isObjectMerge = _.isObject(mergingValue);
        if (_.isUndefined(mergingValue)) {
            mergedValue = options.undefinedKeyword;
        } else if (mergingValue === options.undefinedKeyword) {
            mergedValue = mergingValue;
            mergingValue = undefined;
        } else {
            mergedValue = mergingValue;
        }
    }

    // new
    if (!source.hasOwnProperty(srcKey)) {
        if (options.mergeAdditions && delta[key] !== options.deleteKeyword) {

            // add new object
            if (isObjectMerge) {

                source[srcKey] = Object.create(this.getPrototypeOf(mergingValue));
                childDiffs = this._mergeObject(source[srcKey],
                                               mergingValue,
                                               utils.concatPath(path, key),
                                               options,
                                               _.isArray(source[srcKey]),
                                               override);

                levelDiffs.addChildTracking(childDiffs, key);

            // add new primitive
            } else {

                source[srcKey] = mergingValue;

                levelDiffs.hasAdditions = true;
                levelDiffs.additions[key] = mergedValue;
                levelDiffs.hasDifferences = true;
                levelDiffs.differences[key] = mergedValue;
            }
        }
    // existing
    } else {

        existingValue = source[srcKey];
        isExistingValueArray = _.isArray(existingValue);

        // delete
        if (delta[key] === options.deleteKeyword) {

            levelDiffs = this._deleteAtKey(source, path, key, srcKey, options, existingValue, isSourceArray, levelDiffs);

        // update
        } else {

            // update object
            if (isObjectMerge) {

                // we should replace the source value, todo: array merges check is not sufficient
                if (!isExistingValueArray && !utils.hasSamePrototype(existingValue, mergingValue)) {

                    // this is a restructure
                    // handle prototypes
                    source[srcKey] = Object.create(this.getPrototypeOf(mergingValue));

                }

                childDiffs = this._mergeObject(source[srcKey],
                                               mergingValue,
                                               utils.concatPath(path, key),
                                               options,
                                               isExistingValueArray,
                                               override);

                levelDiffs.addChildTracking(childDiffs, key);

            // update primitive
            } else {

                source[srcKey] = mergingValue;

                levelDiffs.hasUpdates = true;
                levelDiffs.updates[key] = {'old': existingValue, 'new': mergedValue};
                levelDiffs.hasDifferences = true;
                levelDiffs.differences[key] = mergedValue;
            }

        }

    }

    return levelDiffs;
};

MergeDiff.prototype._deleteAtKey = function (source, path, key, srcKey, options, existingValue, isSourceArray, levelDiffs) {
    if (options.mergeDeletions) {
        if (isSourceArray) {
            source.splice(srcKey, 1);
            levelDiffs.arrayOffset--;
        } else {
            delete source[srcKey];
        }

    }

    levelDiffs.deletions[key] = existingValue;
    levelDiffs.differences[key] = options.deleteKeyword;
    levelDiffs.hasDeletions = true;

    this.emitInnerDeletions(path, existingValue);

    return levelDiffs;
};

MergeDiff.prototype._detectDeletionsAtLevel = function (source, key, delta, levelDiffs, path, options, isSourceArray) {
    var keys = _.keys(source),
        length = keys.length,
        existingValue,
        srcKey,
        i;

    // run source object at this level
    for (i = 0; i < length; i++) {
        key = keys[i];
        if (isSourceArray) {
            srcKey = Number(key) + levelDiffs.arrayOffset;
        } else {
            srcKey = key;
        }

        if (!delta.hasOwnProperty(key)) {
            existingValue = source[key];

            levelDiffs = this._deleteAtKey(source, path, key, srcKey, options, existingValue,
                isSourceArray, levelDiffs);

        }

    }
    return levelDiffs;
};

MergeDiff.prototype.getPrototypeOf = function (object) {
    return Object.getPrototypeOf(object);
};

MergeDiff.prototype.emitInnerDeletions = function (deletedObject, path) {
    var keys, key, i;

    if (!_.isObject(deletedObject)) {
        return;
    }

    keys = _.keys(deletedObject);
    for (i = 0; i < keys.length; i++) {
        key = keys[i];
        if (_.isObject(deletedObject[key])) {
            this.emitInnerDeletions(deletedObject[key], utils.concatPath(path, key));
        }
    }

    this.deletions.emit(path, deletedObject);
    this.difference.emit(path, this.options.deleteKeyword);
};

module.exports = MergeDiff;
<commit_msg>changes in delete<commit_after>/**
 * Created by barakedry on 6/19/15.
 */
'use strict';
var _ = require('lodash');
var util = require('util');
var utils = require('./utils');
var DiffTracker = require('./diff-tracker');
var EventEmitter = require('events').EventEmitter;
var debug = require('debuglog')('merge-diff');

function MergeDiff(object, options) {
    this.options = _.defaults(options || {}, {
        emitEvents: true, // settings this to false should allow faster merging but it is not implemented yet
        undefinedKeyword: '__$$U',
        deleteKeyword: '__$$D',
        protoKeyword: '__$$P',
        mergeDeletions: true,
        mergeAdditions: true,
        emitAdditions: true,
        emitUpdates: true,
        emitDifferences: true,
        maxKeysInLevel: 1000,
        maxLevels: 50,
        maxListeners: 1000000
    });

    this.object = object || {};
    //this.prototypes = []; // prototypes are stored in a special collection

    this.setMaxListeners(this.options.maxListeners);
}

util.inherits(MergeDiff, EventEmitter);

MergeDiff.prototype.merge = function (delta, path, options) {

    options = _.defaults(options || {}, this.options);

    if (!_.isObject(delta) && !path) {
        debug('invalid merge, source and delta must be objects');
        return;
    }

    this._mergeObject(this.object, utils.wrapByPath(delta, path), "", options, 0);
};

MergeDiff.prototype.override = function (delta, path, options) {

    options = _.defaults(options || {}, this.options);

    if (!_.isObject(delta) && !path) {
        debug('invalid merge, source and delta must be objects');
        return;
    }

    this._mergeObject(this.object, utils.wrapByPath(delta, path), "", options, 0, path);
};

MergeDiff.prototype.delete = function (path) {

    if (!(path && _.isString(path))) {
        debug('invalid path, cannot delete');
        return;
    }

    this._mergeObject(this.object, utils.wrapByPath(this.options.deleteKeyword, path), "", this.options, 0);
};

MergeDiff.prototype.get = function (path) {

    if (!(path && _.isString(path))) {
        debug('invalid path, cannot get');
        return;
    }

    return _.get(this.object, path);
};

/************************************************************************************
 * The basic merging recursion implementation:
 * ._mergeObject() -> ._mergeAtKey() -> ._mergeObject() -> ._mergeAtKey() -> ...
 *
 * ._mergeObject() iterate keys at level and calls ._mergeAtKey() for each key,
 * after iteration ends it emits and returns level changes to the caller.
 *
 * ._mergeAtKey() assigns/delete primitives and calls _.mergeObject() for objects
 ************************************************************************************/
MergeDiff.prototype._mergeObject = function (source, delta, path, options, level, override) {
    var keys,
        key,
        i,
        length,
        levelDiffs,
        isSourceArray;

    if (!(_.isObject(source) && _.isObject(delta))) {
        debug('invalid merge, source and delta must be objects');
        this.emit('error', new Error('invalid merge, source and delta must be objects'));
        return;
    }

    if (level > options.maxLevels) {
        debug('Trying to merge too deep, stopping at level %d', level);
        this.emit('error', new Error('Trying to merge too deep, stopping at level ' + level));
        return;
    }

    keys = _.keys(delta);
    length = keys.length;
    isSourceArray = _.isArray(source);

    if (options.emitEvents) {
        levelDiffs = DiffTracker.create();
        levelDiffs.path = path;
    }

    if (isSourceArray) {
        levelDiffs = levelDiffs || {};
        levelDiffs.arrayOffset = 0;
    }

    if (length > options.maxKeysInLevel) {
        debug('Stopping Merge, Too many keys in object - %d out of %d allowed keys.', length, options.maxKeysInLevel);
        this.emit('error', new Error('Stopping Merge, Too many keys in object - ' + length + ' out of ' + options.maxKeysInLevel + ' allowed keys.'));
        return levelDiffs;
    }

    // override is either undefined, a path or true
    if (!_.isUndefined(override) && (override === true || override === path)) {
        override = true;
    }

    for (i = 0; i < length; i++) {
        key = keys[i];

        if (utils.isValid(delta[key]) && delta[key] !== source[key]) {
            levelDiffs = this._mergeAtKey(source, delta, path, key, levelDiffs, options, isSourceArray, override);
        }

    }

    if (override) {
        // find keys at this level that exists at the source object and delete them
        levelDiffs = this._detectDeletionsAtLevel(source, delta, levelDiffs, path, options, isSourceArray);
    }

    if (options.emitEvents) {
        this.emit(path || "*", levelDiffs);
    }

    return levelDiffs;
};

MergeDiff.prototype._mergeAtKey = function (source, delta, path, key, levelDiffs, options, isSourceArray, override) {

    var childDiffs,
        mergingValue,
        existingValue,
        srcKey,
        mergedValue, // this value is what goes out to the tracker its not allways the same as mergingValue
        isExistingValueArray,
        isObjectMerge = false;

    mergingValue = delta[key];

    if (isSourceArray) {
        srcKey = Number(key) + levelDiffs.arrayOffset;
    } else {
        srcKey = key;
    }

    if (_.isFunction(mergingValue)) {
        mergedValue = utils.SERIALIZABLE_FUNCTION;
    } else {
        isObjectMerge = _.isObject(mergingValue);
        if (_.isUndefined(mergingValue)) {
            mergedValue = options.undefinedKeyword;
        } else if (mergingValue === options.undefinedKeyword) {
            mergedValue = mergingValue;
            mergingValue = undefined;
        } else {
            mergedValue = mergingValue;
        }
    }

    // new
    if (!source.hasOwnProperty(srcKey)) {
        if (options.mergeAdditions && delta[key] !== options.deleteKeyword) {

            // add new object
            if (isObjectMerge) {

                source[srcKey] = Object.create(this.getPrototypeOf(mergingValue));
                childDiffs = this._mergeObject(source[srcKey],
                                               mergingValue,
                                               utils.concatPath(path, key),
                                               options,
                                               _.isArray(source[srcKey]),
                                               override);

                levelDiffs.addChildTracking(childDiffs, key);

            // add new primitive
            } else {

                source[srcKey] = mergingValue;

                levelDiffs.hasAdditions = true;
                levelDiffs.additions[key] = mergedValue;
                levelDiffs.hasDifferences = true;
                levelDiffs.differences[key] = mergedValue;
            }
        }
    // existing
    } else {

        existingValue = source[srcKey];
        isExistingValueArray = _.isArray(existingValue);

        // delete
        if (delta[key] === options.deleteKeyword) {

            levelDiffs = this._deleteAtKey(source, path, key, srcKey, options, existingValue, isSourceArray, levelDiffs);

        // update
        } else {

            // update object
            if (isObjectMerge) {

                // we should replace the source value, todo: array merges check is not sufficient
                if (!isExistingValueArray && !utils.hasSamePrototype(existingValue, mergingValue)) {

                    // this is a restructure
                    // handle prototypes
                    source[srcKey] = Object.create(this.getPrototypeOf(mergingValue));

                }

                childDiffs = this._mergeObject(source[srcKey],
                                               mergingValue,
                                               utils.concatPath(path, key),
                                               options,
                                               isExistingValueArray,
                                               override);

                levelDiffs.addChildTracking(childDiffs, key);

            // update primitive
            } else {

                source[srcKey] = mergingValue;

                levelDiffs.hasUpdates = true;
                levelDiffs.updates[key] = {'old': existingValue, 'new': mergedValue};
                levelDiffs.hasDifferences = true;
                levelDiffs.differences[key] = mergedValue;
            }

        }

    }

    return levelDiffs;
};

MergeDiff.prototype._deleteAtKey = function (source, path, key, srcKey, options, existingValue, isSourceArray, levelDiffs) {
    if (options.mergeDeletions) {
        if (isSourceArray) {
            source.splice(srcKey, 1);
            levelDiffs.arrayOffset--;
        } else {
            delete source[srcKey];
        }

    }

    levelDiffs.deletions[key] = existingValue;
    levelDiffs.differences[key] = options.deleteKeyword;
    levelDiffs.hasDeletions = true;

    this.emitInnerDeletions(path, existingValue);

    return levelDiffs;
};

MergeDiff.prototype._detectDeletionsAtLevel = function (source, delta, levelDiffs, path, options, isSourceArray) {
    var keys = _.keys(source),
        length = keys.length,
        existingValue,
        srcKey,
        key,
        i;

    // run source object at this level
    for (i = 0; i < length; i++) {
        key = keys[i];
        if (isSourceArray) {
            srcKey = Number(key) + levelDiffs.arrayOffset;
        } else {
            srcKey = key;
        }

        if (!delta.hasOwnProperty(key)) {
            existingValue = source[key];

            levelDiffs = this._deleteAtKey(source, path, key, srcKey, options, existingValue,
                isSourceArray, levelDiffs);

        }

    }
    return levelDiffs;
};

MergeDiff.prototype.getPrototypeOf = function (object) {
    return Object.getPrototypeOf(object);
};

MergeDiff.prototype.emitInnerDeletions = function (deletedObject, path) {
    var keys, key, i;

    if (!_.isObject(deletedObject)) {
        return;
    }

    keys = _.keys(deletedObject);
    for (i = 0; i < keys.length; i++) {
        key = keys[i];
        if (_.isObject(deletedObject[key])) {
            this.emitInnerDeletions(deletedObject[key], utils.concatPath(path, key));
        }
    }

    this.deletions.emit(path, deletedObject);
    this.difference.emit(path, this.options.deleteKeyword);
};

module.exports = MergeDiff;
<|endoftext|>


<commit_before>/**
 * @author Gilles Coomans <gilles.coomans@gmail.com>
 * magical context that follow promises scopes
 */

if (typeof define !== 'function') {
	var define = require('amdefine')(module);
}
define(["require", "./promise", "./utils", "./logs"], function(require, prom, utils, logs) {
	var Promise = prom.Promise;
	Promise.context = {
		rethrow: false,
		debug: true
	};
	Promise.contextualise = function() {
		// shallow copy all context
		Promise.context = utils.shallowCopy(Promise.context);

		// shallowcopy context's protocols (if any)
		if (Promise.context.protocols)
			Promise.context.protocols = utils.shallowCopy(Promise.context.protocols);

		// hard copy modes
		if (Promise.context.modes)
			Promise.context.modes = utils.copy(Promise.context.modes);

		return Promise.context;
	};

	/**
	 * set key/value in current Promise.context
	 *
	 * @chainable
	 * @method context
	 * @param  {String} key
	 * @param  {*} value
	 * @return {deep.Chain} this
	 */
	Promise.prototype.toContext = function(key, val) {
		var self = this;
		var func = function(s, e) {
			if (!key)
				return errors.Internal(".toContext need key/val couple.");
			val = (typeof val === 'undefined') ? s : val;
			self._context[key] = val;
			return val;
		};
		func._isDone_ = true;
		return self._enqueue(func);
	};
	/**
	 * shallow copy current Promise.context
	 *
	 * @chainable
	 * @method contextualise
	 * @return {deep.Chain} this
	 */
	Promise.prototype.contextualise = function(arg) {
		var self = this;
		var func = function(s, e) {
			self._context = Promise.contextualise();
			if(arg)
				for(var i in arg)
					self._context[i] = arg[i];
			return s;
		};
		func._isDone_ = true;
		return self._enqueue(func);
	};
	/**
	 * read key/value in current Promise.context
	 *
	 * @chainable
	 * @method context
	 * @param  {String} key
	 * @param  {*} value
	 * @return {deep.Chain} this
	 */
	Promise.prototype.fromContext = function(key) {
		var self = this;
		var create = function(s, e) {
			if (!key)
				return self._context;
			var out = self._context[key];
			return (typeof out === 'undefined') ? promise.Undefined : out;
		};
		create._isDone_ = true;
		return self._enqueue(create);
	};
	/**
	 * log current Promise.context. If key is provided : log only this property.
	 *
	 * @chainable
	 * @method logContext
	 * @param  {String} key (optional)
	 * @return {deep.Chain} this
	 */
	Promise.prototype.clog = function(key) {
		var self = this;
		var func = function chainLogHandle(s, e) {
			if (key)
				logs.log("deep.chain.context." + key + " : ", self._context[key]);
			else
				logs.log("deep.chain.context : ", self._context);
		};
		return self._enqueue(func);
	};

	return {
		context:function(name, value) {
			if (!name)
				return Promise.context;
			if (!value)
				return Promise.context[name];
			return Promise.context[name] = value;
		},
		contextualise : function(arg) {
			return new Promise().contextualise(arg).resolve(deep.context());
		},
		fromContext : function(key) {
			return new Promise().resolve(Promise.context[key]);
		}
	};
});<commit_msg>pass promise as argument for Promise.contextualise<commit_after>/**
 * @author Gilles Coomans <gilles.coomans@gmail.com>
 * magical context that follow promises scopes
 */

if (typeof define !== 'function') {
	var define = require('amdefine')(module);
}
define(["require", "./promise", "./utils", "./logs"], function(require, prom, utils, logs) {
	var Promise = prom.Promise;
	Promise.context = {
		rethrow: false,
		debug: true
	};
	Promise.contextualise = function(p) {
		// shallow copy all context
		p._context = Promise.context = utils.shallowCopy(Promise.context);
		p._contextualised = true;
		// shallowcopy context's protocols (if any)
		if (Promise.context.protocols)
			Promise.context.protocols = utils.shallowCopy(Promise.context.protocols);

		// hard copy modes
		if (Promise.context.modes)
			Promise.context.modes = utils.copy(Promise.context.modes);

		return Promise.context;
	};

	/**
	 * set key/value in current Promise.context
	 *
	 * @chainable
	 * @method context
	 * @param  {String} key
	 * @param  {*} value
	 * @return {deep.Chain} this
	 */
	Promise.prototype.toContext = function(key, val) {
		var self = this;
		var func = function(s, e) {
			if (!key)
				return errors.Internal(".toContext need key/val couple.");
			val = (typeof val === 'undefined') ? s : val;
			self._context[key] = val;
			return val;
		};
		func._isDone_ = true;
		return self._enqueue(func);
	};
	/**
	 * shallow copy current Promise.context
	 *
	 * @chainable
	 * @method contextualise
	 * @return {deep.Chain} this
	 */
	Promise.prototype.contextualise = function(arg) {
		var self = this;
		var func = function(s, e) {
			self._context = Promise.contextualise(self);
			self._contextualised = true;
			if(arg)
				for(var i in arg)
					self._context[i] = arg[i];
			return s;
		};
		func._isDone_ = true;
		return self._enqueue(func);
	};
	/**
	 * read key/value in current Promise.context
	 *
	 * @chainable
	 * @method context
	 * @param  {String} key
	 * @param  {*} value
	 * @return {deep.Chain} this
	 */
	Promise.prototype.fromContext = function(key) {
		var self = this;
		var create = function(s, e) {
			if (!key)
				return self._context;
			var out = self._context[key];
			return (typeof out === 'undefined') ? promise.Undefined : out;
		};
		create._isDone_ = true;
		return self._enqueue(create);
	};
	/**
	 * log current Promise.context. If key is provided : log only this property.
	 *
	 * @chainable
	 * @method logContext
	 * @param  {String} key (optional)
	 * @return {deep.Chain} this
	 */
	Promise.prototype.clog = function(key) {
		var self = this;
		var func = function chainLogHandle(s, e) {
			if (key)
				logs.log("deep.chain.context." + key + " : ", self._context[key]);
			else
				logs.log("deep.chain.context : ", self._context);
		};
		return self._enqueue(func);
	};

	return {
		context:function(name, value) {
			if (!name)
				return Promise.context;
			if (!value)
				return Promise.context[name];
			return Promise.context[name] = value;
		},
		contextualise : function(arg) {
			return new Promise().contextualise(arg).resolve(deep.context());
		},
		fromContext : function(key) {
			return new Promise().resolve(Promise.context[key]);
		}
	};
});<|endoftext|>


<commit_before>/* eslint no-param-reassign: off */

const marked = require('marked');
const yamlFront = require('yaml-front-matter');  /* https://www.npmjs.com/package/yaml-front-matter */
const utils = require("../utils");

const blockProxyHandler = {
  set: (target, key, value) => {
    if (key === "url") {
      const urlParts = value.match(/.*(nextstrain.org|localhost).*?\/+([^?\s]+)\??(\S*)/);
      target.dataset = urlParts[2];
      target.query = urlParts[3];
      return true;
    } else if (key === "contents") {
      target.__html = marked(value, {sanitize: false, gfm: true});
      return true;
    } else if (key === "mainDisplayMarkdown") {
      target[key] = value;
      return true;
    }
    return false;
  }
};

const makeFrontMatterBlock = (frontMatter) => {
  if (!frontMatter.title || !frontMatter.dataset) {
    throw new Error("Incorrectly formatted frontmatter in narrative file");
  }
  /* create markdown to represent the title page */
  const markdown = [];
  markdown.push(`## ${frontMatter.title}`);
  if (frontMatter.authors) {
    const authors = parseAttributions(frontMatter, "authors", "authorLinks");
    if (authors) {
      markdown.push(`### Author: ${authors}`);
      if (frontMatter.affiliations && typeof frontMatter.affiliations === "string") {
        markdown[markdown.length-1] += " <sup> 1 </sup>";
        markdown.push(`<sub><sup> 1 </sup> ${frontMatter.affiliations}</sub>`);
      }
    }
  }
  if (frontMatter.translators) {
    const translators = parseAttributions(frontMatter, "translators", "translatorLinks");
    if (translators) {
      markdown.push(`### Translators: ${translators}`);
    }
  }
  if (frontMatter.abstract && typeof frontMatter.abstract === "string") {
    markdown.push(`### ${frontMatter.abstract}`);
  }
  if (frontMatter.date && typeof frontMatter.date === "string") {
    markdown.push(`#### Created: ${frontMatter.date}`);
  }
  if (frontMatter.updated && typeof frontMatter.updated === "string") {
    markdown.push(`#### Updated: ${frontMatter.updated}`);
  }
  if (frontMatter.license) {
    const license = parseAttributions(frontMatter, "license", "licenseLink");
    if (license) {
      markdown.push(`#### License: ${license}`);
    }
  }

  const block = new Proxy({}, blockProxyHandler);
  block.url = frontMatter.dataset;
  block.contents = markdown.join("\n");
  return block;
};

function parseAttributions(frontMatter, attributionsKey, attributionLinksKey) {
  const attributions = frontMatter[attributionsKey];
  const attributionLinks = frontMatter[attributionLinksKey];

  if (Array.isArray(attributions)) {
    return parseAttributionsArray(attributions, attributionLinks, attributionsKey, attributionLinksKey);
  } else if (typeof attributions === 'string') {
    return parseAttributionsString(attributions, attributionLinks, attributionsKey, attributionLinksKey);
  }
  return undefined;
}

function parseAttributionsArray(attributions, attributionLinks, attributionsKey, attributionLinksKey) {
  // validate links
  if (attributionLinks) {
    if (!Array.isArray(attributionLinks)) {
      utils.warn(`Narrative parsing - if ${attributionsKey} is an array, then ${attributionLinksKey} must also be an array. Skipping links.`);
      attributionLinks = undefined;
    } else if (attributionLinks.length !== attributions.length) {
      utils.warn(`Narrative parsing - the length of ${attributionsKey} and ${attributionLinksKey} did not match. Skipping links.`);
      attributionLinks = undefined;
    }
  }
  if (attributionLinks) {
    attributions = attributions.map((attribution, idx) => {
      return attributionLink(attribution, attributionLinks[idx]);
    });
  }
  return attributions.join(", ");
}

function parseAttributionsString(attributions, attributionLinks, attributionsKey, attributionLinksKey) {
  // validate links
  if (attributionLinks) {
    if (typeof attributionLinks !== "string") {
      utils.warn(`Narrative parsing - if ${attributionsKey} is a string, then ${attributionLinksKey} must also be a string. Skipping links.`);
      attributionLinks = undefined;
    }
  }
  return attributionLink(attributions, attributionLinks);
}

function attributionLink(attribution, attributionLinkValue) {
  if (attributionLink) {
    return `[${attribution}](${attributionLinkValue})`;
  }
  return attribution;
}

/**
 * Extract a code-block from the content, if it exists, tagged as auspiceMainDisplayMarkdown
 * Which we send to the client under a special key for parsing in the main display
 * This functionality is experimental, undocumented & subject to change.
 * The implementation relies on regex matching, which is not the best long term solution.
 */
const extractAuspiceMainDisplayMarkdown = (paragraph) => {
  const groups = paragraph.match(/([\s\S]*)```auspiceMainDisplayMarkdown\n([\s\S]+)\n```([\s\S]*)/);
  if (!groups) return [paragraph, false];

  return [
    groups[1]+groups[3], // the content above & below the auspiceMainDisplayMarkdown block
    groups[2]            // the content within the auspiceMainDisplayMarkdown block
  ];
};

/**
 * parses text (from the narrative markdown file)
 * into blocks.
 * Returned object (blocks) has properties "__html", "dataset", "query"
 * and will be sent (in JSON form) to the client
 * @param {string} fileContents string representing the entire markdown file
 * @return {Array} Array of Objects. See `blockProxyHandler` for shape of each object.
 */
const parseNarrativeFile = (fileContents) => {
  const blocks = [];
  const frontMatter = yamlFront.loadFront(fileContents);

  const titlesAndParagraphs = frontMatter.__content
    .replace(/\r\n/g, "\n")                 // handle files with CRLF endings (windows)
    .split(/\n*[#\s]+(\[.+?\]\(.+?\))\n+/)  // matches titles defined as: # [title](url)
    .filter((e) => !e.match(/^\s*$/));      // remove empty paragraphs (often leading / trailing)

  /* process the frontmatter content */
  blocks.push(makeFrontMatterBlock(frontMatter));

  /* process the markdown content */
  const isTitle = (s) => s.match(/^\[.+?\]\(.+?\)$/);
  let idx = 0;
  while (idx < titlesAndParagraphs.length) {
    if (!isTitle(titlesAndParagraphs[idx])) {
      idx++; continue;
    }

    /* want to capture the following groups:
    title text (1st element) and URL, possibly including query (2nd element) */
    const matches = titlesAndParagraphs[idx].match(/\[(.+?)\]\((\S+)\)/);
    let paragraphContents = `# ${matches[1]}\n`; // the title text
    /* add in the next paragraph _if_ it's not itself a title */
    if (!isTitle(titlesAndParagraphs[++idx])) {
      paragraphContents += titlesAndParagraphs[idx++];
    }

    let mainDisplayMarkdown;
    [paragraphContents, mainDisplayMarkdown] = extractAuspiceMainDisplayMarkdown(paragraphContents);

    const block = new Proxy({}, blockProxyHandler);
    block.url = matches[2];
    block.contents = paragraphContents;
    if (mainDisplayMarkdown) block.mainDisplayMarkdown = mainDisplayMarkdown;
    blocks.push(block);
  }

  return blocks;
};

module.exports = {
  default: parseNarrativeFile
};
<commit_msg>rewrite parsing of auspiceMainDisplayMarkdown;<commit_after>/* eslint no-param-reassign: off */

const marked = require('marked');
const yamlFront = require('yaml-front-matter');  /* https://www.npmjs.com/package/yaml-front-matter */
const utils = require("../utils");

const blockProxyHandler = {
  set: (target, key, value) => {
    if (key === "url") {
      const urlParts = value.match(/.*(nextstrain.org|localhost).*?\/+([^?\s]+)\??(\S*)/);
      target.dataset = urlParts[2];
      target.query = urlParts[3];
      return true;
    } else if (key === "contents") {
      target.__html = marked(value, {sanitize: false, gfm: true});
      return true;
    } else if (key === "mainDisplayMarkdown") {
      target[key] = value;
      return true;
    }
    return false;
  }
};

const makeFrontMatterBlock = (frontMatter) => {
  if (!frontMatter.title || !frontMatter.dataset) {
    throw new Error("Incorrectly formatted frontmatter in narrative file");
  }
  /* create markdown to represent the title page */
  const markdown = [];
  markdown.push(`## ${frontMatter.title}`);
  if (frontMatter.authors) {
    const authors = parseAttributions(frontMatter, "authors", "authorLinks");
    if (authors) {
      markdown.push(`### Author: ${authors}`);
      if (frontMatter.affiliations && typeof frontMatter.affiliations === "string") {
        markdown[markdown.length-1] += " <sup> 1 </sup>";
        markdown.push(`<sub><sup> 1 </sup> ${frontMatter.affiliations}</sub>`);
      }
    }
  }
  if (frontMatter.translators) {
    const translators = parseAttributions(frontMatter, "translators", "translatorLinks");
    if (translators) {
      markdown.push(`### Translators: ${translators}`);
    }
  }
  if (frontMatter.abstract && typeof frontMatter.abstract === "string") {
    markdown.push(`### ${frontMatter.abstract}`);
  }
  if (frontMatter.date && typeof frontMatter.date === "string") {
    markdown.push(`#### Created: ${frontMatter.date}`);
  }
  if (frontMatter.updated && typeof frontMatter.updated === "string") {
    markdown.push(`#### Updated: ${frontMatter.updated}`);
  }
  if (frontMatter.license) {
    const license = parseAttributions(frontMatter, "license", "licenseLink");
    if (license) {
      markdown.push(`#### License: ${license}`);
    }
  }

  const block = new Proxy({}, blockProxyHandler);
  block.url = frontMatter.dataset;
  block.contents = markdown.join("\n");
  return block;
};

function parseAttributions(frontMatter, attributionsKey, attributionLinksKey) {
  const attributions = frontMatter[attributionsKey];
  const attributionLinks = frontMatter[attributionLinksKey];

  if (Array.isArray(attributions)) {
    return parseAttributionsArray(attributions, attributionLinks, attributionsKey, attributionLinksKey);
  } else if (typeof attributions === 'string') {
    return parseAttributionsString(attributions, attributionLinks, attributionsKey, attributionLinksKey);
  }
  return undefined;
}

function parseAttributionsArray(attributions, attributionLinks, attributionsKey, attributionLinksKey) {
  // validate links
  if (attributionLinks) {
    if (!Array.isArray(attributionLinks)) {
      utils.warn(`Narrative parsing - if ${attributionsKey} is an array, then ${attributionLinksKey} must also be an array. Skipping links.`);
      attributionLinks = undefined;
    } else if (attributionLinks.length !== attributions.length) {
      utils.warn(`Narrative parsing - the length of ${attributionsKey} and ${attributionLinksKey} did not match. Skipping links.`);
      attributionLinks = undefined;
    }
  }
  if (attributionLinks) {
    attributions = attributions.map((attribution, idx) => {
      return attributionLink(attribution, attributionLinks[idx]);
    });
  }
  return attributions.join(", ");
}

function parseAttributionsString(attributions, attributionLinks, attributionsKey, attributionLinksKey) {
  // validate links
  if (attributionLinks) {
    if (typeof attributionLinks !== "string") {
      utils.warn(`Narrative parsing - if ${attributionsKey} is a string, then ${attributionLinksKey} must also be a string. Skipping links.`);
      attributionLinks = undefined;
    }
  }
  return attributionLink(attributions, attributionLinks);
}

function attributionLink(attribution, attributionLinkValue) {
  if (attributionLink) {
    return `[${attribution}](${attributionLinkValue})`;
  }
  return attribution;
}

/**
 * Extract a code-block from the content, if it exists, tagged as auspiceMainDisplayMarkdown
 * Which we send to the client under a special key for parsing in the main display
 * This functionality is experimental, undocumented & subject to change.
 * The implementation relies exactly one opening string for auspiceMainDisplayMarkdown and exactly one
 * closing string. Otherwise it will return the paragraph as is.
 */
const extractAuspiceMainDisplayMarkdown = (paragraph) => {
  const split1 = paragraph.split("```auspiceMainDisplayMarkdown\n");
  if (split1.length !== 2) {
    return [paragraph, false];
  }
  const [before, start] = split1;
  const split2 = start.split("\n```")
  if (split2.length !== 2) {
    return [paragraph, false];
  }
  const [auspiceMainDisplayMarkdown, after] = split2;

  return [
    before+after,              // the content above & below the auspiceMainDisplayMarkdown block
    auspiceMainDisplayMarkdown // the content within the auspiceMainDisplayMarkdown block
  ];
};

/**
 * parses text (from the narrative markdown file)
 * into blocks.
 * Returned object (blocks) has properties "__html", "dataset", "query"
 * and will be sent (in JSON form) to the client
 * @param {string} fileContents string representing the entire markdown file
 * @return {Array} Array of Objects. See `blockProxyHandler` for shape of each object.
 */
const parseNarrativeFile = (fileContents) => {
  const blocks = [];
   const frontMatter = yamlFront.loadFront(fileContents);

  const titlesAndParagraphs = frontMatter.__content
    .replace(/\r\n/g, "\n")                 // handle files with CRLF endings (windows)
    .split(/\n*[#\s]+(\[.+?\]\(.+?\))\n+/)  // matches titles defined as: # [title](url)
    .filter((e) => !e.match(/^\s*$/));      // remove empty paragraphs (often leading / trailing)

  /* process the frontmatter content */
  blocks.push(makeFrontMatterBlock(frontMatter));
  /* process the markdown content */
  const isTitle = (s) => s.match(/^\[.+?\]\(.+?\)$/);
  let idx = 0;
  while (idx < titlesAndParagraphs.length) {
    if (!isTitle(titlesAndParagraphs[idx])) {
      idx++; continue;
    }

    /* want to capture the following groups:
    title text (1st element) and URL, possibly including query (2nd element) */
    const matches = titlesAndParagraphs[idx].match(/\[(.+?)\]\((\S+)\)/);
    let paragraphContents = `# ${matches[1]}\n`; // the title text
    /* add in the next paragraph _if_ it's not itself a title */
    if (!isTitle(titlesAndParagraphs[++idx])) {
      paragraphContents += titlesAndParagraphs[idx++];
    }

    let mainDisplayMarkdown;
    [paragraphContents, mainDisplayMarkdown] = extractAuspiceMainDisplayMarkdown(paragraphContents);
    const block = new Proxy({}, blockProxyHandler);
    block.url = matches[2];
    block.contents = paragraphContents;
    if (mainDisplayMarkdown) block.mainDisplayMarkdown = mainDisplayMarkdown;
    blocks.push(block);
  }

  return blocks;
};

module.exports = {
  default: parseNarrativeFile
};
<|endoftext|>


<commit_before>//Context class for Noodles
//otherwise just make sure that the browser version loads before this file.
if(typeof process !== "undefined"){
	//if node
	var requirejs = require('require-node');
	var define = requirejs.define;
	requirejs.config({
	    nodeRequire: require,
		baseUrl: __dirname,
	});
}
else{
	requirejs = require;
}
define('_noodles_context',function(require, exports, module){
/**--Noodles--
name:Context
description: Template execution
@param {Noodles.Template}
@param {object}
*/
var Context = exports.Context = function(Template,obj){
	this._others = obj;
	this._meta = Template._metaData;
};
/**--Noodles.Context--
name:Context
description: gets an object from the current Context
@param {string | array}
*/
Context.prototype.getObject = function(key){
	key = typeof key !== "object" ? [key] : key;

	var keyTemp = (typeof key[0].execute !== "undefined" ? key[0].execute(this) : key[0]).toLowerCase(),
		obj,keys,keyIndex,tmp;

	if(typeof this._meta[keyTemp] !== "undefined"){
		return this._meta[keyTemp].execute(this);
	}
	obj = this._others;

	for(var i = 0, l = key.length; i < l; i++){
		keyTemp = typeof key[0].execute !== "undefined" ? key[0].execute(this).toLowerCase() : key[0];
		if(typeof obj[keyTemp] !== "undefined"){
			obj = obj[keyTemp];
		}
		else{
			keys = ',' + Object.keys(obj).join()+ ',';
			keyIndex = keys.toLowerCase().indexOf(',' + keyTemp +',');
			if(keyIndex > -1){
				keyTemp = keys.slice(keyindex + 1, keyindex + 1 + keyTemp.length);
				obj[keyTemp.toLowerCase()] = obj[keyTemp];
				obj = obj[keyTemp];
			}
			else{
				obj[keyTemp] = '';
		 		return '';
			}

		}
	}
	return typeof obj === "object" ? ' ' : obj;
};
/**--Noodles.Context--
name:Context
description: sets an object for the current Context
@param {array}
@param {*}
*/
Context.prototype.setObject = function(key,value){
	key = typeof key !== "object" ? [key] : key;

	var obj = this._others,
		keyTemp,obj,keys,keyIndex;

	for(var i = 0, l = key.length; i < l; i++){
		keyTemp = typeof key[0].execute !== "undefined" ? key[0].execute(this).toLowerCase() : key[0];
		if(typeof obj[keyTemp] !== "undefined"){
			obj = obj[keyTemp];
		}
		else{
			keys = ',' + Object.keys(obj).join()+ ',';
			keyIndex = keys.toLowerCase().indexOf(',' + tmp +',');
			if(keyIndex > -1){
				obj = obj[keys.slice(keyindex + 1, keyindex + 1 + tmp.length)];
			} else{
		 		return '';
			}

		}
	}
	obj = typeof value.execute !== "undefined" ? value.execute(this) : value;
};
});<commit_msg>trying to debug metadata nightmar.<commit_after>//Context class for Noodles
//otherwise just make sure that the browser version loads before this file.
if(typeof process !== "undefined"){
	//if node
	var requirejs = require('require-node');
	var define = requirejs.define;
	requirejs.config({
	    nodeRequire: require,
		baseUrl: __dirname,
	});
}
else{
	requirejs = require;
}
define(function(require, exports, module){
var _Template = require('./template').Template;
/**--Noodles--
name:Context
description: Template execution
@param {Noodles.Template}
@param {object}
*/
var Context = exports.Context = function(Template,obj){
	if(typeof _Template === "undefined") _Template = require('./template').Template;
	this._others = obj;
	this._templateId = Template._id;
};
/**--Noodles.Context--
name:Context
description: gets an object from the current Context
@param {Noodles.Template}
@param {string | array}
*/
Context.prototype.getObject = function(Template,key,callback){
	key = typeof key !== "object" ? [key] : key;

	var keyTemp = (typeof key[0].execute !== "undefined" ? key[0].execute(this) : key[0]).toLowerCase(),
		obj,keys,keyIndex;
	if(typeof _Template.templateCache[this._templateId].meta.hasOwnProperty(keyTemp) ){
		console.log(temp);
		return _Template.templateCache[this._templateId].meta[keyTemp].execute(Template,this,callback);
	}
	obj = this._others;

	for(var i = 0, l = key.length; i < l; i++){
		keyTemp = typeof key[0].execute !== "undefined" ? key[0].execute(this).toLowerCase() : key[0];
		if(typeof obj[keyTemp] !== "undefined"){
			obj = obj[keyTemp];
		}
		else{
			keys = ',' + Object.keys(obj).join()+ ',';
			keyIndex = keys.toLowerCase().indexOf(',' + keyTemp +',');
			if(keyIndex > -1){
				keyTemp = keys.slice(keyindex + 1, keyindex + 1 + keyTemp.length);
				obj[keyTemp.toLowerCase()] = obj[keyTemp];
				obj = obj[keyTemp];
			}
			else{
				obj[keyTemp] = '';
		 		return '';
			}

		}
	}
	return typeof obj === "object" ? ' ' : obj;
};
/**--Noodles.Context--
name:Context
description: sets an object for the current Context
@param {array}
@param {*}
*/
Context.prototype.setObject = function(key,value){
	key = typeof key !== "object" ? [key] : key;

	var obj = this._others,
		keyTemp,obj,keys,keyIndex;

	for(var i = 0, l = key.length; i < l; i++){
		keyTemp = typeof key[0].execute !== "undefined" ? key[0].execute(this).toLowerCase() : key[0];
		if(typeof obj[keyTemp] !== "undefined"){
			obj = obj[keyTemp];
		}
		else{
			keys = ',' + Object.keys(obj).join()+ ',';
			keyIndex = keys.toLowerCase().indexOf(',' + tmp +',');
			if(keyIndex > -1){
				obj = obj[keys.slice(keyindex + 1, keyindex + 1 + tmp.length)];
			} else{
		 		return '';
			}

		}
	}
	obj = typeof value.execute !== "undefined" ? value.execute(this) : value;
};
});<|endoftext|>