<commit_before>(function() {
'use strict';

if (!window.location.origin) {
  window.location.origin =
      window.location.protocol + "//" +
      window.location.hostname +
      (window.location.port ? (':' + window.location.port) : '');
}

Elm.Debugger = null;
Elm.debuggerAttach = function(module, hotSwapState /* =undefined */) {
  return {
    make: function(runtime) {
      var wrappedModule = debugModule(module, runtime);
      Elm.Debugger = debuggerInit(wrappedModule, runtime, hotSwapState);
      dispatchElmDebuggerInit();
      return wrappedModule.debuggedModule;
    }
  };
};

var EVENTS_PER_SAVE = 100;

function dispatchElmDebuggerInit() {
  if (parent.window) {
    var dispatch = function() {
      parent.window.postMessage("elmDebuggerInit", window.location.origin);
    };
    if (parent.window.document.readyState === "complete") {
      dispatch();
    }
    else {
      parent.window.addEventListener("load", dispatch);
    }
  }
}

function debugModule(module, runtime) {
  var programPaused = false;
  var recordedEvents = [];
  var asyncCallbacks = [];
  var snapshots = [];
  var watchTracker = Elm.Native.Debug.make(runtime).watchTracker;
  var pauseTime = 0;
  var eventsUntilSnapshot = EVENTS_PER_SAVE;
  runtime.DebugStatus = runtime.DebugStatus || {};
  runtime.DebugStatus.eventCounter = runtime.DebugStatus.eventCounter || 0;

  // runtime is the prototype of wrappedRuntime
  // so we can access all runtime properties too
  var wrappedRuntime = Object.create(runtime);
  wrappedRuntime.notify = wrapNotify;
  wrappedRuntime.runDelayed = wrapRunDelayed;

  // make a copy of the wrappedRuntime
  var assignedPropTracker = Object.create(wrappedRuntime);
  var debuggedModule = module.make(assignedPropTracker);

  // make sure the signal graph is actually a signal & extract the visual model
  if ( !('recv' in debuggedModule.main) ) {
    debuggedModule.main = Elm.Signal.make(runtime).constant(debuggedModule.main);
  }

  // The main module stores imported modules onto the runtime.
  // To ensure only one instance of each module is created,
  // we assign them back on the original runtime object.
  Object.keys(assignedPropTracker).forEach(function(key) {
    runtime[key] = assignedPropTracker[key];
  });

  var signalGraphNodes = flattenSignalGraph(wrappedRuntime.inputs);
  var tracePath = tracePathInit(runtime, debuggedModule.main);

  snapshots.push(snapshotSignalGraph(signalGraphNodes));

  function wrapNotify(id, v) {
    var timestep = runtime.timer.now();

    if (programPaused) {
      // ignore async events generated while playing back
      // or user events while program is paused
      return false;
    }
    else {
      recordEvent(id, v, timestep);
      var changed = runtime.notify(id, v, timestep);
      snapshotOnCheckpoint();
      if (parent.window) {
        parent.window.postMessage("elmNotify", window.location.origin);
      }
      return changed;
    }
  };

  function wrapRunDelayed(func, delayMs) {
    if (programPaused) {
      // Don't push timers and such to the callback stack while we're paused.
      // It causes too many callbacks to be fired during unpausing.
      return 0;
    }
    var cbObj = { func:func, delayMs:delayMs, timerId:0, executed:false };
    var timerId = setTimeout(function() {
        cbObj.executed = true;
        func();
      }, delayMs);
    cbObj.timerId = timerId;
    asyncCallbacks.push(cbObj);
    return timerId;
  }

  function recordEvent(id, v, timestep) {
    watchTracker.pushFrame();
    recordedEvents.push({ id:id, value:v, timestep:timestep });
    runtime.DebugStatus.eventCounter += 1;
  }

  function clearAsyncCallbacks() {
    asyncCallbacks.forEach(function(timer) {
      if (!timer.executed) {
        clearTimeout(timer.timerId);
      }
    });
  }

  function clearRecordedEvents() {
    recordedEvents = [];
    runtime.DebugStatus.eventCounter = 0;
  }

  function getRecordedEventsLength() {
    return recordedEvents.length;
  }

  function getRecordedEventAt(i) {
    return recordedEvents[i];
  }

  function copyRecordedEvents() {
    return recordedEvents.slice();
  }

  function loadRecordedEvents(events) {
    recordedEvents = events.slice();
  }

  function clearSnapshots() {
    snapshots = [snapshotSignalGraph(signalGraphNodes)];
  }

  function getSnapshotAt(i) {
    var snapshotEvent = Math.floor(i / EVENTS_PER_SAVE);
    assert(snapshotEvent < snapshots.length && snapshotEvent >= 0,
           "Out of bounds index: " + snapshotEvent);
    return snapshots[snapshotEvent];
  }

  function snapshotOnCheckpoint() {
    if (eventsUntilSnapshot === 1) {
      snapshots.push(snapshotSignalGraph(signalGraphNodes));
      eventsUntilSnapshot = EVENTS_PER_SAVE;
    } else {
      eventsUntilSnapshot -= 1;
    }
  }

  function setPaused() {
    programPaused = true;
    clearAsyncCallbacks();
    tracePath.stopRecording();
    pauseTime = Date.now();
  }

  function setContinue(position) {
    var timerDelay = Date.now() - pauseTime;
    runtime.timer.addDelay(timerDelay);
    programPaused = false;
    if (position > 0) {
      // If we're not unpausing at the head, then we need to dump the
      // events that are ahead of where we're continuing.
      var lastSnapshotPosition = Math.floor(position / EVENTS_PER_SAVE);
      eventsUntilSnapshot = EVENTS_PER_SAVE - (position % EVENTS_PER_SAVE);
      snapshots = snapshots.slice(0, lastSnapshotPosition + 1);
      recordedEvents = recordedEvents.slice(0, position);
      tracePath.clearTracesAfter(position);
      runtime.DebugStatus.eventCounter = position;
      executeCallbacks(asyncCallbacks, false);
    }
    tracePath.startRecording();
  }

  function getPaused() {
    return programPaused;
  }

  return {
    debuggedModule: debuggedModule,
    signalGraphNodes: signalGraphNodes,
    initialSnapshot: snapshotSignalGraph(signalGraphNodes),
    initialAsyncCallbacks: asyncCallbacks.slice(),
    // API functions
    clearAsyncCallbacks: clearAsyncCallbacks,
    clearRecordedEvents: clearRecordedEvents,
    getRecordedEventsLength: getRecordedEventsLength,
    getRecordedEventAt: getRecordedEventAt,
    copyRecordedEvents: copyRecordedEvents,
    loadRecordedEvents: loadRecordedEvents,
    clearSnapshots: clearSnapshots,
    getSnapshotAt: getSnapshotAt,
    snapshotOnCheckpoint: snapshotOnCheckpoint,
    getPaused: getPaused,
    setPaused: setPaused,
    setContinue: setContinue,
    tracePath: tracePath,
    watchTracker: watchTracker
  };
}

function debuggerInit(debugModule, runtime, hotSwapState /* =undefined */) {
  var currentEventIndex = 0;

  function resetProgram(position) {
    var closestSnapshot = debugModule.getSnapshotAt(position);
    debugModule.clearAsyncCallbacks();
    restoreSnapshot(debugModule.signalGraphNodes, closestSnapshot);
    redrawGraphics();
  }

  function restartProgram() {
    pauseProgram();
    resetProgram(0);
    debugModule.watchTracker.clear();
    debugModule.tracePath.clearTraces();
    debugModule.clearRecordedEvents();
    debugModule.clearSnapshots();
    debugModule.setContinue(0);
    executeCallbacks(debugModule.initialAsyncCallbacks, true);
  }

  function pauseProgram() {
    debugModule.setPaused();
    currentEventIndex = debugModule.getRecordedEventsLength();
  }

  function continueProgram() {
    if (debugModule.getPaused())
    {
      if (currentEventIndex === 0) {
        restartProgram();
        return;
      }
      var closestSnapshotIndex =
          Math.floor(currentEventIndex / EVENTS_PER_SAVE) * EVENTS_PER_SAVE;
      resetProgram(currentEventIndex);
      var continueIndex = currentEventIndex;
      currentEventIndex = closestSnapshotIndex;
      stepTo(continueIndex);
      debugModule.setContinue(currentEventIndex);
    }
  }

  function stepTo(index) {
    if (!debugModule.getPaused()) {
      debugModule.setPaused();
      resetProgram();
    }

    if (index < 0 || index > getMaxSteps()) {
      throw "Index out of bounds: " + index;
    }

    if (index < currentEventIndex) {
      var closestSnapshotIndex = Math.floor(index / EVENTS_PER_SAVE) * EVENTS_PER_SAVE;
      resetProgram(index);
      currentEventIndex = closestSnapshotIndex;
    }

    while (currentEventIndex < index) {
      var nextEvent = debugModule.getRecordedEventAt(currentEventIndex);
      runtime.notify(nextEvent.id, nextEvent.value, nextEvent.timestep);

      currentEventIndex += 1;
    }
  }

  function getMaxSteps() {
    return debugModule.getRecordedEventsLength();
  }

  function redrawGraphics() {
    var main = debugModule.debuggedModule.main
    for (var i = main.kids.length ; i-- ; ) {
      main.kids[i].recv(runtime.timer.now(), true, main.id);
    }
  }

  function getHotSwapState() {
    var continueIndex = currentEventIndex;
    if (!debugModule.getPaused()) {
      continueIndex = getMaxSteps();
    }
    return {
      recordedEvents: debugModule.copyRecordedEvents(),
      currentEventIndex: continueIndex
    };
  }

  function dispose() {
    var parentNode = runtime.node.parentNode;
    parentNode.removeChild(debugModule.tracePath.canvas);
    parentNode.removeChild(runtime.node);
  }

  if (hotSwapState) {
    // The problem is that we want to previous paused state. But
    // by the time JS reaches here, the old code has been swapped out
    // and the new modules are being generated. So we can ask the
    // debugging console what it thinks the pause state is and go
    // from there.
    var paused = top.debug.paused;
    debugModule.setPaused();
    debugModule.loadRecordedEvents(hotSwapState.recordedEvents);
    var index = getMaxSteps();
    runtime.DebugStatus.eventCounter = 0;
    debugModule.tracePath.clearTraces();

    // draw new trace path
    debugModule.tracePath.startRecording();
    while(currentEventIndex < index) {
      var nextEvent = debugModule.getRecordedEventAt(currentEventIndex);
      runtime.DebugStatus.eventCounter += 1;
      runtime.notify(nextEvent.id, nextEvent.value, nextEvent.timestep);
      debugModule.snapshotOnCheckpoint();
      currentEventIndex += 1;
    }
    debugModule.tracePath.stopRecording();

    stepTo(hotSwapState.currentEventIndex);
    if (!paused) {
      debugModule.setContinue(hotSwapState.currentEventIndex);
    }
  }

  runtime.node.parentNode.appendChild(debugModule.tracePath.canvas);

  var elmDebugger = {
      restart: restartProgram,
      pause: pauseProgram,
      kontinue: continueProgram,
      getMaxSteps: getMaxSteps,
      stepTo: stepTo,
      getPaused: debugModule.getPaused,
      getHotSwapState: getHotSwapState,
      dispose: dispose,
      allNodes: debugModule.signalGraphNodes,
      watchTracker: debugModule.watchTracker
  };

  return elmDebugger;
}

function Point(x, y) {
  this.x = x;
  this.y = y;

  this.translate = function(x, y) {
    return new Point(this.x + x, this.y + y);
  }

  this.equals = function(p) {
    return this.x == p.x && this.y == p.y;
  }
}

function tracePathInit(runtime, signalGraphMain) {
  var List = Elm.List.make(runtime);
  var Signal = Elm.Signal.make(runtime);
  var tracePathNode = A2(Signal.lift, graphicsUpdate, signalGraphMain);
  var tracePathCanvas = createCanvas();
  var tracePositions = {};
  var recordingTraces = true;

  function findPositions(currentScene) {
    var positions = {};
    function processElement(elem, offset) {
      if (elem.element.ctor == "Custom" && elem.element.type == "Collage")
      {
        List.map(F2(processForm)(offset))(elem.element.model.forms);
      }
      if (elem.props.debugTracePathId)
      {
        positions[elem.props.debugTracePathId] = new Point(offset.x, offset.y);
      }
    }

    function processForm(offset, form) {
      if (form.form.ctor == "FElement")
      {
        processElement(form.form._0, offset.translate(form.x, -form.y));
      }
    }

    processElement(currentScene, new Point(0, 0));
    return positions;
  }

  function appendPositions(positions) {
    for (var id in positions) {
      var pos = positions[id];
      if (tracePositions.hasOwnProperty(id)) {
        tracePositions[id].push(pos);
      }
      else {
        tracePositions[id] = [pos];
      }
      if (tracePositions[id].length < runtime.DebugStatus.eventCounter) {
        var padCount = runtime.DebugStatus.eventCounter - tracePositions[id].length;
        var lastTracePosition = tracePositions[id][tracePositions[id].length - 1];
        for (var i = padCount; i--;) {
          tracePositions[id].push(lastTracePosition)
        }
      }
      assert(tracePositions[id].length === runtime.DebugStatus.eventCounter,
             "We don't have a 1-1 mapping of trace positions to events");
    }
  }

  function graphicsUpdate(currentScene) {
    if (!recordingTraces) {
      return;
    }

    var ctx = tracePathCanvas.getContext('2d');
    ctx.clearRect(0, 0, tracePathCanvas.width, tracePathCanvas.height);

    ctx.save();
    ctx.translate(ctx.canvas.width/2, ctx.canvas.height/2);
    appendPositions(findPositions(currentScene));
    for (var id in tracePositions)
    {
      ctx.beginPath();
      var points = tracePositions[id];
      for (var i=0; i < points.length; i++)
      {
        var p = points[i];
        if (i == 0) {
          ctx.moveTo(p.x, p.y);
        }
        else {
          ctx.lineTo(p.x, p.y);
        }
      }
      ctx.lineWidth = 1;
      ctx.strokeStyle = "rgba(50, 50, 50, 0.4)";
      ctx.stroke();
    }

    ctx.restore();
  }

  function clearTraces() {
    tracePositions = {};
  }

  function stopRecording() {
    recordingTraces = false;
  }

  function startRecording() {
    recordingTraces = true;
  }

  function clearTracesAfter(position) {
    var newTraces = {};
    for (var id in tracePositions) {
      newTraces[id] = tracePositions[id].slice(0,position);
    }
    tracePositions = newTraces;
  }

  return {
    graphicsUpdate: graphicsUpdate,
    canvas: tracePathCanvas,
    clearTraces: clearTraces,
    clearTracesAfter: clearTracesAfter,
    stopRecording: stopRecording,
    startRecording: startRecording
  }
}

function executeCallbacks(callbacks, reexecute) {
  callbacks.forEach(function(timer) {
    if (reexecute || !timer.executed) {
      var func = timer.func;
      timer.executed = true;
      func();
    }
  });
}

function createCanvas() {
  var c = document.createElement('canvas');
  c.width = window.innerWidth;
  c.height = window.innerHeight;
  c.style.position = "absolute";
  c.style.top = "0";
  c.style.left = "0";
  c.style.pointerEvents = "none";
  return c;
}

function assert(bool, msg) {
  if (!bool) {
    throw "Assertion error: " + msg;
  }
}

function snapshotSignalGraph(signalGraphNodes) {
  var nodeValues = [];

  signalGraphNodes.forEach(function(node) {
    nodeValues.push({ value: node.value, id: node.id });
  });

  return nodeValues;
};

function restoreSnapshot(signalGraphNodes, snapshot) {
  assert(signalGraphNodes.length == snapshot.length,
         "saved program state has wrong length");
  for (var i=0; i < signalGraphNodes.length; i++) {
    var node = signalGraphNodes[i];
    var state = snapshot[i];
    assert(node.id == state.id, "the nodes moved position");

    node.value = state.value;
  }
}

function flattenSignalGraph(nodes) {
  var nodesById = {};

  function addAllToDict(node) {
    nodesById[node.id] = node;
    node.kids.forEach(addAllToDict);
  }
  nodes.forEach(addAllToDict);

  var allNodes = Object.keys(nodesById).sort().map(function(key) {
    return nodesById[key];
  });
  return allNodes;
};

}());
<commit_msg>elm.debuggerStatus.eventCounter<commit_after>(function() {
'use strict';

if (!window.location.origin) {
  window.location.origin =
      window.location.protocol + "//" +
      window.location.hostname +
      (window.location.port ? (':' + window.location.port) : '');
}

Elm.Debugger = null;
Elm.debuggerAttach = function(module, hotSwapState /* =undefined */) {
  return {
    make: function(runtime) {
      var wrappedModule = debugModule(module, runtime);
      Elm.Debugger = debuggerInit(wrappedModule, runtime, hotSwapState);
      dispatchElmDebuggerInit();
      return wrappedModule.debuggedModule;
    }
  };
};

var EVENTS_PER_SAVE = 100;

function dispatchElmDebuggerInit() {
  if (parent.window) {
    var dispatch = function() {
      parent.window.postMessage("elmDebuggerInit", window.location.origin);
    };
    if (parent.window.document.readyState === "complete") {
      dispatch();
    }
    else {
      parent.window.addEventListener("load", dispatch);
    }
  }
}

function debugModule(module, runtime) {
  var programPaused = false;
  var recordedEvents = [];
  var asyncCallbacks = [];
  var snapshots = [];
  var watchTracker = Elm.Native.Debug.make(runtime).watchTracker;
  var pauseTime = 0;
  var eventsUntilSnapshot = EVENTS_PER_SAVE;
  runtime.debuggerStatus = runtime.debuggerStatus || {};
  runtime.debuggerStatus.eventCounter = runtime.debuggerStatus.eventCounter || 0;

  // runtime is the prototype of wrappedRuntime
  // so we can access all runtime properties too
  var wrappedRuntime = Object.create(runtime);
  wrappedRuntime.notify = wrapNotify;
  wrappedRuntime.runDelayed = wrapRunDelayed;

  // make a copy of the wrappedRuntime
  var assignedPropTracker = Object.create(wrappedRuntime);
  var debuggedModule = module.make(assignedPropTracker);

  // make sure the signal graph is actually a signal & extract the visual model
  if ( !('recv' in debuggedModule.main) ) {
    debuggedModule.main = Elm.Signal.make(runtime).constant(debuggedModule.main);
  }

  // The main module stores imported modules onto the runtime.
  // To ensure only one instance of each module is created,
  // we assign them back on the original runtime object.
  Object.keys(assignedPropTracker).forEach(function(key) {
    runtime[key] = assignedPropTracker[key];
  });

  var signalGraphNodes = flattenSignalGraph(wrappedRuntime.inputs);
  var tracePath = tracePathInit(runtime, debuggedModule.main);

  snapshots.push(snapshotSignalGraph(signalGraphNodes));

  function wrapNotify(id, v) {
    var timestep = runtime.timer.now();

    if (programPaused) {
      // ignore async events generated while playing back
      // or user events while program is paused
      return false;
    }
    else {
      recordEvent(id, v, timestep);
      var changed = runtime.notify(id, v, timestep);
      snapshotOnCheckpoint();
      if (parent.window) {
        parent.window.postMessage("elmNotify", window.location.origin);
      }
      return changed;
    }
  };

  function wrapRunDelayed(func, delayMs) {
    if (programPaused) {
      // Don't push timers and such to the callback stack while we're paused.
      // It causes too many callbacks to be fired during unpausing.
      return 0;
    }
    var cbObj = { func:func, delayMs:delayMs, timerId:0, executed:false };
    var timerId = setTimeout(function() {
        cbObj.executed = true;
        func();
      }, delayMs);
    cbObj.timerId = timerId;
    asyncCallbacks.push(cbObj);
    return timerId;
  }

  function recordEvent(id, v, timestep) {
    watchTracker.pushFrame();
    recordedEvents.push({ id:id, value:v, timestep:timestep });
    runtime.debuggerStatus.eventCounter += 1;
  }

  function clearAsyncCallbacks() {
    asyncCallbacks.forEach(function(timer) {
      if (!timer.executed) {
        clearTimeout(timer.timerId);
      }
    });
  }

  function clearRecordedEvents() {
    recordedEvents = [];
    runtime.debuggerStatus.eventCounter = 0;
  }

  function getRecordedEventsLength() {
    return recordedEvents.length;
  }

  function getRecordedEventAt(i) {
    return recordedEvents[i];
  }

  function copyRecordedEvents() {
    return recordedEvents.slice();
  }

  function loadRecordedEvents(events) {
    recordedEvents = events.slice();
  }

  function clearSnapshots() {
    snapshots = [snapshotSignalGraph(signalGraphNodes)];
  }

  function getSnapshotAt(i) {
    var snapshotEvent = Math.floor(i / EVENTS_PER_SAVE);
    assert(snapshotEvent < snapshots.length && snapshotEvent >= 0,
           "Out of bounds index: " + snapshotEvent);
    return snapshots[snapshotEvent];
  }

  function snapshotOnCheckpoint() {
    if (eventsUntilSnapshot === 1) {
      snapshots.push(snapshotSignalGraph(signalGraphNodes));
      eventsUntilSnapshot = EVENTS_PER_SAVE;
    } else {
      eventsUntilSnapshot -= 1;
    }
  }

  function setPaused() {
    programPaused = true;
    clearAsyncCallbacks();
    tracePath.stopRecording();
    pauseTime = Date.now();
  }

  function setContinue(position) {
    var timerDelay = Date.now() - pauseTime;
    runtime.timer.addDelay(timerDelay);
    programPaused = false;
    if (position > 0) {
      // If we're not unpausing at the head, then we need to dump the
      // events that are ahead of where we're continuing.
      var lastSnapshotPosition = Math.floor(position / EVENTS_PER_SAVE);
      eventsUntilSnapshot = EVENTS_PER_SAVE - (position % EVENTS_PER_SAVE);
      snapshots = snapshots.slice(0, lastSnapshotPosition + 1);
      recordedEvents = recordedEvents.slice(0, position);
      tracePath.clearTracesAfter(position);
      runtime.debuggerStatus.eventCounter = position;
      executeCallbacks(asyncCallbacks, false);
    }
    tracePath.startRecording();
  }

  function getPaused() {
    return programPaused;
  }

  return {
    debuggedModule: debuggedModule,
    signalGraphNodes: signalGraphNodes,
    initialSnapshot: snapshotSignalGraph(signalGraphNodes),
    initialAsyncCallbacks: asyncCallbacks.slice(),
    // API functions
    clearAsyncCallbacks: clearAsyncCallbacks,
    clearRecordedEvents: clearRecordedEvents,
    getRecordedEventsLength: getRecordedEventsLength,
    getRecordedEventAt: getRecordedEventAt,
    copyRecordedEvents: copyRecordedEvents,
    loadRecordedEvents: loadRecordedEvents,
    clearSnapshots: clearSnapshots,
    getSnapshotAt: getSnapshotAt,
    snapshotOnCheckpoint: snapshotOnCheckpoint,
    getPaused: getPaused,
    setPaused: setPaused,
    setContinue: setContinue,
    tracePath: tracePath,
    watchTracker: watchTracker
  };
}

function debuggerInit(debugModule, runtime, hotSwapState /* =undefined */) {
  var currentEventIndex = 0;

  function resetProgram(position) {
    var closestSnapshot = debugModule.getSnapshotAt(position);
    debugModule.clearAsyncCallbacks();
    restoreSnapshot(debugModule.signalGraphNodes, closestSnapshot);
    redrawGraphics();
  }

  function restartProgram() {
    pauseProgram();
    resetProgram(0);
    debugModule.watchTracker.clear();
    debugModule.tracePath.clearTraces();
    debugModule.clearRecordedEvents();
    debugModule.clearSnapshots();
    debugModule.setContinue(0);
    executeCallbacks(debugModule.initialAsyncCallbacks, true);
  }

  function pauseProgram() {
    debugModule.setPaused();
    currentEventIndex = debugModule.getRecordedEventsLength();
  }

  function continueProgram() {
    if (debugModule.getPaused())
    {
      if (currentEventIndex === 0) {
        restartProgram();
        return;
      }
      var closestSnapshotIndex =
          Math.floor(currentEventIndex / EVENTS_PER_SAVE) * EVENTS_PER_SAVE;
      resetProgram(currentEventIndex);
      var continueIndex = currentEventIndex;
      currentEventIndex = closestSnapshotIndex;
      stepTo(continueIndex);
      debugModule.setContinue(currentEventIndex);
    }
  }

  function stepTo(index) {
    if (!debugModule.getPaused()) {
      debugModule.setPaused();
      resetProgram();
    }

    if (index < 0 || index > getMaxSteps()) {
      throw "Index out of bounds: " + index;
    }

    if (index < currentEventIndex) {
      var closestSnapshotIndex = Math.floor(index / EVENTS_PER_SAVE) * EVENTS_PER_SAVE;
      resetProgram(index);
      currentEventIndex = closestSnapshotIndex;
    }

    while (currentEventIndex < index) {
      var nextEvent = debugModule.getRecordedEventAt(currentEventIndex);
      runtime.notify(nextEvent.id, nextEvent.value, nextEvent.timestep);

      currentEventIndex += 1;
    }
  }

  function getMaxSteps() {
    return debugModule.getRecordedEventsLength();
  }

  function redrawGraphics() {
    var main = debugModule.debuggedModule.main
    for (var i = main.kids.length ; i-- ; ) {
      main.kids[i].recv(runtime.timer.now(), true, main.id);
    }
  }

  function getHotSwapState() {
    var continueIndex = currentEventIndex;
    if (!debugModule.getPaused()) {
      continueIndex = getMaxSteps();
    }
    return {
      recordedEvents: debugModule.copyRecordedEvents(),
      currentEventIndex: continueIndex
    };
  }

  function dispose() {
    var parentNode = runtime.node.parentNode;
    parentNode.removeChild(debugModule.tracePath.canvas);
    parentNode.removeChild(runtime.node);
  }

  if (hotSwapState) {
    // The problem is that we want to previous paused state. But
    // by the time JS reaches here, the old code has been swapped out
    // and the new modules are being generated. So we can ask the
    // debugging console what it thinks the pause state is and go
    // from there.
    var paused = top.debug.paused;
    debugModule.setPaused();
    debugModule.loadRecordedEvents(hotSwapState.recordedEvents);
    var index = getMaxSteps();
    runtime.debuggerStatus.eventCounter = 0;
    debugModule.tracePath.clearTraces();

    // draw new trace path
    debugModule.tracePath.startRecording();
    while(currentEventIndex < index) {
      var nextEvent = debugModule.getRecordedEventAt(currentEventIndex);
      runtime.debuggerStatus.eventCounter += 1;
      runtime.notify(nextEvent.id, nextEvent.value, nextEvent.timestep);
      debugModule.snapshotOnCheckpoint();
      currentEventIndex += 1;
    }
    debugModule.tracePath.stopRecording();

    stepTo(hotSwapState.currentEventIndex);
    if (!paused) {
      debugModule.setContinue(hotSwapState.currentEventIndex);
    }
  }

  runtime.node.parentNode.appendChild(debugModule.tracePath.canvas);

  var elmDebugger = {
      restart: restartProgram,
      pause: pauseProgram,
      kontinue: continueProgram,
      getMaxSteps: getMaxSteps,
      stepTo: stepTo,
      getPaused: debugModule.getPaused,
      getHotSwapState: getHotSwapState,
      dispose: dispose,
      allNodes: debugModule.signalGraphNodes,
      watchTracker: debugModule.watchTracker
  };

  return elmDebugger;
}

function Point(x, y) {
  this.x = x;
  this.y = y;

  this.translate = function(x, y) {
    return new Point(this.x + x, this.y + y);
  }

  this.equals = function(p) {
    return this.x == p.x && this.y == p.y;
  }
}

function tracePathInit(runtime, signalGraphMain) {
  var List = Elm.List.make(runtime);
  var Signal = Elm.Signal.make(runtime);
  var tracePathNode = A2(Signal.lift, graphicsUpdate, signalGraphMain);
  var tracePathCanvas = createCanvas();
  var tracePositions = {};
  var recordingTraces = true;

  function findPositions(currentScene) {
    var positions = {};
    function processElement(elem, offset) {
      if (elem.element.ctor == "Custom" && elem.element.type == "Collage")
      {
        List.map(F2(processForm)(offset))(elem.element.model.forms);
      }
      if (elem.props.debugTracePathId)
      {
        positions[elem.props.debugTracePathId] = new Point(offset.x, offset.y);
      }
    }

    function processForm(offset, form) {
      if (form.form.ctor == "FElement")
      {
        processElement(form.form._0, offset.translate(form.x, -form.y));
      }
    }

    processElement(currentScene, new Point(0, 0));
    return positions;
  }

  function appendPositions(positions) {
    for (var id in positions) {
      var pos = positions[id];
      if (tracePositions.hasOwnProperty(id)) {
        tracePositions[id].push(pos);
      }
      else {
        tracePositions[id] = [pos];
      }
      if (tracePositions[id].length < runtime.debuggerStatus.eventCounter) {
        var padCount = runtime.debuggerStatus.eventCounter - tracePositions[id].length;
        var lastTracePosition = tracePositions[id][tracePositions[id].length - 1];
        for (var i = padCount; i--;) {
          tracePositions[id].push(lastTracePosition)
        }
      }
      assert(tracePositions[id].length === runtime.debuggerStatus.eventCounter,
             "We don't have a 1-1 mapping of trace positions to events");
    }
  }

  function graphicsUpdate(currentScene) {
    if (!recordingTraces) {
      return;
    }

    var ctx = tracePathCanvas.getContext('2d');
    ctx.clearRect(0, 0, tracePathCanvas.width, tracePathCanvas.height);

    ctx.save();
    ctx.translate(ctx.canvas.width/2, ctx.canvas.height/2);
    appendPositions(findPositions(currentScene));
    for (var id in tracePositions)
    {
      ctx.beginPath();
      var points = tracePositions[id];
      for (var i=0; i < points.length; i++)
      {
        var p = points[i];
        if (i == 0) {
          ctx.moveTo(p.x, p.y);
        }
        else {
          ctx.lineTo(p.x, p.y);
        }
      }
      ctx.lineWidth = 1;
      ctx.strokeStyle = "rgba(50, 50, 50, 0.4)";
      ctx.stroke();
    }

    ctx.restore();
  }

  function clearTraces() {
    tracePositions = {};
  }

  function stopRecording() {
    recordingTraces = false;
  }

  function startRecording() {
    recordingTraces = true;
  }

  function clearTracesAfter(position) {
    var newTraces = {};
    for (var id in tracePositions) {
      newTraces[id] = tracePositions[id].slice(0,position);
    }
    tracePositions = newTraces;
  }

  return {
    graphicsUpdate: graphicsUpdate,
    canvas: tracePathCanvas,
    clearTraces: clearTraces,
    clearTracesAfter: clearTracesAfter,
    stopRecording: stopRecording,
    startRecording: startRecording
  }
}

function executeCallbacks(callbacks, reexecute) {
  callbacks.forEach(function(timer) {
    if (reexecute || !timer.executed) {
      var func = timer.func;
      timer.executed = true;
      func();
    }
  });
}

function createCanvas() {
  var c = document.createElement('canvas');
  c.width = window.innerWidth;
  c.height = window.innerHeight;
  c.style.position = "absolute";
  c.style.top = "0";
  c.style.left = "0";
  c.style.pointerEvents = "none";
  return c;
}

function assert(bool, msg) {
  if (!bool) {
    throw "Assertion error: " + msg;
  }
}

function snapshotSignalGraph(signalGraphNodes) {
  var nodeValues = [];

  signalGraphNodes.forEach(function(node) {
    nodeValues.push({ value: node.value, id: node.id });
  });

  return nodeValues;
};

function restoreSnapshot(signalGraphNodes, snapshot) {
  assert(signalGraphNodes.length == snapshot.length,
         "saved program state has wrong length");
  for (var i=0; i < signalGraphNodes.length; i++) {
    var node = signalGraphNodes[i];
    var state = snapshot[i];
    assert(node.id == state.id, "the nodes moved position");

    node.value = state.value;
  }
}

function flattenSignalGraph(nodes) {
  var nodesById = {};

  function addAllToDict(node) {
    nodesById[node.id] = node;
    node.kids.forEach(addAllToDict);
  }
  nodes.forEach(addAllToDict);

  var allNodes = Object.keys(nodesById).sort().map(function(key) {
    return nodesById[key];
  });
  return allNodes;
};

}());
<|endoftext|>


<commit_before>const exec = require('child_process').exec;

const AUDIT_EXCEPTIONS = ['handlebars'];

const execute = (command, callback) => {
  exec(
    command, {
      maxBuffer: 2000 * 1024
    },
    (error, stdout, stderr) => {
      callback(stdout);
    }
  );
};
execute('npm audit --json', json => {
  const advisories = JSON.parse(json).advisories;
  if (!advisories) {
    console.info('Most likely npm audit is unavailable', json);
    process.exit(0);
  }
  let auditPass = true;
  for (const id in advisories) {
    if (
      advisories[id].severity === 'high' &&
      !AUDIT_EXCEPTIONS.includes(advisories[id].module_name)
    ) {
      console.error('AUDIT Failed', advisories[id]);
      auditPass = false;
    }
  }
  if (!auditPass) process.exit(1);
});
<commit_msg>Add decompress to audit exceptions<commit_after>const exec = require('child_process').exec;

const AUDIT_EXCEPTIONS = ['handlebars', 'decompress'];

const execute = (command, callback) => {
  exec(
    command, {
      maxBuffer: 2000 * 1024
    },
    (error, stdout, stderr) => {
      callback(stdout);
    }
  );
};
execute('npm audit --json', json => {
  const advisories = JSON.parse(json).advisories;
  if (!advisories) {
    console.info('Most likely npm audit is unavailable', json);
    process.exit(0);
  }
  let auditPass = true;
  for (const id in advisories) {
    if (
      advisories[id].severity === 'high' &&
      !AUDIT_EXCEPTIONS.includes(advisories[id].module_name)
    ) {
      console.error('AUDIT Failed', advisories[id]);
      auditPass = false;
    }
  }
  if (!auditPass) process.exit(1);
});
<|endoftext|>


<commit_before>
const parse = require('..')
const generate = require('csv-generate')
const transform = require('stream-transform')

const generator = generate({
  length: 20
})
const parser = parse({
  delimiter: ':'
})
const transformer = transform(function(record, callback){
  setTimeout(function(){
    callback(null, record.join(' ')+'\n')
  }, 500)
}, {
  parallel: 5
})
input.pipe(parser).pipe(transformer).pipe(process.stdout)
<commit_msg>samples: fix bad variable name<commit_after>
const parse = require('..')
const generate = require('csv-generate')
const transform = require('stream-transform')

const generator = generate({
  length: 20
})
const parser = parse({
  delimiter: ':'
})
const transformer = transform(function(record, callback){
  setTimeout(function(){
    callback(null, record.join(' ')+'\n')
  }, 500)
}, {
  parallel: 5
})
generator.pipe(parser).pipe(transformer).pipe(process.stdout)
<|endoftext|>


<commit_before>var WS = require('pull-ws')
var URL = require('url')
var pull = require('pull-stream/pull')
var Map = require('pull-stream/throughs/map')
var scopes = require('multiserver-scopes')
var http = require('http')

function safe_origin (origin, address, port) {

  //if the connection is not localhost, we shouldn't trust
  //the origin header. So, use address instead of origin
  //if origin not set, then it's definitely not a browser.
  if(!(address === '::1' || address === '128.0.0.1') || origin == undefined)
    return 'ws:' + address + (port ? ':' + port : '')

  //note: origin "null" (as string) can happen a bunch of ways
  //      it can be a html opened as a file
  //      or certain types of CORS
  //      https://www.w3.org/TR/cors/#resource-sharing-check-0
  //      and webworkers if loaded from data-url?
  if(origin === 'null')
    return 'ws:null'

  //a connection from the browser on localhost,
  //we choose to trust this came from a browser.
  return origin.replace(/^http/, 'ws')

}

module.exports = function (opts) {
  opts = opts || {}
  opts.binaryType = (opts.binaryType || 'arraybuffer')
  var scope = opts.scope || 'device'
  function isScoped (s) {
    return s === scope || Array.isArray(scope) && ~scope.indexOf(s)
  }

  var secure = opts.server && !!opts.server.key
  return {
    name: 'ws',
    scope: function() { return opts.scope || 'device' },
    server: function (onConnect, startedCb) {

      if(!WS.createServer) return
      // Choose a dynamic port between 49152 and 65535
      // https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Dynamic,_private_or_ephemeral_ports
      opts.port = opts.port || Math.floor(49152 + (65535 - 49152 + 1) * Math.random())

      var server = opts.server || http.createServer(opts.handler)

      var ws_server = WS.createServer(Object.assign({}, opts, {server: server}), function (stream) {
        stream.address = safe_origin(
          stream.headers.origin,
          stream.remoteAddress,
          stream.remotePort
        )
        onConnect(stream)
      })

      startedCb(null, true)

      if(!opts.server) {
        console.log('Listening on ' + opts.host +':' + opts.port + ' (multiserver ws plugin)')
        server.listen(opts.port)
      }
      return function (cb) {
        console.log('Closing server on ' + opts.host +':' + opts.port + ' (multiserver ws plugin)')
        server.close(function(err) {
          if (err) console.error(err)
          else console.log('No longer listening on ' + opts.host +':' + opts.port + ' (multiserver ws plugin)')
          if (cb) cb(err)
        })
      }
    },
    client: function (addr, cb) {
      if(!addr.host) {
        addr.hostname = addr.hostname || opts.host || 'localhost'
        addr.slashes = true
        addr = URL.format(addr)
      }
      if('string' !== typeof addr)
        addr = URL.format(addr)

      var stream = WS.connect(addr, {
        binaryType: opts.binaryType,
        onConnect: function (err) {
          //ensure stream is a stream of node buffers
          stream.source = pull(stream.source, Map(Buffer.from.bind(Buffer)))
          cb(err, stream)
        }
      })
      stream.address = addr

      return function () {
        stream.close(cb)
      }
    },
    stringify: function (scope) {
      scope = scope || 'device'
      if(!isScoped(scope)) return null
      if(!WS.createServer) return null
      var port
      if(opts.server)
        port = opts.server.address().port
      else
        port = opts.port

      var host = (scope == 'public' && opts.external) || scopes.host(scope)
      //if a public scope was requested, but a public ip is not available, return
      if(!host) return null

      return URL.format({
        protocol: secure ? 'wss' : 'ws',
        slashes: true,
        hostname: host,
        port: (secure ? port == 443 : port == 80) ? undefined : port
      })
    },
    parse: function (str) {
      var addr = URL.parse(str)
      if(!/^wss?\:$/.test(addr.protocol)) return null
      return addr
    }
  }
}

<commit_msg>if ws creates a server, it should wait until it is listening to callback startCb<commit_after>var WS = require('pull-ws')
var URL = require('url')
var pull = require('pull-stream/pull')
var Map = require('pull-stream/throughs/map')
var scopes = require('multiserver-scopes')
var http = require('http')

function safe_origin (origin, address, port) {

  //if the connection is not localhost, we shouldn't trust
  //the origin header. So, use address instead of origin
  //if origin not set, then it's definitely not a browser.
  if(!(address === '::1' || address === '128.0.0.1') || origin == undefined)
    return 'ws:' + address + (port ? ':' + port : '')

  //note: origin "null" (as string) can happen a bunch of ways
  //      it can be a html opened as a file
  //      or certain types of CORS
  //      https://www.w3.org/TR/cors/#resource-sharing-check-0
  //      and webworkers if loaded from data-url?
  if(origin === 'null')
    return 'ws:null'

  //a connection from the browser on localhost,
  //we choose to trust this came from a browser.
  return origin.replace(/^http/, 'ws')

}

module.exports = function (opts) {
  opts = opts || {}
  opts.binaryType = (opts.binaryType || 'arraybuffer')
  var scope = opts.scope || 'device'
  function isScoped (s) {
    return s === scope || Array.isArray(scope) && ~scope.indexOf(s)
  }

  var secure = opts.server && !!opts.server.key
  return {
    name: 'ws',
    scope: function() { return opts.scope || 'device' },
    server: function (onConnect, startedCb) {

      if(!WS.createServer) return
      // Choose a dynamic port between 49152 and 65535
      // https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Dynamic,_private_or_ephemeral_ports
      opts.port = opts.port || Math.floor(49152 + (65535 - 49152 + 1) * Math.random())

      var server = opts.server || http.createServer(opts.handler)

      var ws_server = WS.createServer(Object.assign({}, opts, {server: server}), function (stream) {
        stream.address = safe_origin(
          stream.headers.origin,
          stream.remoteAddress,
          stream.remotePort
        )
        onConnect(stream)
      })

      if(!opts.server) {
        console.log('Listening on ' + opts.host +':' + opts.port + ' (multiserver ws plugin)')
        server.listen(opts.port, function () {
          startedCb && startedCb(null, true)
        })
      }
      else
        startedCb && startedCb(null, true)

      return function (cb) {
        console.log('Closing server on ' + opts.host +':' + opts.port + ' (multiserver ws plugin)')
        server.close(function(err) {
          if (err) console.error(err)
          else console.log('No longer listening on ' + opts.host +':' + opts.port + ' (multiserver ws plugin)')
          if (cb) cb(err)
        })
      }
    },
    client: function (addr, cb) {
      if(!addr.host) {
        addr.hostname = addr.hostname || opts.host || 'localhost'
        addr.slashes = true
        addr = URL.format(addr)
      }
      if('string' !== typeof addr)
        addr = URL.format(addr)

      var stream = WS.connect(addr, {
        binaryType: opts.binaryType,
        onConnect: function (err) {
          //ensure stream is a stream of node buffers
          stream.source = pull(stream.source, Map(Buffer.from.bind(Buffer)))
          cb(err, stream)
        }
      })
      stream.address = addr

      return function () {
        stream.close(cb)
      }
    },
    stringify: function (scope) {
      scope = scope || 'device'
      if(!isScoped(scope)) return null
      if(!WS.createServer) return null
      var port
      if(opts.server)
        port = opts.server.address().port
      else
        port = opts.port

      var host = (scope == 'public' && opts.external) || scopes.host(scope)
      //if a public scope was requested, but a public ip is not available, return
      if(!host) return null

      return URL.format({
        protocol: secure ? 'wss' : 'ws',
        slashes: true,
        hostname: host,
        port: (secure ? port == 443 : port == 80) ? undefined : port
      })
    },
    parse: function (str) {
      var addr = URL.parse(str)
      if(!/^wss?\:$/.test(addr.protocol)) return null
      return addr
    }
  }
}

<|endoftext|>


<commit_before>console.log('hello, world');

var iptc = require('node-iptc');
var fs = require('fs');
var babyParse = require('babyparse');
var glob = require('glob');
var SightingList = require('./server/scripts/sightinglist.js');
var XML = require('pixl-xml');

var gSightingList = null;

var csvData = fs.readFileSync('server/data/ebird.csv', 'utf8');

var ebird = babyParse.parse(csvData, {
    header: true,
});
console.log('parsed ebird', ebird.data.length);

var jsonData = fs.readFileSync('server/data/photos.json');
var photos = JSON.parse(jsonData);
console.log('parsed photos', photos.length);

gSightingList = new SightingList();
gSightingList.addRows(ebird.data);
gSightingList.setGlobalIDs();

function handleXMP(inXMPPath, tmpEbirdDate, n, tmpDate) {
    let data = fs.readFileSync(inXMPPath)

    let result = XML.parse(data)

    let label = result['rdf:RDF']['rdf:Description']['xmp:Label'];
    let location = result['rdf:RDF']['rdf:Description']['Iptc4xmpCore:Location'];
    let daySightingList = new SightingList(gSightingList.filter(function(s) { return s['Date'] == tmpEbirdDate; }));
    let speciesSightings = daySightingList.filter(s => { return s['Common Name'] == label });
    let photosOriginalNameMatch = photos.filter(p => p['Photo URL'].toLowerCase().indexOf(n.toLowerCase()) > 0);

    let info = {
        label: label,
        location: location,
        date: tmpDate,
        ebirdDate: tmpEbirdDate,
        sightingsThatDate: daySightingList.length(),
        speciesSightingsThatDate: speciesSightings,
        locations: daySightingList.getUniqueValues('Location')
    }

    if (photosOriginalNameMatch.length > 0) {
        // console.log('already found', n, label, 'in photos.json', tmpEbirdDate, location);
        info.action = 'already found in photos.json'
        // console.log('already found', n, label, 'in photos.json', tmpEbirdDate, location, photosOriginalNameMatch[0]);
    } else if (speciesSightings.length > 0) {
        let newFilename = tmpDate + '-' + speciesSightings[0]['Scientific Name'].toLowerCase().replace(' ', '-') + '-' + n;
        console.log(n, label, 'sighting', speciesSightings[0].id, location);

        info.action = 'READY to add to photos.json'

        let samplePhoto = {
            Date: speciesSightings[0].Date,
            Location: speciesSightings[0].Location,
            'Scientific Name': speciesSightings[0]['Scientific Name'],
            'Common Name': speciesSightings[0]['Common Name'],
            'Thumbnail URL': 'https://s3.amazonaws.com/birdwalker/thumb/' + newFilename,
            'Photo URL': 'https://s3.amazonaws.com/birdwalker/photo/' + newFilename,
            County: speciesSightings[0].County,
            'State/Province': speciesSightings[0]['State/Province'],
        };
        console.log('\n\n\n');
        console.log(JSON.stringify(samplePhoto, null, '  '));
        console.log('cp /Users/walker/Photography/flickrUP/' + n + ' /Users/walker/Photography/flickrUP/' + newFilename);
        console.log('s3cmd put --acl-public /Users/walker/Photography/flickrUP/' + newFilename + ' s3://birdwalker/photo/ --add-header=Cache-Control:max-age=31536000')
        console.log('s3cmd put --acl-public /Users/walker/Photography/flickrUP/' + newFilename + ' s3://birdwalker/thumb/ --add-header=Cache-Control:max-age=31536000')
    } else if (daySightingList.length() > 0) {
        info.action = 'missing species ' + label + ' from existing trip ' + tmpEbirdDate
        // console.log(n, 'trip yes but', label, 'no', tmpEbirdDate);
    } else {
        info.action = 'missing trip for date ' + tmpEbirdDate
        // console.log(n, label, 'no trip this date', tmpEbirdDate);
    }

    return info
}

function handleJPEG(inJPEGFilename) {
    let tmpFile = fs.readFileSync('/Users/walker/Photography/flickrUP/' + inJPEGFilename);
    let tmpIPTCdate = iptc(tmpFile).date_created;
    let tmpDate = tmpIPTCdate.substring(0,4) + '-' + tmpIPTCdate.substring(4,6) + '-' + tmpIPTCdate.substring(6,8);
    let tmpEbirdDate = tmpIPTCdate.substring(4,6) + '-' + tmpIPTCdate.substring(6,8) + '-' + tmpIPTCdate.substring(0,4);
    let tmpPath = tmpIPTCdate.substring(0,4) + '/' + tmpDate + '/' + inJPEGFilename.replace('jpg', 'xmp');
    let info = {}

    if (fs.existsSync('/Volumes/Big\ Ethel/Photos/' + tmpPath)) {
        info = handleXMP('/Volumes/Big\ Ethel/Photos/' + tmpPath, tmpEbirdDate, inJPEGFilename, tmpDate);
    } else if (fs.existsSync('/Users/walker/Pictures/' + tmpPath)) {
        info = handleXMP('/Users/walker/Pictures/' + tmpPath, tmpEbirdDate, inJPEGFilename, tmpDate);
    } else if (fs.existsSync('/Volumes/Big Ethel/Photos/' + tmpDate + '/' + inJPEGFilename.replace('jpg','xmp'))) {
        info = handleXMP('/Volumes/Big Ethel/Photos/' + tmpDate + '/' + inJPEGFilename.replace('jpg','xmp'), tmpEbirdDate, inJPEGFilename, tmpDate);
    } else {
        info.action = 'no XMP for ' + tmpPath
        // console.log('no XMP', tmpPath, tmpDate);
        // console.log('GLOB', glob.sync('/Volumes/Big Ethel/Photos/'+tmpIPTCdate.substring(0,4)+'/**/' + inJPEGFilename.replace('jpg','xmp')));
        // console.log('GLOB', glob.sync('/Volumes/Big Ethel/Photos/'+tmpDate+'/**/' + inJPEGFilename.replace('jpg','xmp')));
    }

    info.iptcDate = tmpIPTCdate
    info.name = inJPEGFilename
    info.path = tmpPath

    return info
}

function getStatusForRecentJPEGs() {
    // read all the recent JPEG's in flickrUp
    var allTheFiles = fs.readdirSync('/Users/walker/Photography/flickrUP');
    var theNow = new Date();

    console.log('total flickrUP jpegs', allTheFiles.length);

    var jpegs = allTheFiles.filter((n) => {
        var stats = fs.statSync('/Users/walker/Photography/flickrUP/' + n);
        var daysOld = (theNow - stats.birthtime)/(24*60*60*1000);
        return (daysOld < 20);
    });

    console.log('recent jpegs', jpegs.length);

    var infoList = jpegs.map(handleJPEG)

    console.log('infoList', infoList.map(i => [i.name, i.action]))
}

getStatusForRecentJPEGs()

<commit_msg>don't show ones that don't need action group all the photos.json entries together<commit_after>console.log('hello, world');

var iptc = require('node-iptc');
var fs = require('fs');
var babyParse = require('babyparse');
var glob = require('glob');
var SightingList = require('./server/scripts/sightinglist.js');
var XML = require('pixl-xml');

var gSightingList = null;

var csvData = fs.readFileSync('server/data/ebird.csv', 'utf8');

var ebird = babyParse.parse(csvData, {
    header: true,
});
console.log('parsed ebird', ebird.data.length);

var jsonData = fs.readFileSync('server/data/photos.json');
var photos = JSON.parse(jsonData);
console.log('parsed photos', photos.length);

gSightingList = new SightingList();
gSightingList.addRows(ebird.data);
gSightingList.setGlobalIDs();

var newPhotos = []

function handleXMP(inXMPPath, tmpEbirdDate, n, tmpDate) {
    let data = fs.readFileSync(inXMPPath)

    let result = XML.parse(data)

    let label = result['rdf:RDF']['rdf:Description']['xmp:Label'];
    let location = result['rdf:RDF']['rdf:Description']['Iptc4xmpCore:Location'];
    let daySightingList = new SightingList(gSightingList.filter(function(s) { return s['Date'] == tmpEbirdDate; }));
    let speciesSightings = daySightingList.filter(s => { return s['Common Name'] == label });
    let photosOriginalNameMatch = photos.filter(p => p['Photo URL'].toLowerCase().indexOf(n.toLowerCase()) > 0);

    if (label == null) {
        label = 'missing'
    }

    let info = {
        label: label,
        location: location,
        date: tmpDate,
        ebirdDate: tmpEbirdDate,
        sightingsThatDate: daySightingList.length(),
        speciesSightingsThatDate: speciesSightings,
        locations: daySightingList.getUniqueValues('Location'),
        needsAction: true
    }

    if (photosOriginalNameMatch.length > 0) {
        // console.log('already found', n, label, 'in photos.json', tmpEbirdDate, location);
        info.action = 'already found in photos.json'
        info.needsAction = false
        // console.log('already found', n, label, 'in photos.json', tmpEbirdDate, location, photosOriginalNameMatch[0]);
    } else if (speciesSightings.length > 0) {
        let newFilename = tmpDate + '-' + speciesSightings[0]['Scientific Name'].toLowerCase().replace(' ', '-') + '-' + n;

        info.action = 'READY to add to photos.json'

        let samplePhoto = {
            Date: speciesSightings[0].Date,
            Location: speciesSightings[0].Location,
            'Scientific Name': speciesSightings[0]['Scientific Name'],
            'Common Name': speciesSightings[0]['Common Name'],
            'Thumbnail URL': 'https://s3.amazonaws.com/birdwalker/thumb/' + newFilename,
            'Photo URL': 'https://s3.amazonaws.com/birdwalker/photo/' + newFilename,
            County: speciesSightings[0].County,
            'State/Province': speciesSightings[0]['State/Province'],
        };

        newPhotos.push(samplePhoto)

        console.log('cp /Users/walker/Photography/flickrUP/' + n + ' /Users/walker/Photography/flickrUP/' + newFilename);
        console.log('s3cmd put --acl-public /Users/walker/Photography/flickrUP/' + newFilename + ' s3://birdwalker/photo/ --add-header=Cache-Control:max-age=31536000')
        console.log('s3cmd put --acl-public /Users/walker/Photography/flickrUP/' + newFilename + ' s3://birdwalker/thumb/ --add-header=Cache-Control:max-age=31536000')
    } else if (daySightingList.length() > 0) {
        info.action = 'missing species from existing trip ' + tmpEbirdDate
        // console.log(n, 'trip yes but', label, 'no', tmpEbirdDate);
    } else {
        info.action = 'missing trip for date ' + tmpEbirdDate
        // console.log(n, label, 'no trip this date', tmpEbirdDate);
    }

    return info
}

function handleJPEG(inJPEGFilename) {
    let tmpFile = fs.readFileSync('/Users/walker/Photography/flickrUP/' + inJPEGFilename);
    let tmpIPTCdate = iptc(tmpFile).date_created;
    let tmpDate = tmpIPTCdate.substring(0,4) + '-' + tmpIPTCdate.substring(4,6) + '-' + tmpIPTCdate.substring(6,8);
    let tmpEbirdDate = tmpIPTCdate.substring(4,6) + '-' + tmpIPTCdate.substring(6,8) + '-' + tmpIPTCdate.substring(0,4);
    let tmpPath = tmpIPTCdate.substring(0,4) + '/' + tmpDate + '/' + inJPEGFilename.replace('jpg', 'xmp');
    let info = {}

    if (fs.existsSync('/Volumes/Big\ Ethel/Photos/' + tmpPath)) {
        info = handleXMP('/Volumes/Big\ Ethel/Photos/' + tmpPath, tmpEbirdDate, inJPEGFilename, tmpDate);
    } else if (fs.existsSync('/Users/walker/Pictures/' + tmpPath)) {
        info = handleXMP('/Users/walker/Pictures/' + tmpPath, tmpEbirdDate, inJPEGFilename, tmpDate);
    } else if (fs.existsSync('/Volumes/Big Ethel/Photos/' + tmpDate + '/' + inJPEGFilename.replace('jpg','xmp'))) {
        info = handleXMP('/Volumes/Big Ethel/Photos/' + tmpDate + '/' + inJPEGFilename.replace('jpg','xmp'), tmpEbirdDate, inJPEGFilename, tmpDate);
    } else {
        info.action = 'no XMP for ' + tmpPath
        // console.log('no XMP', tmpPath, tmpDate);
        // console.log('GLOB', glob.sync('/Volumes/Big Ethel/Photos/'+tmpIPTCdate.substring(0,4)+'/**/' + inJPEGFilename.replace('jpg','xmp')));
        // console.log('GLOB', glob.sync('/Volumes/Big Ethel/Photos/'+tmpDate+'/**/' + inJPEGFilename.replace('jpg','xmp')));
    }

    info.iptcDate = tmpIPTCdate
    info.name = inJPEGFilename
    info.path = tmpPath

    return info
}

function getStatusForRecentJPEGs() {
    // read all the recent JPEG's in flickrUp
    var allTheFiles = fs.readdirSync('/Users/walker/Photography/flickrUP');
    var theNow = new Date();

    console.log('total flickrUP jpegs', allTheFiles.length);

    var jpegs = allTheFiles.filter((n) => {
        var stats = fs.statSync('/Users/walker/Photography/flickrUP/' + n);
        var daysOld = (theNow - stats.birthtime)/(24*60*60*1000);
        return (daysOld < 20);
    });

    console.log('recent jpegs', jpegs.length);

    var infoList = jpegs.map(handleJPEG)

    console.log(JSON.stringify(newPhotos, null, '  '))

    console.log(infoList.filter(i => i.needsAction).map(i => i.name.padStart(15) + ' ' + i.label.padStart(20) + ' ' +    i.action))
}

getStatusForRecentJPEGs()

<|endoftext|>


<commit_before>var q = require('bluebird');
var registry = require("npm-registry");
var _ = require('underscore');
var NpmPublishStream = require('npm-publish-stream');
var semver = require('semver');
var rp = require('request-promise');
var yaml = require('js-yaml');

var database = require("./lib/database.js");

// crawlers
var keywords = require("npm-keywords");
var ghClient = require("./lib/github.js");
var npmClient = require("./lib/npm.js");
var npmHistory = require("./lib/npmHistory.js");

// helpers
var postProcessor = require("./lib/util/postProcessor.js");

var workflow = function(opts) {
  this.keys = opts.keys || [];
  this.refreshTime = opts.refreshTime || 3600;
  this.searchTime = opts.searchTime || 120;
  this.updateInterval = opts.updateInterval || 60;
  this.iotagurl = opts.iotagurl;
  this.db = new database({
    log: log
  });
  this.dbLoad = this.db.load();
  this.pkgs = [];
  this.debug = opts.debug;
  this.npm = new registry({
    registry: opts.registryURL
  });
};

workflow.prototype.start = function() {
  var self = this;

  if (this.debug) {
    this.pkgs = [{
      name: "angularplasmid",
      version: "0.0.1"
    }, {
      name: "biojs-vis-msa",
      version: "0.0.1"
    }];
    this.updateCronJob();
    this.loadIoTags(this.pkgs);
    //this.searchCronI = setInterval(this.searchCron.bind(this), 5000);
    return q.resolve("a");
  }
  log.debug("starting downloader");
  return this.run().then(function() {
    this.reloadCronI = setInterval(this.run.bind(this), this.refreshTime * 1000);
    this.searchCronI = setInterval(this.searchCron.bind(this), this.searchTime * 1000);
    this.updateCronI = setInterval(this.updateCronJob.bind(this), this.updateInterval * 1000);
    // TODO: check whether it makes sense to enable this
    // the "live-stream" also uses polling
    //this.liveStreamI = this.liveStreamCron();
    return this.pkgs;
  }.bind(this));
};

// 1) download package list
workflow.prototype.run = function run() {
  var self = this;
  return keywords(this.keys, this.npm)
    .map(this.downloadPkg.bind(this))
    .then(function(pkgs) {
      // on succes:  save to DB
      log.info("workflow: trying to save into DB.");
      self.pkgs = pkgs.map(function(el) {
        return {
          name: el.name,
          version: el.version
        };
      });
      // post process the package list
      return self.loadIoTags(pkgs);

    }).then(function(pkgs) {
      // save in DB
      self.dbLoad
        .then(self.db.loadCache.bind(self.db))
        .then(function() {
          return self.db.save(pkgs);
        });
    }).then(function() {
      log.info("workflow: load complete.");
      return self.pkgs;
    });
};

// 2) crawl the package from NPM
workflow.prototype.downloadPkg = function(name) {
  // also support pkg object with name attribute
  if (name.name !== undefined) {
    name = name.name;
  }
  return new npmClient(name, this.npm).then(this.postDownload.bind(this));
};

// 3) other info
workflow.prototype.postDownload = function(pkg) {
  var postOpts = {
    keys: this.keys
  };
  var ps = [];
  var opts = {
    log: log
  };
  ps.push(new ghClient(pkg, opts));
  ps.push(new npmHistory(pkg, opts, this.npm));
  return q.settle(ps).then(function() {
    return postProcessor(pkg, postOpts);
  }).catch(function(err) {
    if (err != undefined && err.name !== "queryEvents") {
      // event errors can happen - ignore them
      log.warn("downloaderr", err);
    } else {
      log.info(arguments);
    }
    return pkg;
  });
};

// update a single pkg
workflow.prototype.updatePkg = function(pkg) {
  return this.downloadPkg(pkg).then(function(newPkg) {
    var index = _.indexOf(_.pluck(this.pkgs, "name"), newPkg.name);
    if (index >= 0) {
      this.pkgs[index] = newPkg;
    } else {
      this.pkgs.push(newPkg);
    }
    this.db.updatePkg(newPkg);
    log.info("package update saved: ", newPkg.name);
    return newPkg;
  }.bind(this)).catch(function(err) {
    log.warn("err during package update: ", pkg.name);
    log.warn(err);
  });
};

// download the tagged list and apply it
workflow.prototype.loadIoTags = function(pkgs) {
  if (!this.iotagurl) {
    return q.resolve(pkgs);
  }
  log.debug("downloading yaml file: " + this.iotagurl);
  return rp(this.iotagurl).then(function(raw){
    log.debug("loading yaml file");
    var data = yaml.safeLoad(raw);
    return data;
  }).then(function(packages) {
    log.warn("found " + Object.keys(packages).length + " packages with iotags");
    for (var name in packages) {
      var ps = pkgs.filter(function(p) {
        return p.name === name;
      });
      if (ps.length === 1) {
        var p = ps[0];
        p.iotags = packages[name];
      }
    }
    return pkgs;
  });
};

// crons the npm registry manually and checks for package updates
workflow.prototype.updateCronJob = function updateCronJob() {
  var self = this;
  this.pkgs.forEach(function(oldPkg) {
    return new npmClient(oldPkg.name + "@latest", self.npm).then(function(newPkg) {
      // we have only the latest version -> download entire package
      if (oldPkg.version != newPkg.version && newPkg.name != undefined) {
        log.info("new package uploaded: ", newPkg.name, newPkg.version, oldPkg.version);
        self.updatePkg(newPkg.name).then(function() {
          newPkg.updateVersionType = semver.diff(oldPkg.version, newPkg.version);
          newPkg.updateType = "update";
          self.trigger("pkg:updated", newPkg);
        });
      }
    }.bind(this));
  });
};

// crons the npm registry for new packages
workflow.prototype.searchCron = function searchCron() {
  var self = this;
  var pkgNames = this.pkgs.map(function(pkg) {
    return pkg.name;
  });
  return keywords(this.keys, this.npm).filter(function(pkg) {
    return pkgNames.indexOf(pkg.name) < 0;
  }).then(function(pkgs) {
    if (pkgs.length > 0) {
      pkgs.forEach(function(pkg) {
        log.info("new package found: ", pkg.name);
        self.updatePkg(pkg.name).then(function() {
          pkg.updateType = "new";
          self.trigger("pkg:new", pkg);
        });
      });
    }
  });
};

workflow.prototype.liveStreamCron = function searchCron() {
  var self = this;
  return new NpmPublishStream()
    .on('data', function(data) {
      var newPkg = data.doc;
      if (_.intersection(newPkg.keywords, self.keys).length > 0) {
        var oldPkg = self.pkgs[newPkg.name];
        if (oldPkg == undefined) {
          log.info("new package found: ", newPkg.name);
          self.updatePkg(newPkg.name);
        } else if (oldPkg.version != newPkg.version) {
          log.info("new package uploaded: ", newPkg.name, newPkg.version, oldPkg.version);
          self.updatePkg(newPkg.name);
        }
      }
    });
};

workflow.prototype.stop = function() {
  clearInterval(this.updateCronI);
  clearInterval(this.reloadCronI);
  clearInterval(this.searchCronI);
  if (this.liveStreamI) {
    this.liveStreamI.end();
  }
};

require("biojs-events").mixin(workflow.prototype);

var log;
module.exports = function(logger) {
  log = logger;
  return workflow;
};
<commit_msg>update debug mode to msa 0.4.11<commit_after>var q = require('bluebird');
var registry = require("npm-registry");
var _ = require('underscore');
var NpmPublishStream = require('npm-publish-stream');
var semver = require('semver');
var rp = require('request-promise');
var yaml = require('js-yaml');

var database = require("./lib/database.js");

// crawlers
var keywords = require("npm-keywords");
var ghClient = require("./lib/github.js");
var npmClient = require("./lib/npm.js");
var npmHistory = require("./lib/npmHistory.js");

// helpers
var postProcessor = require("./lib/util/postProcessor.js");

var workflow = function(opts) {
  this.keys = opts.keys || [];
  this.refreshTime = opts.refreshTime || 3600;
  this.searchTime = opts.searchTime || 120;
  this.updateInterval = opts.updateInterval || 60;
  this.iotagurl = opts.iotagurl;
  this.db = new database({
    log: log
  });
  this.dbLoad = this.db.load();
  this.pkgs = [];
  this.debug = opts.debug;
  this.npm = new registry({
    registry: opts.registryURL
  });
};

workflow.prototype.start = function() {
  var self = this;

  log.debug("starting downloader");
  return this.run().then(function() {
    this.reloadCronI = setInterval(this.run.bind(this), this.refreshTime * 1000);
    this.searchCronI = setInterval(this.searchCron.bind(this), this.searchTime * 1000);
    this.updateCronI = setInterval(this.updateCronJob.bind(this), this.updateInterval * 1000);
    // TODO: check whether it makes sense to enable this
    // the "live-stream" also uses polling
    //this.liveStreamI = this.liveStreamCron();
    return this.pkgs;
  }.bind(this));
};

// 1) download package list
workflow.prototype.run = function run() {
  var self = this;

  var ps;

  if (this.debug) {
    ps = q.resolve([{
      name: "angularplasmid",
      version: "0.0.1"
    }, {
      name: "msa",
      version: "0.4.11"
    }]);
  }else{
    ps = keywords(this.keys, this.npm);
  }
  return ps.map(this.downloadPkg.bind(this))
    .then(function(pkgs) {
      // on succes:  save to DB
      log.info("workflow: trying to save into DB.");
      self.pkgs = pkgs.map(function(el) {
        return {
          name: el.name,
          version: el.version
        };
      });
      // post process the package list
      return self.loadIoTags(pkgs);

    }).then(function(pkgs) {
      // save in DB
      self.dbLoad
        .then(self.db.loadCache.bind(self.db))
        .then(function() {
          return self.db.save(pkgs);
        });
    }).then(function() {
      log.info("workflow: load complete.");
      return self.pkgs;
    });
};

// 2) crawl the package from NPM
workflow.prototype.downloadPkg = function(name) {
  // also support pkg object with name attribute
  if (name.name !== undefined) {
    name = name.name;
  }
  return new npmClient(name, this.npm).then(this.postDownload.bind(this));
};

// 3) other info
workflow.prototype.postDownload = function(pkg) {
  var postOpts = {
    keys: this.keys
  };
  var ps = [];
  var opts = {
    log: log
  };
  ps.push(new ghClient(pkg, opts));
  ps.push(new npmHistory(pkg, opts, this.npm));
  return q.settle(ps).then(function() {
    return postProcessor(pkg, postOpts);
  }).catch(function(err) {
    if (err != undefined && err.name !== "queryEvents") {
      // event errors can happen - ignore them
      log.warn("downloaderr", err);
    } else {
      log.info(arguments);
    }
    return pkg;
  });
};

// update a single pkg
workflow.prototype.updatePkg = function(pkg) {
  return this.downloadPkg(pkg).then(function(newPkg) {
    var index = _.indexOf(_.pluck(this.pkgs, "name"), newPkg.name);
    if (index >= 0) {
      this.pkgs[index] = newPkg;
    } else {
      this.pkgs.push(newPkg);
    }
    this.db.updatePkg(newPkg);
    log.info("package update saved: ", newPkg.name);
    return newPkg;
  }.bind(this)).catch(function(err) {
    log.warn("err during package update: ", pkg.name);
    log.warn(err);
  });
};

// download the tagged list and apply it
workflow.prototype.loadIoTags = function(pkgs) {
  if (!this.iotagurl) {
    return q.resolve(pkgs);
  }
  log.debug("downloading yaml file: " + this.iotagurl);
  return rp(this.iotagurl).then(function(raw){
    log.debug("loading yaml file");
    var data = yaml.safeLoad(raw);
    return data;
  }).then(function(packages) {
    log.warn("found " + Object.keys(packages).length + " packages with iotags");
    for (var name in packages) {
      var ps = pkgs.filter(function(p) {
        return p.name === name;
      });
      if (ps.length === 1) {
        var p = ps[0];
        p.iotags = packages[name];
      }
    }
    return pkgs;
  });
};

// crons the npm registry manually and checks for package updates
workflow.prototype.updateCronJob = function updateCronJob() {
  var self = this;
  this.pkgs.forEach(function(oldPkg) {
    return new npmClient(oldPkg.name + "@latest", self.npm).then(function(newPkg) {
      // we have only the latest version -> download entire package
      if (oldPkg.version != newPkg.version && newPkg.name != undefined) {
        log.info("new package uploaded: ", newPkg.name, newPkg.version, oldPkg.version);
        self.updatePkg(newPkg.name).then(function() {
          newPkg.updateVersionType = semver.diff(oldPkg.version, newPkg.version);
          newPkg.updateType = "update";
          self.trigger("pkg:updated", newPkg);
        });
      }
    }.bind(this));
  });
};

// crons the npm registry for new packages
workflow.prototype.searchCron = function searchCron() {
  var self = this;
  var pkgNames = this.pkgs.map(function(pkg) {
    return pkg.name;
  });
  return keywords(this.keys, this.npm).filter(function(pkg) {
    return pkgNames.indexOf(pkg.name) < 0;
  }).then(function(pkgs) {
    if (pkgs.length > 0) {
      pkgs.forEach(function(pkg) {
        log.info("new package found: ", pkg.name);
        self.updatePkg(pkg.name).then(function() {
          pkg.updateType = "new";
          self.trigger("pkg:new", pkg);
        });
      });
    }
  });
};

workflow.prototype.liveStreamCron = function searchCron() {
  var self = this;
  return new NpmPublishStream()
    .on('data', function(data) {
      var newPkg = data.doc;
      if (_.intersection(newPkg.keywords, self.keys).length > 0) {
        var oldPkg = self.pkgs[newPkg.name];
        if (oldPkg == undefined) {
          log.info("new package found: ", newPkg.name);
          self.updatePkg(newPkg.name);
        } else if (oldPkg.version != newPkg.version) {
          log.info("new package uploaded: ", newPkg.name, newPkg.version, oldPkg.version);
          self.updatePkg(newPkg.name);
        }
      }
    });
};

workflow.prototype.stop = function() {
  clearInterval(this.updateCronI);
  clearInterval(this.reloadCronI);
  clearInterval(this.searchCronI);
  if (this.liveStreamI) {
    this.liveStreamI.end();
  }
};

require("biojs-events").mixin(workflow.prototype);

var log;
module.exports = function(logger) {
  log = logger;
  return workflow;
};
<|endoftext|>


<commit_before>var q = require('bluebird');
var registry = require("npm-registry");
var _ = require('underscore');
var NpmPublishStream = require('npm-publish-stream');
var semver = require('semver');
var rp = require('request-promise');

var database = require("./lib/database.js");

// crawlers
var keywords = require("npm-keywords");
var ghClient = require("./lib/github.js");
var npmClient = require("./lib/npm.js");
var npmHistory = require("./lib/npmHistory.js");

// helpers
var postProcessor = require("./lib/util/postProcessor.js");

var workflow = function(opts) {
  this.keys = opts.keys || [];
  this.refreshTime = opts.refreshTime || 3600;
  this.searchTime = opts.searchTime || 120;
  this.updateInterval = opts.updateInterval || 60;
  this.iotagurl = opts.iotagurl;
  this.db = new database({
    log: log
  });
  this.dbLoad = this.db.load();
  this.pkgs = [];
  this.debug = opts.debug;
  this.npm = new registry({
    registry: opts.registryURL
  });
};

workflow.prototype.start = function() {
  var self = this;

  if (this.debug) {
    this.pkgs = [{
      name: "angularplasmid",
      version: "0.0.1"
    }, {
      name: "biojs-vis-msa",
      version: "0.0.1"
    }];
    this.updateCronJob();
    //this.searchCronI = setInterval(this.searchCron.bind(this), 5000);
    return q.resolve("a");
  }
  log.debug("starting downloader");
  return this.run().then(function() {
    this.reloadCronI = setInterval(this.run.bind(this), this.refreshTime * 1000);
    this.searchCronI = setInterval(this.searchCron.bind(this), this.searchTime * 1000);
    this.updateCronI = setInterval(this.updateCronJob.bind(this), this.updateInterval * 1000);
    // TODO: check whether it makes sense to enable this
    // the "live-stream" also uses polling
    //this.liveStreamI = this.liveStreamCron();
    return this.pkgs;
  }.bind(this));
};

// 1) download package list
workflow.prototype.run = function run() {
  var self = this;
  return keywords(this.keys, this.npm)
    .map(this.downloadPkg.bind(this))
    .then(function(pkgs) {
      // on succes:  save to DB
      log.info("workflow: trying to save into DB.");
      self.pkgs = pkgs.map(function(el) {
        return {
          name: el.name,
          version: el.version
        };
      });
      // post process the package list
      return self.loadIoTags(pkgs);

    }).then(function(pkgs) {
      // save in DB
      self.dbLoad
        .then(self.db.loadCache.bind(self.db))
        .then(function() {
          return self.db.save(pkgs);
        });
    }).then(function() {
      log.info("workflow: load complete.");
      return self.pkgs;
    });
};

// 2) crawl the package from NPM
workflow.prototype.downloadPkg = function(name) {
  // also support pkg object with name attribute
  if (name.name !== undefined) {
    name = name.name;
  }
  return new npmClient(name, this.npm).then(this.postDownload.bind(this));
};

// 3) other info
workflow.prototype.postDownload = function(pkg) {
  var postOpts = {
    keys: this.keys
  };
  var ps = [];
  var opts = {
    log: log
  };
  ps.push(new ghClient(pkg, opts));
  ps.push(new npmHistory(pkg, opts, this.npm));
  return q.settle(ps).then(function() {
    return postProcessor(pkg, postOpts);
  }).catch(function(err) {
    if (err != undefined && err.name !== "queryEvents") {
      // event errors can happen - ignore them
      log.warn("downloaderr", err);
    } else {
      log.info(arguments);
    }
    return pkg;
  });
};

// update a single pkg
workflow.prototype.updatePkg = function(pkg) {
  return this.downloadPkg(pkg).then(function(newPkg) {
    var index = _.indexOf(_.pluck(this.pkgs, "name"), newPkg.name);
    if (index >= 0) {
      this.pkgs[index] = newPkg;
    } else {
      this.pkgs.push(newPkg);
    }
    this.db.updatePkg(newPkg);
    log.info("package update saved: ", newPkg.name);
    return newPkg;
  }.bind(this)).catch(function(err) {
    log.warn("err during package update: ", pkg.name);
    log.warn(err);
  });
};

// download the tagged list and apply it
workflow.prototype.loadIoTags = function(pkgs) {
  return rp(this.iotagurl).then(function(raw) {
    var packages = JSON.parse(raw);
    log.warn("found " + Object.keys(packages).length + " packages with iotags");
    for (var name in packages) {
      var ps = pkgs.filter(function(p) {
        return p.name === name
      });
      if (ps.length == 1) {
        var p = ps[0];
        p.iotags = packages[name];
      }
    }
    return pkgs;
  });
};

// crons the npm registry manually and checks for package updates
workflow.prototype.updateCronJob = function updateCronJob() {
  var self = this;
  this.pkgs.forEach(function(oldPkg) {
    return new npmClient(oldPkg.name + "@latest", self.npm).then(function(newPkg) {
      // we have only the latest version -> download entire package
      if (oldPkg.version != newPkg.version && newPkg.name != undefined) {
        log.info("new package uploaded: ", newPkg.name, newPkg.version, oldPkg.version);
        self.updatePkg(newPkg.name).then(function() {
          newPkg.updateVersionType = semver.diff(oldPkg.version, newPkg.version);
          newPkg.updateType = "update";
          self.trigger("pkg:updated", newPkg);
        });
      }
    }.bind(this));
  });
};

// crons the npm registry for new packages
workflow.prototype.searchCron = function searchCron() {
  var self = this;
  var pkgNames = this.pkgs.map(function(pkg) {
    return pkg.name;
  });
  return keywords(this.keys, this.npm).filter(function(pkg) {
    return pkgNames.indexOf(pkg.name) < 0;
  }).then(function(pkgs) {
    if (pkgs.length > 0) {
      pkgs.forEach(function(pkg) {
        log.info("new package found: ", pkg.name);
        self.updatePkg(pkg.name).then(function() {
          pkg.updateType = "new";
          self.trigger("pkg:new", pkg);
        });
      });
    }
  });
};

workflow.prototype.liveStreamCron = function searchCron() {
  var self = this;
  return new NpmPublishStream()
    .on('data', function(data) {
      var newPkg = data.doc;
      if (_.intersection(newPkg.keywords, self.keys).length > 0) {
        var oldPkg = self.pkgs[newPkg.name];
        if (oldPkg == undefined) {
          log.info("new package found: ", newPkg.name);
          self.updatePkg(newPkg.name);
        } else if (oldPkg.version != newPkg.version) {
          log.info("new package uploaded: ", newPkg.name, newPkg.version, oldPkg.version);
          self.updatePkg(newPkg.name);
        }
      }
    });
};

workflow.prototype.stop = function() {
  clearInterval(this.updateCronI);
  clearInterval(this.reloadCronI);
  clearInterval(this.searchCronI);
  if (this.liveStreamI) {
    this.liveStreamI.end();
  }
};

require("biojs-events").mixin(workflow.prototype);

var log;
module.exports = function(logger) {
  log = logger;
  return workflow;
};
<commit_msg>fixed broken unit test - missing iotag url<commit_after>var q = require('bluebird');
var registry = require("npm-registry");
var _ = require('underscore');
var NpmPublishStream = require('npm-publish-stream');
var semver = require('semver');
var rp = require('request-promise');

var database = require("./lib/database.js");

// crawlers
var keywords = require("npm-keywords");
var ghClient = require("./lib/github.js");
var npmClient = require("./lib/npm.js");
var npmHistory = require("./lib/npmHistory.js");

// helpers
var postProcessor = require("./lib/util/postProcessor.js");

var workflow = function(opts) {
  this.keys = opts.keys || [];
  this.refreshTime = opts.refreshTime || 3600;
  this.searchTime = opts.searchTime || 120;
  this.updateInterval = opts.updateInterval || 60;
  this.iotagurl = opts.iotagurl;
  this.db = new database({
    log: log
  });
  this.dbLoad = this.db.load();
  this.pkgs = [];
  this.debug = opts.debug;
  this.npm = new registry({
    registry: opts.registryURL
  });
};

workflow.prototype.start = function() {
  var self = this;

  if (this.debug) {
    this.pkgs = [{
      name: "angularplasmid",
      version: "0.0.1"
    }, {
      name: "biojs-vis-msa",
      version: "0.0.1"
    }];
    this.updateCronJob();
    //this.searchCronI = setInterval(this.searchCron.bind(this), 5000);
    return q.resolve("a");
  }
  log.debug("starting downloader");
  return this.run().then(function() {
    this.reloadCronI = setInterval(this.run.bind(this), this.refreshTime * 1000);
    this.searchCronI = setInterval(this.searchCron.bind(this), this.searchTime * 1000);
    this.updateCronI = setInterval(this.updateCronJob.bind(this), this.updateInterval * 1000);
    // TODO: check whether it makes sense to enable this
    // the "live-stream" also uses polling
    //this.liveStreamI = this.liveStreamCron();
    return this.pkgs;
  }.bind(this));
};

// 1) download package list
workflow.prototype.run = function run() {
  var self = this;
  return keywords(this.keys, this.npm)
    .map(this.downloadPkg.bind(this))
    .then(function(pkgs) {
      // on succes:  save to DB
      log.info("workflow: trying to save into DB.");
      self.pkgs = pkgs.map(function(el) {
        return {
          name: el.name,
          version: el.version
        };
      });
      // post process the package list
      return self.loadIoTags(pkgs);

    }).then(function(pkgs) {
      // save in DB
      self.dbLoad
        .then(self.db.loadCache.bind(self.db))
        .then(function() {
          return self.db.save(pkgs);
        });
    }).then(function() {
      log.info("workflow: load complete.");
      return self.pkgs;
    });
};

// 2) crawl the package from NPM
workflow.prototype.downloadPkg = function(name) {
  // also support pkg object with name attribute
  if (name.name !== undefined) {
    name = name.name;
  }
  return new npmClient(name, this.npm).then(this.postDownload.bind(this));
};

// 3) other info
workflow.prototype.postDownload = function(pkg) {
  var postOpts = {
    keys: this.keys
  };
  var ps = [];
  var opts = {
    log: log
  };
  ps.push(new ghClient(pkg, opts));
  ps.push(new npmHistory(pkg, opts, this.npm));
  return q.settle(ps).then(function() {
    return postProcessor(pkg, postOpts);
  }).catch(function(err) {
    if (err != undefined && err.name !== "queryEvents") {
      // event errors can happen - ignore them
      log.warn("downloaderr", err);
    } else {
      log.info(arguments);
    }
    return pkg;
  });
};

// update a single pkg
workflow.prototype.updatePkg = function(pkg) {
  return this.downloadPkg(pkg).then(function(newPkg) {
    var index = _.indexOf(_.pluck(this.pkgs, "name"), newPkg.name);
    if (index >= 0) {
      this.pkgs[index] = newPkg;
    } else {
      this.pkgs.push(newPkg);
    }
    this.db.updatePkg(newPkg);
    log.info("package update saved: ", newPkg.name);
    return newPkg;
  }.bind(this)).catch(function(err) {
    log.warn("err during package update: ", pkg.name);
    log.warn(err);
  });
};

// download the tagged list and apply it
workflow.prototype.loadIoTags = function(pkgs) {
  if (!this.iotagurl) {
    return q.resolve(pkgs);
  }
  return rp(this.iotagurl).then(function(raw) {
    var packages = JSON.parse(raw);
    log.warn("found " + Object.keys(packages).length + " packages with iotags");
    for (var name in packages) {
      var ps = pkgs.filter(function(p) {
        return p.name === name
      });
      if (ps.length == 1) {
        var p = ps[0];
        p.iotags = packages[name];
      }
    }
    return pkgs;
  });
};

// crons the npm registry manually and checks for package updates
workflow.prototype.updateCronJob = function updateCronJob() {
  var self = this;
  this.pkgs.forEach(function(oldPkg) {
    return new npmClient(oldPkg.name + "@latest", self.npm).then(function(newPkg) {
      // we have only the latest version -> download entire package
      if (oldPkg.version != newPkg.version && newPkg.name != undefined) {
        log.info("new package uploaded: ", newPkg.name, newPkg.version, oldPkg.version);
        self.updatePkg(newPkg.name).then(function() {
          newPkg.updateVersionType = semver.diff(oldPkg.version, newPkg.version);
          newPkg.updateType = "update";
          self.trigger("pkg:updated", newPkg);
        });
      }
    }.bind(this));
  });
};

// crons the npm registry for new packages
workflow.prototype.searchCron = function searchCron() {
  var self = this;
  var pkgNames = this.pkgs.map(function(pkg) {
    return pkg.name;
  });
  return keywords(this.keys, this.npm).filter(function(pkg) {
    return pkgNames.indexOf(pkg.name) < 0;
  }).then(function(pkgs) {
    if (pkgs.length > 0) {
      pkgs.forEach(function(pkg) {
        log.info("new package found: ", pkg.name);
        self.updatePkg(pkg.name).then(function() {
          pkg.updateType = "new";
          self.trigger("pkg:new", pkg);
        });
      });
    }
  });
};

workflow.prototype.liveStreamCron = function searchCron() {
  var self = this;
  return new NpmPublishStream()
    .on('data', function(data) {
      var newPkg = data.doc;
      if (_.intersection(newPkg.keywords, self.keys).length > 0) {
        var oldPkg = self.pkgs[newPkg.name];
        if (oldPkg == undefined) {
          log.info("new package found: ", newPkg.name);
          self.updatePkg(newPkg.name);
        } else if (oldPkg.version != newPkg.version) {
          log.info("new package uploaded: ", newPkg.name, newPkg.version, oldPkg.version);
          self.updatePkg(newPkg.name);
        }
      }
    });
};

workflow.prototype.stop = function() {
  clearInterval(this.updateCronI);
  clearInterval(this.reloadCronI);
  clearInterval(this.searchCronI);
  if (this.liveStreamI) {
    this.liveStreamI.end();
  }
};

require("biojs-events").mixin(workflow.prototype);

var log;
module.exports = function(logger) {
  log = logger;
  return workflow;
};
<|endoftext|>


<commit_before>const winston = require('winston');

module.exports = ({
  level,
  transports: transports = [
    {
      type: 'Console',
      options: {
        timestamp: true,
        colorize: process.env.NODE_ENV !== 'production',
        prettyPrint: process.env.NODE_ENV !== 'production',
        json: process.env.NODE_ENV === 'production',
        stringify: obj => JSON.stringify(obj),
        silent: process.env.NODE_ENV === 'test',
        level: 'debug',
      },
    },
  ],
}) => {
  if (level) {
    for (const transport of transports) {
      transport.options = transport.options || {};
      transport.options.level = level;
    }
  }
  return (category, options) => {
    const logger = new winston.Logger(options);
    for (const transport of transports) {
      logger.add(
        winston.transports[transport.type],
        Object.assign({}, transport.options, { label: category })
      );
    }
    return logger;
  };
};
<commit_msg>Set default log level to 'info' in winston<commit_after>const winston = require('winston');

module.exports = ({
  level,
  transports: transports = [
    {
      type: 'Console',
      options: {
        timestamp: true,
        colorize: process.env.NODE_ENV !== 'production',
        prettyPrint: process.env.NODE_ENV !== 'production',
        json: process.env.NODE_ENV === 'production',
        stringify: obj => JSON.stringify(obj),
        silent: process.env.NODE_ENV === 'test',
        level: 'info',
      },
    },
  ],
}) => {
  if (level) {
    for (const transport of transports) {
      transport.options = transport.options || {};
      transport.options.level = level;
    }
  }
  return (category, options) => {
    const logger = new winston.Logger(options);
    for (const transport of transports) {
      logger.add(
        winston.transports[transport.type],
        Object.assign({}, transport.options, { label: category })
      );
    }
    return logger;
  };
};
<|endoftext|>


<commit_before>/**
 * Message Proxy scenario. Send the message to the user defined
 * serviceURL address.
 */

var SUPPORT_LIBS = '../lib/support/';
var logger = require('../lib/logger');
var soapErrorMsg = require('../lib/util/errormsg');

exports.executeSample = function(zenithMessage, callback){
	var reqUrl = zenithMessage.transportHeaders.url; //returns url object

	if(reqUrl.pathname === '/services/StockQuoteProxy'){

		var serviceURL = 'http://localhost:9000/services/SimpleStockQuoteService';
		logger.debug('SampleConfig', 'EPR: ' + serviceURL);

		var option = {
				url : serviceURL
			};

		var endpoint = require(SUPPORT_LIBS + 'ws_endpoint');

		endpoint.callService(zenithMessage, option, function(err,message){
				callback(null, message);
			});

	} else {
		var errMsg = 'Invalid EPR value.';
		zenithMessage.body = soapErrorMsg.getSOAP11Fault(errMsg);
		callback(null, zenithMessage);

	}

}<commit_msg>soaperrorMsg commented<commit_after>/**
 * Message Proxy scenario. Send the message to the user defined
 * serviceURL address.
 */

var SUPPORT_LIBS = '../lib/support/';
var logger = require('../lib/logger');
//var soapErrorMsg = require('../lib/util/errormsg');

exports.executeSample = function(zenithMessage, callback){
	var reqUrl = zenithMessage.transportHeaders.url; //returns url object

	if(reqUrl.pathname === '/services/StockQuoteProxy'){

		var serviceURL = 'http://localhost:9000/services/SimpleStockQuoteService';
		logger.debug('SampleConfig', 'EPR: ' + serviceURL);

		var option = {
				url : serviceURL
			};

		var endpoint = require(SUPPORT_LIBS + 'ws_endpoint');

		endpoint.callService(zenithMessage, option, function(err,message){
				callback(null, message);
			});

	} else {
		var errMsg = 'Invalid EPR value.';
		zenithMessage.body = soapErrorMsg.getSOAP11Fault(errMsg);
		callback(null, zenithMessage);

	}

}<|endoftext|>


<commit_before>/**
 * Created by Benjamin on 07.10.2016.
 */
var exec = require('child-process-promise').exec;

predict(0, 0, 0);

function predict(lat, lon, timestamp) {
    var wekaCmd = 'java -classpath ./data/weka.jar'; // add more RAM by option: -Xmx1024m
    var classifier = 'weka.classifiers.bayes.NaiveBayes';
    // no-cv: no cross-validation, v: no training data statistics
    var trainingOptions = '-no-cv -v';
    // p <range>: print predictions and attribute values in range. if range = 0 print no attributes
    // specify the output format of the predictions. Use CSV for easier parsing and PlainText for pretty print.
    var testOptions = '-classifications "weka.classifiers.evaluation.output.prediction.CSV"';
    var modelName = 'data/classifier.model';
    var trainingData = 'data/trainingData.arff';
    var testData = 'data/testData.arff';

    /**
     * train model
     * java -classpath ./data/weka.jar -Xmx1024m weka.classifiers.bayes.NaiveBayes -t weather.arff -d weather-NB.model
     * @return {Array|{index: number, input: string}|*|Promise}
     */
    function trainModel() {
        var cmd = wekaCmd + ' ' + classifier + ' ' + trainingOptions
            + ' -t ' + trainingData     // training data to be used to create the model
            + ' -d ' + modelName;       // store the trained model with the specified name
        return exec(cmd);
    }

    /**
     * test model
     * java -classpath ./data/weka.jar weka.classifiers.bayes.NaiveBayes -l weather-NB.model -T weather-test.arff
     * @return {Array|{index: number, input: string}|*|Promise}
     */
    function test() {
        var cmd = wekaCmd + ' ' + classifier + ' ' + testOptions
            + ' -l ' + modelName        // load the trained model with the specified name
            + ' -T ' + testData;        // test data to be evaluated by the trained model
        return exec(cmd);
    }

    trainModel()
        .then(function (result) {
            console.log('-- training completed');
            // console.log('-- train stderr: ' + result.stderr);
            // console.log('-- train stdout: ' + result.stdout);
            return test();
        })
        .then(function (result) {
            console.log('-- test completed');
            // console.log('-- test stderr: ' + result.stderr);
            console.log('-- test stdout: ' + result.stdout);
        })
        .catch(function (err) {
            console.error('ERROR: ', err);
        });

    // call test only if the model was already trained
    // test()
    //     .then(function (result) {
    //         console.log('-- test completed');
    //         console.log('-- test stderr: ' + result.stderr);
    //         console.log('-- test stdout: ' + result.stdout);
    //     })
    //     .catch(function (err) {
    //         console.error('ERROR: ', err);
    //     });
}<commit_msg>added weka output parsing method<commit_after>/**
 * Created by Benjamin on 07.10.2016.
 */
var exec = require('child-process-promise').exec;

predict(0, 0, 0);

function predict(lat, lon, timestamp) {
    var wekaCmd = 'java -classpath ./data/weka.jar'; // add more RAM by option: -Xmx1024m
    var classifier = 'weka.classifiers.bayes.NaiveBayes';
    // no-cv: no cross-validation, v: no training data statistics
    var trainingOptions = '-no-cv -v';
    // p <range>: print predictions and attribute values in range. if range = 0 print no attributes
    // specify the output format of the predictions. Use CSV for easier parsing and PlainText for pretty print.
    var testOptions = '-classifications "weka.classifiers.evaluation.output.prediction.CSV"';
    var modelName = 'data/classifier.model';
    var trainingData = 'data/trainingData.arff';
    var testData = 'data/testData.arff';

    /**
     * train model
     * java -classpath ./data/weka.jar -Xmx1024m weka.classifiers.bayes.NaiveBayes -t weather.arff -d weather-NB.model
     * @return {Array|{index: number, input: string}|*|Promise}
     */
    function trainModel() {
        var cmd = wekaCmd + ' ' + classifier + ' ' + trainingOptions
            + ' -t ' + trainingData     // training data to be used to create the model
            + ' -d ' + modelName;       // store the trained model with the specified name
        return exec(cmd);
    }

    /**
     * test model
     * java -classpath ./data/weka.jar weka.classifiers.bayes.NaiveBayes -l weather-NB.model -T weather-test.arff
     * @return {Array|{index: number, input: string}|*|Promise}
     */
    function test() {
        var cmd = wekaCmd + ' ' + classifier + ' ' + testOptions
            + ' -l ' + modelName        // load the trained model with the specified name
            + ' -T ' + testData;        // test data to be evaluated by the trained model
        return exec(cmd);
    }

    /**
     * parse the weka output and return an array of the predicted class label and the corresponding confidence
     * for all entries in the tested data set
     * @param wekaOutput
     * @return {Array} array of prediction objects which contain the class label and the corresponding confidence
     */
    function parsePredictionOutput(wekaOutput) {
        var predictions = [];

        var lines = wekaOutput.split('\n');
        if (lines.length > 5) {
            lines.slice(5, -2).forEach(function (line) {
                var components = line.split(',');
                predictions.push({
                    "classLabel": components[2].split(':')[1],
                    "confidence": components[4]
                });
            });
        }

        return predictions;
    }

    trainModel()
        .then(function (result) {
            console.log('-- training completed');
            // console.log('-- train stderr: ' + result.stderr);
            // console.log('-- train stdout: ' + result.stdout);
            return test();
        })
        .then(function (result) {
            console.log('-- test completed');
            // console.log('-- test stderr: ' + result.stderr);
            console.log('-- test stdout: ' + result.stdout);
            var predictions = parsePredictionOutput(result.stdout);
            predictions.forEach(function (prediction) {
                console.log('class: ' + prediction.classLabel + ', confidence: ' +prediction.confidence);
            })
        })
        .catch(function (err) {
            console.error('ERROR: ', err);
        });

    // call test only if the model was already trained
    // test()
    //     .then(function (result) {
    //         console.log('-- test completed');
    //         console.log('-- test stderr: ' + result.stderr);
    //         console.log('-- test stdout: ' + result.stdout);
    //     })
    //     .catch(function (err) {
    //         console.error('ERROR: ', err);
    //     });
}<|endoftext|>