<commit_before>
function calendarHeatmap() {
  // defaults
  var width = 750;
  var height = 110;
  var legendWidth = 150;
  var selector = 'body';
  var SQUARE_LENGTH = 11;
  var SQUARE_PADDING = 2;
  var MONTH_LABEL_PADDING = 6;
  var now = moment().endOf('day').toDate();
  var yearAgo = moment().startOf('day').subtract(1, 'year').toDate();
  var startDate = null;
  var data = [];
  var max = null;
  var colorRange = ['#D8E6E7', '#218380'];
  var tooltipEnabled = true;
  var tooltipUnit = 'contribution';
  var legendEnabled = true;
  var onClick = null;
  var weekStart = 0; //0 for Sunday, 1 for Monday
  var locale = {
    months: ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'],
    days: ['S', 'M', 'T', 'W', 'T', 'F', 'S'],
    No: 'No',
    on: 'on',
    Less: 'Less',
    More: 'More'
  };

  // setters and getters
  chart.data = function (value) {
    if (!arguments.length) { return data; }
    data = value;
    return chart;
  };

  chart.max = function (value) {
    if (!arguments.length) { return max; }
    max = value;
    return chart;
  };

  chart.selector = function (value) {
    if (!arguments.length) { return selector; }
    selector = value;
    return chart;
  };

  chart.startDate = function (value) {
    if (!arguments.length) { return startDate; }
    yearAgo = value;
    now = moment(value).endOf('day').add(1, 'year').toDate();
    return chart;
  };

  chart.colorRange = function (value) {
    if (!arguments.length) { return colorRange; }
    colorRange = value;
    return chart;
  };

  chart.tooltipEnabled = function (value) {
    if (!arguments.length) { return tooltipEnabled; }
    tooltipEnabled = value;
    return chart;
  };

  chart.tooltipUnit = function (value) {
    if (!arguments.length) { return tooltipUnit; }
    tooltipUnit = value;
    return chart;
  };

  chart.legendEnabled = function (value) {
    if (!arguments.length) { return legendEnabled; }
    legendEnabled = value;
    return chart;
  };

  chart.onClick = function (value) {
    if (!arguments.length) { return onClick(); }
    onClick = value;
    return chart;
  };

  chart.locale = function (value) {
    if (!arguments.length) { return locale; }
    locale = value;
    return chart;
  };

  function chart() {

    d3.select(chart.selector()).selectAll('svg.calendar-heatmap').remove(); // remove the existing chart, if it exists

    var dateRange = ((d3.time && d3.time.days) || d3.timeDays)(yearAgo, now); // generates an array of date objects within the specified range
    var monthRange = ((d3.time && d3.time.months) || d3.timeMonths)(moment(yearAgo).startOf('month').toDate(), now); // it ignores the first month if the 1st date is after the start of the month
    var firstDate = moment(dateRange[0]);
    if (chart.data().length == 0) {
      max = 0;
    } else if (max === null) {
      max = d3.max(chart.data(), function (d) { return d.count; }); // max data value
    }

    // color range
    var color = ((d3.scale && d3.scale.linear) || d3.scaleLinear)()
      .range(chart.colorRange())
      .domain([0, max]);

    var tooltip;
    var dayRects;

    drawChart();

    function drawChart() {
      var svg = d3.select(chart.selector())
        .style('position', 'relative')
        .append('svg')
        .attr('width', width)
        .attr('class', 'calendar-heatmap')
        .attr('height', height)
        .style('padding', '36px');

      dayRects = svg.selectAll('.day-cell')
        .data(dateRange);  //  array of days for the last yr

      dayRects.enter().append('rect')
        .attr('class', 'day-cell')
        .attr('width', SQUARE_LENGTH)
        .attr('height', SQUARE_LENGTH)
        .attr('fill', function(d) { return color(countForDate(d)); })
        .attr('x', function (d, i) {
          var cellDate = moment(d);
          var result = cellDate.week() - firstDate.week() + (firstDate.weeksInYear() * (cellDate.weekYear() - firstDate.weekYear()));
          return result * (SQUARE_LENGTH + SQUARE_PADDING);
        })
        .attr('y', function (d, i) {
          return MONTH_LABEL_PADDING + formatWeekday(d.getDay()) * (SQUARE_LENGTH + SQUARE_PADDING);
        });

      if (typeof onClick === 'function') {
        dayRects.on('click', function (d) {
          var count = countForDate(d);
          onClick({ date: d, count: count});
        });
      }

      if (chart.tooltipEnabled()) {
        dayRects.on('mouseover', function (d, i) {
          tooltip = d3.select(chart.selector())
            .append('div')
            .attr('class', 'day-cell-tooltip')
            .html(tooltipHTMLForDate(d))
            .style('left', function () { return Math.floor(i / 7) * SQUARE_LENGTH + 'px'; })
            .style('top', function () {
              return formatWeekday(d.getDay()) * (SQUARE_LENGTH + SQUARE_PADDING) + MONTH_LABEL_PADDING * 2 + 'px';
            });
        })
        .on('mouseout', function (d, i) {
          tooltip.remove();
        });
      }

      if (chart.legendEnabled()) {
        var colorRange = [color(0)];
        for (var i = 3; i > 0; i--) {
          colorRange.push(color(max / i));
        }

        var legendGroup = svg.append('g');
        legendGroup.selectAll('.calendar-heatmap-legend')
            .data(colorRange)
            .enter()
          .append('rect')
            .attr('class', 'calendar-heatmap-legend')
            .attr('width', SQUARE_LENGTH)
            .attr('height', SQUARE_LENGTH)
            .attr('x', function (d, i) { return (width - legendWidth) + (i + 1) * 13; })
            .attr('y', height + SQUARE_PADDING)
            .attr('fill', function (d) { return d; });

        legendGroup.append('text')
          .attr('class', 'calendar-heatmap-legend-text calendar-heatmap-legend-text-less')
          .attr('x', width - legendWidth - 13)
          .attr('y', height + SQUARE_LENGTH)
          .text(locale.Less);

        legendGroup.append('text')
          .attr('class', 'calendar-heatmap-legend-text calendar-heatmap-legend-text-more')
          .attr('x', (width - legendWidth + SQUARE_PADDING) + (colorRange.length + 1) * 13)
          .attr('y', height + SQUARE_LENGTH)
          .text(locale.More);
      }

      dayRects.exit().remove();
      var monthLabels = svg.selectAll('.month')
          .data(monthRange)
          .enter().append('text')
          .attr('class', 'month-name')
          .text(function (d) {
            return locale.months[d.getMonth()];
          })
          .attr('x', function (d, i) {
            var matchIndex = 0;
            dateRange.find(function (element, index) {
              matchIndex = index;
              return moment(d).isSame(element, 'month') && moment(d).isSame(element, 'year');
            });

            return Math.floor(matchIndex / 7) * (SQUARE_LENGTH + SQUARE_PADDING);
          })
          .attr('y', 0);  // fix these to the top

      locale.days.forEach(function (day, index) {
        index = formatWeekday(index);
        if (index % 2) {
          svg.append('text')
            .attr('class', 'day-initial')
            .attr('transform', 'translate(-8,' + (SQUARE_LENGTH + SQUARE_PADDING) * (index + 1) + ')')
            .style('text-anchor', 'middle')
            .attr('dy', '2')
            .text(day);
        }
      });
    }

    function pluralizedTooltipUnit (count) {
      if ('string' === typeof tooltipUnit) {
        return (tooltipUnit + (count === 1 ? '' : 's'));
      }
      for (var i in tooltipUnit) {
        var _rule = tooltipUnit[i];
        var _min = _rule.min;
        var _max = _rule.max || _rule.min;
        _max = _max === 'Infinity' ? Infinity : _max;
        if (count >= _min && count <= _max) {
          return _rule.unit;
        }
      }
    }

    function tooltipHTMLForDate(d) {
      var dateStr = moment(d).format('ddd, MMM Do YYYY');
      var count = countForDate(d);
      return '' + (count ? count : locale.No) + ' ' + pluralizedTooltipUnit(count) + ' ' + locale.on + ' ' + dateStr + '';
    }

    function countForDate(d) {
      var count = 0;
      var match = chart.data().find(function (element, index) {
        return moment(element.date).isSame(d, 'day');
      });
      if (match) {
        count = match.count;
      }
      return count;
    }

    function formatWeekday(weekDay) {
      if (weekStart === 1) {
        if (weekDay === 0) {
          return 6;
        } else {
          return weekDay - 1;
        }
      }
      return weekDay;
    }

    var daysOfChart = chart.data().map(function (day) {
      return day.date.toDateString();
    });

  }

  return chart;
}

// polyfill for Array.find() method
/* jshint ignore:start */
if (!Array.prototype.find) {
  Array.prototype.find = function (predicate) {
    if (this === null) {
      throw new TypeError('Array.prototype.find called on null or undefined');
    }
    if (typeof predicate !== 'function') {
      throw new TypeError('predicate must be a function');
    }
    var list = Object(this);
    var length = list.length >>> 0;
    var thisArg = arguments[1];
    var value;

    for (var i = 0; i < length; i++) {
      value = list[i];
      if (predicate.call(thisArg, value, i, list)) {
        return value;
      }
    }
    return undefined;
  };
}
/* jshint ignore:end */
<commit_msg>Fix #34<commit_after>
function calendarHeatmap() {
  // defaults
  var width = 750;
  var height = 110;
  var legendWidth = 150;
  var selector = 'body';
  var SQUARE_LENGTH = 11;
  var SQUARE_PADDING = 2;
  var MONTH_LABEL_PADDING = 6;
  var now = moment().endOf('day').toDate();
  var yearAgo = moment().startOf('day').subtract(1, 'year').toDate();
  var startDate = null;
  var data = [];
  var max = null;
  var colorRange = ['#D8E6E7', '#218380'];
  var tooltipEnabled = true;
  var tooltipUnit = 'contribution';
  var legendEnabled = true;
  var onClick = null;
  var weekStart = 0; //0 for Sunday, 1 for Monday
  var locale = {
    months: ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'],
    days: ['S', 'M', 'T', 'W', 'T', 'F', 'S'],
    No: 'No',
    on: 'on',
    Less: 'Less',
    More: 'More'
  };
  var v = Number(d3.version.split('.')[0]);

  // setters and getters
  chart.data = function (value) {
    if (!arguments.length) { return data; }
    data = value;
    return chart;
  };

  chart.max = function (value) {
    if (!arguments.length) { return max; }
    max = value;
    return chart;
  };

  chart.selector = function (value) {
    if (!arguments.length) { return selector; }
    selector = value;
    return chart;
  };

  chart.startDate = function (value) {
    if (!arguments.length) { return startDate; }
    yearAgo = value;
    now = moment(value).endOf('day').add(1, 'year').toDate();
    return chart;
  };

  chart.colorRange = function (value) {
    if (!arguments.length) { return colorRange; }
    colorRange = value;
    return chart;
  };

  chart.tooltipEnabled = function (value) {
    if (!arguments.length) { return tooltipEnabled; }
    tooltipEnabled = value;
    return chart;
  };

  chart.tooltipUnit = function (value) {
    if (!arguments.length) { return tooltipUnit; }
    tooltipUnit = value;
    return chart;
  };

  chart.legendEnabled = function (value) {
    if (!arguments.length) { return legendEnabled; }
    legendEnabled = value;
    return chart;
  };

  chart.onClick = function (value) {
    if (!arguments.length) { return onClick(); }
    onClick = value;
    return chart;
  };

  chart.locale = function (value) {
    if (!arguments.length) { return locale; }
    locale = value;
    return chart;
  };

  function chart() {

    d3.select(chart.selector()).selectAll('svg.calendar-heatmap').remove(); // remove the existing chart, if it exists

    var dateRange = ((d3.time && d3.time.days) || d3.timeDays)(yearAgo, now); // generates an array of date objects within the specified range
    var monthRange = ((d3.time && d3.time.months) || d3.timeMonths)(moment(yearAgo).startOf('month').toDate(), now); // it ignores the first month if the 1st date is after the start of the month
    var firstDate = moment(dateRange[0]);
    if (chart.data().length == 0) {
      max = 0;
    } else if (max === null) {
      max = d3.max(chart.data(), function (d) { return d.count; }); // max data value
    }

    // color range
    var color = ((d3.scale && d3.scale.linear) || d3.scaleLinear)()
      .range(chart.colorRange())
      .domain([0, max]);

    var tooltip;
    var dayRects;

    drawChart();

    function drawChart() {
      var svg = d3.select(chart.selector())
        .style('position', 'relative')
        .append('svg')
        .attr('width', width)
        .attr('class', 'calendar-heatmap')
        .attr('height', height)
        .style('padding', '36px');

      dayRects = svg.selectAll('.day-cell')
        .data(dateRange);  //  array of days for the last yr

      var enterSelection = dayRects.enter().append('rect')
        .attr('class', 'day-cell')
        .attr('width', SQUARE_LENGTH)
        .attr('height', SQUARE_LENGTH)
        .attr('fill', function(d) { return color(countForDate(d)); })
        .attr('x', function (d, i) {
          var cellDate = moment(d);
          var result = cellDate.week() - firstDate.week() + (firstDate.weeksInYear() * (cellDate.weekYear() - firstDate.weekYear()));
          return result * (SQUARE_LENGTH + SQUARE_PADDING);
        })
        .attr('y', function (d, i) {
          return MONTH_LABEL_PADDING + formatWeekday(d.getDay()) * (SQUARE_LENGTH + SQUARE_PADDING);
        });

      if (typeof onClick === 'function') {
        (v === 3 ? enterSelection : enterSelection.merge(dayRects)).on('click', function(d) {
          var count = countForDate(d);
          onClick({ date: d, count: count});
        });
      }

      if (chart.tooltipEnabled()) {
        (v === 3 ? enterSelection : enterSelection.merge(dayRects)).on('mouseover', function(d, i) {
          tooltip = d3.select(chart.selector())
            .append('div')
            .attr('class', 'day-cell-tooltip')
            .html(tooltipHTMLForDate(d))
            .style('left', function () { return Math.floor(i / 7) * SQUARE_LENGTH + 'px'; })
            .style('top', function () {
              return formatWeekday(d.getDay()) * (SQUARE_LENGTH + SQUARE_PADDING) + MONTH_LABEL_PADDING * 2 + 'px';
            });
        })
        .on('mouseout', function (d, i) {
          tooltip.remove();
        });
      }

      if (chart.legendEnabled()) {
        var colorRange = [color(0)];
        for (var i = 3; i > 0; i--) {
          colorRange.push(color(max / i));
        }

        var legendGroup = svg.append('g');
        legendGroup.selectAll('.calendar-heatmap-legend')
            .data(colorRange)
            .enter()
          .append('rect')
            .attr('class', 'calendar-heatmap-legend')
            .attr('width', SQUARE_LENGTH)
            .attr('height', SQUARE_LENGTH)
            .attr('x', function (d, i) { return (width - legendWidth) + (i + 1) * 13; })
            .attr('y', height + SQUARE_PADDING)
            .attr('fill', function (d) { return d; });

        legendGroup.append('text')
          .attr('class', 'calendar-heatmap-legend-text calendar-heatmap-legend-text-less')
          .attr('x', width - legendWidth - 13)
          .attr('y', height + SQUARE_LENGTH)
          .text(locale.Less);

        legendGroup.append('text')
          .attr('class', 'calendar-heatmap-legend-text calendar-heatmap-legend-text-more')
          .attr('x', (width - legendWidth + SQUARE_PADDING) + (colorRange.length + 1) * 13)
          .attr('y', height + SQUARE_LENGTH)
          .text(locale.More);
      }

      dayRects.exit().remove();
      var monthLabels = svg.selectAll('.month')
          .data(monthRange)
          .enter().append('text')
          .attr('class', 'month-name')
          .text(function (d) {
            return locale.months[d.getMonth()];
          })
          .attr('x', function (d, i) {
            var matchIndex = 0;
            dateRange.find(function (element, index) {
              matchIndex = index;
              return moment(d).isSame(element, 'month') && moment(d).isSame(element, 'year');
            });

            return Math.floor(matchIndex / 7) * (SQUARE_LENGTH + SQUARE_PADDING);
          })
          .attr('y', 0);  // fix these to the top

      locale.days.forEach(function (day, index) {
        index = formatWeekday(index);
        if (index % 2) {
          svg.append('text')
            .attr('class', 'day-initial')
            .attr('transform', 'translate(-8,' + (SQUARE_LENGTH + SQUARE_PADDING) * (index + 1) + ')')
            .style('text-anchor', 'middle')
            .attr('dy', '2')
            .text(day);
        }
      });
    }

    function pluralizedTooltipUnit (count) {
      if ('string' === typeof tooltipUnit) {
        return (tooltipUnit + (count === 1 ? '' : 's'));
      }
      for (var i in tooltipUnit) {
        var _rule = tooltipUnit[i];
        var _min = _rule.min;
        var _max = _rule.max || _rule.min;
        _max = _max === 'Infinity' ? Infinity : _max;
        if (count >= _min && count <= _max) {
          return _rule.unit;
        }
      }
    }

    function tooltipHTMLForDate(d) {
      var dateStr = moment(d).format('ddd, MMM Do YYYY');
      var count = countForDate(d);
      return '' + (count ? count : locale.No) + ' ' + pluralizedTooltipUnit(count) + ' ' + locale.on + ' ' + dateStr + '';
    }

    function countForDate(d) {
      var count = 0;
      var match = chart.data().find(function (element, index) {
        return moment(element.date).isSame(d, 'day');
      });
      if (match) {
        count = match.count;
      }
      return count;
    }

    function formatWeekday(weekDay) {
      if (weekStart === 1) {
        if (weekDay === 0) {
          return 6;
        } else {
          return weekDay - 1;
        }
      }
      return weekDay;
    }

    var daysOfChart = chart.data().map(function (day) {
      return day.date.toDateString();
    });

  }

  return chart;
}

// polyfill for Array.find() method
/* jshint ignore:start */
if (!Array.prototype.find) {
  Array.prototype.find = function (predicate) {
    if (this === null) {
      throw new TypeError('Array.prototype.find called on null or undefined');
    }
    if (typeof predicate !== 'function') {
      throw new TypeError('predicate must be a function');
    }
    var list = Object(this);
    var length = list.length >>> 0;
    var thisArg = arguments[1];
    var value;

    for (var i = 0; i < length; i++) {
      value = list[i];
      if (predicate.call(thisArg, value, i, list)) {
        return value;
      }
    }
    return undefined;
  };
}
/* jshint ignore:end */
<|endoftext|>


<commit_before>"use strict";

var request = require("request");
var cheerio = require("cheerio");
var moment  = require("moment");
var mongoose = require("mongoose");
var async = require("async");

// just a static test url, later the client will send a week and a year
var url = "https://lsf.hft-stuttgart.de/qisserver/rds?state=wplan&k_abstgv.abstgvnr=262&week=12_2015&act=stg&pool=stg&show=plan&P.vx=lang&P.Print=";

request(url, function(error, response, html) {
  if(!error) {
    parse(html);
  }
});

var parse = function(html) {
  var table = null;
  var days = [];
  var timeTableGrid = [];
  var lectures = [];
  var $ = cheerio.load(html);

  // load days before we cut of td.plan_rahmen stuff
  days = getDaysInThisWeek($);

  // remove td.plan rahmen from the table, this removes a lot of trouble
  table = $("th.plan_rahmen").closest("table");
  table.find(".plan_rahmen").remove();

  // lets parse
  // get all the rows of the timetable (except the header (days) row)
  var rows = table.children("tr:not(:first-child)");

  // create and prefill the grid, which holds our whole timetable
  for (var i = 0; i < rows.length; i++) {     // rows
    timeTableGrid[i] = new Array(days.length);
    for (var j = 0; j < days.length; j++) {   // columns
      timeTableGrid[i][j] = ".";
    }
  }

  // iterate over each row and each cell
  rows.each(function(i) {
    var cells = $(this).children("td:not(:first-child)");

    // it seems like cheerio doesnt support collection.get(), so we just
    // bottle that cheerio collection into a native array.
    var cellsArray = [];
    cells.each(function(j) {
      cellsArray.push($(this));
    });

    for (var j = 0; j < days.length; j++) {
      if (timeTableGrid[i][j] === ".") { // here must be a td-(lecture)-element

        var currCell = cellsArray[0];
        cellsArray.shift(); // removes the first element, its a pseudo queue

        if (currCell.attr("class").indexOf("plan1") > -1) {
          // no lecture here
        } else if (currCell.attr("class").indexOf("plan2") > -1) { // here we have a lecture
          // now mark the whole rowspan down as the same lecture
          var rowspan = currCell.attr("rowspan");
          for (var k = 0; k < rowspan; k++) {
            timeTableGrid[i+k][j] = j;
          }

          /******** PARSE THE LECURE TD *********/
          var moreLectures = parseLecturesFromHtml(currCell, days, j);
          Array.prototype.push.apply(lectures, moreLectures); // join two arrays, the js way ¯\_(ツ)_/¯
        }
      }
    }
  });

  // connect to mongodb
  mongoose.connect("mongodb://localhost:27017/wuw");

  // create models from our schemas
  var Lecture = require("./model_lecture");
  var Deadline = require("./model_deadline");

  // drop current lecture collection to get a fresh result
  mongoose.connection.collections.lectures.drop(function(err) {
    if(err) console.log(err);

    // push every lecture to our db
    async.each(lectures, function(lecture, cb) {
      // create Lecture from our Model
      var Lec = new Lecture();
      // set attributes
      Lec.date = lecture.start;
      Lec.fullLectureName = lecture.lsfName;
      Lec.shortLectureName = lecture.shortName;
      Lec.room = lecture.lsfRoom;
      Lec.startTime = lecture.start;
      Lec.endTime = lecture.end;
      Lec.group = lecture.group;
      Lec.hashCode = lecture.hashCode;
      // save lecture to db
      Lec.save(cb);
    }, function(err) {
      if (err) console.log(err);
      mongoose.disconnect();

      // output lectures
      console.log(JSON.stringify(lectures, null, 2));
      console.log("Success! The parser inserted " + lectures.length + " lectures in the database");
    });
  });
};

var parseLecturesFromHtml = function(html, days, dayPos) {
  var lectures = [];
  var $ = cheerio.load(html);

  // there could be more then one groups in this lecture,
  // we create a lecture for every group
  html.find("table").each(function(i) {
    var time = trimProperty($(this).find("td.notiz").first().text());
    var date = days[dayPos];

    var lecture = {};
    lecture.lsfName = $(this).find("a").first().attr("title");
    lecture.start = parseStartEnd(date, time).start;
    lecture.end = parseStartEnd(date, time).end;
    lecture.lsfDate = date;
    lecture.lsfTime = time;
    lecture.lsfRoom = $(this).find("td.notiz a").first().text();
    lecture.group = parseGroup(lecture.lsfName);
    lecture.shortName = parseShortName(lecture.lsfName);
    lecture.hashCode = hashCode(lecture.lsfName+lecture.lsfDate+lecture.lsfTime+lecture.lsfRoom);

    lectures.push(lecture);
  });
  return lectures;
};

var parseGroup = function(s) {
    return s.split(" ")[0];
};

var parseShortName = function(s) {
    var parts = s.split(" ");
    parts.shift();
    var shortName = parts.join(" ");
    return shortName;
};

var parseStartEnd = function(date, time) {
    var start = null;
    var end = null;
    var timeString = time.split(" ")[0];
    var starTimeString = timeString.split("-")[0];
    var endTimeString = timeString.split("-")[1];

    var dateString = date.split(" ")[1];
    var startString = dateString + " " + starTimeString;
    var endString  = dateString + " " + endTimeString;

    start = moment(startString, "DD-MM-YYYY HH:mm");
    end = moment(endString, "DD-MM-YYYY HH:mm");

    return { start: start, end: end };
}

var trimProperty = function(s) {
  s = s.replace(/ /g, ""); // remove that whitespace
  s = s.replace(/\n/g, " ");
  s = s.trim();
  return s;
};

var outputAsTable = function(timeTableGrid, iteration) {
  for (var i = 0; i < timeTableGrid.length; i++) {
    var row = timeTableGrid[i];
    process.stdout.write(iteration + ". ROW: " + i + ":\t");

    for (var j = 0; j < row.length; j++) {
      process.stdout.write(row[j] + "  ");
    }

    process.stdout.write("\n");
  }
};

var getDaysInThisWeek = function($) {
  var days = [];
  var headerRow = $("th.plan_rahmen").closest("table").children("tr").first();
  headerRow.find("th").each(function(index) {
    var day = $(this).text();
    day = day.replace(/ /g, "");
    day = day.replace(/\n/g, " ");
    day = day.trim();
    days.push(day);
  });
  return days;
};

var hashCode = function(s){
	var hash = 0;
	if (s.length === 0) return hash;
	for (var i = 0; i < s.length; i++) {
		var char = s.charCodeAt(i);
		hash = ((hash<<5)-hash)+char;
		hash = hash & hash; // Convert to 32bit integer
	}
	return hash;
};

module.exports = { parse: parse };
<commit_msg>move database inserting to its own method<commit_after>"use strict";

var request = require("request");
var cheerio = require("cheerio");
var moment  = require("moment");
var mongoose = require("mongoose");
var async = require("async");

// just a static test url, later the client will send a week and a year
var url = "https://lsf.hft-stuttgart.de/qisserver/rds?state=wplan&k_abstgv.abstgvnr=262&week=12_2015&act=stg&pool=stg&show=plan&P.vx=lang&P.Print=";

request(url, function(error, response, html) {
  if(!error) {
    parse(html);
  }
});

var parse = function(html) {
  var table = null;
  var days = [];
  var timeTableGrid = [];
  var lectures = [];
  var $ = cheerio.load(html);

  // load days before we cut of td.plan_rahmen stuff
  days = getDaysInThisWeek($);

  // remove td.plan rahmen from the table, this removes a lot of trouble
  table = $("th.plan_rahmen").closest("table");
  table.find(".plan_rahmen").remove();

  // lets parse
  // get all the rows of the timetable (except the header (days) row)
  var rows = table.children("tr:not(:first-child)");

  // create and prefill the grid, which holds our whole timetable
  for (var i = 0; i < rows.length; i++) {     // rows
    timeTableGrid[i] = new Array(days.length);
    for (var j = 0; j < days.length; j++) {   // columns
      timeTableGrid[i][j] = ".";
    }
  }

  // iterate over each row and each cell
  rows.each(function(i) {
    var cells = $(this).children("td:not(:first-child)");

    // it seems like cheerio doesnt support collection.get(), so we just
    // bottle that cheerio collection into a native array.
    var cellsArray = [];
    cells.each(function(j) {
      cellsArray.push($(this));
    });

    for (var j = 0; j < days.length; j++) {
      if (timeTableGrid[i][j] === ".") { // here must be a td-(lecture)-element

        var currCell = cellsArray[0];
        cellsArray.shift(); // removes the first element, its a pseudo queue

        if (currCell.attr("class").indexOf("plan1") > -1) {
          // no lecture here
        } else if (currCell.attr("class").indexOf("plan2") > -1) { // here we have a lecture
          // now mark the whole rowspan down as the same lecture
          var rowspan = currCell.attr("rowspan");
          for (var k = 0; k < rowspan; k++) {
            timeTableGrid[i+k][j] = j;
          }

          /******** PARSE THE LECURE TD *********/
          var moreLectures = parseLecturesFromHtml(currCell, days, j);
          Array.prototype.push.apply(lectures, moreLectures); // join two arrays, the js way ¯\_(ツ)_/¯
        }
      }
    }
  });
  insertInDatabase(lectures);
};

var insertInDatabase = function(lectures) {
    // connect to mongodb
  mongoose.connect("mongodb://localhost:27017/wuw");

  // create models from our schemas
  var Lecture = require("./model_lecture");
  var Deadline = require("./model_deadline");

  // drop current lecture collection to get a fresh result
  mongoose.connection.collections.lectures.drop(function(err) {
    if(err) console.log(err);

    // push every lecture to our db
    async.each(lectures, function(lecture, cb) {
      // create Lecture from our Model
      var Lec = new Lecture();
      // set attributes
      Lec.date = lecture.start;
      Lec.fullLectureName = lecture.lsfName;
      Lec.shortLectureName = lecture.shortName;
      Lec.room = lecture.lsfRoom;
      Lec.startTime = lecture.start;
      Lec.endTime = lecture.end;
      Lec.group = lecture.group;
      Lec.hashCode = lecture.hashCode;
      // save lecture to db
      Lec.save(cb);
    }, function(err) {
      if (err) console.log(err);
      mongoose.disconnect();

      // output lectures
      console.log(JSON.stringify(lectures, null, 2));
      console.log("Success! The parser inserted " + lectures.length + " lectures in the database");
    });
  });
};

var parseLecturesFromHtml = function(html, days, dayPos) {
  var lectures = [];
  var $ = cheerio.load(html);

  // there could be more then one groups in this lecture,
  // we create a lecture for every group
  html.find("table").each(function(i) {
    var time = trimProperty($(this).find("td.notiz").first().text());
    var date = days[dayPos];

    var lecture = {};
    lecture.lsfName = $(this).find("a").first().attr("title");
    lecture.start = parseStartEnd(date, time).start;
    lecture.end = parseStartEnd(date, time).end;
    lecture.lsfDate = date;
    lecture.lsfTime = time;
    lecture.lsfRoom = $(this).find("td.notiz a").first().text();
    lecture.group = parseGroup(lecture.lsfName);
    lecture.shortName = parseShortName(lecture.lsfName);
    lecture.hashCode = hashCode(lecture.lsfName+lecture.lsfDate+lecture.lsfTime+lecture.lsfRoom);

    lectures.push(lecture);
  });
  return lectures;
};

var parseGroup = function(s) {
    return s.split(" ")[0];
};

var parseShortName = function(s) {
    var parts = s.split(" ");
    parts.shift();
    var shortName = parts.join(" ");
    return shortName;
};

var parseStartEnd = function(date, time) {
    var start = null;
    var end = null;
    var timeString = time.split(" ")[0];
    var starTimeString = timeString.split("-")[0];
    var endTimeString = timeString.split("-")[1];

    var dateString = date.split(" ")[1];
    var startString = dateString + " " + starTimeString;
    var endString  = dateString + " " + endTimeString;

    start = moment(startString, "DD-MM-YYYY HH:mm");
    end = moment(endString, "DD-MM-YYYY HH:mm");

    return { start: start, end: end };
}

var trimProperty = function(s) {
  s = s.replace(/ /g, ""); // remove that whitespace
  s = s.replace(/\n/g, " ");
  s = s.trim();
  return s;
};

var outputAsTable = function(timeTableGrid, iteration) {
  for (var i = 0; i < timeTableGrid.length; i++) {
    var row = timeTableGrid[i];
    process.stdout.write(iteration + ". ROW: " + i + ":\t");

    for (var j = 0; j < row.length; j++) {
      process.stdout.write(row[j] + "  ");
    }

    process.stdout.write("\n");
  }
};

var getDaysInThisWeek = function($) {
  var days = [];
  var headerRow = $("th.plan_rahmen").closest("table").children("tr").first();
  headerRow.find("th").each(function(index) {
    var day = $(this).text();
    day = day.replace(/ /g, "");
    day = day.replace(/\n/g, " ");
    day = day.trim();
    days.push(day);
  });
  return days;
};

var hashCode = function(s){
	var hash = 0;
	if (s.length === 0) return hash;
	for (var i = 0; i < s.length; i++) {
		var char = s.charCodeAt(i);
		hash = ((hash<<5)-hash)+char;
		hash = hash & hash; // Convert to 32bit integer
	}
	return hash;
};

module.exports = { parse: parse };
<|endoftext|>


<commit_before>// parser.js - Julian Rosenblum
// Parses Markato files into Markato object

var _ = require('underscore');
var S = require('string');
var fs = require('fs');

var parseFooterStartLine = function (state, line) {
  state.current.footer = true;
  return state;
};

var parseMetaLine = function (state, line) {
  var prefix = line.split(' ')[0];
  var metaName = prefix.slice(2); // Remove ##
  var metaValue = S(line).chompLeft(prefix).trim().s;

  state.meta[metaName] = metaValue;
  return state;
};

var parseSectionLine = function (state, line) {
  var sectionName = S(line.slice(1)).trim().s; // Remove # and trim

  state.current.sectionName = sectionName;

  // It's a new section if we don't already have it on the list
  state.current.newSection = !_.contains(state.sections, sectionName);

  state.current.line = 0;

  state.sections.push(sectionName);
  state.lyrics.push({
    section: sectionName,
    firstTime: state.current.newSection,
    lines: []
  });

  if (state.current.newSection) state.chords[sectionName] = [];

  return state;
};

var parseChordLine = function (state, line) {
  var chords = S(line.slice(1)).trim().s.split(' '); // Remove : and trim
  var sectionName = state.current.sectionName;

  if (state.current.newSection) {
    state.chords[sectionName].push(chords);
  }

  state.current.lastChords = chords;

  return state;
};

var parseFooterLine = function (state, line) {
  var parts = S(line).strip(' ').split('=>');
  var chord;
  var alts;

  if (parts.length !== 2) return state;

  chord = parts[0];
  alts = parts[1].split(',');

  // Add alternates for each chord and remove ' marks
  _.each(state.lyrics, function (section) {
    _.each(section.lines, function (line) {
      _.each(line, function (phrase) {
        if (phrase.chord === chord) {
          phrase.alts = alts;
          phrase.chord = S(phrase.chord).strip('\'').s;
        }
      });
    });
  });

  return state;
};

var parseLyricLine = function (state, line) {
  var caratSplit = line.split('^'); // Used to figure out phrases
  var chordIndex = 0; // Index in list of chords as we assemble phrases
  var phrases = [];
  var exceptionIndices = []; // Chord indices that are exceptions in this section

  // Start off by assuming the chords are the current chords in this verse
  var chords = state.chords[state.current.sectionName][state.current.line];

  // If there are no current chords in this verse, add an empty array to the verse
  if (!chords) {
    chords = state.chords[state.current.sectionName][state.current.line] = [];
  }

  // If we're not in a new section and we just saw a line of chords, this means
  // substitutions are being made in this section
  if (!state.current.isNew && state.current.lastChords) {
    chords = _.map(state.current.lastChords, function (chord, index) {
      // For *, get the chord at the same index from the chord list
      if (chord === '*') return chords[index];

      exceptionIndices.push(index);
      return chord;
    });
  }

  _.each(caratSplit, function (phrase, index) {
    var addPhrase = function (obj) {
      phrases.push(_.defaults(obj, {
        string: '',
        chord: '',
        exception: false,
        alts: [],
        wordExtension: false
      }));
    };

    var chord;
    var lastPhrase;

    // Special case first phrase
    if (index === 0) {
      if (phrase) {
        addPhrase({string: caratSplit[0]});
      }
      return;
    }

    lastPhrase = caratSplit[index - 1];

    // Get next chord
    chord = chords[chordIndex];

    // 'foo ^ bar' case, we insert the chord with a blank lyric
    if (phrase && phrase[0] == ' ') {
      addPhrase({
        chord: chord,
        exception: _.contains(exceptionIndices, chordIndex)
      });

      addPhrase({
        string: S(phrase).trim().s,
      });
    }
    else {
      addPhrase({
        string: S(phrase).trim().s,
        chord: chord,
        exception: _.contains(exceptionIndices, chordIndex),
      });
    }

    // Check for foo^bar case (doesn't start with space and last phrase doesn't end with one)
    if (phrase && phrase[0] !== ' ' && lastPhrase && S(lastPhrase).right(1).s !== ' ') {
      _.last(phrases).wordExtension = true;
    }

    chordIndex += 1;
  });

  // Add new line to current lyric section
  _.last(state.lyrics).lines.push(phrases);

  state.current.lastChords = null;
  state.current.line += 1;
  return state;
};

// Given a parse state, parses a line of Markato and returns updated state
var parseLine = function (state, line) {
  // Remove extraneous whitespace
  line = S(line).trim().collapseWhitespace().s;

  if (!line) return state;

  if (S(line).startsWith('###')) return parseFooterStartLine(state, line);

  if (S(line).startsWith('##')) return parseMetaLine(state, line);

  if (S(line).startsWith('#')) return parseSectionLine(state, line);

  if (S(line).startsWith(':')) return parseChordLine(state, line);

  if (state.current.footer) return parseFooterLine(state, line);

  return parseLyricLine(state, line);
};

// Given a finished parse state, return a markato object
var markatoObjectFromState = function (state) {
  return _.omit(state, 'current');
};

// Parses a string of Markato and returns a Markato object
var parseString = function (str) {
  var lines = S(str).lines();
  var state = {
    current: {
      sectionName: '', // Current section we're in
      newSection: true, // Is this a section being newly defined?
      line: 0, // What line of the section are we in (incremented on parseLyric)
      footer: false, // Are we in the footer?
      lastChords: null // Array of last line of chords we've seen
    },
    sections: [],
    chords: {},
    lyrics: [],
    meta: {}
  };

  state = _.reduce(lines, parseLine, state);
  return markatoObjectFromState(state);
};

module.exports.parseString = parseString;

module.exports.parseFile = function (path) {
  var file = fs.readFileSync(path);
  return parseString(file);
};<commit_msg>Turned parser into client side js file<commit_after>// parser.js - Julian Rosenblum
// Parses Markato files into Markato object
// Dependencies: underscore.js, string.js

var parser = (function () {
  var parseFooterStartLine = function (state, line) {
    state.current.footer = true;
    return state;
  };

  var parseMetaLine = function (state, line) {
    var prefix = line.split(' ')[0];
    var metaName = prefix.slice(2); // Remove ##
    var metaValue = S(line).chompLeft(prefix).trim().s;

    state.meta[metaName] = metaValue;
    return state;
  };

  var parseSectionLine = function (state, line) {
    var sectionName = S(line.slice(1)).trim().s; // Remove # and trim

    state.current.sectionName = sectionName;

    // It's a new section if we don't already have it on the list
    state.current.newSection = !_.contains(state.sections, sectionName);

    state.current.line = 0;

    state.sections.push(sectionName);
    state.lyrics.push({
      section: sectionName,
      firstTime: state.current.newSection,
      lines: []
    });

    if (state.current.newSection) state.chords[sectionName] = [];

    return state;
  };

  var parseChordLine = function (state, line) {
    var chords = S(line.slice(1)).trim().s.split(' '); // Remove : and trim
    var sectionName = state.current.sectionName;

    if (state.current.newSection) {
      state.chords[sectionName].push(chords);
    }

    state.current.lastChords = chords;

    return state;
  };

  var parseFooterLine = function (state, line) {
    var parts = S(line).strip(' ').split('=>');
    var chord;
    var alts;

    if (parts.length !== 2) return state;

    chord = parts[0];
    alts = parts[1].split(',');

    // Add alternates for each chord and remove ' marks
    _.each(state.lyrics, function (section) {
      _.each(section.lines, function (line) {
        _.each(line, function (phrase) {
          if (phrase.chord === chord) {
            phrase.alts = alts;
            phrase.chord = S(phrase.chord).strip('\'').s;
          }
        });
      });
    });

    return state;
  };

  var parseLyricLine = function (state, line) {
    var caratSplit = line.split('^'); // Used to figure out phrases
    var chordIndex = 0; // Index in list of chords as we assemble phrases
    var phrases = [];
    var exceptionIndices = []; // Chord indices that are exceptions in this section

    // Start off by assuming the chords are the current chords in this verse
    var chords = state.chords[state.current.sectionName][state.current.line];

    // If there are no current chords in this verse, add an empty array to the verse
    if (!chords) {
      chords = state.chords[state.current.sectionName][state.current.line] = [];
    }

    // If we're not in a new section and we just saw a line of chords, this means
    // substitutions are being made in this section
    if (!state.current.isNew && state.current.lastChords) {
      chords = _.map(state.current.lastChords, function (chord, index) {
        // For *, get the chord at the same index from the chord list
        if (chord === '*') return chords[index];

        exceptionIndices.push(index);
        return chord;
      });
    }

    _.each(caratSplit, function (phrase, index) {
      var addPhrase = function (obj) {
        phrases.push(_.defaults(obj, {
          string: '',
          chord: '',
          exception: false,
          alts: [],
          wordExtension: false
        }));
      };

      var chord;
      var lastPhrase;

      // Special case first phrase
      if (index === 0) {
        if (phrase) {
          addPhrase({string: caratSplit[0]});
        }
        return;
      }

      lastPhrase = caratSplit[index - 1];

      // Get next chord
      chord = chords[chordIndex];

      // 'foo ^ bar' case, we insert the chord with a blank lyric
      if (phrase && phrase[0] == ' ') {
        addPhrase({
          chord: chord,
          exception: _.contains(exceptionIndices, chordIndex)
        });

        addPhrase({
          string: S(phrase).trim().s,
        });
      }
      else {
        addPhrase({
          string: S(phrase).trim().s,
          chord: chord,
          exception: _.contains(exceptionIndices, chordIndex),
        });
      }

      // Check for foo^bar case (doesn't start with space and last phrase doesn't end with one)
      if (phrase && phrase[0] !== ' ' && lastPhrase && S(lastPhrase).right(1).s !== ' ') {
        _.last(phrases).wordExtension = true;
      }

      chordIndex += 1;
    });

    // Add new line to current lyric section
    _.last(state.lyrics).lines.push(phrases);

    state.current.lastChords = null;
    state.current.line += 1;
    return state;
  };

  // Given a parse state, parses a line of Markato and returns updated state
  var parseLine = function (state, line) {
    // Remove extraneous whitespace
    line = S(line).trim().collapseWhitespace().s;

    if (!line) return state;

    if (S(line).startsWith('###')) return parseFooterStartLine(state, line);

    if (S(line).startsWith('##')) return parseMetaLine(state, line);

    if (S(line).startsWith('#')) return parseSectionLine(state, line);

    if (S(line).startsWith(':')) return parseChordLine(state, line);

    if (state.current.footer) return parseFooterLine(state, line);

    return parseLyricLine(state, line);
  };

  // Given a finished parse state, return a markato object
  var markatoObjectFromState = function (state) {
    return _.omit(state, 'current');
  };

  // Parses a string of Markato and returns a Markato object
  return {
    parseString: function (str) {
      var lines = S(str).lines();
      var state = {
        current: {
          sectionName: '', // Current section we're in
          newSection: true, // Is this a section being newly defined?
          line: 0, // What line of the section are we in (incremented on parseLyric)
          footer: false, // Are we in the footer?
          lastChords: null // Array of last line of chords we've seen
        },
        sections: [],
        chords: {},
        lyrics: [],
        meta: {}
      };

      state = _.reduce(lines, parseLine, state);
      return markatoObjectFromState(state);
    }
  };
}());<|endoftext|>


<commit_before>// < begin copyright >
// Copyright Ryan Marcus 2018
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.
//
// < end copyright >

"use strict";

let lexer = require("./lexer");

// terminals
const LEX_KV = 0;
const LEX_KVLIST = 1;
const LEX_VLIST = 2;
const LEX_BOOLEAN = 3;
const LEX_COVALUE = 4;
const LEX_CVALUE = 5;
const LEX_FLOAT = 6;
const LEX_INT = 7;
const LEX_KEY = 8;
const LEX_LIST = 9;
const LEX_OBJ = 10;
const LEX_QUOTE = 11;
const LEX_RB = 12;
const LEX_RCB = 13;
const LEX_TOKEN = 14;
const LEX_VALUE = 15;

// non-terminals
const LEX_COLON = -1;
const LEX_COMMA = -2;
const LEX_LCB = -3;
const LEX_LB = -4;
const LEX_DOT = -5;

Array.prototype.peek = function() {
    return this[this.length - 1];
};

Array.prototype.last = function(i) {
    return this[this.length - (1 + i)];
};

function is(obj, prop) {
    return (obj && obj.hasOwnProperty("type") && obj.type == prop);
}

function log(str) {
    //console.log(str);
}

module.exports.parse = parse;
function parse(text) {
    let stack = [];

    let tokens = [];
    let emit = function(t) {
        tokens.push(t);
    };

    lexer.lexString(text, emit);

    // ensure that if we started with a LB or LCB, we end with a
    // RB or RCB.
    if (tokens[0].type == LEX_LB && tokens.last(0).type != LEX_RB) {
        tokens.push({ type: LEX_RB, value: "]", row: -1, col: -1});
    }

    if (tokens[0].type == LEX_LCB && tokens.last(0).type != LEX_RCB) {
        tokens.push({ type: LEX_RCB, value: "}", row: -1, col: -1});
    }

    for (let i = 0; i < tokens.length; i++) {
        log("Shifting " + tokens[i].type);
        stack.push(tokens[i]);
        log(stack);
        log("Reducing...");
        while (reduce(stack)) {
            log(stack);
            log("Reducing...");
        }

    }

    return compileOST(stack[0]);

}

function reduce(stack) {
    let next = stack.pop();

    switch(next.type) {
    case LEX_KEY:
        if (next.value == "true") {
            log("Rule 5");
            stack.push({'type': LEX_BOOLEAN, 'value': "true"});
            return true;
        }

        if (next.value == "false") {
            log("Rule 6");
            stack.push({'type': LEX_BOOLEAN, 'value': "false"});
            return true;
        }

        if (next.value == "null") {
            log("Rule 7");
            stack.push({'type': LEX_VALUE, 'value': null});
            return true;
        }
        break;

    case LEX_TOKEN:
        if (is(stack.peek(), LEX_KEY)) {
            log("Rule 11a");
            stack.peek().value += next.value;
            return true;
        }

        log("Rule 11c");
        stack.push({type: LEX_KEY, value: next.value });
        return true;

    case LEX_INT:
        if (is(next, LEX_INT) && is(stack.peek(), LEX_KEY)) {
            log("Rule 11b");
            stack.peek().value += next.value;
            return true;
        }

        log("Rule 11f");
        next.type = LEX_VALUE;
        stack.push(next);
        return true;

    case LEX_QUOTE:
        log("Rule 11d");
        next.type = LEX_VALUE;
        next.value = next.value;
        stack.push(next);
        return true;

    case LEX_BOOLEAN:
        log("Rule 11e");
        next.type = LEX_VALUE;

        if (next.value == "true") {
            next.value = true;
        } else {
            next.value = false;
        }

        stack.push(next);
        return true;

    case LEX_FLOAT:
        log("Rule 11g");
        next.type = LEX_VALUE;
        stack.push(next);
        return true;

    case LEX_VALUE:
        if (is(stack.peek(), LEX_COMMA)) {
            log("Rule 12");
            next.type = LEX_CVALUE;
            stack.pop();
            stack.push(next);
            return true;
        }

        if (is(stack.peek(), LEX_COLON)) {
            log("Rule 13");
            next.type = LEX_COVALUE;
            stack.pop();
            stack.push(next);
            return true;
        }

        if (is(stack.peek(), LEX_KEY) && is(stack.last(1), LEX_VALUE)) {
            log("Error rule 1");
            let middleVal = stack.pop();
            stack.peek().value += '"' + middleVal.value + '"';
            stack.peek().value += next.value;
            return true;
        }

        if (is(stack.peek(), LEX_KEY) && is(stack.last(1), LEX_VLIST)) {
            log("Error rule 2");
            let middleVal = stack.pop();
            let oldLastVal = stack.peek().value.pop();
            oldLastVal +=  '"' + middleVal.value + '"';
            oldLastVal += next.value;

            stack.peek().value.push(oldLastVal);

            return true;
        }

        if (is(stack.peek(), LEX_KEY) && is(stack.last(1), LEX_KVLIST)) {
            log("Error rule 3");
            let middleVal = stack.pop();
            let oldLastVal = stack.peek().value.pop();
            oldLastVal.value +=  '"' + middleVal.value + '"';
            oldLastVal.value += next.value;

            stack.peek().value.push(oldLastVal);

            return true;
        }

        if (is(stack.peek(), LEX_KEY)) {
            log("Error rule 4");
            let keyValue = stack.pop().value;
            next.value = keyValue + next.value;
            stack.push(next);
            return true;
        }

        break;

    case LEX_LIST:
        if (is(next, LEX_LIST) && is(stack.peek(), LEX_COMMA)) {
            log("Rule 12a");
            next.type = LEX_CVALUE;
            stack.pop();
            stack.push(next);
            return true;
        }

        if (is(stack.peek(), LEX_COLON)) {
            log("Rule 13a");
            next.type = LEX_COVALUE;
            stack.pop();
            stack.push(next);
            return true;
        }
        break;

    case LEX_OBJ:
        if (is(stack.peek(), LEX_COMMA)) {
            log("Rule 12b");
            let toPush = {'type': LEX_CVALUE, 'value': next};
            stack.pop();
            stack.push(toPush);
            return true;
        }

        if (is(stack.peek(), LEX_COLON)) {
            log("Rule 13b");
            let toPush = {'type': LEX_COVALUE, 'value': next};
            stack.pop();
            stack.push(toPush);
            return true;
        }
        break;

    case LEX_CVALUE:
        if (is(stack.peek(), LEX_VLIST)) {
            log("Rule 14");
            stack.peek().value.push(next.value);
            return true;
        }

        log("Rule 15");
        stack.push({'type': LEX_VLIST, 'value': [next.value]});
        return true;

    case LEX_VLIST:
        if (is(stack.peek(), LEX_VALUE)) {
            log("Rule 15a");
            next.value.unshift(stack.peek().value);
            stack.pop();
            stack.push(next);
            return true;
        }

        if (is(stack.peek(), LEX_LIST)) {
            log("Rule 15b");
            next.value.unshift(stack.peek().value);
            stack.pop();
            stack.push(next);
            return true;
        }

        if (is(stack.peek(), LEX_OBJ)) {
            log("Rule 15c");
            next.value.unshift(stack.peek());
            stack.pop();
            stack.push(next);
            return true;
        }

        if (is(stack.peek(), LEX_KEY) && (stack.last(1), LEX_COMMA)) {
            log("Error rule 7");
            let l = stack.pop();
            stack.push({type: LEX_VALUE, 'value': l.value});
            log("Start subreduce... (" + l.value + ")");
            while(reduce(stack));
            log("End subreduce");
            stack.push(next);

            return true;
        }

        if (is(stack.peek(), LEX_VLIST)) {
            log("Error rule 8");
            stack.peek().value.push(next.value[0]);
            return true;
        }
        break;

    case LEX_COVALUE:

        if (is(stack.peek(), LEX_KEY) || is(stack.peek(), LEX_VALUE) || is(stack.peek(), LEX_VLIST)) {
            log("Rule 16");
            let key = stack.pop();
            stack.push({'type': LEX_KV, 'key': key.value, 'value': next.value});
            return true;
        }

        throw new Error("Got a :value that can't be handled at line " +
                       next.row + ":" + next.col);

    case LEX_KV:
        if (is(stack.last(0), LEX_COMMA) && is(stack.last(1), LEX_KVLIST)) {
            log("Rule 17");
            stack.last(1).value.push(next);
            stack.pop();
            return true;
        }

        log("Rule 18");
        stack.push({'type': LEX_KVLIST, 'value': [next]});
        return true;

    case LEX_KVLIST:
        if (is(stack.peek(), LEX_KVLIST)) {
            log("Rule 17a");
            next.value.forEach(function (i) {
                stack.peek().value.push(i);
            });

            return true;
        }

        break;

    case LEX_RB:
        if (is(stack.peek(), LEX_VLIST) && is(stack.last(1), LEX_LB)) {
            log("Rule 19");
            let l = stack.pop();
            stack.pop();
            stack.push({'type': LEX_LIST, 'value': l.value});
            return true;
        }

        if (is(stack.peek(), LEX_LIST) && is(stack.last(1), LEX_LB)) {
            log("Rule 19b");
            let l = stack.pop();
            stack.pop();
            stack.push({'type': LEX_LIST, 'value': [l.value]});
            return true;
        }

        if (is(stack.peek(), LEX_LB)) {
            log("Rule 22");
            stack.pop();
            stack.push({type: LEX_LIST, 'value': []});
            return true;
        }

        if (is(stack.peek(), LEX_VALUE) && is(stack.last(1), LEX_LB)) {
            log("Rule 23");
            let val = stack.pop().value;
            stack.pop();
            stack.push({type: LEX_LIST, 'value': [val]});
            return true;
        }

        if (is(stack.peek(), LEX_OBJ) && is(stack.last(1), LEX_LB)) {
            log("Rule 23b");
            let val = stack.pop();
            stack.pop();
            stack.push({type: LEX_LIST, 'value': [val]});
            return true;
        }

        if (is(stack.peek(), LEX_KEY) && is(stack.last(1), LEX_COMMA)) {
            log("Error rule 5");
            let l = stack.pop();
            stack.push({type: LEX_VALUE, 'value': l.value});
            log("Start subreduce... (" + l.value + ")");
            while(reduce(stack));
            log("End subreduce");
            stack.push({type: LEX_RB});
            return true;
        }

        if (is(stack.peek(), LEX_COMMA) && (
            is(stack.last(1), LEX_KEY)
                || is(stack.last(1), LEX_OBJ)
                || is(stack.last(1), LEX_VALUE))
           ) {
            log("Error rule 5a");
            stack.pop();
            //let l = stack.pop();
            //stack.push({type: LEX_VALUE, 'value': l.value});
            stack.push({type: LEX_RB, 'value': ']'});
            log("Start subreduce...");
            log("Content: " + JSON.stringify(stack));
            while(reduce(stack));
            log("End subreduce");

            return true;
        }

        if (is(stack.peek(), LEX_KEY) && is(stack.last(1), LEX_LB)) {
            log("Error rule 5b");
            let v = stack.pop();
            stack.pop();
            stack.push({type: LEX_LIST, value: [v.value]});
            return true;
        }

        if (is(stack.peek(), LEX_COMMA) && is(stack.last(1), LEX_VLIST)) {
            log("Error rule 5c");
            stack.pop();
            stack.push({type: LEX_RB});
            log("Start subreduce...");
            log("Content: " + JSON.stringify(stack));
            while(reduce(stack));
            log("End subreduce");

            return true;
        }

        break;

    case LEX_RCB:
        if (is(stack.peek(), LEX_KVLIST) && is(stack.last(1), LEX_LCB)) {
            log("Rule 20");
            let l = stack.pop();
            stack.pop();
            stack.push({'type': LEX_OBJ, 'value': l.value});
            return true;
        }

        if (is(stack.peek(), LEX_LCB)) {
            log("Rule 21");
            stack.pop();
            stack.push({type: LEX_OBJ, 'value': null});
            return true;
        }

        if (is(stack.peek(), LEX_KEY) && is(stack.last(1), LEX_COLON)) {
            log("Error rule 4");
            let l = stack.pop();
            //stack.pop();
            stack.push({type: LEX_VALUE, 'value': l.value});
            log("Start subreduce... (" + l.value + ")");
            while(reduce(stack));
            log("End subreduce");
            stack.push({type: LEX_RCB});
            return true;
        }

        throw new Error("Found } that I can't handle at line " +
                        next.row + ":" + next.col);

    case LEX_COMMA:
        if (is(stack.peek(), LEX_COMMA)) {
            log("Comma error rule 1");
            // do nothing -- so don't push the extra comma onto the stack
            return true;
        }
    }

    stack.push(next);
    return false;
}

function compileOST(tree) {
    let rawTypes = ["boolean", "number", "string"];

    if (rawTypes.indexOf((typeof tree)) != -1)
        return tree;

    if (tree === null)
        return null;

    if (Array.isArray(tree)) {
        let toR = [];
        while (tree.length > 0)
            toR.unshift(compileOST(tree.pop()));
        return toR;
    }

    if (is(tree, LEX_OBJ)) {
        let toR = {};
        if (tree.value === null)
            return {};
        tree.value.forEach(function (i) {
            toR[i.key] = compileOST(i.value);
        });
        return toR;
    }

    if (is(tree, LEX_LIST)) {
        return compileOST(tree.value);
    }

    // it must be a value
    return tree.value;
}

<commit_msg>formatting<commit_after>// < begin copyright >
// Copyright Ryan Marcus 2018
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.
//
// < end copyright >

"use strict";

let lexer = require("./lexer");

// terminals
const LEX_KV = 0;
const LEX_KVLIST = 1;
const LEX_VLIST = 2;
const LEX_BOOLEAN = 3;
const LEX_COVALUE = 4;
const LEX_CVALUE = 5;
const LEX_FLOAT = 6;
const LEX_INT = 7;
const LEX_KEY = 8;
const LEX_LIST = 9;
const LEX_OBJ = 10;
const LEX_QUOTE = 11;
const LEX_RB = 12;
const LEX_RCB = 13;
const LEX_TOKEN = 14;
const LEX_VALUE = 15;

// non-terminals
const LEX_COLON = -1;
const LEX_COMMA = -2;
const LEX_LCB = -3;
const LEX_LB = -4;
const LEX_DOT = -5;

Array.prototype.peek = function() {
    return this[this.length - 1];
};

Array.prototype.last = function(i) {
    return this[this.length - (1 + i)];
};

function is(obj, prop) {
    return (obj && obj.hasOwnProperty("type") && obj.type == prop);
}

function log(str) {
    //console.log(str);
}

module.exports.parse = parse;
function parse(text) {
    let stack = [];

    let tokens = [];
    let emit = function(t) {
        tokens.push(t);
    };

    lexer.lexString(text, emit);

    // ensure that if we started with a LB or LCB, we end with a
    // RB or RCB.
    if (tokens[0].type == LEX_LB && tokens.last(0).type != LEX_RB) {
        tokens.push({ type: LEX_RB, value: "]", row: -1, col: -1});
    }

    if (tokens[0].type == LEX_LCB && tokens.last(0).type != LEX_RCB) {
        tokens.push({ type: LEX_RCB, value: "}", row: -1, col: -1});
    }

    for (let i = 0; i < tokens.length; i++) {
        log("Shifting " + tokens[i].type);
        stack.push(tokens[i]);
        log(stack);
        log("Reducing...");
        while (reduce(stack)) {
            log(stack);
            log("Reducing...");
        }

    }

    return compileOST(stack[0]);

}

function reduce(stack) {
    let next = stack.pop();

    switch(next.type) {
    case LEX_KEY:
        if (next.value == "true") {
            log("Rule 5");
            stack.push({'type': LEX_BOOLEAN, 'value': "true"});
            return true;
        }

        if (next.value == "false") {
            log("Rule 6");
            stack.push({'type': LEX_BOOLEAN, 'value': "false"});
            return true;
        }

        if (next.value == "null") {
            log("Rule 7");
            stack.push({'type': LEX_VALUE, 'value': null});
            return true;
        }
        break;

    case LEX_TOKEN:
        if (is(stack.peek(), LEX_KEY)) {
            log("Rule 11a");
            stack.peek().value += next.value;
            return true;
        }

        log("Rule 11c");
        stack.push({type: LEX_KEY, value: next.value });
        return true;

    case LEX_INT:
        if (is(next, LEX_INT) && is(stack.peek(), LEX_KEY)) {
            log("Rule 11b");
            stack.peek().value += next.value;
            return true;
        }

        log("Rule 11f");
        next.type = LEX_VALUE;
        stack.push(next);
        return true;

    case LEX_QUOTE:
        log("Rule 11d");
        next.type = LEX_VALUE;
        next.value = next.value;
        stack.push(next);
        return true;

    case LEX_BOOLEAN:
        log("Rule 11e");
        next.type = LEX_VALUE;

        if (next.value == "true") {
            next.value = true;
        } else {
            next.value = false;
        }

        stack.push(next);
        return true;

    case LEX_FLOAT:
        log("Rule 11g");
        next.type = LEX_VALUE;
        stack.push(next);
        return true;

    case LEX_VALUE:
        if (is(stack.peek(), LEX_COMMA)) {
            log("Rule 12");
            next.type = LEX_CVALUE;
            stack.pop();
            stack.push(next);
            return true;
        }

        if (is(stack.peek(), LEX_COLON)) {
            log("Rule 13");
            next.type = LEX_COVALUE;
            stack.pop();
            stack.push(next);
            return true;
        }

        if (is(stack.peek(), LEX_KEY) && is(stack.last(1), LEX_VALUE)) {
            log("Error rule 1");
            let middleVal = stack.pop();
            stack.peek().value += '"' + middleVal.value + '"';
            stack.peek().value += next.value;
            return true;
        }

        if (is(stack.peek(), LEX_KEY) && is(stack.last(1), LEX_VLIST)) {
            log("Error rule 2");
            let middleVal = stack.pop();
            let oldLastVal = stack.peek().value.pop();
            oldLastVal +=  '"' + middleVal.value + '"';
            oldLastVal += next.value;

            stack.peek().value.push(oldLastVal);

            return true;
        }

        if (is(stack.peek(), LEX_KEY) && is(stack.last(1), LEX_KVLIST)) {
            log("Error rule 3");
            let middleVal = stack.pop();
            let oldLastVal = stack.peek().value.pop();
            oldLastVal.value +=  '"' + middleVal.value + '"';
            oldLastVal.value += next.value;

            stack.peek().value.push(oldLastVal);

            return true;
        }

        if (is(stack.peek(), LEX_KEY)) {
            log("Error rule 4");
            let keyValue = stack.pop().value;
            next.value = keyValue + next.value;
            stack.push(next);
            return true;
        }

        break;

    case LEX_LIST:
        if (is(next, LEX_LIST) && is(stack.peek(), LEX_COMMA)) {
            log("Rule 12a");
            next.type = LEX_CVALUE;
            stack.pop();
            stack.push(next);
            return true;
        }

        if (is(stack.peek(), LEX_COLON)) {
            log("Rule 13a");
            next.type = LEX_COVALUE;
            stack.pop();
            stack.push(next);
            return true;
        }
        break;

    case LEX_OBJ:
        if (is(stack.peek(), LEX_COMMA)) {
            log("Rule 12b");
            let toPush = {'type': LEX_CVALUE, 'value': next};
            stack.pop();
            stack.push(toPush);
            return true;
        }

        if (is(stack.peek(), LEX_COLON)) {
            log("Rule 13b");
            let toPush = {'type': LEX_COVALUE, 'value': next};
            stack.pop();
            stack.push(toPush);
            return true;
        }
        break;

    case LEX_CVALUE:
        if (is(stack.peek(), LEX_VLIST)) {
            log("Rule 14");
            stack.peek().value.push(next.value);
            return true;
        }

        log("Rule 15");
        stack.push({'type': LEX_VLIST, 'value': [next.value]});
        return true;

    case LEX_VLIST:
        if (is(stack.peek(), LEX_VALUE)) {
            log("Rule 15a");
            next.value.unshift(stack.peek().value);
            stack.pop();
            stack.push(next);
            return true;
        }

        if (is(stack.peek(), LEX_LIST)) {
            log("Rule 15b");
            next.value.unshift(stack.peek().value);
            stack.pop();
            stack.push(next);
            return true;
        }

        if (is(stack.peek(), LEX_OBJ)) {
            log("Rule 15c");
            next.value.unshift(stack.peek());
            stack.pop();
            stack.push(next);
            return true;
        }

        if (is(stack.peek(), LEX_KEY) && (stack.last(1), LEX_COMMA)) {
            log("Error rule 7");
            let l = stack.pop();
            stack.push({type: LEX_VALUE, 'value': l.value});
            log("Start subreduce... (" + l.value + ")");
            while(reduce(stack));
            log("End subreduce");
            stack.push(next);

            return true;
        }

        if (is(stack.peek(), LEX_VLIST)) {
            log("Error rule 8");
            stack.peek().value.push(next.value[0]);
            return true;
        }
        break;

    case LEX_COVALUE:

        if (is(stack.peek(), LEX_KEY) || is(stack.peek(), LEX_VALUE) || is(stack.peek(), LEX_VLIST)) {
            log("Rule 16");
            let key = stack.pop();
            stack.push({'type': LEX_KV, 'key': key.value, 'value': next.value});
            return true;
        }

        throw new Error("Got a :value that can't be handled at line " +
                       next.row + ":" + next.col);

    case LEX_KV:
        if (is(stack.last(0), LEX_COMMA) && is(stack.last(1), LEX_KVLIST)) {
            log("Rule 17");
            stack.last(1).value.push(next);
            stack.pop();
            return true;
        }

        log("Rule 18");
        stack.push({'type': LEX_KVLIST, 'value': [next]});
        return true;

    case LEX_KVLIST:
        if (is(stack.peek(), LEX_KVLIST)) {
            log("Rule 17a");
            next.value.forEach(function (i) {
                stack.peek().value.push(i);
            });

            return true;
        }

        break;

    case LEX_RB:
        if (is(stack.peek(), LEX_VLIST) && is(stack.last(1), LEX_LB)) {
            log("Rule 19");
            let l = stack.pop();
            stack.pop();
            stack.push({'type': LEX_LIST, 'value': l.value});
            return true;
        }

        if (is(stack.peek(), LEX_LIST) && is(stack.last(1), LEX_LB)) {
            log("Rule 19b");
            let l = stack.pop();
            stack.pop();
            stack.push({'type': LEX_LIST, 'value': [l.value]});
            return true;
        }

        if (is(stack.peek(), LEX_LB)) {
            log("Rule 22");
            stack.pop();
            stack.push({type: LEX_LIST, 'value': []});
            return true;
        }

        if (is(stack.peek(), LEX_VALUE) && is(stack.last(1), LEX_LB)) {
            log("Rule 23");
            let val = stack.pop().value;
            stack.pop();
            stack.push({type: LEX_LIST, 'value': [val]});
            return true;
        }

        if (is(stack.peek(), LEX_OBJ) && is(stack.last(1), LEX_LB)) {
            log("Rule 23b");
            let val = stack.pop();
            stack.pop();
            stack.push({type: LEX_LIST, 'value': [val]});
            return true;
        }

        if (is(stack.peek(), LEX_KEY) && is(stack.last(1), LEX_COMMA)) {
            log("Error rule 5");
            let l = stack.pop();
            stack.push({type: LEX_VALUE, 'value': l.value});
            log("Start subreduce... (" + l.value + ")");
            while(reduce(stack));
            log("End subreduce");
            stack.push({type: LEX_RB});
            return true;
        }

        if (is(stack.peek(), LEX_COMMA) && (
            is(stack.last(1), LEX_KEY)
                || is(stack.last(1), LEX_OBJ)
                || is(stack.last(1), LEX_VALUE))
           ) {
            log("Error rule 5a");
            stack.pop();
            //let l = stack.pop();
            //stack.push({type: LEX_VALUE, 'value': l.value});
            stack.push({type: LEX_RB, 'value': ']'});
            log("Start subreduce...");
            log("Content: " + JSON.stringify(stack));
            while(reduce(stack));
            log("End subreduce");

            return true;
        }

        if (is(stack.peek(), LEX_KEY) && is(stack.last(1), LEX_LB)) {
            log("Error rule 5b");
            let v = stack.pop();
            stack.pop();
            stack.push({type: LEX_LIST, value: [v.value]});
            return true;
        }

        if (is(stack.peek(), LEX_COMMA) && is(stack.last(1), LEX_VLIST)) {
            log("Error rule 5c");
            stack.pop();
            stack.push({type: LEX_RB});
            log("Start subreduce...");
            log("Content: " + JSON.stringify(stack));
            while(reduce(stack));
            log("End subreduce");

            return true;
        }

        break;

    case LEX_RCB:
        if (is(stack.peek(), LEX_KVLIST) && is(stack.last(1), LEX_LCB)) {
            log("Rule 20");
            let l = stack.pop();
            stack.pop();
            stack.push({'type': LEX_OBJ, 'value': l.value});
            return true;
        }

        if (is(stack.peek(), LEX_LCB)) {
            log("Rule 21");
            stack.pop();
            stack.push({type: LEX_OBJ, 'value': null});
            return true;
        }

        if (is(stack.peek(), LEX_KEY) && is(stack.last(1), LEX_COLON)) {
            log("Error rule 4");
            let l = stack.pop();
            //stack.pop();
            stack.push({type: LEX_VALUE, 'value': l.value});
            log("Start subreduce... (" + l.value + ")");
            while(reduce(stack));
            log("End subreduce");
            stack.push({type: LEX_RCB});
            return true;
        }

        throw new Error("Found } that I can't handle at line " +
                        next.row + ":" + next.col);

    case LEX_COMMA:
        if (is(stack.peek(), LEX_COMMA)) {
            log("Comma error rule 1");
            // do nothing -- so don't push the extra comma onto the stack
            return true;
        }
    }

    stack.push(next);
    return false;
}

function compileOST(tree) {
    let rawTypes = ["boolean", "number", "string"];

    if (rawTypes.indexOf((typeof tree)) != -1)
        return tree;

    if (tree === null)
        return null;

    if (Array.isArray(tree)) {
        let toR = [];
        while (tree.length > 0)
            toR.unshift(compileOST(tree.pop()));
        return toR;
    }

    if (is(tree, LEX_OBJ)) {
        let toR = {};
        if (tree.value === null)
            return {};
        tree.value.forEach(function (i) {
            toR[i.key] = compileOST(i.value);
        });
        return toR;
    }

    if (is(tree, LEX_LIST)) {
        return compileOST(tree.value);
    }

    // it must be a value
    return tree.value;
}

<|endoftext|>


<commit_before>var element, axisSpace, pathClicks, outerElement, data;

$(function() {

	var element = "#graphics";

	axisSpace = 15;

	$("#uploadLink").on("click", function(event) { // code run whenever #uploadLink is clicked

		//#uploadLink is basically just a dummy element that we use to activate the hidden #uploadInput element
    	event.preventDefault();
        $("#uploadInput").trigger("click");

    });

    $("#uploadInput").change(function() { //code called by $("#uploadInput").trigger("click");

		var file = $("#uploadInput")[0].files[0]; //the file uploaded by the user

		if (validateXLSX(file)) {

			showSpinner();
			parseXLS(file);

		} else {

			console.log("invalid input");
			// showError();

		}

	});

	$("input[type=radio]").change(function() {

		data = renderVisualization(this.value == "stream", element, data)

	});

});

function validateXLSX(file) {

	console.log(file.name);

	var extension = file.name.split(".").slice(-1)[0];

	return extension == "xlsx" || extension == "xls";

}

function parseXLS(XLS) {

	var reader = new FileReader();

	reader.onload = function(e) {

		var data = e.target.result;
		var rABS = false; //actually determine this instead of just setting it statically
		var workbook;

		if (rABS) {

		    // if binary string, read with type "binary"
		    workbook = XLSX.read(data, {type: "binary"});

		} else {

		    // if array buffer, convert to base64
		    var arr = fixdata(data);
		    workbook = XLSX.read(btoa(arr), {type: "base64"});

		}

		readWorkbook(workbook);

	};

	reader.readAsArrayBuffer(XLS);
}

function readWorkbook(workbook){

	var sheetNames = workbook.SheetNames;

	console.log(workbook);
	console.log(sheetNames);

	for (i in sheetNames) {

		var sheet = sheetNames[i];

		if (sheet == "Column Descriptions") { //what sheets are we actually parsing? how will they be named?
			console.log("skipping " + sheet);
			continue;
		}

		crudeSheet = XLSX.utils.sheet_to_json(workbook.Sheets[sheet]);
		parsedSheet = parseCrude(crudeSheet);

	}
}

//parses the "crude json" which is sheetJS's export of an xls row to a json
function parseCrude(sheet) {

	//we're going to be rendering each different field specifically
	//i.e., we'll define a way to render the chromosome, and we'll be defining a way to render the clinvar data
	//so is it really necessary to separate the different kinds of fields?

	var visualizationData = [];

	var columns = [

		//basic information
		"Chromosome",
		"Position",
		"Reference Allele",
		"Sample Allele",
		"Variation Type",
		"QUAL",
		"FILTER",
		"GT",

		//gene info
		"Gene Region",
		"Gene Symbol",
		"Transcript ID",
		"Transcript Variant",
		"Protein Variant",
		"Translation Impact", // < missense, frameshift, stop loss, stop gain, ...

		//analytic overview
		"Classification", // < Uncertain Signifi

		//models
		"SIFT Function Prediction",
		"PolyPhen-2 Function Prediction",
		"CADD Score",
		"Phylop",
		"MutationTaster",
		"fathmm",
		"Sift",

		//frequencies:: LOTS OF MISSING CELLS
		"1000 Genomes Frequency",
		"ExAC Frequency",
		"ExAC East Asian Frequency",
		"ExAC South Asian Frequency",
		"ExAC African Frequency",
		"ExAC European Frequency",
		"ExAC Latino Frequency",
		"ExAC Homozygous Count",
		"AN_AFR",
		"AN_AMR",
		"AN_ASJ",
		"AN_EAS",
		"AN_FIN",
		"AN_NFE",
		"AN_OTH",
		"AN_SAS",
		"GNOMADMaxAlleleFreq",
		"GNOMAD_Max_Allele_Freq_POP"

		// "TIER" Tier
	]

	for (i in sheet) {

		var row = sheet[i];

		var variant = {
			"core" : {},
			"metadata" : {
				"metrics": {
					"nClicks" : 0
				}, "workflow": {
					"curationMode" : "sheetname",
					"notes" : "notes"
				}
			}
		};

		function fillTemplate(originalValue, column) {

			var value = parseValue(originalValue, column);

			console.log(column + ": " + originalValue + " - " + value);

			return {
				"value" : value, //if value is null, just assign empty string
				"originalValue" : originalValue,
				"associatedValues" : []
			}

		}

		$.each(columns, (_, column) => {

			//make variant.core a dictionary where the keys are the column names and the values are the template returned by filledTemplate
			variant.core[column] = fillTemplate(row[column], column);

		});

		visualizationData.push(variant);
	}

	console.log(visualizationData);
	renderVisualization(false, "#graphics", visualizationData); //render the visualization
	hideSpinner();
}

function parseValue(originalValue, column) {

	//TODO: empty cells

	//implement universal way to check if originalValue not in dict
	//pass originalValue and dict to some function?

	var value = originalValue;
	var dict;

	//model scores
	//should all be scaled to [0, 1], where 0 is least pathogenic and 1 is most pathogenic (that can be given on that scale)
	//Sift Function Prediction, PolyPhen-2 Function Prediction, CADD Score, Phylop, MutationTaster, fathmm, Sift

	var modelScores = [
		"SIFT Function Prediction","PolyPhen-2 Function Prediction","CADD Score",
		"Phylop","MutationTaster","fathmm","Sift"
	];

	if (column == "SIFT Function Prediction") {

		dict = {"Tolerated" : 0, "Damaging" : 1};
		value = dict[originalValue];

	} else if (column == "PolyPhen-2 Function Prediction") {

		dict = {"Benign" : 0, "Probably Damaging" : 1};
		value = dict[originalValue];

	} else if (column == "CADD Score") {

		//range?? 1 to 99?
		value = (parseFloat(originalValue) - 1) / 98;

	} else if (column == "Phylop") {

		//range?? try 0 to 1

		value = parseFloat(originalValue)

	} else if (column == "MutationTaster") {

		//range?? 0 to 1
		value = parseFloat(originalValue);

	} else if (column == "fathmm") {

		//range: [-16.13, 10.64]

		value = (parseFloat(originalValue) + 16.13) / 26.77;

	} else if (column == "Sift") {

		//goes from 0 to 1, where 0 is most deletrious and 1 is most tolerated??
		value = 1 - parseFloat(originalValue);

	}

	//frequencies
	//should all be in [0, 1]
	//primary: ExAC Frequency
	//GNOMADMaxAlleleFreq
	//population: ExAC East Asian Frequency, ExAC South Asian Frequency, ExAC African Frequency, ExAC European Frequency, ExAC Latino Frequency
	//AF_EAS, AF_NFE, AF_SAS, AF_AMR, AF_AFR
	//AN_AFR, AN_AMR, AN_ASJ, AN_EAS, AN_FIN, AN_NFE, AN_OTH, AN_SAS

	var frequencies = ["1000 Genomes Frequency","ExAC Frequency","GNOMADMaxAlleleFreq",
		"ExAC East Asian Frequency","ExAC South Asian Frequency","ExAC African Frequency","ExAC European Frequency","ExAC Latino Frequency",
		"AF_EAS","AF_NFE","AF_SAS","AF_AMR","AF_AFR"
	];

	if ($.inArray(column, frequencies) !== -1) {
		value = parseFloat(originalValue);
	}

	//scale model scores and frequencies
	if ($.inArray(column, modelScores.concat(frequencies)) !== -1) {
		console.log("scaling")
		return scaleValue(value);
	}

	return value;

}

function scaleValue(value) {

	var scale = d3.scalePow()
		.exponent(3)
		.clamp(true) //always return value inside the range, even if input is outside domain
		.domain([0, 1])
		.range([0, 1]);

	return scale(value);

}

function zeroOneNormalize(val, minVal, maxVal){
	//do normalization  do different distributions??
	return .3;
}

function isChromosome(t) {

	return parseInt(d3.select(t).attr("data-isChromosome"));

}<commit_msg>added function to normalize all values to a 0-1 range and error statements for values in invalid ranges and changes to ranges presented<commit_after>var element, axisSpace, pathClicks, outerElement, data;

$(function() {

	var element = "#graphics";

	axisSpace = 15;

	$("#uploadLink").on("click", function(event) { // code run whenever #uploadLink is clicked

		//#uploadLink is basically just a dummy element that we use to activate the hidden #uploadInput element
    	event.preventDefault();
        $("#uploadInput").trigger("click");

    });

    $("#uploadInput").change(function() { //code called by $("#uploadInput").trigger("click");

		var file = $("#uploadInput")[0].files[0]; //the file uploaded by the user

		if (validateXLSX(file)) {

			showSpinner();
			parseXLS(file);

		} else {

			console.log("invalid input");
			// showError();

		}

	});

	$("input[type=radio]").change(function() {

		data = renderVisualization(this.value == "stream", element, data)

	});

});

function validateXLSX(file) {

	console.log(file.name);

	var extension = file.name.split(".").slice(-1)[0];

	return extension == "xlsx" || extension == "xls";

}

function parseXLS(XLS) {

	var reader = new FileReader();

	reader.onload = function(e) {

		var data = e.target.result;
		var rABS = false; //actually determine this instead of just setting it statically
		var workbook;

		if (rABS) {

		    // if binary string, read with type "binary"
		    workbook = XLSX.read(data, {type: "binary"});

		} else {

		    // if array buffer, convert to base64
		    var arr = fixdata(data);
		    workbook = XLSX.read(btoa(arr), {type: "base64"});

		}

		readWorkbook(workbook);

	};

	reader.readAsArrayBuffer(XLS);
}

function readWorkbook(workbook){

	var sheetNames = workbook.SheetNames;

	console.log(workbook);
	console.log(sheetNames);

	for (i in sheetNames) {

		var sheet = sheetNames[i];

		if (sheet == "Column Descriptions") { //what sheets are we actually parsing? how will they be named?
			console.log("skipping " + sheet);
			continue;
		}

		crudeSheet = XLSX.utils.sheet_to_json(workbook.Sheets[sheet]);
		parsedSheet = parseCrude(crudeSheet);

	}
}

//parses the "crude json" which is sheetJS's export of an xls row to a json
function parseCrude(sheet) {

	//we're going to be rendering each different field specifically
	//i.e., we'll define a way to render the chromosome, and we'll be defining a way to render the clinvar data
	//so is it really necessary to separate the different kinds of fields?

	var visualizationData = [];

	var columns = [

		//basic information
		"Chromosome",
		"Position",
		"Reference Allele",
		"Sample Allele",
		"Variation Type",
		"QUAL",
		"FILTER",
		"GT",

		//gene info
		"Gene Region",
		"Gene Symbol",
		"Transcript ID",
		"Transcript Variant",
		"Protein Variant",
		"Translation Impact", // < missense, frameshift, stop loss, stop gain, ...

		//analytic overview
		"Classification", // < Uncertain Signifi

		//models
		"SIFT Function Prediction",
		"PolyPhen-2 Function Prediction",
		"CADD Score",
		"Phylop",
		"MutationTaster",
		"fathmm",
		"Sift",

		//frequencies:: LOTS OF MISSING CELLS
		"1000 Genomes Frequency",
		"ExAC Frequency",
		"ExAC East Asian Frequency",
		"ExAC South Asian Frequency",
		"ExAC African Frequency",
		"ExAC European Frequency",
		"ExAC Latino Frequency",
		"ExAC Homozygous Count",
		"AN_AFR",
		"AN_AMR",
		"AN_ASJ",
		"AN_EAS",
		"AN_FIN",
		"AN_NFE",
		"AN_OTH",
		"AN_SAS",
		"GNOMADMaxAlleleFreq",
		"GNOMAD_Max_Allele_Freq_POP"

		// "TIER" Tier
	]

	for (i in sheet) {

		var row = sheet[i];

		var variant = {
			"core" : {},
			"metadata" : {
				"metrics": {
					"nClicks" : 0
				}, "workflow": {
					"curationMode" : "sheetname",
					"notes" : "notes"
				}
			}
		};

		function fillTemplate(originalValue, column) {

			var value = parseValue(originalValue, column);

			console.log(column + ": " + originalValue + " - " + value);

			return {
				"value" : value, //if value is null, just assign empty string
				"originalValue" : originalValue,
				"associatedValues" : []
			}

		}

		$.each(columns, (_, column) => {

			//make variant.core a dictionary where the keys are the column names and the values are the template returned by filledTemplate
			variant.core[column] = fillTemplate(row[column], column);

		});

		visualizationData.push(variant);
	}

	console.log(visualizationData);
	renderVisualization(false, "#graphics", visualizationData); //render the visualization
	hideSpinner();
}

//given the original value and the column returns a drawing value
function parseValue(originalValue, column) {

	//TODO: empty cells

	//implement universal way to check if originalValue not in dict
	//pass originalValue and dict to some function?

	var value = originalValue;
	var dict;

	//model scores
	//should all be scaled to [0, 1], where 0 is least pathogenic and 1 is most pathogenic (that can be given on that scale)
	//Sift Function Prediction, PolyPhen-2 Function Prediction, CADD Score, Phylop, MutationTaster, fathmm, Sift

	var modelScores = [
		"SIFT Function Prediction","PolyPhen-2 Function Prediction","CADD Score",
		"Phylop","MutationTaster","fathmm","Sift"
	];

	if (column == "SIFT Function Prediction") {

		dict = {"Tolerated" : 0, "Activating": .5, "Damaging" : 1};
		value = convertStringToNumericAnnotation(originalValue, dict, "SIFT Function Prediction");
		return value;

	} else if (column == "PolyPhen-2 Function Prediction") {
		dict = {"Benign" : 0, "Probably Damaging" : 1};
		value = convertStringToNumericAnnotation(originalValue, dict, "PolyPhen-2 Function Prediction");
		return value;

	} else if (column == "CADD Score") {

		//info: The last column of the provided files is the PHRED-like (-10*log10(rank/total)) scaled C-score ranking a variant relative to all possible substitutions of the human genome (8.6x10^9). Like explained above, a scaled C-score of greater of equal 10 indicates that these are predicted to be the 10% most deleterious substitutions that you can do to the human genome, a score of greater or equal 20 indicates the 1% most deleterious and so on.  http://cadd.gs.washington.edu/info
		var minCaddScore = 1;
		var maxCaddScore = 99;
		var value = parseFloat(originalValue);
		var normValue = zeroOneNormalizeValue(value, minCaddScore, maxCaddScore, "CADD Score", false);
		return scaleValue(normValue);

	} else if (column == "Phylop") {
		//info: we use the mamilian phylop rankscore
		//a rank score is always between 0 and 1 and a score of 0.9 means it is more likely to be damaging than 90% of all potential nsSNVs predicted by that method   https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4752381/
		var value = parseFloat(originalValue);
		var normValue = zeroOneNormalizeValue(value, 0, 1, "Phylop", false);
		return scaleValue(value);

	} else if (column == "MutationTaster") {

		//info: we use mutation taster converted rankscore.  1 is more damaging, 0 is less damaging, see phylop above for more details
		value = parseFloat(originalValue);
		var normValue = zeroOneNormalizeValue(value, 0, 1, "MutationTaster", false);
		return scaleValue(value);

	} else if (column == "fathmm") {

		//range: [-16.13, 10.64]
		//Positive FATHMM scores predict a tolerance to the variation while negative FATHMM scores predict intolerance to the variation, and is subsequently considered to be pathogenic. Following proof of concept analysis it was determined that the best possible cut-off value for the FATHMM score is 1.0  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4929716/
		var value = parseFloat(originalValue);
		var normValue = zeroOneNormalizeValue(value, -16.13, 10.64, "fathmm", true);
		return scaleValue(value);

	} else if (column == "Sift") {

		//Range 0 to 1 with values less than 0.05 usually considered intolerant. 40% of the values in this database are below 0.01.
		value = parseFloat(originalValue);
		var normValue = zeroOneNormalizeValue(value, 0, 1, "Sift", true);
		return scaleValue(value);
	}

	//frequencies
	//should all be in [0, 1]
	//primary: ExAC Frequency
	//GNOMADMaxAlleleFreq
	//population: ExAC East Asian Frequency, ExAC South Asian Frequency, ExAC African Frequency, ExAC European Frequency, ExAC Latino Frequency
	//AF_EAS, AF_NFE, AF_SAS, AF_AMR, AF_AFR
	//AN_AFR, AN_AMR, AN_ASJ, AN_EAS, AN_FIN, AN_NFE, AN_OTH, AN_SAS

	var frequencies = ["1000 Genomes Frequency","ExAC Frequency","GNOMADMaxAlleleFreq",
		"ExAC East Asian Frequency","ExAC South Asian Frequency","ExAC African Frequency","ExAC European Frequency","ExAC Latino Frequency",
		"AF_EAS","AF_NFE","AF_SAS","AF_AMR","AF_AFR"
	];

	if ($.inArray(column, frequencies) !== -1) {
		value = parseFloat(originalValue);
	}

	//scale model scores and frequencies
	if ($.inArray(column, modelScores.concat(frequencies)) !== -1) {
		console.log("scaling")
		return scaleValue(value);
	}

	return value;

}

//normalizes a value in the range 0-1
//takes as parameters: value, min possible value, max possible value, name or the value (for error reporting), boolean invert scale for when we want to flip the high and low values
//we want 1 to represent values that are of interest to us (pathogenic, rare etc).  If a value of 1 means not pathogenic and zero means pathogenic, we invert scale.  Similarly we invert the scale for frequencies so rare variants are most visualized
function zeroOneNormalizeValue(value, min, max, valueName, invertScale){
	var invalidValueSentinel = 0; //what sentinel value do we want?
	if(value < min || value > max){
		console.log('error value ' + value + 'for annotation ' + valueName + ' is invalid');
		return invalidValueSentinel;
	}
	else{
		var scale = d3.scale.linear().
		.clamp(true)
		.domain([min, max])
		.range([0, 1]);
		if(invertScale){
			return 1 - scale(value);
		}
		else{
			return scale(value);
		}
	}
}

function convertStringToNumericAnnotation(value, mappingDictionary, annotationName){
	var invalidValueSentinel = 0; //what sentinel value do we want?
	if(value in mappingDictionary){
		return mappingDictionary[value];
	}
	else{
		console.log('error value ' + value + 'for annotation ' + annotationName + ' is invalid');
		return invalidValueSentinel
	}
}

function scaleValue(value) {

	var scale = d3.scalePow()
		.exponent(3)
		.clamp(true) //always return value inside the range, even if input is outside domain
		.domain([0, 1])
		.range([0, 1]);

	return scale(value);

}

function isChromosome(t) {

	return parseInt(d3.select(t).attr("data-isChromosome"));

}<|endoftext|>


<commit_before>/*
 * By: Frances Go
 * Date: Jan 2015
 * Title: Casper Test Integration
 */

// casper.test.begin(testTitle, numberOfTests, callback)
casper.test.begin('Testing About the Courses page', 13, function(test){
    casper.start('http://localhost:3000/courses.html', function(){
        this.test.assertUrlMatch('/courses.html', 'currently in the courses page');
    	test.assertTitle('About the Courses', 'about the courses title is good');
  	});
    casper.then(function(){
        this.click('a[href="/"]');
    });
    casper.then(function() {
        this.test.assertUrlMatch('/', 'currently in the homepage');
    	this.echo('Page title is: ' + this.getTitle());
    	test.assertTitle(this.getTitle(), 'homepage title is good');
	});
    casper.then(function(){
        this.click('a[href="/courses.html"]');
    });
    casper.then(function(){
        this.test.assertUrlMatch('/courses.html', 'currently in the courses page');
        this.echo('Page title is: ' + this.getTitle());
        test.assertTitle(this.getTitle(), 'about the courses title is good');
    });
    casper.then(function(){
        this.test.assertExists('.breadcrumb li a', 'breadcrumbs found');
        this.test.assertEval(function() {
            return __utils__.findAll('.row .thumbnail img').length >= 6;
        }, 'there are at least 6 course images on the page');
    });
    casper.then(function(){
        this.click('a[href="/instructors.html"]');
    });
    casper.then(function(){
        this.test.assertUrlMatch('/instructors.html', 'currently in the instructors page');
        this.echo('Page title is: ' + this.getTitle());
        test.assertTitle(this.getTitle(), 'instructor title is good');
    });
    casper.then(function(){
        this.click('a[href="/courses.html"]');
    });
    casper.then(function(){
        this.test.assertUrlMatch('/courses.html', 'currently in the courses page');
        this.echo('Page title is: ' + this.getTitle());
        this.test.assertExists('footer div', 'footer information found');
    });
    casper.then(function(){
        this.test.assertEval(function() {
            return __utils__.findAll('.row h3 a').length >= 6;
        }, 'there are at least 6 course title links on the page');
    });
    casper.run(function(){
        test.done();
    });
});<commit_msg>Updating issue8test.js<commit_after>/*
 * Title: Casper Test Integration
 * By: Frances Go
 * Date: Jan 2015
 */

// casper.test.begin(testTitle, numberOfTests, callback)
casper.test.begin('Testing About the Courses page', 13, function(test){
    casper.start('http://localhost:3000/courses.html', function(){
        this.test.assertUrlMatch('/courses.html', 'currently in the courses page');
    	test.assertTitle('About the Courses', 'about the courses title is good');
  	});
    casper.then(function(){
        this.click('a[href="/"]');
    });
    casper.then(function() {
        this.test.assertUrlMatch('/', 'currently in the homepage');
    	this.echo('Page title is: ' + this.getTitle());
    	test.assertTitle(this.getTitle(), 'homepage title is good');
	});
    casper.then(function(){
        this.click('a[href="/courses.html"]');
    });
    casper.then(function(){
        this.test.assertUrlMatch('/courses.html', 'currently in the courses page');
        this.echo('Page title is: ' + this.getTitle());
        test.assertTitle(this.getTitle(), 'about the courses title is good');
    });
    casper.then(function(){
        this.test.assertExists('.breadcrumb li a', 'breadcrumbs found');
        this.test.assertEval(function() {
            return __utils__.findAll('.row .thumbnail img').length >= 6;
        }, 'there are at least 6 course images on the page');
    });
    casper.then(function(){
        this.click('a[href="/instructors.html"]');
    });
    casper.then(function(){
        this.test.assertUrlMatch('/instructors.html', 'currently in the instructors page');
        this.echo('Page title is: ' + this.getTitle());
        test.assertTitle(this.getTitle(), 'instructor title is good');
    });
    casper.then(function(){
        this.click('a[href="/courses.html"]');
    });
    casper.then(function(){
        this.test.assertUrlMatch('/courses.html', 'currently in the courses page');
        this.echo('Page title is: ' + this.getTitle());
        this.test.assertExists('footer div', 'footer information found');
    });
    casper.then(function(){
        this.test.assertEval(function() {
            return __utils__.findAll('.row h3 a').length >= 6;
        }, 'there are at least 6 course title links on the page');
    });
    casper.run(function(){
        test.done();
    });
});<|endoftext|>