<commit_before>lang = [
 ["Random/Crypted", 0.045],
 ["Italian",0.075],
 ["English", 0.065],
]
treshold = 0.0049

String.prototype.count = function (string) {
  return (this.match(new RegExp(string,'g')) || []).length
}

var mod = function(x, y) {
	 return (x % y + y) % y;
}

var ctoi = function(char){
 return char.charCodeAt(0)
}

/*
---- Math Funciton ----
*/

var IC = function(string) {
  var n = string.length
  var freq = IC_dict(string)
  // Sum of - fi*(fi-1)
  var sum = 0
  for(var i=0;i<26;i++){
    temp = freq[i]-1
    sum += (freq[i]*temp)
  }
  return sum/(n*(n-1))
}

var IC_dict = function(string){
  var freq = []
  // Single Char frequency - fi
  for(var i=0;i<26;i++){
    freq.push(string.count(String.fromCharCode("a".charCodeAt(0)+i)))
  }
  return freq
}

var IMC = function(str1,str2){
  var n1 = str1.length
  var n2 = str2.length
  var freq1 = IC_dict(str1)
  var freq2 = IC_dict(str2)
  // Sum of - fi1*fi2
  var sum = 0
  for(var i=0;i<26;i++){
    sum += (freq1[i]*freq2[i])
  }
  return sum/(n1*n2)
}

var shift = function(str,key){
  var result = ""
  for (var i = 0; i < str.length; i++) {
    var c = str.charCodeAt(i)
    if      (c >= 65 && c <=  90) result += String.fromCharCode(mod(c - 65 - key, 26) + 65)  // Uppercase
    else if (c >= 97 && c <= 122) result += String.fromCharCode(mod(c - 97 - key, 26) + 97)  // Lowercase
    else result += str.charAt(i)  // Copy
  }
  return result
}

var filter = function(string) {
  var string = string.toLowerCase()
  var ret = ""
  for(var i=0;i<string.length;i++){
    actual = string.charCodeAt(i)
    if(actual >= 97 && actual <= 122){
      ret += String.fromCharCode(actual)
    }
  }
  return ret
}

var divide = function(string, part){
  var arr = []
  for(var i=0;i<part;i++){
    arr.push("")
  }
  for(var i=0;i<string.length;i++){
    arr[i%part] += string[i]
  }
  return arr
}

var check_lang = function(ic_val){
  var l = []
  for(var i=0;i<lang.length;i++){
    if(i==0){
      if(lang[0][1]+treshold>ic_val){
        l[0] = lang[i][0]
        l[1] = i
        break
      }
    }else{
      if(ic_val<lang[i][1]+treshold && ic_val>lang[i][1]-treshold){
        l[0] = lang[i][0]
        l[1] = i
        break
      }
    }
  }
  return l
}

var get_entropy = function(str,dict) {
 	var sum = 0
 	var ignored = 0
 	for (var i = 0; i < str.length; i++) {
  		var c = str.charCodeAt(i)
  		if      (c >= 65 && c <=  90) sum += Math.log(dict[c - 65])  // Uppercase
  		else if (c >= 97 && c <= 122) sum += Math.log(dict[c - 97])  // Lowercase
  		else ignored++
 	}
 	return -sum / Math.log(2) / (str.length - ignored)
}

var get_all_entropies = function(str,dict) {
 	var result = new Array(26)
 	for (var i = 0; i < 26; i++)
  		result[i] = [i, get_entropy(shift(str, i),dict)]
 	return result
}

/*
---- GUI Funciton ----
*/

function lang_guess(){
  var string = $("#cipher").val()
  string = filter(string)
  var ic_val = IC(string)
  console.log(ic_val)
  var msg = "Text's Coincidence Index is: "+ic_val.toFixed(4)
  var l = check_lang(ic_val)
  if(l[0]!=null){
    msg += "
Language is: "+l[0]
  }
  $("#lang_result").html(msg)
  return l
}

function brute_keylen(){
  for(var i=1;i<15;i++){
    r=keylen_guess(i)
    if(r!=null){
      return r
    }
  }
}

function shift_text(){
  var string = $("#cipher").val()
  string = filter(string)

  var key = $("#shift_key").val()
  if(key=="" || isNaN(key)){
    alert("Insert a valid Shift key")
  }else{
    $("#decipher").val(shift(string,parseInt(key)))
  }
}

function shift_guess(){
  var l = lang_guess()
  var langObj = null
  if(l[1]==1 || l[1]==2){
    langObj = DICT[l[1]]
  }else{
    $("#shift_result").html("Can't detect the text language, probably it's a Polialphabetic Ciphertext")
    return
  }

  var string = $("#cipher").val()
  string = filter(string)

  entropies = get_shift_entropies(string,langObj)
 	entropies.sort(function(x, y) {
  		// Compare by lowest entropy, break ties by lowest shift
  		if (x[1] < y[1]) return -1
  		else if (x[1] > y[1]) return 1
  		else if (x[0] < y[0]) return -1
  		else if (x[0] > y[0]) return 1
  		else return 0
 	});

  var best_shift = entropies[0][0]

  $("#shift_result").html("This text is (probably) shifted by "+best_shift+" with "+entropies[0][1].toFixed(4)+" entropy")

  $("#table_mono").html("")
  var $table = $('
');
  $table.append('
').children('thead')
    .append('
').children('tr').append("Shift	").append("Entropy	")

  var $tbody = $table.append('
').children('tbody')
  $tbody.append('
')

  for(var i=0;i<entropies.length;i++){
    $tbody.append('
').append(""+entropies[i][0]+"	").append(""+entropies[i][1]+"	")
  }
  $table.appendTo('#table_mono')

  $("#decipher").val(shift(string,best_shift))
}

function keylen_guess(kl){
  var keylen = kl
  var found = false
  if(keylen=="" || isNaN(keylen)){
    alert("Insert a valid key-lenght")
  }else{
    keylen = parseInt(keylen)

    $("#table").html("")
    var $table = $('
');
    $table.append('
').children('thead')
      .append('
').children('tr').append('Coincidence Index for Key-Length '+keylen+'	');

    var $tbody = $table.append('
').children('tbody');

    var arr = divide(filter($("#cipher").val()),keylen)
    var sum = 0
    if(arr.length == keylen){
      for(var i=0;i<keylen;i++){
        var ic_arr = IC(arr[i])
        sum += ic_arr
        $tbody.append('
').append(""+ic_arr+"	")
      }
      var media = sum/keylen
      var l = check_lang(media)
      if(l.length!=0 && l[0]!=lang[0][0]){
        $tbody.append('
').append("Found language: "+l[0]+"!	")
        found = true
      }

      $table.appendTo('#table')
      if(found==true) return [ic_arr,arr]
    } else {
      alert("Divide Error!")
    }
  }
  return null
}

function guess_key(){
  var arr = brute_keylen()[1]
  var k = []
  var ic = 0
  var max = [null,0.0000]

  for(var i=0;i<arr.length-1;i++){
    imc = IMC(arr[i],arr[i+1])
    max = [0,imc]
    for(var j=1;j<26;j++){
      shifted = shift(arr[i+1],j)
      imc = IMC(arr[i],shifted)
      if(imc>max[1]){
        max = [j,imc]
      }
    }
    k.push(max[0])
    ic += max[1]
  }
  ic = ic/k.length
  // find out the best K(0)
  var l = check_lang(ic)
  var langObj = null
  if(l[1]==1 || l[1]==2){
    langObj = DICT[l[1]]
  }else{
    $("#key_result").html("Can't detect the text language :(")
    return
  }
  console.log(k+" "+langObj)
  // Bruteforce entroy on C(0) with K(0)
  entropies = get_all_entropies(arr[0],langObj)
  entropies.sort(function(x, y) {
    // Compare by lowest entropy, break ties by lowest shift
    if (x[1] < y[1]) return -1
    else if (x[1] > y[1]) return 1
    else if (x[0] < y[0]) return -1
    else if (x[0] > y[0]) return 1
    else return 0
  });

  var best_shift = entropies[0][0]
  var msg = "Best entropy shift for K0 is: "+best_shift+"
"
  // Now find the other shift
  var key = String.fromCharCode(best_shift+97)
  var prec = best_shift
  for(var i=0;i<k.length;i++){
    prec = mod(prec + k[i],26)
    key += String.fromCharCode(prec+97)
  }
  msg += "The most probable key is: "+key
  $("#key_result").html(msg)
}

function clear_DOM(){
  $("#table").html("")
  $("#table_mono").html("")
  $("#decipher").val("")
  $("#shift_result").html("")
  $("#lang_result").html("")
}
<commit_msg>final<commit_after>lang = [
 ["Random/Crypted", 0.045],
 ["Italian",0.075],
 ["English", 0.065],
]
treshold = 0.0049

String.prototype.count = function (string) {
  return (this.match(new RegExp(string,'g')) || []).length
}

var mod = function(x, y) {
	 return (x % y + y) % y;
}

var ctoi = function(char){
 return char.charCodeAt(0)
}

/*
---- Math Funciton ----
*/

var IC = function(string) {
  var n = string.length
  var freq = IC_dict(string)
  // Sum of - fi*(fi-1)
  var sum = 0
  for(var i=0;i<26;i++){
    temp = freq[i]-1
    sum += (freq[i]*temp)
  }
  return sum/(n*(n-1))
}

var IC_dict = function(string){
  var freq = []
  // Single Char frequency - fi
  for(var i=0;i<26;i++){
    freq.push(string.count(String.fromCharCode("a".charCodeAt(0)+i)))
  }
  return freq
}

var IMC = function(str1,str2){
  var n1 = str1.length
  var n2 = str2.length
  var freq1 = IC_dict(str1)
  var freq2 = IC_dict(str2)
  // Sum of - fi1*fi2
  var sum = 0
  for(var i=0;i<26;i++){
    sum += (freq1[i]*freq2[i])
  }
  return sum/(n1*n2)
}

var shift = function(str,key){
  var result = ""
  if(typeof key == "string"){
    // Vigenere
    for (var i = 0; i < str.length; i++) {
      var index = i%key.length
      var c = str.charCodeAt(i)
      if      (c >= 65 && c <=  90) result += String.fromCharCode(mod(c - key.charCodeAt(index), 26) + 65)  // Uppercase
      else if (c >= 97 && c <= 122) result += String.fromCharCode(mod(c - key.charCodeAt(index), 26) + 97)  // Lowercase
      else result += str.charAt(i)  // Copy
    }
  }else{
    // Shift
    for (var i = 0; i < str.length; i++) {
      var c = str.charCodeAt(i)
      if      (c >= 65 && c <=  90) result += String.fromCharCode(mod(c - 65 - key, 26) + 65)  // Uppercase
      else if (c >= 97 && c <= 122) result += String.fromCharCode(mod(c - 97 - key, 26) + 97)  // Lowercase
      else result += str.charAt(i)  // Copy
    }
  }
  return result
}

var filter = function(string) {
  var string = string.toLowerCase()
  var ret = ""
  for(var i=0;i<string.length;i++){
    actual = string.charCodeAt(i)
    if(actual >= 97 && actual <= 122){
      ret += String.fromCharCode(actual)
    }
  }
  return ret
}

var divide = function(string, part){
  var arr = []
  for(var i=0;i<part;i++){
    arr.push("")
  }
  for(var i=0;i<string.length;i++){
    arr[i%part] += string[i]
  }
  return arr
}

var check_lang = function(ic_val){
  var l = []
  for(var i=0;i<lang.length;i++){
    if(i==0){
      if(lang[0][1]+treshold>ic_val){
        l[0] = lang[i][0]
        l[1] = i
        break
      }
    }else{
      if(ic_val<lang[i][1]+treshold && ic_val>lang[i][1]-treshold){
        l[0] = lang[i][0]
        l[1] = i
        break
      }
    }
  }
  return l
}

var get_entropy = function(str,dict) {
 	var sum = 0
 	var ignored = 0
 	for (var i = 0; i < str.length; i++) {
  		var c = str.charCodeAt(i)
  		if      (c >= 65 && c <=  90) sum += Math.log(dict[c - 65])  // Uppercase
  		else if (c >= 97 && c <= 122) sum += Math.log(dict[c - 97])  // Lowercase
  		else ignored++
 	}
 	return -sum / Math.log(2) / (str.length - ignored)
}

var get_all_entropies = function(str,dict) {
 	var result = new Array(26)
 	for (var i = 0; i < 26; i++)
  		result[i] = [i, get_entropy(shift(str, i),dict)]
 	return result
}

/*
---- GUI Funciton ----
*/

function lang_guess(){
  var string = $("#cipher").val()
  string = filter(string)
  var ic_val = IC(string)
  console.log(ic_val)
  var msg = "Text's Coincidence Index is: "+ic_val.toFixed(4)
  var l = check_lang(ic_val)
  if(l[0]!=null){
    msg += "
Language is: "+l[0]
  }
  $("#lang_result").html(msg)
  return l
}

function brute_keylen(){
  for(var i=1;i<15;i++){
    r=keylen_guess(i)
    if(r!=null){
      return r
    }
  }
}

function shift_text(){
  var string = $("#cipher").val()
  string = filter(string)

  var key = $("#shift_key").val()
  if(key=="" || isNaN(key)){
    alert("Insert a valid Shift key")
  }else{
    $("#decipher").val(shift(string,parseInt(key)))
  }
}

function shift_guess(){
  var l = lang_guess()
  var langObj = null
  if(l[1]==1 || l[1]==2){
    langObj = DICT[l[1]]
  }else{
    $("#shift_result").html("Can't detect the text language, probably it's a Polialphabetic Ciphertext")
    return
  }

  var string = $("#cipher").val()
  string = filter(string)

  entropies = get_shift_entropies(string,langObj)
 	entropies.sort(function(x, y) {
  		// Compare by lowest entropy, break ties by lowest shift
  		if (x[1] < y[1]) return -1
  		else if (x[1] > y[1]) return 1
  		else if (x[0] < y[0]) return -1
  		else if (x[0] > y[0]) return 1
  		else return 0
 	});

  var best_shift = entropies[0][0]

  $("#shift_result").html("This text is (probably) shifted by "+best_shift+" with "+entropies[0][1].toFixed(4)+" entropy")

  $("#table_mono").html("")
  var $table = $('
');
  $table.append('
').children('thead')
    .append('
').children('tr').append("Shift	").append("Entropy	")

  var $tbody = $table.append('
').children('tbody')
  $tbody.append('
')

  for(var i=0;i<entropies.length;i++){
    $tbody.append('
').append(""+entropies[i][0]+"	").append(""+entropies[i][1]+"	")
  }
  $table.appendTo('#table_mono')

  $("#decipher").val(shift(string,best_shift))
}

function keylen_guess(kl){
  var keylen = kl
  var found = false
  if(keylen=="" || isNaN(keylen)){
    alert("Insert a valid key-lenght")
  }else{
    keylen = parseInt(keylen)

    $("#table").html("")
    var $table = $('
');
    $table.append('
').children('thead')
      .append('
').children('tr').append('Coincidence Index for Key-Length '+keylen+'	');

    var $tbody = $table.append('
').children('tbody');

    var arr = divide(filter($("#cipher").val()),keylen)
    var sum = 0
    if(arr.length == keylen){
      for(var i=0;i<keylen;i++){
        var ic_arr = IC(arr[i])
        sum += ic_arr
        $tbody.append('
').append(""+ic_arr+"	")
      }
      var media = sum/keylen
      var l = check_lang(media)
      if(l.length!=0 && l[0]!=lang[0][0]){
        $tbody.append('
').append("Found language: "+l[0]+"!	")
        found = true
      }

      $table.appendTo('#table')
      if(found==true) return [ic_arr,arr]
    } else {
      alert("Divide Error!")
    }
  }
  return null
}

function guess_key(){
  var arr = brute_keylen()[1]
  var k = []
  var ic = 0
  var max = [null,0.0000]

  for(var i=0;i<arr.length-1;i++){
    imc = IMC(arr[i],arr[i+1])
    max = [0,imc]
    for(var j=1;j<26;j++){
      shifted = shift(arr[i+1],j)
      imc = IMC(arr[i],shifted)
      if(imc>max[1]){
        max = [j,imc]
      }
    }
    k.push(max[0])
    ic += max[1]
  }
  ic = ic/k.length
  // find out the best K(0)
  var l = check_lang(ic)
  var langObj = null
  if(l[1]==1 || l[1]==2){
    langObj = DICT[l[1]]
  }else{
    $("#key_result").html("Can't detect the text language :(")
    return
  }
  console.log(k+" "+langObj)
  // Bruteforce entroy on C(0) with K(0)
  entropies = get_all_entropies(arr[0],langObj)
  entropies.sort(function(x, y) {
    // Compare by lowest entropy, break ties by lowest shift
    if (x[1] < y[1]) return -1
    else if (x[1] > y[1]) return 1
    else if (x[0] < y[0]) return -1
    else if (x[0] > y[0]) return 1
    else return 0
  });

  var best_shift = entropies[0][0]
  var msg = "Best entropy shift for K0 is: "+best_shift+"
"
  // Now find the other shift
  var key = String.fromCharCode(best_shift+97)
  var prec = best_shift
  for(var i=0;i<k.length;i++){
    prec = mod(prec + k[i],26)
    key += String.fromCharCode(prec+97)
  }
  msg += "The most probable key is: "+key
  $("#key_result").html(msg)

  var dec = ""
  var string = $("#cipher").val()
  string = filter(string)
  $("#decipher").val(shift(string,key))
}

function clear_DOM(){
  $("#table").html("")
  $("#table_mono").html("")
  $("#decipher").val("")
  $("#shift_result").html("")
  $("#lang_result").html("")
  $("#key_result").html("")
}
<|endoftext|>


<commit_before>var canvas = document.getElementById("mycanvas");
var gl;
var pos;
var pMatrixUniform;
var mvMatrixUniform;
var pMatrix = mat4.create();
var mvMatrix = mat4.create();
var samplerUniform;
var vShader;
var fShader;
var program;
var texCoordAttr;
var texCoords;
var vpWidth;
var vpHeight;
var loading = true;
var realFrame = null;
var loadingT0 = new Date().getTime();
var outstandingImages = 0;

function resizeViewport(width, height)
{
    canvas.width = width;
    canvas.height = height;
    vpWidth = width;
    vpHeight = height;
    gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
}

function getNumericStyleProperty(style, prop){
    return parseInt(style.getPropertyValue(prop),10) ;
}

function element_position(e) {
    var x = 0, y = 0;
    var inner = true ;
    do {
        x += e.offsetLeft;
        y += e.offsetTop;
        var style = getComputedStyle(e,null) ;
        var borderTop = getNumericStyleProperty(style,"border-top-width") ;
        var borderLeft = getNumericStyleProperty(style,"border-left-width") ;
        y += borderTop ;
        x += borderLeft ;
        if (inner){
          var paddingTop = getNumericStyleProperty(style,"padding-top") ;
          var paddingLeft = getNumericStyleProperty(style,"padding-left") ;
          y += paddingTop ;
          x += paddingLeft ;
        }
        inner = false ;
    } while (e = e.offsetParent);
    return { x: x, y: y };
}

function initGL(handleMouseDown, handleMouseUp, handleMouseMove) {
    canvas.onmousedown = function(event) {
        var offsetpos = element_position(canvas);
        handleMouseDown(event.pageX-offsetpos.x, event.pageY-offsetpos.y); }
    document.onmouseup = function(event) {
        var offsetpos = element_position(canvas);
        handleMouseUp(event.pageX-offsetpos.x, event.pageY-offsetpos.y); }
    document.onmousemove = function(event) {
        var offsetpos = element_position(canvas);
        handleMouseMove(event.pageX-offsetpos.x, event.pageY-offsetpos.y); }

    return gl;
}

function loadImage(fn, bg)
{
    if (outstandingImages == 0)
        loadingT0 = new Date().getTime();
    outstandingImages++;
    var tex = gl.createTexture();
    tex.loaded = false;
    tex.deleted = false;
    tex.image = new Image();
    if (bg) {
        tex.image.onload = function() {
            if (!tex.deleted) {
                tex.imgWidth = tex.image.width;
                tex.imgHeight = tex.image.height;
                var newCanvas = document.createElement("canvas");
                newCanvas.width = 1024;
                newCanvas.height = 1024;
                var ctx = newCanvas.getContext("2d");
                ctx.drawImage(tex.image, 0, 0, newCanvas.width, newCanvas.height);
                tex.image = new Image();
                // here is the most important part because if you dont replace you will get a DOM 18 exception.
                tex.image.src = newCanvas.toDataURL("image/png").replace("image/png", "image/octet-stream");
                tex.image.onload = function () { outstandingImages--; handleLoadedTexture(gl,tex); }
            }
        };
    }
    else {
        tex.image.onload = function() { outstandingImages--; if (!tex.deleted) handleLoadedTexture(gl,tex) };
    }
    tex.image.src = fn;
    return tex;
}

function handleLoadedTexture(gl,tex) {
    gl.bindTexture(gl.TEXTURE_2D, tex);
    gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, tex.image);
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
    gl.bindTexture(gl.TEXTURE_2D, null);
    tex.loaded = true;
}

function prepareImage()
{
    gl.bindBuffer(gl.ARRAY_BUFFER, texCoords);
    gl.vertexAttribPointer(texCoordAttr, texCoords.itemSize, gl.FLOAT, false, 0, 0);

    gl.activeTexture(gl.TEXTURE0);

    gl.bindBuffer(gl.ARRAY_BUFFER, squareBuf);
    gl.enableVertexAttribArray(pos);
    gl.vertexAttribPointer(pos, squareBuf.itemSize, gl.FLOAT, false, 0, 0);
}

function startRendering()
{
    gl.clearColor(1, 1, 1, 1);
    gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);

    mat4.identity(pMatrix);
    mat4.scale(pMatrix, pMatrix, [0.001 * vpHeight / vpWidth, 0.001, 1]);
    gl.uniformMatrix4fv(pMatrixUniform, false, pMatrix);

    gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
    gl.enable(gl.BLEND);

    prepareImage();
}

function endRendering()
{
    if (loading || outstandingImages > 0) {
        var dt = new Date().getTime() - loadingT0
        if (dt >= 0.5)
            drawImage(loadingTex, 0, 0, 100, 100, false, dt/400);
    }
}

function requestAnimFrame2(f)
{
    if (loading) {
        realFrame = f;
        loading = false;
    }
    else
        requestAnimFrame(f);
}

function drawImage(tex, x, y, w, h, bg, rot)
{
    if (tex.loaded) {
        if (bg) {
            x = 0;
            y = 0;
            var scrnAspect = vpWidth / vpHeight;
            var imgAspect = tex.imgWidth / tex.imgHeight;
            if (imgAspect > scrnAspect) {
                w = 1000 * imgAspect;
                h = 1000;
            }
            else {
                w = 1000 * scrnAspect;
                h = w / imgAspect;
            }
        }

        mat4.identity(mvMatrix);
        mat4.translate(mvMatrix, mvMatrix, [x, y, 0]);
        if (rot != 0)
            mat4.rotate(mvMatrix, mvMatrix, rot, [0, 0, -1]);
        mat4.scale(mvMatrix, mvMatrix, [w, h, 1]);

        gl.bindTexture(gl.TEXTURE_2D, tex);
        gl.uniform1i(samplerUniform, 0);
        gl.uniformMatrix4fv(mvMatrixUniform, false, mvMatrix);
        gl.drawArrays(gl.TRIANGLE_STRIP, 0, squareBuf.numItems);
    }
}

function destroyImage(tex)
{
    tex.deleted = true;
    gl.deleteTexture(tex);
}

function getWindowWidth()
{
  if (typeof (window.innerWidth) == 'number')
    return window.innerWidth;
  else {
    if (document.documentElement && (document.documentElement.clientWidth || document.documentElement.clientHeight))
      return document.documentElement.clientWidth;
    else {
      if (document.body && (document.body.clientWidth || document.body.clientHeight))
        return document.body.clientWidth;
      else
        alert("no window width");
    }
  }
}

function getWindowHeight()
{
  if (typeof (window.innerHeight) == 'number')
    return window.innerHeight;
  else {
    if (document.documentElement && (document.documentElement.clientHeight || document.documentElement.clientHeight))
      return document.documentElement.clientHeight;
    else {
      if (document.body && (document.body.clientHeight || document.body.clientHeight))
        return document.body.clientHeight;
      else
        alert("no window width");
    }
  }
}

try {
    gl = canvas.getContext("webgl");
}
catch (x) { gl = null; }
if (gl == null) {
    try {
        gl = canvas.getContext("experimental-webgl");
    }
    catch (x) { gl = null; }
}
if (!gl) {
    alert("Could not initialise WebGL, sorry :-(");
}

vShader = createShaderFromScriptElement(gl, "shader-vs");
fShader = createShaderFromScriptElement(gl, "shader-fs");
var program = createProgram(gl, [vShader, fShader]);
gl.useProgram(program);

pos = gl.getAttribLocation(program, "aVertexPosition");
pMatrixUniform = gl.getUniformLocation(program, "uPMatrix");
mvMatrixUniform = gl.getUniformLocation(program, "uMVMatrix");

resizeViewport(canvas.offsetWidth, canvas.offsetHeight);

texCoordAttr = gl.getAttribLocation(program, "aTextureCoord");
gl.enableVertexAttribArray(texCoordAttr);
samplerUniform = gl.getUniformLocation(program, "uSampler");

texCoords = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, texCoords);
var textureCoords = [
  // Front face
  0.0, 0.0,
  1.0, 0.0,
  0.0, 1.0,
  1.0, 1.0
];
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(textureCoords), gl.STATIC_DRAW);
texCoords.itemSize = 2;
texCoords.numItems = 4;

squareBuf = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, squareBuf);
var vertices = [
    -1.0,  1.0,  0.0,
     1.0,  1.0,  0.0,
    -1.0, -1.0,  0.0,
     1.0, -1.0,  0.0
];
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(vertices), gl.STATIC_DRAW);
squareBuf.itemSize = 3;
squareBuf.numItems = 4;

var loadingTex = loadImage("loading.png", false);
function drawWhileLoading() {
    if (!loading) { realFrame(); realFrame = null; } else {
        startRendering();
        endRendering();
        requestAnimFrame(drawWhileLoading);
    }
};
requestAnimFrame(drawWhileLoading);
<commit_msg>Fix timing.<commit_after>var canvas = document.getElementById("mycanvas");
var gl;
var pos;
var pMatrixUniform;
var mvMatrixUniform;
var pMatrix = mat4.create();
var mvMatrix = mat4.create();
var samplerUniform;
var vShader;
var fShader;
var program;
var texCoordAttr;
var texCoords;
var vpWidth;
var vpHeight;
var loading = true;
var realFrame = null;
var loadingT0 = new Date().getTime();
var outstandingImages = 0;

function resizeViewport(width, height)
{
    canvas.width = width;
    canvas.height = height;
    vpWidth = width;
    vpHeight = height;
    gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
}

function getNumericStyleProperty(style, prop){
    return parseInt(style.getPropertyValue(prop),10) ;
}

function element_position(e) {
    var x = 0, y = 0;
    var inner = true ;
    do {
        x += e.offsetLeft;
        y += e.offsetTop;
        var style = getComputedStyle(e,null) ;
        var borderTop = getNumericStyleProperty(style,"border-top-width") ;
        var borderLeft = getNumericStyleProperty(style,"border-left-width") ;
        y += borderTop ;
        x += borderLeft ;
        if (inner){
          var paddingTop = getNumericStyleProperty(style,"padding-top") ;
          var paddingLeft = getNumericStyleProperty(style,"padding-left") ;
          y += paddingTop ;
          x += paddingLeft ;
        }
        inner = false ;
    } while (e = e.offsetParent);
    return { x: x, y: y };
}

function initGL(handleMouseDown, handleMouseUp, handleMouseMove) {
    canvas.onmousedown = function(event) {
        var offsetpos = element_position(canvas);
        handleMouseDown(event.pageX-offsetpos.x, event.pageY-offsetpos.y); }
    document.onmouseup = function(event) {
        var offsetpos = element_position(canvas);
        handleMouseUp(event.pageX-offsetpos.x, event.pageY-offsetpos.y); }
    document.onmousemove = function(event) {
        var offsetpos = element_position(canvas);
        handleMouseMove(event.pageX-offsetpos.x, event.pageY-offsetpos.y); }

    return gl;
}

function loadImage(fn, bg)
{
    if (outstandingImages == 0)
        loadingT0 = new Date().getTime();
    outstandingImages++;
    var tex = gl.createTexture();
    tex.loaded = false;
    tex.deleted = false;
    tex.image = new Image();
    if (bg) {
        tex.image.onload = function() {
            if (!tex.deleted) {
                tex.imgWidth = tex.image.width;
                tex.imgHeight = tex.image.height;
                var newCanvas = document.createElement("canvas");
                newCanvas.width = 1024;
                newCanvas.height = 1024;
                var ctx = newCanvas.getContext("2d");
                ctx.drawImage(tex.image, 0, 0, newCanvas.width, newCanvas.height);
                tex.image = new Image();
                // here is the most important part because if you dont replace you will get a DOM 18 exception.
                tex.image.src = newCanvas.toDataURL("image/png").replace("image/png", "image/octet-stream");
                tex.image.onload = function () { outstandingImages--; handleLoadedTexture(gl,tex); }
            }
        };
    }
    else {
        tex.image.onload = function() { outstandingImages--; if (!tex.deleted) handleLoadedTexture(gl,tex) };
    }
    tex.image.src = fn;
    return tex;
}

function handleLoadedTexture(gl,tex) {
    gl.bindTexture(gl.TEXTURE_2D, tex);
    gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, tex.image);
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
    gl.bindTexture(gl.TEXTURE_2D, null);
    tex.loaded = true;
}

function prepareImage()
{
    gl.bindBuffer(gl.ARRAY_BUFFER, texCoords);
    gl.vertexAttribPointer(texCoordAttr, texCoords.itemSize, gl.FLOAT, false, 0, 0);

    gl.activeTexture(gl.TEXTURE0);

    gl.bindBuffer(gl.ARRAY_BUFFER, squareBuf);
    gl.enableVertexAttribArray(pos);
    gl.vertexAttribPointer(pos, squareBuf.itemSize, gl.FLOAT, false, 0, 0);
}

function startRendering()
{
    gl.clearColor(1, 1, 1, 1);
    gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);

    mat4.identity(pMatrix);
    mat4.scale(pMatrix, pMatrix, [0.001 * vpHeight / vpWidth, 0.001, 1]);
    gl.uniformMatrix4fv(pMatrixUniform, false, pMatrix);

    gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
    gl.enable(gl.BLEND);

    prepareImage();
}

function endRendering()
{
    if (loading || outstandingImages > 0) {
        var dt = new Date().getTime() - loadingT0
        if (dt >= 500)
            drawImage(loadingTex, 0, 0, 100, 100, false, dt/400);
    }
}

function requestAnimFrame2(f)
{
    if (loading) {
        realFrame = f;
        loading = false;
    }
    else
        requestAnimFrame(f);
}

function drawImage(tex, x, y, w, h, bg, rot)
{
    if (tex.loaded) {
        if (bg) {
            x = 0;
            y = 0;
            var scrnAspect = vpWidth / vpHeight;
            var imgAspect = tex.imgWidth / tex.imgHeight;
            if (imgAspect > scrnAspect) {
                w = 1000 * imgAspect;
                h = 1000;
            }
            else {
                w = 1000 * scrnAspect;
                h = w / imgAspect;
            }
        }

        mat4.identity(mvMatrix);
        mat4.translate(mvMatrix, mvMatrix, [x, y, 0]);
        if (rot != 0)
            mat4.rotate(mvMatrix, mvMatrix, rot, [0, 0, -1]);
        mat4.scale(mvMatrix, mvMatrix, [w, h, 1]);

        gl.bindTexture(gl.TEXTURE_2D, tex);
        gl.uniform1i(samplerUniform, 0);
        gl.uniformMatrix4fv(mvMatrixUniform, false, mvMatrix);
        gl.drawArrays(gl.TRIANGLE_STRIP, 0, squareBuf.numItems);
    }
}

function destroyImage(tex)
{
    tex.deleted = true;
    gl.deleteTexture(tex);
}

function getWindowWidth()
{
  if (typeof (window.innerWidth) == 'number')
    return window.innerWidth;
  else {
    if (document.documentElement && (document.documentElement.clientWidth || document.documentElement.clientHeight))
      return document.documentElement.clientWidth;
    else {
      if (document.body && (document.body.clientWidth || document.body.clientHeight))
        return document.body.clientWidth;
      else
        alert("no window width");
    }
  }
}

function getWindowHeight()
{
  if (typeof (window.innerHeight) == 'number')
    return window.innerHeight;
  else {
    if (document.documentElement && (document.documentElement.clientHeight || document.documentElement.clientHeight))
      return document.documentElement.clientHeight;
    else {
      if (document.body && (document.body.clientHeight || document.body.clientHeight))
        return document.body.clientHeight;
      else
        alert("no window width");
    }
  }
}

try {
    gl = canvas.getContext("webgl");
}
catch (x) { gl = null; }
if (gl == null) {
    try {
        gl = canvas.getContext("experimental-webgl");
    }
    catch (x) { gl = null; }
}
if (!gl) {
    alert("Could not initialise WebGL, sorry :-(");
}

vShader = createShaderFromScriptElement(gl, "shader-vs");
fShader = createShaderFromScriptElement(gl, "shader-fs");
var program = createProgram(gl, [vShader, fShader]);
gl.useProgram(program);

pos = gl.getAttribLocation(program, "aVertexPosition");
pMatrixUniform = gl.getUniformLocation(program, "uPMatrix");
mvMatrixUniform = gl.getUniformLocation(program, "uMVMatrix");

resizeViewport(canvas.offsetWidth, canvas.offsetHeight);

texCoordAttr = gl.getAttribLocation(program, "aTextureCoord");
gl.enableVertexAttribArray(texCoordAttr);
samplerUniform = gl.getUniformLocation(program, "uSampler");

texCoords = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, texCoords);
var textureCoords = [
  // Front face
  0.0, 0.0,
  1.0, 0.0,
  0.0, 1.0,
  1.0, 1.0
];
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(textureCoords), gl.STATIC_DRAW);
texCoords.itemSize = 2;
texCoords.numItems = 4;

squareBuf = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, squareBuf);
var vertices = [
    -1.0,  1.0,  0.0,
     1.0,  1.0,  0.0,
    -1.0, -1.0,  0.0,
     1.0, -1.0,  0.0
];
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(vertices), gl.STATIC_DRAW);
squareBuf.itemSize = 3;
squareBuf.numItems = 4;

var loadingTex = loadImage("loading.png", false);
function drawWhileLoading() {
    if (!loading) { realFrame(); realFrame = null; } else {
        startRendering();
        endRendering();
        requestAnimFrame(drawWhileLoading);
    }
};
requestAnimFrame(drawWhileLoading);
<|endoftext|>


<commit_before>/* contains common configorations for webpack configs */
const nodeModules = require('./utils').nodeModules
const path = require('path')
const consts = require('./constants')
const pkg = require(path.resolve(process.cwd(), 'package.json'))
const _ = require('lodash')

const pathToReactDOM = nodeModules('react-dom/dist/react-dom.min.js')
const pathToReactRouter = nodeModules('react-router/umd/ReactRouter.min.js')
const pathToMomentTimezone = nodeModules('moment-timezone/builds/moment-timezone-with-data.min.js')

const getConfig = (env) => {
  //define all vendor depnencies with pathes relative to node_modules/
  //that are pre-compiled, this allows to configure webpack to skip parsing of them
  //and we can use them for prod build instead of minifying libs ourself we would use pre-combiled ones
  const preBuiltVendorDeps = {
    'react': nodeModules('react/dist/react.min.js'),
    'react-dom': pathToReactDOM,
    'react-router': pathToReactRouter,
    'react-router-redux': nodeModules('react-router-redux/dist/ReactRouterRedux.min.js'),
    'redux': nodeModules('redux/dist/redux.min.js'),
    'moment': nodeModules('moment/min/moment.min.js'),
    'moment-timezone': pathToMomentTimezone,
    'redux-saga': nodeModules('redux-saga/dist/redux-saga.min.js'),
    //TODO: modify for node to pull node-fetch
    'isomorphic-fetch': nodeModules('whatwg-fetch/fetch.js'),
  }

  // reading from package.json
  // NOTE: that it should only contains main app (main entry chunk "app") vendor deps, all other deps
  // should go to "devDependencies" and get automatically split between entry chunks
  const allVendorDeps = Object.keys(pkg.dependencies)
    //excluding regenerator-runtime since it results in error "regeneratorRuntime is not defined" in runtime
    //and manually specifying correct path
    .filter(x => x !== 'regenerator-runtime')
    .concat(['regenerator-runtime/runtime'])
  const nonPreBuiltVendorDeps = _.difference(allVendorDeps, Object.keys(preBuiltVendorDeps))

  const toArray = (obj) => {
    return Object.keys(obj).map(key => obj[key])
  }

  const buildResolveAliases = vendorDeps =>
    Object.keys(vendorDeps)
      .map(depName => ({
        //$ is used to exactly match dependency
        [depName + '$']: vendorDeps[depName]
      }))
      //reducing array of objects to one object with "name$":"path" structure as required by Webpack
      .reduce((a, b) => {
        const key = Object.keys(b)[0] //we know that it has only one key
        const value = b[key]

        a[key] = value
        return a
      })

  const getLoaders = (env) => {
    let loaders = []

    if (env === 'prod') {
      loaders = loaders.concat([{
        test: pathToReactDOM,
        loader: 'imports'
      }, {
        test: pathToReactRouter,
        loader: 'imports'
      }, {
        test: pathToMomentTimezone,
        loader: 'imports'
      }, {
        test: /\.(js|jsx?)$/,
        loader: 'babel',
        exclude: /(node_modules|bower_components)/,
        include: path.join(consts.SRC_DIR),
        query: {
          presets: ['react', 'es2015', 'stage-2'],
          cacheDirectory: false,
          compact: true, //so babel wont output whitespaces and stuff, speeds up build a little
          plugins: [
            'transform-react-constant-elements', //compile-time optimizations
            'transform-react-inline-elements' //compile-time optimizations
          ]
        }
      }])
    }

    if (env === 'dev') {
      loaders = loaders.concat([{
        test: /\.(js|jsx?)$/,
        loader: 'babel',
        exclude: /(node_modules|bower_components)/,
        include: [path.join(consts.SRC_DIR)],
        query: {
          presets: ['react', 'es2015', 'stage-2'],
          cacheDirectory: true, //not needed for prod build
          plugins: [
            ['react-transform', {
              'transforms': [{
                'transform': 'react-transform-hmr',
                'imports': ['react'],
                'locals': ['module']
              }, {
                'transform': 'react-transform-catch-errors',
                'imports': ['react', 'redbox-react']
              }]
            }]
          ]
        }
      }, {
        test: /\.scss$/,
        //loaders: ['style', ExtractTextPlugin.extract('css?localIdentName=[name]_[local]_[hash:base64:3]!sass')],
        //loaders: ['style', 'css?localIdentName=[name]_[local]_[hash:base64:3]', 'sass'],
        loader: 'style!css!postcss!sass',
        exclude: /(node_modules|bower_components)/,
        include: path.join(consts.SRC_DIR, 'client')
      }])
    }

    //common loaders
    loaders = loaders.concat([{
      test: /\.json$/,
      loader: 'json-loader',
    }, {
      test: /\.(woff|woff2)(\?v=\d+\.\d+\.\d+)?$/,
      exclude: /(node_modules|bower_components)/,
      loaders: ['url?limit=10000&mimetype=application/font-woff'],
      include: path.join(consts.SRC_DIR, 'client')
    }, {
      test: /\.(jpg|jpeg|gif|png|ico|svg)$/,
      exclude: /(node_modules|bower_components)/,
      include: path.join(consts.SRC_DIR, 'client'),
      loader: 'file-loader?name=[path][name].[ext]&context=' + consts.IMG_DIR
    }])

    return loaders
  }

  return {
    entry: {
      vendor: Object.keys(preBuiltVendorDeps).concat(nonPreBuiltVendorDeps)
    },
    resolve: {
      alias: buildResolveAliases(preBuiltVendorDeps)
    },
    module: {
      noParse: toArray(preBuiltVendorDeps),
      loaders: getLoaders(env),
    }
  }
}

module.exports = {
  getConfig: getConfig
}
<commit_msg>switch moment to 2012-2020 timezones<commit_after>/* contains common configorations for webpack configs */
const nodeModules = require('./utils').nodeModules
const path = require('path')
const consts = require('./constants')
const pkg = require(path.resolve(process.cwd(), 'package.json'))
const _ = require('lodash')

const pathToReactDOM = nodeModules('react-dom/dist/react-dom.min.js')
const pathToReactRouter = nodeModules('react-router/umd/ReactRouter.min.js')
const pathToMomentTimezone = nodeModules('moment-timezone/builds/moment-timezone-with-data-2012-2022.min.js')

const getConfig = (env) => {
  //define all vendor depnencies with pathes relative to node_modules/
  //that are pre-compiled, this allows to configure webpack to skip parsing of them
  //and we can use them for prod build instead of minifying libs ourself we would use pre-combiled ones
  const preBuiltVendorDeps = {
    'react': nodeModules('react/dist/react.min.js'),
    'react-dom': pathToReactDOM,
    'react-router': pathToReactRouter,
    'react-router-redux': nodeModules('react-router-redux/dist/ReactRouterRedux.min.js'),
    'redux': nodeModules('redux/dist/redux.min.js'),
    'moment': nodeModules('moment/min/moment.min.js'),
    'moment-timezone': pathToMomentTimezone,
    'redux-saga': nodeModules('redux-saga/dist/redux-saga.min.js'),
    //TODO: modify for node to pull node-fetch
    'isomorphic-fetch': nodeModules('whatwg-fetch/fetch.js'),
  }

  // reading from package.json
  // NOTE: that it should only contains main app (main entry chunk "app") vendor deps, all other deps
  // should go to "devDependencies" and get automatically split between entry chunks
  const allVendorDeps = Object.keys(pkg.dependencies)
    //excluding regenerator-runtime since it results in error "regeneratorRuntime is not defined" in runtime
    //and manually specifying correct path
    .filter(x => x !== 'regenerator-runtime')
    .concat(['regenerator-runtime/runtime'])
  const nonPreBuiltVendorDeps = _.difference(allVendorDeps, Object.keys(preBuiltVendorDeps))

  const toArray = (obj) => {
    return Object.keys(obj).map(key => obj[key])
  }

  const buildResolveAliases = vendorDeps =>
    Object.keys(vendorDeps)
      .map(depName => ({
        //$ is used to exactly match dependency
        [depName + '$']: vendorDeps[depName]
      }))
      //reducing array of objects to one object with "name$":"path" structure as required by Webpack
      .reduce((a, b) => {
        const key = Object.keys(b)[0] //we know that it has only one key
        const value = b[key]

        a[key] = value
        return a
      })

  const getLoaders = (env) => {
    let loaders = []

    if (env === 'prod') {
      loaders = loaders.concat([{
        test: pathToReactDOM,
        loader: 'imports'
      }, {
        test: pathToReactRouter,
        loader: 'imports'
      }, {
        test: pathToMomentTimezone,
        loader: 'imports'
      }, {
        test: /\.(js|jsx?)$/,
        loader: 'babel',
        exclude: /(node_modules|bower_components)/,
        include: path.join(consts.SRC_DIR),
        query: {
          presets: ['react', 'es2015', 'stage-2'],
          cacheDirectory: false,
          compact: true, //so babel wont output whitespaces and stuff, speeds up build a little
          plugins: [
            'transform-react-constant-elements', //compile-time optimizations
            'transform-react-inline-elements' //compile-time optimizations
          ]
        }
      }])
    }

    if (env === 'dev') {
      loaders = loaders.concat([{
        test: /\.(js|jsx?)$/,
        loader: 'babel',
        exclude: /(node_modules|bower_components)/,
        include: [path.join(consts.SRC_DIR)],
        query: {
          presets: ['react', 'es2015', 'stage-2'],
          cacheDirectory: true, //not needed for prod build
          plugins: [
            ['react-transform', {
              'transforms': [{
                'transform': 'react-transform-hmr',
                'imports': ['react'],
                'locals': ['module']
              }, {
                'transform': 'react-transform-catch-errors',
                'imports': ['react', 'redbox-react']
              }]
            }]
          ]
        }
      }, {
        test: /\.scss$/,
        //loaders: ['style', ExtractTextPlugin.extract('css?localIdentName=[name]_[local]_[hash:base64:3]!sass')],
        //loaders: ['style', 'css?localIdentName=[name]_[local]_[hash:base64:3]', 'sass'],
        loader: 'style!css!postcss!sass',
        exclude: /(node_modules|bower_components)/,
        include: path.join(consts.SRC_DIR, 'client')
      }])
    }

    //common loaders
    loaders = loaders.concat([{
      test: /\.json$/,
      loader: 'json-loader',
    }, {
      test: /\.(woff|woff2)(\?v=\d+\.\d+\.\d+)?$/,
      exclude: /(node_modules|bower_components)/,
      loaders: ['url?limit=10000&mimetype=application/font-woff'],
      include: path.join(consts.SRC_DIR, 'client')
    }, {
      test: /\.(jpg|jpeg|gif|png|ico|svg)$/,
      exclude: /(node_modules|bower_components)/,
      include: path.join(consts.SRC_DIR, 'client'),
      loader: 'file-loader?name=[path][name].[ext]&context=' + consts.IMG_DIR
    }])

    return loaders
  }

  return {
    entry: {
      vendor: Object.keys(preBuiltVendorDeps).concat(nonPreBuiltVendorDeps)
    },
    resolve: {
      alias: buildResolveAliases(preBuiltVendorDeps)
    },
    module: {
      noParse: toArray(preBuiltVendorDeps),
      loaders: getLoaders(env),
    }
  }
}

module.exports = {
  getConfig: getConfig
}
<|endoftext|>


<commit_before>// Deps
var
  requester = require("./requester.js"),
  Events  = require("events").EventEmitter,
  CTI     = require("./ctiMap").CTI,
  moment  = require("moment-timezone"),
  util    = require("util");
var
  log     = util.log;

// Report name of data source.
exports.name = "FlightRadar24 Aircraft Info";
exports.dataType = "track";
exports.keyFields = ["fcode"];

// Other stuff.
var MINUTE = 60000; // Milliseconds.
var HOUR = MINUTE * 60; // Milliseconds.

// Generic request options.
var reqOptions = {
  method: "GET",
  proxy: (process.env ? process.env.http_proxy : null),
  url: null, // Updated in the requester
  headers: {
    Host: "www.flightradar24.com"
  },
  json: false
};

var MINUTE = 60000; // Milliseconds.
var HOUR = MINUTE * 60; // Milliseconds.
var lastRun = {}; // The last time this store collected data.
exports.runCheck = function(url)
{
  // Collect data every hour.
  if (lastRun[url] && Date.now() - lastRun[url] < HOUR)
  {
    return false;
  }
  // This data source was last run at the current time.
  lastRun[url] = Date.now();
  return true;
};

// Collect data from web service providers.
exports.collect = function(url, harvesterCb)
{
  // Use the default requester to obtain a response from the web service.
  return requester.collect(
    reqOptions,
    urls,
    // The default requester requires a callback which will tell it when it can
    // make a new request to the web service. The callback must return true or false.
    function()
    {
      // Collect data every 30 sec.
      if (lastRun !== null && Date.now() - lastRun < (MINUTE * 2))
      {
        return false;
      }
      // This data source was last run at the current time.
      lastRun = Date.now();
      return true;
    },
    // No checking callback is required.
    undefined,
    // This callback is passed through from the harvester.
    harvesterCb
  );
};

// Process aircraft data.
exports.auger = function(input)
{
  var processor = new Events();
  var parsedInput = {};
  var aircrafts = [];
  var flightCodes = {};
  var aircraft;

  // Begin processing.
  process.nextTick(function()
  {
    try
    {
      parsedInput = JSON.parse(input);
    }
    catch(e)
    {
      processor.emit("error", e);
    }

    // Process aircraft.
    for (var item in parsedInput)
    {
      var ary = parsedInput[item];
      // Aircraft items are arrays. Only process those.
      if (typeof ary === "object" && Array.isArray(ary))
      {
        aircraft = {};
        aircraft[CTI.modeSId]         = ary[0] || "empty";
        aircraft[CTI.lat]             = ary[1] || 0;
        aircraft[CTI.lon]             = ary[2] || 0;
        aircraft[CTI.heading]         = ary[3] || 0;
        aircraft[CTI.altitudeFt]      = ary[4] || 0;
        aircraft[CTI.speedKt]         = ary[5] || 0;
        aircraft[CTI.equipment]       = ary[8] || "empty";
        aircraft[CTI.aircraftTail]    = ary[9].replace("-", ""); // This is a primary key. Should be proper rego.
        aircraft[CTI.updateTime]      = ary[10] || moment().utc().unix();
        aircraft[CTI.originAirportCode] = ary[11] || "empty";
        aircraft[CTI.destAirportCode] = ary[12] || "empty";
        aircraft[CTI.fullFlightCode]  = ary[13]; // This is a range key. Must be proper flight code.
        aircraft[CTI.onGround]        = ary[14] || 0;
        aircraft[CTI.vertSpeed]       = ary[15] || 0;
        aircraft[CTI.airlineCode]     = ary[16].substring(0,3);

        // Check integrity of data and add to array if all is well.
        if (aircraft[CTI.aircraftTail].length > 1 &&
          aircraft[CTI.equipment].length > 1 &&
          aircraft[CTI.fullFlightCode].length > 1 &&
          flightCodes[ary[13]] === undefined)
        {
          flightCodes[ary[13]] = true;
          aircrafts.push(aircraft);
        }
      }
    }
    // If we get here, the loop finished iterating over structure without error.
    processor.emit("complete", aircrafts);
  });

  // Return the emitter so the consumer can handle important processsing events.
  return processor;
};

// Save the data. The intention of this function is to provide a place where flight stats
// specific, pre-save manipulation of data could be done.
exports.saveData = function (store, data, something)
{
  // Non deep copy.
  // var data2 = data.slice(0);
  // store.saveAircraft(data);
  // store.saveAircraftPositions(data2);
};
<commit_msg>harvester now holds to a url-centric model<commit_after>// Deps
var
  requester = require("./requester.js"),
  Events  = require("events").EventEmitter,
  CTI     = require("./ctiMap").CTI,
  moment  = require("moment-timezone"),
  util    = require("util");
var
  log     = util.log;

// Report name of data source.
exports.name = "FlightRadar24 Aircraft Info";
<<<<<<< HEAD
exports.dataType = "track";
exports.keyFields = ["fcode"];

// Other stuff.
var MINUTE = 60000; // Milliseconds.
var HOUR = MINUTE * 60; // Milliseconds.

=======
exports.dataType = "fraircraft";
exports.keyFields = ["tail", "fcode"];

>>>>>>> harvester now holds to a url-centric model
// Generic request options.
var reqOptions = {
  method: "GET",
  proxy: (process.env ? process.env.http_proxy : null),
  url: null, // Updated in the requester
  headers: {
    Host: "www.flightradar24.com"
  },
  json: false
};

var MINUTE = 60000; // Milliseconds.
var HOUR = MINUTE * 60; // Milliseconds.
var lastRun = {}; // The last time this store collected data.
exports.runCheck = function(url)
{
  // Collect data every hour.
  if (lastRun[url] && Date.now() - lastRun[url] < HOUR)
  {
    return false;
  }
  // This data source was last run at the current time.
  lastRun[url] = Date.now();
  return true;
};

// Collect data from web service providers.
exports.collect = function(url, harvesterCb)
{
  // Use the default requester to obtain a response from the web service.
  return requester.collect(
    reqOptions,
    url.url,
    // No checking callback is required.
    undefined,
    // This callback is passed through from the harvester.
    harvesterCb
  );
};

// Process aircraft data.
exports.auger = function(input)
{
  var processor = new Events();
  var parsedInput = {};
  var aircrafts = [];
  var flightCodes = {};
  var aircraft;

  // Begin processing.
  process.nextTick(function()
  {
    try
    {
      parsedInput = JSON.parse(input);
    }
    catch(e)
    {
      processor.emit("error", e);
    }

    // Process aircraft.
    for (var item in parsedInput)
    {
      var ary = parsedInput[item];
      // Aircraft items are arrays. Only process those.
      if (typeof ary === "object" && Array.isArray(ary))
      {
        aircraft = {};
        aircraft[CTI.modeSId]         = ary[0] || "empty";
        aircraft[CTI.lat]             = ary[1] || 0;
        aircraft[CTI.lon]             = ary[2] || 0;
        aircraft[CTI.heading]         = ary[3] || 0;
        aircraft[CTI.altitudeFt]      = ary[4] || 0;
        aircraft[CTI.speedKt]         = ary[5] || 0;
        aircraft[CTI.equipment]       = ary[8] || "empty";
        aircraft[CTI.aircraftTail]    = ary[9].replace("-", ""); // This is a primary key. Should be proper rego.
        aircraft[CTI.updateTime]      = ary[10] || moment().utc().unix();
        aircraft[CTI.originAirportCode] = ary[11] || "empty";
        aircraft[CTI.destAirportCode] = ary[12] || "empty";
        aircraft[CTI.fullFlightCode]  = ary[13]; // This is a range key. Must be proper flight code.
        aircraft[CTI.onGround]        = ary[14] || 0;
        aircraft[CTI.vertSpeed]       = ary[15] || 0;
        aircraft[CTI.airlineCode]     = ary[16].substring(0,3);

        // Check integrity of data and add to array if all is well.
        if (aircraft[CTI.aircraftTail].length > 1 &&
          aircraft[CTI.equipment].length > 1 &&
          aircraft[CTI.fullFlightCode].length > 1 &&
          flightCodes[ary[13]] === undefined)
        {
          flightCodes[ary[13]] = true;
          aircrafts.push(aircraft);
        }
      }
    }
    // If we get here, the loop finished iterating over structure without error.
    processor.emit("complete", aircrafts);
  });

  // Return the emitter so the consumer can handle important processsing events.
  return processor;
};

// Save the data. The intention of this function is to provide a place where flight stats
// specific, pre-save manipulation of data could be done.
exports.saveData = function (store, data, something)
{
<<<<<<< HEAD
  // Non deep copy.
  // var data2 = data.slice(0);
  // store.saveAircraft(data);
  // store.saveAircraftPositions(data2);
=======
  log("Saving aircraft data.");

  store.saveAircraft(data);
>>>>>>> harvester now holds to a url-centric model
};
<|endoftext|>


<commit_before>var five = require("johnny-five");
var config = require("./config");
var board = new five.Board({
	port: config.board.port
});

var stdin = process.openStdin();
require('tty').setRawMode(true);

var elbowServo, shoulderServo = null;
var elbowOffset = 38;
var shoulderOffset = 7;
var elbowPosition = 90 + elbowOffset;
var shoulderPosition = 90+shoulderOffset;
var L1 = 7; // panjang bahu ke siku cm
var L2 = 7.5; // panjang siku ke pergelangan tangan dari cm
var drawX = 5;
var drawY = 5;
board.on("ready", function() {

 	elbowServo = new five.Servo({
	  pin: 6,
	  startAt: 45 + elbowOffset
	});
 	//elbowServo.sweep();

 	shoulderServo = new five.Servo({
	  pin: 5,
	  startAt: 135 + shoulderOffset
	});
	armMoveTo(5,5);
 	//shoulderServo.sweep();
 	// deviasi 56derajat
 	board.repl.inject({
 	shoulderTo: function(degree){
 		shoulderServo.to(shoulderPosition + degree);
 	},
 	elbowTo: function(degree){
 		elbowServo.to(elbowPosition - degree);
 	},
 	inverseValueE: inverseValueE,
 	forwardX: forwardX,
 	forwardY: forwardY,
 	inverseE: inverseE,
 	inverseS: inverseS,
 	armMoveTo: armMoveTo
 });
	this.on("exit", function(){
		armMoveTo(5,5);
	});
});
function test(){
	console.log('sparta');
}

stdin.on('keypress', function (chunk, key) {
  	// if (key && key.ctrl && key.name == 'c') process.exit();
  	if(key.name == 'right'){
  		console.log('rgiht');
  	}
});

function inverseValueE(x, y){
	var E = Math.acos( ( Math.pow(x, 2)+ Math.pow(y,2) - Math.pow(L1, 2) - Math.pow(L2,2) )/(2*L1*L2) );
	return E;

}

function inverseE(x,y){
	var E =  Math.acos( ( Math.pow(x, 2)+ Math.pow(y,2) - Math.pow(L1, 2) - Math.pow(L2,2) )/(2*L1*L2) );
	return toDegrees(E);
}
function inverseS(x,y){
	var S = Math.atan(y/x) - Math.acos( ( Math.pow(x,2)+Math.pow(y,2)+Math.pow(L1,2)-Math.pow(L2,2) )/( (2*L1 * Math.sqrt( Math.pow(x,2)+Math.pow(y,2) ) ) ) );
	return toDegrees(S);
}
function forwardX(S, E){
	S = toRadians(S);
	E = toRadians(E);
	var x = ( L1*Math.cos(S) ) + (L2 * Math.cos(S+E));

	return x;
}
function forwardY(S, E){
	S = toRadians(S);
	E = toRadians(E);
	var y = ( L1*Math.sin(S) ) + (L2 * Math.sin(S+E));
	return y;
}
function armMoveTo(x,y){
	var S = inverseS(x,y);
	var E = inverseE(x,y);
	console.log(S+','+E);
	shoulderServo.to(shoulderPosition+S);
	elbowServo.to(elbowPosition - E);
}
function toDegrees (angle) {
  return angle * (180 / Math.PI);
}

 function toRadians (angle) {
  return angle * (Math.PI / 180);
}

 <commit_msg>add 4 direction control using keyboard<commit_after>var five = require("johnny-five");
var config = require("./config");
var board = new five.Board({
	port: config.board.port
});

var stdin = process.openStdin();
require('tty').setRawMode(true);

var elbowServo, shoulderServo = null;
var elbowOffset = 38;
var shoulderOffset = 7;
var elbowPosition = 90 + elbowOffset;
var shoulderPosition = 90+shoulderOffset;
var L1 = 7; // panjang bahu ke siku cm
var L2 = 7.5; // panjang siku ke pergelangan tangan dari cm
var drawX = 5;
var drawY = 5;
board.on("ready", function() {

 	elbowServo = new five.Servo({
	  pin: 6,
	  startAt: 45 + elbowOffset
	});
 	//elbowServo.sweep();

 	shoulderServo = new five.Servo({
	  pin: 5,
	  startAt: 135 + shoulderOffset
	});
	armMoveTo(5,5);
 	//shoulderServo.sweep();
 	// deviasi 56derajat
 	board.repl.inject({
 	shoulderTo: function(degree){
 		shoulderServo.to(shoulderPosition + degree);
 	},
 	elbowTo: function(degree){
 		elbowServo.to(elbowPosition - degree);
 	},
 	inverseValueE: inverseValueE,
 	forwardX: forwardX,
 	forwardY: forwardY,
 	inverseE: inverseE,
 	inverseS: inverseS,
 	armMoveTo: armMoveTo
 });
	this.on("exit", function(){
		// armMoveTo(5,5);
	});
});
function test(){
	console.log('sparta');
}

stdin.on('keypress', function (chunk, key) {
  	// if (key && key.ctrl && key.name == 'c') process.exit();
  	if(key.name == 'right'){
  		console.log('right');
  		if(drawY < 10){
  			drawY += 0.5;
  			armMoveTo(drawX, drawY);
  		}
  	}
  	if(key.name == 'left'){
  		console.log('left');
  		if(drawY > 5){
  			drawY -= 0.5;
  			armMoveTo(drawX, drawY);
  		}
  	}
  	if(key.name == 'up'){
  		console.log('up');
  		if(drawX > 5){
  			drawX -= 0.5;
  			armMoveTo(drawX, drawY);
  		}
  	}
  	if(key.name == 'down'){
  		console.log('down');
  		if(drawX < 10){
  			drawX += 0.5;
  			armMoveTo(drawX, drawY);
  		}
  	}
});

function inverseValueE(x, y){
	var E = Math.acos( ( Math.pow(x, 2)+ Math.pow(y,2) - Math.pow(L1, 2) - Math.pow(L2,2) )/(2*L1*L2) );
	return E;

}

function inverseE(x,y){
	var E =  Math.acos( ( Math.pow(x, 2)+ Math.pow(y,2) - Math.pow(L1, 2) - Math.pow(L2,2) )/(2*L1*L2) );
	return toDegrees(E);
}
function inverseS(x,y){
	var S = Math.atan(y/x) - Math.acos( ( Math.pow(x,2)+Math.pow(y,2)+Math.pow(L1,2)-Math.pow(L2,2) )/( (2*L1 * Math.sqrt( Math.pow(x,2)+Math.pow(y,2) ) ) ) );
	return toDegrees(S);
}
function forwardX(S, E){
	S = toRadians(S);
	E = toRadians(E);
	var x = ( L1*Math.cos(S) ) + (L2 * Math.cos(S+E));

	return x;
}
function forwardY(S, E){
	S = toRadians(S);
	E = toRadians(E);
	var y = ( L1*Math.sin(S) ) + (L2 * Math.sin(S+E));
	return y;
}
function armMoveTo(x,y){
	var S = inverseS(x,y);
	var E = inverseE(x,y);
	console.log(S+','+E);
	shoulderServo.to(shoulderPosition+S);
	elbowServo.to(elbowPosition - E);
}
function toDegrees (angle) {
  return angle * (180 / Math.PI);
}

 function toRadians (angle) {
  return angle * (Math.PI / 180);
}

 <|endoftext|>


<commit_before>
module.exports = request
request.GET = GET
request.PUT = PUT
request.reg = reg
request.upload = upload

var npm = require("../../../npm")
  , http = require("http")
  , url = require("url")
  , log = require("../log")
  , ini = require("../ini")
  , Buffer = require("buffer").Buffer
  , fs = require("../graceful-fs")

function request (method, where, what, etag, nofollow, cb) {
  log.verbose(where||"/", method)

  if (where.match(/^\/?favicon.ico/)) {
    return cb(new Error("favicon.ico isn't a package, it's a picture."))
  }
  if (typeof cb !== "function") cb = nofollow, nofollow = false
  if (typeof cb !== "function") cb = etag, etag = null
  if (typeof cb !== "function") cb = what, what = null
  var registry = reg()
  if (registry instanceof Error) return cb(registry)

  var adduserChange = /^\/?adduser\/org\.couchdb\.user:([^\/]+)\/-rev/
    , authRequired = what && !where.match(/^\/?adduser\/org\.couchdb\.user:/)
                   || where.match(adduserChange)
                   || method === "DELETE"
  if (!where.match(/^https?:\/\//)) {
    where = where.split("/").map(function (p) {
      p = p.trim()
      if (p.match(/^org.couchdb.user/)) {
        return p.replace(/\//g, encodeURIComponent("/"))
      }
      return encodeURIComponent(p)
    }).join("/")
    where = url.resolve(registry, where)
  }
  var u = url.parse(where)
    , proxyConfig = npm.config.get("proxy")
  if (proxyConfig) {
    if (!proxyConfig.match(/^https?:\/\//)) {
      proxyConfig = u.protocol+"//"+proxyConfig
    }
    var proxy = (proxyConfig) ? url.parse(proxyConfig) : null
    if (!proxy.host) return cb(new Error("Bad proxy config: "+proxyConfig))
  }
  var https = (proxy || u).protocol === "https:"
    , port = (proxy || u).port || (https ? 443 : 80)
    , hostname = (proxy || u).hostname
    , client = http.createClient(port, hostname, https)
    , auth = authRequired && npm.config.get("_auth")
    , path = proxy ? u.href : u.pathname

  if (authRequired && !auth) {
    return cb(new Error(
      "Cannot insert data into the registry without authorization\n"
      + "See: npm-adduser(1)"))
  }
  if (auth && !https) {
    log.warn("Sending authorization over insecure channel.")
  }
  var headers = { "host" : u.host
                , "accept" : "application/json"
                }
  if (auth) headers.authorization = "Basic " + auth
  if (proxy && proxy.auth) {
    headers["proxy-authorization"] =
      "Basic " + (new Buffer(proxy.auth).toString("base64"))
  }
  if (what) {
    if (what instanceof File) {
      log.verbose(what.name, "uploading")
      headers["content-type"] = "application/octet-stream"
    } else {
      delete what._etag
      log.silly(what,"writing json")
      what = new Buffer(JSON.stringify(what))
      headers["content-type"] = "application/json"
    }
    headers["content-length"] = what.length
  } else {
    headers["content-length"] = 0
  }
  if (etag) {
    log.verbose(etag, "etag")
    headers[method === "GET" ? "if-none-match" : "if-match"] = etag
  }
  log.silly(headers, "headers")

  var req = client.request(method, path, headers)
  client.on("error", log.er(cb, "Error connecting to server "+where))
  req.on("error", log.er(cb, "Request error to "+where))
  req.on("response", function (response) {
    // if (response.statusCode !== 200) return cb(new Error(
    //   "Status code " + response.statusCode + " from PUT "+where))
    var data = ""
    response.on("error", log.er(cb, "response error from "+where))
    response.on("data", function (chunk) {
      log.silly(chunk+"", "chunk")
      data += chunk
    })
    response.on("end", function () {
      if (!nofollow
          && (response.statusCode === 301 || response.statusCode === 302)) {
        // relative redirects SHOULD be disallowed, but alas...
        var newLoc = response.headers.location
        newLoc = newLoc && url.resolve(where, newLoc)
        if (!newLoc) return cb(new Error(
          response.statusCode + " status code with no location"))
        return request(method, newLoc, what, etag,
                       log.er(cb, "Failed to fetch "+newLoc))
      }
      var parsed
      if (response.statusCode !== 304) {
        try {
          parsed = JSON.parse(data)
        } catch (ex) {
          ex.message += "\n" + data
          log.error("error parsing json", "registry")
          return cb(ex, null, data, response)
        }
      }
      var er = null
      if (parsed && response.headers.etag) {
        parsed._etag = response.headers.etag
      }
      if (parsed && parsed.error) {
        var w = url.parse(where).pathname.substr(1)
          , name = w.split("/")[0]
        if (parsed.error === "not_found") {
          er = new Error("404 Not Found: "+name)
          er.errno = npm.E404
          er.pkgid = name
        } else {
          er = new Error(
            parsed.error + " " + (parsed.reason || "") + ": " + w)
        }
      }
      return cb(er, parsed, data, response)
    })
  })
  if (what instanceof File) {
    var size = Math.min(what.length, 1024*1024*1024)
      , remaining = what.length
    log.verbose(what.length, "bytes")
    ;(function W () {
      var b = new Buffer(size)
      try {
        var bytesRead = fs.readSync(what.fd, b, 0, b.length, null)
      } catch (er) {
        return log.er(cb, "Failure to read tarball")(er)
      }
      remaining -= bytesRead
      if (bytesRead) {
        log(bytesRead, "read")
        log(remaining, "remain")
        return (
            req.write(bytesRead === b.length ? b : b.slice(0, bytesRead))
          ) ? W()
            : req.on("drain", function DRAIN () {
                log.silly(remaining, "drain")
                req.removeListener("drain", DRAIN)
                W()
              })
      }
      if (!remaining) {
        req.end()
        log.verbose(what.name, "written to uploading stream")
        log.verbose("Not done yet! If it hangs/quits now, it didn't work.", "upload")
        return
      }
      // wtf!? No bytes read, but also bytes remaining.
      return cb(new Error("Some kind of weirdness reading the file"))
    })()
    return
  } else if (typeof what === "string" || Buffer.isBuffer(what)) {
    // just a json blob
    req.write(what)
  }
  req.end()
}
function GET (where, etag, nofollow, cb) {
  request("GET", where, null, etag, nofollow, cb)
}
function PUT (where, what, etag, nofollow, cb) {
  request("PUT", where, what, etag, nofollow, cb)
}

function upload (where, filename, etag, nofollow, cb) {
  if (typeof etag === "function") cb = etag, etag = null
  new File(filename, function (er, f) {
    if (er) return log.er(cb, "Couldn't open "+filename)(er)
    PUT(where, f, etag, nofollow, function (er) {
      log.info("done with upload", "publish")
      cb(er)
    })
  })
}
function File (name, cb) {
  var f = this
  f.name = name
  if (f.loaded) return cb(null, f)
  fs.stat(f.name, function (er, stat) {
    if (er) return log.er(cb, "doesn't exist "+f.name)(er)
    log.silly(stat, "stat "+name)
    f.length = stat.size
    fs.open(f.name, "r", 0666, function (er, fd) {
      if (er) return log.er(cb, "Error opening "+f.name)(er)
      f.fd = fd
      cb(null, f)
    })
  })
}

function reg () {
  var r = npm.config.get("registry")
  if (!r) {
    return new Error("Must define registry URL before accessing registry.")
  }
  if (r.substr(-1) !== "/") r += "/"
  npm.config.set("registry", r)
  return r
}
<commit_msg>change adduser url test to match new URL shape<commit_after>
module.exports = request
request.GET = GET
request.PUT = PUT
request.reg = reg
request.upload = upload

var npm = require("../../../npm")
  , http = require("http")
  , url = require("url")
  , log = require("../log")
  , ini = require("../ini")
  , Buffer = require("buffer").Buffer
  , fs = require("../graceful-fs")

function request (method, where, what, etag, nofollow, cb) {
  log.verbose(where||"/", method)

  if (where.match(/^\/?favicon.ico/)) {
    return cb(new Error("favicon.ico isn't a package, it's a picture."))
  }
  if (typeof cb !== "function") cb = nofollow, nofollow = false
  if (typeof cb !== "function") cb = etag, etag = null
  if (typeof cb !== "function") cb = what, what = null
  var registry = reg()
  if (registry instanceof Error) return cb(registry)

  var adduserChange = /^\/?-\/user\/org\.couchdb\.user:([^\/]+)\/-rev/
    , adduserNew = /^\/?-\/user\/org\.couchdb\.user:([^\/]+)/
    , authRequired = what && !where.match(adduserNew)
                   || where.match(adduserChange)
                   || method === "DELETE"
  if (!where.match(/^https?:\/\//)) {
    where = where.split("/").map(function (p) {
      p = p.trim()
      if (p.match(/^org.couchdb.user/)) {
        return p.replace(/\//g, encodeURIComponent("/"))
      }
      return encodeURIComponent(p)
    }).join("/")
    where = url.resolve(registry, where)
  }
  var u = url.parse(where)
    , proxyConfig = npm.config.get("proxy")
  if (proxyConfig) {
    if (!proxyConfig.match(/^https?:\/\//)) {
      proxyConfig = u.protocol+"//"+proxyConfig
    }
    var proxy = (proxyConfig) ? url.parse(proxyConfig) : null
    if (!proxy.host) return cb(new Error("Bad proxy config: "+proxyConfig))
  }
  var https = (proxy || u).protocol === "https:"
    , port = (proxy || u).port || (https ? 443 : 80)
    , hostname = (proxy || u).hostname
    , client = http.createClient(port, hostname, https)
    , auth = authRequired && npm.config.get("_auth")
    , path = proxy ? u.href : u.pathname

  if (authRequired && !auth) {
    return cb(new Error(
      "Cannot insert data into the registry without authorization\n"
      + "See: npm-adduser(1)"))
  }
  if (auth && !https) {
    log.warn("Sending authorization over insecure channel.")
  }
  var headers = { "host" : u.host
                , "accept" : "application/json"
                }
  if (auth) headers.authorization = "Basic " + auth
  if (proxy && proxy.auth) {
    headers["proxy-authorization"] =
      "Basic " + (new Buffer(proxy.auth).toString("base64"))
  }
  if (what) {
    if (what instanceof File) {
      log.verbose(what.name, "uploading")
      headers["content-type"] = "application/octet-stream"
    } else {
      delete what._etag
      log.silly(what,"writing json")
      what = new Buffer(JSON.stringify(what))
      headers["content-type"] = "application/json"
    }
    headers["content-length"] = what.length
  } else {
    headers["content-length"] = 0
  }
  if (etag) {
    log.verbose(etag, "etag")
    headers[method === "GET" ? "if-none-match" : "if-match"] = etag
  }
  log.silly(headers, "headers")

  var req = client.request(method, path, headers)
  client.on("error", log.er(cb, "Error connecting to server "+where))
  req.on("error", log.er(cb, "Request error to "+where))
  req.on("response", function (response) {
    // if (response.statusCode !== 200) return cb(new Error(
    //   "Status code " + response.statusCode + " from PUT "+where))
    var data = ""
    response.on("error", log.er(cb, "response error from "+where))
    response.on("data", function (chunk) {
      log.silly(chunk+"", "chunk")
      data += chunk
    })
    response.on("end", function () {
      if (!nofollow
          && (response.statusCode === 301 || response.statusCode === 302)) {
        // relative redirects SHOULD be disallowed, but alas...
        var newLoc = response.headers.location
        newLoc = newLoc && url.resolve(where, newLoc)
        if (!newLoc) return cb(new Error(
          response.statusCode + " status code with no location"))
        return request(method, newLoc, what, etag,
                       log.er(cb, "Failed to fetch "+newLoc))
      }
      var parsed
      if (response.statusCode !== 304) {
        try {
          parsed = JSON.parse(data)
        } catch (ex) {
          ex.message += "\n" + data
          log.error("error parsing json", "registry")
          return cb(ex, null, data, response)
        }
      }
      var er = null
      if (parsed && response.headers.etag) {
        parsed._etag = response.headers.etag
      }
      if (parsed && parsed.error) {
        var w = url.parse(where).pathname.substr(1)
          , name = w.split("/")[0]
        if (parsed.error === "not_found") {
          er = new Error("404 Not Found: "+name)
          er.errno = npm.E404
          er.pkgid = name
        } else {
          er = new Error(
            parsed.error + " " + (parsed.reason || "") + ": " + w)
        }
      }
      return cb(er, parsed, data, response)
    })
  })
  if (what instanceof File) {
    var size = Math.min(what.length, 1024*1024*1024)
      , remaining = what.length
    log.verbose(what.length, "bytes")
    ;(function W () {
      var b = new Buffer(size)
      try {
        var bytesRead = fs.readSync(what.fd, b, 0, b.length, null)
      } catch (er) {
        return log.er(cb, "Failure to read tarball")(er)
      }
      remaining -= bytesRead
      if (bytesRead) {
        log(bytesRead, "read")
        log(remaining, "remain")
        return (
            req.write(bytesRead === b.length ? b : b.slice(0, bytesRead))
          ) ? W()
            : req.on("drain", function DRAIN () {
                log.silly(remaining, "drain")
                req.removeListener("drain", DRAIN)
                W()
              })
      }
      if (!remaining) {
        req.end()
        log.verbose(what.name, "written to uploading stream")
        log.verbose("Not done yet! If it hangs/quits now, it didn't work.", "upload")
        return
      }
      // wtf!? No bytes read, but also bytes remaining.
      return cb(new Error("Some kind of weirdness reading the file"))
    })()
    return
  } else if (typeof what === "string" || Buffer.isBuffer(what)) {
    // just a json blob
    req.write(what)
  }
  req.end()
}
function GET (where, etag, nofollow, cb) {
  request("GET", where, null, etag, nofollow, cb)
}
function PUT (where, what, etag, nofollow, cb) {
  request("PUT", where, what, etag, nofollow, cb)
}

function upload (where, filename, etag, nofollow, cb) {
  if (typeof etag === "function") cb = etag, etag = null
  new File(filename, function (er, f) {
    if (er) return log.er(cb, "Couldn't open "+filename)(er)
    PUT(where, f, etag, nofollow, function (er) {
      log.info("done with upload", "publish")
      cb(er)
    })
  })
}
function File (name, cb) {
  var f = this
  f.name = name
  if (f.loaded) return cb(null, f)
  fs.stat(f.name, function (er, stat) {
    if (er) return log.er(cb, "doesn't exist "+f.name)(er)
    log.silly(stat, "stat "+name)
    f.length = stat.size
    fs.open(f.name, "r", 0666, function (er, fd) {
      if (er) return log.er(cb, "Error opening "+f.name)(er)
      f.fd = fd
      cb(null, f)
    })
  })
}

function reg () {
  var r = npm.config.get("registry")
  if (!r) {
    return new Error("Must define registry URL before accessing registry.")
  }
  if (r.substr(-1) !== "/") r += "/"
  npm.config.set("registry", r)
  return r
}
<|endoftext|>


<commit_before>/* This script will migrate all ApiUser records into ApiApp records
 *
 * run this script from the root of the project like so:
 * node migrations/20140206-apiApp <existingContact>::<domain> <existingContact>::<domain> ...
 */

var habitat = require( "habitat" ),
    async = require( "async" ),
    validate = require( "mongoose-validator" ).validate,
    slicedArgs = process.argv.slice( 2 ),
    mongoStreamEnded = false,
    env,
    mongoose;

var domainMap = {};

slicedArgs.forEach(function( arg ) {
  arg = arg.split( '::' );
  domainMap[ arg[0] ] = arg[1];
});

function getApiUser( mongoose ) {
  return mongoose.model( "ApiUser" , new mongoose.Schema({
    privatekey: {
      type: String,
      required: true,
      unique: true
    },
    publickey: {
      type: String,
      required: true,
      unique: true
    },
    revoked: {
      type: Boolean,
      required: true,
      "default": false
    },
    contact: {
      type: String,
      required: true,
      validate: validate( "isEmail" )
    },
    admin: {
      type: Boolean,
      required: true,
      "default": false
    }
  }));
}

habitat.load();
env = new habitat();

dbh = require( "../lib/mongoose" )( env, function( err ) {
  if ( err ) {
    console.log( err );
    process.exit( 1 );
  }

  var ApiApp = require( "../lib/models/apiApp" )( env, dbh.mongoInstance() ),
      ApiUser = getApiUser( dbh.mongoInstance() ),
      stream = ApiUser.find().stream(),
      queue = async.queue(function( doc, done ) {

        var app = new ApiApp({
          contact: doc.contact,
          privatekey: doc.privatekey,
          publickey: doc.publickey,
          domain: domainMap[ doc.contact ],
          revoked: doc.revoked,
          admin: doc.admin
        });

        app.save(function( err ) {
          if ( err ) {
            console.log( "Failure saving document:" );
            console.log( app );
            console.log( err );
            process.exit( 1 );
          }
          done();
        });
      }, 4 );// concurrency of 4

  queue.drain = function() {
    if ( mongoStreamEnded ) {
      console.log( "completed!" );
      process.exit( 0 );
    }
  };

  stream.on( "data", function onData( doc ) {
    queue.push( doc );
  }).on( "error", function( err ) {
    console.log( err );
    process.exit( 1 );
  }).on( "end", function() {
    mongoStreamEnded = true;
  });
});
<commit_msg>update migration<commit_after>/* This script will migrate all ApiUser records into ApiApp records
 *
 * run this script from the root of the project like so:
 * node migrations/20140206-apiApp <existingContact>::<domain> <existingContact>::<domain> ...
 */

var async = require( "async" ),
    env = require( "../lib/environment" ),
    validate = require( "mongoose-validator" ).validate,
    slicedArgs = process.argv.slice( 2 ),
    mongoStreamEnded = false;

var domainMap = {};

slicedArgs.forEach(function( arg ) {
  arg = arg.split( '::' );
  domainMap[ arg[0] ] = arg[1];
});

function getApiUser( mongoose ) {
  return mongoose.model( "ApiUser" , new mongoose.Schema({
    privatekey: {
      type: String,
      required: true,
      unique: true
    },
    publickey: {
      type: String,
      required: true,
      unique: true
    },
    revoked: {
      type: Boolean,
      required: true,
      "default": false
    },
    contact: {
      type: String,
      required: true,
      validate: validate( "isEmail" )
    },
    admin: {
      type: Boolean,
      required: true,
      "default": false
    },
    domain: {
      type: String,
      required: true,
    }
  }));
}

var dbh = require( "../lib/mongoose" )(),
    ApiApp = require( "../lib/models/apiApp" )( dbh.mongoInstance() ),
    ApiUser = getApiUser( dbh.mongoInstance() ),
    stream = ApiUser.find().stream(),
    queue = async.queue(function( doc, done ) {

      var app = new ApiApp({
        contact: doc.contact,
        privatekey: doc.privatekey,
        publickey: doc.publickey,
        domain: domainMap[ doc.contact ],
        revoked: doc.revoked,
        admin: doc.admin
      });

      app.save(function( err ) {
        if ( err ) {
          console.log( "Failure saving document:" );
          console.log( app );
          console.log( err );
          process.exit( 1 );
        }
        done();
      });
    }, 4 );// concurrency of 4

queue.drain = function() {
  if ( mongoStreamEnded ) {
    console.log( "completed!" );
    process.exit( 0 );
  }
};

stream.on( "data", function onData( doc ) {
  queue.push( doc );
}).on( "error", function( err ) {
  console.log( err );
  process.exit( 1 );
}).on( "end", function() {
  mongoStreamEnded = true;
});
<|endoftext|>


<commit_before>var _ = require('lodash');
var Bluebird = require('bluebird');

var mongo;

var valid_actions = {
    put_job: 'PUT',
    get_job: 'GET',
    destroy_job: 'DELETE',
    list_jobs: 'GET',
    job_history: 'GET',
    reserve_jobs: 'POST',
    renew_reservation: 'PUT',
    release_reservation: 'DELETE',
};

var webtask = function (context, req, res) {
    var action = validate_method(valid_actions);

    if (!validate_params(['MONGO_COLLECTION']))
        return;

    if (action === 'put_job') {
        // Create or update a scheduled webtask

        if (!validate_params(['container', 'name', 'schedule', 'token']))
            return;

        var now = new Date();

        var item = {
            state: 'active',
            schedule: context.data.schedule,
            token: context.data.token,
            container: context.data.container,
            name: context.data.name,
            last_run_at: null,
            next_available_at: now,
            last_result: null,
            run_count: 0,
            error_count: 0,
        };

        return withMongoCollection(context.data.MONGO_COLLECTION)
            .then(function (coll) {
                return coll.findOneAndUpdateAsync({
                    container: item.container,
                    name: item.name,
                }, item, {
                    returnOriginal: false,
                    upsert: true,
                });
            })
            .get('value')
            .then(stripMongoId)
            .then(function (data) {
                res.writeHead(200, {'Content-Type': 'application/json'});
                res.end(JSON.stringify(data));
            })
            .catch(function (err) {
                error(err.statusCode || 500, err);
            });
    } else if (action === 'reserve_jobs') {
        // Reserve a set of jobs for execution

        if (!validate_params(['count', 'ttl']))
            return;

        var now = new Date();
        var next_available_at = new Date(now.valueOf() + (parseInt(context.data.ttl, 10) * 1000))

        return withMongoCollection(context.data.MONGO_COLLECTION)
            .then(function (coll) {
                return Bluebird.resolve(_.range(context.data.count))
                    // TODO (ggoodman): Refactor this to use reduce to avoid calling N queries if reservation fails before N
                    .map(function (n) {
                        console.log("Attempting reservation", n);
                        var filter = {
                            next_available_at: {
                                $lte: now
                            },
                            state: 'active',
                        };
                        var update = {
                            $set: {
                                next_available_at: next_available_at
                            }
                        };

                        return coll.findOneAndUpdateAsync(filter, update, {
                            returnOriginal: false, // Return modified
                        })
                            .get('value'); // Only pull out the value

                    }, {concurrency: 1}) // Serial mapping
                    // Don't prevent one failure from blocking the entire
                    // reservation (that would mean extra waiting on those that
                    // didn't fail)
                    .settle()
                    // Log errors and continue with safe fallback
                    .map(function (result) {
                        if (result.isRejected()) {
                            console.log("Error: findOneAndUpdate failed ", result.reason());
                            return null;
                        } else {
                            return result.value();
                        }
                    })
                    // Elminate error queries and queries that did not match
                    // (this is the case when there are fewer than N jobs
                    // available for running now)
                    .filter(Boolean);

            })
            .map(stripMongoId)
            .then(function (data) {
                res.writeHead(200, {'Content-Type': 'application/json'});
                res.end(JSON.stringify(data));
            })
            .catch(function (err) {
                error(err.statusCode || 500, err);
            });
    } else if (action === 'list_jobs') {
        // List all scheduled webtasks or only those in a container passed via query

        return withMongoCollection(context.data.MONGO_COLLECTION)
            .then(function (coll) {
                var query = {};

                // Both admin and user will hit this endpoint. When admin, container can be unset
                if (context.data.container) query.container = context.data.container;

                // TODO (ggoodman): Pagination logic
                return coll.findAsync(query)
                    .then(function (cursor) {
                        return Bluebird.promisify(cursor.toArray, cursor)();
                    });
            })
            .map(stripMongoId)
            .then(function (data) {
                res.writeHead(200, {'Content-Type': 'application/json'});
                res.end(JSON.stringify(data));
            })
            .catch(function (err) {
                error(err.statusCode || 500, err);
            });

    } else if (action === 'get_job') {
        // Get a single scheduled webtask

        if (!validate_params(['container', 'name']))
            return;

        return withMongoCollection(context.data.MONGO_COLLECTION)
            .then(function (coll) {
                var query = {
                    container: context.data.container,
                    name: context.data.name,
                };

                return coll.findOneAsync(query, {});
            })
            .then(stripMongoId)
            .then(function (data) {
                res.writeHead(200, {'Content-Type': 'application/json'});
                res.end(JSON.stringify(data));
            })
            .catch(function (err) {
                error(err.statusCode || 500, err);
            });

    } else if (action === 'destroy_job') {
        // Get a single scheduled webtask

        if (!validate_params(['container', 'name']))
            return;

        return withMongoCollection(context.data.MONGO_COLLECTION)
            .then(function (coll) {
                var filter = {
                    container: context.data.container,
                    name: context.data.name,
                };

                return coll.deleteOne(filter, {w: 1});
            })
            .then(function (data) {
                res.writeHead(204);
                res.end();
            })
            .catch(function (err) {
                error(err.statusCode || 500, err);
            });

    } else {
        res.writeHead(404);
        res.end('Method not found');
    }

    // Helper methods

    function stripMongoId (doc) {
        if (doc && doc._id) delete doc._id;

        return doc;
    }

    function withMongoDb () {
        if (mongo) return mongo;

        var MongoClient = require('mongodb').MongoClient;
        var connect = Bluebird.promisify(MongoClient.connect, MongoClient);

        return connect(context.data.MONGO_URL)
            .then(function (db) {
                // Store the settled promise resolving to the db object
                mongo = Bluebird.resolve(db);

                return db;
            });
    }

    function withMongoCollection (collName) {
        return withMongoDb()
            .call('collection', collName) // Get a Collection handle
            .then(Bluebird.promisifyAll); // Promisify all methods of Connection
    }

    function validate_method (valid_actions) {
        var action = context.data.action;
        var expected_method = valid_actions[action];

        if (!action) return error(400, 'Missing action.');
        if (!expected_method) return error(400, 'Invalid action.');
        if (expected_method !== req.method) return error(405, 'Invalid method.');

        return action;
    }

    function validate_params(required_params) {
        for (var i in required_params) {
            if (typeof context.data[required_params[i]] !== 'string') {
                return error(400, 'Missing ' + required_params[i] + '.');
            }
        }
        return true;
    }

    function error(code, err) {
        console.log(code + (err ? (': ' + err) : ''));
        res.writeHead(code);
        res.end(err);
        return false;
    }
};

return webtask;<commit_msg>Add note to consider race condition for destroy vs. renew vs. release<commit_after>var _ = require('lodash');
var Bluebird = require('bluebird');

var mongo;

var valid_actions = {
    put_job: 'PUT',
    get_job: 'GET',
    destroy_job: 'DELETE',
    list_jobs: 'GET',
    job_history: 'GET',
    reserve_jobs: 'POST',
    renew_reservation: 'PUT',
    release_reservation: 'DELETE',
};

var webtask = function (context, req, res) {
    var action = validate_method(valid_actions);

    if (!validate_params(['MONGO_COLLECTION']))
        return;

    if (action === 'put_job') {
        // Create or update a scheduled webtask

        if (!validate_params(['container', 'name', 'schedule', 'token']))
            return;

        var now = new Date();

        var item = {
            state: 'active',
            schedule: context.data.schedule,
            token: context.data.token,
            container: context.data.container,
            name: context.data.name,
            last_run_at: null,
            next_available_at: now,
            last_result: null,
            run_count: 0,
            error_count: 0,
        };

        return withMongoCollection(context.data.MONGO_COLLECTION)
            .then(function (coll) {
                return coll.findOneAndUpdateAsync({
                    container: item.container,
                    name: item.name,
                }, item, {
                    returnOriginal: false,
                    upsert: true,
                });
            })
            .get('value')
            .then(stripMongoId)
            .then(function (data) {
                res.writeHead(200, {'Content-Type': 'application/json'});
                res.end(JSON.stringify(data));
            })
            .catch(function (err) {
                error(err.statusCode || 500, err);
            });
    } else if (action === 'reserve_jobs') {
        // Reserve a set of jobs for execution

        if (!validate_params(['count', 'ttl']))
            return;

        var now = new Date();
        var next_available_at = new Date(now.valueOf() + (parseInt(context.data.ttl, 10) * 1000))

        return withMongoCollection(context.data.MONGO_COLLECTION)
            .then(function (coll) {
                return Bluebird.resolve(_.range(context.data.count))
                    // TODO (ggoodman): Refactor this to use reduce to avoid calling N queries if reservation fails before N
                    .map(function (n) {
                        console.log("Attempting reservation", n);
                        var filter = {
                            next_available_at: {
                                $lte: now
                            },
                            state: 'active',
                        };
                        var update = {
                            $set: {
                                next_available_at: next_available_at
                            }
                        };

                        return coll.findOneAndUpdateAsync(filter, update, {
                            returnOriginal: false, // Return modified
                        })
                            .get('value'); // Only pull out the value

                    }, {concurrency: 1}) // Serial mapping
                    // Don't prevent one failure from blocking the entire
                    // reservation (that would mean extra waiting on those that
                    // didn't fail)
                    .settle()
                    // Log errors and continue with safe fallback
                    .map(function (result) {
                        if (result.isRejected()) {
                            console.log("Error: findOneAndUpdate failed ", result.reason());
                            return null;
                        } else {
                            return result.value();
                        }
                    })
                    // Elminate error queries and queries that did not match
                    // (this is the case when there are fewer than N jobs
                    // available for running now)
                    .filter(Boolean);

            })
            .map(stripMongoId)
            .then(function (data) {
                res.writeHead(200, {'Content-Type': 'application/json'});
                res.end(JSON.stringify(data));
            })
            .catch(function (err) {
                error(err.statusCode || 500, err);
            });
    } else if (action === 'list_jobs') {
        // List all scheduled webtasks or only those in a container passed via query

        return withMongoCollection(context.data.MONGO_COLLECTION)
            .then(function (coll) {
                var query = {};

                // Both admin and user will hit this endpoint. When admin, container can be unset
                if (context.data.container) query.container = context.data.container;

                // TODO (ggoodman): Pagination logic
                return coll.findAsync(query)
                    .then(function (cursor) {
                        return Bluebird.promisify(cursor.toArray, cursor)();
                    });
            })
            .map(stripMongoId)
            .then(function (data) {
                res.writeHead(200, {'Content-Type': 'application/json'});
                res.end(JSON.stringify(data));
            })
            .catch(function (err) {
                error(err.statusCode || 500, err);
            });

    } else if (action === 'get_job') {
        // Get a single scheduled webtask

        if (!validate_params(['container', 'name']))
            return;

        return withMongoCollection(context.data.MONGO_COLLECTION)
            .then(function (coll) {
                var query = {
                    container: context.data.container,
                    name: context.data.name,
                };

                return coll.findOneAsync(query, {});
            })
            .then(stripMongoId)
            .then(function (data) {
                res.writeHead(200, {'Content-Type': 'application/json'});
                res.end(JSON.stringify(data));
            })
            .catch(function (err) {
                error(err.statusCode || 500, err);
            });

    } else if (action === 'destroy_job') {
        // Get a single scheduled webtask

        if (!validate_params(['container', 'name']))
            return;

        return withMongoCollection(context.data.MONGO_COLLECTION)
            .then(function (coll) {
                var filter = {
                    container: context.data.container,
                    name: context.data.name,
                };

                return coll.deleteOne(filter, {w: 1});
            })
            .then(function (data) {
                res.writeHead(204);
                res.end();
            })
            .catch(function (err) {
                error(err.statusCode || 500, err);
            });

    } else {
        // TODO (ggoodman): Consider: https://github.com/auth0/auth0-sandbox-ext/pull/1#discussion_r32170649

        res.writeHead(404);
        res.end('Method not found');
    }

    // Helper methods

    function stripMongoId (doc) {
        if (doc && doc._id) delete doc._id;

        return doc;
    }

    function withMongoDb () {
        if (mongo) return mongo;

        var MongoClient = require('mongodb').MongoClient;
        var connect = Bluebird.promisify(MongoClient.connect, MongoClient);

        return connect(context.data.MONGO_URL)
            .then(function (db) {
                // Store the settled promise resolving to the db object
                mongo = Bluebird.resolve(db);

                return db;
            });
    }

    function withMongoCollection (collName) {
        return withMongoDb()
            .call('collection', collName) // Get a Collection handle
            .then(Bluebird.promisifyAll); // Promisify all methods of Connection
    }

    function validate_method (valid_actions) {
        var action = context.data.action;
        var expected_method = valid_actions[action];

        if (!action) return error(400, 'Missing action.');
        if (!expected_method) return error(400, 'Invalid action.');
        if (expected_method !== req.method) return error(405, 'Invalid method.');

        return action;
    }

    function validate_params(required_params) {
        for (var i in required_params) {
            if (typeof context.data[required_params[i]] !== 'string') {
                return error(400, 'Missing ' + required_params[i] + '.');
            }
        }
        return true;
    }

    function error(code, err) {
        console.log(code + (err ? (': ' + err) : ''));
        res.writeHead(code);
        res.end(err);
        return false;
    }
};

return webtask;<|endoftext|>