<commit_before>var system = require('system');
var apiKey = system.env.API_KEY;

casper.options.waitTimeout = 60000;
casper.start();

casper.thenBypassUnless(function() {
  return apiKey && apiKey.length > 0;
}, 2);

casper.thenOpen('http://localhost:3000/train', function(result) {
  casper.test.assert(result.status === 200, 'Front page opens');
  casper.test.assertSelectorHasText('h1.base--h2.use--header', 'Train a demo classifier');

  // dog breeds
  casper.then(function() {
    this.click('div._training--example:nth-child(1)');
  });

  // dog type examples
  casper.waitForSelector('div._examples[data-kind="dogs"]', function() {
  });

  casper.then(function() {
    this.click('img[data-name="goldenretriever"]');
  });

  // example shots
  casper.waitForSelector('div._examples--contact-sheet[data-kind="dogs"]', function() {
  });

  // click three of deez
  casper.then(function() {
    this.click('button._positive[data-name="goldenretriever"]');
  });
  casper.then(function() {
    this.click('button._positive[data-name="husky"]');
  });
  casper.then(function() {
    this.click('button._positive[data-name="dalmatian"]');
  });
  // TRAAAAIIIIIN!
  casper.then(function() {
    this.click('button.train--train-button');
  });

  // // dogs test page
  // casper.waitUntilVisible('test--section', function() {
  //   casper.test.assertSelectorHasText('base--h2 test--classifier', 'Dogs');
  // });
  //
  // // click on the cat
  // casper.then(function() {
  //   this.click('.test--random-test-image');
  // });
  //
  // // random picture
  // casper.waitUntilVisible('test--output-image.landscape', function() {
  // });

  casper.thenOpen('http://localhost:3000/train', function() {
    // Moleskine
    casper.then(function() {
      this.click('div._training--example:nth-child(2)');
    });

    // dog type examples
    casper.waitForSelector('div._examples[data-kind="moleskine"]', function() {

    });

    // 1 pro and one -non
    casper.then(function() {
      this.click('button._positive[data-name="portrait"]');
    });
    casper.then(function() {
      this.click('button._negative[data-name="negative"]');
    });
    // train
    casper.then(function() {
      this.click('button.train--train-button');
    });

    // Moleskines test page
    // casper.waitUntilVisible('test--section', function() {
    //   casper.test.assertSelectorHasText('base--h2 test--classifier', 'Moleskine Types');
    // });
  });
});

casper.run(function() {
  this.test.done();
});
<commit_msg>upload a test image to the train results and test the random link<commit_after>var system = require('system');
var apiKey = system.env.API_KEY;

casper.options.waitTimeout = 60000;

casper.start();

casper.thenBypassUnless(function() {
  return apiKey && apiKey.length > 0;
}, 2);

casper.thenOpen('http://localhost:3000/train', function(result) {
  casper.test.assert(result.status === 200, 'Front page opens');
  casper.test.assertSelectorHasText('h1.base--h2.use--header', 'Train a demo classifier');

  // dog breeds
  casper.then(function() {
    this.click('div._training--example:nth-child(1)');
  });

  // dog type examples
  casper.waitForSelector('div._examples[data-kind="dogs"]', function() {
  });

  casper.then(function() {
    this.click('img[data-name="goldenretriever"]');
  });

  // example shots
  casper.waitForSelector('div._examples--contact-sheet[data-kind="dogs"]', function() {
  });

  // click three of deez
  casper.then(function() {
    this.click('button._positive[data-name="goldenretriever"]');
  });
  casper.then(function() {
    this.click('button._positive[data-name="husky"]');
  });
  casper.then(function() {
    this.click('button._positive[data-name="dalmatian"]');
  });
  // TRAAAAIIIIIN!
  casper.then(function() {
    this.click('button.train--train-button');
  });

  casper.then(function() {
    casper.test.assertVisible('.train--loading', 'animation displayed during training');
  });

  casper.waitWhileVisible('.train--loading', function() {
    casper.test.assertTextExist('Test your newly trained demo classifier');

    // husky image by url
    // commenting for now because it sometimes gets marked as a dalmation (?)
    // casper.then(function() {
    //   this.sendKeys('input.test--url-input', 'https://watson-test-resources.mybluemix.net/resources/husky.jpg');
    //   this.sendKeys('input.test--url-input', casper.page.event.key.Enter);
    // });
    // //casper.waitForResource('http://localhost:3000/api/classify');
    // // casper.waitUntilVisible('.test--loading', function() {
    // //   this.echo('loading animation for husky url displayed')
    // // });
    // // casper.waitWhileVisible('.test--loading', function() {
    // //   this.echo('loading animation for husky hidden');
    // //   this.capture('husky-post-load.png');
    // // });
    // casper.waitUntilVisible('.results-table', function() {
    //   casper.test.assertSelectorHasText('.results-table--container:first-child tbody .base--tr:first-child .base--td:first-child', 'Husky');
    // });
    // casper.then(function() {
    //   this.capture('./husky.png');
    // });

    // Dalmatian image by file upload
    this.fill('#test--fileupload', {
      'images_file': 'public/images/bundles/dogs/test/2.jpg'
    }, true);
    casper.waitForResource('http://localhost:3000/api/classify');
    casper.waitUntilVisible('.results-table', function() {
      casper.test.assertSelectorHasText('.results-table--container:first-child tbody .base--tr:first-child .base--td:first-child', 'Dalmatian');
    });

    // click on the random picture link
    casper.then(function() {
      this.clickLabel('Use a random image', 'a');
    });
    casper.then(function() {
      casper.test.assertVisible('.test--loading', 'animation displayed during image processing');
    });
    casper.waitWhileVisible('.test--loading');
    casper.then(function() {
      this.capture('./random.jpg');
      casper.test.assertVisible('.test--output', "random image gets some output ");
      // can't really validate much more for a random image
    });
  }, null, 3 * 60 * 1000);
});

casper.run(function() {
  this.test.done();
});
<|endoftext|>


<commit_before>'use strict';

module.exports = {};
<commit_msg>Add actual code from jellyscript project<commit_after>'use strict';

var assert = require('assert');

module.exports = function() {
  var privacy = {};

  var tags = { in: {}, out: {} };

  privacy.wrap = function(asset) {
    return {
      _unwrap: function(tag) {
        assert(tag === tags.in);

        return {
          tag: tags.out,
          asset: asset
        };
      }
    };
  };

  privacy.unwrap = function(wrappedAsset) {
    var unwrapped = wrappedAsset._unwrap(tags.in);
    assert(unwrapped.tag === tags.out);

    return unwrapped.asset;
  };

  return privacy;
};
<|endoftext|>


<commit_before>import { moduleForComponent, test } from 'ember-qunit';
import hbs from 'htmlbars-inline-precompile';
import Ember from 'ember';
import moment from 'moment';
import wait from 'ember-test-helpers/wait';

const { Object: EmberObject, RSVP, Service } = Ember;
const { Promise, resolve } = RSVP;

let ourEvent;
let course;
let session;
let sessionType;
let sessionDescription;
let offering;
let storeMock;
let sessionLearningMaterials;
moduleForComponent('single-event', 'Integration | Component | ilios calendar single event', {
  integration: true,
  beforeEach() {
    window.Promise = Promise;
    const now = moment().hour(8).minute(0).second(0);
    course = EmberObject.create({
      id: 1,
      title: 'test course',
      sortedObjectives: resolve([]),
    });
    sessionType = EmberObject.create({
      id: 1,
      title: 'test type',
    });
    sessionDescription = EmberObject.create({
      id: 1,
      description: 'test description',
    });
    session = EmberObject.create({
      id: 1,
      title: 'test session',
      course: resolve(course),
      sessionType: resolve(sessionType),
      sessionDescription: resolve(sessionDescription),
      sortedObjectives: resolve([]),
    });
    sessionLearningMaterials = [
      EmberObject.create({
        title: 'Lecture Notes',
        sessionLearningMaterial: '1',
        description: 'Lecture Notes in PDF format',
        absoluteFileUri: 'http://example.edu/notes.pdf',
        mimetype: 'application/pdf',
        filesize: 1000,
        required: true,
        publicNotes: 'Lorem Ipsum',
        position: 0,
      }),
      EmberObject.create({
        title: 'Mystery Meat',
        sessionLearningMaterial: '2',
        position: 1,
        isBlanked: true,
        endDate: new Date('2013-03-01T01:10:00')
      })
    ];
    offering = EmberObject.create({
      id: 1,
      session: resolve(session),
    });
    ourEvent = EmberObject.create({
      user: 1,
      courseExternalId: 'ext1',
      sessionTypeTitle: 'session type',
      name: 'test event',
      courseTitle: 'test course',
      startDate: now,
      endDate: now.clone().add(1, 'hour'),
      location: 'here',
      instructors: ['Great Teacher'],
      offering: 1,
      learningMaterials: sessionLearningMaterials
    });

    storeMock = Service.extend({
      findRecord(what){
        if (what === 'offering') {
          return resolve(offering);
        }
      },
    });
    this.register('service:store', storeMock);
  }
});

test('it renders', async function(assert) {
  assert.expect(12);
  this.set('event', ourEvent);
  this.render(hbs`{{single-event event=event}}`);
  await wait();

  assert.ok(this.$('.single-event-summary').text().includes('test course'), 'course title is displayed');
  assert.ok(this.$('.single-event-summary').text().includes('test session'), 'session title is displayed');
  assert.ok(this.$('.single-event-location').text().includes('here'), 'location is displayed');
  assert.ok(this.$('.single-event-instructors').text().includes('Taught By Great Teacher'), 'instructors are displayed');
  assert.ok(this.$('.single-event-session-is').text().includes('This session is "test type"'), 'session type is displayed');
  assert.ok(this.$('.single-event-summary').text().includes('test description'), 'session description is displayed');
  let $sessionLm = this.$('.single-event-learningmaterial-list:eq(0) .single-event-learningmaterial-item:eq(0)');
  assert.equal(this.$('.single-event-learningmaterial-item-notes', $sessionLm).text().trim(), sessionLearningMaterials[0].get('publicNotes'));
  assert.equal(this.$('.single-event-learningmaterial-item-description', $sessionLm).text().trim(), sessionLearningMaterials[0].get('description'));
  assert.ok(this.$('.single-event-learningmaterial-item-title', $sessionLm).text().includes(sessionLearningMaterials[0].get('title')));
  $sessionLm = this.$('.single-event-learningmaterial-list:eq(0) .single-event-learningmaterial-item:eq(1)');
  assert.equal(this.$('.lm-type-icon .fa-clock-o', $sessionLm).length, 1, 'Timed release icon is visible');
  assert.ok(this.$('.single-event-learningmaterial-item-title', $sessionLm).text().includes(sessionLearningMaterials[0].get('title')));
  assert.ok(this.$('.single-event-learningmaterial-item-timing-info', $sessionLm).text().includes('Available before 03/01/2013 1:10 AM'), 'Timed release info is visible');
});
<commit_msg>rm unnecessary promise override.<commit_after>import { moduleForComponent, test } from 'ember-qunit';
import hbs from 'htmlbars-inline-precompile';
import Ember from 'ember';
import moment from 'moment';
import wait from 'ember-test-helpers/wait';

const { Object: EmberObject, RSVP, Service } = Ember;
const { resolve } = RSVP;

let ourEvent;
let course;
let session;
let sessionType;
let sessionDescription;
let offering;
let storeMock;
let sessionLearningMaterials;
moduleForComponent('single-event', 'Integration | Component | ilios calendar single event', {
  integration: true,
  beforeEach() {
    const now = moment().hour(8).minute(0).second(0);
    course = EmberObject.create({
      id: 1,
      title: 'test course',
      sortedObjectives: resolve([]),
    });
    sessionType = EmberObject.create({
      id: 1,
      title: 'test type',
    });
    sessionDescription = EmberObject.create({
      id: 1,
      description: 'test description',
    });
    session = EmberObject.create({
      id: 1,
      title: 'test session',
      course: resolve(course),
      sessionType: resolve(sessionType),
      sessionDescription: resolve(sessionDescription),
      sortedObjectives: resolve([]),
    });
    sessionLearningMaterials = [
      EmberObject.create({
        title: 'Lecture Notes',
        sessionLearningMaterial: '1',
        description: 'Lecture Notes in PDF format',
        absoluteFileUri: 'http://example.edu/notes.pdf',
        mimetype: 'application/pdf',
        filesize: 1000,
        required: true,
        publicNotes: 'Lorem Ipsum',
        position: 0,
      }),
      EmberObject.create({
        title: 'Mystery Meat',
        sessionLearningMaterial: '2',
        position: 1,
        isBlanked: true,
        endDate: new Date('2013-03-01T01:10:00')
      })
    ];
    offering = EmberObject.create({
      id: 1,
      session: resolve(session),
    });
    ourEvent = EmberObject.create({
      user: 1,
      courseExternalId: 'ext1',
      sessionTypeTitle: 'session type',
      name: 'test event',
      courseTitle: 'test course',
      startDate: now,
      endDate: now.clone().add(1, 'hour'),
      location: 'here',
      instructors: ['Great Teacher'],
      offering: 1,
      learningMaterials: sessionLearningMaterials
    });

    storeMock = Service.extend({
      findRecord(what){
        if (what === 'offering') {
          return resolve(offering);
        }
      },
    });
    this.register('service:store', storeMock);
  }
});

test('it renders', async function(assert) {
  assert.expect(12);
  this.set('event', ourEvent);
  this.render(hbs`{{single-event event=event}}`);
  await wait();

  assert.ok(this.$('.single-event-summary').text().includes('test course'), 'course title is displayed');
  assert.ok(this.$('.single-event-summary').text().includes('test session'), 'session title is displayed');
  assert.ok(this.$('.single-event-location').text().includes('here'), 'location is displayed');
  assert.ok(this.$('.single-event-instructors').text().includes('Taught By Great Teacher'), 'instructors are displayed');
  assert.ok(this.$('.single-event-session-is').text().includes('This session is "test type"'), 'session type is displayed');
  assert.ok(this.$('.single-event-summary').text().includes('test description'), 'session description is displayed');
  let $sessionLm = this.$('.single-event-learningmaterial-list:eq(0) .single-event-learningmaterial-item:eq(0)');
  assert.equal(this.$('.single-event-learningmaterial-item-notes', $sessionLm).text().trim(), sessionLearningMaterials[0].get('publicNotes'));
  assert.equal(this.$('.single-event-learningmaterial-item-description', $sessionLm).text().trim(), sessionLearningMaterials[0].get('description'));
  assert.ok(this.$('.single-event-learningmaterial-item-title', $sessionLm).text().includes(sessionLearningMaterials[0].get('title')));
  $sessionLm = this.$('.single-event-learningmaterial-list:eq(0) .single-event-learningmaterial-item:eq(1)');
  assert.equal(this.$('.lm-type-icon .fa-clock-o', $sessionLm).length, 1, 'Timed release icon is visible');
  assert.ok(this.$('.single-event-learningmaterial-item-title', $sessionLm).text().includes(sessionLearningMaterials[0].get('title')));
  assert.ok(this.$('.single-event-learningmaterial-item-timing-info', $sessionLm).text().includes('Available before 03/01/2013 1:10 AM'), 'Timed release info is visible');
});
<|endoftext|>


<commit_before>/**
 * TicTacToe
 *
 * @param nextGame {Function}
 * @constructor
 */
var TicTacToe = function (nextGame) {

    this.nextGame = nextGame;

    this.el = document.getElementById('ticTacToe');

    this.play = true;
    this.icon = false; // false = x; true = o
    this.squares = [];
    this.wins   = [
        [0, 1, 2],
        [3, 4, 5],
        [6, 7, 8],

        [0, 3, 6],
        [1, 4, 7],
        [2, 5, 8],

        [6, 4, 2],
        [0, 4, 8]
    ];

    this.buildBoard();
};

/**
 * getSquare
 *
 * @param index {Number}
 * @returns {Object}
 */
TicTacToe.prototype.getSquare = function (index) {

    var square = this.squares[index];

    if (square) {
        return square;
    }

    throw new Error('Square out of range: ' + index);
};

/**
 * getStatus
 * gets the status of a square and matches the icon in play
 *
 * @returns {Boolean}
 */
TicTacToe.prototype.getStatus = function (index) {

    var square = this.getSquare(index),
        icon = square.className === 'cross';

     return !!square.played && (icon === this.icon);
};

/**
 * getSquaresInPlay
 * gets the remaining squares in play that haven't been played
 * @returns {Number}
 */
TicTacToe.prototype.getSquaresInPlay = function () {

    return this.squares.filter(function (square) {
        return !square.played;
    }).length;

};

/**
 * getCompleteText
 * gets the complete text to display at the end of a game
 *
 * @returns {string}
 */
TicTacToe.prototype.getCompleteText = function () {

    var icon = this.icon ? 'crosses' : 'noughts',
        text = 'Game over, ' + icon + ' win this time. \n',
        squaresLeft = this.getSquaresInPlay();

    if (!squaresLeft) {
        text = 'No winner this time. \n';
    }

    return text;
};

/**
 * buildBoard
 *
 * @returns TicTacToe {Object}
 */
TicTacToe.prototype.buildBoard = function () {

    var tictactoe = this,
        list = document.createElement('ul'),
        item,
        i;

    list.className = 'tictactoe';

    for (i = 0; i < 9; i += 1) {

        item = document.createElement('li');
        item.setAttribute('data-index', i);

        item.addEventListener('click', function (evt) {

            if (tictactoe.play) {
                tictactoe.playSquare.call(tictactoe, evt);
            }
        }, false);

        this.squares.push(item);

        list.appendChild(item);
    }

    this.el.innerHTML = '';
    this.el.appendChild(list);

    return this;
};

/**
 * playSquare
 * event handler for when a square is clicked
 *
 * @param evt {Event.Object}
 */
TicTacToe.prototype.playSquare = function (evt) {

    var index = evt.currentTarget.getAttribute('data-index'),
        square = this.getSquare(index);

    if (square.played) {
        return;
    }

    if (this.icon) {
        square.classList.add('cross');
        square.cross = true;

        //this.computerTurn();
    } else {
        square.classList.add('nought');
        square.nought = true;
    }

    square.played = true;

    this.checkForWin(index);
};

/**
 * checkForWin
 *
 * @param index {Number|String}
 */
TicTacToe.prototype.checkForWin = function (index) {

    var accumulative,
        gameOver = false,
        squaresLeft = this.getSquaresInPlay();

    index = +index;

    // iterate the possibles to check for win
    this.wins.forEach(function (wins) {

        accumulative = true;

        if (wins.indexOf(index) < 0 || gameOver) {
            return;
        }

        wins.forEach(function (win) {
            if (!this.getStatus(win)) {
                accumulative = false;
            }
        }, this);

        if (accumulative || !squaresLeft) {
            gameOver = this.gameOver();
        }
    }, this);

    // only swap to next player if game still in play
    if (this.play) {
        this.icon = !this.icon;
    }
};

/**
 * gameOver
 * stops play
 */
TicTacToe.prototype.gameOver = function () {

    var score = document.createElement('div'),
        button = document.createElement('button');

    this.play = false;

    button.innerText = 'Play Again';

    score.className = 'score';
    score.innerText = this.getCompleteText();
    score.appendChild(button);

    this.el.appendChild(score);

    button.addEventListener('click', this.nextGame, false);

    return this;
};

// loop for continual play
var newGame = function () {
    return new TicTacToe(newGame);
};

// start the game
newGame();
<commit_msg>Starting the computer moves logic<commit_after>/**
 * TicTacToe
 *
 * @param nextGame {Function}
 * @constructor
 */
var TicTacToe = function (nextGame) {

    this.nextGame = nextGame;

    this.el = document.getElementById('ticTacToe');

    this.play = true;
    this.icon = true; // false = x; true = o
    this.squares = [];
    this.wins   = [
        [0, 1, 2],
        [3, 4, 5],
        [6, 7, 8],

        [0, 3, 6],
        [1, 4, 7],
        [2, 5, 8],

        [6, 4, 2],
        [0, 4, 8]
    ];

    this.buildBoard();
};

/**
 * getSquare
 *
 * @param index {Number}
 * @returns {Object}
 */
TicTacToe.prototype.getSquare = function (index) {

    var square = this.squares[index];

    if (square) {
        return square;
    }

    throw new Error('Square out of range: ' + index);
};

/**
 * getStatus
 * gets the status of a square and matches the icon in play
 *
 * @returns {Boolean}
 */
TicTacToe.prototype.getStatus = function (index) {

    var square = this.getSquare(index),
        icon = square.className === 'cross';

     return !!square.played && (icon === this.icon);
};

/**
 * getSquaresInPlay
 * gets the remaining squares in play that haven't been played
 * @returns {Number}
 */
TicTacToe.prototype.getSquaresInPlay = function () {

    return this.squares.filter(function (square) {
        return !square.played;
    }).length;

};

/**
 * getCompleteText
 * gets the complete text to display at the end of a game
 *
 * @returns {string}
 */
TicTacToe.prototype.getCompleteText = function () {

    var icon = this.icon ? 'crosses' : 'noughts',
        text = 'Game over, ' + icon + ' win this time. \n',
        squaresLeft = this.getSquaresInPlay();

    if (!squaresLeft) {
        text = 'No winner this time. \n';
    }

    return text;
};

/**
 * buildBoard
 *
 * @returns TicTacToe {Object}
 */
TicTacToe.prototype.buildBoard = function () {

    var tictactoe = this,
        list = document.createElement('ul'),
        item,
        i;

    list.className = 'tictactoe';

    for (i = 0; i < 9; i += 1) {

        item = document.createElement('li');
        item.setAttribute('data-index', i);

        item.addEventListener('click', function (evt) {

            if (tictactoe.play) {
                tictactoe.playSquare.call(tictactoe, evt);
            }
        }, false);

        this.squares.push(item);

        list.appendChild(item);
    }

    this.el.innerHTML = '';
    this.el.appendChild(list);

    return this;
};

/**
 * playSquare
 * event handler for when a square is clicked
 *
 * @param evt {Event.Object} !optional
 * @param index {Number} !optional
 */
TicTacToe.prototype.playSquare = function (evt, index) {

    var square;

    index = evt ? evt.currentTarget.getAttribute('data-index') : index;

    square = this.getSquare(index);

    if (square.played) {
        return;
    }

    if (this.icon) {
        square.classList.add('cross');
        square.cross = true;
    } else {
        square.classList.add('nought');
        square.nought = true;
    }

    square.played = true;

    this.checkForWin(index);
};

/**
 * checkForWin
 *
 * @param index {Number|String}
 */
TicTacToe.prototype.checkForWin = function (index) {

    var accumulative,
        gameOver = false,
        squaresLeft = this.getSquaresInPlay();

    index = +index;

    // iterate the possibles to check for win
    this.wins.forEach(function (wins) {

        accumulative = true;

        if (wins.indexOf(index) < 0 || gameOver) {
            return;
        }

        wins.forEach(function (win) {
            if (!this.getStatus(win)) {
                accumulative = false;
            }
        }, this);

        if (accumulative || !squaresLeft) {
            gameOver = this.gameOver();
        }
    }, this);

    // only swap to next player if game still in play
    if (this.play) {
        this.icon = !this.icon;

        if (!this.icon) {
            this.computerTurn();
        }
    }
};

/**
 * gameOver
 * stops play
 */
TicTacToe.prototype.gameOver = function () {

    var score = document.createElement('div'),
        button = document.createElement('button');

    this.play = false;

    button.innerText = 'Play Again';

    score.className = 'score';
    score.innerText = this.getCompleteText();
    score.appendChild(button);

    this.el.appendChild(score);

    button.addEventListener('click', this.nextGame, false);

    return this;
};

/**
 * computerTurn
 * works out the computers next move and stops the user getting three in a row
 */
TicTacToe.prototype.computerTurn = function () {

    // @TODO work out from the available wins where to place
    // also stop them getting three in a row ;)
    this.playSquare(null, 1);

    // maybe put a small delay on the move, so as to emulate the computer is
    // thinking about the move
};

// loop for continual play
var newGame = function () {
    return new TicTacToe(newGame);
};

// start the game
newGame();
<|endoftext|>


<commit_before>'use strict';

var Promise = require('bluebird');
var fs = Promise.promisifyAll(require("fs"));
var Queue = require('../utils/queue');
var getFileNames = require('../utils/file_utils').getFileNames;

var fileQueue = new Queue;

function newReader(context, opConfig, jobConfig) {
    var context = context;
    var opConfig = opConfig;

    return function(msg) {
        var data = fs.readFileSync(msg, 'utf-8');
        return JSON.parse(data);
    }
}

function newSlicer(context, job, retryData) {
    var opConfig = job.readerConfig;
    var jobConfig = job.jobConfig;

    getFileNames(jobConfig, opConfig, fileQueue);

    return function() {
        return fileQueue.dequeue();
    }
}

function schema() {
    return {
        path: {
            doc: 'Path to where the directory is located to read from',
            default: null,
            format: 'required_String'
        }
    };
}

module.exports = {
    newReader: newReader,
    newSlicer: newSlicer,
    schema: schema
};<commit_msg>file_import uses new slicers call signiture<commit_after>'use strict';

var Promise = require('bluebird');
var fs = Promise.promisifyAll(require("fs"));
var Queue = require('../utils/queue');
var getFileNames = require('../utils/file_utils').getFileNames;

var fileQueue = new Queue;

function newReader(context, opConfig, jobConfig) {
    var context = context;
    var opConfig = opConfig;

    return function(msg) {
        var data = fs.readFileSync(msg, 'utf-8');
        return JSON.parse(data);
    }
}

function newSlicer(context, job, retryData) {
    var opConfig = job.readerConfig;
    var jobConfig = job.jobConfig;
    var slicers = [];
    //TODO review performance implications
    getFileNames(jobConfig, opConfig, fileQueue);

    slicers.push(function() {return fileQueue.dequeue()});

    return Promise.resolve(slicers)
}

function schema() {
    return {
        path: {
            doc: 'Path to where the directory is located to read from',
            default: null,
            format: 'required_String'
        }
    };
}

var parallelSlicers = false;

module.exports = {
    newReader: newReader,
    newSlicer: newSlicer,
    schema: schema,
    parallelSlicers: parallelSlicers
};<|endoftext|>


<commit_before>
var dom      = exports.dom = require("./jsdom/level3/index").dom,
    fs       = require("fs"),
    pkg      = JSON.parse(fs.readFileSync(__dirname + "/../package.json")),
    request  = require('request'),
    URL      = require('url');

var style = require('./jsdom/level2/style');
exports.defaultLevel = dom.level3.html;
exports.browserAugmentation = require("./jsdom/browser/index").browserAugmentation;
exports.windowAugmentation = require("./jsdom/browser/index").windowAugmentation;

exports.debugMode = false;

var createWindow = exports.createWindow = require("./jsdom/browser/index").createWindow;

exports.__defineGetter__('version', function() {
  return pkg.version;
});

exports.jsdom = function (html, level, options) {

  options = options || {};
  level   = level || exports.defaultLevel;

  if (!options.url) {
    options.url = (module.parent.id === 'jsdom') ?
                  module.parent.parent.filename  :
                  module.parent.filename;
  }

  if (options.features && options.features.QuerySelector) {
    require("./jsdom/selectors/index").applyQuerySelectorPrototype(level);
  }

  var browser = exports.browserAugmentation(level, options),
      doc     = (browser.HTMLDocument)             ?
                 new browser.HTMLDocument(options) :
                 new browser.Document(options);

  exports.applyDocumentFeatures(doc, options.features);

  if (!!html) {
    doc.write(html + '');
  } else {
    doc.write('');
  }

  if (doc.close && !options.deferClose) {
    doc.close();
  }

  // Kept for backwards-compatibility. The window is lazily created when
  // document.parentWindow or document.defaultView is accessed.
  doc.createWindow = function() {
    // Remove ourself
    if (doc.createWindow) {
      delete doc.createWindow;
    }
    return doc.parentWindow;
  };

  return doc;
};

exports.html = function(html, level, options) {
  html += '';

  // TODO: cache a regex and use it here instead
  //       or make the parser handle it
  var htmlLowered = html.toLowerCase();

  // body
  if (!~htmlLowered.indexOf('<body')) {
    html = '' + html + '';
  }

  // html
  if (!~htmlLowered.indexOf('<html')) {
    html = '' + html + '';
  }
  return exports.jsdom(html, level, options);
};

exports.availableDocumentFeatures = [
  'FetchExternalResources',
  'ProcessExternalResources',
  'MutationEvents',
  'QuerySelector'
];

exports.defaultDocumentFeatures = {
  "FetchExternalResources"   : ['script'/*, 'img', 'css', 'frame', 'link'*/],
  "ProcessExternalResources" : ['script'/*, 'frame', 'iframe'*/],
  "MutationEvents"           : '2.0',
  "QuerySelector"            : false
};

exports.applyDocumentFeatures = function(doc, features) {
  var i, maxFeatures = exports.availableDocumentFeatures.length,
      defaultFeatures = exports.defaultDocumentFeatures,
      j,
      k,
      featureName,
      featureSource;

  features = features || {};

  for (i=0; i<maxFeatures; i++) {
    featureName = exports.availableDocumentFeatures[i];
    if (typeof features[featureName] !== 'undefined') {
      featureSource = features[featureName];
    } else if (defaultFeatures[featureName]) {
      featureSource = defaultFeatures[featureName];
    } else {
      continue;
    }

    doc.implementation.removeFeature(featureName);

    if (typeof featureSource !== 'undefined') {
      if (featureSource instanceof Array) {
        k = featureSource.length;
        for (j=0; j<k; j++) {
          doc.implementation.addFeature(featureName, featureSource[j]);
        }
      } else {
        doc.implementation.addFeature(featureName, featureSource);
      }
    }
  }
};

exports.jQueryify = exports.jsdom.jQueryify = function (window /* path [optional], callback */) {

  if (!window || !window.document) { return; }

  var args = Array.prototype.slice.call(arguments),
      callback = (typeof(args[args.length - 1]) === 'function') && args.pop(),
      path,
      jQueryTag = window.document.createElement("script");

  if (args.length > 1 && typeof(args[1] === 'string')) {
    path = args[1];
  }

  var features = window.document.implementation._features;

  window.document.implementation.addFeature('FetchExternalResources', ['script']);
  window.document.implementation.addFeature('ProcessExternalResources', ['script']);
  window.document.implementation.addFeature('MutationEvents', ["1.0"]);
  jQueryTag.src = path || 'http://code.jquery.com/jquery-latest.js';
  window.document.body.appendChild(jQueryTag);

  jQueryTag.onload = function() {
    if (callback) {
      callback(window, window.jQuery);
    }

    window.document.implementation._features = features;
  };

  return window;
};

exports.env = exports.jsdom.env = function() {
  var
  args        = Array.prototype.slice.call(arguments),
  config      = exports.env.processArguments(args),
  callback    = config.done,
  processHTML = function(err, html) {

    html += '';
    if(err) {
      return callback(err);
    }

    config.scripts = config.scripts || [];
    if (typeof config.scripts === 'string') {
      config.scripts = [config.scripts];
    }

    var
    options    = {
      features: {
        'FetchExternalResources' : false,
        'ProcessExternalResources' : false
      },
      url: config.url
    },
    window     = exports.html(html, null, options).createWindow(),
    features   = window.document.implementation._features,
    docsLoaded = 0,
    totalDocs  = config.scripts.length,
    readyState = null,
    errors     = null;

    if (!window || !window.document) {
      return callback(new Error('JSDOM: a window object could not be created.'));
    }

    window.document.implementation.addFeature('FetchExternalResources', ['script']);
    window.document.implementation.addFeature('ProcessExternalResources', ['script']);
    window.document.implementation.addFeature('MutationEvents', ['1.0']);

    if (totalDocs > 0) {
      config.scripts.forEach(function(src) {
        var
        script = window.document.createElement('script'),
        scriptComplete = function() {
          docsLoaded++;
          if (docsLoaded >= totalDocs) {
            window.document.implementation._features = features;
            callback(errors, window);
          }
        };

        script.onload = function() {
          process.nextTick(scriptComplete);
        };

        script.onerror = function(e) {
          if (!errors) {
            errors = [];
          }
          errors.push(e.error || e.message);
          // nextTick so that an exception within scriptComplete won't cause
          // another script onerror (which would be an infinite loop)
          process.nextTick(scriptComplete);
        };

        script.src = src;
        window.document.documentElement.appendChild(script);
        window.document.documentElement.removeChild(script);
      });
    } else {
      callback(errors, window);
    }
  };

  config.html += '';

  // Handle markup
  if (config.html.indexOf("\n") > 0 || config.html.match(/^\W*</)) {
    processHTML(null, config.html);

  // Handle url/file
  } else {
    var url = URL.parse(config.html);
    config.url = config.url || url.href;
    if (url.hostname) {
      request({uri: url, encoding: config.encoding || 'utf8'}, function(err, request, body) {
        processHTML(err, body);
      });
    } else {
      fs.readFile(url.pathname, processHTML);
    }
  }
};

/*
  Since jsdom.env() is a helper for quickly and easily setting up a
  window with scripts and such already loaded into it, the arguments
  should be fairly flexible.  Here are the requirements

  1) collect `html` (url, string, or file on disk)  (STRING)
  2) load `code` into the window (array of scripts) (ARRAY)
  3) callback when resources are `done`             (FUNCTION)
  4) configuration                                  (OBJECT)

  Rules:
  + if there is one argument it had better be an object with atleast
    a `html` and `done` property (other properties are gravy)

  + arguments above are pulled out of the arguments and put into the
    config object that is returned
*/
exports.env.processArguments = function(args) {
  if (!args || !args.length || args.length < 1) {
    throw new Error('No arguments passed to jsdom.env().');
  }

  var
  props = {
    'html'    : true,
    'done'    : true,
    'scripts' : false,
    'config'  : false,
    'url'     : false  // the URL for location.href if different from 'html'
  },
  propKeys = Object.keys(props),
  config = {
    code : []
  },
  l    = args.length
  ;
  if (l === 1) {
    config = args[0];
  } else {
    args.forEach(function(v) {
      var type = typeof v;
      if (!v) {
        return;
      }
      if (type === 'string' || v + '' === v) {
        config.html = v;
      } else if (type === 'object') {
        // Array
        if (v.length && v[0]) {
          config.scripts = v;
        } else {
          // apply missing required properties if appropriate
          propKeys.forEach(function(req) {

            if (typeof v[req] !== 'undefined' &&
                typeof config[req] === 'undefined') {

              config[req] = v[req];
              delete v[req];
            }
          });
          config.config = v;
        }
      } else if (type === 'function') {
        config.done = v;
      }
    });
  }

  propKeys.forEach(function(req) {
    var required = props[req];
    if (required && typeof config[req] === 'undefined') {
      throw new Error("jsdom.env requires a '" + req + "' argument");
    }
  });
  return config;
};
<commit_msg>Edit comment.<commit_after>
var dom      = exports.dom = require("./jsdom/level3/index").dom,
    fs       = require("fs"),
    pkg      = JSON.parse(fs.readFileSync(__dirname + "/../package.json")),
    request  = require('request'),
    URL      = require('url');

var style = require('./jsdom/level2/style');
exports.defaultLevel = dom.level3.html;
exports.browserAugmentation = require("./jsdom/browser/index").browserAugmentation;
exports.windowAugmentation = require("./jsdom/browser/index").windowAugmentation;

exports.debugMode = false;

var createWindow = exports.createWindow = require("./jsdom/browser/index").createWindow;

exports.__defineGetter__('version', function() {
  return pkg.version;
});

exports.jsdom = function (html, level, options) {

  options = options || {};
  level   = level || exports.defaultLevel;

  if (!options.url) {
    options.url = (module.parent.id === 'jsdom') ?
                  module.parent.parent.filename  :
                  module.parent.filename;
  }

  if (options.features && options.features.QuerySelector) {
    require("./jsdom/selectors/index").applyQuerySelectorPrototype(level);
  }

  var browser = exports.browserAugmentation(level, options),
      doc     = (browser.HTMLDocument)             ?
                 new browser.HTMLDocument(options) :
                 new browser.Document(options);

  exports.applyDocumentFeatures(doc, options.features);

  if (!!html) {
    doc.write(html + '');
  } else {
    doc.write('');
  }

  if (doc.close && !options.deferClose) {
    doc.close();
  }

  // Kept for backwards-compatibility. The window is lazily created when
  // document.parentWindow or document.defaultView is accessed.
  doc.createWindow = function() {
    // Remove ourself
    if (doc.createWindow) {
      delete doc.createWindow;
    }
    return doc.parentWindow;
  };

  return doc;
};

exports.html = function(html, level, options) {
  html += '';

  // TODO: cache a regex and use it here instead
  //       or make the parser handle it
  var htmlLowered = html.toLowerCase();

  // body
  if (!~htmlLowered.indexOf('<body')) {
    html = '' + html + '';
  }

  // html
  if (!~htmlLowered.indexOf('<html')) {
    html = '' + html + '';
  }
  return exports.jsdom(html, level, options);
};

exports.availableDocumentFeatures = [
  'FetchExternalResources',
  'ProcessExternalResources',
  'MutationEvents',
  'QuerySelector'
];

exports.defaultDocumentFeatures = {
  "FetchExternalResources"   : ['script'/*, 'img', 'css', 'frame', 'link'*/],
  "ProcessExternalResources" : ['script'/*, 'frame', 'iframe'*/],
  "MutationEvents"           : '2.0',
  "QuerySelector"            : false
};

exports.applyDocumentFeatures = function(doc, features) {
  var i, maxFeatures = exports.availableDocumentFeatures.length,
      defaultFeatures = exports.defaultDocumentFeatures,
      j,
      k,
      featureName,
      featureSource;

  features = features || {};

  for (i=0; i<maxFeatures; i++) {
    featureName = exports.availableDocumentFeatures[i];
    if (typeof features[featureName] !== 'undefined') {
      featureSource = features[featureName];
    } else if (defaultFeatures[featureName]) {
      featureSource = defaultFeatures[featureName];
    } else {
      continue;
    }

    doc.implementation.removeFeature(featureName);

    if (typeof featureSource !== 'undefined') {
      if (featureSource instanceof Array) {
        k = featureSource.length;
        for (j=0; j<k; j++) {
          doc.implementation.addFeature(featureName, featureSource[j]);
        }
      } else {
        doc.implementation.addFeature(featureName, featureSource);
      }
    }
  }
};

exports.jQueryify = exports.jsdom.jQueryify = function (window /* path [optional], callback */) {

  if (!window || !window.document) { return; }

  var args = Array.prototype.slice.call(arguments),
      callback = (typeof(args[args.length - 1]) === 'function') && args.pop(),
      path,
      jQueryTag = window.document.createElement("script");

  if (args.length > 1 && typeof(args[1] === 'string')) {
    path = args[1];
  }

  var features = window.document.implementation._features;

  window.document.implementation.addFeature('FetchExternalResources', ['script']);
  window.document.implementation.addFeature('ProcessExternalResources', ['script']);
  window.document.implementation.addFeature('MutationEvents', ["1.0"]);
  jQueryTag.src = path || 'http://code.jquery.com/jquery-latest.js';
  window.document.body.appendChild(jQueryTag);

  jQueryTag.onload = function() {
    if (callback) {
      callback(window, window.jQuery);
    }

    window.document.implementation._features = features;
  };

  return window;
};

exports.env = exports.jsdom.env = function() {
  var
  args        = Array.prototype.slice.call(arguments),
  config      = exports.env.processArguments(args),
  callback    = config.done,
  processHTML = function(err, html) {

    html += '';
    if(err) {
      return callback(err);
    }

    config.scripts = config.scripts || [];
    if (typeof config.scripts === 'string') {
      config.scripts = [config.scripts];
    }

    var
    options    = {
      features: {
        'FetchExternalResources' : false,
        'ProcessExternalResources' : false
      },
      url: config.url
    },
    window     = exports.html(html, null, options).createWindow(),
    features   = window.document.implementation._features,
    docsLoaded = 0,
    totalDocs  = config.scripts.length,
    readyState = null,
    errors     = null;

    if (!window || !window.document) {
      return callback(new Error('JSDOM: a window object could not be created.'));
    }

    window.document.implementation.addFeature('FetchExternalResources', ['script']);
    window.document.implementation.addFeature('ProcessExternalResources', ['script']);
    window.document.implementation.addFeature('MutationEvents', ['1.0']);

    if (totalDocs > 0) {
      config.scripts.forEach(function(src) {
        var
        script = window.document.createElement('script'),
        scriptComplete = function() {
          docsLoaded++;
          if (docsLoaded >= totalDocs) {
            window.document.implementation._features = features;
            callback(errors, window);
          }
        };

        script.onload = function() {
          process.nextTick(scriptComplete);
        };

        script.onerror = function(e) {
          if (!errors) {
            errors = [];
          }
          errors.push(e.error || e.message);
          // nextTick so that an exception within scriptComplete won't cause
          // another script onerror (which would be an infinite loop)
          process.nextTick(scriptComplete);
        };

        script.src = src;
        window.document.documentElement.appendChild(script);
        window.document.documentElement.removeChild(script);
      });
    } else {
      callback(errors, window);
    }
  };

  config.html += '';

  // Handle markup
  if (config.html.indexOf("\n") > 0 || config.html.match(/^\W*</)) {
    processHTML(null, config.html);

  // Handle url/file
  } else {
    var url = URL.parse(config.html);
    config.url = config.url || url.href;
    if (url.hostname) {
      request({uri: url, encoding: config.encoding || 'utf8'}, function(err, request, body) {
        processHTML(err, body);
      });
    } else {
      fs.readFile(url.pathname, processHTML);
    }
  }
};

/*
  Since jsdom.env() is a helper for quickly and easily setting up a
  window with scripts and such already loaded into it, the arguments
  should be fairly flexible.  Here are the requirements

  1) collect `html` (url, string, or file on disk)  (STRING)
  2) load `code` into the window (array of scripts) (ARRAY)
  3) callback when resources are `done`             (FUNCTION)
  4) configuration                                  (OBJECT)

  Rules:
  + if there is one argument it had better be an object with atleast
    a `html` and `done` property (other properties are gravy)

  + arguments above are pulled out of the arguments and put into the
    config object that is returned
*/
exports.env.processArguments = function(args) {
  if (!args || !args.length || args.length < 1) {
    throw new Error('No arguments passed to jsdom.env().');
  }

  var
  props = {
    'html'    : true,
    'done'    : true,
    'scripts' : false,
    'config'  : false,
    'url'     : false  // the URL for location.href if different from html
  },
  propKeys = Object.keys(props),
  config = {
    code : []
  },
  l    = args.length
  ;
  if (l === 1) {
    config = args[0];
  } else {
    args.forEach(function(v) {
      var type = typeof v;
      if (!v) {
        return;
      }
      if (type === 'string' || v + '' === v) {
        config.html = v;
      } else if (type === 'object') {
        // Array
        if (v.length && v[0]) {
          config.scripts = v;
        } else {
          // apply missing required properties if appropriate
          propKeys.forEach(function(req) {

            if (typeof v[req] !== 'undefined' &&
                typeof config[req] === 'undefined') {

              config[req] = v[req];
              delete v[req];
            }
          });
          config.config = v;
        }
      } else if (type === 'function') {
        config.done = v;
      }
    });
  }

  propKeys.forEach(function(req) {
    var required = props[req];
    if (required && typeof config[req] === 'undefined') {
      throw new Error("jsdom.env requires a '" + req + "' argument");
    }
  });
  return config;
};
<|endoftext|>


<commit_before>var webpack = require('webpack');

module.exports = function(config) {
  config.set({
    autoWatch: true,
    browsers: ['PhantomJS'],
    files: [
      // React explodes in PhantomJS without a bind polyfill
      './node_modules/phantomjs-polyfill/bind-polyfill.js',
      'tests.webpack.js'
    ],
    reporters: [ 'progress', 'coverage' ],
    coverageReporter: {
        type: 'text'
    },
  	frameworks: ['jasmine'],
    preprocessors: {
      'tests.webpack.js': ['webpack', 'sourcemap']
    },
    webpack: {
      devtool: 'inline-source-map',
      module: {
        preLoaders: [
          {test: /\.js$/, loader: 'eslint-loader', exclude: /node_modules/},
          {test: /\.js$/, loader: 'isparta', include: /src/, exclude: /(\__tests__|node_modules)/} // Only check test coverage on source files, ignore tests and node_modules
        ],
        loaders: [
          {test: /\.js$/, loaders: ['babel-loader?optional[]=runtime'], exclude: /node_modules/}
        ]
      }
    },
    webpackServer: {
      noInfo: true
    }
  });
};
<commit_msg>Fixes indentation in karma.conf.js<commit_after>var webpack = require('webpack');

module.exports = function(config) {
  config.set({
    autoWatch: true,
    browsers: ['PhantomJS'],
    files: [
      // React explodes in PhantomJS without a bind polyfill
      './node_modules/phantomjs-polyfill/bind-polyfill.js',
      'tests.webpack.js'
    ],
    reporters: [ 'progress', 'coverage' ],
    coverageReporter: {
      type: 'text'
    },
  	frameworks: ['jasmine'],
    preprocessors: {
      'tests.webpack.js': ['webpack', 'sourcemap']
    },
    webpack: {
      devtool: 'inline-source-map',
      module: {
        preLoaders: [
          {test: /\.js$/, loader: 'eslint-loader', exclude: /node_modules/},
          {test: /\.js$/, loader: 'isparta', include: /src/, exclude: /(\__tests__|node_modules)/} // Only check test coverage on source files, ignore tests and node_modules
        ],
        loaders: [
          {test: /\.js$/, loaders: ['babel-loader?optional[]=runtime'], exclude: /node_modules/}
        ]
      }
    },
    webpackServer: {
      noInfo: true
    }
  });
};
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2011-2012 by Nikhil Marathe <nsm.nikhil@gmail.com>
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */
"use strict";

var assert = require('assert');
var node_constants = require('constants');
var async = require('async');
var constants = require('./constants');
var util = require('./util');
var rpc = require('./rpc');
var Bucket = require('./bucket').Bucket;
var _ = require('underscore');
_.str = require('underscore.string');

// abbreviate message utilities
var MC = util.message_contact;
var MID = util.message_rpcID;

exports.KNode = function(desc) {
    // TODO: probably want to persist nodeID
    this.self = _.defaults({ nodeID: util.nodeID(desc.address, desc.port) }, desc);
    Object.freeze(this.self);

    this._storage = {};
    // object treated as an array
    this._buckets = {};
    this._rpc = new rpc.RPC(this.self, _.bind(this._onMessage, this));

    // DEBUG
    setInterval(_.bind(this.debug, this), 5000);
}

exports.KNode.prototype._MSG = function(type, params) {
    // NOTE: always keep this.self last. This way users of _MSG
    // don't have to worry about accidentally overriding self properties
    return _.extend({ type: type}, params, this.self);
}

exports.KNode.prototype._onMessage = function(message) {
    var methodName = '_on' + _.str.titleize(_.str.camelize(message.type.toLowerCase()));
    var action = this[methodName];
    if (action) {
        this._updateContact(MC(message));
        _.bind(action, this)(message);
    }
    else
        console.warn("Unknown message", message);
}

exports.KNode.prototype._ping = function(contact) {

    function onPong(err, message) {
        if (err && err.errno == node_constants.ETIMEDOUT) {
            // TODO: handle peer death
        }
        else {
            assert.equal(message.type, 'PONG');
            this._updateContact(MC(message));
        }
    }

    this._rpc.send(contact, this._MSG('PING'), _.bind(onPong, this));
}

exports.KNode.prototype._onPing = function(message) {
    // this can be made more intelligent such that
    // if an outgoing message is present, piggyback the pong
    // onto it rather than sending it separately
    this._rpc.send(MC(message), this._MSG('PONG', {'replyTo': MID(message)}));
}

exports.KNode.prototype._updateContact = function(contact, cb) {
    var callback = cb || function() {};
    var bucketIndex = util.bucketIndex(this.self.nodeID, contact.nodeID);
    assert.ok(bucketIndex < constants.B);
    if (!this._buckets[bucketIndex])
        this._buckets[bucketIndex] = new Bucket();

    var bucket = this._buckets[bucketIndex];
    contact.lastSeen = Date.now();

    var exists = bucket.contains(contact);
    if (exists) {
        // move to the end of the bucket
        bucket.remove(contact);
        bucket.add(contact);
        callback();
    }
    else if (bucket.size() < constants.K) {
        bucket.add(contact);
        callback();
    }
    else { // TODO:
        this._rpc.send(bucket.get(0), this._MSG('PING'),
            _.bind(function(err, message) {
                if( err ) {
                    // add new contact, old one is dead
                    bucket.removeIndex(0);
                    bucket.add(contact);
                }
                else {
                }
                callback();
            }), this);
    }
}

// TODO: handle large values which
// won't fit in single UDP packets
exports.KNode.prototype._onStore = function(message) {
    assert.equal(message.key.length, constants.B/4);
    this._storage[message.key] = _.clone(message.value);
    this._rpc.send(MC(message), this._MSG('STORE_REPLY', {
        'replyTo': MID(message),
        'status': true
    }));
}

// This is just to prevent Unknown message errors
exports.KNode.prototype._onStoreReply = function() {}

// debug methods FIXME: remove
exports.KNode.prototype._onGet = function(message) {
    this.get(message.key, function(err, val) {
        console.log('ONGET', val);
    });
}

exports.KNode.prototype._onFindValue = function(message) {
    assert.ok(message.key);
    assert.equal(message.key.length, constants.B/4);
    if (this._storage.hasOwnProperty(message.key)) {
        this._rpc.send(MC(message), this._MSG('FIND_VALUE_REPLY', {
            'replyTo': MID(message),
            'found': true,
            'value': this._storage[message.key]
        }));
    }
    else {
        var contacts = this._findClosestNodes(message.key, constants.K, MC(message).nodeID);

        this._rpc.send(MC(message), this._MSG('FIND_NODE_REPLY', {
            'replyTo': MID(message),
            'contacts': contacts
        }));
    }
}

exports.KNode.prototype._findClosestNodes = function(key, howMany, exclude) {
    var contacts = [];
    function addContact(contact) {
        if (contacts.length >= howMany)
            return;

        if (contact.nodeID == exclude)
            return;

        contacts.push(contact);
    }

    function addClosestFromBucket(bucket) {
        var distances = _.map(bucket.contacts(), function(contact) {
            return {
                distance: util.distance(contact.nodeID, key),
                contact: contact
            };
        });

        distances.sort(function(a, b) {
            return util.buffer_compare(a.distance, b.distance);
        });

        _(distances).chain()
         .first(howMany - contacts.length)
         .pluck('contact')
         .map(MC)
         .value()
         .forEach(addContact);
    }

    // first check the same bucket
    // what bucket would key go into, that is the closest
    // bucket we start from, hence bucketIndex
    // is with reference to self.nodeID
    var bucketIndex = util.bucketIndex(this.self.nodeID, key);
    if (this._buckets.hasOwnProperty(bucketIndex))
        addClosestFromBucket(this._buckets[bucketIndex]);

    var oldBucketIndex = bucketIndex;
    // then check buckets higher up
    while (contacts.length < howMany && bucketIndex < constants.B) {
        bucketIndex++;
        if (this._buckets.hasOwnProperty(bucketIndex))
            addClosestFromBucket(this._buckets[bucketIndex]);
    }

    // then check buckets lower down
    // since Kademlia does not define the search strategy, we can cheat
    // and use this strategy although it may not actually return the closest
    // FIXME: be intelligent to choose actual closest
    bucketIndex = oldBucketIndex;
    while (contacts.length < howMany && bucketIndex >= 0) {
        bucketIndex--;
        if (this._buckets.hasOwnProperty(bucketIndex))
            addClosestFromBucket(this._buckets[bucketIndex]);
    }
    return contacts;
}

exports.KNode.prototype._refreshBucket = function(bucketIndex) {
    var random = util.randomInBucketRangeBuffer(bucketIndex);
    this._iterativeFindNode(random.toString('hex'));
}

// this is a primitive operation, no network activity allowed
exports.KNode.prototype._onFindNode = function(message) {
    assert.ok(message.key);

    var contacts = this._findClosestNodes(message.key, constants.K, MC(message).nodeID);

    this._rpc.send(MC(message), this._MSG('FIND_NODE_REPLY', {
        'replyTo': MID(message),
        'contacts': contacts
    }));
}

// cb should be function(err, type, result)
// where type == 'VALUE' -> result is the value
//       type == 'NODE'  -> result is [list of contacts]
exports.KNode.prototype._iterativeFind = function(key, mode, cb) {
    assert.ok(_.include(['NODE', 'VALUE'], mode));
    var externalCallback = cb || function() {};

    var closestNode = null, previousClosestNode = null;
    var closestNodeDistance = -1;
    var shortlist = this._findClosestNodes(key, constants.ALPHA, this.self.nodeID);
    var contacted = {};
    var foundValue = false;
    var value = null;
    var contactsWhichHadValue = [];
    closestNode = shortlist[0];
    closestNodeDistance = util.distance(key, closestNode.nodeID);

    function xyz(alphaContacts) {
        // clone because we're going to be modifying inside
        async.forEach(alphaContacts, _.bind(function(contact, callback) {
            this._rpc.send(contact, this._MSG('FIND_'+mode, {
                key: key
            }), _.bind(function(err, message) {
                if (err) {
                    console.log("ERROR in iterativeFind"+_.str.titleize(mode)+" send to", contact);
                    shortlist = _.reject(shortlist, function(el) { return el.nodeID == contact.nodeID; });
                }
                else {
                    this._updateContact(contact);
                    contacted[contact.nodeID] = true;
                    var dist = util.distance(key, contact.nodeID);
                    if (util.buffer_compare(dist, closestNodeDistance) == -1) {
                        previousClosestNode = closestNode;
                        closestNode = contact;
                        closestNodeDistance = dist;
                    }

                    if (message.found && mode == 'VALUE') {
                        // TODO: iterative find value when it does find the
                        // value should store the value at the closest node which
                        // DID NOT have the value
                        foundValue = true;
                        value = message.value;
                        contactsWhichHadValue.push(contact.nodeID);
                    }
                    else {
                        shortlist = shortlist.concat(message.contacts);
                        shortlist = _.uniq(shortlist, false /* is sorted? */, function(contact) {
                            return contact.nodeID;
                        });
                    }
                }
                callback();
            }, this));
        }, this), _.bind(function(err) {
            if (foundValue) {
                console.log("Found value from ", contactsWhichHadValue);
                // closest node without value

                var thisNodeID = this.self.nodeID;
                var contactsWithoutValue = _.reject(shortlist, function(contact) {
                    return _.include(contactsWhichHadValue, contact.nodeID);
                });

                var distances = _.map(contactsWithoutValue, function(contact) {
                    return {
                        distance: util.distance(contact.nodeID, thisNodeID),
                        contact: contact
                    };
                });

                distances.sort(function(a, b) {
                    return util.buffer_compare(a.distance, b.distance);
                });

                if (distances.length >= 1) {
                    var closestWithoutValue = distances[0].contact;

                    console.log("Closest is ", closestWithoutValue);
                    var message = this._MSG('STORE', {
                        'key': key,
                        'value': value
                    });
                    this._rpc.send(closestWithoutValue, message);
                }
                externalCallback(null, 'VALUE', value);
                return;
            }

            if (closestNode == previousClosestNode || shortlist.length >= constants.K) {
                // TODO: clarify we might have to do a FIND_NODE here too
                externalCallback(null, 'NODE', shortlist);
                return;
            }

            var remain = _.reject(shortlist, function(el) { return contacted[el.nodeID]; })
            // TODO: call only on alpha random
            if (remain.length == 0)
                externalCallback(null, 'NODE', shortlist);
            else
                _.bind(xyz, this)(remain);
        }, this));
    }
    _.bind(xyz, this)(shortlist);
}

exports.KNode.prototype._iterativeFindNode = function(nodeID, cb) {
    this._iterativeFind(nodeID, 'NODE', cb);
}

// cb -> function(err, value)
// this does not map over directly to the spec
// rather iterativeFind already does the related things
// if the callback gets a list of contacts, it simply
// assumes the key does not exist in the DHT (atleast with
// available knowledge)
exports.KNode.prototype._iterativeFindValue = function(key, cb) {
    var callback = cb || function() {};
    this._iterativeFind(key, 'VALUE', _.bind(function(err, type, result) {
        if (type == 'VALUE')
            callback(null, result);
        else
            callback({
                'code': 'NOTFOUND',
                'key': key
            }, null);
    }, this));
}

exports.KNode.prototype.toString = function() {
    return "Node " + this.self.nodeID + ":" + this.self.address + ":" + this.self.port;
}

exports.KNode.prototype.debug = function() {
    console.log(this.toString());
    _(this._buckets).each(function(bucket, j) {
        //console.log("bucket", j, bucket.toString());
        console.log("bucket", j, bucket.size(), "peers");
    });
    console.log("store", this._storage);
}

/***** Public API *****/
exports.KNode.prototype.connect = function(address, port, cb) {
    var callback = cb || function() {};
    assert.ok(this.self.nodeID);
    var contact = util.make_contact(address, port);
    async.waterfall([
        _.bind(this._updateContact, this, contact),
        _.bind(this._iterativeFindNode, this, this.self.nodeID),
        _.bind(function(err, type, contacts, cb) {
            if (err) {
                callback(err);
                return;
            }
            // FIXME: Do we update buckets or does iterativeFindNode do it?
            var leastBucket = _.min(_.keys(this._buckets));
            var bucketsToRefresh = _.filter(_.keys(this._buckets), function(num) { return num >= leastBucket; });
            // TODO: do each refresh only after earlier finishes
            _.each(bucketsToRefresh, this._refreshBucket, this);
            // TODO: use async module to trigger callback when all done
            callback(null);
        }, this)
    ]);
}

exports.KNode.prototype.get = function(key, cb) {
    var callback = cb || function() {};
    this._iterativeFindValue(util.id(key), callback);
}

exports.KNode.prototype.set = function(key, value, cb) {
    var callback = cb || function() {};
    var message = this._MSG('STORE', {
        'key': util.id(key),
        'value': value
    });
    this._iterativeFindNode(util.id(key), _.bind(function(err, type, contacts) {
        async.forEach(contacts, _.bind(function(contact, asyncCb) {
            this._rpc.send(contact, message, function() {
                // TODO handle error
                asyncCb(null);
            });
        }, this), callback);
    }, this));
}
<commit_msg>Get rid of junk TODO<commit_after>/*
 * Copyright (C) 2011-2012 by Nikhil Marathe <nsm.nikhil@gmail.com>
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */
"use strict";

var assert = require('assert');
var node_constants = require('constants');
var async = require('async');
var constants = require('./constants');
var util = require('./util');
var rpc = require('./rpc');
var Bucket = require('./bucket').Bucket;
var _ = require('underscore');
_.str = require('underscore.string');

// abbreviate message utilities
var MC = util.message_contact;
var MID = util.message_rpcID;

exports.KNode = function(desc) {
    // TODO: probably want to persist nodeID
    this.self = _.defaults({ nodeID: util.nodeID(desc.address, desc.port) }, desc);
    Object.freeze(this.self);

    this._storage = {};
    // object treated as an array
    this._buckets = {};
    this._rpc = new rpc.RPC(this.self, _.bind(this._onMessage, this));

    // DEBUG
    setInterval(_.bind(this.debug, this), 5000);
}

exports.KNode.prototype._MSG = function(type, params) {
    // NOTE: always keep this.self last. This way users of _MSG
    // don't have to worry about accidentally overriding self properties
    return _.extend({ type: type}, params, this.self);
}

exports.KNode.prototype._onMessage = function(message) {
    var methodName = '_on' + _.str.titleize(_.str.camelize(message.type.toLowerCase()));
    var action = this[methodName];
    if (action) {
        this._updateContact(MC(message));
        _.bind(action, this)(message);
    }
    else
        console.warn("Unknown message", message);
}

exports.KNode.prototype._ping = function(contact) {

    function onPong(err, message) {
        if (err && err.errno == node_constants.ETIMEDOUT) {
            // TODO: handle peer death
        }
        else {
            assert.equal(message.type, 'PONG');
            this._updateContact(MC(message));
        }
    }

    this._rpc.send(contact, this._MSG('PING'), _.bind(onPong, this));
}

exports.KNode.prototype._onPing = function(message) {
    // this can be made more intelligent such that
    // if an outgoing message is present, piggyback the pong
    // onto it rather than sending it separately
    this._rpc.send(MC(message), this._MSG('PONG', {'replyTo': MID(message)}));
}

exports.KNode.prototype._updateContact = function(contact, cb) {
    var callback = cb || function() {};
    var bucketIndex = util.bucketIndex(this.self.nodeID, contact.nodeID);
    assert.ok(bucketIndex < constants.B);
    if (!this._buckets[bucketIndex])
        this._buckets[bucketIndex] = new Bucket();

    var bucket = this._buckets[bucketIndex];
    contact.lastSeen = Date.now();

    var exists = bucket.contains(contact);
    if (exists) {
        // move to the end of the bucket
        bucket.remove(contact);
        bucket.add(contact);
        callback();
    }
    else if (bucket.size() < constants.K) {
        bucket.add(contact);
        callback();
    }
    else {
        this._rpc.send(bucket.get(0), this._MSG('PING'),
            _.bind(function(err, message) {
                if( err ) {
                    // add new contact, old one is dead
                    bucket.removeIndex(0);
                    bucket.add(contact);
                }
                else {
                }
                callback();
            }), this);
    }
}

// TODO: handle large values which
// won't fit in single UDP packets
exports.KNode.prototype._onStore = function(message) {
    assert.equal(message.key.length, constants.B/4);
    this._storage[message.key] = _.clone(message.value);
    this._rpc.send(MC(message), this._MSG('STORE_REPLY', {
        'replyTo': MID(message),
        'status': true
    }));
}

// This is just to prevent Unknown message errors
exports.KNode.prototype._onStoreReply = function() {}

// debug methods FIXME: remove
exports.KNode.prototype._onGet = function(message) {
    this.get(message.key, function(err, val) {
        console.log('ONGET', val);
    });
}

exports.KNode.prototype._onFindValue = function(message) {
    assert.ok(message.key);
    assert.equal(message.key.length, constants.B/4);
    if (this._storage.hasOwnProperty(message.key)) {
        this._rpc.send(MC(message), this._MSG('FIND_VALUE_REPLY', {
            'replyTo': MID(message),
            'found': true,
            'value': this._storage[message.key]
        }));
    }
    else {
        var contacts = this._findClosestNodes(message.key, constants.K, MC(message).nodeID);

        this._rpc.send(MC(message), this._MSG('FIND_NODE_REPLY', {
            'replyTo': MID(message),
            'contacts': contacts
        }));
    }
}

exports.KNode.prototype._findClosestNodes = function(key, howMany, exclude) {
    var contacts = [];
    function addContact(contact) {
        if (contacts.length >= howMany)
            return;

        if (contact.nodeID == exclude)
            return;

        contacts.push(contact);
    }

    function addClosestFromBucket(bucket) {
        var distances = _.map(bucket.contacts(), function(contact) {
            return {
                distance: util.distance(contact.nodeID, key),
                contact: contact
            };
        });

        distances.sort(function(a, b) {
            return util.buffer_compare(a.distance, b.distance);
        });

        _(distances).chain()
         .first(howMany - contacts.length)
         .pluck('contact')
         .map(MC)
         .value()
         .forEach(addContact);
    }

    // first check the same bucket
    // what bucket would key go into, that is the closest
    // bucket we start from, hence bucketIndex
    // is with reference to self.nodeID
    var bucketIndex = util.bucketIndex(this.self.nodeID, key);
    if (this._buckets.hasOwnProperty(bucketIndex))
        addClosestFromBucket(this._buckets[bucketIndex]);

    var oldBucketIndex = bucketIndex;
    // then check buckets higher up
    while (contacts.length < howMany && bucketIndex < constants.B) {
        bucketIndex++;
        if (this._buckets.hasOwnProperty(bucketIndex))
            addClosestFromBucket(this._buckets[bucketIndex]);
    }

    // then check buckets lower down
    // since Kademlia does not define the search strategy, we can cheat
    // and use this strategy although it may not actually return the closest
    // FIXME: be intelligent to choose actual closest
    bucketIndex = oldBucketIndex;
    while (contacts.length < howMany && bucketIndex >= 0) {
        bucketIndex--;
        if (this._buckets.hasOwnProperty(bucketIndex))
            addClosestFromBucket(this._buckets[bucketIndex]);
    }
    return contacts;
}

exports.KNode.prototype._refreshBucket = function(bucketIndex) {
    var random = util.randomInBucketRangeBuffer(bucketIndex);
    this._iterativeFindNode(random.toString('hex'));
}

// this is a primitive operation, no network activity allowed
exports.KNode.prototype._onFindNode = function(message) {
    assert.ok(message.key);

    var contacts = this._findClosestNodes(message.key, constants.K, MC(message).nodeID);

    this._rpc.send(MC(message), this._MSG('FIND_NODE_REPLY', {
        'replyTo': MID(message),
        'contacts': contacts
    }));
}

// cb should be function(err, type, result)
// where type == 'VALUE' -> result is the value
//       type == 'NODE'  -> result is [list of contacts]
exports.KNode.prototype._iterativeFind = function(key, mode, cb) {
    assert.ok(_.include(['NODE', 'VALUE'], mode));
    var externalCallback = cb || function() {};

    var closestNode = null, previousClosestNode = null;
    var closestNodeDistance = -1;
    var shortlist = this._findClosestNodes(key, constants.ALPHA, this.self.nodeID);
    var contacted = {};
    var foundValue = false;
    var value = null;
    var contactsWhichHadValue = [];
    closestNode = shortlist[0];
    closestNodeDistance = util.distance(key, closestNode.nodeID);

    function xyz(alphaContacts) {
        // clone because we're going to be modifying inside
        async.forEach(alphaContacts, _.bind(function(contact, callback) {
            this._rpc.send(contact, this._MSG('FIND_'+mode, {
                key: key
            }), _.bind(function(err, message) {
                if (err) {
                    console.log("ERROR in iterativeFind"+_.str.titleize(mode)+" send to", contact);
                    shortlist = _.reject(shortlist, function(el) { return el.nodeID == contact.nodeID; });
                }
                else {
                    this._updateContact(contact);
                    contacted[contact.nodeID] = true;
                    var dist = util.distance(key, contact.nodeID);
                    if (util.buffer_compare(dist, closestNodeDistance) == -1) {
                        previousClosestNode = closestNode;
                        closestNode = contact;
                        closestNodeDistance = dist;
                    }

                    if (message.found && mode == 'VALUE') {
                        // TODO: iterative find value when it does find the
                        // value should store the value at the closest node which
                        // DID NOT have the value
                        foundValue = true;
                        value = message.value;
                        contactsWhichHadValue.push(contact.nodeID);
                    }
                    else {
                        shortlist = shortlist.concat(message.contacts);
                        shortlist = _.uniq(shortlist, false /* is sorted? */, function(contact) {
                            return contact.nodeID;
                        });
                    }
                }
                callback();
            }, this));
        }, this), _.bind(function(err) {
            if (foundValue) {
                console.log("Found value from ", contactsWhichHadValue);
                // closest node without value

                var thisNodeID = this.self.nodeID;
                var contactsWithoutValue = _.reject(shortlist, function(contact) {
                    return _.include(contactsWhichHadValue, contact.nodeID);
                });

                var distances = _.map(contactsWithoutValue, function(contact) {
                    return {
                        distance: util.distance(contact.nodeID, thisNodeID),
                        contact: contact
                    };
                });

                distances.sort(function(a, b) {
                    return util.buffer_compare(a.distance, b.distance);
                });

                if (distances.length >= 1) {
                    var closestWithoutValue = distances[0].contact;

                    console.log("Closest is ", closestWithoutValue);
                    var message = this._MSG('STORE', {
                        'key': key,
                        'value': value
                    });
                    this._rpc.send(closestWithoutValue, message);
                }
                externalCallback(null, 'VALUE', value);
                return;
            }

            if (closestNode == previousClosestNode || shortlist.length >= constants.K) {
                // TODO: clarify we might have to do a FIND_NODE here too
                externalCallback(null, 'NODE', shortlist);
                return;
            }

            var remain = _.reject(shortlist, function(el) { return contacted[el.nodeID]; })
            // TODO: call only on alpha random
            if (remain.length == 0)
                externalCallback(null, 'NODE', shortlist);
            else
                _.bind(xyz, this)(remain);
        }, this));
    }
    _.bind(xyz, this)(shortlist);
}

exports.KNode.prototype._iterativeFindNode = function(nodeID, cb) {
    this._iterativeFind(nodeID, 'NODE', cb);
}

// cb -> function(err, value)
// this does not map over directly to the spec
// rather iterativeFind already does the related things
// if the callback gets a list of contacts, it simply
// assumes the key does not exist in the DHT (atleast with
// available knowledge)
exports.KNode.prototype._iterativeFindValue = function(key, cb) {
    var callback = cb || function() {};
    this._iterativeFind(key, 'VALUE', _.bind(function(err, type, result) {
        if (type == 'VALUE')
            callback(null, result);
        else
            callback({
                'code': 'NOTFOUND',
                'key': key
            }, null);
    }, this));
}

exports.KNode.prototype.toString = function() {
    return "Node " + this.self.nodeID + ":" + this.self.address + ":" + this.self.port;
}

exports.KNode.prototype.debug = function() {
    console.log(this.toString());
    _(this._buckets).each(function(bucket, j) {
        //console.log("bucket", j, bucket.toString());
        console.log("bucket", j, bucket.size(), "peers");
    });
    console.log("store", this._storage);
}

/***** Public API *****/
exports.KNode.prototype.connect = function(address, port, cb) {
    var callback = cb || function() {};
    assert.ok(this.self.nodeID);
    var contact = util.make_contact(address, port);
    async.waterfall([
        _.bind(this._updateContact, this, contact),
        _.bind(this._iterativeFindNode, this, this.self.nodeID),
        _.bind(function(err, type, contacts, cb) {
            if (err) {
                callback(err);
                return;
            }
            // FIXME: Do we update buckets or does iterativeFindNode do it?
            var leastBucket = _.min(_.keys(this._buckets));
            var bucketsToRefresh = _.filter(_.keys(this._buckets), function(num) { return num >= leastBucket; });
            // TODO: do each refresh only after earlier finishes
            _.each(bucketsToRefresh, this._refreshBucket, this);
            // TODO: use async module to trigger callback when all done
            callback(null);
        }, this)
    ]);
}

exports.KNode.prototype.get = function(key, cb) {
    var callback = cb || function() {};
    this._iterativeFindValue(util.id(key), callback);
}

exports.KNode.prototype.set = function(key, value, cb) {
    var callback = cb || function() {};
    var message = this._MSG('STORE', {
        'key': util.id(key),
        'value': value
    });
    this._iterativeFindNode(util.id(key), _.bind(function(err, type, contacts) {
        async.forEach(contacts, _.bind(function(contact, asyncCb) {
            this._rpc.send(contact, message, function() {
                // TODO handle error
                asyncCb(null);
            });
        }, this), callback);
    }, this));
}
<|endoftext|>