<commit_before>splitKeypath =  function ( keypath ) {
	var index, startIndex, keys, remaining, part;

	// We should only have to do all the heavy regex stuff once... caching FTW
	if ( keypathCache[ keypath ] ) {
		return keypathCache[ keypath ].concat();
	}

	keys = [];
	remaining = keypath;

	startIndex = 0;

	// Split into keys
	while ( remaining.length ) {
		// Find next dot
		index = remaining.indexOf( '.', startIndex );

		// Final part?
		if ( index === -1 ) {
			part = remaining;
			remaining = '';
		}

		else {
			// If this dot is preceded by a backslash, which isn't
			// itself preceded by a backslash, we consider it escaped
			if ( remaining.charAt( index - 1) === '\\' && remaining.charAt( index - 2 ) !== '\\' ) {
				// we don't want to keep this part, we want to keep looking
				// for the separator
				startIndex = index + 1;
				continue;
			}

			// Otherwise, we have our next part
			part = remaining.substr( 0, index );
			startIndex = 0;
		}

		if ( /\[/.test( part ) ) {
			keys = keys.concat( part.replace( /\[\s*([0-9]+)\s*\]/g, '.$1' ).split( '.' ) );
		} else {
			keys[ keys.length ] = part;
		}

		remaining = remaining.substring( index + 1 );
	}

	keypathCache[ keypath ] = keys;
	return keys.concat();
};

// thanks, http://perfectionkills.com/instanceof-considered-harmful-or-how-to-write-a-robust-isarray/
isArray = function ( obj ) {
	return Object.prototype.toString.call( obj ) === '[object Array]';
};

isEqual = function ( a, b ) {
	if ( a === null && b === null ) {
		return true;
	}

	if ( typeof a === 'object' || typeof b === 'object' ) {
		return false;
	}

	return a === b;
};

// http://stackoverflow.com/questions/18082/validate-numbers-in-javascript-isnumeric
isNumeric = function ( n ) {
	return !isNaN( parseFloat( n ) ) && isFinite( n );
};

isObject = function ( obj ) {
	return ( Object.prototype.toString.call( obj ) === '[object Object]' ) && ( typeof obj !== 'function' );
};

getEl = function ( input ) {
	var output;

	if ( typeof window === 'undefined' ) {
		return;
	}

	if ( !input ) {
		throw new Error( 'No container element specified' );
	}

	// We already have a DOM node - no work to do
	if ( input.tagName ) {
		return input;
	}

	// Get node from string
	if ( typeof input === 'string' ) {
		// try ID first
		output = doc.getElementById( input );

		// then as selector, if possible
		if ( !output && doc.querySelector ) {
			output = doc.querySelector( input );
		}

		// did it work?
		if ( output.tagName ) {
			return output;
		}
	}

	// If we've been given a collection (jQuery, Zepto etc), extract the first item
	if ( input[0] && input[0].tagName ) {
		return input[0];
	}

	throw new Error( 'Could not find container element' );
};

try {
	Object.create( null );

	createFromNull = function () {
		return Object.create( null );
	};
} catch ( err ) {
	// sigh
	createFromNull = function () {
		return {}; // hope you're not modifying the Object prototype
	}
}<commit_msg>moved createFromNull to begin.js, where it is needed<commit_after>splitKeypath =  function ( keypath ) {
	var index, startIndex, keys, remaining, part;

	// We should only have to do all the heavy regex stuff once... caching FTW
	if ( keypathCache[ keypath ] ) {
		return keypathCache[ keypath ].concat();
	}

	keys = [];
	remaining = keypath;

	startIndex = 0;

	// Split into keys
	while ( remaining.length ) {
		// Find next dot
		index = remaining.indexOf( '.', startIndex );

		// Final part?
		if ( index === -1 ) {
			part = remaining;
			remaining = '';
		}

		else {
			// If this dot is preceded by a backslash, which isn't
			// itself preceded by a backslash, we consider it escaped
			if ( remaining.charAt( index - 1) === '\\' && remaining.charAt( index - 2 ) !== '\\' ) {
				// we don't want to keep this part, we want to keep looking
				// for the separator
				startIndex = index + 1;
				continue;
			}

			// Otherwise, we have our next part
			part = remaining.substr( 0, index );
			startIndex = 0;
		}

		if ( /\[/.test( part ) ) {
			keys = keys.concat( part.replace( /\[\s*([0-9]+)\s*\]/g, '.$1' ).split( '.' ) );
		} else {
			keys[ keys.length ] = part;
		}

		remaining = remaining.substring( index + 1 );
	}

	keypathCache[ keypath ] = keys;
	return keys.concat();
};

// thanks, http://perfectionkills.com/instanceof-considered-harmful-or-how-to-write-a-robust-isarray/
isArray = function ( obj ) {
	return Object.prototype.toString.call( obj ) === '[object Array]';
};

isEqual = function ( a, b ) {
	if ( a === null && b === null ) {
		return true;
	}

	if ( typeof a === 'object' || typeof b === 'object' ) {
		return false;
	}

	return a === b;
};

// http://stackoverflow.com/questions/18082/validate-numbers-in-javascript-isnumeric
isNumeric = function ( n ) {
	return !isNaN( parseFloat( n ) ) && isFinite( n );
};

isObject = function ( obj ) {
	return ( Object.prototype.toString.call( obj ) === '[object Object]' ) && ( typeof obj !== 'function' );
};

getEl = function ( input ) {
	var output;

	if ( typeof window === 'undefined' ) {
		return;
	}

	if ( !input ) {
		throw new Error( 'No container element specified' );
	}

	// We already have a DOM node - no work to do
	if ( input.tagName ) {
		return input;
	}

	// Get node from string
	if ( typeof input === 'string' ) {
		// try ID first
		output = doc.getElementById( input );

		// then as selector, if possible
		if ( !output && doc.querySelector ) {
			output = doc.querySelector( input );
		}

		// did it work?
		if ( output.tagName ) {
			return output;
		}
	}

	// If we've been given a collection (jQuery, Zepto etc), extract the first item
	if ( input[0] && input[0].tagName ) {
		return input[0];
	}

	throw new Error( 'Could not find container element' );
};<|endoftext|>


<commit_before>var config = require('../config');
var chanService = require('./4ChanService');
var telegramService = require('./telegramAPI');

var bot = {};

// MESSAGES

bot.readMessage = function(message) {
  message.text = bot.normalizeMessage(message);
  if (bot.isMessageNew(message)) {
    if (bot.isMessageCommand(message)) {
      bot.executeCommand(message);
    } else {
      var response = message.text + ' is not a command. Read /help to learn how to use it.';
      telegramService.postMessage(config.TOKEN, message.chat.id, config.START_MESSAGE,function(err, res, body) {
        if (err) {
          return console.log(err);
        } else {
          return console.log('start message posted!');
        }
      });
    }
  } else {
    console.log('message ' + message.text + ' is old, dropped.');
  }
};

bot.executeCommand = function(message) {
  if(config.BOARD_COMMANDS.indexOf(message.text) >= 0) {
    bot.executeBoardCommand(message);
  } else if(config.GENERIC_COMMANDS.indexOf(message.text) >= 0) {
    bot.executeGenericCommand(message);
  }
};

bot.executeBoardCommand = function (message) {
  chanService.getRandomImage(message.text, function(err, localPath){
    if (err) {
      return console.log(err);
    } else {
      var extension = localPath.split('.').pop();
      if (extension == 'png' || extension == 'jpg') {
        telegramService.postImage(config.TOKEN, localPath, message.chat.id, function(err, res, body) {
          if (err) {
            return console.log(err);
          } else {
            return console.log('image posted!');
          }
        });
      } else {
        telegramService.postDocument(config.TOKEN, localPath, message.chat.id, function(err, res, body) {
          if (err) {
            return console.log(err);
          } else {
            return console.log('document posted!');
          }
        });
      }
    }
  });
};

bot.executeGenericCommand = function (message) {
  if (message.text == "/start") {
    telegramService.postMessage(config.TOKEN, message.chat.id, config.START_MESSAGE,function(err, res, body) {
      if (err) {
        return console.log(err);
      } else {
        return console.log('start message posted!');
      }
    });
  } else if (message.text == "/help") {
    telegramService.postMessage(config.TOKEN, message.chat.id, config.HELP_MESSAGE, function(err, res, body) {
      if (err) {
        return console.log(err);
      } else {
        return console.log('help message posted!');
      }
    });
  }
  return console.log('command ' + message.text);
};

bot.isMessageCommand = function(message) {
  if (!message.hasOwnProperty('text')) {
    return false;
  }
  return config.COMMANDS.indexOf(message.text) >= 0;
};

bot.normalizeMessage = function(message) {
  if (!message.hasOwnProperty('text')) {
    return false;
  }
  return message.text.replace("@" + config.BOT_NAME, "");
};

bot.isMessageNew = function(message) {
  if (!message.hasOwnProperty('date')) {
    return false;
  }
  if (!Number.isInteger(message.date)) {
    return false;
  }
  var currentDate = Date.now();
  var ONE_MINUTE = 60 * 60 * 1000;
  return (currentDate/1000 - message.date) < ONE_MINUTE;
};

// INLINE QUERIES

bot.readQuery = function(inline_query) {
  console.log('reading query...');
  if (bot.isQueryValid(inline_query)) {
    bot.executeQuery(inline_query);
  } else {
    console.log('query ' + inline_query.query + ' is invalid.');
  }
};

bot.isQueryValid = function(inline_query) {
  if (!inline_query.hasOwnProperty('query')) {
    return false;
  }
  return config.VALID_QUERIES.indexOf(inline_query.query) >= 0;
};

bot.executeQuery = function (inline_query) {
  chanService.getRandomMediaURLsFromBoard(inline_query.query, config.QUERY_RESULT_COUNT, function(err, mediaURLs) {
    if (err) {
      return console.log(err);
    } else {
      telegramService.answerQueryWithMedia(config.TOKEN, inline_query.id, mediaURLs, function(err, res, body) {
        if (err) {
          return console.log(err);
        } else {
          return console.log('query ' + inline_query.id + ' answered.');
        }
      });
    }
  });
};

module.exports = bot;
<commit_msg>Fixed answer message.<commit_after>var config = require('../config');
var chanService = require('./4ChanService');
var telegramService = require('./telegramAPI');

var bot = {};

// MESSAGES

bot.readMessage = function(message) {
  message.text = bot.normalizeMessage(message);
  if (bot.isMessageNew(message)) {
    if (bot.isMessageCommand(message)) {
      bot.executeCommand(message);
    } else {
      var response = message.text + ' is not a command. Read /help to learn how to use it.';
      telegramService.postMessage(config.TOKEN, message.chat.id, response, function(err, res, body) {
        if (err) {
          return console.log(err);
        } else {
          return console.log('start message posted!');
        }
      });
    }
  } else {
    console.log('message ' + message.text + ' is old, dropped.');
  }
};

bot.executeCommand = function(message) {
  if(config.BOARD_COMMANDS.indexOf(message.text) >= 0) {
    bot.executeBoardCommand(message);
  } else if(config.GENERIC_COMMANDS.indexOf(message.text) >= 0) {
    bot.executeGenericCommand(message);
  }
};

bot.executeBoardCommand = function (message) {
  chanService.getRandomImage(message.text, function(err, localPath){
    if (err) {
      return console.log(err);
    } else {
      var extension = localPath.split('.').pop();
      if (extension == 'png' || extension == 'jpg') {
        telegramService.postImage(config.TOKEN, localPath, message.chat.id, function(err, res, body) {
          if (err) {
            return console.log(err);
          } else {
            return console.log('image posted!');
          }
        });
      } else {
        telegramService.postDocument(config.TOKEN, localPath, message.chat.id, function(err, res, body) {
          if (err) {
            return console.log(err);
          } else {
            return console.log('document posted!');
          }
        });
      }
    }
  });
};

bot.executeGenericCommand = function (message) {
  if (message.text == "/start") {
    telegramService.postMessage(config.TOKEN, message.chat.id, config.START_MESSAGE,function(err, res, body) {
      if (err) {
        return console.log(err);
      } else {
        return console.log('start message posted!');
      }
    });
  } else if (message.text == "/help") {
    telegramService.postMessage(config.TOKEN, message.chat.id, config.HELP_MESSAGE, function(err, res, body) {
      if (err) {
        return console.log(err);
      } else {
        return console.log('help message posted!');
      }
    });
  }
  return console.log('command ' + message.text);
};

bot.isMessageCommand = function(message) {
  if (!message.hasOwnProperty('text')) {
    return false;
  }
  return config.COMMANDS.indexOf(message.text) >= 0;
};

bot.normalizeMessage = function(message) {
  if (!message.hasOwnProperty('text')) {
    return false;
  }
  return message.text.replace("@" + config.BOT_NAME, "");
};

bot.isMessageNew = function(message) {
  if (!message.hasOwnProperty('date')) {
    return false;
  }
  if (!Number.isInteger(message.date)) {
    return false;
  }
  var currentDate = Date.now();
  var ONE_MINUTE = 60 * 60 * 1000;
  return (currentDate/1000 - message.date) < ONE_MINUTE;
};

// INLINE QUERIES

bot.readQuery = function(inline_query) {
  console.log('reading query...');
  if (bot.isQueryValid(inline_query)) {
    bot.executeQuery(inline_query);
  } else {
    console.log('query ' + inline_query.query + ' is invalid.');
  }
};

bot.isQueryValid = function(inline_query) {
  if (!inline_query.hasOwnProperty('query')) {
    return false;
  }
  return config.VALID_QUERIES.indexOf(inline_query.query) >= 0;
};

bot.executeQuery = function (inline_query) {
  chanService.getRandomMediaURLsFromBoard(inline_query.query, config.QUERY_RESULT_COUNT, function(err, mediaURLs) {
    if (err) {
      return console.log(err);
    } else {
      telegramService.answerQueryWithMedia(config.TOKEN, inline_query.id, mediaURLs, function(err, res, body) {
        if (err) {
          return console.log(err);
        } else {
          return console.log('query ' + inline_query.id + ' answered.');
        }
      });
    }
  });
};

module.exports = bot;
<|endoftext|>


<commit_before>import ReactDOM from 'react-dom';
import React from 'react';
import Request from 'request';
import moment from 'moment';

import Dropdown from './components/shared/dropdown';

/**
 * Admin
 */
export default class Admin extends React.Component {
  constructor(props) {
    super(props);

    this.state = {
      artists: [],
      allArtist: [],
      filters: {
        published: 'all',
        clip: 'all'
      }
    };

    this.filterArtists = this.filterArtists.bind(this);
    this.handleFilterChange = this.handleFilterChange.bind(this);
  }

  componentWillMount() {
    const artistsUrl = window.location.origin + '/artists';

    let artistsPromise = new Promise(function(resolve) {
      Request(artistsUrl, (error, response, body) => {
        if (!error && response.statusCode === 200) {
          // Success
          resolve(JSON.parse(body));
        } else {
          console.error(error);
        }
      });
    });

    artistsPromise.then((res) => {
      // Prepare loaderCell destruction
      const loaderCell = document.getElementById('loaderCell');

      loaderCell.remove();

      this.setState({
        artists: res,
        allArtist: res,
        artistsTotal: res.length,
      });

    });
  }

  filterArtists() {

    let filteredArtists  = [];

    // Published Filter
    let publishedArtistFilter;

    switch(this.state.filters.published) {
      case 'published': {
        publishedArtistFilter = this.state.allArtist.filter( function (artist) {
          return artist.published === true;
        });
        break;
      }
      case 'not-published': {
        publishedArtistFilter = this.state.allArtist.filter( function (artist) {
          return artist.published === false;
        });
        break;
      }
      default: {
        publishedArtistFilter = this.state.allArtist.filter( function (artist) {
          return artist.published === false || artist.published === true;
        });
        break;
      }
    }

    // Clip Filter
    let clipArtistFilter;

    switch(this.state.filters.clip) {
      case 'with-clip': {
        clipArtistFilter = publishedArtistFilter.filter( function (artist) {
          return artist.youtube.clipExampleUrl.length > 0;
        });
        break;
      }
      case 'without-clip': {
        clipArtistFilter = publishedArtistFilter.filter( function (artist) {
          return artist.youtube.clipExampleUrl.length === 0;
        });
        break;
      }
      default: {
        clipArtistFilter = publishedArtistFilter.filter( function (artist) {
          return artist.youtube.clipExampleUrl.length > 0 || artist.youtube.clipExampleUrl.length === 0;
        });
        break;
      }
    }

    // Apply filters
    filteredArtists = clipArtistFilter;

    const mergedFilteredArtist = [].concat.apply([], filteredArtists);

    this.setState({
      artists: mergedFilteredArtist
    });

  }

  handleFilterChange(filter, value) {
    if (value !== this.state.filters[filter]) {

      const filters = Object.assign(this.state.filters, {
        [filter]: value
      });

      this.setState({
        filters: filters
      }, () => {
        this.filterArtists();
      });
    }
  }

  render() {

    // Artists row
    const artistsResult = this.state.artists.map((artist, step) => {
      // Artist Bio Url
      let artistBioUrl;
      if (artist.bio.url) {
        artistBioUrl = (
          {artist.name}
        );
      } else {
        artistBioUrl = (

{artist.name}

        );
      }

      // Artist Categories
      const artistCategories = artist.categories.map((category) =>
        {category}
      );

      return (

            { artistBioUrl }

            {artist.location.city}

              {artistCategories}

            {artist.youtube.clipExampleUrl.length > 0 &&
              Link
            }

            {moment(artist.createDate).format('MM/DD/YYYY HH:mm')}

            {String(artist.published)}

            Edit
            Delete

      );
    });

    return (

        Create a new artist

Filters:

              <Dropdown label={'Published ' + '(' + this.state.filters.published + ')'}>

                     this.handleFilterChange('published', 'all')}>All

                     this.handleFilterChange('published', 'published')}>Published

                     this.handleFilterChange('published', 'not-published')}>Not Published

              </Dropdown>

              <Dropdown label={'Clip ' + '(' + this.state.filters.clip + ')'}>

                     this.handleFilterChange('clip', 'all')}>All

                     this.handleFilterChange('clip', 'with-clip')}>With clip

                     this.handleFilterChange('clip', 'without-clip')}>Without clip

              </Dropdown>

                    Name
                    City
                    Categorie(s)
                    Clip
                    Creation Date
                    Published

              {this.state.artists.length > 0 &&
                artistsResult
              }

    );
  }
}

ReactDOM.render(<Admin />,
  document.getElementById('app')
);<commit_msg>Add filter 'category' into admin<commit_after>import ReactDOM from 'react-dom';
import React from 'react';
import Request from 'request';
import moment from 'moment';

import Dropdown from './components/shared/dropdown';

/**
 * Admin
 */
export default class Admin extends React.Component {
  constructor(props) {
    super(props);

    this.state = {
      artists: [],
      allArtist: [],
      filters: {
        published: 'all',
        clip: 'all',
        category: 'all'
      }
    };

    this.filterArtists = this.filterArtists.bind(this);
    this.handleFilterChange = this.handleFilterChange.bind(this);
  }

  componentWillMount() {
    const artistsUrl = window.location.origin + '/artists';

    let artistsPromise = new Promise(function(resolve) {
      Request(artistsUrl, (error, response, body) => {
        if (!error && response.statusCode === 200) {
          // Success
          resolve(JSON.parse(body));
        } else {
          console.error(error);
        }
      });
    });

    artistsPromise.then((res) => {
      // Prepare loaderCell destruction
      const loaderCell = document.getElementById('loaderCell');

      loaderCell.remove();

      this.setState({
        artists: res,
        allArtist: res,
        artistsTotal: res.length,
      });

    });
  }

  filterArtists() {

    let filteredArtists  = [];

    // Published Filter
    let publishedArtistFilter;

    switch(this.state.filters.published) {
      case 'published': {
        publishedArtistFilter = this.state.allArtist.filter( function (artist) {
          return artist.published === true;
        });
        break;
      }
      case 'not-published': {
        publishedArtistFilter = this.state.allArtist.filter( function (artist) {
          return artist.published === false;
        });
        break;
      }
      default: {
        publishedArtistFilter = this.state.allArtist.filter( function (artist) {
          return artist.published === false || artist.published === true;
        });
        break;
      }
    }

    // Clip Filter
    let clipArtistFilter;

    switch(this.state.filters.clip) {
      case 'with-clip': {
        clipArtistFilter = publishedArtistFilter.filter( function (artist) {
          return artist.youtube.clipExampleUrl.length > 0;
        });
        break;
      }
      case 'without-clip': {
        clipArtistFilter = publishedArtistFilter.filter( function (artist) {
          return artist.youtube.clipExampleUrl.length === 0;
        });
        break;
      }
      default: {
        clipArtistFilter = publishedArtistFilter.filter( function (artist) {
          return artist.youtube.clipExampleUrl.length > 0 || artist.youtube.clipExampleUrl.length === 0;
        });
        break;
      }
    }

    // Category Filter
    let categoryArtistFilter;

    switch(this.state.filters.category) {
      case 'rapper': {
        categoryArtistFilter = clipArtistFilter.filter( function (artist) {
          return artist.categories.indexOf('rapper') > -1;
        });
        break;
      }
      case 'singer': {
        categoryArtistFilter = clipArtistFilter.filter( function (artist) {
          return artist.categories.indexOf('singer') > -1;
        });
        break;
      }
      case 'dj-producer': {
        categoryArtistFilter = clipArtistFilter.filter( function (artist) {
          return artist.categories.indexOf('dj / producer') > -1;
        });
        break;
      }
      default: {
        categoryArtistFilter = clipArtistFilter;
        break;
      }
    }

    // Apply filters
    filteredArtists = categoryArtistFilter;

    const mergedFilteredArtist = [].concat.apply([], filteredArtists);

    this.setState({
      artists: mergedFilteredArtist
    });

  }

  handleFilterChange(filter, value) {
    if (value !== this.state.filters[filter]) {

      const filters = Object.assign(this.state.filters, {
        [filter]: value
      });

      this.setState({
        filters: filters
      }, () => {
        this.filterArtists();
      });
    }
  }

  render() {

    // Artists row
    const artistsResult = this.state.artists.map((artist, step) => {
      // Artist Bio Url
      let artistBioUrl;
      if (artist.bio.url) {
        artistBioUrl = (
          {artist.name}
        );
      } else {
        artistBioUrl = (

{artist.name}

        );
      }

      // Artist Categories
      const artistCategories = artist.categories.map((category) =>
        {category}
      );

      return (

            { artistBioUrl }

            {artist.location.city}

              {artistCategories}

            {artist.youtube.clipExampleUrl.length > 0 &&
              Link
            }

            {moment(artist.createDate).format('MM/DD/YYYY HH:mm')}

            {String(artist.published)}

            Edit
            Delete

      );
    });

    return (

        Create a new artist

Filters:

              <Dropdown label={'Published ' + '(' + this.state.filters.published + ')'}>

                     this.handleFilterChange('published', 'all')}>All

                     this.handleFilterChange('published', 'published')}>Published

                     this.handleFilterChange('published', 'not-published')}>Not Published

              </Dropdown>

              <Dropdown label={'Clip ' + '(' + this.state.filters.clip + ')'}>

                     this.handleFilterChange('clip', 'all')}>All

                     this.handleFilterChange('clip', 'with-clip')}>With clip

                     this.handleFilterChange('clip', 'without-clip')}>Without clip

              </Dropdown>

              <Dropdown label={'Category ' + '(' + this.state.filters.category + ')'}>

                     this.handleFilterChange('category', 'all')}>All

                     this.handleFilterChange('category', 'rapper')}>Rapper

                     this.handleFilterChange('category', 'singer')}>Singer

                     this.handleFilterChange('category', 'dj-producer')}>Dj / Producder

              </Dropdown>

                    Name
                    City
                    Categorie(s)
                    Clip
                    Creation Date
                    Published

              {this.state.artists.length > 0 &&
                artistsResult
              }

    );
  }
}

ReactDOM.render(<Admin />,
  document.getElementById('app')
);<|endoftext|>


<commit_before>/**
 * This is a class that defines the methods for an object that
 * creates the inverted index
 */
class InvertedIndex {
  /**
   * Constructor for the Inverted Index class
   */
  constructor() {
    // Object to store all indexes created
    this.indexes = {};
  }

  /**
   * Removes non alphanumeric chracters from a string and tokenizes it
   *
   * @param {String} text - Words to get tokens from
   * @return {Array}  An array of sorted strings without non-alphanumeric symbols
   */
  static clean(text) {
    return text.replace(/[^a-z0-9\s]+/gi, '')
      .replace(/\s{2,}/g, ' ')
      .toLowerCase()
      .split(' ')
      .sort();
  }

  /**
   * Removes duplicate strings from an array of strings
   *
   * @param {Array} words - An array of strings
   * @return {Array} uniqueWords - An array non-duplicate strings
   */
  static removeDuplicates(words) {
    return words.filter((item, index) => words.indexOf(item) === index);
  }

  /**
   * Sorts the keys of an Object
   *
   * @param {Object} index - Efizi function to sort an Object
   * @return {Object} sortedObject - Object with keys sorted
   */
  static sortObjectKeys(index) {
    const sortedKeys = Object.keys(index).sort();
    // Object that will contain the sorted object
    const sortedObject = {};
    sortedKeys.forEach((key) => {
      sortedObject[key] = index[key];
    });
    return (sortedObject);
  }

  /**
   * Checks if a file is empty or invalid
   *
   * @param {Object} file - JSON object from which index is to be created
   * @returns{string}  A string describing the file's status
   */
  static validateFile(file) {
    if (file.length === 0) {
      return 'Empty file';
    }
    // get keys of all objects in JSON file
    let keys = [];
    file.forEach((doc) => {
      keys.push(Object.keys(doc));
    });
    // 'flatten' it to an array`
    keys = keys.toString().split(',');
    if (keys.includes('title') || keys.includes('text')) {
      return 'Valid file';
    }
    return 'Invalid file';
  }

  /**
   * Creates the Inverted Index data structure
   *
   * @param{String} filename - Name of the file for which index is to be created
   * @param{Object} fileToIndex - JSON file
   * @returns{undefined} - This function does not return anything it just
   * updates the this.indexes Object
   */
  createIndex(filename, fileToIndex) {
    // This object stores the index of the current document
    let index = {};
    // Ensure JSON file is not empty or invalid
    if (InvertedIndex.validateFile(fileToIndex) === 'Empty file') {
      this.indexes[filename] = ['JSON file is empty'];
    } else if (InvertedIndex.validateFile(fileToIndex) === 'Invalid file') {
      this.indexes[filename] = ['JSON file is invalid'];
    } else {
      fileToIndex.forEach((file, docIndex) => {
        const cleanWords = InvertedIndex.clean(`${file.title} ${file.text}`);
        const uniqueWords = InvertedIndex.removeDuplicates(cleanWords);
        uniqueWords.forEach((word) => {
          /**
           * If word does not exist as a key in the index object, first create
           * an empty array value for it
           */
          if (!Object.keys(index).includes(word)) {
            index[word] = [];
          }
          index[word].push(docIndex + 1);
        });
      });
      // Sort the index keys
      index = InvertedIndex.sortObjectKeys(index);
      /**
       * Simply update the this.indexes object
       * Don't return it yet
       */
      this.indexes[filename] = index;
    }
  }

  /**
   * Returns all indexes or an index for a specified file
   *
   * @param{String} filename - Optional parameter specifying a file whose index is required
   * @returns{Object} - Object that contains all indexes or a required index
   */
  getIndex(filename) {
    return (filename === undefined) ? this.indexes : this.indexes[filename];
  }

  /**
   * Checks that index for file exists before search
   *
   * @param{String} file - Name of file which we want to search
   * @returns{Boolean} - A boolean indicating whether or not index for that file exists
   */
  indexExists(file) {
    return (Object.keys(this.getIndex()).includes(file));
  }

  /**
   * Searches all the indexes created or a particular index
   *
   * @param{Array} args - This is an array that can contain an infinite
   * number of arguments. The first element of this array maybe a filename,
   * if it is not all files are searched
   * @returns{Object} - The results of the search
   */
  searchIndex(...args) {
    // This regex allows us to split on both whitespace and commas
    const allArgs = args.toString().toLowerCase().split(/[\s,]+/);
    const searchResults = {};
    let filesToSearch;

    /**
     * Query is an array that contains the terms we want to search for
     * It is simply a copy of the allArgs array
     */
    const query = allArgs.slice();

    if (allArgs[0].slice(-5) === '.json') {
      /**
       * Assign filesToSearch to the first element of allArgs if it is a
       * filename
       */
      filesToSearch = Array(allArgs[0]);
      // Remove the filename because we don't want to search for it
      query.shift();
      // check that index for the file exists before search
      if (!this.indexExists(allArgs[0])) {
        return ('Index for file does not exist');
      }
    } else {
      filesToSearch = Object.keys(this.getIndex());
    }

    filesToSearch.forEach((file) => {
      searchResults[file] = {};
      query.forEach((term) => {
        if (term in this.getIndex(file)) {
          searchResults[file][term] = this.getIndex(file)[term];
        } else {
          searchResults[file][term] = 'Word not found!';
        }
      });
    });
    return (allArgs[0].slice(-5) === '.json') ? searchResults[allArgs[0]] : searchResults;
  }
}
<commit_msg>Use sensible variable name<commit_after>/**
 * This is a class that defines the methods for an object that
 * creates the inverted index
 */
class InvertedIndex {
  /**
   * Constructor for the Inverted Index class
   */
  constructor() {
    // Object to store all indexes created
    this.indexes = {};
  }

  /**
   * Removes non alphanumeric chracters from a string and tokenizes it
   *
   * @param {String} text - Words to get tokens from
   * @return {Array}  An array of sorted strings without non-alphanumeric symbols
   */
  static clean(text) {
    return text.replace(/[^a-z0-9\s]+/gi, '')
      .replace(/\s{2,}/g, ' ')
      .toLowerCase()
      .split(' ')
      .sort();
  }

  /**
   * Removes duplicate strings from an array of strings
   *
   * @param {Array} words - An array of strings
   * @return {Array} uniqueWords - An array non-duplicate strings
   */
  static removeDuplicates(words) {
    return words.filter((item, index) => words.indexOf(item) === index);
  }

  /**
   * Sorts the keys of an Object
   *
   * @param {Object} index - Efizi function to sort an Object
   * @return {Object} sortedObject - Object with keys sorted
   */
  static sortObjectKeys(index) {
    const sortedKeys = Object.keys(index).sort();
    // Object that will contain the sorted object
    const sortedObject = {};
    sortedKeys.forEach((key) => {
      sortedObject[key] = index[key];
    });
    return (sortedObject);
  }

  /**
   * Checks if a file is empty or invalid
   *
   * @param {Object} file - JSON object from which index is to be created
   * @returns{string}  A string describing the file's status
   */
  static validateFile(file) {
    if (file.length === 0) {
      return 'Empty file';
    }
    // get keys of all objects in JSON file
    let keys = [];
    file.forEach((doc) => {
      keys.push(Object.keys(doc));
    });
    // 'flatten' it to an array`
    keys = keys.toString().split(',');
    if (keys.includes('title') || keys.includes('text')) {
      return 'Valid file';
    }
    return 'Invalid file';
  }

  /**
   * Creates the Inverted Index data structure
   *
   * @param{String} filename - Name of the file for which index is to be created
   * @param{Object} fileToIndex - JSON file
   * @returns{undefined} - This function does not return anything it just
   * updates the this.indexes Object
   */
  createIndex(filename, fileToIndex) {
    // This object stores the index of the current document
    let index = {};
    // Ensure JSON file is not empty or invalid
    if (InvertedIndex.validateFile(fileToIndex) === 'Empty file') {
      this.indexes[filename] = ['JSON file is empty'];
    } else if (InvertedIndex.validateFile(fileToIndex) === 'Invalid file') {
      this.indexes[filename] = ['JSON file is invalid'];
    } else {
      fileToIndex.forEach((doc, docIndex) => {
        const cleanWords = InvertedIndex.clean(`${doc.title} ${doc.text}`);
        const uniqueWords = InvertedIndex.removeDuplicates(cleanWords);
        uniqueWords.forEach((word) => {
          /**
           * If word does not exist as a key in the index object, first create
           * an empty array value for it
           */
          if (!Object.keys(index).includes(word)) {
            index[word] = [];
          }
          index[word].push(docIndex + 1);
        });
      });
      // Sort the index keys
      index = InvertedIndex.sortObjectKeys(index);
      /**
       * Simply update the this.indexes object
       * Don't return it yet
       */
      this.indexes[filename] = index;
    }
  }

  /**
   * Returns all indexes or an index for a specified file
   *
   * @param{String} filename - Optional parameter specifying a file whose index is required
   * @returns{Object} - Object that contains all indexes or a required index
   */
  getIndex(filename) {
    return (filename === undefined) ? this.indexes : this.indexes[filename];
  }

  /**
   * Checks that index for file exists before search
   *
   * @param{String} file - Name of file which we want to search
   * @returns{Boolean} - A boolean indicating whether or not index for that file exists
   */
  indexExists(file) {
    return (Object.keys(this.getIndex()).includes(file));
  }

  /**
   * Searches all the indexes created or a particular index
   *
   * @param{Array} args - This is an array that can contain an infinite
   * number of arguments. The first element of this array maybe a filename,
   * if it is not all files are searched
   * @returns{Object} - The results of the search
   */
  searchIndex(...args) {
    // This regex allows us to split on both whitespace and commas
    const allArgs = args.toString().toLowerCase().split(/[\s,]+/);
    const searchResults = {};
    let filesToSearch;

    /**
     * Query is an array that contains the terms we want to search for
     * It is simply a copy of the allArgs array
     */
    const query = allArgs.slice();

    if (allArgs[0].slice(-5) === '.json') {
      /**
       * Assign filesToSearch to the first element of allArgs if it is a
       * filename
       */
      filesToSearch = Array(allArgs[0]);
      // Remove the filename because we don't want to search for it
      query.shift();
      // check that index for the file exists before search
      if (!this.indexExists(allArgs[0])) {
        return ('Index for file does not exist');
      }
    } else {
      filesToSearch = Object.keys(this.getIndex());
    }

    filesToSearch.forEach((file) => {
      searchResults[file] = {};
      query.forEach((term) => {
        if (term in this.getIndex(file)) {
          searchResults[file][term] = this.getIndex(file)[term];
        } else {
          searchResults[file][term] = 'Word not found!';
        }
      });
    });
    return (allArgs[0].slice(-5) === '.json') ? searchResults[allArgs[0]] : searchResults;
  }
}
<|endoftext|>


<commit_before>var EventEmitter = require('events'),
events = new EventEmitter();
var average = [];
var lastAverage;
var averageNumber = 400;
var averageCount =0;
var signalMax = 0;
var signalMin = 4096;
var peakToPeak;
var localSpi;
var sampleinterval;
var averageinterval;
var audioArray = [];
var eventIntervalTime = 60000;
var eventInterval;
var audioDataIntervalTime = 20000;
var audioDataInterval;
var maxPeak = 0;

var txbuf = new Buffer([ 0x23, 0x48, 0xAF, 0x19, 0x19, 0x19 ]);
var rxbuf = new Buffer([ 0x00, 0x00, 0x00]);
exports.events = events;
exports.setup = function(SPI,callback){
  localSpi = new SPI.Spi('/dev/spidev0.1', {
      'mode': SPI.MODE['MODE_0'],  // always set mode as the first option
      'chipSelect': SPI.CS['none'] // 'none', 'high' - defaults to low
    }, function(s){
      s.maxSpeed(6000000);
      s.open();
    });
  checkValues(callback);
  //startReadings();
}
function checkValues(callback){
  localSpi.read(rxbuf, function(device, buf) {
    var sample = buf.readUInt16BE(0);
    if(sample >0 || sample < 4096){
      startReadings();
      callback(true);
    }else{
      callback(false);
    }
  });
}

function startReadings(){
  sampleinterval = setInterval(function(){
    localSpi.read(rxbuf, function(device, buf) {
      //console.log(buf.readUInt16BE(0));
      var sample = buf.readUInt16BE(0);
      if (sample > signalMax)
           {
              signalMax = sample;  // save just the max levels
           }
           else if (sample < signalMin)
           {
              signalMin = sample;  // save just the min levels
           }
    });
  },1);
  averageinterval = setInterval(function(){
   peakToPeak = 0;
   peakToPeak = signalMax - signalMin;  // max - min = peak-peak amplitude
   if(peakToPeak > maxPeak){
     maxPeak = peakToPeak;
   }
   average.push(peakToPeak);
   if(average.length == averageNumber){
     var first = average.shift();
   }
   signalMax = 0;
   signalMin = 4096;
  },50);
  eventInterval = setInterval(function(){
    events.emit('data',audioArray.slice(0));
    audioArray = [];
  },eventIntervalTime);
  audioDataInterval = setInterval(function{
    var value = medianFun(average.slice(0));
    var outString = "median value is " + value;
    for(var i = 0; i<maxPeak/10;i++){
     outString += " ";
    }
    outString += "*";
    maxPeak = 0;
    average = [];
    console.log(outString);
    var micData = {"average":vale,"max":maxPeak};
    audioArray.push(micData);
  })
}

// setInterval(function(){
//   spi.read(rxbuf, function(device, buf) {
//     //console.log(buf.readUInt16BE(0));
//     var sample = buf.readUInt16BE(0);
//     if (sample > signalMax)
//          {
//             signalMax = sample;  // save just the max levels
//          }
//          else if (sample < signalMin)
//          {
//             signalMin = sample;  // save just the min levels
//          }
//   });
// },1);

// setInterval(function(){
//  peakToPeak = 0;
//  peakToPeak = signalMax - signalMin;  // max - min = peak-peak amplitude
//  average.push(peakToPeak);
//  if(average.length == 20){
//    var first = average.shift();
//  }
//  var value = medianFun(average.slice(0));
//  var outString = "median value is " + value;
//  //var outString = "";
//  for(var i = 0; i<peakToPeak/10;i++){
//    outString += " ";
//  }
//  outString += "*";
//  //var averageToMedian = average.slice(0);
//  //averageToMedian = average
//  console.log(outString);
//  signalMax = 0;
//  signalMin = 4096;
// },50);

function medianFun(values) {

    values.sort( function(a,b) {return a - b;} );

    var half = Math.floor(values.length/2);

    if(values.length % 2)
        return values[half];
    else
        return (values[half-1] + values[half]) / 2.0;
}
<commit_msg>many updates<commit_after>var EventEmitter = require('events'),
events = new EventEmitter();
var average = [];
var lastAverage;
var averageNumber = 400;
var averageCount =0;
var signalMax = 0;
var signalMin = 4096;
var peakToPeak;
var localSpi;
var sampleinterval;
var averageinterval;
var audioArray = [];
var eventIntervalTime = 60000;
var eventInterval;
var audioDataIntervalTime = 20000;
var audioDataInterval;
var maxPeak = 0;

var txbuf = new Buffer([ 0x23, 0x48, 0xAF, 0x19, 0x19, 0x19 ]);
var rxbuf = new Buffer([ 0x00, 0x00, 0x00]);
exports.events = events;
exports.setup = function(SPI,callback){
  localSpi = new SPI.Spi('/dev/spidev0.1', {
      'mode': SPI.MODE['MODE_0'],  // always set mode as the first option
      'chipSelect': SPI.CS['none'] // 'none', 'high' - defaults to low
    }, function(s){
      s.maxSpeed(6000000);
      s.open();
    });
  checkValues(callback);
  //startReadings();
}
function checkValues(callback){
  localSpi.read(rxbuf, function(device, buf) {
    var sample = buf.readUInt16BE(0);
    if(sample >0 || sample < 4096){
      startReadings();
      callback(true);
    }else{
      callback(false);
    }
  });
}

function startReadings(){
  sampleinterval = setInterval(function(){
    localSpi.read(rxbuf, function(device, buf) {
      //console.log(buf.readUInt16BE(0));
      var sample = buf.readUInt16BE(0);
      if (sample > signalMax)
           {
              signalMax = sample;  // save just the max levels
           }
           else if (sample < signalMin)
           {
              signalMin = sample;  // save just the min levels
           }
    });
  },1);
  averageinterval = setInterval(function(){
   peakToPeak = 0;
   peakToPeak = signalMax - signalMin;  // max - min = peak-peak amplitude
   if(peakToPeak > maxPeak){
     maxPeak = peakToPeak;
   }
   average.push(peakToPeak);
   if(average.length == averageNumber){
     var first = average.shift();
   }
   signalMax = 0;
   signalMin = 4096;
  },50);
  eventInterval = setInterval(function(){
    events.emit('data',audioArray.slice(0));
    audioArray = [];
  },eventIntervalTime);
  audioDataInterval = setInterval(function(){
    var value = medianFun(average.slice(0));
    var outString = "median value is " + value;
    for(var i = 0; i<maxPeak/10;i++){
     outString += " ";
    }
    outString += "*";
    maxPeak = 0;
    average = [];
    console.log(outString);
    var micData = {"average":vale,"max":maxPeak};
    audioArray.push(micData);
  })
}

// setInterval(function(){
//   spi.read(rxbuf, function(device, buf) {
//     //console.log(buf.readUInt16BE(0));
//     var sample = buf.readUInt16BE(0);
//     if (sample > signalMax)
//          {
//             signalMax = sample;  // save just the max levels
//          }
//          else if (sample < signalMin)
//          {
//             signalMin = sample;  // save just the min levels
//          }
//   });
// },1);

// setInterval(function(){
//  peakToPeak = 0;
//  peakToPeak = signalMax - signalMin;  // max - min = peak-peak amplitude
//  average.push(peakToPeak);
//  if(average.length == 20){
//    var first = average.shift();
//  }
//  var value = medianFun(average.slice(0));
//  var outString = "median value is " + value;
//  //var outString = "";
//  for(var i = 0; i<peakToPeak/10;i++){
//    outString += " ";
//  }
//  outString += "*";
//  //var averageToMedian = average.slice(0);
//  //averageToMedian = average
//  console.log(outString);
//  signalMax = 0;
//  signalMin = 4096;
// },50);

function medianFun(values) {

    values.sort( function(a,b) {return a - b;} );

    var half = Math.floor(values.length/2);

    if(values.length % 2)
        return values[half];
    else
        return (values[half-1] + values[half]) / 2.0;
}
<|endoftext|>


<commit_before>/*
 * A module to deal with storing and retrieving guild variables. This module handles
 * getting and setting all the variables, as well as creating/removing/listing/truncating
 * the main storage table. I'll probably change this code a lot as I start storing more
 * and more data, but the interface will remain the same. The idea is to blackbox this
 * part of the code and not worry about it in my other modules.
 */

// TODO: tbh this is complete garbage, just rewrite this completely, maybe use MongoDB or something
const sql = require('sqlite');
const path = require('path');

const CVTableName = 'guildsCV';
const CVDBFilePath = path.join(__dirname, '..', 'data', 'CV.sqlite');

const allowedVariables = [
    'prefix',
    'aliases'
];

sql.open(CVDBFilePath.toString());

const scvFunctions = {
    create: () => {
        return sql.run(`CREATE TABLE ${CVTableName} (id bigint, prefix varchar(255), aliases varchar(1000))`);
    },
    drop: () => {
        return sql.run(`DROP TABLE ${CVTableName}`);
    },
    add: (variable, datatype) => {
        return sql.run(`ALTER TABLE ${CVTableName} ADD ${variable} ${datatype}`);
    },
    listTable: () => {
        return sql.all(`SELECT * FROM ${CVTableName}`).then((rows) => {
            rows.forEach((row) => {
                console.log(row);
            });
        });
    },
    truncate: () => {
        return sql.run(`DROP TABLE ${CVTableName}`).then(() => {
            module.exports.create();
        });
    },
    set: (channelID, variable, value) => {
        return new Promise((resolve, reject) => {
            if (allowedVariables.includes(variable)) {
                sql.get(`SELECT * FROM ${CVTableName} WHERE id = "${channelID}"`).then(row => {
                    if (!row) {
                        // if the guild has no entry in the db, create one
                        sql.run(`INSERT INTO ${CVTableName} (id) VALUES (?)`, channelID).then(() => {
                            return sql.run(`UPDATE ${CVTableName} SET ${variable} = ? WHERE id = "${channelID}"`, value);
                        }).then(() => {
                            resolve(value);
                        });
                    } else {
                        sql.run(`UPDATE ${CVTableName} SET ${variable} = ? WHERE id = "${channelID}"`, value).then(() => {
                            resolve(value);
                        }).catch(err => {
                            console.log(err);
                            reject(err);
                        });
                    }
                }).catch(err => {
                    console.log(err);
                    reject(err);
                });
            } else {
                // do nothing
                reject("No such variable");
            }
        });
    },
    get: (channelID, variable) => {
        return new Promise((resolve, reject) => {
            if (allowedVariables.includes(variable)) {
                sql.get(`SELECT * FROM ${CVTableName} WHERE id = "${channelID}"`).then(row => {
                    if (!row) {
                        // if the guild has no entry in the db, create one
                        sql.run(`INSERT INTO ${CVTableName} (id) VALUES (?)`, channelID);
                        resolve(false);
                    } else {
                        resolve(row[variable] === null ? false : row[variable]);
                    }
                }).catch(err => {
                    console.log(err);
                    reject(err);
                });
            } else {
                resolve(false);
            }
        });
    },
    getMultiple: (channelID, variables) => {
        return new Promise((resolve, reject) => {
            let getPromises = variables.map((elem) => {
                return scvFunctions.get(channelID, elem);
            });

            Promise.all(getPromises).then((values) => {
                let varValues = {};

                values.forEach((varValue) => {
                    let varName = variables[i];
                    varValues[varName] = varValue;
                });

                resolve(varValues);
            }).catch((err) => {
                reject(err);
            });
        });
    }
};

module.exports = scvFunctions;<commit_msg>Make persistence methods nicer<commit_after>/*
 * A module to deal with storing and retrieving guild variables. This module handles
 * getting and setting all the variables, as well as creating/removing/listing/truncating
 * the main storage table. I'll probably change this code a lot as I start storing more
 * and more data, but the interface will remain the same. The idea is to blackbox this
 * part of the code and not worry about it in my other modules.
 */

// TODO: tbh this is complete garbage, just rewrite this completely, maybe use MongoDB or something
const sql = require('sqlite');
const path = require('path');

const CVTableName = 'guildsCV';
const CVDBFilePath = path.join(__dirname, '..', 'data', 'CV.sqlite');

const allowedVariables = [
    'prefix',
    'aliases'
];

sql.open(CVDBFilePath.toString());

const scvFunctions = {
    create: () => {
        return sql.run(`CREATE TABLE ${CVTableName} (id bigint, prefix varchar(255), aliases varchar(1000))`);
    },
    drop: () => {
        return sql.run(`DROP TABLE ${CVTableName}`);
    },
    add: (variable, datatype) => {
        return sql.run(`ALTER TABLE ${CVTableName} ADD ${variable} ${datatype}`);
    },
    listTable: () => {
        return sql.all(`SELECT * FROM ${CVTableName}`).then((rows) => {
            rows.forEach((row) => {
                console.log(row);
            });
        });
    },
    truncate: () => {
        return sql.run(`DROP TABLE ${CVTableName}`).then(() => {
            module.exports.create();
        });
    },
    set: async (channelID, variable, value) => {
        if (allowedVariables.includes(variable)) {
            let row = await sql.get(`SELECT * FROM ${CVTableName} WHERE id = "${channelID}"`);
            if (!row) {
                // if the guild has no entry in the db, create one
                await sql.run(`INSERT INTO ${CVTableName} (id) VALUES (?)`, channelID);
                await sql.run(`UPDATE ${CVTableName} SET ${variable} = ? WHERE id = "${channelID}"`, value);
            } else {
                await sql.run(`UPDATE ${CVTableName} SET ${variable} = ? WHERE id = "${channelID}"`, value);
            }
            return value;
        } else {
            // do nothing
            throw new Error("No such variable");
        }
    },
    get: async (channelID, variable) => {
        if (allowedVariables.includes(variable)) {
            let row = await sql.get(`SELECT * FROM ${CVTableName} WHERE id = "${channelID}"`);
            if (!row) {
                // if the guild has no entry in the db, create one
                await sql.run(`INSERT INTO ${CVTableName} (id) VALUES (?)`, channelID);
                return false;
            } else {
                return row[variable] === null ? false : row[variable];
            }
        } else {
            return false;
        }
    },
    getMultiple: (channelID, variables) => {
        let getPromises = variables.map((elem) => {
            return scvFunctions.get(channelID, elem);
        });

        return Promise.all(getPromises).then((values) => {
            let varValues = {};

            values.forEach((varValue) => {
                let varName = variables[i];
                varValues[varName] = varValue;
            });

            return varValues;
        });
    }
};

module.exports = scvFunctions;<|endoftext|>


<commit_before>const urijs = require("urijs")
const path = require("path")

module.exports = ({
  files,
  filesToTargets,
  fileExists,
  fileHasTarget,
  link: {
    target,
    type,
    filename,
  },
  options,
}) => {
  // Allow anchors before checking for a missing href
  if (options.allowAnchors && type === "anchor") {
    return false
  }

  // Missing href is always broken
  if ((target == null)) {
    return true
  }

  const uri = urijs(target)

  // Allow anything matching the options.allowRegex regex
  if ((options.allowRegex != null) && options.allowRegex.exec(target)) {
    return false
  }

  // Empty link is always broken
  if (target === "") {
    return true
  }

  // Allow link to '#'
  if (target === "#") {
    return false
  }

  // Automatically accept all external links (could change later)
  if (uri.hostname()) {
    return false
  }

  // Ignore mailto and other non-http/https links
  if (uri.protocol() && !["http", "https"].includes(uri.protocol)) {
    return false
  }

  const fragment = uri.fragment()

  // Allow links to elements on the same page
  if (fragment && !uri.path()) {
    return options.checkAnchors && !fileHasTarget(filesToTargets, filename, fragment)
  }

  // Add baseURL in here so that the linkPath resolves to it in the case of
  // a relative link
  if (options.baseURL) {
    filename = path.join(options.baseURL, filename)
  }

  // Need to transform uri.path() into something Metalsmith can recognise
  const unixFilename = filename.replace(/\\/g, "/")
  let linkPath = uri.absoluteTo(unixFilename).path()

  // If baseURL then all internal links should be prefixed by it.
  if (options.baseURL) {

    // If the linkPath does not start with the baseURL then it is broken
    if (linkPath.indexOf(options.baseURL) !== 0) {
      return true
    }

    // Strip the baseURL out for checking whether the file exists in metalsmith
    linkPath = linkPath.replace(options.baseURL, "")

    // Fix bug where you were linking directly to the linkPath
    if (linkPath === "") {
      linkPath = "/"
    }
  }

  // Special case for link to root
  if (linkPath === "/") {
    return !fileExists(files, "index.html") || (options.checkAnchors && !fileHasTarget(filesToTargets, "index.html", fragment))
  }

  // Allow links to directories with a trailing slash
  if (linkPath.slice(-1) === "/") {
    linkPath += "index.html"
  }

  // Allow links to directories without a trailing slash with allowRedirects option
  if (options.allowRedirects && fileExists(files, linkPath + "/index.html")) {
    return false
  }

  return !fileExists(files, linkPath) || (options.checkAnchors && !fileHasTarget(filesToTargets, linkPath, fragment))
}
<commit_msg>Decode links paths<commit_after>const urijs = require("urijs")
const path = require("path")

module.exports = ({
  files,
  filesToTargets,
  fileExists,
  fileHasTarget,
  link: {
    target,
    type,
    filename,
  },
  options,
}) => {
  // Allow anchors before checking for a missing href
  if (options.allowAnchors && type === "anchor") {
    return false
  }

  // Missing href is always broken
  if ((target == null)) {
    return true
  }

  const uri = urijs(target)

  // Allow anything matching the options.allowRegex regex
  if ((options.allowRegex != null) && options.allowRegex.exec(target)) {
    return false
  }

  // Empty link is always broken
  if (target === "") {
    return true
  }

  // Allow link to '#'
  if (target === "#") {
    return false
  }

  // Automatically accept all external links (could change later)
  if (uri.hostname()) {
    return false
  }

  // Ignore mailto and other non-http/https links
  if (uri.protocol() && !["http", "https"].includes(uri.protocol)) {
    return false
  }

  const fragment = uri.fragment()

  // Allow links to elements on the same page
  if (fragment && !uri.path()) {
    return options.checkAnchors && !fileHasTarget(filesToTargets, filename, fragment)
  }

  // Add baseURL in here so that the linkPath resolves to it in the case of
  // a relative link
  if (options.baseURL) {
    filename = path.join(options.baseURL, filename)
  }

  // Need to transform uri.path() into something Metalsmith can recognise
  const unixFilename = filename.replace(/\\/g, "/")
  let linkPath = decodeURIComponent(uri.absoluteTo(unixFilename).path())

  // If baseURL then all internal links should be prefixed by it.
  if (options.baseURL) {

    // If the linkPath does not start with the baseURL then it is broken
    if (linkPath.indexOf(options.baseURL) !== 0) {
      return true
    }

    // Strip the baseURL out for checking whether the file exists in metalsmith
    linkPath = linkPath.replace(options.baseURL, "")

    // Fix bug where you were linking directly to the linkPath
    if (linkPath === "") {
      linkPath = "/"
    }
  }

  // Special case for link to root
  if (linkPath === "/") {
    return !fileExists(files, "index.html") || (options.checkAnchors && !fileHasTarget(filesToTargets, "index.html", fragment))
  }

  // Allow links to directories with a trailing slash
  if (linkPath.slice(-1) === "/") {
    linkPath += "index.html"
  }

  // Allow links to directories without a trailing slash with allowRedirects option
  if (options.allowRedirects && fileExists(files, linkPath + "/index.html")) {
    return false
  }

  return !fileExists(files, linkPath) || (options.checkAnchors && !fileHasTarget(filesToTargets, linkPath, fragment))
}
<|endoftext|>


<commit_before>"use strict";

const JSZip = require("jszip"),
      modFs = require("fs"),
      modPath = require("path"),
      modStream = require("stream");

const modVfs = require("./vfs.js");

var CountToken = function (count, callback)
{
    this.decrease = function ()
    {
        --count;
        if (count === 0)
        {
            callback();
        }
    };

    if (count === 0)
    {
        callback();
    }
};

var zipCache = { };

function openZipFile(zipPath, callback)
{
    if (zipCache[zipPath])
    {
        callback(zipCache[zipPath]);
    }
    else
    {
        modVfs.readFile(zipPath, function (err, buffer)
        {
            if (err)
            {
                callback(null);
                return;
            }

            var zipper = new JSZip();
            zipper.loadAsync(buffer).then(function (zip)
            {
                if (zip)
                {
                    zipCache[zipPath] = zip;
                }
                callback(zip);
            });
        });
    }
}

function addFileSync(zip, path)
{
    var stats = modFs.statSync(path);
    if (stats.isDirectory())
    {
        var zipFolder = zip.folder(modPath.basename(path));
        var files = modFs.readdirSync(path);
        for (var i = 0; i < files.length; ++i)
        {
            addFileSync(zipFolder, modPath.join(path, files[i]));
        }
    }
    else
    {
        zip.file(modPath.basename(path), modFs.readFileSync(path));
    }
}

function addFile(zip, path, callback)
{
    modFs.stat(path, function (err, stats)
    {
        if (err)
        {
            callback(err);
            return;
        }

        if (stats.isDirectory())
        {
            var zipFolder = zip.folder(modPath.basename(path));
            modFs.readdir(path, function (err, files)
            {
                if (err)
                {
                    callback(err);
                    return;
                }

                var token = new CountToken(files.length, function ()
                {
                    callback(null);
                });

                for (var i = 0; i < files.length; ++i)
                {
                    addFile(zipFolder, modPath.join(path, files[i]), function ()
                    {
                        token.decrease();
                    });
                }
            });
        }
        else
        {
            var stream = modFs.createReadStream(path);
            zip.file(modPath.basename(path), stream);
            callback(null);
        }
    });
}

exports.makeZip = function(path, callback)
{
    var zip = new JSZip();

    addFile(zip, path, function (err)
    {
        callback(err, zip.generateNodeStream({ streamFiles:true }));
    });
};

exports.readZip = function(zipPath, subPath, callback)
{
    function handle(zip)
    {
        if (! zip)
        {
            callback("failed to open ZIP", []);
            return;
        }

        var files = [];

        for (var file in zip.files)
        {
            var fileObj = zip.files[file];
            var filePath = fileObj.name.replace(/\/$/, "");
            var dir = filePath.substr(0, filePath.lastIndexOf("/"));

            if (dir === subPath)
            {
                var parts = filePath.split("/");
                var name = parts[parts.length - 1];
                files.push(name);
            }
        }

        callback(null, files);
    }

    subPath = subPath.replace(/\/$/, "");

    console.log("readZip: " + zipPath + "#" + subPath);
    openZipFile(zipPath, handle);
};

/* Returns a stream for reading the given file in a ZIP.
 */
exports.createReadStream = function (zipPath, innerPath, callback)
{
    function handle(zip)
    {
        if (! zip)
        {
            callback("failed to open ZIP", []);
            return;
        }

        zip.file(innerPath).async("nodebuffer").then(function (b)
        {
            var readable = new modStream.Readable();
            readable._read = function () { };
            readable.push(b);
            readable.push(null);
            callback(b.length, readable);
        });
    }

    openZipFile(zipPath, handle);
};

exports.createReadStreamRanged = function (zipPath, innerPath, from, to, callback)
{
    function handle(zip)
    {
        if (! zip)
        {
            callback("failed to open ZIP", []);
            return;
        }

        zip.file(innerPath).async("nodebuffer").then(function (b)
        {
            var readable = new modStream.Readable();
            readable._read = function () { };
            readable.push(b.slice(from, to - from));
            readable.push(null);
            callback(b.length, readable);
        });
    }

    openZipFile(zipPath, handle);
};

/* Opens a file in the ZIP and passes a handle to the callback.
 */
exports.open = function (zipPath, innerPath, mode, callback)
{
    function handle(zip)
    {
        if (! zip)
        {
            callback("failed to open ZIP", null);
            return;
        }

        var fd = {
            isZip: true,
            file: zip.file(innerPath)
        };
        callback(null, fd);
    }

    openZipFile(zipPath, handle);
};

/* Reads from the given file entry of a ZIP.
 */
exports.readFile = function (file, buffer, offset, length, position, callback)
{
    file.async("nodebuffer").then(function (b)
    {
        var readable = new modStream.Readable();
        readable._read = function () { };
        readable.push(b.slice(position, length));
        readable.push(null);

        var data = readable.read(length);
        data.copy(buffer, offset);
        callback(null, length, buffer);
    });
};

/* Closes the given handle.
 */
exports.close = function (fd, callback)
{
    callback(null);
};

exports.stat = function (zipPath, innerPath, callback)
{
    function handle(zip)
    {
        if (! zip)
        {
            callback("failed to open ZIP", []);
            return;
        }

        var stats = [];

        for (var file in zip.files)
        {
            var fileObj = zip.files[file];
            var filePath = fileObj.name.replace(/\/$/, "");
            var dir = filePath.substr(0, filePath.lastIndexOf("/"));

            if (filePath === innerPath)
            {
                var parts = filePath.split("/");
                var name = parts[parts.length - 1];
                var isDir = fileObj.dir;
                stats.push({
                    isDirectory: function (isDir) { return function () { return isDir; }; }(isDir),
                    mtime: new Date(fileObj.date),
                    size: fileObj._data.uncompressedSize
                });
            }
        }

        if (stats.length === 0)
        {
            callback("no such file", null);
        }
        else
        {
            callback(null, stats[0]);
        }
    }

    innerPath = innerPath.replace(/\/$/, "");

    if (innerPath === "")
    {
        modVfs.stat(zipPath, callback);
        return;
    }

    openZipFile(zipPath, handle);
};
<commit_msg>fixed some ZIP reading issues<commit_after>"use strict";

const JSZip = require("jszip"),
      modFs = require("fs"),
      modPath = require("path"),
      modStream = require("stream");

const modVfs = require("./vfs.js");

var CountToken = function (count, callback)
{
    this.decrease = function ()
    {
        --count;
        if (count === 0)
        {
            callback();
        }
    };

    if (count === 0)
    {
        callback();
    }
};

var zipCache = { };

function openZipFile(zipPath, callback)
{
    var now = Date.now();
    for (var key in zipCache)
    {
        if (zipCache[key][0] + 5 * 60 * 1000 < now)
        {
            delete zipCache[key];
        }
    }

    if (zipCache[zipPath])
    {
        zipCache[zipPath][0] = now;
        callback(zipCache[zipPath][1]);
    }
    else
    {
        modVfs.readFile(zipPath, function (err, buffer)
        {
            if (err)
            {
                callback(null);
                return;
            }

            var zipper = new JSZip();
            zipper.loadAsync(buffer).then(function (zip)
            {
                if (zip)
                {
                    zipCache[zipPath] = [now, zip];
                }
                callback(zip);
            });
        });
    }
}

function addFileSync(zip, path)
{
    var stats = modFs.statSync(path);
    if (stats.isDirectory())
    {
        var zipFolder = zip.folder(modPath.basename(path));
        var files = modFs.readdirSync(path);
        for (var i = 0; i < files.length; ++i)
        {
            addFileSync(zipFolder, modPath.join(path, files[i]));
        }
    }
    else
    {
        zip.file(modPath.basename(path), modFs.readFileSync(path));
    }
}

function addFile(zip, path, callback)
{
    modFs.stat(path, function (err, stats)
    {
        if (err)
        {
            callback(err);
            return;
        }

        if (stats.isDirectory())
        {
            var zipFolder = zip.folder(modPath.basename(path));
            modFs.readdir(path, function (err, files)
            {
                if (err)
                {
                    callback(err);
                    return;
                }

                var token = new CountToken(files.length, function ()
                {
                    callback(null);
                });

                for (var i = 0; i < files.length; ++i)
                {
                    addFile(zipFolder, modPath.join(path, files[i]), function ()
                    {
                        token.decrease();
                    });
                }
            });
        }
        else
        {
            var stream = modFs.createReadStream(path);
            zip.file(modPath.basename(path), stream);
            callback(null);
        }
    });
}

exports.makeZip = function(path, callback)
{
    var zip = new JSZip();

    addFile(zip, path, function (err)
    {
        callback(err, zip.generateNodeStream({ streamFiles:true }));
    });
};

exports.readZip = function(zipPath, subPath, callback)
{
    function handle(zip)
    {
        if (! zip)
        {
            callback("failed to open ZIP", []);
            return;
        }

        var files = [];
        var seen = { };

        for (var file in zip.files)
        {
            var fileObj = zip.files[file];
            var filePath = fileObj.name.replace(/\/$/, "");
            var dir = filePath.substr(0, filePath.lastIndexOf("/"));

            if (dir === subPath)
            {
                var parts = filePath.split("/");
                var name = parts[parts.length - 1];
                files.push(name);
                seen[name] = true;
            }
        }

        for (var file in zip.files)
        {
            var fileObj = zip.files[file];
            var filePath = fileObj.name.replace(/\/$/, "");
            var dir = filePath.substr(0, filePath.lastIndexOf("/"));

            if (subPath === "" || dir.startsWith(subPath))
            {
                var pos = dir.indexOf("/", subPath.length);
                if (pos !== -1)
                {
                    var subDir = dir.substr(0, pos);
                    var parts = subDir.split("/");
                    var name = parts[parts.length - 1];

                    if (! seen[name])
                    {
                        files.push(name);
                        seen[name] = true;
                    }
                }
            }
        }

        callback(null, files);
    }

    subPath = subPath.replace(/\/$/, "");

    console.log("readZip: " + zipPath + "#" + subPath);
    openZipFile(zipPath, handle);
};

/* Returns a stream for reading the given file in a ZIP.
 */
exports.createReadStream = function (zipPath, innerPath, callback)
{
    function handle(zip)
    {
        if (! zip)
        {
            callback("failed to open ZIP", []);
            return;
        }

        zip.file(innerPath).async("nodebuffer").then(function (b)
        {
            var readable = new modStream.Readable();
            readable._read = function () { };
            readable.push(b);
            readable.push(null);
            callback(b.length, readable);
        });
    }

    openZipFile(zipPath, handle);
};

exports.createReadStreamRanged = function (zipPath, innerPath, from, to, callback)
{
    function handle(zip)
    {
        if (! zip)
        {
            callback("failed to open ZIP", []);
            return;
        }

        zip.file(innerPath).async("nodebuffer").then(function (b)
        {
            var readable = new modStream.Readable();
            readable._read = function () { };
            readable.push(b.slice(from, to - from));
            readable.push(null);
            callback(b.length, readable);
        });
    }

    openZipFile(zipPath, handle);
};

/* Opens a file in the ZIP and passes a handle to the callback.
 */
exports.open = function (zipPath, innerPath, mode, callback)
{
    function handle(zip)
    {
        if (! zip)
        {
            callback("failed to open ZIP", null);
            return;
        }

        var fd = {
            isZip: true,
            file: zip.file(innerPath)
        };
        callback(null, fd);
    }

    openZipFile(zipPath, handle);
};

/* Reads from the given file entry of a ZIP.
 */
exports.readFile = function (file, buffer, offset, length, position, callback)
{
    file.async("nodebuffer").then(function (b)
    {
        var readable = new modStream.Readable();
        readable._read = function () { };
        readable.push(b.slice(position, length));
        readable.push(null);

        var data = readable.read(length);
        data.copy(buffer, offset);
        callback(null, length, buffer);
    });
};

/* Closes the given handle.
 */
exports.close = function (fd, callback)
{
    callback(null);
};

exports.stat = function (zipPath, innerPath, callback)
{
    function handle(zip)
    {
        if (! zip)
        {
            callback("failed to open ZIP", []);
            return;
        }

        var stats = [];

        var existsAsPrefix = false;
        for (var file in zip.files)
        {
            var fileObj = zip.files[file];
            var filePath = fileObj.name.replace(/\/$/, "");
            var dir = filePath.substr(0, filePath.lastIndexOf("/"));

            if (filePath === innerPath)
            {
                var parts = filePath.split("/");
                var name = parts[parts.length - 1];
                var isDir = fileObj.dir;
                stats.push({
                    isDirectory: function (isDir) { return function () { return isDir; }; }(isDir),
                    mtime: new Date(fileObj.date),
                    size: fileObj._data.uncompressedSize
                });
            }
            else if (filePath.startsWith(innerPath))
            {
                existsAsPrefix = true;
            }
        }

        if (stats.length === 0 && existsAsPrefix)
        {
            stats.push({
                isDirectory: function () { return true; },
                mtime: new Date(),
                size: 0
            });
        }

        if (stats.length === 0)
        {
            callback("no such file", null);
        }
        else
        {
            callback(null, stats[0]);
        }
    }

    innerPath = innerPath.replace(/\/$/, "");

    if (innerPath === "")
    {
        modVfs.stat(zipPath, callback);
        return;
    }

    openZipFile(zipPath, handle);
};
<|endoftext|>