<commit_before>const path = require('path');
const should = require('should');
const supertest = require('supertest');
const sinon = require('sinon');
const testUtils = require('../../../../utils');
const localUtils = require('./utils');
const config = require('../../../../../server/config');
const labs = require('../../../../../server/services/labs');

const ghost = testUtils.startGhost;

let request;

describe('Members API', function () {
    before(function () {
        sinon.stub(labs, 'isSet').withArgs('members').returns(true);
    });

    after(function () {
        sinon.restore();
    });

    before(function () {
        return ghost()
            .then(function () {
                request = supertest.agent(config.get('url'));
            })
            .then(function () {
                return localUtils.doAuth(request, 'member');
            });
    });

    it('Can browse', function () {
        return request
            .get(localUtils.API.getApiQuery('members/'))
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(200)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);
                const jsonResponse = res.body;
                should.exist(jsonResponse);
                should.exist(jsonResponse.members);
                jsonResponse.members.should.have.length(1);
                localUtils.API.checkResponse(jsonResponse.members[0], 'member',  'stripe');

                testUtils.API.isISO8601(jsonResponse.members[0].created_at).should.be.true();
                jsonResponse.members[0].created_at.should.be.an.instanceof(String);

                jsonResponse.meta.pagination.should.have.property('page', 1);
                jsonResponse.meta.pagination.should.have.property('limit', 15);
                jsonResponse.meta.pagination.should.have.property('pages', 1);
                jsonResponse.meta.pagination.should.have.property('total', 1);
                jsonResponse.meta.pagination.should.have.property('next', null);
                jsonResponse.meta.pagination.should.have.property('prev', null);
            });
    });

    it('Can read', function () {
        return request
            .get(localUtils.API.getApiQuery(`members/${testUtils.DataGenerator.Content.members[0].id}/`))
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(200)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);
                const jsonResponse = res.body;
                should.exist(jsonResponse);
                should.exist(jsonResponse.members);
                jsonResponse.members.should.have.length(1);
                localUtils.API.checkResponse(jsonResponse.members[0], 'member', 'stripe');
            });
    });

    it('Can add', function () {
        const member = {
            name: 'test',
            email: 'memberTestAdd@test.com'
        };

        return request
            .post(localUtils.API.getApiQuery(`members/`))
            .send({members: [member]})
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(201)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);
                const jsonResponse = res.body;
                should.exist(jsonResponse);
                should.exist(jsonResponse.members);
                jsonResponse.members.should.have.length(1);

                jsonResponse.members[0].name.should.equal(member.name);
                jsonResponse.members[0].email.should.equal(member.email);
            })
            .then(() => {
                return request
                    .post(localUtils.API.getApiQuery(`members/`))
                    .send({members: [member]})
                    .set('Origin', config.get('url'))
                    .expect('Content-Type', /json/)
                    .expect('Cache-Control', testUtils.cacheRules.private)
                    .expect(422);
            });
    });

    it('Should fail when passing incorrect email_type query parameter', function () {
        const member = {
            name: 'test',
            email: 'memberTestAdd@test.com'
        };

        return request
            .post(localUtils.API.getApiQuery(`members/?send_email=true&email_type=lel`))
            .send({members: [member]})
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(422);
    });

    it('Can edit by id', function () {
        const memberToChange = {
            name: 'change me',
            email: 'member2Change@test.com'
        };

        const memberChanged = {
            name: 'changed',
            email: 'cantChangeMe@test.com'
        };

        return request
            .post(localUtils.API.getApiQuery(`members/`))
            .send({members: [memberToChange]})
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(201)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);
                const jsonResponse = res.body;
                should.exist(jsonResponse);
                should.exist(jsonResponse.members);
                jsonResponse.members.should.have.length(1);

                return jsonResponse.members[0];
            })
            .then((newMember) => {
                return request
                    .put(localUtils.API.getApiQuery(`members/${newMember.id}/`))
                    .send({members: [memberChanged]})
                    .set('Origin', config.get('url'))
                    .expect('Content-Type', /json/)
                    .expect('Cache-Control', testUtils.cacheRules.private)
                    .expect(200)
                    .then((res) => {
                        should.not.exist(res.headers['x-cache-invalidate']);

                        const jsonResponse = res.body;

                        should.exist(jsonResponse);
                        should.exist(jsonResponse.members);
                        jsonResponse.members.should.have.length(1);
                        localUtils.API.checkResponse(jsonResponse.members[0], 'member');
                        jsonResponse.members[0].name.should.equal(memberChanged.name);
                        jsonResponse.members[0].email.should.not.equal(memberChanged.email);
                        jsonResponse.members[0].email.should.equal(memberToChange.email);
                    });
            });
    });

    it('Can destroy', function () {
        const member = {
            name: 'test',
            email: 'memberTestDestroy@test.com'
        };

        return request
            .post(localUtils.API.getApiQuery(`members/`))
            .send({members: [member]})
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(201)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);

                const jsonResponse = res.body;

                should.exist(jsonResponse);
                should.exist(jsonResponse.members);

                return jsonResponse.members[0];
            })
            .then((newMember) => {
                return request
                    .delete(localUtils.API.getApiQuery(`members/${newMember.id}`))
                    .set('Origin', config.get('url'))
                    .expect('Cache-Control', testUtils.cacheRules.private)
                    .expect(204)
                    .then(() => newMember);
            })
            .then((newMember) => {
                return request
                    .get(localUtils.API.getApiQuery(`members/${newMember.id}/`))
                    .set('Origin', config.get('url'))
                    .expect('Content-Type', /json/)
                    .expect('Cache-Control', testUtils.cacheRules.private)
                    .expect(404);
            });
    });

    it('Can export CSV', function () {
        return request
            .get(localUtils.API.getApiQuery(`members/csv/`))
            .set('Origin', config.get('url'))
            .expect('Content-Type', /text\/csv/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(200)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);
                res.headers['content-disposition'].should.match(/Attachment;\sfilename="members/);
                res.text.should.match(/id,email,name,note,created_at,deleted_at/);
                res.text.should.match(/member1@test.com/);
                res.text.should.match(/Mr Egg/);
            });
    });

    it('Can import CSV', function () {
        return request
            .post(localUtils.API.getApiQuery(`members/csv/`))
            .attach('membersfile', path.join(__dirname, '/../../../../utils/fixtures/csv/valid-members-import.csv'))
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(201)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);
                const jsonResponse = res.body;

                should.exist(jsonResponse);
                should.exist(jsonResponse.meta);
                should.exist(jsonResponse.meta.stats);

                jsonResponse.meta.stats.imported.should.equal(2);
                jsonResponse.meta.stats.duplicates.should.equal(0);
                jsonResponse.meta.stats.invalid.should.equal(0);
            });
    });
});
<commit_msg>Fixed linter<commit_after>const path = require('path');
const should = require('should');
const supertest = require('supertest');
const sinon = require('sinon');
const testUtils = require('../../../../utils');
const localUtils = require('./utils');
const config = require('../../../../../server/config');
const labs = require('../../../../../server/services/labs');

const ghost = testUtils.startGhost;

let request;

describe('Members API', function () {
    before(function () {
        sinon.stub(labs, 'isSet').withArgs('members').returns(true);
    });

    after(function () {
        sinon.restore();
    });

    before(function () {
        return ghost()
            .then(function () {
                request = supertest.agent(config.get('url'));
            })
            .then(function () {
                return localUtils.doAuth(request, 'member');
            });
    });

    it('Can browse', function () {
        return request
            .get(localUtils.API.getApiQuery('members/'))
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(200)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);
                const jsonResponse = res.body;
                should.exist(jsonResponse);
                should.exist(jsonResponse.members);
                jsonResponse.members.should.have.length(1);
                localUtils.API.checkResponse(jsonResponse.members[0], 'member', 'stripe');

                testUtils.API.isISO8601(jsonResponse.members[0].created_at).should.be.true();
                jsonResponse.members[0].created_at.should.be.an.instanceof(String);

                jsonResponse.meta.pagination.should.have.property('page', 1);
                jsonResponse.meta.pagination.should.have.property('limit', 15);
                jsonResponse.meta.pagination.should.have.property('pages', 1);
                jsonResponse.meta.pagination.should.have.property('total', 1);
                jsonResponse.meta.pagination.should.have.property('next', null);
                jsonResponse.meta.pagination.should.have.property('prev', null);
            });
    });

    it('Can read', function () {
        return request
            .get(localUtils.API.getApiQuery(`members/${testUtils.DataGenerator.Content.members[0].id}/`))
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(200)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);
                const jsonResponse = res.body;
                should.exist(jsonResponse);
                should.exist(jsonResponse.members);
                jsonResponse.members.should.have.length(1);
                localUtils.API.checkResponse(jsonResponse.members[0], 'member', 'stripe');
            });
    });

    it('Can add', function () {
        const member = {
            name: 'test',
            email: 'memberTestAdd@test.com'
        };

        return request
            .post(localUtils.API.getApiQuery(`members/`))
            .send({members: [member]})
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(201)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);
                const jsonResponse = res.body;
                should.exist(jsonResponse);
                should.exist(jsonResponse.members);
                jsonResponse.members.should.have.length(1);

                jsonResponse.members[0].name.should.equal(member.name);
                jsonResponse.members[0].email.should.equal(member.email);
            })
            .then(() => {
                return request
                    .post(localUtils.API.getApiQuery(`members/`))
                    .send({members: [member]})
                    .set('Origin', config.get('url'))
                    .expect('Content-Type', /json/)
                    .expect('Cache-Control', testUtils.cacheRules.private)
                    .expect(422);
            });
    });

    it('Should fail when passing incorrect email_type query parameter', function () {
        const member = {
            name: 'test',
            email: 'memberTestAdd@test.com'
        };

        return request
            .post(localUtils.API.getApiQuery(`members/?send_email=true&email_type=lel`))
            .send({members: [member]})
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(422);
    });

    it('Can edit by id', function () {
        const memberToChange = {
            name: 'change me',
            email: 'member2Change@test.com'
        };

        const memberChanged = {
            name: 'changed',
            email: 'cantChangeMe@test.com'
        };

        return request
            .post(localUtils.API.getApiQuery(`members/`))
            .send({members: [memberToChange]})
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(201)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);
                const jsonResponse = res.body;
                should.exist(jsonResponse);
                should.exist(jsonResponse.members);
                jsonResponse.members.should.have.length(1);

                return jsonResponse.members[0];
            })
            .then((newMember) => {
                return request
                    .put(localUtils.API.getApiQuery(`members/${newMember.id}/`))
                    .send({members: [memberChanged]})
                    .set('Origin', config.get('url'))
                    .expect('Content-Type', /json/)
                    .expect('Cache-Control', testUtils.cacheRules.private)
                    .expect(200)
                    .then((res) => {
                        should.not.exist(res.headers['x-cache-invalidate']);

                        const jsonResponse = res.body;

                        should.exist(jsonResponse);
                        should.exist(jsonResponse.members);
                        jsonResponse.members.should.have.length(1);
                        localUtils.API.checkResponse(jsonResponse.members[0], 'member');
                        jsonResponse.members[0].name.should.equal(memberChanged.name);
                        jsonResponse.members[0].email.should.not.equal(memberChanged.email);
                        jsonResponse.members[0].email.should.equal(memberToChange.email);
                    });
            });
    });

    it('Can destroy', function () {
        const member = {
            name: 'test',
            email: 'memberTestDestroy@test.com'
        };

        return request
            .post(localUtils.API.getApiQuery(`members/`))
            .send({members: [member]})
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(201)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);

                const jsonResponse = res.body;

                should.exist(jsonResponse);
                should.exist(jsonResponse.members);

                return jsonResponse.members[0];
            })
            .then((newMember) => {
                return request
                    .delete(localUtils.API.getApiQuery(`members/${newMember.id}`))
                    .set('Origin', config.get('url'))
                    .expect('Cache-Control', testUtils.cacheRules.private)
                    .expect(204)
                    .then(() => newMember);
            })
            .then((newMember) => {
                return request
                    .get(localUtils.API.getApiQuery(`members/${newMember.id}/`))
                    .set('Origin', config.get('url'))
                    .expect('Content-Type', /json/)
                    .expect('Cache-Control', testUtils.cacheRules.private)
                    .expect(404);
            });
    });

    it('Can export CSV', function () {
        return request
            .get(localUtils.API.getApiQuery(`members/csv/`))
            .set('Origin', config.get('url'))
            .expect('Content-Type', /text\/csv/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(200)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);
                res.headers['content-disposition'].should.match(/Attachment;\sfilename="members/);
                res.text.should.match(/id,email,name,note,created_at,deleted_at/);
                res.text.should.match(/member1@test.com/);
                res.text.should.match(/Mr Egg/);
            });
    });

    it('Can import CSV', function () {
        return request
            .post(localUtils.API.getApiQuery(`members/csv/`))
            .attach('membersfile', path.join(__dirname, '/../../../../utils/fixtures/csv/valid-members-import.csv'))
            .set('Origin', config.get('url'))
            .expect('Content-Type', /json/)
            .expect('Cache-Control', testUtils.cacheRules.private)
            .expect(201)
            .then((res) => {
                should.not.exist(res.headers['x-cache-invalidate']);
                const jsonResponse = res.body;

                should.exist(jsonResponse);
                should.exist(jsonResponse.meta);
                should.exist(jsonResponse.meta.stats);

                jsonResponse.meta.stats.imported.should.equal(2);
                jsonResponse.meta.stats.duplicates.should.equal(0);
                jsonResponse.meta.stats.invalid.should.equal(0);
            });
    });
});
<|endoftext|>


<commit_before>{% load vv_tags vvpages_tags %}
{% use_local_storage as storage %}
{% if storage %}
	isInStorage: function(resturl) {
		var url = store.get(resturl);
		if (url == undefined) {
			return false
		}
		return true
	},
	loadFromStorage: function(resturl) {
		var data = store.get(resturl);
		return data
	},
{% endif %}
loadHtml: function(resturl){
	var fetch = true;
	var data = {};
	{% if storage %}
	if (this.isInStorage(resturl) === true) {
		//console.log("Data in storage: "+resturl);
		data = this.loadFromStorage(resturl);
		now = new Date();
		var is_expired = false;
		if (data.expires < now) {
			var is_expired = true;
		}
		if (is_expired === false) {
			//console.log("Data not expired");
			fetch = false;
		} else {
			//console.log("Data expired");
			if (navigator.onLine === false) {
				fetch = false;
				//console.log("Navigator not online");
			} else {
				//console.log("Navigator online");
			}
		}
	}
	{% endif %}
	if (fetch === true){
		//console.log("Fetchind data from "+resturl+" //");
		promise.get(resturl,{},{"Accept":"application/json"}).then(function(error, data, xhr) {
		    if (error) {
		    	console.log('Error ' + xhr.status);return;
		    }
		    {% if isdebug is True %}console.log("Raw data: "+data);{% endif %}
		    data = JSON.parse(data);
			app.flush();
			app.pageContent = data.content;
		    app.pageTitle = data.title;
		    top.document.pageTitle = data.title;
		    app.activate(["pageContent", "pageTitle"]);
		    var now = new Date();
			var exp = new Date();
			var mins = 1;
			var duration = 1000*60*mins;
			exp.setTime(now.getTime() + (duration));
			data.expires = exp;
			{% if storage %}
				//console.log("Setting key in local storage");
				store.set(resturl, data);
			{% endif %}
			{% if perms.vvpages.change_page %}
		    	app.adminPageUrl ="/admin/vvpages/page/"+data.pk+"/change/";
		    {% endif %}
		});
	} else {
		app.flush();
		app.pageContent = data.content;
	    app.pageTitle = data.title;
	    top.document.pageTitle = data.title;
	    app.activate(["pageContent", "pageTitle"]);
	    {% if perms.vvpages.change_page %}
	    	app.adminPageUrl ="/admin/vvpages/page/"+data.pk+"/change/";
	    {% endif %}
	}
	return
},
loadChunk: function (resturl, pageTitle){
	promise.get(resturl).then(function(error, data, xhr) {
	    if (error) {console.log('Error ' + xhr.status);return;}
	    {% if isdebug is True %}console.log("Raw chunk: "+data);{% endif %}
	    app.flush();
	    app.pageContent = data;
	    app.pageTitle = title;
	    top.document.title = title;
	    app.activate(["pageContent", "pageTitle"]);
	});
	return
},<commit_msg>Correction<commit_after>{% load vv_tags vvpages_tags %}
{% use_local_storage as storage %}
{% if storage %}
	isInStorage: function(resturl) {
		var url = store.get(resturl);
		if (url == undefined) {
			return false
		}
		return true
	},
	loadFromStorage: function(resturl) {
		var data = store.get(resturl);
		return data
	},
{% endif %}
loadHtml: function(resturl){
	var fetch = true;
	var data = {};
	{% if storage %}
	if (this.isInStorage(resturl) === true) {
		//console.log("Data in storage: "+resturl);
		data = this.loadFromStorage(resturl);
		now = new Date();
		var is_expired = false;
		if (data.expires < now) {
			var is_expired = true;
		}
		if (is_expired === false) {
			//console.log("Data not expired");
			fetch = false;
		} else {
			//console.log("Data expired");
			if (navigator.onLine === false) {
				fetch = false;
				//console.log("Navigator not online");
			} else {
				//console.log("Navigator online");
			}
		}
	}
	{% endif %}
	if (fetch === true){
		//console.log("Fetchind data from "+resturl+" //");
		promise.get(resturl,{},{"Accept":"application/json"}).then(function(error, data, xhr) {
		    if (error) {
		    	console.log('Error ' + xhr.status);return;
		    }
		    {% if isdebug is True %}console.log("Raw data: "+data);{% endif %}
		    data = JSON.parse(data);
			app.flush();
			app.pageContent = data.content;
		    app.pageTitle = data.title;
		    top.document.pageTitle = data.title;
		    app.activate(["pageContent", "pageTitle"]);
		    var now = new Date();
			var exp = new Date();
			var mins = 1;
			var duration = 1000*60*mins;
			exp.setTime(now.getTime() + (duration));
			data.expires = exp;
			{% if storage %}
				//console.log("Setting key in local storage");
				store.set(resturl, data);
			{% endif %}
			{% if perms.vvpages.change_page %}
		    	app.adminPageUrl ="/admin/vvpages/page/"+data.pk+"/change/";
		    {% endif %}
		});
	} else {
		app.flush();
		app.pageContent = data.content;
	    app.pageTitle = data.title;
	    top.document.pageTitle = data.title;
	    app.activate(["pageContent", "pageTitle"]);
	    {% if perms.vvpages.change_page %}
	    	app.adminPageUrl ="/admin/vvpages/page/"+data.pk+"/change/";
	    {% endif %}
	}
	return
},
loadChunk: function (resturl, title){
	promise.get(resturl).then(function(error, data, xhr) {
	    if (error) {console.log('Error ' + xhr.status);return;}
	    {% if isdebug is True %}console.log("Raw chunk: "+data);{% endif %}
	    app.flush();
	    app.pageContent = data;
	    app.pageTitle = title;
	    top.document.title = title;
	    app.activate(["pageContent", "pageTitle"]);
	});
	return
},<|endoftext|>


<commit_before>import { applyMiddleware as reduxApplyMiddleware, combine } from 'redux'
import thunk from 'redux-thunk'

export default function applyMiddleware (...additionalMiddlewares) {
  return createStore => (reducer, initial, enhancer) => {
    const middleware = reduxApplyMiddleware(thunk)
    const enhancerFunc = enhancer ? combine(middleware, enhancer) : middleware
    return createStore(reducer, initial, enhancerFunc)
  }
}
<commit_msg>Simplify applyMiddleware<commit_after>import { applyMiddleware as reduxApplyMiddleware, combine } from 'redux'
import thunk from 'redux-thunk'

export default function applyMiddleware (...additionalMiddlewares) {
  return reduxApplyMiddleware(thunk, ...additionalMiddlewares)
}
<|endoftext|>


<commit_before>const errors = require('../../../errors');
const list = require('../../../algos/list/exportAlgos');
const genVID =
    require('../../../versioning/VersionID').generateVersionId;
const { serializeCircJSON } = require('../../utils');

const getMultipartUploadListing = require('./getMultipartUploadListing');
const { metadata } = require('./metadata');

// const genVID = versioning.VersionID.generateVersionId;

const defaultMaxKeys = 1000;
let uidCounter = 0;

function generateVersionId(replicationGroupId) {
    return genVID(uidCounter++, replicationGroupId);
}

function formatVersionKey(key, versionId) {
    return `${key}\0${versionId}`;
}

function inc(str) {
    return str ? (str.slice(0, str.length - 1) +
            String.fromCharCode(str.charCodeAt(str.length - 1) + 1)) : str;
}

const metastore = {
    errors: {}, // Used for simulation of metadata errors.

    createBucket: (bucketName, bucketMD, log, cb) => {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, (err, bucket) => {
                // TODO Check whether user already owns the bucket,
                // if so return "BucketAlreadyOwnedByYou"
                // If not owned by user, return "BucketAlreadyExists"
                if (bucket) {
                    return cb(errors.BucketAlreadyExists);
                }
                metadata.buckets.set(bucketName, bucketMD);
                metadata.keyMaps.set(bucketName, new Map);
                return cb();
            });
        });
    },

    putBucketAttributes: (bucketName, bucketMD, log, cb) => {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, err => {
                if (err) {
                    return cb(err);
                }
                metadata.buckets.set(bucketName, bucketMD);
                return cb();
            });
        });
    },

    getBucketAttributes: (bucketName, log, cb) => {
        process.nextTick(() => {
            if (!metadata.buckets.has(bucketName)) {
                return cb(errors.NoSuchBucket);
            }
            return cb(null, metadata.buckets.get(bucketName));
        });
    },

    deleteBucket: (bucketName, log, cb) => {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, err => {
                if (err) {
                    return cb(err);
                }
                if (metadata.keyMaps.has(bucketName)
                    && metadata.keyMaps.get(bucketName).length > 0) {
                    return cb(errors.BucketNotEmpty);
                }
                metadata.buckets.delete(bucketName);
                metadata.keyMaps.delete(bucketName);
                return cb(null);
            });
        });
    },

    putObject: (bucketName, objName, objVal, params, log, cb) => {
        if (metastore.errors.putObject) {
            return process.nextTick(() => cb(metastore.errors.putObject));
        }
        return process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, err => {
                if (err) {
                    return cb(err);
                }
                /*
                    valid combinations of versioning options:
                    - !versioning && !versionId: normal non-versioning put
                    - versioning && !versionId: create a new version
                    - versionId: update (PUT/DELETE) an existing version,
                        and also update master version in case the put
                        version is newer or same version than master.
                        if versionId === '' update master version
                */

                if (params && params.versionId) {
                    objVal.versionId = params.versionId; // eslint-disable-line
                    const mst = metadata.keyMaps.get(bucketName).get(objName);
                    if (mst && mst.versionId === params.versionId || !mst) {
                        metadata.keyMaps.get(bucketName).set(objName, objVal);
                    }
                    // eslint-disable-next-line
                    objName = formatVersionKey(objName, params.versionId);
                    metadata.keyMaps.get(bucketName).set(objName, objVal);
                    return cb(null, `{"versionId":"${objVal.versionId}"}`);
                }
                if (params && params.versioning) {
                    const versionId = generateVersionId();
                    objVal.versionId = versionId; // eslint-disable-line
                    metadata.keyMaps.get(bucketName).set(objName, objVal);
                    // eslint-disable-next-line
                    objName = formatVersionKey(objName, versionId);
                    metadata.keyMaps.get(bucketName).set(objName, objVal);
                    return cb(null, `{"versionId":"${versionId}"}`);
                }
                if (params && params.versionId === '') {
                    const versionId = generateVersionId();
                    objVal.versionId = versionId; // eslint-disable-line
                    metadata.keyMaps.get(bucketName).set(objName, objVal);
                    return cb(null, `{"versionId":"${objVal.versionId}"}`);
                }
                metadata.keyMaps.get(bucketName).set(objName, objVal);
                return cb(null);
            });
        });
    },

    getBucketAndObject: (bucketName, objName, params, log, cb) => {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, (err, bucket) => {
                if (err) {
                    return cb(err, { bucket });
                }
                if (params && params.versionId) {
                    // eslint-disable-next-line
                    objName = formatVersionKey(objName, params.versionId);
                }
                if (!metadata.keyMaps.has(bucketName)
                    || !metadata.keyMaps.get(bucketName).has(objName)) {
                    return cb(null, { bucket: bucket.serialize() });
                }
                const objMD = metadata.keyMaps.get(bucketName).get(objName);
                return cb(null, {
                    bucket: bucket.serialize(),
                    obj: JSON.stringify(objMD, serializeCircJSON()),
                });
            });
        });
    },

    getObject: (bucketName, objName, params, log, cb) => {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, err => {
                if (err) {
                    return cb(err);
                }
                if (params && params.versionId) {
                    // eslint-disable-next-line
                    objName = formatVersionKey(objName, params.versionId);
                }
                if (!metadata.keyMaps.has(bucketName)
                    || !metadata.keyMaps.get(bucketName).has(objName)) {
                    return cb(errors.NoSuchKey);
                }
                return cb(null, metadata.keyMaps.get(bucketName).get(objName));
            });
        });
    },

    deleteObject: (bucketName, objName, params, log, cb) => {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, err => {
                if (err) {
                    return cb(err);
                }
                if (!metadata.keyMaps.get(bucketName).has(objName)) {
                    return cb(errors.NoSuchKey);
                }
                if (params && params.versionId) {
                    const baseKey = inc(formatVersionKey(objName, ''));
                    const vobjName = formatVersionKey(objName,
                            params.versionId);
                    metadata.keyMaps.get(bucketName).delete(vobjName);
                    const mst = metadata.keyMaps.get(bucketName).get(objName);
                    if (mst.versionId === params.versionId) {
                        const keys = [];
                        metadata.keyMaps.get(bucketName).forEach((val, key) => {
                            if (key < baseKey && key > vobjName) {
                                keys.push(key);
                            }
                        });
                        if (keys.length === 0) {
                            metadata.keyMaps.get(bucketName).delete(objName);
                            return cb();
                        }
                        const key = keys.sort()[0];
                        const value = metadata.keyMaps.get(bucketName).get(key);
                        metadata.keyMaps.get(bucketName).set(objName, value);
                    }
                    return cb();
                }
                metadata.keyMaps.get(bucketName).delete(objName);
                return cb();
            });
        });
    },

    _hasDeleteMarker(key, keyMap) {
        const objectMD = keyMap.get(key);
        if (objectMD['x-amz-delete-marker'] !== undefined) {
            return (objectMD['x-amz-delete-marker'] === true);
        }
        return false;
    },

    listObject(bucketName, params, log, cb) {
        process.nextTick(() => {
            const {
                prefix,
                marker,
                delimiter,
                maxKeys,
                continuationToken,
                startAfter,
            } = params;
            if (prefix && typeof prefix !== 'string') {
                return cb(errors.InvalidArgument);
            }

            if (marker && typeof marker !== 'string') {
                return cb(errors.InvalidArgument);
            }

            if (delimiter && typeof delimiter !== 'string') {
                return cb(errors.InvalidArgument);
            }

            if (maxKeys && typeof maxKeys !== 'number') {
                return cb(errors.InvalidArgument);
            }

            if (continuationToken && typeof continuationToken !== 'string') {
                return cb(errors.InvalidArgument);
            }

            if (startAfter && typeof startAfter !== 'string') {
                return cb(errors.InvalidArgument);
            }

            // If paramMaxKeys is undefined, the default parameter will set it.
            // However, if it is null, the default parameter will not set it.
            let numKeys = maxKeys;
            if (numKeys === null) {
                numKeys = defaultMaxKeys;
            }

            if (!metadata.keyMaps.has(bucketName)) {
                return cb(errors.NoSuchBucket);
            }

            // If marker specified, edit the keys array so it
            // only contains keys that occur alphabetically after the marker
            const listingType = params.listingType;
            const extension = new list[listingType](params, log);
            const listingParams = extension.genMDParams();

            const keys = [];
            metadata.keyMaps.get(bucketName).forEach((val, key) => {
                if (listingParams.gt && listingParams.gt >= key) {
                    return null;
                }
                if (listingParams.gte && listingParams.gte > key) {
                    return null;
                }
                if (listingParams.lt && key >= listingParams.lt) {
                    return null;
                }
                if (listingParams.lte && key > listingParams.lte) {
                    return null;
                }
                return keys.push(key);
            });
            keys.sort();

            // Iterate through keys array and filter keys containing
            // delimiter into response.CommonPrefixes and filter remaining
            // keys into response.Contents
            for (let i = 0; i < keys.length; ++i) {
                const currentKey = keys[i];
                // Do not list object with delete markers
                if (this._hasDeleteMarker(currentKey,
                    metadata.keyMaps.get(bucketName))) {
                    continue;
                }
                const objMD = metadata.keyMaps.get(bucketName).get(currentKey);
                const value = JSON.stringify(objMD, serializeCircJSON());
                const obj = {
                    key: currentKey,
                    value,
                };
                // calling Ext.filter(obj) adds the obj to the Ext result if
                // not filtered.
                // Also, Ext.filter returns false when hit max keys.
                // What a nifty function!
                if (extension.filter(obj) < 0) {
                    break;
                }
            }
            return cb(null, extension.result());
        });
    },

    listMultipartUploads(bucketName, listingParams, log, cb) {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, (err, bucket) => {
                if (bucket === undefined) {
                    // no on going multipart uploads, return empty listing
                    return cb(null, {
                        IsTruncated: false,
                        NextMarker: undefined,
                        MaxKeys: 0,
                    });
                }
                return getMultipartUploadListing(bucket, listingParams, cb);
            });
        });
    },
};

module.exports = metastore;
<commit_msg>bugfix: S3C-2052 Add error functions<commit_after>const errors = require('../../../errors');
const list = require('../../../algos/list/exportAlgos');
const genVID =
    require('../../../versioning/VersionID').generateVersionId;
const { serializeCircJSON } = require('../../utils');

const getMultipartUploadListing = require('./getMultipartUploadListing');
const { metadata } = require('./metadata');

// const genVID = versioning.VersionID.generateVersionId;

const defaultMaxKeys = 1000;
let uidCounter = 0;

function generateVersionId(replicationGroupId) {
    return genVID(uidCounter++, replicationGroupId);
}

function formatVersionKey(key, versionId) {
    return `${key}\0${versionId}`;
}

function inc(str) {
    return str ? (str.slice(0, str.length - 1) +
            String.fromCharCode(str.charCodeAt(str.length - 1) + 1)) : str;
}

const metastore = {
    // Used for simulation of metadata errors.
    error: () => {},

    setErrorFunc(func) {
        this.error = func;
        return this;
    },

    clearErrorFunc() {
        this.error = () => {};
        return this;
    },

    createBucket: (bucketName, bucketMD, log, cb) => {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, (err, bucket) => {
                // TODO Check whether user already owns the bucket,
                // if so return "BucketAlreadyOwnedByYou"
                // If not owned by user, return "BucketAlreadyExists"
                if (bucket) {
                    return cb(errors.BucketAlreadyExists);
                }
                metadata.buckets.set(bucketName, bucketMD);
                metadata.keyMaps.set(bucketName, new Map);
                return cb();
            });
        });
    },

    putBucketAttributes: (bucketName, bucketMD, log, cb) => {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, err => {
                if (err) {
                    return cb(err);
                }
                metadata.buckets.set(bucketName, bucketMD);
                return cb();
            });
        });
    },

    getBucketAttributes: (bucketName, log, cb) => {
        process.nextTick(() => {
            if (!metadata.buckets.has(bucketName)) {
                return cb(errors.NoSuchBucket);
            }
            return cb(null, metadata.buckets.get(bucketName));
        });
    },

    deleteBucket: (bucketName, log, cb) => {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, err => {
                if (err) {
                    return cb(err);
                }
                if (metadata.keyMaps.has(bucketName)
                    && metadata.keyMaps.get(bucketName).length > 0) {
                    return cb(errors.BucketNotEmpty);
                }
                metadata.buckets.delete(bucketName);
                metadata.keyMaps.delete(bucketName);
                return cb(null);
            });
        });
    },

    putObject: (bucketName, objName, objVal, params, log, cb) => {
        const err = metastore.error(metastore.putObject.name);
        if (err) {
            return process.nextTick(() => cb(err));
        }
        return process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, err => {
                if (err) {
                    return cb(err);
                }
                /*
                    valid combinations of versioning options:
                    - !versioning && !versionId: normal non-versioning put
                    - versioning && !versionId: create a new version
                    - versionId: update (PUT/DELETE) an existing version,
                        and also update master version in case the put
                        version is newer or same version than master.
                        if versionId === '' update master version
                */

                if (params && params.versionId) {
                    objVal.versionId = params.versionId; // eslint-disable-line
                    const mst = metadata.keyMaps.get(bucketName).get(objName);
                    if (mst && mst.versionId === params.versionId || !mst) {
                        metadata.keyMaps.get(bucketName).set(objName, objVal);
                    }
                    // eslint-disable-next-line
                    objName = formatVersionKey(objName, params.versionId);
                    metadata.keyMaps.get(bucketName).set(objName, objVal);
                    return cb(null, `{"versionId":"${objVal.versionId}"}`);
                }
                if (params && params.versioning) {
                    const versionId = generateVersionId();
                    objVal.versionId = versionId; // eslint-disable-line
                    metadata.keyMaps.get(bucketName).set(objName, objVal);
                    // eslint-disable-next-line
                    objName = formatVersionKey(objName, versionId);
                    metadata.keyMaps.get(bucketName).set(objName, objVal);
                    return cb(null, `{"versionId":"${versionId}"}`);
                }
                if (params && params.versionId === '') {
                    const versionId = generateVersionId();
                    objVal.versionId = versionId; // eslint-disable-line
                    metadata.keyMaps.get(bucketName).set(objName, objVal);
                    return cb(null, `{"versionId":"${objVal.versionId}"}`);
                }
                metadata.keyMaps.get(bucketName).set(objName, objVal);
                return cb(null);
            });
        });
    },

    getBucketAndObject: (bucketName, objName, params, log, cb) => {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, (err, bucket) => {
                if (err) {
                    return cb(err, { bucket });
                }
                if (params && params.versionId) {
                    // eslint-disable-next-line
                    objName = formatVersionKey(objName, params.versionId);
                }
                if (!metadata.keyMaps.has(bucketName)
                    || !metadata.keyMaps.get(bucketName).has(objName)) {
                    return cb(null, { bucket: bucket.serialize() });
                }
                const objMD = metadata.keyMaps.get(bucketName).get(objName);
                return cb(null, {
                    bucket: bucket.serialize(),
                    obj: JSON.stringify(objMD, serializeCircJSON()),
                });
            });
        });
    },

    getObject: (bucketName, objName, params, log, cb) => {
        const err = metastore.error(metastore.getObject.name, objName);
        if (err) {
            return process.nextTick(() => cb(err));
        }
        return process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, err => {
                if (err) {
                    return cb(err);
                }
                if (params && params.versionId) {
                    // eslint-disable-next-line
                    objName = formatVersionKey(objName, params.versionId);
                }
                if (!metadata.keyMaps.has(bucketName)
                    || !metadata.keyMaps.get(bucketName).has(objName)) {
                    return cb(errors.NoSuchKey);
                }
                return cb(null, metadata.keyMaps.get(bucketName).get(objName));
            });
        });
    },

    deleteObject: (bucketName, objName, params, log, cb) => {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, err => {
                if (err) {
                    return cb(err);
                }
                if (!metadata.keyMaps.get(bucketName).has(objName)) {
                    return cb(errors.NoSuchKey);
                }
                if (params && params.versionId) {
                    const baseKey = inc(formatVersionKey(objName, ''));
                    const vobjName = formatVersionKey(objName,
                            params.versionId);
                    metadata.keyMaps.get(bucketName).delete(vobjName);
                    const mst = metadata.keyMaps.get(bucketName).get(objName);
                    if (mst.versionId === params.versionId) {
                        const keys = [];
                        metadata.keyMaps.get(bucketName).forEach((val, key) => {
                            if (key < baseKey && key > vobjName) {
                                keys.push(key);
                            }
                        });
                        if (keys.length === 0) {
                            metadata.keyMaps.get(bucketName).delete(objName);
                            return cb();
                        }
                        const key = keys.sort()[0];
                        const value = metadata.keyMaps.get(bucketName).get(key);
                        metadata.keyMaps.get(bucketName).set(objName, value);
                    }
                    return cb();
                }
                metadata.keyMaps.get(bucketName).delete(objName);
                return cb();
            });
        });
    },

    _hasDeleteMarker(key, keyMap) {
        const objectMD = keyMap.get(key);
        if (objectMD['x-amz-delete-marker'] !== undefined) {
            return (objectMD['x-amz-delete-marker'] === true);
        }
        return false;
    },

    listObject(bucketName, params, log, cb) {
        process.nextTick(() => {
            const {
                prefix,
                marker,
                delimiter,
                maxKeys,
                continuationToken,
                startAfter,
            } = params;
            if (prefix && typeof prefix !== 'string') {
                return cb(errors.InvalidArgument);
            }

            if (marker && typeof marker !== 'string') {
                return cb(errors.InvalidArgument);
            }

            if (delimiter && typeof delimiter !== 'string') {
                return cb(errors.InvalidArgument);
            }

            if (maxKeys && typeof maxKeys !== 'number') {
                return cb(errors.InvalidArgument);
            }

            if (continuationToken && typeof continuationToken !== 'string') {
                return cb(errors.InvalidArgument);
            }

            if (startAfter && typeof startAfter !== 'string') {
                return cb(errors.InvalidArgument);
            }

            // If paramMaxKeys is undefined, the default parameter will set it.
            // However, if it is null, the default parameter will not set it.
            let numKeys = maxKeys;
            if (numKeys === null) {
                numKeys = defaultMaxKeys;
            }

            if (!metadata.keyMaps.has(bucketName)) {
                return cb(errors.NoSuchBucket);
            }

            // If marker specified, edit the keys array so it
            // only contains keys that occur alphabetically after the marker
            const listingType = params.listingType;
            const extension = new list[listingType](params, log);
            const listingParams = extension.genMDParams();

            const keys = [];
            metadata.keyMaps.get(bucketName).forEach((val, key) => {
                if (listingParams.gt && listingParams.gt >= key) {
                    return null;
                }
                if (listingParams.gte && listingParams.gte > key) {
                    return null;
                }
                if (listingParams.lt && key >= listingParams.lt) {
                    return null;
                }
                if (listingParams.lte && key > listingParams.lte) {
                    return null;
                }
                return keys.push(key);
            });
            keys.sort();

            // Iterate through keys array and filter keys containing
            // delimiter into response.CommonPrefixes and filter remaining
            // keys into response.Contents
            for (let i = 0; i < keys.length; ++i) {
                const currentKey = keys[i];
                // Do not list object with delete markers
                if (this._hasDeleteMarker(currentKey,
                    metadata.keyMaps.get(bucketName))) {
                    continue;
                }
                const objMD = metadata.keyMaps.get(bucketName).get(currentKey);
                const value = JSON.stringify(objMD, serializeCircJSON());
                const obj = {
                    key: currentKey,
                    value,
                };
                // calling Ext.filter(obj) adds the obj to the Ext result if
                // not filtered.
                // Also, Ext.filter returns false when hit max keys.
                // What a nifty function!
                if (extension.filter(obj) < 0) {
                    break;
                }
            }
            return cb(null, extension.result());
        });
    },

    listMultipartUploads(bucketName, listingParams, log, cb) {
        process.nextTick(() => {
            metastore.getBucketAttributes(bucketName, log, (err, bucket) => {
                if (bucket === undefined) {
                    // no on going multipart uploads, return empty listing
                    return cb(null, {
                        IsTruncated: false,
                        NextMarker: undefined,
                        MaxKeys: 0,
                    });
                }
                return getMultipartUploadListing(bucket, listingParams, cb);
            });
        });
    },
};

module.exports = metastore;
<|endoftext|>


<commit_before>'use strict';

/* eslint-disable no-console */

/**
 * ConsoleLogger.js -- Provides basic logging functionality with all log messages posting to `console.log`.
 * Trace will also output "console.trace()" after the log message is posted.
 */
export default class ConsoleLogger
{
   /**
    * Post debug message.
    */
   debug()
   {
      let message;

      // Attempt to convert the arguments to a string.
      try { message = JSON.stringify(arguments); }
      catch(ignore) { /* ignore */}

      if (message !== undefined) { console.log(`Debug: ${message}`); }
   }

   /**
    * Post error message.
    */
   error()
   {
      let message;

      // Attempt to convert the arguments to a string.
      try { message = JSON.stringify(arguments); }
      catch(ignore) { /* ignore */}

      if (message !== undefined) { console.log(`Error: ${message}`); }
   }

   /**
    * Post fatal message.
    */
   fatal()
   {
      let message;

      // Attempt to convert the arguments to a string.
      try { message = JSON.stringify(arguments); }
      catch(ignore) { /* ignore */}

      if (message !== undefined) { console.log(`Fatal: ${message}`); }
   }

   /**
    * Post info message.
    */
   info()
   {
      let message;

      // Attempt to convert the arguments to a string.
      try { message = JSON.stringify(arguments); }
      catch(ignore) { /* ignore */}

      if (message !== undefined) { console.log(`Info: ${message}`); }
   }

   /**
    * Post trace message.
    */
   trace()
   {
      let message;

      // Attempt to convert the arguments to a string.
      try { message = JSON.stringify(arguments); }
      catch(ignore) { /* ignore */}

      if (message !== undefined) { console.log(`Trace: ${message}`); console.trace(); }
   }

   /**
    * Post warn message.
    */
   warn()
   {
      let message;

      // Attempt to convert the arguments to a string.
      try { message = JSON.stringify(arguments); }
      catch(ignore) { /* ignore */}

      if (message !== undefined) { console.log(`Warn: ${message}`); }
   }
}<commit_msg>Added ability to set custom serializer to ConsoleLogger.<commit_after>'use strict';

/* eslint-disable no-console */

/**
 * ConsoleLogger.js -- Provides basic logging functionality with all log messages posting to `console.log`.
 * Trace will also output "console.trace()" after the log message is posted.
 */
export default class ConsoleLogger
{
   constructor(serializer = JSON)
   {
      this.setSerializer(serializer);
   }

   /**
    * Post debug message.
    */
   debug()
   {
      let message;

      // Attempt to convert the arguments to a string.
      try { message = this._serializer.stringify(arguments); }
      catch(ignore) { /* ignore */}

      if (message !== undefined) { console.log(`Debug: ${message}`); }
   }

   /**
    * Post error message.
    */
   error()
   {
      let message;

      // Attempt to convert the arguments to a string.
      try { message = this._serializer.stringify(arguments); }
      catch(ignore) { /* ignore */}

      if (message !== undefined) { console.log(`Error: ${message}`); }
   }

   /**
    * Post fatal message.
    */
   fatal()
   {
      let message;

      // Attempt to convert the arguments to a string.
      try { message = this._serializer.stringify(arguments); }
      catch(ignore) { /* ignore */}

      if (message !== undefined) { console.log(`Fatal: ${message}`); }
   }

   /**
    * Returns the serializer.
    *
    * @returns {Object|*}
    */
   getSerializer()
   {
      return this._serializer;
   }

   /**
    * Post info message.
    */
   info()
   {
      let message;

      // Attempt to convert the arguments to a string.
      try { message = this._serializer.stringify(arguments); }
      catch(ignore) { /* ignore */}

      if (message !== undefined) { console.log(`Info: ${message}`); }
   }

   /**
    * Sets the serializer which must conform to the JSON API.
    *
    * @param {Object}   serializer - An instance of a JSON like serializer.
    */
   setSerializer(serializer)
   {
      if (typeof serializer !== 'object' || typeof serializer.stringify !== 'function' ||
       typeof serializer.parse !== 'function')
      {
         throw new TypeError('setSerializer - `serializer` does not conform to the JSON API.');
      }

      this._serializer = serializer;
   }

   /**
    * Post trace message.
    */
   trace()
   {
      let message;

      // Attempt to convert the arguments to a string.
      try { message = this._serializer.stringify(arguments); }
      catch(ignore) { /* ignore */}

      if (message !== undefined) { console.log(`Trace: ${message}`); console.trace(); }
   }

   /**
    * Post warn message.
    */
   warn()
   {
      let message;

      // Attempt to convert the arguments to a string.
      try { message = this._serializer.stringify(arguments); }
      catch(ignore) { /* ignore */}

      if (message !== undefined) { console.log(`Warn: ${message}`); }
   }
}<|endoftext|>


<commit_before>var recursive=require('recursive-readdir')
var elasticsearch = require('elasticsearch');
import * as _ from 'lodash'
var path=require('path')
import * as fs from 'fs'
import {XMLHttpRequest as xmhtrq} from 'xmlhttprequest';
var AgentKeepAlive = require('agentkeepalive');

// =====================================================================================
// FUNCTIONS TO GET CONTENT INTO THE INDEX
// recursively looks through a given folder to upload eupmc_result.json contents
// to elastic search
var ESClient = function () {
	var client = new elasticsearch.Client({
		host: Meteor.settings.elastichost+':'+Meteor.settings.elasticport,
    createNodeAgent(connection, config) {
	    return new AgentKeepAlive(connection.makeAgentConfig(config));
    }
	})
	return client
}

var uploadJSONFileToES =  function(file, index, type, client, cprojectID) {
	fs.readFile(file, function (err, data) {
		document = JSON.parse(data)
		document.cprojectID = cprojectID
		client.create({
			index: index,
			type: type,
			body: document
		})
	})
}

var uploadXMLFileToES = function(file, index, type, client, cprojectID, cb) {
	fs.readFile(file, function (err, data) {
		client.create({
			index: index,
			type: type,
			body: {
				"fulltext": data.toString('utf8'),
				"cprojectID": cprojectID
			}
		}, cb)
	})
}

var loadEuPMCFullTexts = function(folder, cb) {
	var client = ESClient()
	console.log("reading fulltexts from disk")
	recursive(folder, function(err, files) {
		var done = _.after(files.length-1, function() { cb()
			console.log("done all loading of files")
		})
		files.forEach(function (file) {
			if(path.basename(file)=="fulltext.xml") {
				var cprojectID = path.basename(path.dirname(file))
				console.log("uploading fulltext from CProject: " + cprojectID)
				uploadXMLFileToES(file, 'fulltext', 'unstructured', client, cprojectID, done)
			}
		})
	})
}

var indexEuPMCMetadata = function(folder) {
	var client = ESClient()
	console.log(folder)
	recursive(folder, function(err, files) {
		files.forEach(function (file) {
	    if(path.basename(file)=="eupmc_result.json") {
				cprojectID = path.basename(path.dirname(file))
				//console.log("Uploading file with cprojectID: " + cprojectID)
	    	uploadJSONFileToES(file, 'metadata', 'eupmc', client, cprojectID)
	    }
	  })
	})
}

var indexMetadata = function(dailyset) {
	indexEuPMCMetadata(Meteor.settings.storedir + '/' + dailyset)
  // for every metadata record, use the folder name (the URL uid) as the _id for the record
  //bulkload(metadata,'/catalogue/'+dailyset,true);
  var mdwithflattext = [];
  //var mdwithstructuredtext = [];
  // for every metadata record look in its folder and see if there is a fulltext.txt
  //for ( var m in metadata) {
    // look for the fulltext.txt file and read in the content of it to metadata[m].text
    // if the fulltext was found, then mdwithflattext.push(metadata[m])
  //}
  // if (mdwithflattext.length > 0) bulkload(mdwithflattext,'/flat/'+dailyset,true);
  // if norma processing is enabled, and structured fulltext can also be found in a scholarly.html file,
  // then the loop above could also popualte the mdwithstructuredtext array, and that too could be bulkloaded to '/structured/'+dailyset
}

var query = function(qry,from,size,set,index) {
  if (from === undefined) from = 0;
  if (size === undefined) size = 100;
  if (index === undefined) index = 'facts';
  // filter queries are much faster, but cannot get highlight matches out of them - so use standard queries
  // https://www.elastic.co/guide/en/elasticsearch/reference/1.4/search-request-highlighting.html
  var qr = {
    query: {},
    from: from,
    size:size
  }
  if (index !== 'facts') {
    qr.highlight = {
      fields: {
        text:{
          fragment_size:200,
          number_of_fragments:200 // this is a max
        }
      }
    }
  }
  if (qry.indexOf('/') === 0) {
    // https://www.elastic.co/guide/en/elasticsearch/reference/1.4/query-dsl-regexp-filter.html
    qr.query.regexp = {text:qry};
  } else if (typeof qry === 'object') {
    // allows to pass any query in. There are many highly customisable query types that ES could handle,
    // and these could be passed straight in from dicts with query pointing to full query objects (if they are in json this would be easy)
    // read ES docs https://www.elastic.co/guide/en/elasticsearch/reference/1.4/query-dsl.html and/or ask MM
    qr.query = qry;
  } else {
    // https://www.elastic.co/guide/en/elasticsearch/reference/1.4/query-dsl-term-filter.html
    qr.query.term = {text:qry};
  }
  var qrl = Meteor.settings.indexurl + '/' + index + '/';
  if (set) qrl += set + '/';
  qrl += '_search';
  return Meteor.http.call('POST',qrl,{data:qr}).data;
}

var bulkload = function(records,route,create) {
  if (create) {
    // delete the index at the route it if exists
    // create the index with mapping
  }
	var bulk = '';
	for ( var i in records ) {
		if ( records[i]._id === undefined ) {
  		bulk +=  '{create:{}}\n'; // TODO check this is correct syntax for ES bulk create
    } else {
  		bulk +=  '{index:{_id:"' + records[i]._id + '"}}\n';
    }
		bulk += JSON.stringify( records[i] ) + '\n';
	}
	var frl = Meteor.settings.indexurl + route + '/_bulk';
	var xhr = new xmhtrq();
	xhr.open('POST', frl, true);
	xhr.send(bulk);
	console.log(records.length + ' records sent to ' + route);
}

module.exports.indexMetadata = indexMetadata
module.exports.bulkload = bulkload
module.exports.loadEuPMCFullTexts = loadEuPMCFullTexts
module.exports.ESClient = ESClient
<commit_msg>make done func run everytime<commit_after>var recursive=require('recursive-readdir')
var elasticsearch = require('elasticsearch');
import * as _ from 'lodash'
var path=require('path')
import * as fs from 'fs'
import {XMLHttpRequest as xmhtrq} from 'xmlhttprequest';
var AgentKeepAlive = require('agentkeepalive');

// =====================================================================================
// FUNCTIONS TO GET CONTENT INTO THE INDEX
// recursively looks through a given folder to upload eupmc_result.json contents
// to elastic search
var ESClient = function () {
	var client = new elasticsearch.Client({
		host: Meteor.settings.elastichost+':'+Meteor.settings.elasticport,
    createNodeAgent(connection, config) {
	    return new AgentKeepAlive(connection.makeAgentConfig(config));
    }
	})
	return client
}

var uploadJSONFileToES =  function(file, index, type, client, cprojectID) {
	fs.readFile(file, function (err, data) {
		document = JSON.parse(data)
		document.cprojectID = cprojectID
		client.create({
			index: index,
			type: type,
			body: document
		})
	})
}

var uploadXMLFileToES = function(file, index, type, client, cprojectID, cb) {
	fs.readFile(file, function (err, data) {
		client.create({
			index: index,
			type: type,
			body: {
				"fulltext": data.toString('utf8'),
				"cprojectID": cprojectID
			}
		}, cb)
	})
}

var loadEuPMCFullTexts = function(folder, cb) {
	var client = ESClient()
	console.log("reading fulltexts from disk")
	recursive(folder, function(err, files) {
		var done = _.after(0, function() { cb()
			console.log("done all loading of files")
		})
		files.forEach(function (file) {
			if(path.basename(file)=="fulltext.xml") {
				var cprojectID = path.basename(path.dirname(file))
				console.log("uploading fulltext from CProject: " + cprojectID)
				uploadXMLFileToES(file, 'fulltext', 'unstructured', client, cprojectID, done)
			}
		})
	})
}

var indexEuPMCMetadata = function(folder) {
	var client = ESClient()
	console.log(folder)
	recursive(folder, function(err, files) {
		files.forEach(function (file) {
	    if(path.basename(file)=="eupmc_result.json") {
				cprojectID = path.basename(path.dirname(file))
				//console.log("Uploading file with cprojectID: " + cprojectID)
	    	uploadJSONFileToES(file, 'metadata', 'eupmc', client, cprojectID)
	    }
	  })
	})
}

var indexMetadata = function(dailyset) {
	indexEuPMCMetadata(Meteor.settings.storedir + '/' + dailyset)
  // for every metadata record, use the folder name (the URL uid) as the _id for the record
  //bulkload(metadata,'/catalogue/'+dailyset,true);
  var mdwithflattext = [];
  //var mdwithstructuredtext = [];
  // for every metadata record look in its folder and see if there is a fulltext.txt
  //for ( var m in metadata) {
    // look for the fulltext.txt file and read in the content of it to metadata[m].text
    // if the fulltext was found, then mdwithflattext.push(metadata[m])
  //}
  // if (mdwithflattext.length > 0) bulkload(mdwithflattext,'/flat/'+dailyset,true);
  // if norma processing is enabled, and structured fulltext can also be found in a scholarly.html file,
  // then the loop above could also popualte the mdwithstructuredtext array, and that too could be bulkloaded to '/structured/'+dailyset
}

var query = function(qry,from,size,set,index) {
  if (from === undefined) from = 0;
  if (size === undefined) size = 100;
  if (index === undefined) index = 'facts';
  // filter queries are much faster, but cannot get highlight matches out of them - so use standard queries
  // https://www.elastic.co/guide/en/elasticsearch/reference/1.4/search-request-highlighting.html
  var qr = {
    query: {},
    from: from,
    size:size
  }
  if (index !== 'facts') {
    qr.highlight = {
      fields: {
        text:{
          fragment_size:200,
          number_of_fragments:200 // this is a max
        }
      }
    }
  }
  if (qry.indexOf('/') === 0) {
    // https://www.elastic.co/guide/en/elasticsearch/reference/1.4/query-dsl-regexp-filter.html
    qr.query.regexp = {text:qry};
  } else if (typeof qry === 'object') {
    // allows to pass any query in. There are many highly customisable query types that ES could handle,
    // and these could be passed straight in from dicts with query pointing to full query objects (if they are in json this would be easy)
    // read ES docs https://www.elastic.co/guide/en/elasticsearch/reference/1.4/query-dsl.html and/or ask MM
    qr.query = qry;
  } else {
    // https://www.elastic.co/guide/en/elasticsearch/reference/1.4/query-dsl-term-filter.html
    qr.query.term = {text:qry};
  }
  var qrl = Meteor.settings.indexurl + '/' + index + '/';
  if (set) qrl += set + '/';
  qrl += '_search';
  return Meteor.http.call('POST',qrl,{data:qr}).data;
}

var bulkload = function(records,route,create) {
  if (create) {
    // delete the index at the route it if exists
    // create the index with mapping
  }
	var bulk = '';
	for ( var i in records ) {
		if ( records[i]._id === undefined ) {
  		bulk +=  '{create:{}}\n'; // TODO check this is correct syntax for ES bulk create
    } else {
  		bulk +=  '{index:{_id:"' + records[i]._id + '"}}\n';
    }
		bulk += JSON.stringify( records[i] ) + '\n';
	}
	var frl = Meteor.settings.indexurl + route + '/_bulk';
	var xhr = new xmhtrq();
	xhr.open('POST', frl, true);
	xhr.send(bulk);
	console.log(records.length + ' records sent to ' + route);
}

module.exports.indexMetadata = indexMetadata
module.exports.bulkload = bulkload
module.exports.loadEuPMCFullTexts = loadEuPMCFullTexts
module.exports.ESClient = ESClient
<|endoftext|>


<commit_before>import classNames from 'classnames';
import React, {PropTypes} from 'react/addons';

import Modal from '../Modal/Modal';
import Util from '../Util/Util';

export default class Confirm extends React.Component {
  getButtons() {
    let disabledConfig = {disabled: this.props.disabled};

    let leftButtonClassName = classNames(
      this.props.leftButtonClassName, disabledConfig
    );
    let rightButtonClassName = classNames(
      this.props.rightButtonClassName, disabledConfig
    );

    let extraAttributes = {};
    if (this.props.disabled) {
      extraAttributes = disabledConfig;
    }

    return (

        <button
          className={leftButtonClassName}
          onClick={this.props.leftButtonCallback}
          {...extraAttributes}>
          {this.props.leftButtonText}
        </button>
        <button
          className={rightButtonClassName}
          onClick={this.props.rightButtonCallback}
          {...extraAttributes}>
          {this.props.rightButtonText}
        </button>

    );
  }

  render() {
    let props = Util.exclude(this.props, 'children', 'disabled',
      'leftButtonText', 'leftButtonClassName', 'leftButtonCallback',
      'rightButtonText', 'rightButtonClassName', 'rightButtonCallback'
      );

    return (
      <Modal
        closeByBackdropClick={!this.props.disabled}
        modalClass="modal confirm-modal"
        showCloseButton={false}
        showFooter={true}
        footer={this.getButtons()}
        titleClass="modal-header-title text-align-center flush-top
          flush-bottom"
        innerBodyClass="modal-content-inner container container-pod
          container-pod-short flex-container-col flush-bottom"
        {...props}
        >
        {this.props.children}
      </Modal>
    );
  }
}

Confirm.defaultProps = {
  open: true,
  disabled: false,
  onClose: function () {},

  // Left button properties
  leftButtonText: 'Cancel',
  leftButtonClassName: 'button',
  leftButtonCallback: function () {},
  // Right button properties
  rightButtonText: 'Confirm',
  rightButtonClassName: 'button button-primary',
  rightButtonCallback: function () {}
};

Confirm.propTypes = {
  children: React.PropTypes.element.isRequired,

  open: PropTypes.bool.isRequired,
  disabled: PropTypes.bool,
  // This will be triggered by backdrop click
  onClose: PropTypes.func,

  // Left button properties
  leftButtonText: PropTypes.string.isRequired,
  leftButtonClassName: PropTypes.string,
  leftButtonCallback: PropTypes.func.isRequired,
  // Right button properties
  rightButtonText: PropTypes.string.isRequired,
  rightButtonClassName: PropTypes.string,
  rightButtonCallback: PropTypes.func.isRequired
};
<commit_msg>fixed spacing<commit_after>import classNames from 'classnames';
import React, {PropTypes} from 'react/addons';

import Modal from '../Modal/Modal';
import Util from '../Util/Util';

export default class Confirm extends React.Component {
  getButtons() {
    let disabledConfig = {disabled: this.props.disabled};

    let leftButtonClassName = classNames(
      this.props.leftButtonClassName, disabledConfig
    );
    let rightButtonClassName = classNames(
      this.props.rightButtonClassName, disabledConfig
    );

    let extraAttributes = {};
    if (this.props.disabled) {
      extraAttributes = disabledConfig;
    }

    return (

        <button
          className={leftButtonClassName}
          onClick={this.props.leftButtonCallback}
          {...extraAttributes}>
          {this.props.leftButtonText}
        </button>
        <button
          className={rightButtonClassName}
          onClick={this.props.rightButtonCallback}
          {...extraAttributes}>
          {this.props.rightButtonText}
        </button>

    );
  }

  render() {
    let props = Util.exclude(this.props, 'children', 'disabled',
      'leftButtonText', 'leftButtonClassName', 'leftButtonCallback',
      'rightButtonText', 'rightButtonClassName', 'rightButtonCallback'
    );

    return (
      <Modal
        closeByBackdropClick={!this.props.disabled}
        modalClass="modal confirm-modal"
        showCloseButton={false}
        showFooter={true}
        footer={this.getButtons()}
        titleClass="modal-header-title text-align-center flush-top
          flush-bottom"
        innerBodyClass="modal-content-inner container container-pod
          container-pod-short flex-container-col flush-bottom"
        {...props}
        >
        {this.props.children}
      </Modal>
    );
  }
}

Confirm.defaultProps = {
  open: true,
  disabled: false,
  onClose: function () {},

  // Left button properties
  leftButtonText: 'Cancel',
  leftButtonClassName: 'button',
  leftButtonCallback: function () {},
  // Right button properties
  rightButtonText: 'Confirm',
  rightButtonClassName: 'button button-primary',
  rightButtonCallback: function () {}
};

Confirm.propTypes = {
  children: React.PropTypes.element.isRequired,

  open: PropTypes.bool.isRequired,
  disabled: PropTypes.bool,
  // This will be triggered by backdrop click
  onClose: PropTypes.func,

  // Left button properties
  leftButtonText: PropTypes.string.isRequired,
  leftButtonClassName: PropTypes.string,
  leftButtonCallback: PropTypes.func.isRequired,
  // Right button properties
  rightButtonText: PropTypes.string.isRequired,
  rightButtonClassName: PropTypes.string,
  rightButtonCallback: PropTypes.func.isRequired
};
<|endoftext|>


<commit_before>/**
  * Copyright (C) 2016 yanni4night.com
  * read-transformer.js
  *
  * changelog
  * 2016-06-19[08:41:50]:revised
  *
  * @author yanni4night@gmail.com
  * @version 0.1.0
  * @since 0.1.0
  */

'use strict';

const Transformer = require('./transformer');

const {
    isBinary,
    R,
    NAMESPACE
} = require('../config');
const extend = require('lodash/extend');

class ReadTransformer extends Transformer {
    _transform(seed) {
        const {
            file,
            content
        } = seed;
        if (content !== null && content !== undefined) {
            return Promise.resolve(extend({}, seed));
        } else {
            return R(file, isBinary(file)).then(content => {
                return extend({}, seed, {
                    content,
                    moduleId: `${NAMESPACE}/${file}`
                });
            });
        }
    }
}

module.exports = ReadTransformer;<commit_msg>use isNil of lodash<commit_after>/**
  * Copyright (C) 2016 yanni4night.com
  * read-transformer.js
  *
  * changelog
  * 2016-06-19[08:41:50]:revised
  *
  * @author yanni4night@gmail.com
  * @version 0.1.0
  * @since 0.1.0
  */

'use strict';

const Transformer = require('./transformer');

const {
    isBinary,
    R,
    NAMESPACE
} = require('../config');
const extend = require('lodash/extend');
const isNil = require('lodash/isNil');

class ReadTransformer extends Transformer {
    _transform(seed) {
        const {
            file,
            content
        } = seed;
        // use cache if possible
        if (!isNil(content)) {
            return Promise.resolve(extend({}, seed));
        } else {
            return R(file, isBinary(file)).then(content => {
                return extend({}, seed, {
                    content,
                    moduleId: `${NAMESPACE}/${file}`
                });
            });
        }
    }
}

module.exports = ReadTransformer;<|endoftext|>