Commit 94ab7fc2 authored by Kelly Davis's avatar Kelly Davis

Fixed #255

parent 8429f9b6
......@@ -3148,8 +3148,7 @@
"through": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=",
"dev": true
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
},
"through2": {
"version": "2.0.3",
......
......@@ -6,7 +6,6 @@
"dependencies": {
"aws-sdk": "^2.67.0",
"bcrypt": "^1.0.2",
"better-queue": "^3.8.5",
"bluebird": "^3.5.0",
"ff": "0.2.1",
"find-remove": "^1.0.1",
......
......@@ -4,11 +4,10 @@ import { getFileExt } from './utility';
const MemoryStream = require('memorystream');
const path = require('path');
const Promise = require('bluebird');
const Queue = require('better-queue');
const AWS = require('./aws');
const BATCH_SIZE = 5;
const REFRESH_INTERVAL = 20000; // how often we refresh our S3 list
const REFRESH_INTERVAL = 10000; // how often we refresh our S3 list
const MP3_EXT = '.mp3';
const CONVERTABLE_EXTS = ['.ogg', '.m4a'];
const CONFIG_PATH = path.resolve(__dirname, '../../..', 'config.json');
......@@ -19,18 +18,17 @@ export default class Files {
private s3: any;
private files: {
// fileGlob: [
// sentence: 'the text of the sentenct',
// exts : ['.txt', '.ogg', '.mp3', etc..]
// sentence: 'the text of the sentenct'
// ]
};
private paths: string[];
private mp3s: string[];
private continuationToken: string;
constructor() {
this.s3 = new AWS.S3();
this.files = {};
this.paths = [];
this.mp3s = [];
this.continuationToken = undefined;
this.init().then(() => {
setInterval(this.init.bind(this), REFRESH_INTERVAL);
});
......@@ -46,37 +44,17 @@ export default class Files {
/**
* Read a sentence in from s3.
*/
private process(data: any, cb: Function) {
let params = {Bucket: BUCKET_NAME, Key: data.path};
private getSentence(glob: string, key: string) {
let params = {Bucket: BUCKET_NAME, Key: key};
this.s3.getObject(params, (err: any, s3Data: any) => {
if (err) {
console.error('Could not read from s3', data.path, err);
return cb(err);
console.error('Could not read from s3', key, err);
delete this.files[glob];
return;
}
let sentence = s3Data.Body.toString();
this.files[data.glob].sentence = sentence;
cb(null);
});
}
private processBatch(batches: any[], cb: Function) {
map(this, this.process, batches)
.then(cb)
.error((err: any) => {
console.log('got an error pocessing the batches', err);
});
}
/**
* Make a list of mp3s so we can randomly choose one later.
*/
private generateMP3List() {
this.mp3s = [];
this.paths.forEach(glob => {
if (this.files[glob].exts.indexOf(MP3_EXT) !== -1) {
this.mp3s.push(glob);
}
this.files[glob].sentence = sentence;
});
}
......@@ -84,47 +62,29 @@ export default class Files {
* Load a list of files from S3.
*/
private init(): Promise<any> {
// Create our batch processor to help us read all sentences
// from the filesystem without overloading the server.
let batches = new Queue(this.processBatch.bind(this), { batchSize: BATCH_SIZE });
return new Promise((res: Function, rej: Function) => {
let searchParam = {Bucket: BUCKET_NAME, MaxKeys: 1000};
let searchParam = {Bucket: BUCKET_NAME, MaxKeys: 50, ContinuationToken: this.continuationToken};
let awsRequest = this.s3.listObjectsV2(searchParam);
awsRequest.on('success', (response) => {
this.continuationToken = response['data']['NextContinuationToken'];
let contents = response['data']['Contents'];
for (let i = 0; i < contents.length; i++) {
let key = contents[i].Key;
let glob = this.getGlob(key);
let ext = getFileExt(key);
// Ignore directories
if (!glob) {
// Ignore non-text files
if (ext !== '.txt') {
continue;
}
// Ignore non-text and non-mp3 files
if ((ext !== '.txt') && (ext !== MP3_EXT)) {
continue;
}
// Track file gobs and extensions of the voice clips.
// Track gobs and sentence of the voice clips.
if (!this.files[glob]) {
this.files[glob] = {
sentence: null,
exts: []
sentence: null
}
}
// Text files go into our batch processing queue for later reading.
if (ext === '.txt') {
batches.push({
path: key,
glob: glob
});
} else {
this.files[glob].exts.push(ext);
this.getSentence(glob, key);
}
}
});
......@@ -144,8 +104,6 @@ export default class Files {
return;
}
// Convert any files that haven't been converted to mp3 yet.
this.generateMP3List();
res();
});
......@@ -158,11 +116,11 @@ export default class Files {
*/
getRandomClip(uid: string): Promise<string[2]> {
// Make sure we have at least 1 file to choose from.
if (this.mp3s.length === 0) {
if (this.paths.length === 0) {
return Promise.reject('No files.');
}
let items = this.mp3s.filter(glob => !glob.includes(uid));
let items = this.paths.filter(glob => !glob.includes(uid));
// Make sure we have at least 1 file to choose from that's not from us.
if (items.length === 0) {
return Promise.reject('No files not from us.');
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment