Fixed #255

package-lock.json

@@ -3148,8 +3148,7 @@
     "through": {
       "version": "2.3.8",
       "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
-      "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=",
-      "dev": true
+      "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
     },
     "through2": {
       "version": "2.0.3",

package.json

@@ -6,7 +6,6 @@
   "dependencies": {
     "aws-sdk": "^2.67.0",
     "bcrypt": "^1.0.2",
-    "better-queue": "^3.8.5",
     "bluebird": "^3.5.0",
     "ff": "0.2.1",
     "find-remove": "^1.0.1",

server/src/lib/files.ts

@@ -4,11 +4,10 @@ import { getFileExt } from './utility';
 const MemoryStream = require('memorystream');
 const path = require('path');
 const Promise = require('bluebird');
-const Queue = require('better-queue');
 const AWS = require('./aws');
 
 const BATCH_SIZE = 5;
-const REFRESH_INTERVAL = 20000; // how often we refresh our S3 list
+const REFRESH_INTERVAL = 10000; // how often we refresh our S3 list
 const MP3_EXT = '.mp3';
 const CONVERTABLE_EXTS = ['.ogg', '.m4a'];
 const CONFIG_PATH = path.resolve(__dirname, '../../..', 'config.json');
@@ -19,18 +18,17 @@ export default class Files {
   private s3: any;
   private files: {
     // fileGlob: [
-    //   sentence: 'the text of the sentenct',
-    //   exts : ['.txt', '.ogg', '.mp3', etc..]
+    //   sentence: 'the text of the sentenct'
     // ]
   };
   private paths: string[];
-  private mp3s: string[];
+  private continuationToken: string;
 
   constructor() {
     this.s3 = new AWS.S3();
     this.files = {};
     this.paths = [];
-    this.mp3s = [];
+    this.continuationToken = undefined;
     this.init().then(() => {
       setInterval(this.init.bind(this), REFRESH_INTERVAL);
     });
@@ -46,37 +44,17 @@ export default class Files {
   /**
    * Read a sentence in from s3.
    */
-  private process(data: any, cb: Function) {
-    let params = {Bucket: BUCKET_NAME, Key: data.path};
+  private getSentence(glob: string, key: string) {
+    let params = {Bucket: BUCKET_NAME, Key: key};
     this.s3.getObject(params, (err: any, s3Data: any) => {
       if (err) {
-        console.error('Could not read from s3', data.path, err);
-        return cb(err);
+        console.error('Could not read from s3', key, err);
+        delete this.files[glob];
+        return;
       }
 
       let sentence = s3Data.Body.toString();
-      this.files[data.glob].sentence = sentence;
-      cb(null);
-    });
-  }
-
-  private processBatch(batches: any[], cb: Function) {
-    map(this, this.process, batches)
-      .then(cb)
-      .error((err: any) => {
-        console.log('got an error pocessing the batches', err);
-      });
-  }
-
-  /**
-   * Make a list of mp3s so we can randomly choose one later.
-   */
-  private generateMP3List() {
-    this.mp3s = [];
-    this.paths.forEach(glob => {
-      if (this.files[glob].exts.indexOf(MP3_EXT) !== -1) {
-        this.mp3s.push(glob);
-      }
+      this.files[glob].sentence = sentence;
     });
   }
 
@@ -84,47 +62,29 @@ export default class Files {
    * Load a list of files from S3.
    */
   private init(): Promise<any> {
-    // Create our batch processor to help us read all sentences
-    // from the filesystem without overloading the server.
-    let batches = new Queue(this.processBatch.bind(this), { batchSize: BATCH_SIZE });
-
     return new Promise((res: Function, rej: Function) => {
-      let searchParam = {Bucket: BUCKET_NAME, MaxKeys: 1000};
+      let searchParam = {Bucket: BUCKET_NAME, MaxKeys: 50, ContinuationToken: this.continuationToken};
       let awsRequest = this.s3.listObjectsV2(searchParam);
 
       awsRequest.on('success', (response) => {
+        this.continuationToken = response['data']['NextContinuationToken'];
         let contents = response['data']['Contents'];
         for (let i = 0; i < contents.length; i++) {
           let key = contents[i].Key;
           let glob = this.getGlob(key);
           let ext = getFileExt(key);
 
-          // Ignore directories
-          if (!glob) {
+          // Ignore non-text files
+          if (ext !== '.txt') {
             continue;
           }
 
-          // Ignore non-text and non-mp3 files
-          if ((ext !== '.txt') && (ext !== MP3_EXT)) {
-            continue;
-          }
-
-          // Track file gobs and extensions of the voice clips.
+          // Track gobs and sentence of the voice clips.
           if (!this.files[glob]) {
             this.files[glob] = {
-              sentence: null,
-              exts: []
+              sentence: null
             }
-          }
-
-          // Text files go into our batch processing queue for later reading.
-          if (ext === '.txt') {
-            batches.push({
-              path: key,
-              glob: glob
-            });
-          } else {
-            this.files[glob].exts.push(ext);
+            this.getSentence(glob, key);
           }
         }
       });
@@ -144,8 +104,6 @@ export default class Files {
           return;
         }
 
-        // Convert any files that haven't been converted to mp3 yet.
-       this.generateMP3List();
        res();
       });
 
@@ -158,11 +116,11 @@ export default class Files {
    */
   getRandomClip(uid: string): Promise<string[2]> {
     // Make sure we have at least 1 file to choose from.
-    if (this.mp3s.length === 0) {
+    if (this.paths.length === 0) {
       return Promise.reject('No files.');
     }
 
-    let items = this.mp3s.filter(glob => !glob.includes(uid));
+    let items = this.paths.filter(glob => !glob.includes(uid));
     // Make sure we have at least 1 file to choose from that's not from us.
     if (items.length === 0) {
       return Promise.reject('No files not from us.');