Skip to content

GitLab

Commit 255a0874 authored by Michael Henretty's avatar Michael Henretty Committed by GitHub
Browse files

Add linter to build process, fixes #444 (#455) 

* add basic linter (using prettier) to the build process
parent 16d3bfdd
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1279 additions and 377 deletions
.eslintrc.json 0 → 100644
View file @ 255a0874
{
"env": {
},
"parser": "typescript-eslint-parser",
"parserOptions": {
"ecmaFeatures": {
"jsx": true
},
"sourceType": "module"
},
"plugins": [
"prettier"
],
"rules": {
"prettier/prettier": "error"
}
}
.prettierrc 0 → 100644
View file @ 255a0874
{
"parser": "typescript",
"jsxBracketSameLine": true,
"singleQuote": true,
"trailingComma": "es5"
}
gulpfile.js
View file @ 255a0874
'use strict';
const CWD = process.cwd() + '/';
const APP_NAME = 'common-voice';
const TS_CONFIG = 'tsconfig.json';
const TS_GLOB = 'src/**/*';
const DIR_CLIENT = './web/';
const DIR_SERVER = './server/';
const TS_GLOB = '**/*.ts*';
const DIR_CLIENT = CWD + 'web/';
const DIR_CLIENT_SRC = DIR_CLIENT + 'src/';
const DIR_SERVER = CWD + 'server/';
const DIR_SERVER_SRC = DIR_SERVER + 'src/';
const DIR_UPLOAD = DIR_SERVER + 'upload/';
const DIR_SERVER_JS = DIR_SERVER + 'js/';
const DIR_DIST = DIR_CLIENT + 'dist/';
const PATH_CSS = DIR_CLIENT + 'css/*.css';
const PATH_TS = DIR_CLIENT + TS_GLOB;
const PATH_TS_SERVER = DIR_SERVER + TS_GLOB;
const PATH_TS = DIR_CLIENT_SRC + TS_GLOB;
const PATH_TS_CONFIG = DIR_CLIENT + TS_CONFIG;
const PATH_TS_SERVER = DIR_SERVER_SRC + TS_GLOB;
const PATH_TS_CONFIG_SERVER = DIR_SERVER + TS_CONFIG;
const PATH_VENDOR = DIR_CLIENT + 'vendor/';
const RELOAD_DELAY = 2500;
const SERVER_SCRIPT = 'server/js/server.js'
const RELOAD_DELAY = 2500;
// Add gulp help functionality.
let gulp = require('gulp-help')(require('gulp'));
......@@ -23,10 +29,7 @@ let shell = require('gulp-shell');
let path = require('path');
let ts = require('gulp-typescript');
let insert = require('gulp-insert');
function compile(project) {
return project.src().pipe(project()).js;
}
let eslint = require('gulp-eslint');
function listen() {
require('gulp-nodemon')({
......@@ -63,6 +66,39 @@ function getVendorJS() {
}, '');
}
function compile(pathConfig, pathSrc) {
let project = ts.createProject(pathConfig);
// Always lint before compiling.
return lint(pathSrc)
.pipe(project())
.js;
}
function compileClient() {
let insert = require('gulp-insert');
let uglify = require('gulp-uglify');
let uglifyOptions = {
mangle: false,
compress: false,
output: {
beautify: true,
indent_level: 2,
semicolons: false
}
};
return compile(PATH_TS_CONFIG, PATH_TS)
.pipe(uglify(uglifyOptions))
.pipe(insert.prepend(getVendorJS()))
.pipe(gulp.dest(DIR_DIST));
}
function compileServer() {
return compile(PATH_TS_CONFIG_SERVER, PATH_TS_SERVER)
.pipe(gulp.dest(DIR_SERVER_JS));
}
function compileCSS() {
var postcss = require('gulp-postcss');
var cssnext = require('postcss-cssnext');
......@@ -83,32 +119,31 @@ function compileCSS() {
.pipe(gulp.dest(DIR_DIST));
}
function compileClient() {
let project = ts.createProject(DIR_CLIENT + TS_CONFIG);
let insert = require('gulp-insert');
let uglify = require('gulp-uglify');
let uglifyOptions = {
mangle: false,
compress: false,
output: {
beautify: true,
indent_level: 2,
semicolons: false
}
};
function lint(src) {
return gulp.src(src)
.pipe(eslint())
.pipe(eslint.format())
.pipe(eslint.failAfterError());
}
return compile(project)
.pipe(uglify(uglifyOptions))
.pipe(insert.prepend(getVendorJS()))
.pipe(gulp.dest(DIR_DIST));
function lintAll() {
return lint([PATH_TS, PATH_TS_SERVER]);
}
function compileServer() {
let project = ts.createProject(DIR_SERVER + TS_CONFIG);
return compile(project)
.pipe(gulp.dest(DIR_SERVER_JS));
function prettify(src) {
return gulp.src(src, { base: CWD })
.pipe(eslint({ fix: true }))
.pipe(gulp.dest(CWD));
}
function prettifyAll() {
return prettify([PATH_TS, PATH_TS_SERVER]);
}
gulp.task('lint', 'Perform style checks on all typescript code', lintAll);
gulp.task('prettify', 'Auto-format all typescript code', prettifyAll);
gulp.task('css', 'Minify CSS files', compileCSS);
gulp.task('ts', 'Compile typescript files into bundle.js', compileClient);
......
package-lock.json
View file @ 255a0874
This diff is collapsed.
package.json
View file @ 255a0874
......@@ -9,9 +9,12 @@
"aws-sdk": "^2.67.0",
"bcrypt": "^1.0.2",
"bluebird": "^3.5.0",
"eslint": "^4.7.2",
"eslint-plugin-prettier": "^2.3.1",
"ff": "0.2.1",
"glob": "7.1.1 ",
"gulp": "3.9.1",
"gulp-eslint": "^4.0.0",
"gulp-help": "^1.6.1",
"gulp-insert": "^0.5.0",
"gulp-shell": "0.6.3",
......@@ -23,12 +26,14 @@
"pm2": "^2.4.6",
"postcss-cssnext": "^3.0.2",
"preact": "^8.2.5",
"prettier": "^1.7.0",
"proxy-agent": "^2.0.0",
"random-js": "^1.0.8",
"readline": "^1.3.0",
"simple-git": "^1.72.0",
"stream-transcoder": "0.0.5",
"typescript": "^2.5.2",
"typescript-eslint-parser": "^8.0.0",
"walk": "^2.3.9"
},
"devDependencies": {
......
server/src/lib/api.ts
View file @ 255a0874
......@@ -12,7 +12,7 @@ const SENTENCE_FOLDER = '../../data/';
export default class API {
sentencesCache: String[];
webhook: WebHook;
randomEngine: any
randomEngine: any;
constructor() {
this.webhook = new WebHook();
......@@ -38,33 +38,39 @@ export default class API {
}
private getFilesInFolder(folderpath: string) {
return new Promise((res: (files: string[]) => void,
rej: (error: any) => void) => {
fs.readdir(folderpath, (err: any, files: string[]) => {
if (err) {
rej(err);
return;
}
res(files);
});
});
return new Promise(
(res: (files: string[]) => void, rej: (error: any) => void) => {
fs.readdir(folderpath, (err: any, files: string[]) => {
if (err) {
rej(err);
return;
}
res(files);
});
}
);
}
private getFileContents(filepath: string) {
return new Promise((res: (contents: string) => void,
rej: (error: any) => void) => {
fs.readFile(filepath, {
contents: 'utf8'
}, (err: any, data: Buffer) => {
if (err) {
rej(err);
return;
}
res(data.toString());
});
});
return new Promise(
(res: (contents: string) => void, rej: (error: any) => void) => {
fs.readFile(
filepath,
{
contents: 'utf8',
},
(err: any, data: Buffer) => {
if (err) {
rej(err);
return;
}
res(data.toString());
}
);
}
);
}
/**
* Is this request directed at the api?
......@@ -76,23 +82,21 @@ export default class API {
/**
* Give api response.
*/
handleRequest(request: http.IncomingMessage,
response: http.ServerResponse) {
handleRequest(request: http.IncomingMessage, response: http.ServerResponse) {
// Most often this will be a sentence request.
if (request.url.includes('/sentence')) {
let parts = request.url.split('/');
let index = parts.indexOf('sentence');
let count = parts[index + 1] && parseInt(parts[index + 1], 10);
this.returnRandomSentence(response, count);
// Webhooks from github.
// Webhooks from github.
} else if (this.webhook.isHookRequest(request)) {
this.webhook.handleWebhookRequest(request, response);
// Unrecognized requests get here.
// Unrecognized requests get here.
} else {
console.error('unrecongized api url', request.url);
respond(response, 'I\'m not sure what you want.', 404);
respond(response, "I'm not sure what you want.", 404);
}
}
......@@ -101,41 +105,44 @@ export default class API {
return Promise.resolve(this.sentencesCache);
}
return this.getFilesInFolder(this.getSentenceFolder())
.then((files: string[]) => {
return Promise.all(files.map(filename => {
// Only parse the top-level text files, not any sub folders.
if (filename.split('.').pop() !== 'txt') {
return null;
}
let filepath = path.join(this.getSentenceFolder(), filename);
return this.getFileContents(filepath);
}));
})
// Chop the array of content strings into an array of sentences.
.then((values: string[]) => {
let sentences: string[] = [];
let sentenceArrays = values.map(fileContents => {
if (!fileContents) {
return [];
}
// Remove any blank line sentences.
let fileSentences = fileContents.split('\n');
return fileSentences.filter(sentence => { return !!sentence; });
});
sentences = sentences.concat.apply(sentences, sentenceArrays);
console.log('sentences found', sentences.length);
this.sentencesCache = sentences;
})
.catch((err: any) => {
console.error('could not retrieve sentences', err);
});
return (
this.getFilesInFolder(this.getSentenceFolder())
.then((files: string[]) => {
return Promise.all(
files.map(filename => {
// Only parse the top-level text files, not any sub folders.
if (filename.split('.').pop() !== 'txt') {
return null;
}
let filepath = path.join(this.getSentenceFolder(), filename);
return this.getFileContents(filepath);
})
);
})
// Chop the array of content strings into an array of sentences.
.then((values: string[]) => {
let sentences: string[] = [];
let sentenceArrays = values.map(fileContents => {
if (!fileContents) {
return [];
}
// Remove any blank line sentences.
let fileSentences = fileContents.split('\n');
return fileSentences.filter(sentence => {
return !!sentence;
});
});
sentences = sentences.concat.apply(sentences, sentenceArrays);
console.log('sentences found', sentences.length);
this.sentencesCache = sentences;
})
.catch((err: any) => {
console.error('could not retrieve sentences', err);
})
);
}
/**
......@@ -144,13 +151,16 @@ export default class API {
returnRandomSentence(response: http.ServerResponse, count: number) {
count = count || 1;
this.getSentences().then((sentences: String[]) => {
return this.getRandomSentences(count);
}).then((randoms: string[]) => {
respond(response, randoms.join('\n'));
}).catch((err: any) => {
console.error('Could not load sentences', err);
respond(response, 'No sentences right now', 500);
});
this.getSentences()
.then((sentences: String[]) => {
return this.getRandomSentences(count);
})
.then((randoms: string[]) => {
respond(response, randoms.join('\n'));
})
.catch((err: any) => {
console.error('Could not load sentences', err);
respond(response, 'No sentences right now', 500);
});
}
}
server/src/lib/aws.ts
View file @ 255a0874
var AWS = require('aws-sdk');
if(process.env.HTTP_PROXY) {
if (process.env.HTTP_PROXY) {
var proxy = require('proxy-agent');
AWS.config.update({
httpOptions: { agent: proxy(process.env.HTTP_PROXY) }
httpOptions: { agent: proxy(process.env.HTTP_PROXY) },
});
}
......
server/src/lib/clip.ts
View file @ 255a0874
......@@ -16,7 +16,7 @@ const Transcoder = require('stream-transcoder');
const UPLOAD_PATH = path.resolve(__dirname, '../..', 'upload');
const CONFIG_PATH = path.resolve(__dirname, '../../..', 'config.json');
const ACCEPTED_EXT = [ '.mp3', '.ogg', '.webm', '.m4a' ];
const ACCEPTED_EXT = ['.mp3', '.ogg', '.webm', '.m4a'];
const DEFAULT_SALT = '8hd3e8sddFSdfj';
const config = require(CONFIG_PATH);
const salt = config.salt || DEFAULT_SALT;
......@@ -30,7 +30,7 @@ export default class Clip {
private files: Files;
constructor() {
this.s3 = new AWS.S3({signatureVersion: 'v4'});
this.s3 = new AWS.S3({ signatureVersion: 'v4' });
this.files = new Files();
}
......@@ -42,28 +42,38 @@ export default class Clip {
}
private hash(str: string): string {
return crypto.createHmac('sha256', salt).update(str).digest('hex');
return crypto
.createHmac('sha256', salt)
.update(str)
.digest('hex');
}
private streamAudio(request: http.IncomingMessage,
response: http.ServerResponse,
key: string): void {
private streamAudio(
request: http.IncomingMessage,
response: http.ServerResponse,
key: string
): void {
// Save the data locally, stream to client, remove local data (Performance?)
let tmpFilePath = path.join(UPLOAD_PATH, key);
let tmpFileDirectory = path.dirname(tmpFilePath);
let f = ff(() => {
mkdirp(tmpFileDirectory, f.wait());
}, () => {
let retrieveParam = {Bucket: BUCKET_NAME, Key: key};
let awsResult = this.s3.getObject(retrieveParam);
f.pass(awsResult);
}, (awsResult: any) => {
let tmpFile = fs.createWriteStream(tmpFilePath);
tmpFile = awsResult.createReadStream().pipe(tmpFile);
tmpFile.on('finish', f.wait());
}, () => {
ms.pipe(request, response, tmpFilePath);
}).onError((err: any) => {
let f = ff(
() => {
mkdirp(tmpFileDirectory, f.wait());
},
() => {
let retrieveParam = { Bucket: BUCKET_NAME, Key: key };
let awsResult = this.s3.getObject(retrieveParam);
f.pass(awsResult);
},
(awsResult: any) => {
let tmpFile = fs.createWriteStream(tmpFilePath);
tmpFile = awsResult.createReadStream().pipe(tmpFile);
tmpFile.on('finish', f.wait());
},
() => {
ms.pipe(request, response, tmpFilePath);
}
).onError((err: any) => {
console.error('streaming audio error', err, err.stack);
respond(response, 'Server error, could not fetch audio data.', 500);
});
......@@ -121,14 +131,16 @@ export default class Clip {
return request.url.includes('/upload/demographic');
}
/**
* Distinguish between uploading and listening requests.
*/
handleRequest(request: http.IncomingMessage,
response: http.ServerResponse): void {
handleRequest(
request: http.IncomingMessage,
response: http.ServerResponse
): void {
if (request.method === 'POST') {
if (this.isClipVoteRequest(request)) { // Note: Check must occur first
if (this.isClipVoteRequest(request)) {
// Note: Check must occur first
this.saveClipVote(request, response);
} else if (this.isClipDemographic(request)) {
this.saveClipDemographic(request, response);
......@@ -147,11 +159,12 @@ export default class Clip {
/**
* Save clip vote posted to server
*/
saveClipVote(request: http.IncomingMessage,
response: http.ServerResponse) {
this.saveVote(request).then(timestamp => {
saveClipVote(request: http.IncomingMessage, response: http.ServerResponse) {
this.saveVote(request)
.then(timestamp => {
respond(response, '' + timestamp);
}).catch(e => {
})
.catch(e => {
console.error('saving clip vote error', e, e.stack);
respond(response, 'Error', 500);
});
......@@ -173,28 +186,33 @@ export default class Clip {
// Where is the clip vote going to be located?
let voteFile = glob + '-by-' + uid + '.vote';
let f = ff(() => {
// Save vote to S3
let params = {Bucket: BUCKET_NAME, Key: voteFile, Body: vote};
this.s3.putObject(params, f());
}, () => {
// File saving is now complete.
console.log('clip vote written to s3', voteFile);
resolve(glob);
}).onError(reject);
let f = ff(
() => {
// Save vote to S3
let params = { Bucket: BUCKET_NAME, Key: voteFile, Body: vote };
this.s3.putObject(params, f());
},
() => {
// File saving is now complete.
console.log('clip vote written to s3', voteFile);
resolve(glob);
}
).onError(reject);
});
}
/**
* Save clip demographic posted to server
*/
saveClipDemographic(request: http.IncomingMessage,
response: http.ServerResponse) {
this.saveDemographic(request).then(timestamp => {
saveClipDemographic(
request: http.IncomingMessage,
response: http.ServerResponse
) {
this.saveDemographic(request)
.then(timestamp => {
respond(response, '' + timestamp);
}).catch(e => {
})
.catch(e => {
console.error('saving clip demographic error', e, e.stack);
respond(response, 'Error', 500);
});
......@@ -215,28 +233,34 @@ export default class Clip {
// Where is the clip demographic going to be located?
let demographicFile = uid + '/demographic.json';
let f = ff(() => {
// Save demographic to S3
let params = {Bucket: BUCKET_NAME, Key: demographicFile, Body: demographic};
this.s3.putObject(params, f());
}, () => {
// File saving is now complete.
console.log('clip demographic written to s3', demographicFile);
resolve(uid);
}).onError(reject);
let f = ff(
() => {
// Save demographic to S3
let params = {
Bucket: BUCKET_NAME,
Key: demographicFile,
Body: demographic,
};
this.s3.putObject(params, f());
},
() => {
// File saving is now complete.
console.log('clip demographic written to s3', demographicFile);
resolve(uid);
}
).onError(reject);
});
}