diff --git a/client/robot/robot.js b/client/robot/robot.js index fdfabff2a97bba6472482ba76e903f6c967fd089..bc8ca5b898afd2856fc3c4004c236a559d50605d 100644 --- a/client/robot/robot.js +++ b/client/robot/robot.js @@ -32,12 +32,18 @@ robot = { previousReco: [], recordedParticipantsWS: undefined, participantsMediaRecorders: {}, + participantsStream: {}, isDisconnected: false, intervalList: [], - speakerId : {}, + speakerId: {}, + processAudio(stream, callback, interval) { - const mediaRecorder = new MediaRecorder(stream); + var options = { + video : false, + mimeType : 'audio/webm' + } + const mediaRecorder = new MediaRecorder(stream, options); mediaRecorder.ondataavailable = callback; mediaRecorder.start(interval); @@ -92,18 +98,18 @@ robot = { }, openSTTSocket() { - if(robot.recordedParticipantsWS === undefined){ + if (robot.recordedParticipantsWS === undefined) { const ws = robotLib.stt.getTranscriptSocket(e => { console.log('> ' + e.text); robotLib.reco.send( { from: robot.room, - text: e.from + '\t' + e.until + '\t' + speakerId + '\t' + e.text + text: e.from + '\t' + e.until + '\t' + robot.speakerId + '\t' + e.text }); }); ws.addEventListener('error', e => { // Try to open new connection on error - console.error('STT ws for ' + easyrtcid + ' error. Trying to reopen'); + console.error('STT ws for ' + robot.speakerId + ' error. Trying to reopen'); console.error(e); robot.openSTTSocket(); }); @@ -120,13 +126,45 @@ robot = { }); }, - audioGenerator(easyrtcid, e) { - speakerId = easyrtcid; - robot.recordedParticipantsWS.send(e.data); + activityVoiceDetection() { + const keyMap = Object.keys(robot.participantsStream) + + let selectedStream; + if (keyMap) { + for (let i = 0; i < keyMap.length; i++) { + console.log('easyrtcid keyList value', keyMap[i]); + selectedStream = robot.participantsStream[keyMap[i]].shift(); + robot.speakerId = keyMap[i]; + } + } + /* + var arrayBuffer; + var fileReader = new FileReader(); + fileReader.onload = function() { + arrayBuffer = this.result; + var byteArray = new Int16Array(arrayBuffer); + for (var i = 0; i < byteArray.byteLength; i++) { + // do something with each byte in the array + } + }; + fileReader.readAsArrayBuffer(blob);*/ + + return selectedStream; }, + audioGenerator(easyrtcid, e) { + robot.participantsStream[easyrtcid].push(e.data); + + const selectedStream = robot.activityVoiceDetection(); + + robot.recordedParticipantsWS.send(selectedStream); + }, recordParticipant(easyrtcid) { + if (robot.participantsStream[easyrtcid] === undefined) { + robot.participantsStream[easyrtcid] = []; + } + robot.openSTTSocket(); robot.getUserStream(easyrtcid); }, @@ -178,7 +216,7 @@ robot = { }); }; - robotController.onAttendeeRemove = (e, data) => { + robotController.onAttendeeRemove = e => { robot.checkDisconnect(); }; diff --git a/package.json b/package.json index a3e7db712a5d227ba5ee19b87667b2b0eb173a07..7945d5a0c81b25d15dff57d68a2d81f30573c483 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,11 @@ "webdriver-stop": "./bin/webdriver-stop" }, "dependencies": { + "@ffmpeg-installer/ffmpeg": "^1.0.13", + "@ffprobe-installer/ffprobe": "^1.0.8", "body-parser": "^1.18.2", "express": "^4.16.2", + "fluent-ffmpeg": "^2.1.2", "mz": "2.6.0", "puppeteer": "0.12.0", "sockjs-client": "1.1.4",