I want to send streaming data (as sequences of ArrayBuffer) from a Chrome extension to a Chrome App, since Chrome message API (includes chrome.runtime.sendMessage, postMessage...) does not support ArrayBuffer and JS arrays have poor performance, I have to try other methods. Eventually, I found WebRTC over RTCDataChannel might a good solution in my case.
I have succeeded to send string over a RTCDataChannel, but when I tried to send ArrayBuffer I got:
code: 19
message: "Failed to execute 'send' on 'RTCDataChannel': Could not send data"
name: "NetworkError"
It seems that it's not a bandwidths limits problem since it failed even though I sent one byte of data. Here is my code:
pc = new RTCPeerConnection(configuration, { optional: [ { RtpDataChannels: true } ]});
//...
var dataChannel = m.pc.createDataChannel("mydata", {reliable: true});
//...
var ab = new ArrayBuffer(8);
dataChannel.send(ab);
Tested on OSX 10.10.1, Chrome M40 (Stnble), M42(Canary); and on Chromebook M40.
I have filed a bug for WebRTC here.
I modified my code, now everything worked amazing:
removed RtpDataChannels option when creating RTCPeerConnection.(YES, remove RtpDataChannels option if you want data channel, what a magic world!)
on receiver side: no need createDataChannel, instead, handle onmessage, onxxx by using event.channle from pc.ondatachannel callback:
pc.ondatachannel function(event)
var receiveChannel = event.channel;
receiveChannel.onmessage = function(event){
console.log("Got Data Channel Message:", event.data);
};
};
Related
I am currently looking to find a best way to store a incoming webrtc video streams. I am joining the videocall using webrtc (via chrome) and I would like to record every incoming video stream to from each participant to the browser.
The solutions I am researching are:
Intercept network packets coming to the browsers e.g. using Whireshark and then decode. Following this article: https://webrtchacks.com/video_replay/
Modifying a browser to store recording as a file e.g. by modifying Chromium itself
Any screen-recorders or using solutions like xvfb & ffmpeg is not an options due the resources constrains. Is there any other way that could let me capture packets or encoded video as a file? The solution must be working on Linux.
if the media stream is what you want a method is to override the browser's PeerConnection. Here is an example:
In an extension manifest add the following content script:
content_scripts": [
{
"matches": ["http://*/*", "https://*/*"],
"js": ["payload/inject.js"],
"all_frames": true,
"match_about_blank": true,
"run_at": "document_start"
}
]
inject.js
var inject = '('+function() {
//overide the browser's default RTCPeerConnection.
var origPeerConnection = window.RTCPeerConnection || window.webkitRTCPeerConnection || window.mozRTCPeerConnection;
//make sure it is supported
if (origPeerConnection) {
//our own RTCPeerConnection
var newPeerConnection = function(config, constraints) {
console.log('PeerConnection created with config', config);
//proxy the orginal peer connection
var pc = new origPeerConnection(config, constraints);
//store the old addStream
var oldAddStream = pc.addStream;
//addStream is called when a local stream is added.
//arguments[0] is a local media stream
pc.addStream = function() {
console.log("our add stream called!")
//our mediaStream object
console.dir(arguments[0])
return oldAddStream.apply(this, arguments);
}
//ontrack is called when a remote track is added.
//the media stream(s) are located in event.streams
pc.ontrack = function(event) {
console.log("ontrack got a track")
console.dir(event);
}
window.ourPC = pc;
return pc;
};
['RTCPeerConnection', 'webkitRTCPeerConnection', 'mozRTCPeerConnection'].forEach(function(obj) {
// Override objects if they exist in the window object
if (window.hasOwnProperty(obj)) {
window[obj] = newPeerConnection;
// Copy the static methods
Object.keys(origPeerConnection).forEach(function(x){
window[obj][x] = origPeerConnection[x];
})
window[obj].prototype = origPeerConnection.prototype;
}
});
}
}+')();';
var script = document.createElement('script');
script.textContent = inject;
(document.head||document.documentElement).appendChild(script);
script.parentNode.removeChild(script);
I tested this with a voice call in google hangouts and saw that two mediaStreams where added via pc.addStream and one track was added via pc.ontrack. addStream would seem to be local media streams and the event object in ontrack is a RTCTrackEvent which has a streams object. which I assume are what you are looking for.
To access these streams from your extenion's content script you will need to create audio elements and set the "srcObject" property to the media stream: e.g.
pc.ontrack = function(event) {
//check if our element exists
var elm = document.getElementById("remoteStream");
if(elm == null) {
//create an audio element
elm = document.createElement("audio");
elm.id = "remoteStream";
}
//set the srcObject to our stream. not sure if you need to clone it
elm.srcObject = event.streams[0].clone();
//write the elment to the body
document.body.appendChild(elm);
//fire a custom event so our content script knows the stream is available.
// you could pass the id in the "detail" object. for example:
//CustomEvent("remoteStreamAdded", {"detail":{"id":"audio_element_id"}})
//then access if via e.detail.id in your event listener.
var e = CustomEvent("remoteStreamAdded");
window.dispatchEvent(e);
}
Then in your content script you can listen for that event/access the mediastream like so:
window.addEventListener("remoteStreamAdded", function(e) {
elm = document.getElementById("remoteStream");
var stream = elm.captureStream();
})
With the capture stream available to your content script you can do pretty much anything you want with it. For example, MediaRecorder works really well for recording the stream(s) or you could use something like peer.js or maybe binary.js to stream to another source.
I haven't tested this but it should also be possible to override the local streams. For example, in the inject.js you could establish some blank mediastream, override navigator.mediaDevices.getUserMedia and instead of returning the local mediastream return your own mediastream.
This method should work in firefox and maybe others as well assuming you use an extenion/app to load the inject.js script at the start of the document. It being loaded before any of the target's libs is key to making this work.
edited for more detail
edited for even more detail
Capturing packets will only give you the network packets which you would then need to turn into frames and put into a container. A server such as Janus can record videos.
Running headless chrome and using the javascript MediaRecorder API is another option but much more heavy on resources.
I have a very simple service worker to add offline support. The fetch handler looks like
self.addEventListener("fetch", function (event) {
var url = event.request.url;
event.respondWith(fetch(event.request).then(function (response) {
//var cacheResponse: Response = response.clone();
//caches.open(CURRENT_CACHES.offline).then((cache: Cache) => {
// cache.put(url, cacheResponse).catch(() => {
// // ignore error
// });
//});
return response;
}).catch(function () {
// check the cache
return getCachedContent(event.request);
}));
});
Intermittently we are seeing a net::ERR_CONNECTION_RESET error for a particular script we load into the page when online. The error is not coming from the server as the service worker is picking up the file from the browser cache. Chrome's network tab shows the service worker has successfully fetched the file from the disk cache but the request from the browser to the service worker shows as (failed)
Does anyone know the underlying issue causing this? Is there a problem with my service worker implementation?
This is likely due to a bug in Chrome (and potentially other browsers as well) that could result in a garbage collection event removing a reference to the response stream while it's still being read.
Its fix in Chrome is being tracked at https://bugs.chromium.org/p/chromium/issues/detail?id=934386.
I am writing an Angular 2 app (built with angular cli), and trying to use AWS Polly text-to-speech API.
According to the API you can request audio output as well as "Speech Marks" which can describe word timing, visemes, etc. The audio is delivered as "mp3" format, and the speech marks as "application/x-json-stream", which I understand as a "new line" delimited JSON. It cannot be parsed with JSON.parse() due to the new lines. I have yet been unable to read/parse this data. I have looked at several libs that are for "json streaming" but they are all built for node.js and won't work with Angular 2. My code is as follows...
onClick() {
AWS.config.region = 'us-west-2';
AWS.config.accessKeyId = 'xxxxx';
AWS.config.secretAccessKey = 'yyyyy';
let polly = new AWS.Polly();
var params = {
OutputFormat: 'json',
Text: 'Hello world',
VoiceId: 'Joanna',
SpeechMarkTypes:['viseme']
};
polly.synthesizeSpeech(params, (err, data) => {
if (err) {
console.log(err, err.stack);
} else {
var uInt8Array = new Uint8Array(data.AudioStream);
var arrayBuffer = uInt8Array.buffer;
var blob = new Blob([arrayBuffer]);
var url = URL.createObjectURL(blob);
this.audio.src = url;
this.audio.play(); // works fine
// speech marks info displays "application/x-json-stream"
console.log(data.ContentType);
}
});
Strangely enough Chrome browser knows how to read this data and displays it in the response.
Any help would be greatly appreciated.
I had the same problem. I saved the file so I could then read it line by line, accessing the JSON objects when I need to highlight words being read. Mind you this is probably not the most effective way, but an easy way to move on and get working on the fun stuff.
I am trying out different ways to work with Polly, will update answer if I find a better way
You can do it with:
https://www.npmjs.com/package/ndjson-parse
That worked for me.
But I can't play audio, I tried your code it says
DOMException: Failed to load because no supported source was found.
I'm struggling for a couple of days. Question is simple, is there a way that can I create a server on Raspberry PI that will return current status of GPIO ports in JSON format?
Example:
Http://192.168.1.109:3000/led
{
"Name": "green led",
"Status": "on"
}
I found Adafruit gpio-stream library useful but don't know how to send data to JSON format.
Thank you
There are a variety of libraries for gpio interaction for node.js. One issue is that you might need to run it as root to have access to gpio, unless you can adjust the read access for those devices. This is supposed to be fixed in the latest version of rasbian though.
I recently built a node.js application that was triggered from a motion sensor, in order to activate the screen (and deactivate it after a period of time). I tried various gpio libraries but the one that I ended up using was "onoff" https://www.npmjs.com/package/onoff mainly because it seemed to use an appropriate way to identify changes on the GPIO pins (using interrupts).
Now, you say that you want to send data, but you don't specify how that is supposed to happen. If we use the example that you want to send data using a POST request via HTTP, and send the JSON as body, that would mean that you would initialize the GPIO pins that you have connected, and then attach event handlers for them (to listen for changes).
Upon a change, you would invoke the http request and serialize the JSON from a javascript object (there are libraries that would take care of this as well). You would need to keep a name reference yourself since you only address the GPIO pins by number.
Example:
var GPIO = require('onoff').Gpio;
var request = require('request');
var x = new GPIO(4, 'in', 'both');
function exit() {
x.unexport();
}
x.watch(function (err, value) {
if (err) {
console.error(err);
return;
}
request({
uri: 'http://example.org/',
method: 'POST',
json: true,
body: { x: value } // This is the actual JSON data that you are sending
}, function () {
// this is the callback from when the request is finished
});
});
process.on('SIGINT', exit);
I'm using the npm modules onoff and request. request is used for simplifying the JSON serialization over a http request.
As you can see, I only set up one GPIO here. If you need to track multiple, you must make sure to initialize them all, distinguish them with some sort of name and also remember to unexport them in the exit callback. Not sure what happens if you don't do it, but you might lock it for other processes.
Thank You, this was very helpful. I did not express myself well, sorry for that. I don't want to send data (for now) i just want to enter web address like 192.168.1.109/led and receive json response. This is what I manage to do for now. I don't know if this is the right way. PLS can you review this or suggest better method..
var http = require('http');
var url = require('url');
var Gpio = require('onoff').Gpio;
var led = new Gpio(23, 'out');
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
var command = url.parse(req.url).pathname.slice(1);
switch(command) {
case "on":
//led.writeSync(1);
var x = led.readSync();
res.write(JSON.stringify({ msgId: x }));
//res.end("It's ON");
res.end();
break;
case "off":
led.writeSync(0);
res.end("It's OFF");
break;
default:
res.end('Hello? yes, this is pi!');
}
}).listen(8080);
I have created two Chrome apps and I want to pass some data (string format) from one Chrome app to another Chrome app. Appreciate if someone can help me with showing the correct way of doing this?
It's an RTFM question.
From Messaging documentation (note that it mentions extensions, but it works for apps):
In addition to sending messages between different components in your extension, you can use the messaging API to communicate with other extensions. This lets you expose a public API that other extensions can take advantage of.
You need to send messages using chrome.runtime.sendMessage (using app ID) and receive them using chrome.runtime.onMessageExternal event. If required, long-lived connections can also be established.
// App 1
var app2id = "abcdefghijklmnoabcdefhijklmnoab2";
chrome.runtime.onMessageExternal.addListener(
// This should fire even if the app is not running, as long as it is
// included in the event page (background script)
function(request, sender, sendResponse) {
if(sender.id == app2id && request.data) {
// Use data passed
// Pass an answer with sendResponse() if needed
}
}
);
// App 2
var app1id = "abcdefghijklmnoabcdefhijklmnoab1";
chrome.runtime.sendMessage(app1id, {data: /* some data */},
function(response) {
if(response) {
// Installed and responded
} else {
// Could not connect; not installed
// Maybe inspect chrome.runtime.lastError
}
}
);