diff options
Diffstat (limited to 'testing/web-platform/tests/mediacapture-insertable-streams')
14 files changed, 1102 insertions, 0 deletions
diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-audio.https.html b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-audio.https.html new file mode 100644 index 0000000000..c2f0bfc011 --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-audio.https.html @@ -0,0 +1,97 @@ +<!doctype html> +<html> + +<head> + <title>MediaStreamTrackGenerator</title> + <link rel="help" href="https://w3c.github.io/mediacapture-insertable-streams"> +</head> + +<body> + <p class="instructions">When prompted, use the accept button to give permission to use your audio and video devices.</p> + <h1 class="instructions">Description</h1> + <p class="instructions">This test checks that generating audio MediaStreamTracks works as expected.</p> + <audio id="audioElement" autoplay=true></audio> + <script src=/resources/testharness.js></script> + <script src=/resources/testharnessreport.js></script> + <script src=/resources/testdriver.js></script> + <script src=/resources/testdriver-vendor.js></script> + <script src='../mediacapture-streams/permission-helper.js'></script> + <script> + + function makeAudioData(timestamp) { + const sampleRate = 30000; + + let frames = sampleRate / 10; + let channels = 1; + + // Generate a simple sin wave, so we have something. + let data = new Float32Array(frames*channels); + const hz = 100; // sound frequency + for (let i = 0; i < data.length; i++) { + const t = (i / sampleRate) * hz * (Math.PI * 2); + data[i] = Math.sin(t); + } + + return new AudioData({ + timestamp: timestamp, + numberOfFrames: frames, + numberOfChannels: channels, + sampleRate: sampleRate, + data: data, + format: "f32", + }); + } + + promise_test(async t => { + const generator = new MediaStreamTrackGenerator("audio"); + + const writer = generator.writable.getWriter(); + await writer.write(makeAudioData(1)); + + assert_equals(generator.kind, "audio"); + assert_equals(generator.readyState, "live"); + + t.add_cleanup(() => generator.stop()); + }, "Tests that creating a Audio MediaStreamTrackGenerator works as expected"); + + promise_test(async t => { + assert_throws_js(TypeError, () => { new MediaStreamTrackGenerator({ kind: "invalid kind" }) }); + }, "Creating Generator with an invalid kind throws"); + + promise_test(async t => { + await setMediaPermission(); + const capturedStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + assert_equals(capturedStream.getAudioTracks().length, 1); + const upstreamTrack = capturedStream.getAudioTracks()[0]; + t.add_cleanup(() => upstreamTrack.stop()); + + assert_throws_js(TypeError, () => { new MediaStreamTrackGenerator() }); + }, "Creating Generator with a missing kind throws"); + + promise_test(async t => { + const generator = new MediaStreamTrackGenerator({ kind: "video" }); + t.add_cleanup(() => generator.stop()); + + const writer = generator.writable.getWriter(); + const data = makeAudioData(1); + + writer.write(data).then(t.step_func(() => assert_unreached("Write should reject")), t.step_func(f => assert_true(f instanceof TypeError, "write rejects with a TypeError"))); + }, "Mismatched data and generator kind throws on write."); + + promise_test(async t => { + const generator = new MediaStreamTrackGenerator("audio"); + t.add_cleanup(() => generator.stop()); + + const audioElement = document.getElementById("audioElement"); + audioElement.srcObject = new MediaStream([generator]); + await audioElement.play(); + + const writer = generator.writable.getWriter(); + await writer.write(makeAudioData(1)); + + // Wait for audio playout to actually happen. + await t.step_wait(() => audioElement.currentTime > 0, "audioElement played out generated track"); + }, "Tests that audio actually flows to a connected audio element"); + </script> +</body> +</html> diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-in-service-worker.https.html b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-in-service-worker.https.html new file mode 100644 index 0000000000..389a30d0d9 --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-in-service-worker.https.html @@ -0,0 +1,24 @@ +<!doctype html> +<title>Test initialize MediaStreamTrackGenerator in a service worker</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src='/service-workers/service-worker/resources/test-helpers.sub.js'></script> +<script> +'use strict'; + +promise_test(async t => { + const registration = await navigator.serviceWorker.register('service-worker.js'); + await wait_for_state(t, registration.installing, 'activated'); + const result = new Promise((resolve, reject) => { + navigator.serviceWorker.addEventListener('message', (e) => { + if (e.data.result === 'Failure') { + reject('Failed with error ' + e.data.error); + } else { + resolve(); + } + }); + }); + registration.active.postMessage('hello world'); + return result; +}, 'A service worker is able to initialize a MediaStreamTrackGenerator without crashing'); +</script>
\ No newline at end of file diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-in-shared-worker.https.html b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-in-shared-worker.https.html new file mode 100644 index 0000000000..deecfccad1 --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-in-shared-worker.https.html @@ -0,0 +1,22 @@ +<!doctype html> +<title>Test initialize MediaStreamTrackGenerator in a shared worker</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script> +'use strict'; + +promise_test(async t => { + const worker = new SharedWorker('shared-worker.js'); + const result = new Promise((resolve, reject) => { + worker.port.onmessage = (e) => { + if (e.data.result === 'Failure') { + reject('Failed with error ' + e.data.error); + } else { + resolve(); + } + }; + }); + worker.port.postMessage('Hello world'); + return result; +}, 'A shared worker is able to initialize a MediaStreamTrackGenerator without crashing'); +</script>
\ No newline at end of file diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-in-worker.https.html b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-in-worker.https.html new file mode 100644 index 0000000000..e0a8f2fc27 --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-in-worker.https.html @@ -0,0 +1,39 @@ +<!doctype html> +<title>Test creation of MediaStreamTrackGenerator in a worker</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script> +'use strict'; + +function initWorker(){ + const worker = new Worker('dedicated-worker.js'); + const result = new Promise((resolve, reject) => { + worker.onmessage = (e) => { + if (e.data.result === 'Failure') { + reject('Failed with error ' + e.data.error); + } else { + resolve(); + } + }; + }); + return [worker,result]; +} + +promise_test(async t => { + const [worker,result] = initWorker(); + worker.postMessage({msg: 'Hello there'}); + return result; +}, 'A worker is able to initialize a MediaStreamTrackGenerator without crashing'); + +promise_test(async t => { + const [worker,result] = initWorker(); + worker.postMessage({enable: true}); + return result; +}, 'A worker is able to enable a MediaStreamTrackGenerator without crashing'); + +promise_test(async t => { + const [worker,result] = initWorker(); + worker.postMessage({enable: false}); + return result; +}, 'A worker is able to disable a MediaStreamTrackGenerator without crashing'); +</script> diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-pipes-data-in-worker.https.html b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-pipes-data-in-worker.https.html new file mode 100644 index 0000000000..61a6f038c4 --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-pipes-data-in-worker.https.html @@ -0,0 +1,41 @@ +<!doctype html> +<title>Test piping data through MediaStreamTrackGenerator in a worker</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script id="workerCode" type="javascript/worker"> +self.onmessage = (e) => { + try { + const generator = new MediaStreamTrackGenerator({kind: 'video'}); + e.data.readable.pipeTo(generator.writable); + self.postMessage({result: 'Success'}); + } catch (e) { + self.postMessage({result: 'Failure', error: e}); + } +} +</script> +<script> +'use strict'; + +promise_test(async t => { + const workerBlob = new Blob([document.querySelector('#workerCode').textContent], + { type: "text/javascript" }); + const workerUrl = window.URL.createObjectURL(workerBlob); + const worker = new Worker(workerUrl); + window.URL.revokeObjectURL(workerUrl); + const result = new Promise((resolve, reject) => { + worker.onmessage = (e) => { + if (e.data.result === 'Failure') { + reject('Failed with error ' + e.data.error); + } else { + resolve(); + } + }; + }); + const stream = await navigator.mediaDevices.getUserMedia({ video: true }); + const track = stream.getVideoTracks()[0]; + const processor = new MediaStreamTrackProcessor({ track: track }); + worker.postMessage({ readable: processor.readable }, + [processor.readable]); + return result; +}, 'A worker is able to pipe data through a MediaStreamTrackGenerator without crashing'); +</script>
\ No newline at end of file diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-video.https.html b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-video.https.html new file mode 100644 index 0000000000..a6f73f009c --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackGenerator-video.https.html @@ -0,0 +1,285 @@ +<!DOCTYPE html> +<html> +<head> +<title>MediaStream Insertable Streams - Video</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src="/webrtc/RTCPeerConnection-helper.js"></script> +</head> +<body> + <p class="instructions">When prompted, use the accept button to give permission to use your audio and video devices.</p> + <h1 class="instructions">Description</h1> + <p class="instructions">This test checks that generating video MediaStreamTracks works as expected.</p> + <script> + + const pixelColour = [50, 100, 150, 255]; + const height = 240; + const width = 320; + function makeVideoFrame(timestamp) { + const canvas = new OffscreenCanvas(width, height); + + const ctx = canvas.getContext('2d', {alpha: false}); + ctx.fillStyle = `rgba(${pixelColour.join()})`; + ctx.fillRect(0, 0, width, height); + + return new VideoFrame(canvas, {timestamp, alpha: 'discard'}); + } + + async function getVideoFrame() { + const stream = await getNoiseStream({video: true}); + const input_track = stream.getTracks()[0]; + const processor = new MediaStreamTrackProcessor(input_track); + const reader = processor.readable.getReader(); + const result = await reader.read(); + input_track.stop(); + return result.value; + } + + function assertPixel(t, bytes, expected, epsilon = 5) { + for (let i = 0; i < bytes.length; i++) { + t.step(() => { + assert_less_than(Math.abs(bytes[i] - expected[i]), epsilon, "Mismatched pixel"); + }); + } + } + + async function initiateSingleTrackCall(t, track, output) { + const caller = new RTCPeerConnection(); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(); + t.add_cleanup(() => callee.close()); + caller.addTrack(track); + t.add_cleanup(() => track.stop()); + + exchangeIceCandidates(caller, callee); + // Wait for the first track. + const e = await exchangeOfferAndListenToOntrack(t, caller, callee); + output.srcObject = new MediaStream([e.track]); + // Exchange answer. + await exchangeAnswer(caller, callee); + await waitForConnectionStateChange(callee, ['connected']); + } + + promise_test(async t => { + const videoFrame = await getVideoFrame(); + const originalWidth = videoFrame.displayWidth; + const originalHeight = videoFrame.displayHeight; + const originalTimestamp = videoFrame.timestamp; + const generator = new MediaStreamTrackGenerator({kind: 'video'}); + t.add_cleanup(() => generator.stop()); + + // Use a MediaStreamTrackProcessor as a sink for |generator| to verify + // that |processor| actually forwards the frames written to its writable + // field. + const processor = new MediaStreamTrackProcessor(generator); + const reader = processor.readable.getReader(); + const readerPromise = new Promise(async resolve => { + const result = await reader.read(); + assert_equals(result.value.displayWidth, originalWidth); + assert_equals(result.value.displayHeight, originalHeight); + assert_equals(result.value.timestamp, originalTimestamp); + resolve(); + }); + + generator.writable.getWriter().write(videoFrame); + + return readerPromise; + }, 'Tests that MediaStreamTrackGenerator forwards frames to sink'); + + promise_test(async t => { + const videoFrame = makeVideoFrame(1); + const originalWidth = videoFrame.displayWidth; + const originalHeight = videoFrame.displayHeight; + const generator = new MediaStreamTrackGenerator({ kind: 'video' }); + t.add_cleanup(() => generator.stop()); + + const video = document.createElement("video"); + video.autoplay = true; + video.width = 320; + video.height = 240; + video.srcObject = new MediaStream([generator]); + video.play(); + + // Wait for the video element to be connected to the generator and + // generate the frame. + video.onloadstart = () => generator.writable.getWriter().write(videoFrame); + + return new Promise((resolve)=> { + video.ontimeupdate = t.step_func(() => { + const canvas = document.createElement("canvas"); + canvas.width = originalWidth; + canvas.height = originalHeight; + const context = canvas.getContext('2d'); + context.drawImage(video, 0, 0); + // Pick a pixel in the centre of the video and check that it has the colour of the frame provided. + const pixel = context.getImageData(videoFrame.displayWidth/2, videoFrame.displayHeight/2, 1, 1); + assertPixel(t, pixel.data, pixelColour); + resolve(); + }); + }); + }, 'Tests that frames are actually rendered correctly in a stream used for a video element.'); + + promise_test(async t => { + const generator = new MediaStreamTrackGenerator({kind: 'video'}); + t.add_cleanup(() => generator.stop()); + + // Write frames for the duration of the test. + const writer = generator.writable.getWriter(); + let timestamp = 0; + const intervalId = setInterval( + t.step_func(async () => { + if (generator.readyState === 'live') { + timestamp++; + await writer.write(makeVideoFrame(timestamp)); + } + }), + 40); + t.add_cleanup(() => clearInterval(intervalId)); + + const video = document.createElement('video'); + video.autoplay = true; + video.width = width; + video.height = height; + video.muted = true; + + await initiateSingleTrackCall(t, generator, video); + + return new Promise(resolve => { + video.ontimeupdate = t.step_func(() => { + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext('2d'); + context.drawImage(video, 0, 0); + // Pick a pixel in the centre of the video and check that it has the + // colour of the frame provided. + const pixel = context.getImageData(width / 2, height / 2, 1, 1); + // Encoding/decoding can add noise, so increase the threshhold to 8. + assertPixel(t, pixel.data, pixelColour, 8); + resolve(); + }); + }); + }, 'Tests that frames are actually rendered correctly in a stream sent over a peer connection.'); + + promise_test(async t => { + const generator = new MediaStreamTrackGenerator({kind: 'video'}); + t.add_cleanup(() => generator.stop()); + + const inputCanvas = new OffscreenCanvas(width, height); + + const inputContext = inputCanvas.getContext('2d', {alpha: false}); + // draw four quadrants + const colorUL = [255, 0, 0, 255]; + inputContext.fillStyle = `rgba(${colorUL.join()})`; + inputContext.fillRect(0, 0, width / 2, height / 2); + const colorUR = [255, 255, 0, 255]; + inputContext.fillStyle = `rgba(${colorUR.join()})`; + inputContext.fillRect(width / 2, 0, width / 2, height / 2); + const colorLL = [0, 255, 0, 255]; + inputContext.fillStyle = `rgba(${colorLL.join()})`; + inputContext.fillRect(0, height / 2, width / 2, height / 2); + const colorLR = [0, 255, 255, 255]; + inputContext.fillStyle = `rgba(${colorLR.join()})`; + inputContext.fillRect(width / 2, height / 2, width / 2, height / 2); + + // Write frames for the duration of the test. + const writer = generator.writable.getWriter(); + let timestamp = 0; + const intervalId = setInterval( + t.step_func(async () => { + if (generator.readyState === 'live') { + timestamp++; + await writer.write(new VideoFrame( + inputCanvas, {timestamp: timestamp, alpha: 'discard'})); + } + }), + 40); + t.add_cleanup(() => clearInterval(intervalId)); + + const caller = new RTCPeerConnection(); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(); + t.add_cleanup(() => callee.close()); + const sender = caller.addTrack(generator); + + exchangeIceCandidates(caller, callee); + // Wait for the first track. + const e = await exchangeOfferAndListenToOntrack(t, caller, callee); + + // Exchange answer. + await exchangeAnswer(caller, callee); + await waitForConnectionStateChange(callee, ['connected']); + const params = sender.getParameters(); + params.encodings.forEach(e => e.scaleResolutionDownBy = 2); + sender.setParameters(params); + + const processor = new MediaStreamTrackProcessor(e.track); + const reader = processor.readable.getReader(); + + // The first frame may not have had scaleResolutionDownBy applied + const numTries = 5; + for (let i = 1; i <= numTries; i++) { + const {value: outputFrame} = await reader.read(); + if (outputFrame.displayWidth !== width / 2) { + assert_less_than(i, numTries, `First ${numTries} frames were the wrong size.`); + outputFrame.close(); + continue; + } + + assert_equals(outputFrame.displayWidth, width / 2); + assert_equals(outputFrame.displayHeight, height / 2); + + const outputCanvas = new OffscreenCanvas(width / 2, height / 2); + const outputContext = outputCanvas.getContext('2d', {alpha: false}); + outputContext.drawImage(outputFrame, 0, 0); + outputFrame.close(); + // Check the four quadrants + const pixelUL = outputContext.getImageData(width / 8, height / 8, 1, 1); + assertPixel(t, pixelUL.data, colorUL); + const pixelUR = + outputContext.getImageData(width * 3 / 8, height / 8, 1, 1); + assertPixel(t, pixelUR.data, colorUR); + const pixelLL = + outputContext.getImageData(width / 8, height * 3 / 8, 1, 1); + assertPixel(t, pixelLL.data, colorLL); + const pixelLR = + outputContext.getImageData(width * 3 / 8, height * 3 / 8, 1, 1); + assertPixel(t, pixelLR.data, colorLR); + break; + } + }, 'Tests that frames are sent correctly with RTCRtpEncodingParameters.scaleResolutionDownBy.'); + + promise_test(async t => { + const generator = new MediaStreamTrackGenerator("video"); + t.add_cleanup(() => generator.stop()); + + const writer = generator.writable.getWriter(); + const frame = makeVideoFrame(1); + await writer.write(frame); + + assert_equals(generator.kind, "video"); + assert_equals(generator.readyState, "live"); + }, "Tests that creating a Video MediaStreamTrackGenerator works as expected"); + + promise_test(async t => { + const generator = new MediaStreamTrackGenerator("video"); + t.add_cleanup(() => generator.stop()); + + const writer = generator.writable.getWriter(); + const frame = makeVideoFrame(1); + await writer.write(frame); + + assert_throws_dom("InvalidStateError", () => frame.clone(), "VideoFrame wasn't destroyed on write."); + }, "Tests that VideoFrames are destroyed on write."); + + promise_test(async t => { + const generator = new MediaStreamTrackGenerator("audio"); + t.add_cleanup(() => generator.stop()); + + const writer = generator.writable.getWriter(); + const frame = makeVideoFrame(1); + assert_throws_js(TypeError, writer.write(frame)); + }, "Mismatched frame and generator kind throws on write."); + </script> +</body> +</html> diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackProcessor-audio.https.html b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackProcessor-audio.https.html new file mode 100644 index 0000000000..449b4833a2 --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackProcessor-audio.https.html @@ -0,0 +1,54 @@ +<!doctype html> +<html> +<head> + <title>MediaStreamTrackProcessor</title> + <link rel="help" href="https://w3c.github.io/mediacapture-insertable-streams"> +</head> +<body> +<p class="instructions">When prompted, use the accept button to give permission to use your audio and video devices.</p> +<h1 class="instructions">Description</h1> +<p class="instructions">This test checks that MediaStreamTrackProcessor works as expected on audio MediaStreamTracks.</p> +<script src=/resources/testharness.js></script> +<script src=/resources/testharnessreport.js></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../mediacapture-streams/permission-helper.js'></script> +<script> +promise_test(async t => { + await setMediaPermission("granted", ["microphone"]); + const stream = await navigator.mediaDevices.getUserMedia({audio: true}); + const track = stream.getTracks()[0]; + const processor = new MediaStreamTrackProcessor({track: track}); + const reader = processor.readable.getReader(); + const readResult = await reader.read(); + assert_false(readResult.done) + assert_true(readResult.value instanceof AudioData); + readResult.value.close(); + track.stop(); + return reader.closed; +}, "Tests that the reader of an audio MediaStreamTrackProcessor produces AudioData objects and is closed on track stop"); + +promise_test(async t => { + const stream = await navigator.mediaDevices.getUserMedia({audio: true}); + const track = stream.getTracks()[0]; + const processor = new MediaStreamTrackProcessor({track: track}); + const worker = new Worker('MediaStreamTrackProcessor-worker.js'); + const promise = new Promise(resolve => { + worker.onmessage = t.step_func(msg => { + if (msg.data instanceof AudioData) { + msg.data.close(); + track.stop(); + } else if (msg.data == 'closed') { + resolve(); + } else { + assert_unreached(); + } + }) + }); + worker.postMessage({readable: processor.readable}, + [processor.readable]); + return promise; +}, "Tests that the reader of an audio MediaStreamTrackProcessor produces AudioData objects and is closed on track stop while running on a worker"); +</script> +</body> +</html> diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackProcessor-backpressure.https.html b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackProcessor-backpressure.https.html new file mode 100644 index 0000000000..7b4f88e944 --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackProcessor-backpressure.https.html @@ -0,0 +1,69 @@ +<!doctype html> +<html> +<head> + <title>MediaStreamTrackProcessor backpressure</title> + <link rel="help" href="https://w3c.github.io/mediacapture-insertable-streams"> +</head> +<body> + <h1 class="instructions">Description</h1> +<p class="instructions">This test checks that MediaStreamTrackProcessor handles backpressure from a WHATWG stream pipeline.</p> +<script src=/resources/testharness.js></script> +<script src=/resources/testharnessreport.js></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script> + + const height = 240; + const width = 320; + + const inputCanvas = new OffscreenCanvas(width, height); + const inputCtx = inputCanvas.getContext('2d', {alpha: false}); + inputCtx.fillStyle = 'black'; + inputCtx.fillRect(0, 0, width, height); + + const frameDuration = 40; + + function makeUniformVideoFrame(timestamp) { + return new VideoFrame(inputCanvas, {timestamp, alpha: 'discard'}); + } + + promise_test(async t => { + // TODO: use "new VideoTrackGenerator" + const generator = new MediaStreamTrackGenerator({kind: 'video'}); + t.add_cleanup(() => generator.stop()); + + // Write frames for the duration of the test. + const writer = generator.writable.getWriter(); + let timestamp = 0; + const intervalId = setInterval( + t.step_func(async () => { + if (generator.readyState === 'live') { + timestamp++; + await writer.write(makeUniformVideoFrame(timestamp)); + } + }), + frameDuration); + t.add_cleanup(() => clearInterval(intervalId)); + t.step_timeout(function() { + clearInterval(intervalId); + generator.stop(); + }, 2000); + const processor = new MediaStreamTrackProcessor({track: generator}); + let ts = 1; + await processor.readable.pipeTo(new WritableStream({ + async write(frame) { + if (ts === 1) { + assert_equals(frame.timestamp, ts, "Timestamp mismatch"); + } else { + assert_greater_than_equal(frame.timestamp, ts, "Backpressure should have resulted in skipping at least 3 frames"); + } + frame.close(); + ts+=3; + // Wait the equivalent of 3 frames + return new Promise((res) => t.step_timeout(res, 3*frameDuration)); + } + })); + }, "Tests that backpressure forces MediaStreamTrackProcess to skip frames"); +</script> +</body> +</html> diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackProcessor-video.https.html b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackProcessor-video.https.html new file mode 100644 index 0000000000..4182505ada --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackProcessor-video.https.html @@ -0,0 +1,97 @@ +<!doctype html> +<html> +<head> + <title>MediaStreamTrackProcessor</title> + <link rel="help" href="https://w3c.github.io/mediacapture-insertable-streams"> +</head> +<body> +<p class="instructions">When prompted, use the accept button to give permission to use your audio and video devices.</p> +<h1 class="instructions">Description</h1> +<p class="instructions">This test checks that MediaStreamTrackProcessor works as expected on video MediaStreamTracks.</p> +<script src=/resources/testharness.js></script> +<script src=/resources/testharnessreport.js></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../mediacapture-streams/permission-helper.js'></script> +<script> +promise_test(async t => { + await setMediaPermission("granted", ["camera"]); + const stream = await navigator.mediaDevices.getUserMedia({video: true}); + const track = stream.getTracks()[0]; + const processor = new MediaStreamTrackProcessor({track: track}); + const reader = processor.readable.getReader(); + const readResult = await reader.read(); + assert_false(readResult.done) + assert_true(readResult.value instanceof VideoFrame); + readResult.value.close(); + track.stop(); + return reader.closed; +}, "Tests that the reader of a video MediaStreamTrackProcessor produces video frames and is closed on track stop"); + +promise_test(async t => { + const stream = await navigator.mediaDevices.getUserMedia({video: true}); + const track = stream.getTracks()[0]; + const processor = new MediaStreamTrackProcessor({track: track}); + const worker = new Worker('MediaStreamTrackProcessor-worker.js'); + const promise = new Promise(resolve => { + worker.onmessage = t.step_func(msg => { + if (msg.data instanceof VideoFrame) { + msg.data.close(); + track.stop(); + } else if (msg.data == 'closed') { + resolve(); + } else { + assert_unreached(); + } + }) + }); + worker.postMessage({readable: processor.readable}, + [processor.readable]); + return promise; +}, "Tests that the reader of a video MediaStreamTrackProcessor produces VideoFrame objects and is closed on track stop while running on a worker"); + +function makeVideoFrame(timestamp) { + const canvas = new OffscreenCanvas(100, 100); + const ctx = canvas.getContext('2d'); + return new VideoFrame(canvas, {timestamp}); +} + +promise_test(async t => { + // The generator will be used as the source for the processor to + // produce frames in a controlled manner. + const generator = new MediaStreamTrackGenerator('video'); + t.add_cleanup(() => generator.stop()); + // Use a larger maxBufferSize than the default to ensure no frames + // will be dropped. + const processor = new MediaStreamTrackProcessor({track: generator, maxBufferSize:10}); + const reader = processor.readable.getReader(); + const writer = generator.writable.getWriter(); + + let numReads = 0; + let resolve = null; + const promise = new Promise(r => resolve = r); + + const numOperations = 4; + // Issue reads without waiting for the frames to arrive. + for (let i = 0; i < numOperations; i++) { + reader.read().then(dv=> { + dv.value.close(); + if (++numReads == numOperations) + resolve(); + }); + } + + // Write video frames in different tasks to "slowly" settle the pending read + // requests. + for (let i = 0; i<numOperations; i++) { + await writer.write(makeVideoFrame(i)); + await new Promise(r=>setTimeout(r,0)); + } + + return promise; + +}, "Tests that multiple read requests are eventually settled"); + +</script> +</body> +</html> diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackProcessor-worker.js b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackProcessor-worker.js new file mode 100644 index 0000000000..51eaef80a9 --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/MediaStreamTrackProcessor-worker.js @@ -0,0 +1,17 @@ +onmessage = async msg => { + const reader = msg.data.readable.getReader(); + let readResult = await reader.read(); + postMessage(readResult.value); + readResult.value.close(); + // Continue reading until the stream is done due to a track.stop() + while (true) { + readResult = await reader.read(); + if (readResult.done) { + break; + } else { + readResult.value.close(); + } + } + await reader.closed; + postMessage('closed'); +} diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/VideoTrackGenerator.https.html b/testing/web-platform/tests/mediacapture-insertable-streams/VideoTrackGenerator.https.html new file mode 100644 index 0000000000..2c81c7604a --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/VideoTrackGenerator.https.html @@ -0,0 +1,327 @@ +<!DOCTYPE html> +<html> +<head> +<title>MediaStream Insertable Streams - VideoTrackGenerator</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src="../webrtc/RTCPeerConnection-helper.js"></script> +</head> +<body> + <p class="instructions">If prompted, use the accept button to give permission to use your audio and video devices.</p> + <h1 class="instructions">Description</h1> + <p class="instructions">This test checks that generating video MediaStreamTracks from VideoTrackGenerator works as expected.</p> + <script> + + const pixelColour = [50, 100, 150, 255]; + const height = 240; + const width = 320; + function makeVideoFrame(timestamp) { + const canvas = new OffscreenCanvas(width, height); + + const ctx = canvas.getContext('2d', {alpha: false}); + ctx.fillStyle = `rgba(${pixelColour.join()})`; + ctx.fillRect(0, 0, width, height); + + return new VideoFrame(canvas, {timestamp, alpha: 'discard'}); + } + + async function getVideoFrame() { + const stream = await getNoiseStream({video: true}); + const input_track = stream.getTracks()[0]; + const processor = new MediaStreamTrackProcessor(input_track); + const reader = processor.readable.getReader(); + const result = await reader.read(); + input_track.stop(); + return result.value; + } + + function assertPixel(t, bytes, expected, epsilon = 5) { + for (let i = 0; i < bytes.length; i++) { + t.step(() => { + assert_less_than(Math.abs(bytes[i] - expected[i]), epsilon, "Mismatched pixel"); + }); + } + } + + async function initiateSingleTrackCall(t, track, output) { + const caller = new RTCPeerConnection(); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(); + t.add_cleanup(() => callee.close()); + caller.addTrack(track); + t.add_cleanup(() => track.stop()); + + exchangeIceCandidates(caller, callee); + // Wait for the first track. + const e = await exchangeOfferAndListenToOntrack(t, caller, callee); + output.srcObject = new MediaStream([e.track]); + // Exchange answer. + await exchangeAnswer(caller, callee); + await waitForConnectionStateChange(callee, ['connected']); + } + + promise_test(async t => { + const videoFrame = await getVideoFrame(); + const originalWidth = videoFrame.displayWidth; + const originalHeight = videoFrame.displayHeight; + const originalTimestamp = videoFrame.timestamp; + const generator = new VideoTrackGenerator(); + t.add_cleanup(() => generator.track.stop()); + + // Use a MediaStreamTrackProcessor as a sink for |generator| to verify + // that |processor| actually forwards the frames written to its writable + // field. + const processor = new MediaStreamTrackProcessor(generator); + const reader = processor.readable.getReader(); + const readerPromise = new Promise(async resolve => { + const result = await reader.read(); + assert_equals(result.value.displayWidth, originalWidth); + assert_equals(result.value.displayHeight, originalHeight); + assert_equals(result.value.timestamp, originalTimestamp); + resolve(); + }); + + generator.writable.getWriter().write(videoFrame); + return readerPromise; + }, 'Tests that VideoTrackGenerator forwards frames to sink'); + + promise_test(async t => { + const videoFrame = makeVideoFrame(1); + const originalWidth = videoFrame.displayWidth; + const originalHeight = videoFrame.displayHeight; + const generator = new VideoTrackGenerator(); + t.add_cleanup(() => generator.track.stop()); + + const video = document.createElement("video"); + video.autoplay = true; + video.width = 320; + video.height = 240; + video.srcObject = new MediaStream([generator.track]); + video.play(); + + // Wait for the video element to be connected to the generator and + // generate the frame. + video.onloadstart = () => generator.writable.getWriter().write(videoFrame); + + return new Promise((resolve)=> { + video.ontimeupdate = t.step_func(() => { + const canvas = document.createElement("canvas"); + canvas.width = originalWidth; + canvas.height = originalHeight; + const context = canvas.getContext('2d'); + context.drawImage(video, 0, 0); + // Pick a pixel in the centre of the video and check that it has the colour of the frame provided. + const pixel = context.getImageData(videoFrame.displayWidth/2, videoFrame.displayHeight/2, 1, 1); + assertPixel(t, pixel.data, pixelColour); + resolve(); + }); + }); + }, 'Tests that frames are actually rendered correctly in a stream used for a video element.'); + + promise_test(async t => { + const generator = new VideoTrackGenerator(); + t.add_cleanup(() => generator.track.stop()); + + // Write frames for the duration of the test. + const writer = generator.writable.getWriter(); + let timestamp = 0; + const intervalId = setInterval( + t.step_func(async () => { + if (generator.track.readyState === 'live') { + timestamp++; + await writer.write(makeVideoFrame(timestamp)); + } + }), + 40); + t.add_cleanup(() => clearInterval(intervalId)); + + const video = document.createElement('video'); + video.autoplay = true; + video.width = width; + video.height = height; + video.muted = true; + + await initiateSingleTrackCall(t, generator.track, video); + + return new Promise(resolve => { + video.ontimeupdate = t.step_func(() => { + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext('2d'); + context.drawImage(video, 0, 0); + // Pick a pixel in the centre of the video and check that it has the + // colour of the frame provided. + const pixel = context.getImageData(width / 2, height / 2, 1, 1); + // Encoding/decoding can add noise, so increase the threshhold to 8. + assertPixel(t, pixel.data, pixelColour, 8); + resolve(); + }); + }); + }, 'Tests that frames are actually rendered correctly in a stream sent over a peer connection.'); + + + promise_test(async t => { + const generator = new VideoTrackGenerator(); + t.add_cleanup(() => generator.track.stop()); + + const inputCanvas = new OffscreenCanvas(width, height); + + const inputContext = inputCanvas.getContext('2d', {alpha: false}); + // draw four quadrants + const colorUL = [255, 0, 0, 255]; + inputContext.fillStyle = `rgba(${colorUL.join()})`; + inputContext.fillRect(0, 0, width / 2, height / 2); + const colorUR = [255, 255, 0, 255]; + inputContext.fillStyle = `rgba(${colorUR.join()})`; + inputContext.fillRect(width / 2, 0, width / 2, height / 2); + const colorLL = [0, 255, 0, 255]; + inputContext.fillStyle = `rgba(${colorLL.join()})`; + inputContext.fillRect(0, height / 2, width / 2, height / 2); + const colorLR = [0, 255, 255, 255]; + inputContext.fillStyle = `rgba(${colorLR.join()})`; + inputContext.fillRect(width / 2, height / 2, width / 2, height / 2); + + // Write frames for the duration of the test. + const writer = generator.writable.getWriter(); + let timestamp = 0; + const intervalId = setInterval( + t.step_func(async () => { + if (generator.track.readyState === 'live') { + timestamp++; + await writer.write(new VideoFrame( + inputCanvas, {timestamp: timestamp, alpha: 'discard'})); + } + }), + 40); + t.add_cleanup(() => clearInterval(intervalId)); + + const caller = new RTCPeerConnection(); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(); + t.add_cleanup(() => callee.close()); + const sender = caller.addTrack(generator.track); + + exchangeIceCandidates(caller, callee); + // Wait for the first track. + const e = await exchangeOfferAndListenToOntrack(t, caller, callee); + + // Exchange answer. + await exchangeAnswer(caller, callee); + await waitForConnectionStateChange(callee, ['connected']); + const params = sender.getParameters(); + params.encodings.forEach(e => e.scaleResolutionDownBy = 2); + sender.setParameters(params); + + const processor = new MediaStreamTrackProcessor(e.track); + const reader = processor.readable.getReader(); + + // The first frame may not have had scaleResolutionDownBy applied + const numTries = 5; + for (let i = 1; i <= numTries; i++) { + const {value: outputFrame} = await reader.read(); + if (outputFrame.displayWidth !== width / 2) { + assert_less_than(i, numTries, `First ${numTries} frames were the wrong size.`); + outputFrame.close(); + continue; + } + + assert_equals(outputFrame.displayWidth, width / 2); + assert_equals(outputFrame.displayHeight, height / 2); + + const outputCanvas = new OffscreenCanvas(width / 2, height / 2); + const outputContext = outputCanvas.getContext('2d', {alpha: false}); + outputContext.drawImage(outputFrame, 0, 0); + outputFrame.close(); + // Check the four quadrants + const pixelUL = outputContext.getImageData(width / 8, height / 8, 1, 1); + assertPixel(t, pixelUL.data, colorUL); + const pixelUR = + outputContext.getImageData(width * 3 / 8, height / 8, 1, 1); + assertPixel(t, pixelUR.data, colorUR); + const pixelLL = + outputContext.getImageData(width / 8, height * 3 / 8, 1, 1); + assertPixel(t, pixelLL.data, colorLL); + const pixelLR = + outputContext.getImageData(width * 3 / 8, height * 3 / 8, 1, 1); + assertPixel(t, pixelLR.data, colorLR); + break; + } + }, 'Tests that frames are sent correctly with RTCRtpEncodingParameters.scaleResolutionDownBy.'); + + promise_test(async t => { + const generator = new VideoTrackGenerator(); + t.add_cleanup(() => generator.track.stop()); + + const writer = generator.writable.getWriter(); + const frame = makeVideoFrame(1); + await writer.write(frame); + + assert_equals(generator.track.kind, "video"); + assert_equals(generator.track.readyState, "live"); + }, "Tests that creating a VideoTrackGenerator works as expected"); + + promise_test(async t => { + const generator = new VideoTrackGenerator(); + t.add_cleanup(() => generator.track.stop()); + + const writer = generator.writable.getWriter(); + const frame = makeVideoFrame(1); + await writer.write(frame); + + assert_throws_dom("InvalidStateError", () => frame.clone(), "VideoFrame wasn't destroyed on write."); + }, "Tests that VideoFrames are destroyed on write."); + + promise_test(async t => { + const generator = new VideoTrackGenerator(); + t.add_cleanup(() => generator.track.stop()); + + const writer = generator.writable.getWriter(); + const frame = makeVideoFrame(1); + assert_throws_js(TypeError, writer.write(frame)); + }, "Mismatched frame and generator kind throws on write."); + + promise_test(async t => { + const generator = new VideoTrackGenerator(); + t.add_cleanup(() => generator.track.stop()); + + // Use a MediaStreamTrackProcessor as a sink for |generator| to verify + // that |processor| actually forwards the frames written to its writable + // field. + const processor = new MediaStreamTrackProcessor(generator.track); + const reader = processor.readable.getReader(); + const videoFrame = makeVideoFrame(1); + + const writer = generator.writable.getWriter(); + const videoFrame1 = makeVideoFrame(1); + writer.write(videoFrame1); + const result1 = await reader.read(); + assert_equals(result1.value.timestamp, 1); + generator.muted = true; + + // This frame is expected to be discarded. + const videoFrame2 = makeVideoFrame(2); + writer.write(videoFrame2); + generator.muted = false; + + const videoFrame3 = makeVideoFrame(3); + writer.write(videoFrame3); + const result3 = await reader.read(); + assert_equals(result3.value.timestamp, 3); + + // Set up a read ahead of time, then mute, enqueue and unmute. + const promise5 = reader.read(); + generator.muted = true; + writer.write(makeVideoFrame(4)); // Expected to be discarded. + generator.muted = false; + writer.write(makeVideoFrame(5)); + const result5 = await promise5; + assert_equals(result5.value.timestamp, 5); + }, 'Tests that VideoTrackGenerator forwards frames only when unmuted'); + + // Note - tests for mute/unmute events will be added once + // https://github.com/w3c/mediacapture-transform/issues/81 is resolved + + </script> +</body> +</html> diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/dedicated-worker.js b/testing/web-platform/tests/mediacapture-insertable-streams/dedicated-worker.js new file mode 100644 index 0000000000..0dbcc32d0b --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/dedicated-worker.js @@ -0,0 +1,11 @@ +self.onmessage = (e) => { + try { + const mstg = new MediaStreamTrackGenerator({kind: 'video'}); + if ('enable' in e.data) { + mstg.enabled = e.data.enable; + } + self.postMessage({result: 'Success'}); + } catch (e) { + self.postMessage({result: 'Failure', error: e}); + } +}
\ No newline at end of file diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/service-worker.js b/testing/web-platform/tests/mediacapture-insertable-streams/service-worker.js new file mode 100644 index 0000000000..05a8b99ad8 --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/service-worker.js @@ -0,0 +1,8 @@ +self.addEventListener('message', (event) => { + try { + const mstg = new MediaStreamTrackGenerator({ kind: 'video' }); + event.source.postMessage({ result: 'Success' }); + } catch (e) { + event.source.postMessage({ result: 'Failure', error: e }); + }; +});
\ No newline at end of file diff --git a/testing/web-platform/tests/mediacapture-insertable-streams/shared-worker.js b/testing/web-platform/tests/mediacapture-insertable-streams/shared-worker.js new file mode 100644 index 0000000000..61ff67bcff --- /dev/null +++ b/testing/web-platform/tests/mediacapture-insertable-streams/shared-worker.js @@ -0,0 +1,11 @@ +onconnect = (e) => { + const port = e.ports[0]; + port.onmessage = (e) => { + try { + const generator = new MediaStreamTrackGenerator({kind: 'video'}); + port.postMessage({result: 'Success'}); + } catch (e) { + port.postMessage({result: 'Failure', error: e}); + } + } +}
\ No newline at end of file |