diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
| commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
| tree | f435a8308119effd964b339f76abb83a57c29483 /testing/web-platform/tests/webrtc-encoded-transform/tentative | |
| parent | Initial commit. (diff) | |
| download | firefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz firefox-26a029d407be480d791972afb5975cf62c9360a6.zip | |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'testing/web-platform/tests/webrtc-encoded-transform/tentative')
17 files changed, 1503 insertions, 0 deletions
diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedAudioFrame-clone.https.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedAudioFrame-clone.https.html new file mode 100644 index 0000000000..9f07713d44 --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedAudioFrame-clone.https.html @@ -0,0 +1,51 @@ +<!DOCTYPE html> +<meta charset="utf-8"> +<title>RTCEncodedAudioFrame can be cloned and distributed</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../../mediacapture-streams/permission-helper.js'></script> +<script src="../../webrtc/RTCPeerConnection-helper.js"></script> +<script src="../../service-workers/service-worker/resources/test-helpers.sub.js"></script> + +<script> +"use strict"; +promise_test(async t => { + const caller1 = new RTCPeerConnection(); + t.add_cleanup(() => caller1.close()); + const callee1 = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => callee1.close()); + await setMediaPermission("granted", ["microphone"]); + const inputStream = await navigator.mediaDevices.getUserMedia({audio:true}); + const inputTrack = inputStream.getAudioTracks()[0]; + t.add_cleanup(() => inputTrack.stop()); + caller1.addTrack(inputTrack) + exchangeIceCandidates(caller1, callee1); + + const caller2 = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller2.close()); + const sender2 = caller2.addTransceiver("audio").sender; + const writer2 = sender2.createEncodedStreams().writable.getWriter(); + sender2.replaceTrack(new MediaStreamTrackGenerator({ kind: 'audio' })); + + const framesReceivedCorrectly = new Promise((resolve, reject) => { + callee1.ontrack = async e => { + const receiverStreams = e.receiver.createEncodedStreams(); + const receiverReader = receiverStreams.readable.getReader(); + const result = await receiverReader.read(); + const original = result.value; + let clone = structuredClone(original); + assert_equals(original.timestamp, clone.timestamp); + assert_equals(original.getMetadata().absCaptureTime, clone.getMetadata().absCaptureTime); + assert_array_equals(Array.from(original.data), Array.from(clone.data)); + await writer2.write(clone); + resolve(); + } + }); + + await exchangeOfferAnswer(caller1, callee1); + + return framesReceivedCorrectly; +}, "Cloning before sending works"); +</script> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedAudioFrame-receive-cloned.https.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedAudioFrame-receive-cloned.https.html new file mode 100644 index 0000000000..3077632a3b --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedAudioFrame-receive-cloned.https.html @@ -0,0 +1,52 @@ +<!DOCTYPE html> +<meta charset="utf-8"> +<title>RTCEncodedAudioFrame can be cloned and distributed</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../../mediacapture-streams/permission-helper.js'></script> +<script src="../../webrtc/RTCPeerConnection-helper.js"></script> +<script src="../../service-workers/service-worker/resources/test-helpers.sub.js"></script> + +<script> +"use strict"; +promise_test(async t => { + const caller1 = new RTCPeerConnection(); + t.add_cleanup(() => caller1.close()); + const callee1 = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => callee1.close()); + await setMediaPermission("granted", ["microphone"]); + const inputStream = await navigator.mediaDevices.getUserMedia({audio:true}); + const inputTrack = inputStream.getAudioTracks()[0]; + t.add_cleanup(() => inputTrack.stop()); + caller1.addTrack(inputTrack) + exchangeIceCandidates(caller1, callee1); + + const caller2 = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller2.close()); + const sender2 = caller2.addTransceiver("audio").sender; + const writer2 = sender2.createEncodedStreams().writable.getWriter(); + sender2.replaceTrack(new MediaStreamTrackGenerator({ kind: 'audio' })); + + const framesReceivedCorrectly = new Promise((resolve, reject) => { + callee1.ontrack = async e => { + const receiverStreams = e.receiver.createEncodedStreams(); + const receiverReader = receiverStreams.readable.getReader(); + const receiverWriter = receiverStreams.writable.getWriter(); + const result = await receiverReader.read(); + const originalAudioFrame = result.value; + let clonedAudioFrame = structuredClone(originalAudioFrame); + assert_equals(originalAudioFrame.timestamp, clonedAudioFrame.timestamp); + assert_array_equals(Array.from(originalAudioFrame.data), Array.from(clonedAudioFrame.data)); + await writer2.write(clonedAudioFrame); + await receiverWriter.write(structuredClone(originalAudioFrame)); + resolve(); + } + }); + + await exchangeOfferAnswer(caller1, callee1); + + return framesReceivedCorrectly; +}, "Cloning before sending works"); +</script> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedAudioFrame-send-incoming.https.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedAudioFrame-send-incoming.https.html new file mode 100644 index 0000000000..02f3b17e0c --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedAudioFrame-send-incoming.https.html @@ -0,0 +1,63 @@ +<!DOCTYPE html> +<meta charset="utf-8"> +<title>RTCEncodedAudioFrame can be cloned and distributed</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../../mediacapture-streams/permission-helper.js'></script> +<script src="../../webrtc/RTCPeerConnection-helper.js"></script> +<script src="../../service-workers/service-worker/resources/test-helpers.sub.js"></script> + +<script> +"use strict"; +promise_test(async t => { + const caller1 = new RTCPeerConnection(); + t.add_cleanup(() => caller1.close()); + const callee1 = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => callee1.close()); + await setMediaPermission("granted", ["microphone"]); + const inputStream = await navigator.mediaDevices.getUserMedia({audio:true}); + const inputTrack = inputStream.getAudioTracks()[0]; + t.add_cleanup(() => inputTrack.stop()); + caller1.addTrack(inputTrack) + exchangeIceCandidates(caller1, callee1); + + const caller2 = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller2.close()); + const callee2 = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => callee2.close()); + const sender2 = caller2.addTransceiver("audio").sender; + const writer2 = sender2.createEncodedStreams().writable.getWriter(); + sender2.replaceTrack(new MediaStreamTrackGenerator({ kind: 'audio' })); + exchangeIceCandidates(caller2, callee2); + + const IncomingframesSentAndReceivedCorrectly = new Promise((resolve, reject) => { + // Write the received incoming frames on callee1 to caller2. + callee1.ontrack = async e => { + const receiverStreams = e.receiver.createEncodedStreams(); + const receiverReader = receiverStreams.readable.getReader(); + const result = await receiverReader.read(); + const original = result.value; + await writer2.write(original); + resolve(); + } + + // callee2 receives frames over the PC from caller2. + callee2.ontrack = async e => { + const receiverStreams = e.receiver.createEncodedStreams(); + const receiverReader = receiverStreams.readable.getReader(); + const receiverWriter = receiverStreams.writable.getWriter(); + const result = await receiverReader.read(); + receiverWriter.write(result.value); + resolve(); + } + + }); + + await exchangeOfferAnswer(caller1, callee1); + await exchangeOfferAnswer(caller2, callee2); + + return IncomingframesSentAndReceivedCorrectly; +}, "Send endoded incoming frame"); +</script> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedAudioFrame-serviceworker-failure.https.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedAudioFrame-serviceworker-failure.https.html new file mode 100644 index 0000000000..b2f5f5e94c --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedAudioFrame-serviceworker-failure.https.html @@ -0,0 +1,60 @@ +<!DOCTYPE html> +<meta charset="utf-8"> +<!-- Based on similar tests in html/infrastructure/safe-passing-of-structured-data/shared-array-buffers/ --> +<title>RTCEncodedVideoFrame cannot cross agent clusters, service worker edition</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../../mediacapture-streams/permission-helper.js'></script> +<script src="../../webrtc/RTCPeerConnection-helper.js"></script> +<script src="../../service-workers/service-worker/resources/test-helpers.sub.js"></script> + +<script> +"use strict"; +promise_test(async t => { + const caller = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(); + t.add_cleanup(() => callee.close()); + + await setMediaPermission("granted", ["microphone"]); + const stream = await navigator.mediaDevices.getUserMedia({audio:true}); + const track = stream.getTracks()[0]; + t.add_cleanup(() => track.stop()); + + const sender = caller.addTrack(track) + const streams = sender.createEncodedStreams(); + const reader = streams.readable.getReader(); + const writer = streams.writable.getWriter(); + + exchangeIceCandidates(caller, callee); + await exchangeOfferAnswer(caller, callee); + + const result = await reader.read(); + const scope = "resources/blank.html"; + let reg = await service_worker_unregister_and_register(t, "resources/serviceworker-failure.js", scope); + t.add_cleanup(() => service_worker_unregister(t, scope)); + await wait_for_state(t, reg.installing, "activated"); + let iframe = await with_iframe(scope); + t.add_cleanup(() => iframe.remove()); + const sw = iframe.contentWindow.navigator.serviceWorker; + let state = "start in window"; + return new Promise(resolve => { + sw.onmessage = t.step_func(e => { + if (e.data === "start in worker") { + assert_equals(state, "start in window"); + sw.controller.postMessage(result.value); + state = "we are expecting confirmation of an onmessageerror in the worker"; + } else if (e.data === "onmessageerror was received in worker") { + assert_equals(state, "we are expecting confirmation of an onmessageerror in the worker"); + resolve(); + } else { + assert_unreached("Got an unexpected message from the service worker: " + e.data); + } + }); + + sw.controller.postMessage(state); + }); +}); +</script> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedVideoFrame-clone.https.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedVideoFrame-clone.https.html new file mode 100644 index 0000000000..324c44f193 --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedVideoFrame-clone.https.html @@ -0,0 +1,61 @@ +<!DOCTYPE html> +<meta charset="utf-8"> +<title>RTCEncodedVideoFrame can be cloned and distributed</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../../mediacapture-streams/permission-helper.js'></script> +<script src="../../webrtc/RTCPeerConnection-helper.js"></script> +<script src="../../service-workers/service-worker/resources/test-helpers.sub.js"></script> + +<script> +"use strict"; +promise_test(async t => { + const caller = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => callee.close()); + + await setMediaPermission("granted", ["camera"]); + const stream = await navigator.mediaDevices.getUserMedia({video:true}); + const videoTrack = stream.getVideoTracks()[0]; + t.add_cleanup(() => videoTrack.stop()); + + const videoSender = caller.addTrack(videoTrack) + const senderStreams = videoSender.createEncodedStreams(); + const senderReader = senderStreams.readable.getReader(); + const senderWriter = senderStreams.writable.getWriter(); + + exchangeIceCandidates(caller, callee); + + // Send 10 frames and stop + const numFramesToSend = 10; + + const framesReceivedCorrectly = new Promise((resolve, reject) => { + callee.ontrack = async e => { + const receiverStreams = e.receiver.createEncodedStreams(); + const receiverReader = receiverStreams.readable.getReader(); + const receiverWriter = receiverStreams.writable.getWriter(); + + // This should all be able to happen in 5 seconds. + // For fast failures, uncomment this line. + // t.step_timeout(reject, 5000); + for (let i = 0; i < numFramesToSend; i++) { + const result = await receiverReader.read(); + // Write upstream, purely to avoid "no frame received" error messages + receiverWriter.write(result.value); + } + resolve(); + } + }); + + await exchangeOfferAnswer(caller, callee); + + for (let i = 0; i < numFramesToSend; i++) { + const result = await senderReader.read(); + senderWriter.write(structuredClone(result.value)); + } + return framesReceivedCorrectly; +}, "Cloning before sending works"); +</script> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedVideoFrame-serviceworker-failure.https.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedVideoFrame-serviceworker-failure.https.html new file mode 100644 index 0000000000..e725e5ce12 --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCEncodedVideoFrame-serviceworker-failure.https.html @@ -0,0 +1,60 @@ +<!DOCTYPE html> +<meta charset="utf-8"> +<!-- Based on similar tests in html/infrastructure/safe-passing-of-structured-data/shared-array-buffers/ --> +<title>RTCEncodedVideoFrame cannot cross agent clusters, service worker edition</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../../mediacapture-streams/permission-helper.js'></script> +<script src="../../webrtc/RTCPeerConnection-helper.js"></script> +<script src="../../service-workers/service-worker/resources/test-helpers.sub.js"></script> + +<script> +"use strict"; +promise_test(async t => { + const caller = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(); + t.add_cleanup(() => callee.close()); + + await setMediaPermission("granted", ["camera"]); + const stream = await navigator.mediaDevices.getUserMedia({video:true}); + const videoTrack = stream.getVideoTracks()[0]; + t.add_cleanup(() => videoTrack.stop()); + + const videoSender = caller.addTrack(videoTrack) + const senderStreams = videoSender.createEncodedStreams(); + const senderReader = senderStreams.readable.getReader(); + const senderWriter = senderStreams.writable.getWriter(); + + exchangeIceCandidates(caller, callee); + await exchangeOfferAnswer(caller, callee); + + const result = await senderReader.read(); + const scope = "resources/blank.html"; + const reg = await service_worker_unregister_and_register(t, "resources/serviceworker-failure.js", scope) + t.add_cleanup(() => service_worker_unregister(t, scope)); + await wait_for_state(t, reg.installing, "activated"); + const iframe = await with_iframe(scope); + t.add_cleanup(() => iframe.remove()); + const sw = iframe.contentWindow.navigator.serviceWorker; + let state = "start in window"; + return new Promise(resolve => { + sw.onmessage = t.step_func(e => { + if (e.data === "start in worker") { + assert_equals(state, "start in window"); + sw.controller.postMessage(result.value); + state = "we are expecting confirmation of an onmessageerror in the worker"; + } else if (e.data === "onmessageerror was received in worker") { + assert_equals(state, "we are expecting confirmation of an onmessageerror in the worker"); + resolve(); + } else { + assert_unreached("Got an unexpected message from the service worker: " + e.data); + } + }); + + sw.controller.postMessage(state); + }); +}); +</script> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-audio.https.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-audio.https.html new file mode 100644 index 0000000000..83d284146a --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-audio.https.html @@ -0,0 +1,228 @@ +<!DOCTYPE html> +<html> +<head> +<title>RTCPeerConnection Insertable Streams Audio</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../../mediacapture-streams/permission-helper.js'></script> +<script src="../../webrtc/RTCPeerConnection-helper.js"></script> +<script src="./RTCPeerConnection-insertable-streams.js"></script> +</head> +<body> +<script> +async function testAudioFlow(t, negotiationFunction, setConstructorParam, perFrameCallback = () => {}) { + const caller = new RTCPeerConnection(setConstructorParam ? {encodedInsertableStreams:true} : {}); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(setConstructorParam ? {encodedInsertableStreams:true} : {}); + t.add_cleanup(() => callee.close()); + + await setMediaPermission("granted", ["microphone"]); + const stream = await navigator.mediaDevices.getUserMedia({audio:true}); + const audioTrack = stream.getAudioTracks()[0]; + t.add_cleanup(() => audioTrack.stop()); + + const audioSender = caller.addTrack(audioTrack) + const senderStreams = audioSender.createEncodedStreams(); + const senderReader = senderStreams.readable.getReader(); + const senderWriter = senderStreams.writable.getWriter(); + + const frameInfos = []; + const numFramesPassthrough = 5; + const numFramesReplaceData = 5; + const numFramesModifyData = 5; + const numFramesToSend = numFramesPassthrough + numFramesReplaceData + numFramesModifyData; + + let streamsCreatedAtNegotiation; + + const ontrackPromise = new Promise(resolve => { + callee.ontrack = t.step_func(() => { + const audioReceiver = callee.getReceivers().find(r => r.track.kind === 'audio'); + assert_not_equals(audioReceiver, undefined); + + let receiverReader; + let receiverWriter; + if (streamsCreatedAtNegotiation) { + const audioStreams = streamsCreatedAtNegotiation.find(r => r.kind === 'audio'); + assert_true(!!audioStreams); + receiverReader = audioStreams.streams.readable.getReader(); + receiverWriter = audioStreams.streams.writable.getWriter(); + } else { + const receiverStreams = + audioReceiver.createEncodedStreams(); + receiverReader = receiverStreams.readable.getReader(); + receiverWriter = receiverStreams.writable.getWriter(); + } + + const maxFramesToReceive = numFramesToSend; + let numVerifiedFrames = 0; + for (let i = 0; i < maxFramesToReceive; i++) { + receiverReader.read().then(t.step_func(result => { + if (frameInfos[numVerifiedFrames] && + areFrameInfosEqual(result.value, frameInfos[numVerifiedFrames])) { + numVerifiedFrames++; + } else { + // Receiving unexpected frames is an indication that + // frames are not passed correctly between sender and receiver. + assert_unreached("Incorrect frame received"); + } + assert_not_equals(result.value.getMetadata().sequenceNumber, undefined); + + if (numVerifiedFrames == numFramesToSend) + resolve(); + })); + } + }); + }); + + exchangeIceCandidates(caller, callee); + await negotiationFunction(caller, callee, (streams) => {streamsCreatedAtNegotiation = streams;}); + + // Pass frames as they come from the encoder. + for (let i = 0; i < numFramesPassthrough; i++) { + const result = await senderReader.read(); + const frame = result.value; + perFrameCallback(frame); + frameInfos.push({ + data: frame.data, + timestamp: frame.timestamp, + type: frame.type, + metadata: frame.getMetadata(), + getMetadata() { return this.metadata; } + }); + senderWriter.write(frame); + } + + // Replace frame data with arbitrary buffers. + for (let i = 0; i < numFramesReplaceData; i++) { + const result = await senderReader.read() + const frame = result.value; + + const buffer = new ArrayBuffer(100); + const int8View = new Int8Array(buffer); + int8View.fill(i); + + frame.data = buffer; + perFrameCallback(frame); + frameInfos.push({ + data: frame.data, + timestamp: frame.timestamp, + type: frame.type, + metadata: frame.getMetadata(), + getMetadata() { return this.metadata; } + }); + senderWriter.write(frame); + } + + // Modify frame data. + for (let i = 0; i < numFramesReplaceData; i++) { + const result = await senderReader.read() + const frame = result.value; + const int8View = new Int8Array(frame.data); + int8View.fill(i); + + perFrameCallback(frame); + frameInfos.push({ + data: frame.data, + timestamp: frame.timestamp, + type: frame.type, + metadata: frame.getMetadata(), + getMetadata() { return this.metadata; } + }); + senderWriter.write(frame); + } + + return ontrackPromise; +} + +for (const setConstructorParam of [false, true]) { + promise_test(async t => { + return testAudioFlow(t, exchangeOfferAnswer, setConstructorParam); + }, 'Frames flow correctly using insertable streams' + (setConstructorParam ? ' with param' : '')); + + promise_test(async t => { + return testAudioFlow(t, exchangeOfferAnswerReverse, setConstructorParam); + }, 'Frames flow correctly using insertable streams when receiver starts negotiation' + (setConstructorParam ? ' with param' : '')); +} + +promise_test(async t => { + const caller = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(); + t.add_cleanup(() => callee.close()); + + const stream = await navigator.mediaDevices.getUserMedia({audio:true}); + const track = stream.getTracks()[0]; + t.add_cleanup(() => track.stop()); + + const sender = caller.addTrack(track) + const streams = sender.createEncodedStreams(); + const transformer = new TransformStream({ + transform(frame, controller) { + // Inserting the same frame twice will result in failure since the frame + // will be neutered after the first insertion is processed. + controller.enqueue(frame); + controller.enqueue(frame); + } + }); + + exchangeIceCandidates(caller, callee); + await exchangeOfferAnswer(caller, callee); + + await promise_rejects_dom( + t, 'OperationError', + streams.readable.pipeThrough(transformer).pipeTo(streams.writable)); +}, 'Enqueuing the same frame twice fails'); + +promise_test(async t => { + const caller = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller.close()); + const stream = await navigator.mediaDevices.getUserMedia({audio:true}); + const track = stream.getTracks()[0]; + t.add_cleanup(() => track.stop()); + const sender = caller.addTrack(track) + sender.createEncodedStreams(); + assert_throws_dom("InvalidStateError", () => sender.createEncodedStreams()); +}, 'Creating streams twice throws'); + +promise_test(async t => { + let clonedFrames = []; + function verifyFramesSerializeAndDeserialize(frame) { + // Clone encoded frames using structedClone (ie serialize + deserialize) and + // keep a reference. + const clone = structuredClone(frame); + clonedFrames.push(clone); + }; + + await testAudioFlow( + t, exchangeOfferAnswer, /*setConstructorParam=*/false, verifyFramesSerializeAndDeserialize); + + // Ensure all of our cloned frames are still alive and well, despite the + // originals having been sent through the PeerConnection. + clonedFrames.forEach((clonedFrame) => { + assert_not_equals(clonedFrame.data.size, 0); + assert_not_equals(clonedFrame.timestamp, 0); + }); +}, 'Encoded frames serialize and deserialize into a deep clone'); + +promise_test(async t => { + let clonedFrames = []; + function rewriteFrameTimestamps(frame) { + // Add 1 to the rtp timestamp of the frame. + const metadata = frame.getMetadata(); + metadata.rtpTimestamp += 1; + frame.setMetadata(metadata); + + assert_equals(frame.getMetadata().rtpTimestamp, metadata.rtpTimestamp) + }; + + // Run audio flows which will assert that the frames received have the + // rtp timestamp set by our modification. + await testAudioFlow( + t, exchangeOfferAnswer, /*setConstructorParam=*/false, rewriteFrameTimestamps); +}, 'Modifying rtp timestamp'); + +</script> +</body> +</html> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-errors.https.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-errors.https.html new file mode 100644 index 0000000000..a0c68c400a --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-errors.https.html @@ -0,0 +1,46 @@ +<!DOCTYPE html> +<html> +<head> +<title>RTCPeerConnection Insertable Streams - Errors</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../../mediacapture-streams/permission-helper.js'></script> +<script src="../../webrtc/RTCPeerConnection-helper.js"></script> +<script src="./RTCPeerConnection-insertable-streams.js"></script> +</head> +<body> +<script> +promise_test(async t => { + const caller = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(); + t.add_cleanup(() => callee.close()); + + const stream = await navigator.mediaDevices.getUserMedia({video:true}); + const track = stream.getTracks()[0]; + t.add_cleanup(() => track.stop()); + + const sender = caller.addTrack(track) + const streams = sender.createEncodedStreams(); + const transformer = new TransformStream({ + transform(frame, controller) { + // Inserting the same frame twice will result in failure since the frame + // will be neutered after the first insertion is processed. + controller.enqueue(frame); + controller.enqueue(frame); + } + }); + + exchangeIceCandidates(caller, callee); + await exchangeOfferAnswer(caller, callee); + + await promise_rejects_dom( + t, 'OperationError', + streams.readable.pipeThrough(transformer).pipeTo(streams.writable)); +}, 'Enqueuing the same frame twice fails'); + +</script> +</body> +</html> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-simulcast.https.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-simulcast.https.html new file mode 100644 index 0000000000..834644674e --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-simulcast.https.html @@ -0,0 +1,89 @@ +<!doctype html> +<meta charset=utf-8> +<title>RTCPeerConnection Insertable Streams Simulcast</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../../mediacapture-streams/permission-helper.js'></script> +<script src="../../webrtc/RTCPeerConnection-helper.js"></script> +<script src="../../webrtc/third_party/sdp/sdp.js"></script> +<script src="../../webrtc/simulcast/simulcast.js"></script> +<script> +// Test based on wpt/webrtc/simulcast/basic.https.html +promise_test(async t => { + const rids = [0, 1, 2]; + const pc1 = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => pc1.close()); + const pc2 = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => pc2.close()); + + exchangeIceCandidates(pc1, pc2); + + const metadataToBeLoaded = []; + let receiverSSRCs = [] + pc2.ontrack = t.step_func(e => { + const receiverTransformer = new TransformStream({ + async transform(encodedFrame, controller) { + let ssrc = encodedFrame.getMetadata().synchronizationSource; + if (receiverSSRCs.indexOf(ssrc) == -1) + receiverSSRCs.push(ssrc); + controller.enqueue(encodedFrame); + } + }); + const receiverStreams = e.receiver.createEncodedStreams(); + receiverStreams.readable + .pipeThrough(receiverTransformer) + .pipeTo(receiverStreams.writable); + + const stream = e.streams[0]; + const v = document.createElement('video'); + v.autoplay = true; + v.srcObject = stream; + v.id = stream.id + metadataToBeLoaded.push(new Promise((resolve) => { + v.addEventListener('loadedmetadata', () => { + resolve(); + }); + })); + }); + + await setMediaPermission("granted", ["camera"]); + const stream = await navigator.mediaDevices.getUserMedia({video: {width: 1280, height: 720}}); + t.add_cleanup(() => stream.getTracks().forEach(track => track.stop())); + const transceiver = pc1.addTransceiver(stream.getVideoTracks()[0], { + streams: [stream], + sendEncodings: rids.map(rid => {rid}), + }); + const senderStreams = transceiver.sender.createEncodedStreams(); + let senderSSRCs = []; + const senderTransformer = new TransformStream({ + async transform(encodedFrame, controller) { + if (senderSSRCs.indexOf(encodedFrame.getMetadata().synchronizationSource) == -1) + senderSSRCs.push(encodedFrame.getMetadata().synchronizationSource); + controller.enqueue(encodedFrame); + } + }); + senderStreams.readable + .pipeThrough(senderTransformer) + .pipeTo(senderStreams.writable); + + const offer = await pc1.createOffer(); + await pc1.setLocalDescription(offer), + await pc2.setRemoteDescription({ + type: 'offer', + sdp: swapRidAndMidExtensionsInSimulcastOffer(offer, rids), + }); + const answer = await pc2.createAnswer(); + await pc2.setLocalDescription(answer); + await pc1.setRemoteDescription({ + type: 'answer', + sdp: swapRidAndMidExtensionsInSimulcastAnswer(answer, pc1.localDescription, rids), + }); + assert_equals(metadataToBeLoaded.length, 3); + await Promise.all(metadataToBeLoaded); + // Ensure that frames from the 3 simulcast layers are exposed. + assert_equals(senderSSRCs.length, 3); + assert_equals(receiverSSRCs.length, 3); +}, 'Basic simulcast setup with three spatial layers'); +</script> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-video-frames.https.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-video-frames.https.html new file mode 100644 index 0000000000..d3db116ff6 --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-video-frames.https.html @@ -0,0 +1,80 @@ +<!DOCTYPE html> +<html> +<head> +<title>RTCPeerConnection Insertable Streams - Video Frames</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../../mediacapture-streams/permission-helper.js'></script> +<script src="../../webrtc/RTCPeerConnection-helper.js"></script> +<script src="./RTCPeerConnection-insertable-streams.js"></script> +</head> +<body> +<script> +promise_test(async t => { + const caller = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => callee.close()); + + await setMediaPermission("granted", ["camera"]); + const stream = await navigator.mediaDevices.getUserMedia({video:true}); + const track = stream.getTracks()[0]; + t.add_cleanup(() => track.stop()); + + const sender = caller.addTrack(track) + const senderStreams = sender.createEncodedStreams(); + const senderReader = senderStreams.readable.getReader(); + const senderWriter = senderStreams.writable.getWriter(); + const numFramesToSend = 20; + + const ontrackPromise = new Promise((resolve, reject) => { + callee.ontrack = async e => { + const receiverStreams = e.receiver.createEncodedStreams(); + const receiverReader = receiverStreams.readable.getReader(); + + let numReceivedKeyFrames = 0; + let numReceivedDeltaFrames = 0; + for (let i = 0; i < numFramesToSend; i++) { + const result = await receiverReader.read(); + if (result.value.type == 'key') + numReceivedKeyFrames++; + else if (result.value.type == 'delta') + numReceivedDeltaFrames++; + + if (numReceivedKeyFrames > 0 && numReceivedDeltaFrames > 0) + resolve(); + else if (numReceivedKeyFrames + numReceivedDeltaFrames >= numFramesToSend) + reject(); + } + } + }); + + exchangeIceCandidates(caller, callee); + await exchangeOfferAnswer(caller, callee); + + let numSentKeyFrames = 0; + let numSentDeltaFrames = 0; + // Pass frames as they come from the encoder. + for (let i = 0; i < numFramesToSend; i++) { + const result = await senderReader.read(); + verifyNonstandardAdditionalDataIfPresent(result.value); + if (result.value.type == 'key') { + numSentKeyFrames++; + } else { + numSentDeltaFrames++; + } + + senderWriter.write(result.value); + } + + assert_greater_than(numSentKeyFrames, 0); + assert_greater_than(numSentDeltaFrames, 0); + + return ontrackPromise; +}, 'Key and Delta frames are sent and received'); + +</script> +</body> +</html> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-video.https.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-video.https.html new file mode 100644 index 0000000000..5334b8d1f9 --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-video.https.html @@ -0,0 +1,182 @@ +<!DOCTYPE html> +<html> +<head> +<title>RTCPeerConnection Insertable Streams - Video</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../../mediacapture-streams/permission-helper.js'></script> +<script src="../../webrtc/RTCPeerConnection-helper.js"></script> +<script src="./RTCPeerConnection-insertable-streams.js"></script> +</head> +<body> +<script> +async function testVideoFlow(t, negotiationFunction, setConstructorParam, frameCallback = () => {}) { + const caller = new RTCPeerConnection(setConstructorParam ? {encodedInsertableStreams:true} : {}); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(setConstructorParam ? {encodedInsertableStreams:true} : {}); + t.add_cleanup(() => callee.close()); + + await setMediaPermission("granted", ["camera"]); + const stream = await navigator.mediaDevices.getUserMedia({video:true}); + const videoTrack = stream.getVideoTracks()[0]; + t.add_cleanup(() => videoTrack.stop()); + + const videoSender = caller.addTrack(videoTrack) + const senderStreams = videoSender.createEncodedStreams(); + const senderReader = senderStreams.readable.getReader(); + const senderWriter = senderStreams.writable.getWriter(); + + const frameInfos = []; + const numFramesPassthrough = 5; + const numFramesReplaceData = 5; + const numFramesModifyData = 5; + const numFramesToSend = numFramesPassthrough + numFramesReplaceData + numFramesModifyData; + + let streamsCreatedAtNegotiation; + const ontrackPromise = new Promise(resolve => { + callee.ontrack = t.step_func(() => { + const videoReceiver = callee.getReceivers().find(r => r.track.kind === 'video'); + assert_not_equals(videoReceiver, undefined); + + let receiverReader; + let receiverWriter; + if (streamsCreatedAtNegotiation) { + const videoStreams = streamsCreatedAtNegotiation.find(r => r.kind === 'video'); + assert_true(!!videoStreams); + receiverReader = videoStreams.streams.readable.getReader(); + receiverWriter = videoStreams.streams.writable.getWriter(); + } else { + const receiverStreams = + videoReceiver.createEncodedStreams(); + receiverReader = receiverStreams.readable.getReader(); + receiverWriter = receiverStreams.writable.getWriter(); + } + + const maxFramesToReceive = numFramesToSend; + let numVerifiedFrames = 0; + for (let i = 0; i < maxFramesToReceive; i++) { + receiverReader.read().then(t.step_func(result => { + verifyNonstandardAdditionalDataIfPresent(result.value); + if (frameInfos[numVerifiedFrames] && + areFrameInfosEqual(result.value, frameInfos[numVerifiedFrames])) { + numVerifiedFrames++; + } else { + // Receiving unexpected frames is an indication that + // frames are not passed correctly between sender and receiver. + assert_unreached("Incorrect frame received"); + } + + if (numVerifiedFrames == numFramesToSend) + resolve(); + })); + } + }); + }); + + exchangeIceCandidates(caller, callee); + await negotiationFunction(caller, callee, (streams) => {streamsCreatedAtNegotiation = streams;}); + + // Pass frames as they come from the encoder. + for (let i = 0; i < numFramesPassthrough; i++) { + const result = await senderReader.read(); + const frame = result.value; + const metadata = frame.getMetadata(); + assert_true(containsVideoMetadata(metadata)); + verifyNonstandardAdditionalDataIfPresent(frame); + frameInfos.push({ + timestamp: frame.timestamp, + type: frame.type, + data: frame.data, + metadata: metadata, + getMetadata() { return this.metadata; } + }); + frameCallback(frame); + senderWriter.write(frame); + } + + // Replace frame data with arbitrary buffers. + for (let i = 0; i < numFramesReplaceData; i++) { + const result = await senderReader.read(); + const metadata = result.value.getMetadata(); + assert_true(containsVideoMetadata(metadata)); + const buffer = new ArrayBuffer(100); + const int8View = new Int8Array(buffer); + int8View.fill(i); + + result.value.data = buffer; + frameInfos.push({ + timestamp: result.value.timestamp, + type: result.value.type, + data: result.value.data, + metadata: metadata, + getMetadata() { return this.metadata; } + }); + senderWriter.write(result.value); + } + + // Modify frame data. + for (let i = 0; i < numFramesReplaceData; i++) { + const result = await senderReader.read(); + const metadata = result.value.getMetadata(); + assert_true(containsVideoMetadata(metadata)); + const int8View = new Int8Array(result.value.data); + int8View.fill(i); + + frameInfos.push({ + timestamp: result.value.timestamp, + type: result.value.type, + data: result.value.data, + metadata: metadata, + getMetadata() { return this.metadata; } + }); + senderWriter.write(result.value); + } + + return ontrackPromise; +} + +for (const setConstructorParam of [false, true]) { + promise_test(async t => { + return testVideoFlow(t, exchangeOfferAnswer, setConstructorParam); + }, 'Frames flow correctly using insertable streams' + (setConstructorParam ? ' with param' : '')); + + promise_test(async t => { + return testVideoFlow(t, exchangeOfferAnswerReverse, setConstructorParam); + }, 'Frames flow correctly using insertable streams when receiver starts negotiation' + (setConstructorParam ? ' with param' : '')); +} + +promise_test(async t => { + const caller = new RTCPeerConnection(); + t.add_cleanup(() => caller.close()); + const stream = await navigator.mediaDevices.getUserMedia({video:true}); + const track = stream.getTracks()[0]; + t.add_cleanup(() => track.stop()); + const sender = caller.addTrack(track) + sender.createEncodedStreams(); + assert_throws_dom("InvalidStateError", () => sender.createEncodedStreams()); +}, 'Creating streams twice throws'); + +promise_test(async t => { + let clonedFrames = []; + function verifyFramesSerializeAndDeserialize(frame) { + // Clone encoded frames using structedClone (ie serialize + deserialize) and + // keep a reference. + const clone = structuredClone(frame); + clonedFrames.push(clone); + }; + + await testVideoFlow(t, exchangeOfferAnswer, /*setConstructorParam=*/false, verifyFramesSerializeAndDeserialize); + + // Ensure all of our cloned frames are still alive and well, despite the + // originals having been sent through the PeerConnection. + clonedFrames.forEach((clonedFrame) => { + assert_not_equals(clonedFrame.data.size, 0); + assert_not_equals(clonedFrame.type, "empty") + }); +}, 'Encoded frames serialize and deserialize into a deep clone'); + +</script> +</body> +</html> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-worker.https.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-worker.https.html new file mode 100644 index 0000000000..94943f8b69 --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams-worker.https.html @@ -0,0 +1,196 @@ +<!DOCTYPE html> +<html> +<head> +<title>RTCPeerConnection Insertable Streams - Worker</title> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script src=/resources/testdriver.js></script> +<script src=/resources/testdriver-vendor.js></script> +<script src='../../mediacapture-streams/permission-helper.js'></script> +<script src="../../webrtc/RTCPeerConnection-helper.js"></script> +<script src="./RTCPeerConnection-insertable-streams.js"></script> +</head> +<body> +<script> +promise_test(async t => { + const caller = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(); + t.add_cleanup(() => callee.close()); + + // Video is used in a later test, so we ask for both permissions + await setMediaPermission(); + const stream = await navigator.mediaDevices.getUserMedia({audio:true}); + const track = stream.getTracks()[0]; + t.add_cleanup(() => track.stop()); + + const sender = caller.addTrack(track) + const senderStreams = sender.createEncodedStreams(); + + const senderWorker = new Worker('RTCPeerConnection-sender-worker-single-frame.js') + t.add_cleanup(() => senderWorker.terminate()); + senderWorker.postMessage( + {readableStream: senderStreams.readable}, + [senderStreams.readable]); + + let expectedFrameData = null; + let verifiedFrameData = false; + let numVerifiedFrames = 0; + const onmessagePromise = new Promise(resolve => { + senderWorker.onmessage = t.step_func(message => { + if (!(message.data instanceof RTCEncodedAudioFrame)) { + // This is the first message sent from the Worker to the test. + // It contains an object (not an RTCEncodedAudioFrame) with the same + // fields as the RTCEncodedAudioFrame to be sent in follow-up messages. + // These serve as expected values to validate that the + // RTCEncodedAudioFrame is sent correctly back to the test in the next + // message. + expectedFrameData = message.data; + } else { + // This is the frame sent by the Worker after reading it from the + // readable stream. The Worker sends it twice after sending the + // verification message. + assert_equals(message.data.type, expectedFrameData.type); + assert_equals(message.data.timestamp, expectedFrameData.timestamp); + assert_true(areArrayBuffersEqual(message.data.data, expectedFrameData.data)); + if (++numVerifiedFrames == 2) + resolve(); + } + }); + }); + + exchangeIceCandidates(caller, callee); + await exchangeOfferAnswer(caller, callee); + + return onmessagePromise; +}, 'RTCRtpSender readable stream transferred to a Worker and the Worker sends an RTCEncodedAudioFrame back'); + +promise_test(async t => { + const caller = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(); + t.add_cleanup(() => callee.close()); + + const stream = await navigator.mediaDevices.getUserMedia({video:true}); + const videoTrack = stream.getVideoTracks()[0]; + t.add_cleanup(() => videoTrack.stop()); + + const videoSender = caller.addTrack(videoTrack) + const senderStreams = videoSender.createEncodedStreams(); + + const senderWorker = new Worker('RTCPeerConnection-sender-worker-single-frame.js') + t.add_cleanup(() => senderWorker.terminate()); + senderWorker.postMessage( + {readableStream: senderStreams.readable}, + [senderStreams.readable]); + + let expectedFrameData = null; + let verifiedFrameData = false; + let numVerifiedFrames = 0; + const onmessagePromise = new Promise(resolve => { + senderWorker.onmessage = t.step_func(message => { + if (!(message.data instanceof RTCEncodedVideoFrame)) { + // This is the first message sent from the Worker to the test. + // It contains an object (not an RTCEncodedVideoFrame) with the same + // fields as the RTCEncodedVideoFrame to be sent in follow-up messages. + // These serve as expected values to validate that the + // RTCEncodedVideoFrame is sent correctly back to the test in the next + // message. + expectedFrameData = message.data; + } else { + // This is the frame sent by the Worker after reading it from the + // readable stream. The Worker sends it twice after sending the + // verification message. + assert_equals(message.data.type, expectedFrameData.type); + assert_equals(message.data.timestamp, expectedFrameData.timestamp); + assert_true(areArrayBuffersEqual(message.data.data, expectedFrameData.data)); + assert_equals(message.data.getMetadata().synchronizationSource, expectedFrameData.metadata.synchronizationSource); + if (++numVerifiedFrames == 2) + resolve(); + } + }); + }); + + exchangeIceCandidates(caller, callee); + await exchangeOfferAnswer(caller, callee); + + return onmessagePromise; +}, 'RTCRtpSender readable stream transferred to a Worker and the Worker sends an RTCEncodedVideoFrame back'); + +promise_test(async t => { + const caller = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(); + t.add_cleanup(() => callee.close()); + + const stream = await navigator.mediaDevices.getUserMedia({video:true}); + const videoTrack = stream.getVideoTracks()[0]; + t.add_cleanup(() => videoTrack.stop()); + + const videoSender = caller.addTrack(videoTrack) + const senderStreams = videoSender.createEncodedStreams(); + + const senderWorker = new Worker('RTCPeerConnection-worker-transform.js') + t.add_cleanup(() => senderWorker.terminate()); + senderWorker.postMessage( + { + readableStream: senderStreams.readable, + writableStream: senderStreams.writable, + insertError: true + }, + [senderStreams.readable, senderStreams.writable]); + + const onmessagePromise = new Promise(resolve => { + senderWorker.onmessage = t.step_func(message => { + assert_false(message.data.success); + assert_true(message.data.error instanceof TypeError); + resolve(); + }); + }); + + exchangeIceCandidates(caller, callee); + await exchangeOfferAnswer(caller, callee); + + return onmessagePromise; +}, 'Video RTCRtpSender insertable streams transferred to a worker, which tries to write an invalid frame'); + +promise_test(async t => { + const caller = new RTCPeerConnection({encodedInsertableStreams:true}); + t.add_cleanup(() => caller.close()); + const callee = new RTCPeerConnection(); + t.add_cleanup(() => callee.close()); + + const stream = await navigator.mediaDevices.getUserMedia({audio:true}); + const audioTrack = stream.getAudioTracks()[0]; + t.add_cleanup(() => audioTrack.stop()); + + const audioSender = caller.addTrack(audioTrack) + const senderStreams = audioSender.createEncodedStreams(); + + const senderWorker = new Worker('RTCPeerConnection-worker-transform.js') + t.add_cleanup(() => senderWorker.terminate()); + senderWorker.postMessage( + { + readableStream: senderStreams.readable, + writableStream: senderStreams.writable, + insertError: true + }, + [senderStreams.readable, senderStreams.writable]); + + const onmessagePromise = new Promise(resolve => { + senderWorker.onmessage = t.step_func(message => { + assert_false(message.data.success); + assert_true(message.data.error instanceof TypeError); + resolve(); + }); + }); + + exchangeIceCandidates(caller, callee); + await exchangeOfferAnswer(caller, callee); + + return onmessagePromise; +}, 'Audio RTCRtpSender insertable streams transferred to a worker, which tries to write an invalid frame'); + +</script> +</body> +</html> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams.js b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams.js new file mode 100644 index 0000000000..0bf820acde --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-insertable-streams.js @@ -0,0 +1,262 @@ +function areArrayBuffersEqual(buffer1, buffer2) +{ + if (buffer1.byteLength !== buffer2.byteLength) { + return false; + } + let array1 = new Int8Array(buffer1); + var array2 = new Int8Array(buffer2); + for (let i = 0 ; i < buffer1.byteLength ; ++i) { + if (array1[i] !== array2[i]) { + return false; + } + } + return true; +} + +function areArraysEqual(a1, a2) { + if (a1 === a1) + return true; + if (a1.length != a2.length) + return false; + for (let i = 0; i < a1.length; i++) { + if (a1[i] != a2[i]) + return false; + } + return true; +} + +function areMetadataEqual(metadata1, metadata2, type) { + return metadata1.synchronizationSource === metadata2.synchronizationSource && + metadata1.payloadType == metadata2.payloadType && + areArraysEqual( + metadata1.contributingSources, metadata2.contributingSources) && + metadata1.absCaptureTime == metadata2.absCaptureTime && + metadata1.frameId === metadata2.frameId && + areArraysEqual(metadata1.dependencies, metadata2.dependencies) && + metadata1.spatialIndex === metadata2.spatialIndex && + metadata1.temporalIndex === metadata2.temporalIndex && + // Width and height are reported only for key frames on the receiver + // side. + type == 'key' ? + metadata1.width === metadata2.width && + metadata1.height === metadata2.height : + true; +} + +function areFrameInfosEqual(frame1, frame2) { + return frame1.timestamp === frame2.timestamp && + frame1.type === frame2.type && + areMetadataEqual(frame1.getMetadata(), frame2.getMetadata(), frame1.type) && + areArrayBuffersEqual(frame1.data, frame2.data); +} + +function containsVideoMetadata(metadata) { + return metadata.synchronizationSource !== undefined && + metadata.width !== undefined && + metadata.height !== undefined && + metadata.spatialIndex !== undefined && + metadata.temporalIndex !== undefined && + metadata.dependencies !== undefined; +} + +function enableExtension(sdp, extension) { + if (sdp.indexOf(extension) !== -1) + return sdp; + + const extensionIds = sdp.trim().split('\n') + .map(line => line.trim()) + .filter(line => line.startsWith('a=extmap:')) + .map(line => line.split(' ')[0].substr(9)) + .map(id => parseInt(id, 10)) + .sort((a, b) => a - b); + for (let newId = 1; newId <= 15; newId++) { + if (!extensionIds.includes(newId)) { + return sdp += 'a=extmap:' + newId + ' ' + extension + '\r\n'; + } + } + if (sdp.indexOf('a=extmap-allow-mixed') !== -1) { // Pick the next highest one. + const newId = extensionIds[extensionIds.length - 1] + 1; + return sdp += 'a=extmap:' + newId + ' ' + extension + '\r\n'; + } + throw 'Could not find free extension id to use for ' + extension; +} + +const GFD_V00_EXTENSION = + 'http://www.webrtc.org/experiments/rtp-hdrext/generic-frame-descriptor-00'; +const ABS_V00_EXTENSION = + 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time'; + +async function exchangeOfferAnswer(pc1, pc2) { + const offer = await pc1.createOffer(); + // Munge the SDP to enable the GFD and ACT extension in order to get correct + // metadata. + const sdpABS = enableExtension(offer.sdp, ABS_V00_EXTENSION); + const sdpGFD = enableExtension(sdpABS, GFD_V00_EXTENSION); + await pc1.setLocalDescription({type: offer.type, sdp: sdpGFD}); + // Munge the SDP to disable bandwidth probing via RTX. + // TODO(crbug.com/1066819): remove this hack when we do not receive duplicates from RTX + // anymore. + const sdpRTX = sdpGFD.replace(new RegExp('rtx', 'g'), 'invalid'); + await pc2.setRemoteDescription({type: 'offer', sdp: sdpRTX}); + + const answer = await pc2.createAnswer(); + await pc2.setLocalDescription(answer); + await pc1.setRemoteDescription(answer); +} + +async function exchangeOfferAnswerReverse(pc1, pc2, encodedStreamsCallback) { + const offer = await pc2.createOffer({offerToReceiveAudio: true, offerToReceiveVideo: true}); + if (encodedStreamsCallback) { + // RTCRtpReceivers will have been created during the above createOffer call, so if the caller + // wants to createEncodedStreams synchronously after creation to ensure all frames pass + // through the transform, it will have to be done now. + encodedStreamsCallback( + pc2.getReceivers().map(r => { + return {kind: r.track.kind, streams: r.createEncodedStreams()}; + })); + } + + // Munge the SDP to enable the GFD extension in order to get correct metadata. + const sdpABS = enableExtension(offer.sdp, ABS_V00_EXTENSION); + const sdpGFD = enableExtension(sdpABS, GFD_V00_EXTENSION); + // Munge the SDP to disable bandwidth probing via RTX. + // TODO(crbug.com/1066819): remove this hack when we do not receive duplicates from RTX + // anymore. + const sdpRTX = sdpGFD.replace(new RegExp('rtx', 'g'), 'invalid'); + await pc1.setRemoteDescription({type: 'offer', sdp: sdpRTX}); + await pc2.setLocalDescription({type: 'offer', sdp: sdpGFD}); + + const answer = await pc1.createAnswer(); + await pc2.setRemoteDescription(answer); + await pc1.setLocalDescription(answer); +} + +function createFrameDescriptor(videoFrame) { + const kMaxSpatialLayers = 8; + const kMaxTemporalLayers = 8; + const kMaxNumFrameDependencies = 8; + + const metadata = videoFrame.getMetadata(); + let frameDescriptor = { + beginningOfSubFrame: true, + endOfSubframe: false, + frameId: metadata.frameId & 0xFFFF, + spatialLayers: 1 << metadata.spatialIndex, + temporalLayer: metadata.temporalLayer, + frameDependenciesDiffs: [], + width: 0, + height: 0 + }; + + for (const dependency of metadata.dependencies) { + frameDescriptor.frameDependenciesDiffs.push(metadata.frameId - dependency); + } + if (metadata.dependencies.length == 0) { + frameDescriptor.width = metadata.width; + frameDescriptor.height = metadata.height; + } + return frameDescriptor; +} + +function additionalDataSize(descriptor) { + if (!descriptor.beginningOfSubFrame) { + return 1; + } + + let size = 4; + for (const fdiff of descriptor.frameDependenciesDiffs) { + size += (fdiff >= (1 << 6)) ? 2 : 1; + } + if (descriptor.beginningOfSubFrame && + descriptor.frameDependenciesDiffs.length == 0 && + descriptor.width > 0 && + descriptor.height > 0) { + size += 4; + } + + return size; +} + +// Compute the buffer reported in the additionalData field using the metadata +// provided by a video frame. +// Based on the webrtc::RtpDescriptorAuthentication() C++ function at +// https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/rtp_rtcp/source/rtp_descriptor_authentication.cc +function computeAdditionalData(videoFrame) { + const kMaxSpatialLayers = 8; + const kMaxTemporalLayers = 8; + const kMaxNumFrameDependencies = 8; + + const metadata = videoFrame.getMetadata(); + if (metadata.spatialIndex < 0 || + metadata.temporalIndex < 0 || + metadata.spatialIndex >= kMaxSpatialLayers || + metadata.temporalIndex >= kMaxTemporalLayers || + metadata.dependencies.length > kMaxNumFrameDependencies) { + return new ArrayBuffer(0); + } + + const descriptor = createFrameDescriptor(videoFrame); + const size = additionalDataSize(descriptor); + const additionalData = new ArrayBuffer(size); + const data = new Uint8Array(additionalData); + + const kFlagBeginOfSubframe = 0x80; + const kFlagEndOfSubframe = 0x40; + const kFlagFirstSubframeV00 = 0x20; + const kFlagLastSubframeV00 = 0x10; + + const kFlagDependencies = 0x08; + const kFlagMoreDependencies = 0x01; + const kFlageXtendedOffset = 0x02; + + let baseHeader = + (descriptor.beginningOfSubFrame ? kFlagBeginOfSubframe : 0) | + (descriptor.endOfSubFrame ? kFlagEndOfSubframe : 0); + baseHeader |= kFlagFirstSubframeV00; + baseHeader |= kFlagLastSubframeV00; + + if (!descriptor.beginningOfSubFrame) { + data[0] = baseHeader; + return additionalData; + } + + data[0] = + baseHeader | + (descriptor.frameDependenciesDiffs.length == 0 ? 0 : kFlagDependencies) | + descriptor.temporalLayer; + data[1] = descriptor.spatialLayers; + data[2] = descriptor.frameId & 0xFF; + data[3] = descriptor.frameId >> 8; + + const fdiffs = descriptor.frameDependenciesDiffs; + let offset = 4; + if (descriptor.beginningOfSubFrame && + fdiffs.length == 0 && + descriptor.width > 0 && + descriptor.height > 0) { + data[offset++] = (descriptor.width >> 8); + data[offset++] = (descriptor.width & 0xFF); + data[offset++] = (descriptor.height >> 8); + data[offset++] = (descriptor.height & 0xFF); + } + for (let i = 0; i < fdiffs.length; i++) { + const extended = fdiffs[i] >= (1 << 6); + const more = i < fdiffs.length - 1; + data[offset++] = ((fdiffs[i] & 0x3f) << 2) | + (extended ? kFlageXtendedOffset : 0) | + (more ? kFlagMoreDependencies : 0); + if (extended) { + data[offset++] = fdiffs[i] >> 6; + } + } + return additionalData; +} + +function verifyNonstandardAdditionalDataIfPresent(videoFrame) { + if (videoFrame.additionalData === undefined) + return; + + const computedData = computeAdditionalData(videoFrame); + assert_true(areArrayBuffersEqual(videoFrame.additionalData, computedData)); +} + diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-sender-worker-single-frame.js b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-sender-worker-single-frame.js new file mode 100644 index 0000000000..c943dafe5b --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-sender-worker-single-frame.js @@ -0,0 +1,19 @@ +onmessage = async (event) => { + const readableStream = event.data.readableStream; + const reader = readableStream.getReader(); + const result = await reader.read(); + + // Post an object with individual fields so that the test side has + // values to verify the serialization of the RTCEncodedVideoFrame. + postMessage({ + type: result.value.type, + timestamp: result.value.timestamp, + data: result.value.data, + metadata: result.value.getMetadata(), + }); + + // Send the frame twice to verify that the frame does not change after the + // first serialization. + postMessage(result.value); + postMessage(result.value); +} diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-worker-transform.js b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-worker-transform.js new file mode 100644 index 0000000000..36e3949e4d --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/RTCPeerConnection-worker-transform.js @@ -0,0 +1,22 @@ +onmessage = async (event) => { + const readableStream = event.data.readableStream; + const writableStream = event.data.writableStream; + const insertError = event.data.insertError; + + try { + await readableStream.pipeThrough(new TransformStream({ + transform: (chunk, controller) => { + if (insertError) { + controller.enqueue("This is not a valid frame"); + } else { + controller.enqueue(chunk); + } + } + })).pipeTo(writableStream); + + postMessage({success:true}); + } catch(e) { + postMessage({success:false, error: e}); + } + +} diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/resources/blank.html b/testing/web-platform/tests/webrtc-encoded-transform/tentative/resources/blank.html new file mode 100644 index 0000000000..a3c3a4689a --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/resources/blank.html @@ -0,0 +1,2 @@ +<!DOCTYPE html> +<title>Empty doc</title> diff --git a/testing/web-platform/tests/webrtc-encoded-transform/tentative/resources/serviceworker-failure.js b/testing/web-platform/tests/webrtc-encoded-transform/tentative/resources/serviceworker-failure.js new file mode 100644 index 0000000000..e7aa8e11be --- /dev/null +++ b/testing/web-platform/tests/webrtc-encoded-transform/tentative/resources/serviceworker-failure.js @@ -0,0 +1,30 @@ +// Based on similar tests in html/infrastructure/safe-passing-of-structured-data/shared-array-buffers/. +"use strict"; +self.importScripts("/resources/testharness.js"); + +let state = "start in worker"; + +self.onmessage = e => { + if (e.data === "start in window") { + assert_equals(state, "start in worker"); + e.source.postMessage(state); + state = "we are expecting a messageerror due to the window sending us an RTCEncodedVideoFrame or RTCEncodedAudioFrame"; + } else { + e.source.postMessage(`worker onmessage was reached when in state "${state}" and data ${e.data}`); + } +}; + +self.onmessageerror = e => { + if (state === "we are expecting a messageerror due to the window sending us an RTCEncodedVideoFrame or RTCEncodedAudioFrame") { + assert_equals(e.constructor.name, "ExtendableMessageEvent", "type"); + assert_equals(e.data, null, "data"); + assert_equals(e.origin, self.origin, "origin"); + assert_not_equals(e.source, null, "source"); + assert_equals(e.ports.length, 0, "ports length"); + + state = "onmessageerror was received in worker"; + e.source.postMessage(state); + } else { + e.source.postMessage(`worker onmessageerror was reached when in state "${state}" and data ${e.data}`); + } +}; |