From 086c044dc34dfc0f74fbe41f4ecb402b2cd34884 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 03:13:33 +0200 Subject: Merging upstream version 125.0.1. Signed-off-by: Daniel Baumann --- dom/media/ADTSDemuxer.cpp | 247 +------ dom/media/ADTSDemuxer.h | 16 +- dom/media/AudibilityMonitor.h | 2 +- dom/media/AudioSampleFormat.h | 234 ++++--- dom/media/AudioSegment.h | 10 +- dom/media/CallbackThreadRegistry.cpp | 1 + dom/media/ChannelMediaDecoder.cpp | 8 +- dom/media/ExternalEngineStateMachine.cpp | 126 ++-- dom/media/ExternalEngineStateMachine.h | 15 +- dom/media/IdpSandbox.sys.mjs | 2 +- dom/media/ImageToI420.cpp | 8 +- dom/media/MediaData.h | 14 +- dom/media/MediaDecoder.cpp | 18 +- dom/media/MediaDecoder.h | 25 + dom/media/MediaDecoderStateMachineBase.cpp | 4 +- dom/media/MediaDecoderStateMachineBase.h | 4 +- dom/media/MediaDevices.cpp | 6 +- dom/media/MediaInfo.h | 14 +- dom/media/MediaManager.cpp | 75 +-- dom/media/MediaQueue.h | 86 ++- dom/media/MediaStreamTrack.cpp | 2 +- dom/media/MediaTrackGraph.cpp | 6 + dom/media/PeerConnection.sys.mjs | 139 ++-- dom/media/VideoFrameConverter.h | 21 +- dom/media/VideoUtils.cpp | 3 - .../test/browser/browser_autoplay_videoDocument.js | 4 +- dom/media/autoplay/test/browser/head.js | 8 +- .../file_autoplay_policy_eventdown_activation.html | 4 +- .../file_autoplay_policy_key_blacklist.html | 2 +- ...autoplay_policy_play_before_loadedmetadata.html | 2 +- .../file_autoplay_policy_unmute_pauses.html | 4 +- .../test/browser/browser_doctor_notification.js | 14 +- dom/media/eme/EMEUtils.cpp | 39 +- dom/media/eme/EMEUtils.h | 3 + dom/media/eme/KeySystemConfig.cpp | 37 +- dom/media/eme/KeySystemConfig.h | 7 + dom/media/eme/MediaKeySession.cpp | 23 +- dom/media/eme/MediaKeySession.h | 7 +- dom/media/eme/MediaKeySystemAccess.cpp | 85 ++- dom/media/eme/MediaKeySystemAccess.h | 3 +- dom/media/eme/MediaKeySystemAccessManager.cpp | 3 +- dom/media/eme/MediaKeys.cpp | 17 +- .../eme/mediadrm/MediaDrmCDMCallbackProxy.cpp | 1 - dom/media/eme/mediafoundation/WMFCDMImpl.cpp | 107 +-- dom/media/eme/mediafoundation/WMFCDMImpl.h | 4 +- dom/media/eme/mediafoundation/WMFCDMProxy.cpp | 3 +- dom/media/flac/FlacDecoder.cpp | 8 +- dom/media/gmp/ChromiumCDMProxy.cpp | 3 +- dom/media/gmp/GMPChild.cpp | 47 +- dom/media/gmp/GMPDiskStorage.cpp | 1 - dom/media/gmp/GMPLoader.cpp | 2 +- dom/media/gmp/GMPParent.cpp | 16 +- dom/media/gmp/moz.build | 2 +- dom/media/gmp/rlz/moz.build | 6 +- dom/media/gtest/TestAudioSampleFormat.cpp | 116 ++++ dom/media/gtest/TestAudioSegment.cpp | 16 +- dom/media/gtest/TestAudioTrackGraph.cpp | 2 +- dom/media/gtest/TestMediaDataDecoder.cpp | 3 +- dom/media/gtest/TestMediaDataEncoder.cpp | 9 +- dom/media/gtest/TestMediaQueue.cpp | 16 + dom/media/gtest/TestMediaUtils.cpp | 2 +- dom/media/gtest/moz.build | 1 + dom/media/ipc/MFCDMSerializers.h | 16 +- dom/media/ipc/MFMediaEngineParent.cpp | 4 +- dom/media/ipc/RDDProcessHost.cpp | 4 - dom/media/ipc/RDDProcessManager.cpp | 8 +- dom/media/ipc/RemoteDecoderManagerChild.cpp | 32 +- dom/media/mediacapabilities/MediaCapabilities.cpp | 25 +- dom/media/mediacontrol/ContentMediaController.cpp | 4 +- dom/media/mediacontrol/ContentMediaController.h | 2 +- .../mediacontrol/ContentPlaybackController.cpp | 9 +- dom/media/mediacontrol/MediaControlKeyManager.cpp | 14 +- dom/media/mediacontrol/MediaControlKeyManager.h | 2 +- dom/media/mediacontrol/MediaControlKeySource.cpp | 6 +- dom/media/mediacontrol/MediaControlKeySource.h | 8 +- dom/media/mediacontrol/MediaControlService.cpp | 10 +- dom/media/mediacontrol/MediaControlUtils.h | 13 +- dom/media/mediacontrol/MediaController.cpp | 20 +- dom/media/mediacontrol/MediaController.h | 2 +- dom/media/mediacontrol/MediaStatusManager.cpp | 27 +- dom/media/mediacontrol/MediaStatusManager.h | 13 +- .../browser_media_control_before_media_starts.js | 2 +- .../tests/gtest/MediaKeyListenerTest.h | 2 +- dom/media/mediacontrol/tests/gtest/moz.build | 2 +- dom/media/mediasession/MediaSession.cpp | 30 +- dom/media/mediasession/MediaSession.h | 22 +- dom/media/mediasession/MediaSessionIPCUtils.h | 17 +- dom/media/mediasession/moz.build | 4 + .../mediasession/test/gtest/TestPositionState.cpp | 127 ++++ dom/media/mediasession/test/gtest/moz.build | 15 + dom/media/mediasource/MediaSourceDecoder.cpp | 33 +- dom/media/mediasource/MediaSourceDecoder.h | 5 + dom/media/mediasource/test/mochitest.toml | 2 + .../test/test_BufferedSeekCanPlayThrough.html | 46 ++ dom/media/mediasource/test/test_EndOfStream.html | 2 +- .../mediasource/test/test_EndOfStream_mp4.html | 2 +- .../mediasource/test/test_ExperimentalAsync.html | 4 +- dom/media/mediasource/test/test_SetModeThrows.html | 2 +- dom/media/metrics.yaml | 44 ++ dom/media/nsIMediaManager.idl | 5 - dom/media/platforms/PDMFactory.cpp | 41 +- dom/media/platforms/PEMFactory.cpp | 10 +- dom/media/platforms/agnostic/bytestreams/Adts.cpp | 234 ++++++- dom/media/platforms/agnostic/bytestreams/Adts.h | 117 +++- .../platforms/agnostic/bytestreams/AnnexB.cpp | 12 +- .../agnostic/eme/ChromiumCDMVideoDecoder.cpp | 39 +- .../platforms/agnostic/eme/EMEDecoderModule.cpp | 18 +- .../platforms/agnostic/gmp/GMPDecoderModule.cpp | 10 + dom/media/platforms/apple/AppleATDecoder.cpp | 90 ++- dom/media/platforms/apple/AppleATDecoder.h | 5 +- dom/media/platforms/apple/AppleDecoderModule.cpp | 7 +- dom/media/platforms/apple/AppleVTDecoder.cpp | 10 +- dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp | 12 +- dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp | 7 +- dom/media/platforms/ffmpeg/FFmpegLibWrapper.h | 11 +- dom/media/platforms/ffmpeg/FFmpegRDFTTypes.h | 34 - dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp | 255 +++---- dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h | 13 + .../platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp | 14 +- .../platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.h | 8 +- dom/media/platforms/moz.build | 18 +- dom/media/platforms/wmf/MFCDMSession.cpp | 3 +- .../platforms/wmf/MFMediaEngineAudioStream.cpp | 2 +- .../platforms/wmf/MFMediaEngineDecoderModule.cpp | 16 +- .../platforms/wmf/MFMediaEngineDecoderModule.h | 2 + dom/media/platforms/wmf/MFMediaEngineStream.cpp | 33 +- dom/media/platforms/wmf/MFMediaEngineStream.h | 8 +- .../platforms/wmf/MFMediaEngineVideoStream.cpp | 108 ++- dom/media/platforms/wmf/MFMediaEngineVideoStream.h | 27 +- dom/media/platforms/wmf/MFMediaSource.h | 2 - dom/media/platforms/wmf/WMFAudioMFTManager.cpp | 25 +- dom/media/platforms/wmf/WMFAudioMFTManager.h | 1 + dom/media/platforms/wmf/WMFMediaDataEncoder.h | 5 +- dom/media/platforms/wmf/WMFUtils.cpp | 34 +- .../webrtc/videoengine/VideoCaptureAndroid.java | 35 +- .../android_video_capture/video_capture_android.cc | 12 +- .../video_capture_avfoundation.mm | 5 +- dom/media/test/background_video.js | 2 +- dom/media/test/bipbop-clearkey-video-av1.mp4 | Bin 0 -> 52054 bytes .../test/bipbop-clearkey-video-av1.mp4^headers^ | 1 + dom/media/test/bipbop-clearkey-video-av1.webm | Bin 0 -> 50186 bytes .../test/bipbop-clearkey-video-av1.webm^headers^ | 1 + dom/media/test/browser/browser.toml | 4 + .../browser_encrypted_play_time_telemetry.js | 70 -- .../browser_glean_first_frame_loaded_time.js | 81 +++ .../browser_tab_visibility_and_play_time.js | 2 +- dom/media/test/browser/head.js | 119 ++++ dom/media/test/browser/wmfme/browser.toml | 8 +- .../browser_wmfme_glean_first_frame_loaded_time.js | 98 +++ dom/media/test/can_play_type_ogg.js | 2 +- dom/media/test/crashtests/crashtests.list | 10 +- dom/media/test/eme.js | 4 +- dom/media/test/manifest.js | 46 +- dom/media/test/mochitest.toml | 5 + dom/media/test/mochitest_background_video.toml | 4 + dom/media/test/mochitest_bugs.toml | 17 +- dom/media/test/mochitest_compat.toml | 13 +- dom/media/test/mochitest_eme.toml | 4 + dom/media/test/mochitest_media_recorder.toml | 4 + dom/media/test/mochitest_seek.toml | 4 + dom/media/test/mochitest_stream.toml | 4 + dom/media/test/reftest/color_quads/reftest.list | 26 +- dom/media/test/reftest/reftest.list | 14 +- dom/media/test/test_arraybuffer.html | 4 +- dom/media/test/test_aspectratio_mp4.html | 2 +- ...und_video_resume_after_end_show_last_frame.html | 2 +- dom/media/test/test_buffered.html | 2 +- .../test/test_bug1431810_opus_downmix_to_mono.html | 8 +- dom/media/test/test_cueless_webm_seek-1.html | 2 +- dom/media/test/test_cueless_webm_seek-2.html | 2 +- dom/media/test/test_cueless_webm_seek-3.html | 2 +- dom/media/test/test_decode_error_crossorigin.html | 2 +- dom/media/test/test_delay_load.html | 2 +- dom/media/test/test_eme_detach_media_keys.html | 2 +- ...ch_reattach_same_mediakeys_during_playback.html | 4 +- dom/media/test/test_eme_getstatusforpolicy.html | 2 +- dom/media/test/test_eme_initDataTypes.html | 4 +- .../test/test_eme_mfcdm_generate_request.html | 6 +- .../test/test_eme_mfcdm_getstatusforpolicy.html | 2 +- dom/media/test/test_eme_missing_pssh.html | 2 +- dom/media/test/test_eme_non_mse_fails.html | 2 +- dom/media/test/test_eme_playback.html | 4 +- dom/media/test/test_eme_protection_query.html | 2 +- dom/media/test/test_eme_pssh_in_moof.html | 2 +- .../test/test_eme_requestKeySystemAccess.html | 4 +- dom/media/test/test_eme_request_notifications.html | 4 +- .../test/test_eme_sample_groups_playback.html | 4 +- .../test_eme_stream_capture_blocked_case1.html | 2 +- .../test_eme_stream_capture_blocked_case2.html | 2 +- .../test_eme_stream_capture_blocked_case3.html | 2 +- .../test/test_eme_unsetMediaKeys_then_capture.html | 4 +- .../test/test_eme_wideinve_l1_installation.html | 4 +- dom/media/test/test_eme_wv_privacy.html | 2 +- dom/media/test/test_hls_player_independency.html | 4 +- dom/media/test/test_hw_video_decoding.html | 5 +- dom/media/test/test_invalid_reject.html | 4 +- dom/media/test/test_invalid_reject_play.html | 4 +- dom/media/test/test_load.html | 8 +- dom/media/test/test_load_source.html | 4 +- dom/media/test/test_media_selection.html | 4 +- ..._mediarecorder_record_canvas_captureStream.html | 2 +- ...arecorder_record_changing_video_resolution.html | 2 +- ...t_mediarecorder_record_downsize_resolution.html | 2 +- ...t_mediarecorder_record_gum_video_timeslice.html | 2 +- .../test/test_mediarecorder_record_session.html | 2 +- .../test_mediarecorder_record_startstopstart.html | 4 +- .../test/test_mediarecorder_record_timeslice.html | 2 +- ...est_mediarecorder_record_upsize_resolution.html | 2 +- .../test_mediatrack_consuming_mediaresource.html | 12 +- .../test_mediatrack_consuming_mediastream.html | 8 +- .../test/test_mediatrack_replay_from_end.html | 8 +- .../test/test_midflight_redirect_blocked.html | 2 +- dom/media/test/test_mixed_principals.html | 2 +- dom/media/test/test_new_audio.html | 2 +- dom/media/test/test_periodic_timeupdate.html | 2 +- dom/media/test/test_play_twice.html | 2 +- dom/media/test/test_playback.html | 2 +- dom/media/test/test_playback_errors.html | 2 +- dom/media/test/test_playback_hls.html | 2 +- dom/media/test/test_preload_actions.html | 54 +- dom/media/test/test_preload_suspend.html | 2 +- dom/media/test/test_reset_events_async.html | 6 +- dom/media/test/test_seek-10.html | 2 +- dom/media/test/test_seekToNextFrame.html | 2 +- .../test/test_temporary_file_blob_video_plays.html | 2 +- dom/media/test/test_video_dimensions.html | 2 +- dom/media/test/test_video_low_power_telemetry.html | 2 +- dom/media/utils/TelemetryProbesReporter.cpp | 46 ++ dom/media/utils/TelemetryProbesReporter.h | 2 + dom/media/webaudio/FFTBlock.cpp | 14 +- dom/media/webaudio/FFTBlock.h | 290 +++----- dom/media/webaudio/OscillatorNode.cpp | 6 +- dom/media/webaudio/blink/FFTConvolver.cpp | 2 +- dom/media/webaudio/blink/HRTFKernel.cpp | 2 +- dom/media/webaudio/blink/PeriodicWave.cpp | 2 +- dom/media/webaudio/moz.build | 2 - .../test/audioBufferSourceNodeDetached_worker.js | 2 +- .../webaudio/test/test_OfflineAudioContext.html | 4 +- .../test/test_WebAudioMemoryReporting.html | 2 +- .../test/test_audioBufferSourceNodeOffset.html | 2 +- .../test/test_audioContextSuspendResumeClose.html | 8 +- dom/media/webaudio/test/test_bug1056032.html | 2 +- dom/media/webaudio/test/test_bug867174.html | 2 +- .../test/test_convolverNodeNormalization.html | 2 +- .../test/test_decodeAudioDataOnDetachedBuffer.html | 2 +- .../webaudio/test/test_decodeAudioDataPromise.html | 2 +- dom/media/webaudio/test/test_decodeAudioError.html | 4 +- .../webaudio/test/test_dynamicsCompressorNode.html | 2 +- .../webaudio/test/test_event_listener_leaks.html | 2 +- .../test/test_mediaStreamAudioSourceNodeNoGC.html | 2 +- dom/media/webaudio/test/test_pannerNodeTail.html | 2 +- .../test/test_scriptProcessorNodeNotConnected.html | 2 +- dom/media/webaudio/test/webaudio.js | 2 +- dom/media/webcodecs/AudioData.cpp | 731 +++++++++++++++++++++ dom/media/webcodecs/AudioData.h | 176 +++++ dom/media/webcodecs/AudioDecoder.cpp | 481 ++++++++++++++ dom/media/webcodecs/AudioDecoder.h | 83 +++ dom/media/webcodecs/DecoderAgent.cpp | 2 - dom/media/webcodecs/DecoderTemplate.cpp | 15 +- dom/media/webcodecs/DecoderTypes.h | 64 +- dom/media/webcodecs/EncodedAudioChunk.cpp | 260 ++++++++ dom/media/webcodecs/EncodedAudioChunk.h | 117 ++++ dom/media/webcodecs/VideoDecoder.cpp | 97 +-- dom/media/webcodecs/VideoEncoder.cpp | 103 ++- dom/media/webcodecs/VideoFrame.cpp | 41 +- dom/media/webcodecs/WebCodecsUtils.cpp | 109 +-- dom/media/webcodecs/WebCodecsUtils.h | 9 +- dom/media/webcodecs/moz.build | 6 + dom/media/webrtc/MediaEngineFake.cpp | 6 +- dom/media/webrtc/MediaEngineRemoteVideoSource.cpp | 3 +- dom/media/webrtc/MediaEngineWebRTCAudio.cpp | 6 +- dom/media/webrtc/MediaTrackConstraints.h | 11 - dom/media/webrtc/MediaTransportChild.h | 6 +- dom/media/webrtc/PMediaTransport.ipdl | 4 +- dom/media/webrtc/WebrtcGlobal.h | 33 +- dom/media/webrtc/WebrtcIPCTraits.h | 11 +- dom/media/webrtc/jsapi/MediaTransportHandler.cpp | 62 +- dom/media/webrtc/jsapi/MediaTransportHandler.h | 15 +- .../webrtc/jsapi/MediaTransportHandlerIPC.cpp | 10 +- dom/media/webrtc/jsapi/MediaTransportParent.cpp | 14 +- dom/media/webrtc/jsapi/PeerConnectionImpl.cpp | 562 +++++++++++----- dom/media/webrtc/jsapi/PeerConnectionImpl.h | 42 +- dom/media/webrtc/jsapi/RTCDtlsTransport.cpp | 7 +- dom/media/webrtc/jsapi/RTCDtlsTransport.h | 3 + dom/media/webrtc/jsapi/RTCIceTransport.cpp | 60 ++ dom/media/webrtc/jsapi/RTCIceTransport.h | 53 ++ dom/media/webrtc/jsapi/RTCRtpReceiver.cpp | 4 +- dom/media/webrtc/jsapi/RTCRtpSender.cpp | 1 - dom/media/webrtc/jsapi/RTCRtpTransceiver.cpp | 33 +- dom/media/webrtc/jsapi/RTCRtpTransceiver.h | 3 +- dom/media/webrtc/jsapi/moz.build | 2 + dom/media/webrtc/jsep/JsepSessionImpl.cpp | 12 +- dom/media/webrtc/jsep/JsepTrack.cpp | 4 +- dom/media/webrtc/jsep/JsepTrack.h | 10 +- dom/media/webrtc/metrics.yaml | 80 ++- dom/media/webrtc/tests/mochitests/head.js | 1 + dom/media/webrtc/tests/mochitests/iceTestUtils.js | 126 ++++ dom/media/webrtc/tests/mochitests/pc.js | 21 +- .../test_peerConnection_RTCIceTransport.html | 197 ++++++ .../test_peerConnection_basicAudioNATRelay.html | 2 + .../test_peerConnection_basicAudioNATRelayTCP.html | 2 + .../test_peerConnection_basicAudioNATRelayTLS.html | 2 + .../test_peerConnection_basicAudioNATSrflx.html | 2 + ...est_peerConnection_basicAudioNoisyUDPBlock.html | 2 + .../test_peerConnection_basicAudioRelayPolicy.html | 5 +- .../mochitests/test_peerConnection_bug825703.html | 79 +-- .../mochitests/test_peerConnection_callbacks.html | 29 +- .../test_peerConnection_localRollback.html | 2 +- .../test_peerConnection_portRestrictions.html | 4 + .../mochitests/test_peerConnection_restartIce.html | 12 +- ...onnection_restartIceLocalAndRemoteRollback.html | 10 - ...est_peerConnection_restartIceLocalRollback.html | 9 - .../test_peerConnection_restartIceNoBundle.html | 12 +- ...peerConnection_restartIceNoBundleNoRtcpMux.html | 12 +- .../test_peerConnection_restartIceNoRtcpMux.html | 13 +- .../test_peerConnection_sillyCodecPriorities.html | 12 +- .../test_peerConnection_stats_relayProtocol.html | 2 + ...t_peerConnection_threeUnbundledConnections.html | 4 +- .../test_peerConnection_throwInCallbacks.html | 7 +- .../test_peerConnection_verifyDescriptions.html | 89 ++- .../webrtc/third_party_build/default_config_env | 20 +- .../webrtc/third_party_build/filter_git_changes.py | 2 +- dom/media/webrtc/third_party_build/prep_repo.sh | 8 +- .../webrtc/third_party_build/save_patch_stack.py | 2 +- .../webrtc/third_party_build/verify_vendoring.sh | 16 +- dom/media/webrtc/transport/nricectx.cpp | 180 +++-- dom/media/webrtc/transport/nricectx.h | 39 +- dom/media/webrtc/transport/nricemediastream.cpp | 55 ++ dom/media/webrtc/transport/nricemediastream.h | 14 + dom/media/webrtc/transport/test/ice_unittest.cpp | 134 ++-- .../transport/test/test_nr_socket_ice_unittest.cpp | 3 +- .../webrtc/transport/test/transport_unittests.cpp | 16 +- .../transport/third_party/nICEr/src/ice/ice_ctx.c | 38 +- .../transport/third_party/nICEr/src/ice/ice_ctx.h | 21 +- .../third_party/nICEr/src/ice/ice_handler.h | 6 + .../third_party/nICEr/src/ice/ice_media_stream.c | 38 ++ .../third_party/nICEr/src/ice/ice_media_stream.h | 1 + .../third_party/nICEr/src/ice/ice_peer_ctx.c | 6 + .../webrtc/transportbridge/MediaPipelineFilter.cpp | 34 +- .../webrtc/transportbridge/MediaPipelineFilter.h | 4 +- .../webspeech/recognition/SpeechRecognition.cpp | 13 +- .../webspeech/synth/nsISynthVoiceRegistry.idl | 12 - dom/media/webvtt/WebVTTListener.cpp | 2 +- dom/media/webvtt/test/mochitest/manifest.js | 2 +- dom/media/webvtt/test/mochitest/mochitest.toml | 12 +- dom/media/webvtt/update-webvtt.js | 4 +- 346 files changed, 6850 insertions(+), 2628 deletions(-) create mode 100644 dom/media/gtest/TestAudioSampleFormat.cpp create mode 100644 dom/media/mediasession/test/gtest/TestPositionState.cpp create mode 100644 dom/media/mediasession/test/gtest/moz.build create mode 100644 dom/media/mediasource/test/test_BufferedSeekCanPlayThrough.html delete mode 100644 dom/media/platforms/ffmpeg/FFmpegRDFTTypes.h create mode 100644 dom/media/test/bipbop-clearkey-video-av1.mp4 create mode 100644 dom/media/test/bipbop-clearkey-video-av1.mp4^headers^ create mode 100644 dom/media/test/bipbop-clearkey-video-av1.webm create mode 100644 dom/media/test/bipbop-clearkey-video-av1.webm^headers^ create mode 100644 dom/media/test/browser/browser_glean_first_frame_loaded_time.js create mode 100644 dom/media/test/browser/head.js create mode 100644 dom/media/test/browser/wmfme/browser_wmfme_glean_first_frame_loaded_time.js create mode 100644 dom/media/webcodecs/AudioData.cpp create mode 100644 dom/media/webcodecs/AudioData.h create mode 100644 dom/media/webcodecs/AudioDecoder.cpp create mode 100644 dom/media/webcodecs/AudioDecoder.h create mode 100644 dom/media/webcodecs/EncodedAudioChunk.cpp create mode 100644 dom/media/webcodecs/EncodedAudioChunk.h create mode 100644 dom/media/webrtc/jsapi/RTCIceTransport.cpp create mode 100644 dom/media/webrtc/jsapi/RTCIceTransport.h create mode 100644 dom/media/webrtc/tests/mochitests/test_peerConnection_RTCIceTransport.html (limited to 'dom/media') diff --git a/dom/media/ADTSDemuxer.cpp b/dom/media/ADTSDemuxer.cpp index 29ea270461..49135efc52 100644 --- a/dom/media/ADTSDemuxer.cpp +++ b/dom/media/ADTSDemuxer.cpp @@ -10,6 +10,7 @@ #include "VideoUtils.h" #include "mozilla/Logging.h" #include "mozilla/UniquePtr.h" +#include "Adts.h" #include extern mozilla::LazyLogModule gMediaDemuxerLog; @@ -21,227 +22,6 @@ extern mozilla::LazyLogModule gMediaDemuxerLog; DDMOZ_LOG(gMediaDemuxerLog, LogLevel::Verbose, msg, ##__VA_ARGS__) namespace mozilla { -namespace adts { - -// adts::FrameHeader - Holds the ADTS frame header and its parsing -// state. -// -// ADTS Frame Structure -// -// 11111111 1111BCCD EEFFFFGH HHIJKLMM MMMMMMMM MMMOOOOO OOOOOOPP(QQQQQQQQ -// QQQQQQQQ) -// -// Header consists of 7 or 9 bytes(without or with CRC). -// Letter Length(bits) Description -// { sync } 12 syncword 0xFFF, all bits must be 1 -// B 1 MPEG Version: 0 for MPEG-4, 1 for MPEG-2 -// C 2 Layer: always 0 -// D 1 protection absent, Warning, set to 1 if there is no -// CRC and 0 if there is CRC -// E 2 profile, the MPEG-4 Audio Object Type minus 1 -// F 4 MPEG-4 Sampling Frequency Index (15 is forbidden) -// H 3 MPEG-4 Channel Configuration (in the case of 0, the -// channel configuration is sent via an in-band PCE) -// M 13 frame length, this value must include 7 or 9 bytes of -// header length: FrameLength = -// (ProtectionAbsent == 1 ? 7 : 9) + size(AACFrame) -// O 11 Buffer fullness -// P 2 Number of AAC frames(RDBs) in ADTS frame minus 1, for -// maximum compatibility always use 1 AAC frame per ADTS -// frame -// Q 16 CRC if protection absent is 0 -class FrameHeader { - public: - uint32_t mFrameLength{}; - uint32_t mSampleRate{}; - uint32_t mSamples{}; - uint32_t mChannels{}; - uint8_t mObjectType{}; - uint8_t mSamplingIndex{}; - uint8_t mChannelConfig{}; - uint8_t mNumAACFrames{}; - bool mHaveCrc{}; - - // Returns whether aPtr matches a valid ADTS header sync marker - static bool MatchesSync(const uint8_t* aPtr) { - return aPtr[0] == 0xFF && (aPtr[1] & 0xF6) == 0xF0; - } - - FrameHeader() { Reset(); } - - // Header size - uint64_t HeaderSize() const { return (mHaveCrc) ? 9 : 7; } - - bool IsValid() const { return mFrameLength > 0; } - - // Resets the state to allow for a new parsing session. - void Reset() { PodZero(this); } - - // Returns whether the byte creates a valid sequence up to this point. - bool Parse(const uint8_t* aPtr) { - const uint8_t* p = aPtr; - - if (!MatchesSync(p)) { - return false; - } - - // AAC has 1024 samples per frame per channel. - mSamples = 1024; - - mHaveCrc = !(p[1] & 0x01); - mObjectType = ((p[2] & 0xC0) >> 6) + 1; - mSamplingIndex = (p[2] & 0x3C) >> 2; - mChannelConfig = (p[2] & 0x01) << 2 | (p[3] & 0xC0) >> 6; - mFrameLength = static_cast( - (p[3] & 0x03) << 11 | (p[4] & 0xFF) << 3 | (p[5] & 0xE0) >> 5); - mNumAACFrames = (p[6] & 0x03) + 1; - - static const uint32_t SAMPLE_RATES[] = {96000, 88200, 64000, 48000, 44100, - 32000, 24000, 22050, 16000, 12000, - 11025, 8000, 7350}; - if (mSamplingIndex >= ArrayLength(SAMPLE_RATES)) { - LOG(("ADTS: Init() failure: invalid sample-rate index value: %" PRIu32 - ".", - mSamplingIndex)); - return false; - } - mSampleRate = SAMPLE_RATES[mSamplingIndex]; - - MOZ_ASSERT(mChannelConfig < 8); - mChannels = (mChannelConfig == 7) ? 8 : mChannelConfig; - - return true; - } -}; - -// adts::Frame - Frame meta container used to parse and hold a frame -// header and side info. -class Frame { - public: - Frame() : mOffset(0) {} - - uint64_t Offset() const { return mOffset; } - size_t Length() const { - // TODO: If fields are zero'd when invalid, this check wouldn't be - // necessary. - if (!mHeader.IsValid()) { - return 0; - } - - return mHeader.mFrameLength; - } - - // Returns the offset to the start of frame's raw data. - uint64_t PayloadOffset() const { return mOffset + mHeader.HeaderSize(); } - - // Returns the length of the frame's raw data (excluding the header) in bytes. - size_t PayloadLength() const { - // TODO: If fields are zero'd when invalid, this check wouldn't be - // necessary. - if (!mHeader.IsValid()) { - return 0; - } - - return mHeader.mFrameLength - mHeader.HeaderSize(); - } - - // Returns the parsed frame header. - const FrameHeader& Header() const { return mHeader; } - - bool IsValid() const { return mHeader.IsValid(); } - - // Resets the frame header and data. - void Reset() { - mHeader.Reset(); - mOffset = 0; - } - - // Returns whether the valid - bool Parse(uint64_t aOffset, const uint8_t* aStart, const uint8_t* aEnd) { - MOZ_ASSERT(aStart && aEnd); - - bool found = false; - const uint8_t* ptr = aStart; - // Require at least 7 bytes of data at the end of the buffer for the minimum - // ADTS frame header. - while (ptr < aEnd - 7 && !found) { - found = mHeader.Parse(ptr); - ptr++; - } - - mOffset = aOffset + (static_cast(ptr - aStart)) - 1u; - - return found; - } - - private: - // The offset to the start of the header. - uint64_t mOffset; - - // The currently parsed frame header. - FrameHeader mHeader; -}; - -class FrameParser { - public: - // Returns the currently parsed frame. Reset via Reset or EndFrameSession. - const Frame& CurrentFrame() const { return mFrame; } - - // Returns the first parsed frame. Reset via Reset. - const Frame& FirstFrame() const { return mFirstFrame; } - - // Resets the parser. Don't use between frames as first frame data is reset. - void Reset() { - EndFrameSession(); - mFirstFrame.Reset(); - } - - // Clear the last parsed frame to allow for next frame parsing, i.e.: - // - sets PrevFrame to CurrentFrame - // - resets the CurrentFrame - // - resets ID3Header if no valid header was parsed yet - void EndFrameSession() { mFrame.Reset(); } - - // Parses contents of given ByteReader for a valid frame header and returns - // true if one was found. After returning, the variable passed to - // 'aBytesToSkip' holds the amount of bytes to be skipped (if any) in order to - // jump across a large ID3v2 tag spanning multiple buffers. - bool Parse(uint64_t aOffset, const uint8_t* aStart, const uint8_t* aEnd) { - const bool found = mFrame.Parse(aOffset, aStart, aEnd); - - if (mFrame.Length() && !mFirstFrame.Length()) { - mFirstFrame = mFrame; - } - - return found; - } - - private: - // We keep the first parsed frame around for static info access, the - // previously parsed frame for debugging and the currently parsed frame. - Frame mFirstFrame; - Frame mFrame; -}; - -// Initialize the AAC AudioSpecificConfig. -// Only handles two-byte version for AAC-LC. -static void InitAudioSpecificConfig(const Frame& frame, - MediaByteBuffer* aBuffer) { - const FrameHeader& header = frame.Header(); - MOZ_ASSERT(header.IsValid()); - - int audioObjectType = header.mObjectType; - int samplingFrequencyIndex = header.mSamplingIndex; - int channelConfig = header.mChannelConfig; - - uint8_t asc[2]; - asc[0] = (audioObjectType & 0x1F) << 3 | (samplingFrequencyIndex & 0x0E) >> 1; - asc[1] = (samplingFrequencyIndex & 0x01) << 7 | (channelConfig & 0x0F) << 3; - - aBuffer->AppendElements(asc, 2); -} - -} // namespace adts using media::TimeUnit; @@ -292,7 +72,7 @@ bool ADTSDemuxer::IsSeekable() const { // ADTSTrackDemuxer ADTSTrackDemuxer::ADTSTrackDemuxer(MediaResource* aSource) : mSource(aSource), - mParser(new adts::FrameParser()), + mParser(new ADTS::FrameParser()), mOffset(0), mNumParsedFrames(0), mFrameIndex(0), @@ -535,7 +315,7 @@ TimeUnit ADTSTrackDemuxer::Duration(int64_t aNumFrames) const { return TimeUnit(aNumFrames * mSamplesPerFrame, mSamplesPerSecond); } -const adts::Frame& ADTSTrackDemuxer::FindNextFrame( +const ADTS::Frame& ADTSTrackDemuxer::FindNextFrame( bool findFirstFrame /*= false*/) { static const int BUFFER_SIZE = 4096; static const int MAX_SKIPPED_BYTES = 10 * BUFFER_SIZE; @@ -568,7 +348,7 @@ const adts::Frame& ADTSTrackDemuxer::FindNextFrame( break; } - const adts::Frame& currentFrame = mParser->CurrentFrame(); + const ADTS::Frame& currentFrame = mParser->CurrentFrame(); foundFrame = mParser->Parse(frameHeaderOffset, buffer, buffer + read); if (findFirstFrame && foundFrame) { // Check for sync marker after the found frame, since it's @@ -579,7 +359,7 @@ const adts::Frame& ADTSTrackDemuxer::FindNextFrame( currentFrame.Offset() + currentFrame.Length(); uint32_t read = Read(buffer, AssertedCast(nextFrameHeaderOffset), 2); - if (read != 2 || !adts::FrameHeader::MatchesSync(buffer)) { + if (read != 2 || !ADTS::FrameHeader::MatchesSync(buffer)) { frameHeaderOffset = currentFrame.Offset() + 1; mParser->Reset(); foundFrame = false; @@ -621,7 +401,7 @@ const adts::Frame& ADTSTrackDemuxer::FindNextFrame( return mParser->CurrentFrame(); } -bool ADTSTrackDemuxer::SkipNextFrame(const adts::Frame& aFrame) { +bool ADTSTrackDemuxer::SkipNextFrame(const ADTS::Frame& aFrame) { if (!mNumParsedFrames || !aFrame.Length()) { RefPtr frame(GetNextFrame(aFrame)); return frame; @@ -639,7 +419,7 @@ bool ADTSTrackDemuxer::SkipNextFrame(const adts::Frame& aFrame) { } already_AddRefed ADTSTrackDemuxer::GetNextFrame( - const adts::Frame& aFrame) { + const ADTS::Frame& aFrame) { ADTSLOG("GetNext() Begin({mOffset=%" PRIu64 " HeaderSize()=%" PRIu64 " Length()=%zu})", aFrame.Offset(), aFrame.Header().HeaderSize(), @@ -735,7 +515,7 @@ int64_t ADTSTrackDemuxer::FrameIndexFromTime(const TimeUnit& aTime) const { return std::max(0, frameIndex); } -void ADTSTrackDemuxer::UpdateState(const adts::Frame& aFrame) { +void ADTSTrackDemuxer::UpdateState(const ADTS::Frame& aFrame) { uint32_t frameLength = aFrame.Length(); // Prevent overflow. if (mTotalFrameLen + frameLength < mTotalFrameLen) { @@ -750,7 +530,7 @@ void ADTSTrackDemuxer::UpdateState(const adts::Frame& aFrame) { mTotalFrameLen += frameLength; if (!mSamplesPerFrame) { - const adts::FrameHeader& header = aFrame.Header(); + const ADTS::FrameHeader& header = aFrame.Header(); mSamplesPerFrame = header.mSamples; mSamplesPerSecond = header.mSampleRate; mChannels = header.mChannels; @@ -795,15 +575,15 @@ bool ADTSDemuxer::ADTSSniffer(const uint8_t* aData, const uint32_t aLength) { if (aLength < 7) { return false; } - if (!adts::FrameHeader::MatchesSync(aData)) { + if (!ADTS::FrameHeader::MatchesSync(Span(aData, aLength))) { return false; } - auto parser = MakeUnique(); + auto parser = MakeUnique(); if (!parser->Parse(0, aData, aData + aLength)) { return false; } - const adts::Frame& currentFrame = parser->CurrentFrame(); + const ADTS::Frame& currentFrame = parser->CurrentFrame(); // Check for sync marker after the found frame, since it's // possible to find sync marker in AAC data. If sync marker // exists after the current frame then we've found a frame @@ -812,7 +592,8 @@ bool ADTSDemuxer::ADTSSniffer(const uint8_t* aData, const uint32_t aLength) { currentFrame.Offset() + currentFrame.Length(); return aLength > nextFrameHeaderOffset && aLength - nextFrameHeaderOffset >= 2 && - adts::FrameHeader::MatchesSync(aData + nextFrameHeaderOffset); + ADTS::FrameHeader::MatchesSync(Span(aData + nextFrameHeaderOffset, + aLength - nextFrameHeaderOffset)); } } // namespace mozilla diff --git a/dom/media/ADTSDemuxer.h b/dom/media/ADTSDemuxer.h index 40ff44898e..67a661883f 100644 --- a/dom/media/ADTSDemuxer.h +++ b/dom/media/ADTSDemuxer.h @@ -11,14 +11,10 @@ #include "mozilla/Maybe.h" #include "MediaDataDemuxer.h" #include "MediaResource.h" +#include "Adts.h" namespace mozilla { -namespace adts { -class Frame; -class FrameParser; -} // namespace adts - class ADTSTrackDemuxer; DDLoggedTypeDeclNameAndBase(ADTSDemuxer, MediaDataDemuxer); @@ -87,16 +83,16 @@ class ADTSTrackDemuxer : public MediaTrackDemuxer, media::TimeUnit ScanUntil(const media::TimeUnit& aTime); // Finds the next valid frame and returns its byte range. - const adts::Frame& FindNextFrame(bool findFirstFrame = false); + const ADTS::Frame& FindNextFrame(bool findFirstFrame = false); // Skips the next frame given the provided byte range. - bool SkipNextFrame(const adts::Frame& aFrame); + bool SkipNextFrame(const ADTS::Frame& aFrame); // Returns the next ADTS frame, if available. - already_AddRefed GetNextFrame(const adts::Frame& aFrame); + already_AddRefed GetNextFrame(const ADTS::Frame& aFrame); // Updates post-read meta data. - void UpdateState(const adts::Frame& aFrame); + void UpdateState(const ADTS::Frame& aFrame); // Returns the frame index for the given offset. int64_t FrameIndexFromOffset(uint64_t aOffset) const; @@ -115,7 +111,7 @@ class ADTSTrackDemuxer : public MediaTrackDemuxer, MediaResourceIndex mSource; // ADTS frame parser used to detect frames and extract side info. - adts::FrameParser* mParser; + ADTS::FrameParser* mParser; // Current byte offset in the source stream. uint64_t mOffset; diff --git a/dom/media/AudibilityMonitor.h b/dom/media/AudibilityMonitor.h index fdcf474403..25a715cebf 100644 --- a/dom/media/AudibilityMonitor.h +++ b/dom/media/AudibilityMonitor.h @@ -67,7 +67,7 @@ class AudibilityMonitor { for (uint32_t i = 0; i < frameCount; i++) { bool atLeastOneAudible = false; for (uint32_t j = 0; j < aChannels; j++) { - if (std::fabs(AudioSampleToFloat(samples[readIndex++])) > + if (std::fabs(ConvertAudioSample(samples[readIndex++])) > AUDIBILITY_THRESHOLD) { atLeastOneAudible = true; } diff --git a/dom/media/AudioSampleFormat.h b/dom/media/AudioSampleFormat.h index 1cec31a385..7a329d06df 100644 --- a/dom/media/AudioSampleFormat.h +++ b/dom/media/AudioSampleFormat.h @@ -1,13 +1,16 @@ /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* vim: set ts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ #ifndef MOZILLA_AUDIOSAMPLEFORMAT_H_ #define MOZILLA_AUDIOSAMPLEFORMAT_H_ #include "mozilla/Assertions.h" +#include "mozilla/PodOperations.h" #include +#include +#include namespace mozilla { @@ -62,113 +65,191 @@ class AudioSampleTypeToFormat { static const AudioSampleFormat Format = AUDIO_FORMAT_S16; }; -// Single-sample conversion -/* - * Use "2^N" conversion since it's simple, fast, "bit transparent", used by - * many other libraries and apparently behaves reasonably. - * http://blog.bjornroche.com/2009/12/int-float-int-its-jungle-out-there.html - * http://blog.bjornroche.com/2009/12/linearity-and-dynamic-range-in-int.html - */ -inline float AudioSampleToFloat(float aValue) { return aValue; } -inline float AudioSampleToFloat(int16_t aValue) { - return static_cast(aValue) / 32768.0f; -} -inline float AudioSampleToFloat(int32_t aValue) { - return static_cast(aValue) / (float)(1U << 31); +template +constexpr float MaxAsFloat() { + return static_cast(std::numeric_limits::max()); } template -T FloatToAudioSample(float aValue); - -template <> -inline float FloatToAudioSample(float aValue) { - return aValue; -} -template <> -inline int16_t FloatToAudioSample(float aValue) { - float v = aValue * 32768.0f; - float clamped = std::max(-32768.0f, std::min(32767.0f, v)); - return int16_t(clamped); +constexpr float LowestAsFloat() { + return static_cast(std::numeric_limits::lowest()); } +// The maximum value for an audio sample. If T is signed, the absolute value of +// this number is smaller (by exactly 1) than ::Min(). template -T UInt8bitToAudioSample(uint8_t aValue); - -template <> -inline float UInt8bitToAudioSample(uint8_t aValue) { - return static_cast(aValue) * (static_cast(2) / UINT8_MAX) - - static_cast(1); -} -template <> -inline int16_t UInt8bitToAudioSample(uint8_t aValue) { - return static_cast((aValue << 8) + aValue + INT16_MIN); +constexpr T Max() { + return std::numeric_limits::max(); } +// The minimum value for an audio sample. If T is signed, the absolute value of +// this number is greater (by exactly 1) than ::Max() template -T IntegerToAudioSample(int16_t aValue); +constexpr T Min() { + return std::numeric_limits::lowest(); +} template <> -inline float IntegerToAudioSample(int16_t aValue) { - return static_cast(aValue) / 32768.0f; +constexpr float Max() { + return 1.0f; } + template <> -inline int16_t IntegerToAudioSample(int16_t aValue) { - return aValue; +constexpr float Min() { + return -1.0f; } +// The bias value is the middle of the range. In linear PCM audio, if the +// values are all equal to the bias value, the audio is silent. template -T Int24bitToAudioSample(int32_t aValue); +constexpr T Bias() { + return 0; +} template <> -inline float Int24bitToAudioSample(int32_t aValue) { - return static_cast(aValue) / static_cast(1 << 23); +constexpr uint8_t Bias() { + return 128; } -template <> -inline int16_t Int24bitToAudioSample(int32_t aValue) { - return static_cast(aValue / 256); + +// Clip a floating point audio sample to its nominal range. This is +// destructive, and is only used here for avoiding overflow in some edge cases, +// so it's not going to be generally audible. +inline float Clip(float aValue) { return std::clamp(aValue, -1.0f, 1.0f); } + +template +T FloatToAudioSample(float aValue) { + if constexpr (std::is_same_v) { + return aValue; + } + if constexpr (std::is_same_v) { + return static_cast(std::clamp((aValue + 1.0f) * 128.f, + LowestAsFloat(), MaxAsFloat())); + } else if constexpr (std::is_same_v) { + // This produces correct results accross the range. + return static_cast(std::clamp(aValue * -LowestAsFloat(), + LowestAsFloat(), MaxAsFloat())); + } else if constexpr (std::is_same_v) { + // We need to handle this differently because of rounding between INT32_MAX + // and float 32-bits, to maximise precision. + if (aValue >= 0.) { + // if the input sample is greater OR EQUAL to 1.0, then clip and return + // the max value. + if (aValue >= 1.0) { + return std::numeric_limits::max(); + } + // otherwise cast to a double and map to the positive range. + // float 32-bits cannot represent int32_max (but can represent int32_min) + constexpr double magnitudePos = std::numeric_limits::max(); + return static_cast(aValue * magnitudePos); + } + // Similarly for the negative range. + if (aValue <= -1.0) { + return std::numeric_limits::lowest(); + } + constexpr double magnitudeNegative = + -1.0 * std::numeric_limits::lowest(); + return static_cast(aValue * magnitudeNegative); + } } -template -inline void ConvertAudioSample(SrcT aIn, DstT& aOut); +template +T UInt8bitToAudioSample(uint8_t aValue) { + if constexpr (std::is_same_v) { + return aValue; + } else if constexpr (std::is_same_v) { + return (static_cast(aValue) << 8) - (1 << 15); + } else if constexpr (std::is_same_v) { + return (static_cast(aValue) << 24) - (1 << 31); + } else if constexpr (std::is_same_v) { + float biased = static_cast(aValue) - Bias(); + if (aValue >= Bias()) { + return Clip(biased / MaxAsFloat()); + } + return Clip(biased / -LowestAsFloat()); + } +} -template <> -inline void ConvertAudioSample(int16_t aIn, int16_t& aOut) { - aOut = aIn; +template +T Int16ToAudioSample(int16_t aValue) { + if constexpr (std::is_same_v) { + return static_cast(aValue >> 8) + 128; + } else if constexpr (std::is_same_v) { + return aValue; + } else if constexpr (std::is_same_v) { + return aValue << 16; + } else if constexpr (std::is_same_v) { + if (aValue >= 0) { + return Clip(static_cast(aValue) / MaxAsFloat()); + } + return Clip(static_cast(aValue) / -LowestAsFloat()); + } } -template <> -inline void ConvertAudioSample(int16_t aIn, float& aOut) { - aOut = AudioSampleToFloat(aIn); +// 24-bits audio samples are stored in 32-bits variables. +template +T Int24ToAudioSample(int32_t aValue) { + if constexpr (std::is_same_v) { + return static_cast(aValue >> 16) + 128; + } else if constexpr (std::is_same_v) { + return static_cast(aValue >> 8); + } else if constexpr (std::is_same_v) { + return aValue << 8; + } else if constexpr (std::is_same_v) { + const int32_t min = -(2 << 22); + const int32_t max = (2 << 22) - 1; + if (aValue >= 0) { + return Clip(static_cast(aValue) / static_cast(max)); + } + return Clip(static_cast(aValue) / -static_cast(min)); + } } -template <> -inline void ConvertAudioSample(float aIn, float& aOut) { - aOut = aIn; +template +T Int32ToAudioSample(int32_t aValue) { + if constexpr (std::is_same_v) { + return static_cast(aValue >> 24) + 128; + } else if constexpr (std::is_same_v) { + return aValue >> 16; + } else if constexpr (std::is_same_v) { + return aValue; + } else if constexpr (std::is_same_v) { + if (aValue >= 0) { + return Clip(static_cast(aValue) / MaxAsFloat()); + } + return Clip(static_cast(aValue) / -LowestAsFloat()); + } } -template <> -inline void ConvertAudioSample(float aIn, int16_t& aOut) { - aOut = FloatToAudioSample(aIn); +// This does not handle 24-bits audio, call the function explicitly when +// needed. +template +inline D ConvertAudioSample(const S& aSource) { + if constexpr (std::is_same_v) { + return aSource; + } else if constexpr (std::is_same_v) { + return UInt8bitToAudioSample(aSource); + } else if constexpr (std::is_same_v) { + return Int16ToAudioSample(aSource); + } else if constexpr (std::is_same_v) { + return Int32ToAudioSample(aSource); + } else if constexpr (std::is_same_v) { + return FloatToAudioSample(aSource); + } } // Sample buffer conversion - template inline void ConvertAudioSamples(const From* aFrom, To* aTo, int aCount) { + if constexpr (std::is_same_v) { + PodCopy(aTo, aFrom, aCount); + return; + } for (int i = 0; i < aCount; ++i) { - aTo[i] = FloatToAudioSample(AudioSampleToFloat(aFrom[i])); + aTo[i] = ConvertAudioSample(aFrom[i]); } } -inline void ConvertAudioSamples(const int16_t* aFrom, int16_t* aTo, - int aCount) { - memcpy(aTo, aFrom, sizeof(*aTo) * aCount); -} -inline void ConvertAudioSamples(const float* aFrom, float* aTo, int aCount) { - memcpy(aTo, aFrom, sizeof(*aTo) * aCount); -} // Sample buffer conversion with scale - template inline void ConvertAudioSamplesWithScale(const From* aFrom, To* aTo, int aCount, float aScale) { @@ -177,7 +258,8 @@ inline void ConvertAudioSamplesWithScale(const From* aFrom, To* aTo, int aCount, return; } for (int i = 0; i < aCount; ++i) { - aTo[i] = FloatToAudioSample(AudioSampleToFloat(aFrom[i]) * aScale); + aTo[i] = + ConvertAudioSample(ConvertAudioSample(aFrom[i]) * aScale); } } inline void ConvertAudioSamplesWithScale(const int16_t* aFrom, int16_t* aTo, @@ -194,7 +276,8 @@ inline void ConvertAudioSamplesWithScale(const int16_t* aFrom, int16_t* aTo, return; } for (int i = 0; i < aCount; ++i) { - aTo[i] = FloatToAudioSample(AudioSampleToFloat(aFrom[i]) * aScale); + aTo[i] = FloatToAudioSample(ConvertAudioSample(aFrom[i]) * + aScale); } } @@ -202,8 +285,9 @@ template inline void AddAudioSamplesWithScale(const From* aFrom, To* aTo, int aCount, float aScale) { for (int i = 0; i < aCount; ++i) { - aTo[i] = FloatToAudioSample(AudioSampleToFloat(aTo[i]) + - AudioSampleToFloat(aFrom[i]) * aScale); + aTo[i] = + ConvertAudioSample(ConvertAudioSample(aTo[i]) + + ConvertAudioSample(aFrom[i]) * aScale); } } diff --git a/dom/media/AudioSegment.h b/dom/media/AudioSegment.h index 006f996c39..7d60bf7ca5 100644 --- a/dom/media/AudioSegment.h +++ b/dom/media/AudioSegment.h @@ -79,7 +79,8 @@ static void InterleaveAndConvertBuffer(const SrcT* const* aSourceChannels, DestT* output = aOutput; for (size_t i = 0; i < aLength; ++i) { for (size_t channel = 0; channel < aChannels; ++channel) { - float v = AudioSampleToFloat(aSourceChannels[channel][i]) * aVolume; + float v = + ConvertAudioSample(aSourceChannels[channel][i]) * aVolume; *output = FloatToAudioSample(v); ++output; } @@ -93,7 +94,8 @@ static void DeinterleaveAndConvertBuffer(const SrcT* aSourceBuffer, for (size_t i = 0; i < aChannels; i++) { size_t interleavedIndex = i; for (size_t j = 0; j < aFrames; j++) { - ConvertAudioSample(aSourceBuffer[interleavedIndex], aOutput[i][j]); + aOutput[i][j] = + ConvertAudioSample(aSourceBuffer[interleavedIndex]); interleavedIndex += aChannels; } } @@ -148,7 +150,7 @@ void DownmixAndInterleave(Span aChannelData, * separate pointers to each channel's buffer. */ struct AudioChunk { - typedef mozilla::AudioSampleFormat SampleFormat; + using SampleFormat = mozilla::AudioSampleFormat; AudioChunk() = default; @@ -318,7 +320,7 @@ struct AudioChunk { * A list of audio samples consisting of a sequence of slices of SharedBuffers. * The audio rate is determined by the track, not stored in this class. */ -class AudioSegment : public MediaSegmentBase { +class AudioSegment final : public MediaSegmentBase { // The channel count that MaxChannelCount() returned last time it was called. uint32_t mMemoizedMaxChannelCount = 0; diff --git a/dom/media/CallbackThreadRegistry.cpp b/dom/media/CallbackThreadRegistry.cpp index f4d2af5bd1..84ef7b7cb4 100644 --- a/dom/media/CallbackThreadRegistry.cpp +++ b/dom/media/CallbackThreadRegistry.cpp @@ -6,6 +6,7 @@ #include "CallbackThreadRegistry.h" #include "mozilla/ClearOnShutdown.h" +#include "nsThreadUtils.h" namespace mozilla { struct CallbackThreadRegistrySingleton { diff --git a/dom/media/ChannelMediaDecoder.cpp b/dom/media/ChannelMediaDecoder.cpp index c6da221f94..12f6c11e47 100644 --- a/dom/media/ChannelMediaDecoder.cpp +++ b/dom/media/ChannelMediaDecoder.cpp @@ -221,9 +221,11 @@ MediaDecoderStateMachineBase* ChannelMediaDecoder::CreateStateMachine( mReader = DecoderTraits::CreateReader(ContainerType(), init); #ifdef MOZ_WMF_MEDIA_ENGINE - // TODO : Only for testing development for now. In the future this should be - // used for encrypted content only. - if (StaticPrefs::media_wmf_media_engine_enabled() && + // This state machine is mainly used for the encrypted playback. However, for + // testing purpose we would also use it the non-encrypted playback. + // 1=enabled encrypted and clear, 3=enabled clear + if ((StaticPrefs::media_wmf_media_engine_enabled() == 1 || + StaticPrefs::media_wmf_media_engine_enabled() == 3) && StaticPrefs::media_wmf_media_engine_channel_decoder_enabled() && !aDisableExternalEngine) { return new ExternalEngineStateMachine(this, mReader); diff --git a/dom/media/ExternalEngineStateMachine.cpp b/dom/media/ExternalEngineStateMachine.cpp index 68fb053b83..acfc1f5fa2 100644 --- a/dom/media/ExternalEngineStateMachine.cpp +++ b/dom/media/ExternalEngineStateMachine.cpp @@ -10,6 +10,7 @@ # include "mozilla/MFMediaEngineChild.h" # include "mozilla/StaticPrefs_media.h" #endif +#include "mozilla/AppShutdown.h" #include "mozilla/Atomics.h" #include "mozilla/ClearOnShutdown.h" #include "mozilla/ProfilerLabels.h" @@ -203,6 +204,10 @@ ExternalEngineStateMachine::ExternalEngineStateMachine( InitEngine(); } +ExternalEngineStateMachine::~ExternalEngineStateMachine() { + LOG("ExternalEngineStateMachine is destroyed"); +} + void ExternalEngineStateMachine::InitEngine() { MOZ_ASSERT(mState.IsInitEngine() || mState.IsRecoverEngine()); #ifdef MOZ_WMF_MEDIA_ENGINE @@ -565,6 +570,9 @@ RefPtr ExternalEngineStateMachine::Shutdown() { mSetCDMProxyPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_ABORT_ERR, __func__); mSetCDMProxyRequest.DisconnectIfExists(); + mInitEngineForCDMRequest.DisconnectIfExists(); + + mPendingTasks.Clear(); mEngine->Shutdown(); @@ -607,49 +615,59 @@ void ExternalEngineStateMachine::BufferedRangeUpdated() { } } -// Note: the variadic only supports passing member variables. -#define PERFORM_WHEN_ALLOW(Func, ...) \ - do { \ - /* Initialzation is not done yet, postpone the operation */ \ - if ((mState.IsInitEngine() || mState.IsRecoverEngine()) && \ - mState.AsInitEngine()->mInitPromise) { \ - LOG("%s is called before init", __func__); \ - mState.AsInitEngine()->mInitPromise->Then( \ - OwnerThread(), __func__, \ - [self = RefPtr{this}, this]( \ - const GenericNonExclusivePromise::ResolveOrRejectValue& aVal) { \ - if (aVal.IsResolve()) { \ - Func(__VA_ARGS__); \ - } \ - }); \ - return; \ - } else if (mState.IsShutdownEngine()) { \ - return; \ - } \ +#define PERFORM_WHEN_ALLOW(Func) \ + do { \ + if (mState.IsShutdownEngine() || mHasFatalError || \ + AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed)) { \ + return; \ + } \ + /* Initialzation is not done yet, postpone the operation */ \ + if ((mState.IsInitEngine() || mState.IsRecoverEngine()) && \ + mState.AsInitEngine()->mInitPromise) { \ + LOG("%s is called before init", __func__); \ + mPendingTasks.AppendElement(NewRunnableMethod( \ + __func__, this, &ExternalEngineStateMachine::Func)); \ + return; \ + } \ } while (false) void ExternalEngineStateMachine::SetPlaybackRate(double aPlaybackRate) { AssertOnTaskQueue(); + // TODO : consider to make `mPlaybackRate` a mirror to fit other usages like + // `mVolume` and `mPreservesPitch`. mPlaybackRate = aPlaybackRate; - PERFORM_WHEN_ALLOW(SetPlaybackRate, mPlaybackRate); - mEngine->SetPlaybackRate(aPlaybackRate); + PlaybackRateChanged(); +} + +void ExternalEngineStateMachine::PlaybackRateChanged() { + AssertOnTaskQueue(); + PERFORM_WHEN_ALLOW(PlaybackRateChanged); + MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || + mState.IsSeekingData()); + mEngine->SetPlaybackRate(mPlaybackRate); } void ExternalEngineStateMachine::VolumeChanged() { AssertOnTaskQueue(); PERFORM_WHEN_ALLOW(VolumeChanged); + MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || + mState.IsSeekingData()); mEngine->SetVolume(mVolume); } void ExternalEngineStateMachine::PreservesPitchChanged() { AssertOnTaskQueue(); PERFORM_WHEN_ALLOW(PreservesPitchChanged); + MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || + mState.IsSeekingData()); mEngine->SetPreservesPitch(mPreservesPitch); } void ExternalEngineStateMachine::PlayStateChanged() { AssertOnTaskQueue(); PERFORM_WHEN_ALLOW(PlayStateChanged); + MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || + mState.IsSeekingData()); if (mPlayState == MediaDecoder::PLAY_STATE_PLAYING) { mEngine->Play(); } else if (mPlayState == MediaDecoder::PLAY_STATE_PAUSED) { @@ -660,6 +678,8 @@ void ExternalEngineStateMachine::PlayStateChanged() { void ExternalEngineStateMachine::LoopingChanged() { AssertOnTaskQueue(); PERFORM_WHEN_ALLOW(LoopingChanged); + MOZ_ASSERT(mState.IsReadingMetadata() || mState.IsRunningEngine() || + mState.IsSeekingData()); mEngine->SetLooping(mLooping); } @@ -775,6 +795,13 @@ void ExternalEngineStateMachine::StartRunningEngine() { if (HasVideo()) { RunningEngineUpdate(MediaData::Type::VIDEO_DATA); } + // Run tasks which was called before the engine is ready. + if (!mPendingTasks.IsEmpty()) { + for (auto& task : mPendingTasks) { + Unused << OwnerThread()->Dispatch(task.forget()); + } + mPendingTasks.Clear(); + } } void ExternalEngineStateMachine::RunningEngineUpdate(MediaData::Type aType) { @@ -1141,6 +1168,14 @@ void ExternalEngineStateMachine::RecoverFromCDMProcessCrashIfNeeded() { return; } + if (mState.IsInitEngine()) { + LOG("Failed on the engine initialization, the media engine playback might " + "not be supported"); + DecodeError( + MediaResult(NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR)); + return; + } + LOG("CDM process crashed, recover the engine again (last time=%" PRId64 ")", mCurrentPosition.Ref().ToMicroseconds()); ChangeStateTo(State::RecoverEngine); @@ -1180,24 +1215,28 @@ RefPtr ExternalEngineStateMachine::SetCDMProxy( if (mState.IsInitEngine() && mState.AsInitEngine()->mInitPromise) { LOG("SetCDMProxy is called before init"); - mState.AsInitEngine()->mInitPromise->Then( - OwnerThread(), __func__, - [self = RefPtr{this}, proxy = RefPtr{aProxy}, - this](const GenericNonExclusivePromise::ResolveOrRejectValue& aVal) { - SetCDMProxy(proxy) - ->Then(OwnerThread(), __func__, - [self = RefPtr{this}, - this](const SetCDMPromise::ResolveOrRejectValue& aVal) { - mSetCDMProxyRequest.Complete(); - if (aVal.IsResolve()) { - mSetCDMProxyPromise.Resolve(true, __func__); - } else { - mSetCDMProxyPromise.Reject(NS_ERROR_DOM_MEDIA_CDM_ERR, - __func__); - } - }) - ->Track(mSetCDMProxyRequest); - }); + mState.AsInitEngine() + ->mInitPromise + ->Then( + OwnerThread(), __func__, + [self = RefPtr{this}, proxy = RefPtr{aProxy}, this]( + const GenericNonExclusivePromise::ResolveOrRejectValue& aVal) { + mInitEngineForCDMRequest.Complete(); + SetCDMProxy(proxy) + ->Then(OwnerThread(), __func__, + [self = RefPtr{this}, this]( + const SetCDMPromise::ResolveOrRejectValue& aVal) { + mSetCDMProxyRequest.Complete(); + if (aVal.IsResolve()) { + mSetCDMProxyPromise.Resolve(true, __func__); + } else { + mSetCDMProxyPromise.Reject( + NS_ERROR_DOM_MEDIA_CDM_ERR, __func__); + } + }) + ->Track(mSetCDMProxyRequest); + }) + ->Track(mInitEngineForCDMRequest); return mSetCDMProxyPromise.Ensure(__func__); } @@ -1232,6 +1271,7 @@ bool ExternalEngineStateMachine::IsCDMProxySupported(CDMProxy* aProxy) { void ExternalEngineStateMachine::ReportTelemetry(const MediaResult& aError) { glean::mfcdm::ErrorExtra extraData; extraData.errorName = Some(aError.ErrorName()); + extraData.currentState = Some(nsAutoCString{StateToStr(mState.mName)}); nsAutoCString resolution; if (mInfo) { if (mInfo->HasAudio()) { @@ -1268,6 +1308,14 @@ void ExternalEngineStateMachine::ReportTelemetry(const MediaResult& aError) { } } +void ExternalEngineStateMachine::DecodeError(const MediaResult& aError) { + if (aError != NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA || + aError != NS_ERROR_DOM_MEDIA_CANCELED) { + mHasFatalError = true; + } + MediaDecoderStateMachineBase ::DecodeError(aError); +} + #undef FMT #undef LOG #undef LOGV diff --git a/dom/media/ExternalEngineStateMachine.h b/dom/media/ExternalEngineStateMachine.h index 84dedbe717..79183f894d 100644 --- a/dom/media/ExternalEngineStateMachine.h +++ b/dom/media/ExternalEngineStateMachine.h @@ -102,8 +102,10 @@ class ExternalEngineStateMachine final bool IsCDMProxySupported(CDMProxy* aProxy) override; + bool IsExternalEngineStateMachine() const override { return true; } + private: - ~ExternalEngineStateMachine() = default; + ~ExternalEngineStateMachine(); void AssertOnTaskQueue() const { MOZ_ASSERT(OnTaskQueue()); } @@ -233,6 +235,7 @@ class ExternalEngineStateMachine final void PreservesPitchChanged() override; void PlayStateChanged() override; void LoopingChanged() override; + void PlaybackRateChanged(); // Not supported. void SetIsLiveStream(bool aIsLiveStream) override {} @@ -293,6 +296,8 @@ class ExternalEngineStateMachine final void ReportTelemetry(const MediaResult& aError); + void DecodeError(const MediaResult& aError) override; + UniquePtr mEngine; bool mHasEnoughAudio = false; @@ -303,12 +308,20 @@ class ExternalEngineStateMachine final // Only used if setting CDM happens before the engine finishes initialization. MozPromiseHolder mSetCDMProxyPromise; MozPromiseRequestHolder mSetCDMProxyRequest; + MozPromiseRequestHolder mInitEngineForCDMRequest; // It would be zero for audio-only playback. gfx::IntSize mVideoDisplay; // It would be set if playback is encrypted. nsCString mKeySystem; + + // This array stores the tasks which needs to be executed only after the + // engine is ready but is called before that. It will be executed when + // starting running the engine. + nsTArray> mPendingTasks; + + bool mHasFatalError = false; }; class ExternalPlaybackEngine { diff --git a/dom/media/IdpSandbox.sys.mjs b/dom/media/IdpSandbox.sys.mjs index 8b94abf7dd..c8735c223d 100644 --- a/dom/media/IdpSandbox.sys.mjs +++ b/dom/media/IdpSandbox.sys.mjs @@ -56,7 +56,7 @@ ResourceLoader.prototype = { this.data += stream.read(count); }, - onStartRequest(request) {}, + onStartRequest() {}, onStopRequest(request, status) { if (Components.isSuccessCode(status)) { diff --git a/dom/media/ImageToI420.cpp b/dom/media/ImageToI420.cpp index 8fc5198b4a..0f7976cb63 100644 --- a/dom/media/ImageToI420.cpp +++ b/dom/media/ImageToI420.cpp @@ -11,6 +11,7 @@ #include "mozilla/dom/ImageUtils.h" #include "mozilla/gfx/Point.h" #include "mozilla/RefPtr.h" +#include "mozilla/Result.h" #include "nsThreadUtils.h" using mozilla::ImageFormat; @@ -75,7 +76,12 @@ nsresult ConvertToI420(Image* aImage, uint8_t* aDestY, int aDestStrideY, if (const PlanarYCbCrData* data = GetPlanarYCbCrData(aImage)) { const ImageUtils imageUtils(aImage); - switch (imageUtils.GetFormat()) { + Maybe format = imageUtils.GetFormat(); + if (format.isNothing()) { + MOZ_ASSERT_UNREACHABLE("YUV format conversion not implemented"); + return NS_ERROR_NOT_IMPLEMENTED; + } + switch (format.value()) { case ImageBitmapFormat::YUV420P: return MapRv(libyuv::I420ToI420( data->mYChannel, data->mYStride, data->mCbChannel, diff --git a/dom/media/MediaData.h b/dom/media/MediaData.h index ee1e204815..3ae8c1dbc2 100644 --- a/dom/media/MediaData.h +++ b/dom/media/MediaData.h @@ -97,8 +97,16 @@ class AlignedBuffer { } AlignedBuffer& operator=(AlignedBuffer&& aOther) noexcept { - this->~AlignedBuffer(); - new (this) AlignedBuffer(std::move(aOther)); + if (&aOther == this) { + return *this; + } + mData = aOther.mData; + mLength = aOther.mLength; + mBuffer = std::move(aOther.mBuffer); + mCapacity = aOther.mCapacity; + aOther.mData = nullptr; + aOther.mLength = 0; + aOther.mCapacity = 0; return *this; } @@ -268,7 +276,7 @@ class InflatableShortBuffer { // capacity, and the loop goes backward. float* output = reinterpret_cast(mBuffer.mData); for (size_t i = Length(); i--;) { - output[i] = AudioSampleToFloat(mBuffer.mData[i]); + output[i] = ConvertAudioSample(mBuffer.mData[i]); } AlignedFloatBuffer rv; rv.mBuffer = std::move(mBuffer.mBuffer); diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp index c7fdcb6844..23c30eed2a 100644 --- a/dom/media/MediaDecoder.cpp +++ b/dom/media/MediaDecoder.cpp @@ -140,9 +140,7 @@ void MediaDecoder::InitStatics() { # if defined(MOZ_FFMPEG) Preferences::Lock("media.utility-ffmpeg.enabled"); # endif // defined(MOZ_FFMPEG) -# if defined(MOZ_FFVPX) Preferences::Lock("media.utility-ffvpx.enabled"); -# endif // defined(MOZ_FFVPX) # if defined(MOZ_WMF) Preferences::Lock("media.utility-wmf.enabled"); # endif // defined(MOZ_WMF) @@ -194,6 +192,12 @@ void MediaDecoder::SetOutputCaptureState(OutputCaptureState aState, MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); MOZ_ASSERT_IF(aState == OutputCaptureState::Capture, aDummyTrack); + + if (mOutputCaptureState.Ref() != aState) { + LOG("Capture state change from %s to %s", + OutputCaptureStateToStr(mOutputCaptureState.Ref()), + OutputCaptureStateToStr(aState)); + } mOutputCaptureState = aState; if (mOutputDummyTrack.Ref().get() != aDummyTrack) { mOutputDummyTrack = nsMainThreadPtrHandle( @@ -450,6 +454,7 @@ void MediaDecoder::OnPlaybackErrorEvent(const MediaResult& aError) { } LOG("Need to create a new %s state machine", needExternalEngine ? "external engine" : "normal"); + mStateMachineRecreated = true; nsresult rv = CreateAndInitStateMachine( false /* live stream */, @@ -611,6 +616,7 @@ nsresult MediaDecoder::CreateAndInitStateMachine(bool aIsLiveStream, NS_ENSURE_TRUE(GetStateMachine(), NS_ERROR_FAILURE); GetStateMachine()->DispatchIsLiveStream(aIsLiveStream); + mMDSMCreationTime = Some(TimeStamp::Now()); nsresult rv = mDecoderStateMachine->Init(this); NS_ENSURE_SUCCESS(rv, rv); @@ -885,6 +891,14 @@ void MediaDecoder::FirstFrameLoaded( ChangeState(mNextState); } + // We only care about video first frame. + if (mInfo->HasVideo() && mMDSMCreationTime) { + mTelemetryProbesReporter->OntFirstFrameLoaded( + TimeStamp::Now() - *mMDSMCreationTime, IsMSE(), + mDecoderStateMachine->IsExternalEngineStateMachine()); + mMDSMCreationTime.reset(); + } + // GetOwner()->FirstFrameLoaded() might call us back. Put it at the bottom of // this function to avoid unexpected shutdown from reentrant calls. if (aEventVisibility != MediaDecoderEventVisibility::Suppressed) { diff --git a/dom/media/MediaDecoder.h b/dom/media/MediaDecoder.h index f2f10a67c6..a5494e9a84 100644 --- a/dom/media/MediaDecoder.h +++ b/dom/media/MediaDecoder.h @@ -207,6 +207,20 @@ class MediaDecoder : public DecoderDoctorLifeLogger { // not connected to streams created by captureStreamUntilEnded. enum class OutputCaptureState { Capture, Halt, None }; + const char* OutputCaptureStateToStr(OutputCaptureState aState) const { + switch (aState) { + case OutputCaptureState::Capture: + return "Capture"; + case OutputCaptureState::Halt: + return "Halt"; + case OutputCaptureState::None: + return "None"; + default: + MOZ_ASSERT_UNREACHABLE("Not defined state!"); + return "Not-defined"; + } + } + // Set the output capture state of this decoder. // @param aState Capture: Output is captured into output tracks, and // aDummyTrack must be provided. @@ -742,6 +756,12 @@ class MediaDecoder : public DecoderDoctorLifeLogger { bool mShouldDelaySeek = false; Maybe mDelayedSeekTarget; +# ifdef MOZ_WMF_MEDIA_ENGINE + // True if we've ever recreated a new state machine due to the previous state + // didn't support the media format or key system. + bool mStateMachineRecreated = false; +# endif + public: Canonical& CanonicalVolume() { return mVolume; } Canonical& CanonicalPreservesPitch() { return mPreservesPitch; } @@ -815,6 +835,11 @@ class MediaDecoder : public DecoderDoctorLifeLogger { // consistent with the previous destroyed one. bool mPendingStatusUpdateForNewlyCreatedStateMachine = false; # endif + + // The time of creating the media decoder state machine, it's used to record + // the probe for measuring the first video frame loaded time. Reset after + // reporting the measurement to avoid a dulpicated report. + Maybe mMDSMCreationTime; }; } // namespace mozilla diff --git a/dom/media/MediaDecoderStateMachineBase.cpp b/dom/media/MediaDecoderStateMachineBase.cpp index e60937c31f..38d51dbd8e 100644 --- a/dom/media/MediaDecoderStateMachineBase.cpp +++ b/dom/media/MediaDecoderStateMachineBase.cpp @@ -163,7 +163,9 @@ bool MediaDecoderStateMachineBase::OnTaskQueue() const { void MediaDecoderStateMachineBase::DecodeError(const MediaResult& aError) { MOZ_ASSERT(OnTaskQueue()); - LOGE("Decode error: %s", aError.Description().get()); + if (aError != NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR) { + LOGE("Decode error: %s", aError.Description().get()); + } PROFILER_MARKER_TEXT("MDSMBase::DecodeError", MEDIA_PLAYBACK, {}, aError.Description()); // Notify the decode error and MediaDecoder will shut down MDSM. diff --git a/dom/media/MediaDecoderStateMachineBase.h b/dom/media/MediaDecoderStateMachineBase.h index b4950746e8..5872151015 100644 --- a/dom/media/MediaDecoderStateMachineBase.h +++ b/dom/media/MediaDecoderStateMachineBase.h @@ -169,6 +169,8 @@ class MediaDecoderStateMachineBase { virtual bool IsCDMProxySupported(CDMProxy* aProxy) = 0; + virtual bool IsExternalEngineStateMachine() const { return false; } + protected: virtual ~MediaDecoderStateMachineBase() = default; @@ -195,7 +197,7 @@ class MediaDecoderStateMachineBase { virtual RefPtr Seek(const SeekTarget& aTarget) = 0; - void DecodeError(const MediaResult& aError); + virtual void DecodeError(const MediaResult& aError); // Functions used by assertions to ensure we're calling things // on the appropriate threads. diff --git a/dom/media/MediaDevices.cpp b/dom/media/MediaDevices.cpp index cfbc148337..b312fc1ec3 100644 --- a/dom/media/MediaDevices.cpp +++ b/dom/media/MediaDevices.cpp @@ -315,8 +315,6 @@ RefPtr MediaDevices::FilterExposedDevices( } haveDefaultOutput = true; break; - case MediaDeviceKind::EndGuard_: - continue; // Avoid `default:` so that `-Wswitch` catches missing // enumerators at compile time. } @@ -334,8 +332,6 @@ bool MediaDevices::CanExposeInfo(MediaDeviceKind aKind) const { case MediaDeviceKind::Audiooutput: // Assumes caller has used FilterExposedDevices() return true; - case MediaDeviceKind::EndGuard_: - break; // Avoid `default:` so that `-Wswitch` catches missing enumerators at // compile time. } @@ -550,7 +546,7 @@ already_AddRefed MediaDevices::GetDisplayMedia( // for us. vc.mMediaSource.Reset(); vc.mMediaSource.Construct().AssignASCII( - dom::MediaSourceEnumValues::GetString(MediaSourceEnum::Screen)); + dom::GetEnumString(MediaSourceEnum::Screen)); RefPtr self(this); MediaManager::Get() diff --git a/dom/media/MediaInfo.h b/dom/media/MediaInfo.h index abe5dfd9da..73704d1593 100644 --- a/dom/media/MediaInfo.h +++ b/dom/media/MediaInfo.h @@ -219,14 +219,6 @@ inline already_AddRefed ForceGetAudioCodecSpecificBlob( // information as a blob or where a blob is ambiguous. inline already_AddRefed GetAudioCodecSpecificBlob( const AudioCodecSpecificVariant& v) { - MOZ_ASSERT(!v.is(), - "NoCodecSpecificData shouldn't be used as a blob"); - MOZ_ASSERT(!v.is(), - "AacCodecSpecificData has 2 blobs internally, one should " - "explicitly be selected"); - MOZ_ASSERT(!v.is(), - "Mp3CodecSpecificData shouldn't be used as a blob"); - return ForceGetAudioCodecSpecificBlob(v); } @@ -470,7 +462,8 @@ class VideoInfo : public TrackInfo { rv.AppendPrintf("extra data: %zu bytes", mExtraData->Length()); } rv.AppendPrintf("rotation: %d", static_cast(mRotation)); - rv.AppendPrintf("colors: %s", ColorDepthStrings[static_cast(mColorDepth)]); + rv.AppendPrintf("colors: %s", + ColorDepthStrings[static_cast(mColorDepth)]); if (mColorSpace) { rv.AppendPrintf( "YUV colorspace: %s ", @@ -486,7 +479,8 @@ class VideoInfo : public TrackInfo { "transfer function %s ", TransferFunctionStrings[static_cast(mTransferFunction.value())]); } - rv.AppendPrintf("color range: %s", ColorRangeStrings[static_cast(mColorRange)]); + rv.AppendPrintf("color range: %s", + ColorRangeStrings[static_cast(mColorRange)]); if (mImageRect) { rv.AppendPrintf("image rect: %dx%d", mImageRect->Width(), mImageRect->Height()); diff --git a/dom/media/MediaManager.cpp b/dom/media/MediaManager.cpp index 25eeb876d4..422769587a 100644 --- a/dom/media/MediaManager.cpp +++ b/dom/media/MediaManager.cpp @@ -872,8 +872,7 @@ MediaDevice::MediaDevice(MediaEngine* aEngine, MediaSourceEnum aMediaSource, mCanRequestOsLevelPrompt(canRequestOsLevelPrompt == OsPromptable::Yes), mIsFake(mEngine->IsFake()), mIsPlaceholder(aIsPlaceholder == IsPlaceholder::Yes), - mType( - NS_ConvertASCIItoUTF16(dom::MediaDeviceKindValues::GetString(mKind))), + mType(NS_ConvertASCIItoUTF16(dom::GetEnumString(mKind))), mRawID(aRawID), mRawGroupID(aRawGroupID), mRawName(aRawName) { @@ -895,8 +894,7 @@ MediaDevice::MediaDevice(MediaEngine* aEngine, mCanRequestOsLevelPrompt(false), mIsFake(false), mIsPlaceholder(false), - mType( - NS_ConvertASCIItoUTF16(dom::MediaDeviceKindValues::GetString(mKind))), + mType(NS_ConvertASCIItoUTF16(dom::GetEnumString(mKind))), mRawID(aRawID), mRawGroupID(mAudioDeviceInfo->GroupID()), mRawName(mAudioDeviceInfo->Name()) {} @@ -1064,8 +1062,7 @@ LocalMediaDevice::GetMediaSource(nsAString& aMediaSource) { if (Kind() == MediaDeviceKind::Audiooutput) { aMediaSource.Truncate(); } else { - aMediaSource.AssignASCII( - dom::MediaSourceEnumValues::GetString(GetMediaSource())); + aMediaSource.AssignASCII(dom::GetEnumString(GetMediaSource())); } return NS_OK; } @@ -2249,14 +2246,6 @@ MediaManager::MediaManager(already_AddRefed aMediaThread) GetPrefs(branch, nullptr); } } - LOG("%s: default prefs: %dx%d @%dfps, %dHz test tones, aec: %s," - "agc: %s, hpf: %s, noise: %s, agc level: %d, agc version: %s, noise " - "level: %d, transient: %s, channels %d", - __FUNCTION__, mPrefs.mWidth, mPrefs.mHeight, mPrefs.mFPS, mPrefs.mFreq, - mPrefs.mAecOn ? "on" : "off", mPrefs.mAgcOn ? "on" : "off", - mPrefs.mHPFOn ? "on" : "off", mPrefs.mNoiseOn ? "on" : "off", mPrefs.mAgc, - mPrefs.mAgc2Forced ? "2" : "1", mPrefs.mNoise, - mPrefs.mTransientOn ? "on" : "off", mPrefs.mChannels); } NS_IMPL_ISUPPORTS(MediaManager, nsIMediaManagerService, nsIMemoryReporter, @@ -2755,10 +2744,10 @@ RefPtr MediaManager::GetUserMedia( auto& vc = c.mVideo.GetAsMediaTrackConstraints(); if (!vc.mMediaSource.WasPassed()) { vc.mMediaSource.Construct().AssignASCII( - dom::MediaSourceEnumValues::GetString(MediaSourceEnum::Camera)); + dom::GetEnumString(MediaSourceEnum::Camera)); } - videoType = StringToEnum(dom::MediaSourceEnumValues::strings, - vc.mMediaSource.Value(), MediaSourceEnum::Other); + videoType = dom::StringToEnum(vc.mMediaSource.Value()) + .valueOr(MediaSourceEnum::Other); Telemetry::Accumulate(Telemetry::WEBRTC_GET_USER_MEDIA_TYPE, (uint32_t)videoType); switch (videoType) { @@ -2823,8 +2812,7 @@ RefPtr MediaManager::GetUserMedia( if (videoType == MediaSourceEnum::Screen || videoType == MediaSourceEnum::Browser) { videoType = MediaSourceEnum::Window; - vc.mMediaSource.Value().AssignASCII( - dom::MediaSourceEnumValues::GetString(videoType)); + vc.mMediaSource.Value().AssignASCII(dom::GetEnumString(videoType)); } // only allow privileged content to set the window id if (vc.mBrowserWindow.WasPassed()) { @@ -2848,10 +2836,10 @@ RefPtr MediaManager::GetUserMedia( auto& ac = c.mAudio.GetAsMediaTrackConstraints(); if (!ac.mMediaSource.WasPassed()) { ac.mMediaSource.Construct(NS_ConvertASCIItoUTF16( - dom::MediaSourceEnumValues::GetString(MediaSourceEnum::Microphone))); + dom::GetEnumString(MediaSourceEnum::Microphone))); } - audioType = StringToEnum(dom::MediaSourceEnumValues::strings, - ac.mMediaSource.Value(), MediaSourceEnum::Other); + audioType = dom::StringToEnum(ac.mMediaSource.Value()) + .valueOr(MediaSourceEnum::Other); Telemetry::Accumulate(Telemetry::WEBRTC_GET_USER_MEDIA_TYPE, (uint32_t)audioType); @@ -3498,6 +3486,14 @@ void MediaManager::GetPrefs(nsIPrefBranch* aBranch, const char* aData) { GetPref(aBranch, "media.getusermedia.noise", aData, &mPrefs.mNoise); GetPref(aBranch, "media.getusermedia.channels", aData, &mPrefs.mChannels); #endif + LOG("%s: default prefs: %dx%d @%dfps, %dHz test tones, aec: %s, " + "agc: %s, hpf: %s, noise: %s, agc level: %d, agc version: %s, noise " + "level: %d, transient: %s, channels %d", + __FUNCTION__, mPrefs.mWidth, mPrefs.mHeight, mPrefs.mFPS, mPrefs.mFreq, + mPrefs.mAecOn ? "on" : "off", mPrefs.mAgcOn ? "on" : "off", + mPrefs.mHPFOn ? "on" : "off", mPrefs.mNoiseOn ? "on" : "off", mPrefs.mAgc, + mPrefs.mAgc2Forced ? "2" : "1", mPrefs.mNoise, + mPrefs.mTransientOn ? "on" : "off", mPrefs.mChannels); } void MediaManager::Shutdown() { @@ -3650,8 +3646,6 @@ nsresult MediaManager::Observe(nsISupports* aSubject, const char* aTopic, nsCOMPtr branch(do_QueryInterface(aSubject)); if (branch) { GetPrefs(branch, NS_ConvertUTF16toUTF8(aData).get()); - LOG("%s: %dx%d @%dfps", __FUNCTION__, mPrefs.mWidth, mPrefs.mHeight, - mPrefs.mFPS); DeviceListChanged(); } } else if (!strcmp(aTopic, "last-pb-context-exited")) { @@ -4006,8 +4000,7 @@ void DeviceListener::Activate(RefPtr aDevice, MOZ_ASSERT(NS_IsMainThread(), "Only call on main thread"); LOG("DeviceListener %p activating %s device %p", this, - nsCString(dom::MediaDeviceKindValues::GetString(aDevice->Kind())).get(), - aDevice.get()); + dom::GetEnumString(aDevice->Kind()).get(), aDevice.get()); MOZ_ASSERT(!mStopped, "Cannot activate stopped device listener"); MOZ_ASSERT(!Activated(), "Already activated"); @@ -4063,18 +4056,15 @@ DeviceListener::InitializeAsync() { } if (NS_FAILED(rv)) { nsCString log; - log.AppendPrintf( - "Starting %s failed", - nsCString(dom::MediaDeviceKindValues::GetString(kind)) - .get()); + log.AppendPrintf("Starting %s failed", + dom::GetEnumString(kind).get()); aHolder.Reject( MakeRefPtr(MediaMgrError::Name::AbortError, std::move(log)), __func__); return; } - LOG("started %s device %p", - nsCString(dom::MediaDeviceKindValues::GetString(kind)).get(), + LOG("started %s device %p", dom::GetEnumString(kind).get(), device.get()); aHolder.Resolve(true, __func__); }) @@ -4182,9 +4172,7 @@ auto DeviceListener::UpdateDevice(bool aOn) -> RefPtr { } LOG("DeviceListener %p turning %s %s input device %s", this, aOn ? "on" : "off", - nsCString( - dom::MediaDeviceKindValues::GetString(GetDevice()->Kind())) - .get(), + dom::GetEnumString(GetDevice()->Kind()).get(), NS_SUCCEEDED(aResult) ? "succeeded" : "failed"); if (NS_FAILED(aResult) && aResult != NS_ERROR_ABORT) { @@ -4217,8 +4205,7 @@ void DeviceListener::SetDeviceEnabled(bool aEnable) { LOG("DeviceListener %p %s %s device", this, aEnable ? "enabling" : "disabling", - nsCString(dom::MediaDeviceKindValues::GetString(GetDevice()->Kind())) - .get()); + dom::GetEnumString(GetDevice()->Kind()).get()); state.mTrackEnabled = aEnable; @@ -4274,9 +4261,7 @@ void DeviceListener::SetDeviceEnabled(bool aEnable) { LOG("DeviceListener %p %s %s device - starting device operation", this, aEnable ? "enabling" : "disabling", - nsCString( - dom::MediaDeviceKindValues::GetString(GetDevice()->Kind())) - .get()); + dom::GetEnumString(GetDevice()->Kind()).get()); if (state.mStopped) { // Source was stopped between timer resolving and this runnable. @@ -4345,8 +4330,7 @@ void DeviceListener::SetDeviceMuted(bool aMute) { DeviceState& state = *mDeviceState; LOG("DeviceListener %p %s %s device", this, aMute ? "muting" : "unmuting", - nsCString(dom::MediaDeviceKindValues::GetString(GetDevice()->Kind())) - .get()); + dom::GetEnumString(GetDevice()->Kind()).get()); if (state.mStopped) { // Device terminally stopped. Updating device state is pointless. @@ -4360,8 +4344,7 @@ void DeviceListener::SetDeviceMuted(bool aMute) { LOG("DeviceListener %p %s %s device - starting device operation", this, aMute ? "muting" : "unmuting", - nsCString(dom::MediaDeviceKindValues::GetString(GetDevice()->Kind())) - .get()); + dom::GetEnumString(GetDevice()->Kind()).get()); state.mDeviceMuted = aMute; @@ -4467,9 +4450,7 @@ RefPtr DeviceListener::ApplyConstraints( if (mStopped || mDeviceState->mStopped) { LOG("DeviceListener %p %s device applyConstraints, but device is stopped", - this, - nsCString(dom::MediaDeviceKindValues::GetString(GetDevice()->Kind())) - .get()); + this, dom::GetEnumString(GetDevice()->Kind()).get()); return DeviceListenerPromise::CreateAndResolve(false, __func__); } diff --git a/dom/media/MediaQueue.h b/dom/media/MediaQueue.h index 4609493339..9c4c829c87 100644 --- a/dom/media/MediaQueue.h +++ b/dom/media/MediaQueue.h @@ -25,6 +25,7 @@ extern LazyLogModule gMediaDecoderLog; class AudioData; class VideoData; +class EncodedFrame; template struct TimestampAdjustmentTrait { @@ -46,13 +47,24 @@ struct NonTimestampAdjustmentTrait { static const bool mValue = !TimestampAdjustmentTrait::mValue; }; +template +struct DurationTypeTrait { + using type = media::TimeUnit; +}; + +template <> +struct DurationTypeTrait { + using type = uint64_t; +}; + template class MediaQueue : private nsRefPtrDeque { public: - MediaQueue() + explicit MediaQueue(bool aEnablePreciseDuration = false) : nsRefPtrDeque(), mRecursiveMutex("mediaqueue"), - mEndOfStream(false) {} + mEndOfStream(false), + mEnablePreciseDuration(aEnablePreciseDuration) {} ~MediaQueue() { Reset(); } @@ -97,6 +109,7 @@ class MediaQueue : private nsRefPtrDeque { AdjustTimeStampIfNeeded(aItem); } nsRefPtrDeque::PushFront(aItem); + AddDurationToPreciseDuration(aItem); } inline void Push(T* aItem) { @@ -112,6 +125,7 @@ class MediaQueue : private nsRefPtrDeque { MOZ_DIAGNOSTIC_ASSERT(item->GetEndTime() >= item->mTime); AdjustTimeStampIfNeeded(item); nsRefPtrDeque::Push(dont_AddRef(item)); + AddDurationToPreciseDuration(item); mPushEvent.Notify(RefPtr(item)); // Pushing new data after queue has ended means that the stream is active @@ -126,6 +140,7 @@ class MediaQueue : private nsRefPtrDeque { RefPtr rv = nsRefPtrDeque::PopFront(); if (rv) { MOZ_DIAGNOSTIC_ASSERT(rv->GetEndTime() >= rv->mTime); + SubtractDurationFromPreciseDuration(rv); mPopFrontEvent.Notify(RefPtr(rv)); } return rv.forget(); @@ -133,7 +148,12 @@ class MediaQueue : private nsRefPtrDeque { inline already_AddRefed PopBack() { RecursiveMutexAutoLock lock(mRecursiveMutex); - return nsRefPtrDeque::Pop(); + RefPtr rv = nsRefPtrDeque::Pop(); + if (rv) { + MOZ_DIAGNOSTIC_ASSERT(rv->GetEndTime() >= rv->mTime); + SubtractDurationFromPreciseDuration(rv); + } + return rv.forget(); } inline RefPtr PeekFront() const { @@ -151,6 +171,7 @@ class MediaQueue : private nsRefPtrDeque { nsRefPtrDeque::Erase(); SetOffset(media::TimeUnit::Zero()); mEndOfStream = false; + ResetPreciseDuration(); } bool AtEndOfStream() const { @@ -186,6 +207,12 @@ class MediaQueue : private nsRefPtrDeque { return (last->GetEndTime() - first->mTime).ToMicroseconds(); } + // Return a precise duration if the feature is enabled. Otherwise, return -1. + int64_t PreciseDuration() const { + RecursiveMutexAutoLock lock(mRecursiveMutex); + return GetPreciseDuration(); + } + void LockedForEach(nsDequeFunctor& aFunctor) const { RecursiveMutexAutoLock lock(mRecursiveMutex); nsRefPtrDeque::ForEach(aFunctor); @@ -268,6 +295,59 @@ class MediaQueue : private nsRefPtrDeque { // the media queue starts receiving looped data, which timestamp needs to be // modified. media::TimeUnit mOffset; + + inline void AddDurationToPreciseDuration(T* aItem) { + if (!mEnablePreciseDuration) { + return; + } + if constexpr (std::is_same_v::type, + media::TimeUnit> || + std::is_same_v::type, + uint64_t>) { + mPreciseDuration += aItem->mDuration; + } + } + + inline void SubtractDurationFromPreciseDuration(T* aItem) { + if (!mEnablePreciseDuration) { + return; + } + if constexpr (std::is_same_v::type, + media::TimeUnit> || + std::is_same_v::type, + uint64_t>) { + mPreciseDuration -= aItem->mDuration; + } + } + + inline void ResetPreciseDuration() { + if (!mEnablePreciseDuration) { + return; + } + if constexpr (std::is_same_v::type, + media::TimeUnit>) { + mPreciseDuration = media::TimeUnit::Zero(); + } else if constexpr (std::is_same_v::type, + uint64_t>) { + mPreciseDuration = 0; + } + } + + inline int64_t GetPreciseDuration() const { + if (mEnablePreciseDuration) { + if constexpr (std::is_same_v::type, + media::TimeUnit>) { + return mPreciseDuration.ToMicroseconds(); + } else if constexpr (std::is_same_v::type, + uint64_t>) { + return mPreciseDuration; + } + } + return -1; + } + + typename DurationTypeTrait::type mPreciseDuration; + const bool mEnablePreciseDuration = false; }; } // namespace mozilla diff --git a/dom/media/MediaStreamTrack.cpp b/dom/media/MediaStreamTrack.cpp index 8ddb16d8e1..4235d96d13 100644 --- a/dom/media/MediaStreamTrack.cpp +++ b/dom/media/MediaStreamTrack.cpp @@ -324,7 +324,7 @@ void MediaStreamTrack::GetSettings(dom::MediaTrackSettings& aResult, } if (aResult.mFacingMode.WasPassed()) { aResult.mFacingMode.Value().AssignASCII( - VideoFacingModeEnumValues::GetString(VideoFacingModeEnum::User)); + GetEnumString(VideoFacingModeEnum::User)); } } diff --git a/dom/media/MediaTrackGraph.cpp b/dom/media/MediaTrackGraph.cpp index 157ad403d2..2af7aacb1f 100644 --- a/dom/media/MediaTrackGraph.cpp +++ b/dom/media/MediaTrackGraph.cpp @@ -1393,6 +1393,9 @@ void MediaTrackGraphImpl::SelectOutputDeviceForAEC() { if (currentDeviceIndex == mOutputDevices.NoIndex) { // Outputs for this device have been removed. // Fall back to the primary output device. + LOG(LogLevel::Info, ("%p: No remaining outputs to device %p. " + "Switch to primary output device %p for AEC", + this, mOutputDeviceForAEC, PrimaryOutputDeviceID())); mOutputDeviceForAEC = PrimaryOutputDeviceID(); currentDeviceIndex = 0; MOZ_ASSERT(mOutputDevices[0].mDeviceID == mOutputDeviceForAEC); @@ -1425,6 +1428,9 @@ void MediaTrackGraphImpl::SelectOutputDeviceForAEC() { for (const auto& output : outputDeviceEntry.mTrackOutputs) { if (HasNonNullAudio(output)) { // Switch to this device. + LOG(LogLevel::Info, + ("%p: Switch output device for AEC from silent %p to non-null %p", + this, mOutputDeviceForAEC, outputDeviceEntry.mDeviceID)); mOutputDeviceForAEC = outputDeviceEntry.mDeviceID; return; } diff --git a/dom/media/PeerConnection.sys.mjs b/dom/media/PeerConnection.sys.mjs index 1bebde8159..00b4023c2f 100644 --- a/dom/media/PeerConnection.sys.mjs +++ b/dom/media/PeerConnection.sys.mjs @@ -24,7 +24,7 @@ const PC_COREQUEST_CID = Components.ID( "{74b2122d-65a8-4824-aa9e-3d664cb75dc2}" ); -function logMsg(msg, file, line, flag, winID) { +function logWebRTCMsg(msg, file, line, flag, win) { let scriptErrorClass = Cc["@mozilla.org/scripterror;1"]; let scriptError = scriptErrorClass.createInstance(Ci.nsIScriptError); scriptError.initWithWindowID( @@ -35,9 +35,14 @@ function logMsg(msg, file, line, flag, winID) { 0, flag, "content javascript", - winID + win.windowGlobalChild.innerWindowId ); Services.console.logMessage(scriptError); + if ( + Services.prefs.getBoolPref("media.peerconnection.treat_warnings_as_errors") + ) { + throw new win.TypeError(msg); + } } let setupPrototype = (_class, dict) => { @@ -100,7 +105,7 @@ export class GlobalPCList { if (this._list[winID] === undefined) { return; } - this._list[winID] = this._list[winID].filter(function (e, i, a) { + this._list[winID] = this._list[winID].filter(function (e) { return e.get() !== null; }); @@ -237,14 +242,12 @@ export class RTCSessionDescription { init(win) { this._win = win; this._winID = this._win.windowGlobalChild.innerWindowId; + this._legacyPref = Services.prefs.getBoolPref( + "media.peerconnection.description.legacy.enabled" + ); } __init({ type, sdp }) { - if (!type) { - throw new this._win.TypeError( - "Missing required 'type' member of RTCSessionDescriptionInit" - ); - } Object.assign(this, { _type: type, _sdp: sdp }); } @@ -252,6 +255,10 @@ export class RTCSessionDescription { return this._type; } set type(type) { + if (!this._legacyPref) { + // TODO: this throws even in sloppy mode. Remove in bug 1883992 + throw new this._win.TypeError("setting getter-only property type"); + } this.warn(); this._type = type; } @@ -260,6 +267,10 @@ export class RTCSessionDescription { return this._sdp; } set sdp(sdp) { + if (!this._legacyPref) { + // TODO: this throws even in sloppy mode. Remove in bug 1883992 + throw new this._win.TypeError("setting getter-only property sdp"); + } this.warn(); this._sdp = sdp; } @@ -267,23 +278,26 @@ export class RTCSessionDescription { warn() { if (!this._warned) { // Warn once per RTCSessionDescription about deprecated writable usage. - this.logWarning( - "RTCSessionDescription's members are readonly! " + - "Writing to them is deprecated and will break soon!" - ); + if (this._legacyPref) { + this.logMsg( + "RTCSessionDescription's members are readonly! " + + "Writing to them is deprecated and will break soon!", + Ci.nsIScriptError.warningFlag + ); + } else { + this.logMsg( + "RTCSessionDescription's members are readonly! " + + "Writing to them no longer works!", + Ci.nsIScriptError.errorFlag + ); + } this._warned = true; } } - logWarning(msg) { + logMsg(msg, flag) { let err = this._win.Error(); - logMsg( - msg, - err.fileName, - err.lineNumber, - Ci.nsIScriptError.warningFlag, - this._winID - ); + logWebRTCMsg(msg, err.fileName, err.lineNumber, flag, this._win); } } @@ -350,6 +364,13 @@ export class RTCPeerConnection { constructor() { this._pc = null; this._closed = false; + this._pendingLocalDescription = null; + this._pendingRemoteDescription = null; + this._currentLocalDescription = null; + this._currentRemoteDescription = null; + this._legacyPref = Services.prefs.getBoolPref( + "media.peerconnection.description.legacy.enabled" + ); // http://rtcweb-wg.github.io/jsep/#rfc.section.4.1.9 // canTrickle == null means unknown; when a remote description is received it @@ -723,7 +744,7 @@ export class RTCPeerConnection { } }; - var stunServers = 0; + let stunServers = 0; iceServers.forEach(({ urls, username, credential, credentialType }) => { if (!urls) { @@ -788,11 +809,7 @@ export class RTCPeerConnection { } if (stunServers >= 5) { this.logError( - "Using five or more STUN/TURN servers causes problems" - ); - } else if (stunServers > 2) { - this.logWarning( - "Using more than two STUN/TURN servers slows down discovery" + "Using five or more STUN/TURN servers slows down discovery" ); } }); @@ -861,7 +878,7 @@ export class RTCPeerConnection { } logMsg(msg, file, line, flag) { - return logMsg(msg, file, line, flag, this._winID); + return logWebRTCMsg(msg, file, line, flag, this._win); } getEH(type) { @@ -996,7 +1013,7 @@ export class RTCPeerConnection { return this._async(() => this._createAnswer(optionsOrOnSucc)); } - _createAnswer(options) { + _createAnswer() { this._checkClosed(); return this._chain(() => this._createAnAnswer()); } @@ -1192,11 +1209,6 @@ export class RTCPeerConnection { } _setRemoteDescription({ type, sdp }) { - if (!type) { - throw new this._win.TypeError( - "Missing required 'type' member of RTCSessionDescriptionInit" - ); - } if (type == "pranswer") { throw new this._win.DOMException( "pranswer not yet implemented", @@ -1541,24 +1553,36 @@ export class RTCPeerConnection { return this.pendingLocalDescription || this.currentLocalDescription; } + cacheDescription(name, type, sdp) { + if ( + !this[name] || + this[name].type != type || + this[name].sdp != sdp || + this._legacyPref + ) { + this[name] = sdp.length + ? new this._win.RTCSessionDescription({ type, sdp }) + : null; + } + return this[name]; + } + get currentLocalDescription() { this._checkClosed(); - const sdp = this._pc.currentLocalDescription; - if (!sdp.length) { - return null; - } - const type = this._pc.currentOfferer ? "offer" : "answer"; - return new this._win.RTCSessionDescription({ type, sdp }); + return this.cacheDescription( + "_currentLocalDescription", + this._pc.currentOfferer ? "offer" : "answer", + this._pc.currentLocalDescription + ); } get pendingLocalDescription() { this._checkClosed(); - const sdp = this._pc.pendingLocalDescription; - if (!sdp.length) { - return null; - } - const type = this._pc.pendingOfferer ? "offer" : "answer"; - return new this._win.RTCSessionDescription({ type, sdp }); + return this.cacheDescription( + "_pendingLocalDescription", + this._pc.pendingOfferer ? "offer" : "answer", + this._pc.pendingLocalDescription + ); } get remoteDescription() { @@ -1567,22 +1591,20 @@ export class RTCPeerConnection { get currentRemoteDescription() { this._checkClosed(); - const sdp = this._pc.currentRemoteDescription; - if (!sdp.length) { - return null; - } - const type = this._pc.currentOfferer ? "answer" : "offer"; - return new this._win.RTCSessionDescription({ type, sdp }); + return this.cacheDescription( + "_currentRemoteDescription", + this._pc.currentOfferer ? "answer" : "offer", + this._pc.currentRemoteDescription + ); } get pendingRemoteDescription() { this._checkClosed(); - const sdp = this._pc.pendingRemoteDescription; - if (!sdp.length) { - return null; - } - const type = this._pc.pendingOfferer ? "answer" : "offer"; - return new this._win.RTCSessionDescription({ type, sdp }); + return this.cacheDescription( + "_pendingRemoteDescription", + this._pc.pendingOfferer ? "answer" : "offer", + this._pc.pendingRemoteDescription + ); } get peerIdentity() { @@ -1918,8 +1940,7 @@ export class PeerConnectionObserver { switch (state) { case "IceConnectionState": - let connState = this._dompc._pc.iceConnectionState; - this.handleIceConnectionStateChange(connState); + this.handleIceConnectionStateChange(this._dompc._pc.iceConnectionState); break; case "IceGatheringState": diff --git a/dom/media/VideoFrameConverter.h b/dom/media/VideoFrameConverter.h index 36132c1b45..31b3104955 100644 --- a/dom/media/VideoFrameConverter.h +++ b/dom/media/VideoFrameConverter.h @@ -109,8 +109,8 @@ class VideoFrameConverter { // for processing so it can be immediately sent. mLastFrameQueuedForProcessing.mTime = time; - MOZ_ALWAYS_SUCCEEDS(mTaskQueue->Dispatch( - NewRunnableMethod>( + MOZ_ALWAYS_SUCCEEDS( + mTaskQueue->Dispatch(NewRunnableMethod( "VideoFrameConverter::ProcessVideoFrame", this, &VideoFrameConverter::ProcessVideoFrame, mLastFrameQueuedForProcessing))); @@ -138,8 +138,8 @@ class VideoFrameConverter { mLastFrameQueuedForProcessing.mForceBlack = true; mLastFrameQueuedForProcessing.mImage = nullptr; - MOZ_ALWAYS_SUCCEEDS(mTaskQueue->Dispatch( - NewRunnableMethod>( + MOZ_ALWAYS_SUCCEEDS( + mTaskQueue->Dispatch(NewRunnableMethod( "VideoFrameConverter::ProcessVideoFrame", this, &VideoFrameConverter::ProcessVideoFrame, mLastFrameQueuedForProcessing))); @@ -293,11 +293,10 @@ class VideoFrameConverter { return; } - MOZ_ALWAYS_SUCCEEDS(mTaskQueue->Dispatch( - NewRunnableMethod>( - "VideoFrameConverter::ProcessVideoFrame", this, - &VideoFrameConverter::ProcessVideoFrame, - mLastFrameQueuedForProcessing))); + MOZ_ALWAYS_SUCCEEDS(mTaskQueue->Dispatch(NewRunnableMethod( + "VideoFrameConverter::ProcessVideoFrame", this, + &VideoFrameConverter::ProcessVideoFrame, + mLastFrameQueuedForProcessing))); } void ProcessVideoFrame(const FrameToProcess& aFrame) { @@ -364,7 +363,9 @@ class VideoFrameConverter { aFrame.mImage->AsPlanarYCbCrImage(); if (image) { dom::ImageUtils utils(image); - if (utils.GetFormat() == dom::ImageBitmapFormat::YUV420P && + Maybe format = utils.GetFormat(); + if (format.isSome() && + format.value() == dom::ImageBitmapFormat::YUV420P && image->GetData()) { const layers::PlanarYCbCrData* data = image->GetData(); rtc::scoped_refptr video_frame_buffer = diff --git a/dom/media/VideoUtils.cpp b/dom/media/VideoUtils.cpp index 279ce9b912..90057f7c83 100644 --- a/dom/media/VideoUtils.cpp +++ b/dom/media/VideoUtils.cpp @@ -4,7 +4,6 @@ #include "VideoUtils.h" -#include #include #include "CubebUtils.h" @@ -19,7 +18,6 @@ #include "mozilla/StaticPrefs_accessibility.h" #include "mozilla/StaticPrefs_media.h" #include "mozilla/TaskQueue.h" -#include "mozilla/Telemetry.h" #include "nsCharSeparatedTokenizer.h" #include "nsContentTypeParser.h" #include "nsIConsoleService.h" @@ -29,7 +27,6 @@ #include "nsNetCID.h" #include "nsServiceManagerUtils.h" #include "nsThreadUtils.h" -#include "AudioStream.h" namespace mozilla { diff --git a/dom/media/autoplay/test/browser/browser_autoplay_videoDocument.js b/dom/media/autoplay/test/browser/browser_autoplay_videoDocument.js index 77ce4ddbc1..433e4fd51e 100644 --- a/dom/media/autoplay/test/browser/browser_autoplay_videoDocument.js +++ b/dom/media/autoplay/test/browser/browser_autoplay_videoDocument.js @@ -31,8 +31,8 @@ async function checkIsIframeVideoDocumentAutoplay(browser) { const iframe = content.document.createElement("iframe"); iframe.src = pageURL; content.document.body.appendChild(iframe); - const iframeLoaded = new Promise((resolve, reject) => { - iframe.addEventListener("load", e => resolve(), { once: true }); + const iframeLoaded = new Promise(resolve => { + iframe.addEventListener("load", () => resolve(), { once: true }); }); await iframeLoaded; return iframe.browsingContext; diff --git a/dom/media/autoplay/test/browser/head.js b/dom/media/autoplay/test/browser/head.js index c84850900a..6522104b7f 100644 --- a/dom/media/autoplay/test/browser/head.js +++ b/dom/media/autoplay/test/browser/head.js @@ -29,10 +29,10 @@ function loadAutoplayVideo(browser, args) { info("- create a new autoplay video -"); let video = content.document.createElement("video"); video.id = "v1"; - video.didPlayPromise = new Promise((resolve, reject) => { + video.didPlayPromise = new Promise(resolve => { video.addEventListener( "playing", - e => { + () => { video.didPlay = true; resolve(); }, @@ -40,7 +40,7 @@ function loadAutoplayVideo(browser, args) { ); video.addEventListener( "blocked", - e => { + () => { video.didPlay = false; resolve(); }, @@ -54,7 +54,7 @@ function loadAutoplayVideo(browser, args) { info("will call play() when reached loadedmetadata"); video.addEventListener( "loadedmetadata", - e => { + () => { video.play().then( () => { info("video play() resolved"); diff --git a/dom/media/autoplay/test/mochitest/file_autoplay_policy_eventdown_activation.html b/dom/media/autoplay/test/mochitest/file_autoplay_policy_eventdown_activation.html index e25b6401d1..9fcda2afe9 100644 --- a/dom/media/autoplay/test/mochitest/file_autoplay_policy_eventdown_activation.html +++ b/dom/media/autoplay/test/mochitest/file_autoplay_policy_eventdown_activation.html @@ -36,8 +36,8 @@ let x = eventNames.map( (eventName) => { - return new Promise(function (resolve, reject) { - window.addEventListener(eventName, async function (event) { + return new Promise(function (resolve) { + window.addEventListener(eventName, async function () { let p = await element.play().then(() => true, () => false); ok(p, "Expect to be activated already in " + eventName); resolve(); diff --git a/dom/media/autoplay/test/mochitest/file_autoplay_policy_key_blacklist.html b/dom/media/autoplay/test/mochitest/file_autoplay_policy_key_blacklist.html index b901df3324..fc8c47065e 100644 --- a/dom/media/autoplay/test/mochitest/file_autoplay_policy_key_blacklist.html +++ b/dom/media/autoplay/test/mochitest/file_autoplay_policy_key_blacklist.html @@ -91,7 +91,7 @@ input.blur(); } - async function testAutoplayKeyBlacklist(testCase, parent_window) { + async function testAutoplayKeyBlacklist() { let element = document.createElement("video"); element.preload = "auto"; element.src = "short.mp4"; diff --git a/dom/media/autoplay/test/mochitest/file_autoplay_policy_play_before_loadedmetadata.html b/dom/media/autoplay/test/mochitest/file_autoplay_policy_play_before_loadedmetadata.html index 3594d0f236..dd358e6dc1 100644 --- a/dom/media/autoplay/test/mochitest/file_autoplay_policy_play_before_loadedmetadata.html +++ b/dom/media/autoplay/test/mochitest/file_autoplay_policy_play_before_loadedmetadata.html @@ -22,7 +22,7 @@ window.is = window.opener.is; window.info = window.opener.info; - async function testPlayBeforeLoadedMetata(testCase, parent_window) { + async function testPlayBeforeLoadedMetata(testCase) { info("testPlayBeforeLoadedMetata: " + testCase.resource); let element = document.createElement("video"); diff --git a/dom/media/autoplay/test/mochitest/file_autoplay_policy_unmute_pauses.html b/dom/media/autoplay/test/mochitest/file_autoplay_policy_unmute_pauses.html index 125ee156b6..f3b4e7da4d 100644 --- a/dom/media/autoplay/test/mochitest/file_autoplay_policy_unmute_pauses.html +++ b/dom/media/autoplay/test/mochitest/file_autoplay_policy_unmute_pauses.html @@ -22,8 +22,8 @@ window.is = window.opener.is; window.info = window.opener.info; - function testAutoplayUnmutePauses(testCase, parent_window) { - return new Promise(function (resolve, reject) { + function testAutoplayUnmutePauses(testCase) { + return new Promise(function (resolve) { info("testAutoplayUnmutePauses: " + testCase.property); diff --git a/dom/media/doctor/test/browser/browser_doctor_notification.js b/dom/media/doctor/test/browser/browser_doctor_notification.js index 5789622e23..6068304dfa 100644 --- a/dom/media/doctor/test/browser/browser_doctor_notification.js +++ b/dom/media/doctor/test/browser/browser_doctor_notification.js @@ -233,15 +233,11 @@ async function setFormatDiagnosticsReportForMimeType(tab, params) { async function setDecodeError(tab, params) { info(`start check for ${params.error}`); - await SpecialPowers.spawn( - tab.linkedBrowser, - [params], - async (params, shouldReportNotification) => { - const video = content.document.createElement("video"); - SpecialPowers.wrap(video).setDecodeError(params.error); - await content._waitForReport(params, params.shouldReportNotification); - } - ); + await SpecialPowers.spawn(tab.linkedBrowser, [params], async params => { + const video = content.document.createElement("video"); + SpecialPowers.wrap(video).setDecodeError(params.error); + await content._waitForReport(params, params.shouldReportNotification); + }); ok(true, `finished check for ${params.error}`); } diff --git a/dom/media/eme/EMEUtils.cpp b/dom/media/eme/EMEUtils.cpp index 294951e7e6..5a6b645df2 100644 --- a/dom/media/eme/EMEUtils.cpp +++ b/dom/media/eme/EMEUtils.cpp @@ -8,13 +8,13 @@ #include "jsfriendapi.h" #include "MediaData.h" +#include "KeySystemConfig.h" #include "mozilla/StaticPrefs_media.h" #include "mozilla/dom/KeySystemNames.h" #include "mozilla/dom/UnionTypes.h" #ifdef MOZ_WMF_CDM # include "mozilla/PMFCDM.h" -# include "KeySystemConfig.h" #endif namespace mozilla { @@ -143,22 +143,35 @@ const char* ToMediaKeyStatusStr(dom::MediaKeyStatus aStatus) { bool IsHardwareDecryptionSupported( const dom::MediaKeySystemConfiguration& aConfig) { - bool supportHardwareDecryption = false; for (const auto& capabilities : aConfig.mAudioCapabilities) { if (capabilities.mRobustness.EqualsLiteral("HW_SECURE_ALL")) { - supportHardwareDecryption = true; - break; + return true; } } for (const auto& capabilities : aConfig.mVideoCapabilities) { if (capabilities.mRobustness.EqualsLiteral("3000") || capabilities.mRobustness.EqualsLiteral("HW_SECURE_ALL") || capabilities.mRobustness.EqualsLiteral("HW_SECURE_DECODE")) { - supportHardwareDecryption = true; - break; + return true; + } + } + return false; +} + +bool IsHardwareDecryptionSupported(const KeySystemConfig& aConfig) { + for (const auto& robustness : aConfig.mAudioRobustness) { + if (robustness.EqualsLiteral("HW_SECURE_ALL")) { + return true; + } + } + for (const auto& robustness : aConfig.mVideoRobustness) { + if (robustness.EqualsLiteral("3000") || + robustness.EqualsLiteral("HW_SECURE_ALL") || + robustness.EqualsLiteral("HW_SECURE_DECODE")) { + return true; } } - return supportHardwareDecryption; + return false; } const char* EncryptionSchemeStr(const CryptoScheme& aScheme) { @@ -245,4 +258,16 @@ bool CheckIfHarewareDRMConfigExists( return foundHWDRMconfig; } +bool DoesKeySystemSupportHardwareDecryption(const nsAString& aKeySystem) { +#ifdef MOZ_WMF_CDM + if (aKeySystem.EqualsLiteral(kPlayReadyKeySystemHardware) || + aKeySystem.EqualsLiteral(kPlayReadyHardwareClearLeadKeySystemName) || + aKeySystem.EqualsLiteral(kWidevineExperimentKeySystemName) || + aKeySystem.EqualsLiteral(kWidevineExperiment2KeySystemName)) { + return true; + } +#endif + return false; +} + } // namespace mozilla diff --git a/dom/media/eme/EMEUtils.h b/dom/media/eme/EMEUtils.h index ca5c684a9f..3fbf22f359 100644 --- a/dom/media/eme/EMEUtils.h +++ b/dom/media/eme/EMEUtils.h @@ -89,6 +89,7 @@ const char* ToMediaKeyStatusStr(dom::MediaKeyStatus aStatus); // Return true if given config supports hardware decryption (SL3000 or L1). bool IsHardwareDecryptionSupported( const dom::MediaKeySystemConfiguration& aConfig); +bool IsHardwareDecryptionSupported(const KeySystemConfig& aConfig); const char* EncryptionSchemeStr(const CryptoScheme& aScheme); @@ -104,6 +105,8 @@ bool DoesKeySystemSupportClearLead(const nsAString& aKeySystem); bool CheckIfHarewareDRMConfigExists( const nsTArray& aConfigs); +bool DoesKeySystemSupportHardwareDecryption(const nsAString& aKeySystem); + } // namespace mozilla #endif // EME_LOG_H_ diff --git a/dom/media/eme/KeySystemConfig.cpp b/dom/media/eme/KeySystemConfig.cpp index b1b1f9ff63..0cb5da1a56 100644 --- a/dom/media/eme/KeySystemConfig.cpp +++ b/dom/media/eme/KeySystemConfig.cpp @@ -57,9 +57,10 @@ bool KeySystemConfig::Supports(const nsAString& aKeySystem) { {nsCString(kWidevineExperimentKeySystemName)}); } - if ((IsPlayReadyKeySystemAndSupported(aKeySystem) || - IsWMFClearKeySystemAndSupported(aKeySystem)) && - WMFCDMImpl::Supports(aKeySystem)) { + // PlayReady and WMF-based ClearKey are always installed, we don't need to + // download them. + if (IsPlayReadyKeySystemAndSupported(aKeySystem) || + IsWMFClearKeySystemAndSupported(aKeySystem)) { return true; } #endif @@ -69,7 +70,8 @@ bool KeySystemConfig::Supports(const nsAString& aKeySystem) { /* static */ bool KeySystemConfig::CreateKeySystemConfigs( - const nsAString& aKeySystem, nsTArray& aOutConfigs) { + const nsAString& aKeySystem, const DecryptionInfo aDecryption, + nsTArray& aOutConfigs) { if (!Supports(aKeySystem)) { return false; } @@ -103,10 +105,16 @@ bool KeySystemConfig::CreateKeySystemConfigs( config->mMP4.SetCanDecrypt(EME_CODEC_FLAC); config->mMP4.SetCanDecrypt(EME_CODEC_OPUS); config->mMP4.SetCanDecrypt(EME_CODEC_VP9); +#ifdef MOZ_AV1 + config->mMP4.SetCanDecrypt(EME_CODEC_AV1); +#endif config->mWebM.SetCanDecrypt(EME_CODEC_VORBIS); config->mWebM.SetCanDecrypt(EME_CODEC_OPUS); config->mWebM.SetCanDecrypt(EME_CODEC_VP8); config->mWebM.SetCanDecrypt(EME_CODEC_VP9); +#ifdef MOZ_AV1 + config->mWebM.SetCanDecrypt(EME_CODEC_AV1); +#endif if (StaticPrefs::media_clearkey_test_key_systems_enabled()) { // Add testing key systems. These offer the same capabilities as the @@ -156,6 +164,10 @@ bool KeySystemConfig::CreateKeySystemConfigs( &config->mMP4}, {nsCString(VIDEO_MP4), EME_CODEC_VP9, java::MediaDrmProxy::AVC, &config->mMP4}, +# ifdef MOZ_AV1 + {nsCString(VIDEO_MP4), EME_CODEC_AV1, java::MediaDrmProxy::AV1, + &config->mMP4}, +# endif {nsCString(AUDIO_MP4), EME_CODEC_AAC, java::MediaDrmProxy::AAC, &config->mMP4}, {nsCString(AUDIO_MP4), EME_CODEC_FLAC, java::MediaDrmProxy::FLAC, @@ -166,6 +178,10 @@ bool KeySystemConfig::CreateKeySystemConfigs( &config->mWebM}, {nsCString(VIDEO_WEBM), EME_CODEC_VP9, java::MediaDrmProxy::VP9, &config->mWebM}, +# ifdef MOZ_AV1 + {nsCString(VIDEO_WEBM), EME_CODEC_AV1, java::MediaDrmProxy::AV1, + &config->mWebM}, +# endif {nsCString(AUDIO_WEBM), EME_CODEC_VORBIS, java::MediaDrmProxy::VORBIS, &config->mWebM}, {nsCString(AUDIO_WEBM), EME_CODEC_OPUS, java::MediaDrmProxy::OPUS, @@ -198,10 +214,16 @@ bool KeySystemConfig::CreateKeySystemConfigs( config->mMP4.SetCanDecrypt(EME_CODEC_OPUS); config->mMP4.SetCanDecryptAndDecode(EME_CODEC_H264); config->mMP4.SetCanDecryptAndDecode(EME_CODEC_VP9); +# ifdef MOZ_AV1 + config->mMP4.SetCanDecryptAndDecode(EME_CODEC_AV1); +# endif config->mWebM.SetCanDecrypt(EME_CODEC_VORBIS); config->mWebM.SetCanDecrypt(EME_CODEC_OPUS); config->mWebM.SetCanDecryptAndDecode(EME_CODEC_VP8); config->mWebM.SetCanDecryptAndDecode(EME_CODEC_VP9); +# ifdef MOZ_AV1 + config->mWebM.SetCanDecryptAndDecode(EME_CODEC_AV1); +# endif #endif return true; } @@ -209,7 +231,8 @@ bool KeySystemConfig::CreateKeySystemConfigs( if (IsPlayReadyKeySystemAndSupported(aKeySystem) || IsWidevineExperimentKeySystemAndSupported(aKeySystem)) { RefPtr cdm = MakeRefPtr(aKeySystem); - return cdm->GetCapabilities(aOutConfigs); + return cdm->GetCapabilities(aDecryption == DecryptionInfo::Hardware, + aOutConfigs); } #endif return false; @@ -243,7 +266,9 @@ void KeySystemConfig::GetGMPKeySystemConfigs(dom::Promise* aPromise) { continue; } #endif - if (KeySystemConfig::CreateKeySystemConfigs(name, keySystemConfigs)) { + if (KeySystemConfig::CreateKeySystemConfigs( + name, KeySystemConfig::DecryptionInfo::Software, + keySystemConfigs)) { auto* info = cdmInfo.AppendElement(fallible); if (!info) { aPromise->MaybeReject(NS_ERROR_OUT_OF_MEMORY); diff --git a/dom/media/eme/KeySystemConfig.h b/dom/media/eme/KeySystemConfig.h index 8bd7d98217..cc35ba76de 100644 --- a/dom/media/eme/KeySystemConfig.h +++ b/dom/media/eme/KeySystemConfig.h @@ -36,6 +36,7 @@ struct KeySystemConfig { static constexpr auto EME_CODEC_VORBIS = "vorbis"_ns; static constexpr auto EME_CODEC_FLAC = "flac"_ns; static constexpr auto EME_CODEC_H264 = "h264"_ns; + static constexpr auto EME_CODEC_AV1 = "av1"_ns; static constexpr auto EME_CODEC_VP8 = "vp8"_ns; static constexpr auto EME_CODEC_VP9 = "vp9"_ns; static constexpr auto EME_CODEC_HEVC = "hevc"_ns; @@ -123,7 +124,13 @@ struct KeySystemConfig { // Return true if given key system is supported on the current device. static bool Supports(const nsAString& aKeySystem); + + enum class DecryptionInfo : uint8_t { + Software, + Hardware, + }; static bool CreateKeySystemConfigs(const nsAString& aKeySystem, + const DecryptionInfo aDecryption, nsTArray& aOutConfigs); static void GetGMPKeySystemConfigs(dom::Promise* aPromise); diff --git a/dom/media/eme/MediaKeySession.cpp b/dom/media/eme/MediaKeySession.cpp index 908df5f7c8..8a3a01dd5c 100644 --- a/dom/media/eme/MediaKeySession.cpp +++ b/dom/media/eme/MediaKeySession.cpp @@ -52,7 +52,7 @@ static const uint32_t MAX_CENC_INIT_DATA_LENGTH = 64 * 1024; MediaKeySession::MediaKeySession(nsPIDOMWindowInner* aParent, MediaKeys* aKeys, const nsAString& aKeySystem, MediaKeySessionType aSessionType, - ErrorResult& aRv) + bool aHardwareDecryption, ErrorResult& aRv) : DOMEventTargetHelper(aParent), mKeys(aKeys), mKeySystem(aKeySystem), @@ -61,7 +61,8 @@ MediaKeySession::MediaKeySession(nsPIDOMWindowInner* aParent, MediaKeys* aKeys, mIsClosed(false), mUninitialized(true), mKeyStatusMap(new MediaKeyStatusMap(aParent)), - mExpiration(JS::GenericNaN()) { + mExpiration(JS::GenericNaN()), + mHardwareDecryption(aHardwareDecryption) { EME_LOG("MediaKeySession[%p,''] ctor", this); MOZ_ASSERT(aParent); @@ -123,9 +124,8 @@ void MediaKeySession::UpdateKeyStatusMap() { nsPrintfCString("MediaKeySession[%p,'%s'] key statuses change {", this, NS_ConvertUTF16toUTF8(mSessionId).get())); for (const CDMCaps::KeyStatus& status : keyStatuses) { - message.AppendPrintf( - " (%s,%s)", ToHexString(status.mId).get(), - nsCString(MediaKeyStatusValues::GetString(status.mStatus)).get()); + message.AppendPrintf(" (%s,%s)", ToHexString(status.mId).get(), + GetEnumString(status.mStatus).get()); } message.AppendLiteral(" }"); // Use %s so we aren't exposing random strings to printf interpolation. @@ -250,8 +250,8 @@ already_AddRefed MediaKeySession::GenerateRequest( // cdm implementation value does not support initDataType as an // Initialization Data Type, return a promise rejected with a // NotSupportedError. String comparison is case-sensitive. - if (!MediaKeySystemAccess::KeySystemSupportsInitDataType(mKeySystem, - aInitDataType)) { + if (!MediaKeySystemAccess::KeySystemSupportsInitDataType( + mKeySystem, aInitDataType, mHardwareDecryption)) { promise->MaybeRejectWithNotSupportedError( "Unsupported initDataType passed to MediaKeySession.generateRequest()"); EME_LOG( @@ -542,8 +542,7 @@ void MediaKeySession::DispatchKeyMessage(MediaKeyMessageType aMessageType, EME_LOG( "MediaKeySession[%p,'%s'] DispatchKeyMessage() type=%s message='%s'", this, NS_ConvertUTF16toUTF8(mSessionId).get(), - nsCString(MediaKeyMessageTypeValues::GetString(aMessageType)).get(), - ToHexString(aMessage).get()); + GetEnumString(aMessageType).get(), ToHexString(aMessage).get()); } RefPtr event( @@ -611,12 +610,8 @@ void MediaKeySession::SetOnmessage(EventHandlerNonNull* aCallback) { SetEventHandler(nsGkAtoms::onmessage, aCallback); } -nsCString ToCString(MediaKeySessionType aType) { - return nsCString(MediaKeySessionTypeValues::GetString(aType)); -} - nsString ToString(MediaKeySessionType aType) { - return NS_ConvertUTF8toUTF16(ToCString(aType)); + return NS_ConvertUTF8toUTF16(GetEnumString(aType)); } } // namespace mozilla::dom diff --git a/dom/media/eme/MediaKeySession.h b/dom/media/eme/MediaKeySession.h index e19488c311..7204f99eef 100644 --- a/dom/media/eme/MediaKeySession.h +++ b/dom/media/eme/MediaKeySession.h @@ -36,8 +36,6 @@ class ArrayBufferViewOrArrayBuffer; class MediaKeyError; class MediaKeyStatusMap; -nsCString ToCString(MediaKeySessionType aType); - nsString ToString(MediaKeySessionType aType); class MediaKeySession final : public DOMEventTargetHelper, @@ -49,7 +47,7 @@ class MediaKeySession final : public DOMEventTargetHelper, public: MediaKeySession(nsPIDOMWindowInner* aParent, MediaKeys* aKeys, const nsAString& aKeySystem, MediaKeySessionType aSessionType, - ErrorResult& aRv); + bool aHardwareDecryption, ErrorResult& aRv); void SetSessionId(const nsAString& aSessionId); @@ -134,6 +132,9 @@ class MediaKeySession final : public DOMEventTargetHelper, bool mUninitialized; RefPtr mKeyStatusMap; double mExpiration; + + // True if this key session is related with hardware decryption. + bool mHardwareDecryption; }; } // namespace dom diff --git a/dom/media/eme/MediaKeySystemAccess.cpp b/dom/media/eme/MediaKeySystemAccess.cpp index b58ff76424..d498c2a773 100644 --- a/dom/media/eme/MediaKeySystemAccess.cpp +++ b/dom/media/eme/MediaKeySystemAccess.cpp @@ -214,6 +214,9 @@ static KeySystemConfig::EMECodecString ToEMEAPICodecString( if (IsH264CodecString(aCodec)) { return KeySystemConfig::EME_CODEC_H264; } + if (IsAV1CodecString(aCodec)) { + return KeySystemConfig::EME_CODEC_AV1; + } if (IsVP8CodecString(aCodec)) { return KeySystemConfig::EME_CODEC_VP8; } @@ -228,31 +231,53 @@ static KeySystemConfig::EMECodecString ToEMEAPICodecString( return ""_ns; } -static nsTArray GetSupportedKeySystems() { +static nsTArray GetSupportedKeySystems( + const nsAString& aKeySystem, bool aIsHardwareDecryption) { + using DecryptionInfo = KeySystemConfig::DecryptionInfo; nsTArray keySystemConfigs; - - const nsTArray keySystemNames{ - NS_ConvertUTF8toUTF16(kClearKeyKeySystemName), - NS_ConvertUTF8toUTF16(kWidevineKeySystemName), + if (IsWidevineKeySystem(aKeySystem) || IsClearkeyKeySystem(aKeySystem)) { + Unused << KeySystemConfig::CreateKeySystemConfigs( + aKeySystem, DecryptionInfo::Software, keySystemConfigs); + } #ifdef MOZ_WMF_CDM - NS_ConvertUTF8toUTF16(kPlayReadyKeySystemName), - NS_ConvertUTF8toUTF16(kPlayReadyKeySystemHardware), - NS_ConvertUTF8toUTF16(kPlayReadyHardwareClearLeadKeySystemName), - NS_ConvertUTF8toUTF16(kWidevineExperimentKeySystemName), - NS_ConvertUTF8toUTF16(kWidevineExperiment2KeySystemName), -#endif - }; - for (const auto& name : keySystemNames) { - Unused << KeySystemConfig::CreateKeySystemConfigs(name, keySystemConfigs); + if (IsPlayReadyKeySystem(aKeySystem)) { + Unused << KeySystemConfig::CreateKeySystemConfigs( + NS_ConvertUTF8toUTF16(kPlayReadyKeySystemName), + DecryptionInfo::Software, keySystemConfigs); + if (aIsHardwareDecryption) { + Unused << KeySystemConfig::CreateKeySystemConfigs( + NS_ConvertUTF8toUTF16(kPlayReadyKeySystemName), + DecryptionInfo::Hardware, keySystemConfigs); + Unused << KeySystemConfig::CreateKeySystemConfigs( + NS_ConvertUTF8toUTF16(kPlayReadyKeySystemHardware), + DecryptionInfo::Hardware, keySystemConfigs); + Unused << KeySystemConfig::CreateKeySystemConfigs( + NS_ConvertUTF8toUTF16(kPlayReadyHardwareClearLeadKeySystemName), + DecryptionInfo::Hardware, keySystemConfigs); + } + } + // If key system is kWidevineKeySystemName but with hardware decryption + // requirement, then we need to check those experiement key systems which are + // used for hardware decryption. + if (IsWidevineExperimentKeySystem(aKeySystem) || + (IsWidevineKeySystem(aKeySystem) && aIsHardwareDecryption)) { + Unused << KeySystemConfig::CreateKeySystemConfigs( + NS_ConvertUTF8toUTF16(kWidevineExperimentKeySystemName), + DecryptionInfo::Hardware, keySystemConfigs); + Unused << KeySystemConfig::CreateKeySystemConfigs( + NS_ConvertUTF8toUTF16(kWidevineExperiment2KeySystemName), + DecryptionInfo::Hardware, keySystemConfigs); } +#endif return keySystemConfigs; } static bool GetKeySystemConfigs( - const nsAString& aKeySystem, + const nsAString& aKeySystem, bool aIsHardwareDecryption, nsTArray& aOutKeySystemConfig) { bool foundConfigs = false; - for (auto& config : GetSupportedKeySystems()) { + for (auto& config : + GetSupportedKeySystems(aKeySystem, aIsHardwareDecryption)) { if (config.IsSameKeySystem(aKeySystem)) { aOutKeySystemConfig.AppendElement(std::move(config)); foundConfigs = true; @@ -263,9 +288,10 @@ static bool GetKeySystemConfigs( /* static */ bool MediaKeySystemAccess::KeySystemSupportsInitDataType( - const nsAString& aKeySystem, const nsAString& aInitDataType) { + const nsAString& aKeySystem, const nsAString& aInitDataType, + bool aIsHardwareDecryption) { nsTArray implementations; - GetKeySystemConfigs(aKeySystem, implementations); + GetKeySystemConfigs(aKeySystem, aIsHardwareDecryption, implementations); bool containInitType = false; for (const auto& config : implementations) { if (config.mInitDataTypes.Contains(aInitDataType)) { @@ -352,15 +378,13 @@ static bool SupportsEncryptionScheme( static bool ToSessionType(const nsAString& aSessionType, MediaKeySessionType& aOutType) { - if (aSessionType.Equals(ToString(MediaKeySessionType::Temporary))) { - aOutType = MediaKeySessionType::Temporary; - return true; - } - if (aSessionType.Equals(ToString(MediaKeySessionType::Persistent_license))) { - aOutType = MediaKeySessionType::Persistent_license; - return true; + Maybe type = + StringToEnum(aSessionType); + if (type.isNothing()) { + return false; } - return false; + aOutType = type.value(); + return true; } // 5.1.1 Is persistent session type? @@ -395,6 +419,7 @@ static CodecType GetCodecType(const KeySystemConfig::EMECodecString& aCodec) { return Audio; } if (aCodec.Equals(KeySystemConfig::EME_CODEC_H264) || + aCodec.Equals(KeySystemConfig::EME_CODEC_AV1) || aCodec.Equals(KeySystemConfig::EME_CODEC_VP8) || aCodec.Equals(KeySystemConfig::EME_CODEC_VP9) || aCodec.Equals(KeySystemConfig::EME_CODEC_HEVC)) { @@ -1040,7 +1065,11 @@ bool MediaKeySystemAccess::GetSupportedConfig( DecoderDoctorDiagnostics* aDiagnostics, bool aIsPrivateBrowsing, const std::function& aDeprecationLogFn) { nsTArray implementations; - if (!GetKeySystemConfigs(aKeySystem, implementations)) { + const bool isHardwareDecryptionRequest = + CheckIfHarewareDRMConfigExists(aConfigs) || + DoesKeySystemSupportHardwareDecryption(aKeySystem); + if (!GetKeySystemConfigs(aKeySystem, isHardwareDecryptionRequest, + implementations)) { return false; } for (const auto& implementation : implementations) { @@ -1082,7 +1111,7 @@ static nsCString ToCString(const nsString& aString) { static nsCString ToCString(const MediaKeysRequirement aValue) { nsCString str("'"); - str.AppendASCII(MediaKeysRequirementValues::GetString(aValue)); + str.AppendASCII(GetEnumString(aValue)); str.AppendLiteral("'"); return str; } diff --git a/dom/media/eme/MediaKeySystemAccess.h b/dom/media/eme/MediaKeySystemAccess.h index 954ff7adf3..18eec47008 100644 --- a/dom/media/eme/MediaKeySystemAccess.h +++ b/dom/media/eme/MediaKeySystemAccess.h @@ -67,7 +67,8 @@ class MediaKeySystemAccess final : public nsISupports, public nsWrapperCache { const std::function& aDeprecationLogFn); static bool KeySystemSupportsInitDataType(const nsAString& aKeySystem, - const nsAString& aInitDataType); + const nsAString& aInitDataType, + bool aIsHardwareDecryption); static nsCString ToCString( const Sequence& aConfig); diff --git a/dom/media/eme/MediaKeySystemAccessManager.cpp b/dom/media/eme/MediaKeySystemAccessManager.cpp index 2bc12d57d7..8ebe7ceee7 100644 --- a/dom/media/eme/MediaKeySystemAccessManager.cpp +++ b/dom/media/eme/MediaKeySystemAccessManager.cpp @@ -412,8 +412,7 @@ void MediaKeySystemAccessManager::RequestMediaKeySystemAccess( "MediaKeySystemAccess::GetKeySystemStatus(%s) " "result=%s msg='%s'", NS_ConvertUTF16toUTF8(aRequest->mKeySystem).get(), - nsCString(MediaKeySystemStatusValues::GetString(status)).get(), - message.get()); + GetEnumString(status).get(), message.get()); LogToBrowserConsole(NS_ConvertUTF8toUTF16(msg)); EME_LOG("%s", msg.get()); diff --git a/dom/media/eme/MediaKeys.cpp b/dom/media/eme/MediaKeys.cpp index c4340885a3..9a3f4bba7a 100644 --- a/dom/media/eme/MediaKeys.cpp +++ b/dom/media/eme/MediaKeys.cpp @@ -23,7 +23,6 @@ #include "mozilla/dom/UnionTypes.h" #include "mozilla/dom/WindowContext.h" #include "mozilla/dom/WindowGlobalChild.h" -#include "nsContentCID.h" #include "nsContentTypeParser.h" #include "nsContentUtils.h" #include "nsIScriptObjectPrincipal.h" @@ -429,7 +428,8 @@ class MediaKeysGMPCrashHelper : public GMPCrashHelper { already_AddRefed MediaKeys::CreateCDMProxy() { const bool isHardwareDecryptionSupported = - IsHardwareDecryptionSupported(mConfig); + IsHardwareDecryptionSupported(mConfig) || + DoesKeySystemSupportHardwareDecryption(mKeySystem); EME_LOG("MediaKeys[%p]::CreateCDMProxy(), isHardwareDecryptionSupported=%d", this, isHardwareDecryptionSupported); RefPtr proxy; @@ -662,8 +662,12 @@ already_AddRefed MediaKeys::CreateSession( EME_LOG("MediaKeys[%p] Creating session", this); - RefPtr session = new MediaKeySession( - GetParentObject(), this, mKeySystem, aSessionType, aRv); + const bool isHardwareDecryption = + IsHardwareDecryptionSupported(mConfig) || + DoesKeySystemSupportHardwareDecryption(mKeySystem); + RefPtr session = + new MediaKeySession(GetParentObject(), this, mKeySystem, aSessionType, + isHardwareDecryption, aRv); if (aRv.Failed()) { return nullptr; @@ -792,8 +796,7 @@ void MediaKeys::GetSessionsInfo(nsString& sessionsInfo) { sessionsInfo.AppendLiteral("(kid="); sessionsInfo.Append(keyID); sessionsInfo.AppendLiteral(" status="); - sessionsInfo.AppendASCII( - MediaKeyStatusValues::GetString(keyStatusMap->GetValueAtIndex(i))); + sessionsInfo.AppendASCII(GetEnumString(keyStatusMap->GetValueAtIndex(i))); sessionsInfo.AppendLiteral(")"); } sessionsInfo.AppendLiteral(")"); @@ -824,7 +827,7 @@ already_AddRefed MediaKeys::GetStatusForPolicy( } EME_LOG("GetStatusForPolicy minHdcpVersion = %s.", - HDCPVersionValues::GetString(aPolicy.mMinHdcpVersion.Value()).data()); + GetEnumString(aPolicy.mMinHdcpVersion.Value()).get()); mProxy->GetStatusForPolicy(StorePromise(promise), aPolicy.mMinHdcpVersion.Value()); return promise.forget(); diff --git a/dom/media/eme/mediadrm/MediaDrmCDMCallbackProxy.cpp b/dom/media/eme/mediadrm/MediaDrmCDMCallbackProxy.cpp index e5431f50fd..f3cfb6c026 100644 --- a/dom/media/eme/mediadrm/MediaDrmCDMCallbackProxy.cpp +++ b/dom/media/eme/mediadrm/MediaDrmCDMCallbackProxy.cpp @@ -9,7 +9,6 @@ #include "nsString.h" #include "mozilla/dom/MediaKeys.h" #include "mozilla/dom/MediaKeySession.h" -#include "nsContentCID.h" #include "nsServiceManagerUtils.h" #include "MainThreadUtils.h" #include "mozilla/EMEUtils.h" diff --git a/dom/media/eme/mediafoundation/WMFCDMImpl.cpp b/dom/media/eme/mediafoundation/WMFCDMImpl.cpp index 1fe42aa8e2..add978f755 100644 --- a/dom/media/eme/mediafoundation/WMFCDMImpl.cpp +++ b/dom/media/eme/mediafoundation/WMFCDMImpl.cpp @@ -15,37 +15,8 @@ namespace mozilla { -/* static */ -bool WMFCDMImpl::Supports(const nsAString& aKeySystem) { - MOZ_ASSERT(NS_IsMainThread()); - if (AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed)) { - return false; - } - - static std::map sSupports; - static bool sSetRunOnShutdown = false; - if (!sSetRunOnShutdown) { - GetMainThreadSerialEventTarget()->Dispatch( - NS_NewRunnableFunction("WMFCDMImpl::Supports", [&] { - RunOnShutdown([&] { sSupports.clear(); }, - ShutdownPhase::XPCOMShutdown); - })); - sSetRunOnShutdown = true; - } - - nsString key(aKeySystem); - if (const auto& s = sSupports.find(key); s != sSupports.end()) { - return s->second; - } - - RefPtr cdm = MakeRefPtr(aKeySystem); - nsTArray configs; - bool s = cdm->GetCapabilities(configs); - sSupports[key] = s; - return s; -} - -bool WMFCDMImpl::GetCapabilities(nsTArray& aOutConfigs) { +bool WMFCDMImpl::GetCapabilities(bool aIsHardwareDecryption, + nsTArray& aOutConfigs) { MOZ_ASSERT(NS_IsMainThread()); if (AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed)) { return false; @@ -67,13 +38,14 @@ bool WMFCDMImpl::GetCapabilities(nsTArray& aOutConfigs) { auto keySystem = std::string{NS_ConvertUTF16toUTF8(mKeySystem).get()}; if (auto rv = sKeySystemConfigs.find(keySystem); rv != sKeySystemConfigs.end()) { - EME_LOG("Return cached capabilities for %s", keySystem.c_str()); for (const auto& config : rv->second) { - aOutConfigs.AppendElement(config); - EME_LOG("-- capabilities (%s)", - NS_ConvertUTF16toUTF8(config.GetDebugInfo()).get()); + if (IsHardwareDecryptionSupported(config) == aIsHardwareDecryption) { + EME_LOG("Return cached capabilities for %s (%s)", keySystem.c_str(), + NS_ConvertUTF16toUTF8(config.GetDebugInfo()).get()); + aOutConfigs.AppendElement(config); + return true; + } } - return true; } // Not cached result, ask the remote process. @@ -83,42 +55,33 @@ bool WMFCDMImpl::GetCapabilities(nsTArray& aOutConfigs) { mCDM = MakeRefPtr(mKeySystem); } bool ok = false; - static const bool sIsHwSecure[2] = {false, true}; - for (const auto& isHWSecure : sIsHwSecure) { - media::Await( - do_AddRef(backgroundTaskQueue), mCDM->GetCapabilities(isHWSecure), - [&ok, &aOutConfigs, keySystem, - isHWSecure](const MFCDMCapabilitiesIPDL& capabilities) { - EME_LOG("capabilities: keySystem=%s (hw-secure=%d)", - keySystem.c_str(), isHWSecure); - for (const auto& v : capabilities.videoCapabilities()) { - EME_LOG("capabilities: video=%s", - NS_ConvertUTF16toUTF8(v.contentType()).get()); - } - for (const auto& a : capabilities.audioCapabilities()) { - EME_LOG("capabilities: audio=%s", - NS_ConvertUTF16toUTF8(a.contentType()).get()); - } - for (const auto& v : capabilities.encryptionSchemes()) { - EME_LOG("capabilities: encryptionScheme=%s", - EncryptionSchemeStr(v)); - } - KeySystemConfig* config = aOutConfigs.AppendElement(); - MFCDMCapabilitiesIPDLToKeySystemConfig(capabilities, *config); - sKeySystemConfigs[keySystem].AppendElement(*config); - // This is equal to "com.microsoft.playready.recommendation.3000", so - // we can store it directly without asking the remote process again. - if (keySystem.compare(kPlayReadyKeySystemName) == 0 && isHWSecure) { - config->mKeySystem.AssignLiteral(kPlayReadyKeySystemHardware); - sKeySystemConfigs["com.microsoft.playready.recommendation.3000"] - .AppendElement(*config); - } - ok = true; - }, - [](nsresult rv) { - EME_LOG("Fail to get key system capabilities. rv=%x", uint32_t(rv)); - }); - } + media::Await( + do_AddRef(backgroundTaskQueue), + mCDM->GetCapabilities(aIsHardwareDecryption), + [&ok, &aOutConfigs, keySystem, + aIsHardwareDecryption](const MFCDMCapabilitiesIPDL& capabilities) { + EME_LOG("capabilities: keySystem=%s (hw-secure=%d)", keySystem.c_str(), + aIsHardwareDecryption); + for (const auto& v : capabilities.videoCapabilities()) { + EME_LOG("capabilities: video=%s", + NS_ConvertUTF16toUTF8(v.contentType()).get()); + } + for (const auto& a : capabilities.audioCapabilities()) { + EME_LOG("capabilities: audio=%s", + NS_ConvertUTF16toUTF8(a.contentType()).get()); + } + for (const auto& v : capabilities.encryptionSchemes()) { + EME_LOG("capabilities: encryptionScheme=%s", EncryptionSchemeStr(v)); + } + KeySystemConfig* config = aOutConfigs.AppendElement(); + MFCDMCapabilitiesIPDLToKeySystemConfig(capabilities, *config); + sKeySystemConfigs[keySystem].AppendElement(*config); + ok = true; + }, + [](nsresult rv) { + EME_LOG("Fail to get key system capabilities. rv=%x", uint32_t(rv)); + }); + return ok; } diff --git a/dom/media/eme/mediafoundation/WMFCDMImpl.h b/dom/media/eme/mediafoundation/WMFCDMImpl.h index 452629ec84..b7e6308848 100644 --- a/dom/media/eme/mediafoundation/WMFCDMImpl.h +++ b/dom/media/eme/mediafoundation/WMFCDMImpl.h @@ -34,9 +34,9 @@ class WMFCDMImpl final { explicit WMFCDMImpl(const nsAString& aKeySystem) : mKeySystem(aKeySystem) {} - static bool Supports(const nsAString& aKeySystem); // TODO: make this async? - bool GetCapabilities(nsTArray& aOutConfigs); + bool GetCapabilities(bool aIsHardwareDecryption, + nsTArray& aOutConfigs); using InitPromise = GenericPromise; struct InitParams { diff --git a/dom/media/eme/mediafoundation/WMFCDMProxy.cpp b/dom/media/eme/mediafoundation/WMFCDMProxy.cpp index 21207ecc22..f7e05dfb6a 100644 --- a/dom/media/eme/mediafoundation/WMFCDMProxy.cpp +++ b/dom/media/eme/mediafoundation/WMFCDMProxy.cpp @@ -381,8 +381,7 @@ void WMFCDMProxy::GetStatusForPolicy(PromiseId aPromiseId, RETURN_IF_SHUTDOWN(); EME_LOG("WMFCDMProxy::GetStatusForPolicy(this=%p, pid=%" PRIu32 ", minHDCP=%s)", - this, aPromiseId, - dom::HDCPVersionValues::GetString(aMinHdcpVersion).data()); + this, aPromiseId, dom::GetEnumString(aMinHdcpVersion).get()); mCDM->GetStatusForPolicy(aPromiseId, aMinHdcpVersion) ->Then( mMainThread, __func__, diff --git a/dom/media/flac/FlacDecoder.cpp b/dom/media/flac/FlacDecoder.cpp index 2f205c9aae..07d9f10983 100644 --- a/dom/media/flac/FlacDecoder.cpp +++ b/dom/media/flac/FlacDecoder.cpp @@ -11,13 +11,7 @@ namespace mozilla { /* static */ -bool FlacDecoder::IsEnabled() { -#ifdef MOZ_FFVPX - return StaticPrefs::media_flac_enabled(); -#else - return false; -#endif -} +bool FlacDecoder::IsEnabled() { return StaticPrefs::media_flac_enabled(); } /* static */ bool FlacDecoder::IsSupportedType(const MediaContainerType& aContainerType) { diff --git a/dom/media/gmp/ChromiumCDMProxy.cpp b/dom/media/gmp/ChromiumCDMProxy.cpp index e8d871a113..566b386b0b 100644 --- a/dom/media/gmp/ChromiumCDMProxy.cpp +++ b/dom/media/gmp/ChromiumCDMProxy.cpp @@ -605,8 +605,7 @@ void ChromiumCDMProxy::GetStatusForPolicy( MOZ_ASSERT(NS_IsMainThread()); EME_LOG("ChromiumCDMProxy::GetStatusForPolicy(this=%p, pid=%" PRIu32 ") minHdcpVersion=%s", - this, aPromiseId, - dom::HDCPVersionValues::GetString(aMinHdcpVersion).data()); + this, aPromiseId, dom::GetEnumString(aMinHdcpVersion).get()); RefPtr cdm = GetCDMParent(); if (!cdm) { diff --git a/dom/media/gmp/GMPChild.cpp b/dom/media/gmp/GMPChild.cpp index 1eea8b108b..d543d46387 100644 --- a/dom/media/gmp/GMPChild.cpp +++ b/dom/media/gmp/GMPChild.cpp @@ -237,16 +237,15 @@ mozilla::ipc::IPCResult GMPChild::RecvPreloadLibs(const nsCString& aLibs) { bool GMPChild::GetUTF8LibPath(nsACString& aOutLibPath) { nsCOMPtr libFile; -#define GMP_PATH_CRASH(explain) \ - do { \ - nsAutoString path; \ - if (!libFile || NS_FAILED(libFile->GetPath(path))) { \ - path = mPluginPath; \ - } \ - CrashReporter::AnnotateCrashReport( \ - CrashReporter::Annotation::GMPLibraryPath, \ - NS_ConvertUTF16toUTF8(path)); \ - MOZ_CRASH(explain); \ +#define GMP_PATH_CRASH(explain) \ + do { \ + nsAutoString path; \ + if (!libFile || NS_FAILED(libFile->GetPath(path))) { \ + path = mPluginPath; \ + } \ + CrashReporter::RecordAnnotationNSString( \ + CrashReporter::Annotation::GMPLibraryPath, path); \ + MOZ_CRASH(explain); \ } while (false) nsresult rv = NS_NewLocalFile(mPluginPath, true, getter_AddRefs(libFile)); @@ -514,7 +513,7 @@ mozilla::ipc::IPCResult GMPChild::RecvStartPlugin(const nsString& aAdapter) { nsAutoCString libPath; if (!GetUTF8LibPath(libPath)) { - CrashReporter::AnnotateCrashReport( + CrashReporter::RecordAnnotationNSCString( CrashReporter::Annotation::GMPLibraryPath, NS_ConvertUTF16toUTF8(mPluginPath)); @@ -556,7 +555,7 @@ mozilla::ipc::IPCResult GMPChild::RecvStartPlugin(const nsString& aAdapter) { NS_WARNING("Failed to load GMP"); #endif delete platformAPI; - CrashReporter::AnnotateCrashReport( + CrashReporter::RecordAnnotationNSCString( CrashReporter::Annotation::GMPLibraryPath, NS_ConvertUTF16toUTF8(mPluginPath)); @@ -727,21 +726,21 @@ mozilla::ipc::IPCResult GMPChild::RecvShutdown(ShutdownResolver&& aResolver) { } const bool isProfiling = profiler_is_active(); - CrashReporter::AnnotateCrashReport( + CrashReporter::RecordAnnotationCString( CrashReporter::Annotation::ProfilerChildShutdownPhase, - isProfiling ? "Profiling - GrabShutdownProfileAndShutdown"_ns - : "Not profiling - GrabShutdownProfileAndShutdown"_ns); + isProfiling ? "Profiling - GrabShutdownProfileAndShutdown" + : "Not profiling - GrabShutdownProfileAndShutdown"); ProfileAndAdditionalInformation shutdownProfileAndAdditionalInformation = mProfilerController->GrabShutdownProfileAndShutdown(); - CrashReporter::AnnotateCrashReport( + CrashReporter::RecordAnnotationCString( CrashReporter::Annotation::ProfilerChildShutdownPhase, - isProfiling ? "Profiling - Destroying ChildProfilerController"_ns - : "Not profiling - Destroying ChildProfilerController"_ns); + isProfiling ? "Profiling - Destroying ChildProfilerController" + : "Not profiling - Destroying ChildProfilerController"); mProfilerController = nullptr; - CrashReporter::AnnotateCrashReport( + CrashReporter::RecordAnnotationCString( CrashReporter::Annotation::ProfilerChildShutdownPhase, - isProfiling ? "Profiling - SendShutdownProfile (resovling)"_ns - : "Not profiling - SendShutdownProfile (resolving)"_ns); + isProfiling ? "Profiling - SendShutdownProfile (resovling)" + : "Not profiling - SendShutdownProfile (resolving)"); if (const size_t len = shutdownProfileAndAdditionalInformation.SizeOf(); len >= size_t(IPC::Channel::kMaximumMessageSize)) { shutdownProfileAndAdditionalInformation.mProfile = @@ -752,10 +751,10 @@ mozilla::ipc::IPCResult GMPChild::RecvShutdown(ShutdownResolver&& aResolver) { // Send the shutdown profile to the parent process through our own // message channel, which we know will survive for long enough. aResolver(shutdownProfileAndAdditionalInformation.mProfile); - CrashReporter::AnnotateCrashReport( + CrashReporter::RecordAnnotationCString( CrashReporter::Annotation::ProfilerChildShutdownPhase, - isProfiling ? "Profiling - SendShutdownProfile (resolved)"_ns - : "Not profiling - SendShutdownProfile (resolved)"_ns); + isProfiling ? "Profiling - SendShutdownProfile (resolved)" + : "Not profiling - SendShutdownProfile (resolved)"); return IPC_OK(); } diff --git a/dom/media/gmp/GMPDiskStorage.cpp b/dom/media/gmp/GMPDiskStorage.cpp index 5f7a023b9b..c1c0c8215a 100644 --- a/dom/media/gmp/GMPDiskStorage.cpp +++ b/dom/media/gmp/GMPDiskStorage.cpp @@ -13,7 +13,6 @@ #include "mozilla/EndianUtils.h" #include "nsClassHashtable.h" #include "prio.h" -#include "nsContentCID.h" #include "nsServiceManagerUtils.h" namespace mozilla::gmp { diff --git a/dom/media/gmp/GMPLoader.cpp b/dom/media/gmp/GMPLoader.cpp index 9fa7756165..c0c9364c7b 100644 --- a/dom/media/gmp/GMPLoader.cpp +++ b/dom/media/gmp/GMPLoader.cpp @@ -84,7 +84,7 @@ class PassThroughGMPAdapter : public GMPAdapter { bool GMPLoader::Load(const char* aUTF8LibPath, uint32_t aUTF8LibPathLen, const GMPPlatformAPI* aPlatformAPI, GMPAdapter* aAdapter) { - CrashReporter::AutoAnnotateCrashReport autoLibPath( + CrashReporter::AutoRecordAnnotation autoLibPath( CrashReporter::Annotation::GMPLibraryPath, nsDependentCString(aUTF8LibPath)); diff --git a/dom/media/gmp/GMPParent.cpp b/dom/media/gmp/GMPParent.cpp index 115e9b7392..0830313be9 100644 --- a/dom/media/gmp/GMPParent.cpp +++ b/dom/media/gmp/GMPParent.cpp @@ -770,13 +770,15 @@ bool GMPParent::EnsureProcessLoaded() { void GMPParent::AddCrashAnnotations() { if (mCrashReporter) { - mCrashReporter->AddAnnotation(CrashReporter::Annotation::GMPPlugin, true); - mCrashReporter->AddAnnotation(CrashReporter::Annotation::PluginFilename, - NS_ConvertUTF16toUTF8(mName)); - mCrashReporter->AddAnnotation(CrashReporter::Annotation::PluginName, - mDisplayName); - mCrashReporter->AddAnnotation(CrashReporter::Annotation::PluginVersion, - mVersion); + mCrashReporter->AddAnnotationBool(CrashReporter::Annotation::GMPPlugin, + true); + mCrashReporter->AddAnnotationNSCString( + CrashReporter::Annotation::PluginFilename, + NS_ConvertUTF16toUTF8(mName)); + mCrashReporter->AddAnnotationNSCString( + CrashReporter::Annotation::PluginName, mDisplayName); + mCrashReporter->AddAnnotationNSCString( + CrashReporter::Annotation::PluginVersion, mVersion); } } diff --git a/dom/media/gmp/moz.build b/dom/media/gmp/moz.build index a2b1be5ed8..3c34021506 100644 --- a/dom/media/gmp/moz.build +++ b/dom/media/gmp/moz.build @@ -126,7 +126,7 @@ PREPROCESSED_IPDL_SOURCES += [ "PGMPContent.ipdl", ] -if CONFIG["OS_TARGET"] in ["WINNT", "Darwin"]: +if CONFIG["TARGET_OS"] in ["WINNT", "OSX"]: DEFINES["SUPPORT_STORAGE_ID"] = 1 include("/ipc/chromium/chromium-config.mozbuild") diff --git a/dom/media/gmp/rlz/moz.build b/dom/media/gmp/rlz/moz.build index 8e2d9ea5d1..a874d54c65 100644 --- a/dom/media/gmp/rlz/moz.build +++ b/dom/media/gmp/rlz/moz.build @@ -9,19 +9,19 @@ FINAL_LIBRARY = 'xul' -if CONFIG['OS_TARGET'] in ['WINNT', 'Darwin']: +if CONFIG['TARGET_OS'] in ['WINNT', 'OSX']: UNIFIED_SOURCES += [ 'lib/crc8.cc', 'lib/machine_id.cc', 'lib/string_utils.cc', ] -if CONFIG['OS_TARGET'] == 'WINNT': +if CONFIG['TARGET_OS'] == 'WINNT': UNIFIED_SOURCES += [ 'win/lib/machine_id_win.cc', ] -if CONFIG['OS_TARGET'] == 'Darwin': +if CONFIG['TARGET_OS'] == 'OSX': UNIFIED_SOURCES += [ 'mac/lib/machine_id_mac.cc', ] diff --git a/dom/media/gtest/TestAudioSampleFormat.cpp b/dom/media/gtest/TestAudioSampleFormat.cpp new file mode 100644 index 0000000000..4737a73a23 --- /dev/null +++ b/dom/media/gtest/TestAudioSampleFormat.cpp @@ -0,0 +1,116 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AudioSampleFormat.h" +#include "gtest/gtest.h" +#include + +using namespace mozilla; + +template +constexpr T LowestSample() { + if constexpr (std::is_integral_v) { + return std::numeric_limits::lowest(); + } else { + return -1.0f; + } +} + +// When converting a sample-type to another sample-type, this returns the +// maximum value possible in the destination format +template +constexpr Dest HighestSample() { + if constexpr (std::is_integral_v) { + return std::numeric_limits::max(); + } else { + return +1.0f; + } +} + +// When converting a sample-type to another sample-type, this returns the +// maximum value expected in the destination format +template +constexpr Dest HighestSampleExpected() { + // When converting small integer samples to large integer sample, the higher + // bound isn't reached because of positive / negative integer assymetry. + if constexpr (std::is_same_v && + std::is_same_v) { + return 32512; // INT16_MAX - 2 << 7 + 1 + } else if constexpr (std::is_same_v && + std::is_same_v) { + return 2130706432; // INT32_MAX - (2 << 23) + 1 + } else if constexpr (std::is_same_v && + std::is_same_v) { + return 2147418112; // INT32_MAX - UINT16_MAX + } + + if constexpr (std::is_integral_v) { + return std::numeric_limits::max(); + } else { + return +1.0f; + } +} + +template +void TestSampleTypePair() { + std::cout << __PRETTY_FUNCTION__ << std::endl; + + ASSERT_EQ(LowestSample(), + ConvertAudioSample(LowestSample())); + Dest expected = HighestSampleExpected(); + ASSERT_EQ(expected, ConvertAudioSample(HighestSample())); + ASSERT_EQ(Bias(), ConvertAudioSample(Bias())); +} + +template +void TestSampleType24bits() { + std::cout << __PRETTY_FUNCTION__ << std::endl; + + int32_t max_sample_24bits = (2 << 22) - 1; + int32_t min_sample_24bits = -(2 << 22); + int32_t silence_24bits = 0; + + ASSERT_EQ(LowestSample(), Int24ToAudioSample(min_sample_24bits)); + ASSERT_EQ(Int24ToAudioSample(min_sample_24bits), LowestSample()); + if constexpr (std::is_same_v) { + // Quantization issue: 2147483392 + (2<<8 - 1) == INT32_MAX + // See comment on HighestSampleExpected above + const int32_t HIGHEST_FROM_24BITS = 2147483392; + ASSERT_EQ(HIGHEST_FROM_24BITS, Int24ToAudioSample(max_sample_24bits)); + ASSERT_EQ(Int24ToAudioSample(max_sample_24bits), HIGHEST_FROM_24BITS); + } else { + ASSERT_EQ(HighestSample(), Int24ToAudioSample(max_sample_24bits)); + ASSERT_EQ(Int24ToAudioSample(max_sample_24bits), HighestSample()); + } + ASSERT_EQ(Bias(), Int24ToAudioSample(silence_24bits)); + ASSERT_EQ(Int24ToAudioSample(silence_24bits), Bias()); +} + +TEST(AudioSampleFormat, Boundaries) +{ + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + TestSampleTypePair(); + + // Separately test 24-bit audio stored in 32-bits integers. + TestSampleType24bits(); + TestSampleType24bits(); + TestSampleType24bits(); + TestSampleType24bits(); +} diff --git a/dom/media/gtest/TestAudioSegment.cpp b/dom/media/gtest/TestAudioSegment.cpp index ee44839283..ea0e43f0cd 100644 --- a/dom/media/gtest/TestAudioSegment.cpp +++ b/dom/media/gtest/TestAudioSegment.cpp @@ -31,7 +31,7 @@ float GetLowValue() { template <> int16_t GetLowValue() { - return -INT16_MAX; + return INT16_MIN; } template <> @@ -62,7 +62,7 @@ const T* const* GetPlanarChannelArray(size_t aChannels, size_t aSize) { for (size_t c = 0; c < aChannels; c++) { channels[c] = new T[aSize]; for (size_t i = 0; i < aSize; i++) { - channels[c][i] = FloatToAudioSample(1. / (c + 1)); + channels[c][i] = ConvertAudioSample(1.f / static_cast(c + 1)); } } return channels; @@ -104,7 +104,7 @@ const T* GetInterleavedChannelArray(size_t aChannels, size_t aSize) { T* samples = new T[sampleCount]; for (size_t i = 0; i < sampleCount; i++) { uint32_t channel = (i % aChannels) + 1; - samples[i] = FloatToAudioSample(1. / channel); + samples[i] = ConvertAudioSample(1.f / static_cast(channel)); } return samples; } @@ -128,8 +128,9 @@ void TestInterleaveAndConvert() { uint32_t channelIndex = 0; for (size_t i = 0; i < arraySize * channels; i++) { - ASSERT_TRUE(FuzzyEqual( - dst[i], FloatToAudioSample(1. / (channelIndex + 1)))); + ASSERT_TRUE( + FuzzyEqual(dst[i], ConvertAudioSample( + 1.f / static_cast(channelIndex + 1)))); channelIndex++; channelIndex %= channels; } @@ -151,8 +152,9 @@ void TestDeinterleaveAndConvert() { for (size_t channel = 0; channel < channels; channel++) { for (size_t i = 0; i < arraySize; i++) { - ASSERT_TRUE(FuzzyEqual(dst[channel][i], - FloatToAudioSample(1. / (channel + 1)))); + ASSERT_TRUE(FuzzyEqual( + dst[channel][i], + ConvertAudioSample(1.f / static_cast(channel + 1)))); } } diff --git a/dom/media/gtest/TestAudioTrackGraph.cpp b/dom/media/gtest/TestAudioTrackGraph.cpp index 457c50e731..1bd255bed1 100644 --- a/dom/media/gtest/TestAudioTrackGraph.cpp +++ b/dom/media/gtest/TestAudioTrackGraph.cpp @@ -1462,7 +1462,7 @@ float rmsf32(AudioDataValue* aSamples, uint32_t aChannels, uint32_t aFrames) { for (uint32_t i = 0; i < aFrames; i++) { downmixed = 0.; for (uint32_t j = 0; j < aChannels; j++) { - downmixed += AudioSampleToFloat(aSamples[readIdx++]); + downmixed += ConvertAudioSample(aSamples[readIdx++]); } rms += downmixed * downmixed; } diff --git a/dom/media/gtest/TestMediaDataDecoder.cpp b/dom/media/gtest/TestMediaDataDecoder.cpp index 79a92842b6..820b15b718 100644 --- a/dom/media/gtest/TestMediaDataDecoder.cpp +++ b/dom/media/gtest/TestMediaDataDecoder.cpp @@ -63,8 +63,7 @@ TEST(MediaDataDecoder, H264) } // Decoding AV1 via. ffvpx is supported on Linux only. -#if defined(MOZ_AV1) && defined(MOZ_WIDGET_GTK) && defined(MOZ_FFVPX) && \ - !defined(MOZ_FFVPX_AUDIOONLY) +#if defined(MOZ_AV1) && defined(MOZ_WIDGET_GTK) && !defined(MOZ_FFVPX_AUDIOONLY) TEST(MediaDataDecoder, AV1) { if (!MP4Decoder::IsSupportedType(MediaContainerType(MEDIAMIMETYPE(VIDEO_MP4)), diff --git a/dom/media/gtest/TestMediaDataEncoder.cpp b/dom/media/gtest/TestMediaDataEncoder.cpp index bdab94cfe5..27a6b7cd07 100644 --- a/dom/media/gtest/TestMediaDataEncoder.cpp +++ b/dom/media/gtest/TestMediaDataEncoder.cpp @@ -382,8 +382,7 @@ TEST_F(MediaDataEncoderTest, AndroidNotSupportedSize) { } #endif -#if defined(XP_LINUX) && !defined(ANDROID) && \ - (defined(MOZ_FFMPEG) || defined(MOZ_FFVPX)) +#if defined(XP_LINUX) && !defined(ANDROID) TEST_F(MediaDataEncoderTest, H264AVCC) { RUN_IF_SUPPORTED(CodecType::H264, [this]() { // Encod frames in avcC format. @@ -508,8 +507,7 @@ TEST_F(MediaDataEncoderTest, VP8Duration) { }); } -#if defined(XP_LINUX) && !defined(ANDROID) && \ - (defined(MOZ_FFMPEG) || defined(MOZ_FFVPX)) +#if defined(XP_LINUX) && !defined(ANDROID) TEST_F(MediaDataEncoderTest, VP8EncodeAfterDrain) { RUN_IF_SUPPORTED(CodecType::VP8, [this]() { RefPtr e = CreateVP8Encoder(); @@ -673,8 +671,7 @@ TEST_F(MediaDataEncoderTest, VP9Duration) { }); } -#if defined(XP_LINUX) && !defined(ANDROID) && \ - (defined(MOZ_FFMPEG) || defined(MOZ_FFVPX)) +#if defined(XP_LINUX) && !defined(ANDROID) TEST_F(MediaDataEncoderTest, VP9EncodeAfterDrain) { RUN_IF_SUPPORTED(CodecType::VP9, [this]() { RefPtr e = CreateVP9Encoder(); diff --git a/dom/media/gtest/TestMediaQueue.cpp b/dom/media/gtest/TestMediaQueue.cpp index 5b049dc7fe..7176de069f 100644 --- a/dom/media/gtest/TestMediaQueue.cpp +++ b/dom/media/gtest/TestMediaQueue.cpp @@ -6,6 +6,7 @@ #include "MediaData.h" #include "MediaQueue.h" +#include "nsISupportsImpl.h" using namespace mozilla; using mozilla::media::TimeUnit; @@ -285,4 +286,19 @@ TEST(MediaQueue, TimestampAdjustmentForNotSupportDataType) EXPECT_EQ(data->GetEndTime(), TimeUnit::FromMicroseconds(10)); } +TEST(MediaQueue, PreciseDuration) +{ + MediaQueue queueOff; + queueOff.Push(CreateDataRawPtr(5, 10)); + queueOff.Push(CreateDataRawPtr(0, 5)); + EXPECT_EQ(queueOff.Duration(), 0); + EXPECT_EQ(queueOff.PreciseDuration(), -1); + + MediaQueue queueOn(true /* aEnablePreciseDuration */); + queueOn.Push(CreateDataRawPtr(5, 10)); + queueOn.Push(CreateDataRawPtr(0, 5)); + EXPECT_EQ(queueOn.Duration(), 0); + EXPECT_EQ(queueOn.PreciseDuration(), 10); +} + #undef EXPECT_EQUAL_SIZE_T diff --git a/dom/media/gtest/TestMediaUtils.cpp b/dom/media/gtest/TestMediaUtils.cpp index 33a32b7ea0..3708f43f01 100644 --- a/dom/media/gtest/TestMediaUtils.cpp +++ b/dom/media/gtest/TestMediaUtils.cpp @@ -15,7 +15,7 @@ using namespace mozilla::gtest; using namespace mozilla::media; // Spawning the death test child process aborts on Android. -#if !defined(ANDROID) +#if !defined(ANDROID) && defined(GTEST_HAS_DEATH_TEST) // Kept here for reference as it can be handy during development. # define DISABLE_CRASH_REPORTING \ diff --git a/dom/media/gtest/moz.build b/dom/media/gtest/moz.build index 581be004ef..df67aeef18 100644 --- a/dom/media/gtest/moz.build +++ b/dom/media/gtest/moz.build @@ -31,6 +31,7 @@ UNIFIED_SOURCES += [ "TestAudioMixer.cpp", "TestAudioPacketizer.cpp", "TestAudioRingBuffer.cpp", + "TestAudioSampleFormat.cpp", "TestAudioSegment.cpp", "TestAudioSinkWrapper.cpp", "TestAudioTrackEncoder.cpp", diff --git a/dom/media/ipc/MFCDMSerializers.h b/dom/media/ipc/MFCDMSerializers.h index 587f30ae09..64867f4dbb 100644 --- a/dom/media/ipc/MFCDMSerializers.h +++ b/dom/media/ipc/MFCDMSerializers.h @@ -5,9 +5,9 @@ #ifndef DOM_MEDIA_IPC_MFCDMSERIALIZERS_H_ #define DOM_MEDIA_IPC_MFCDMSERIALIZERS_H_ -#include "ipc/EnumSerializer.h" #include "MediaData.h" #include "mozilla/KeySystemConfig.h" +#include "mozilla/dom/BindingIPCUtils.h" #include "mozilla/dom/MediaKeyMessageEventBinding.h" #include "mozilla/dom/MediaKeyStatusMapBinding.h" @@ -35,23 +35,17 @@ struct ParamTraits template <> struct ParamTraits - : public ContiguousEnumSerializer< - mozilla::dom::MediaKeyMessageType, - mozilla::dom::MediaKeyMessageType::License_request, - mozilla::dom::MediaKeyMessageType::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer< + mozilla::dom::MediaKeyMessageType> {}; template <> struct ParamTraits - : public ContiguousEnumSerializer { + : public mozilla::dom::WebIDLEnumSerializer { }; template <> struct ParamTraits - : public ContiguousEnumSerializer {}; + : public mozilla::dom::WebIDLEnumSerializer {}; } // namespace IPC diff --git a/dom/media/ipc/MFMediaEngineParent.cpp b/dom/media/ipc/MFMediaEngineParent.cpp index 0238956781..5ed1b71160 100644 --- a/dom/media/ipc/MFMediaEngineParent.cpp +++ b/dom/media/ipc/MFMediaEngineParent.cpp @@ -113,7 +113,6 @@ void MFMediaEngineParent::DestroyEngineIfExists( mRequestSampleListener.DisconnectIfExists(); if (mDXGIDeviceManager) { mDXGIDeviceManager = nullptr; - wmf::MFUnlockDXGIDeviceManager(); } if (aError) { Unused << SendNotifyError(*aError); @@ -191,6 +190,9 @@ void MFMediaEngineParent::InitializeDXGIDeviceManager() { UINT deviceResetToken; RETURN_VOID_IF_FAILED( wmf::MFLockDXGIDeviceManager(&deviceResetToken, &mDXGIDeviceManager)); + if (!mDXGIDeviceManager) { + return; + } RETURN_VOID_IF_FAILED( mDXGIDeviceManager->ResetDevice(d3d11Device.get(), deviceResetToken)); LOG("Initialized DXGI manager"); diff --git a/dom/media/ipc/RDDProcessHost.cpp b/dom/media/ipc/RDDProcessHost.cpp index 0c0d35e7b5..d855b7804b 100644 --- a/dom/media/ipc/RDDProcessHost.cpp +++ b/dom/media/ipc/RDDProcessHost.cpp @@ -53,10 +53,6 @@ bool RDDProcessHost::Launch(StringVector aExtraOpts) { } mPrefSerializer->AddSharedPrefCmdLineArgs(*this, aExtraOpts); -#if defined(XP_WIN) && defined(MOZ_SANDBOX) - mSandboxLevel = Preferences::GetInt("security.sandbox.rdd.level"); -#endif - mLaunchPhase = LaunchPhase::Waiting; mLaunchTime = TimeStamp::Now(); diff --git a/dom/media/ipc/RDDProcessManager.cpp b/dom/media/ipc/RDDProcessManager.cpp index e7da3c3569..cb51b68282 100644 --- a/dom/media/ipc/RDDProcessManager.cpp +++ b/dom/media/ipc/RDDProcessManager.cpp @@ -170,8 +170,8 @@ RefPtr RDDProcessManager::LaunchRDDProcess() { } mQueuedPrefs.Clear(); - CrashReporter::AnnotateCrashReport( - CrashReporter::Annotation::RDDProcessStatus, "Running"_ns); + CrashReporter::RecordAnnotationCString( + CrashReporter::Annotation::RDDProcessStatus, "Running"); if (!CreateVideoBridge()) { mNumProcessAttempts++; @@ -271,8 +271,8 @@ void RDDProcessManager::DestroyProcess() { mRDDChild = nullptr; mQueuedPrefs.Clear(); - CrashReporter::AnnotateCrashReport( - CrashReporter::Annotation::RDDProcessStatus, "Destroyed"_ns); + CrashReporter::RecordAnnotationCString( + CrashReporter::Annotation::RDDProcessStatus, "Destroyed"); } bool RDDProcessManager::CreateContentBridge( diff --git a/dom/media/ipc/RemoteDecoderManagerChild.cpp b/dom/media/ipc/RemoteDecoderManagerChild.cpp index 48da254f39..b0b075e4d2 100644 --- a/dom/media/ipc/RemoteDecoderManagerChild.cpp +++ b/dom/media/ipc/RemoteDecoderManagerChild.cpp @@ -50,8 +50,8 @@ using namespace gfx; // Used so that we only ever attempt to check if the RDD/GPU/Utility processes // should be launched serially. Protects sLaunchPromise StaticMutex sLaunchMutex; -static EnumeratedArray> +static EnumeratedArray, + size_t(RemoteDecodeIn::SENTINEL)> sLaunchPromises MOZ_GUARDED_BY(sLaunchMutex); // Only modified on the main-thread, read on any thread. While it could be read @@ -61,8 +61,8 @@ static StaticDataMutex> sRemoteDecoderManagerChildThread("sRemoteDecoderManagerChildThread"); // Only accessed from sRemoteDecoderManagerChildThread -static EnumeratedArray> +static EnumeratedArray, + size_t(RemoteDecodeIn::SENTINEL)> sRemoteDecoderManagerChildForProcesses; static StaticAutoPtr>> sRecreateTasks; @@ -70,8 +70,8 @@ static StaticAutoPtr>> sRecreateTasks; // Used for protecting codec support information collected from different remote // processes. StaticMutex sProcessSupportedMutex; -static EnumeratedArray> +static EnumeratedArray, + size_t(RemoteDecodeIn::SENTINEL)> sProcessSupported MOZ_GUARDED_BY(sProcessSupportedMutex); class ShutdownObserver final : public nsIObserver { @@ -311,6 +311,16 @@ RemoteDecoderManagerChild::CreateAudioDecoder( __func__); } + if (!aParams.mMediaEngineId && + aLocation == RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM) { + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, + nsPrintfCString("%s only support for media engine playback", + RemoteDecodeInToStr(aLocation)) + .get()), + __func__); + } + RefPtr launchPromise; if (StaticPrefs::media_utility_process_enabled() && (aLocation == RemoteDecodeIn::UtilityProcess_Generic || @@ -385,6 +395,16 @@ RemoteDecoderManagerChild::CreateVideoDecoder( __func__); } + if (!aParams.mMediaEngineId && + aLocation == RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM) { + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, + nsPrintfCString("%s only support for media engine playback", + RemoteDecodeInToStr(aLocation)) + .get()), + __func__); + } + MOZ_ASSERT(aLocation != RemoteDecodeIn::Unspecified); RefPtr p; diff --git a/dom/media/mediacapabilities/MediaCapabilities.cpp b/dom/media/mediacapabilities/MediaCapabilities.cpp index abc83dae4d..d3123f913b 100644 --- a/dom/media/mediacapabilities/MediaCapabilities.cpp +++ b/dom/media/mediacapabilities/MediaCapabilities.cpp @@ -45,21 +45,6 @@ static nsCString VideoConfigurationToStr(const VideoConfiguration* aConfig) { return nsCString(); } - nsCString hdrMetaType( - aConfig->mHdrMetadataType.WasPassed() - ? HdrMetadataTypeValues::GetString(aConfig->mHdrMetadataType.Value()) - : "?"); - - nsCString colorGamut( - aConfig->mColorGamut.WasPassed() - ? ColorGamutValues::GetString(aConfig->mColorGamut.Value()) - : "?"); - - nsCString transferFunction(aConfig->mTransferFunction.WasPassed() - ? TransferFunctionValues::GetString( - aConfig->mTransferFunction.Value()) - : "?"); - auto str = nsPrintfCString( "[contentType:%s width:%d height:%d bitrate:%" PRIu64 " framerate:%lf hasAlphaChannel:%s hdrMetadataType:%s colorGamut:%s " @@ -69,7 +54,15 @@ static nsCString VideoConfigurationToStr(const VideoConfiguration* aConfig) { aConfig->mHasAlphaChannel.WasPassed() ? aConfig->mHasAlphaChannel.Value() ? "true" : "false" : "?", - hdrMetaType.get(), colorGamut.get(), transferFunction.get(), + aConfig->mHdrMetadataType.WasPassed() + ? GetEnumString(aConfig->mHdrMetadataType.Value()).get() + : "?", + aConfig->mColorGamut.WasPassed() + ? GetEnumString(aConfig->mColorGamut.Value()).get() + : "?", + aConfig->mTransferFunction.WasPassed() + ? GetEnumString(aConfig->mTransferFunction.Value()).get() + : "?", aConfig->mScalabilityMode.WasPassed() ? NS_ConvertUTF16toUTF8(aConfig->mScalabilityMode.Value()).get() : "?"); diff --git a/dom/media/mediacontrol/ContentMediaController.cpp b/dom/media/mediacontrol/ContentMediaController.cpp index 1171785fe4..c0b466ff0f 100644 --- a/dom/media/mediacontrol/ContentMediaController.cpp +++ b/dom/media/mediacontrol/ContentMediaController.cpp @@ -286,8 +286,8 @@ void ContentMediaAgent::NotifyMediaFullScreenState(uint64_t aBrowsingContextId, } } -void ContentMediaAgent::UpdatePositionState(uint64_t aBrowsingContextId, - const PositionState& aState) { +void ContentMediaAgent::UpdatePositionState( + uint64_t aBrowsingContextId, const Maybe& aState) { RefPtr bc = GetBrowsingContextForAgent(aBrowsingContextId); if (!bc || bc->IsDiscarded()) { return; diff --git a/dom/media/mediacontrol/ContentMediaController.h b/dom/media/mediacontrol/ContentMediaController.h index 9e162dbb27..a58be24b9d 100644 --- a/dom/media/mediacontrol/ContentMediaController.h +++ b/dom/media/mediacontrol/ContentMediaController.h @@ -66,7 +66,7 @@ class ContentMediaAgent : public IMediaInfoUpdater { void NotifyMediaFullScreenState(uint64_t aBrowsingContextId, bool aIsInFullScreen) override; void UpdatePositionState(uint64_t aBrowsingContextId, - const PositionState& aState) override; + const Maybe& aState) override; // Use these methods to register/unregister `ContentMediaControlKeyReceiver` // in order to listen to media control key events. diff --git a/dom/media/mediacontrol/ContentPlaybackController.cpp b/dom/media/mediacontrol/ContentPlaybackController.cpp index ba48c0a5ce..fcc8e3ab58 100644 --- a/dom/media/mediacontrol/ContentPlaybackController.cpp +++ b/dom/media/mediacontrol/ContentPlaybackController.cpp @@ -168,8 +168,12 @@ void ContentMediaControlKeyHandler::HandleMediaControlAction( if (!aContext->GetDocShell()) { return; } + if (aAction.mKey.isNothing()) { + MOZ_ASSERT_UNREACHABLE("Invalid media control key."); + return; + } ContentPlaybackController controller(aContext); - switch (aAction.mKey) { + switch (aAction.mKey.value()) { case MediaControlKey::Focus: controller.Focus(); return; @@ -179,6 +183,9 @@ void ContentMediaControlKeyHandler::HandleMediaControlAction( case MediaControlKey::Pause: controller.Pause(); return; + case MediaControlKey::Playpause: + MOZ_ASSERT_UNREACHABLE("Invalid media control key."); + return; case MediaControlKey::Stop: controller.Stop(); return; diff --git a/dom/media/mediacontrol/MediaControlKeyManager.cpp b/dom/media/mediacontrol/MediaControlKeyManager.cpp index 4cb562aa84..ba6ed3a524 100644 --- a/dom/media/mediacontrol/MediaControlKeyManager.cpp +++ b/dom/media/mediacontrol/MediaControlKeyManager.cpp @@ -184,10 +184,16 @@ void MediaControlKeyManager::SetEnablePictureInPictureMode(bool aIsEnabled) { } } -void MediaControlKeyManager::SetPositionState(const PositionState& aState) { - LOG_INFO("Set PositionState, duration=%f, playbackRate=%f, position=%f", - aState.mDuration, aState.mPlaybackRate, - aState.mLastReportedPlaybackPosition); +void MediaControlKeyManager::SetPositionState( + const Maybe& aState) { + if (aState) { + LOG_INFO("Set PositionState, duration=%f, playbackRate=%f, position=%f", + aState->mDuration, aState->mPlaybackRate, + aState->mLastReportedPlaybackPosition); + } else { + LOG_INFO("Set PositionState, Nothing"); + } + if (mEventSource && mEventSource->IsOpened()) { mEventSource->SetPositionState(aState); } diff --git a/dom/media/mediacontrol/MediaControlKeyManager.h b/dom/media/mediacontrol/MediaControlKeyManager.h index feb857e335..9b029f3bc9 100644 --- a/dom/media/mediacontrol/MediaControlKeyManager.h +++ b/dom/media/mediacontrol/MediaControlKeyManager.h @@ -41,7 +41,7 @@ class MediaControlKeyManager final : public MediaControlKeySource, void SetSupportedMediaKeys(const MediaKeysArray& aSupportedKeys) override; void SetEnableFullScreen(bool aIsEnabled) override; void SetEnablePictureInPictureMode(bool aIsEnabled) override; - void SetPositionState(const PositionState& aState) override; + void SetPositionState(const Maybe& aState) override; private: ~MediaControlKeyManager(); diff --git a/dom/media/mediacontrol/MediaControlKeySource.cpp b/dom/media/mediacontrol/MediaControlKeySource.cpp index 22756c860a..db372fc57b 100644 --- a/dom/media/mediacontrol/MediaControlKeySource.cpp +++ b/dom/media/mediacontrol/MediaControlKeySource.cpp @@ -36,7 +36,11 @@ void MediaControlKeyHandler::OnActionPerformed( return; } - switch (aAction.mKey) { + if (aAction.mKey.isNothing()) { + MOZ_ASSERT_UNREACHABLE("Error : undefined media key!"); + return; + } + switch (aAction.mKey.value()) { case MediaControlKey::Focus: controller->Focus(); return; diff --git a/dom/media/mediacontrol/MediaControlKeySource.h b/dom/media/mediacontrol/MediaControlKeySource.h index f5d62a429e..4ab2c8b8be 100644 --- a/dom/media/mediacontrol/MediaControlKeySource.h +++ b/dom/media/mediacontrol/MediaControlKeySource.h @@ -28,10 +28,10 @@ struct SeekDetails { struct MediaControlAction { MediaControlAction() = default; - explicit MediaControlAction(MediaControlKey aKey) : mKey(aKey) {} + explicit MediaControlAction(MediaControlKey aKey) : mKey(Some(aKey)) {} MediaControlAction(MediaControlKey aKey, const SeekDetails& aDetails) - : mKey(aKey), mDetails(Some(aDetails)) {} - MediaControlKey mKey = MediaControlKey::EndGuard_; + : mKey(Some(aKey)), mDetails(Some(aDetails)) {} + Maybe mKey; Maybe mDetails; }; @@ -109,7 +109,7 @@ class MediaControlKeySource { // to notify change to the embedded application. virtual void SetEnableFullScreen(bool aIsEnabled){}; virtual void SetEnablePictureInPictureMode(bool aIsEnabled){}; - virtual void SetPositionState(const PositionState& aState){}; + virtual void SetPositionState(const Maybe& aState){}; protected: virtual ~MediaControlKeySource() = default; diff --git a/dom/media/mediacontrol/MediaControlService.cpp b/dom/media/mediacontrol/MediaControlService.cpp index c321e080d2..f45ab4253d 100644 --- a/dom/media/mediacontrol/MediaControlService.cpp +++ b/dom/media/mediacontrol/MediaControlService.cpp @@ -157,6 +157,10 @@ void MediaControlService::NotifyMediaControlHasEverBeenUsed() { Telemetry::ScalarSet(Telemetry::ScalarID::MEDIA_CONTROL_PLATFORM_USAGE, u"Android"_ns, usedOnMediaControl); #endif +#ifdef MOZ_WIDGET_UIKIT + Telemetry::ScalarSet(Telemetry::ScalarID::MEDIA_CONTROL_PLATFORM_USAGE, + u"iOS"_ns, usedOnMediaControl); +#endif } void MediaControlService::NotifyMediaControlHasEverBeenEnabled() { @@ -182,6 +186,10 @@ void MediaControlService::NotifyMediaControlHasEverBeenEnabled() { Telemetry::ScalarSet(Telemetry::ScalarID::MEDIA_CONTROL_PLATFORM_USAGE, u"Android"_ns, enableOnMediaControl); #endif +#ifdef MOZ_WIDGET_UIKIT + Telemetry::ScalarSet(Telemetry::ScalarID::MEDIA_CONTROL_PLATFORM_USAGE, + u"iOS"_ns, enableOnMediaControl); +#endif } NS_IMETHODIMP @@ -510,7 +518,7 @@ void MediaControlService::ControllerManager::ConnectMainControllerEvents() { mSource->SetEnablePictureInPictureMode(aIsEnabled); }); mPositionChangedListener = mMainController->PositionChangedEvent().Connect( - AbstractThread::MainThread(), [this](const PositionState& aState) { + AbstractThread::MainThread(), [this](const Maybe& aState) { mSource->SetPositionState(aState); }); } diff --git a/dom/media/mediacontrol/MediaControlUtils.h b/dom/media/mediacontrol/MediaControlUtils.h index a327c2f3d8..e4e75e7c97 100644 --- a/dom/media/mediacontrol/MediaControlUtils.h +++ b/dom/media/mediacontrol/MediaControlUtils.h @@ -50,6 +50,14 @@ inline const char* ToMediaControlKeyStr(MediaControlKey aKey) { } } +inline const char* ToMediaControlKeyStr(const Maybe& aKey) { + if (aKey.isNothing()) { + MOZ_ASSERT_UNREACHABLE("Invalid action."); + return "Unknown"; + } + return ToMediaControlKeyStr(aKey.value()); +} + inline const char* ToMediaSessionActionStr(MediaSessionAction aAction) { switch (aAction) { case MediaSessionAction::Play: @@ -99,11 +107,6 @@ inline MediaControlKey ConvertMediaSessionActionToControlKey( } } -inline MediaSessionAction ConvertToMediaSessionAction(uint8_t aActionValue) { - MOZ_DIAGNOSTIC_ASSERT(aActionValue < uint8_t(MediaSessionAction::EndGuard_)); - return static_cast(aActionValue); -} - inline const char* ToMediaPlaybackStateStr(MediaPlaybackState aState) { switch (aState) { case MediaPlaybackState::eStarted: diff --git a/dom/media/mediacontrol/MediaController.cpp b/dom/media/mediacontrol/MediaController.cpp index bfb98f24c9..4290e952b0 100644 --- a/dom/media/mediacontrol/MediaController.cpp +++ b/dom/media/mediacontrol/MediaController.cpp @@ -186,9 +186,10 @@ bool MediaController::ShouldPropagateActionToAllContexts( // These three actions have default action handler for each frame, so we // need to propagate to all contexts. We would handle default handlers in // `ContentMediaController::HandleMediaKey`. - return aAction.mKey == MediaControlKey::Play || - aAction.mKey == MediaControlKey::Pause || - aAction.mKey == MediaControlKey::Stop; + return aAction.mKey.isSome() && + (aAction.mKey.value() == MediaControlKey::Play || + aAction.mKey.value() == MediaControlKey::Pause || + aAction.mKey.value() == MediaControlKey::Stop); } void MediaController::UpdateMediaControlActionToContentMediaIfNeeded( @@ -493,11 +494,16 @@ void MediaController::HandleSupportedMediaSessionActionsChanged( MediaController_Binding::ClearCachedSupportedKeysValue(this); } -void MediaController::HandlePositionStateChanged(const PositionState& aState) { +void MediaController::HandlePositionStateChanged( + const Maybe& aState) { + if (!aState) { + return; + } + PositionStateEventInit init; - init.mDuration = aState.mDuration; - init.mPlaybackRate = aState.mPlaybackRate; - init.mPosition = aState.mLastReportedPlaybackPosition; + init.mDuration = aState->mDuration; + init.mPlaybackRate = aState->mPlaybackRate; + init.mPosition = aState->mLastReportedPlaybackPosition; RefPtr event = PositionStateEvent::Constructor(this, u"positionstatechange"_ns, init); DispatchAsyncEvent(event.forget()); diff --git a/dom/media/mediacontrol/MediaController.h b/dom/media/mediacontrol/MediaController.h index 82b351ead7..f95ff6368e 100644 --- a/dom/media/mediacontrol/MediaController.h +++ b/dom/media/mediacontrol/MediaController.h @@ -159,7 +159,7 @@ class MediaController final : public DOMEventTargetHelper, void HandleSupportedMediaSessionActionsChanged( const nsTArray& aSupportedAction); - void HandlePositionStateChanged(const PositionState& aState); + void HandlePositionStateChanged(const Maybe& aState); void HandleMetadataChanged(const MediaMetadataBase& aMetadata); // This would register controller to the media control service that takes a diff --git a/dom/media/mediacontrol/MediaStatusManager.cpp b/dom/media/mediacontrol/MediaStatusManager.cpp index 4365e6b531..9187e56f25 100644 --- a/dom/media/mediacontrol/MediaStatusManager.cpp +++ b/dom/media/mediacontrol/MediaStatusManager.cpp @@ -154,6 +154,7 @@ void MediaStatusManager::SetActiveMediaSessionContextId( *mActiveMediaSessionContextId); mMetadataChangedEvent.Notify(GetCurrentMediaMetadata()); mSupportedActionsChangedEvent.Notify(GetSupportedActions()); + mPositionStateChangedEvent.Notify(GetCurrentPositionState()); if (StaticPrefs::media_mediacontrol_testingevents_enabled()) { if (nsCOMPtr obs = services::GetObserverService()) { obs->NotifyObservers(nullptr, "active-media-session-changed", nullptr); @@ -170,6 +171,7 @@ void MediaStatusManager::ClearActiveMediaSessionContextIdIfNeeded() { StoreMediaSessionContextIdOnWindowContext(); mMetadataChangedEvent.Notify(GetCurrentMediaMetadata()); mSupportedActionsChangedEvent.Notify(GetSupportedActions()); + mPositionStateChangedEvent.Notify(GetCurrentPositionState()); if (StaticPrefs::media_mediacontrol_testingevents_enabled()) { if (nsCOMPtr obs = services::GetObserverService()) { obs->NotifyObservers(nullptr, "active-media-session-changed", nullptr); @@ -362,8 +364,14 @@ void MediaStatusManager::DisableAction(uint64_t aBrowsingContextId, NotifySupportedKeysChangedIfNeeded(aBrowsingContextId); } -void MediaStatusManager::UpdatePositionState(uint64_t aBrowsingContextId, - const PositionState& aState) { +void MediaStatusManager::UpdatePositionState( + uint64_t aBrowsingContextId, const Maybe& aState) { + auto info = mMediaSessionInfoMap.Lookup(aBrowsingContextId); + if (info) { + LOG("Update position state for context %" PRIu64, aBrowsingContextId); + info->mPositionState = aState; + } + // The position state comes from non-active media session which we don't care. if (!mActiveMediaSessionContextId || *mActiveMediaSessionContextId != aBrowsingContextId) { @@ -393,9 +401,8 @@ CopyableTArray MediaStatusManager::GetSupportedActions() MediaSessionInfo info = mMediaSessionInfoMap.Get(*mActiveMediaSessionContextId); - const uint8_t actionNums = uint8_t(MediaSessionAction::EndGuard_); - for (uint8_t actionValue = 0; actionValue < actionNums; actionValue++) { - MediaSessionAction action = ConvertToMediaSessionAction(actionValue); + for (MediaSessionAction action : + MakeWebIDLEnumeratedRange()) { if (info.IsActionSupported(action)) { supportedActions.AppendElement(action); } @@ -421,6 +428,16 @@ MediaMetadataBase MediaStatusManager::GetCurrentMediaMetadata() const { return CreateDefaultMetadata(); } +Maybe MediaStatusManager::GetCurrentPositionState() const { + if (mActiveMediaSessionContextId) { + auto info = mMediaSessionInfoMap.Lookup(*mActiveMediaSessionContextId); + if (info) { + return info->mPositionState; + } + } + return Nothing(); +} + void MediaStatusManager::FillMissingTitleAndArtworkIfNeeded( MediaMetadataBase& aMetadata) const { // If the metadata doesn't set its title and artwork properly, we would like diff --git a/dom/media/mediacontrol/MediaStatusManager.h b/dom/media/mediacontrol/MediaStatusManager.h index 24247d119d..a4216c8453 100644 --- a/dom/media/mediacontrol/MediaStatusManager.h +++ b/dom/media/mediacontrol/MediaStatusManager.h @@ -53,6 +53,7 @@ class MediaSessionInfo { Maybe mMetadata; MediaSessionPlaybackState mDeclaredPlaybackState = MediaSessionPlaybackState::None; + Maybe mPositionState; // Use bitwise to store the supported actions. uint32_t mSupportedActions = 0; }; @@ -118,7 +119,7 @@ class IMediaInfoUpdater { // Use this method when media session update its position state. virtual void UpdatePositionState(uint64_t aBrowsingContextId, - const PositionState& aState) = 0; + const Maybe& aState) = 0; }; /** @@ -163,7 +164,7 @@ class MediaStatusManager : public IMediaInfoUpdater { void DisableAction(uint64_t aBrowsingContextId, MediaSessionAction aAction) override; void UpdatePositionState(uint64_t aBrowsingContextId, - const PositionState& aState) override; + const Maybe& aState) override; // Return active media session's metadata if active media session exists and // it has already set its metadata. Otherwise, return default media metadata @@ -180,7 +181,7 @@ class MediaStatusManager : public IMediaInfoUpdater { return mMetadataChangedEvent; } - MediaEventSource& PositionChangedEvent() { + MediaEventSource>& PositionChangedEvent() { return mPositionStateChangedEvent; } @@ -246,6 +247,10 @@ class MediaStatusManager : public IMediaInfoUpdater { // media session doesn't exist, return 'None' instead. MediaSessionPlaybackState GetCurrentDeclaredPlaybackState() const; + // Return the active media session's position state. If the active media + // session doesn't exist or doesn't have any state, Nothing is returned. + Maybe GetCurrentPositionState() const; + // This state can match to the `guessed playback state` in the spec [1], it // indicates if we have any media element playing within the tab which this // controller belongs to. But currently we only take media elements into @@ -266,7 +271,7 @@ class MediaStatusManager : public IMediaInfoUpdater { MediaEventProducer mMetadataChangedEvent; MediaEventProducer> mSupportedActionsChangedEvent; - MediaEventProducer mPositionStateChangedEvent; + MediaEventProducer> mPositionStateChangedEvent; MediaEventProducer mPlaybackStateChangedEvent; MediaPlaybackStatus mPlaybackStatusDelegate; }; diff --git a/dom/media/mediacontrol/tests/browser/browser_media_control_before_media_starts.js b/dom/media/mediacontrol/tests/browser/browser_media_control_before_media_starts.js index 292c2f521f..a0c75bfa7c 100644 --- a/dom/media/mediacontrol/tests/browser/browser_media_control_before_media_starts.js +++ b/dom/media/mediacontrol/tests/browser/browser_media_control_before_media_starts.js @@ -184,7 +184,7 @@ function enableMediaFullScreenInIframe(tab) { } function waitUntilIframeMediaStartedPlaying(tab) { - return SpecialPowers.spawn(tab.linkedBrowser, [IFRAME_URL], async url => { + return SpecialPowers.spawn(tab.linkedBrowser, [IFRAME_URL], async () => { info(`check if media in iframe starts playing`); const iframe = content.document.getElementById("iframe"); iframe.contentWindow.postMessage("check-playing", "*"); diff --git a/dom/media/mediacontrol/tests/gtest/MediaKeyListenerTest.h b/dom/media/mediacontrol/tests/gtest/MediaKeyListenerTest.h index 5145eb7dbb..50ef254947 100644 --- a/dom/media/mediacontrol/tests/gtest/MediaKeyListenerTest.h +++ b/dom/media/mediacontrol/tests/gtest/MediaKeyListenerTest.h @@ -18,7 +18,7 @@ class MediaKeyListenerTest : public MediaControlKeyListener { void Clear() { mReceivedKey = mozilla::Nothing(); } void OnActionPerformed(const MediaControlAction& aAction) override { - mReceivedKey = mozilla::Some(aAction.mKey); + mReceivedKey = aAction.mKey; } bool IsResultEqualTo(MediaControlKey aResult) const { if (mReceivedKey) { diff --git a/dom/media/mediacontrol/tests/gtest/moz.build b/dom/media/mediacontrol/tests/gtest/moz.build index 7043bfcd5e..a8602f6718 100644 --- a/dom/media/mediacontrol/tests/gtest/moz.build +++ b/dom/media/mediacontrol/tests/gtest/moz.build @@ -11,7 +11,7 @@ UNIFIED_SOURCES += [ "TestMediaKeysEvent.cpp", ] -if CONFIG["MOZ_APPLEMEDIA"]: +if CONFIG["MOZ_APPLEMEDIA"] and CONFIG["TARGET_OS"] == "OSX": UNIFIED_SOURCES += ["TestMediaKeysEventMac.mm", "TestMediaKeysEventMediaCenter.mm"] include("/ipc/chromium/chromium-config.mozbuild") diff --git a/dom/media/mediasession/MediaSession.cpp b/dom/media/mediasession/MediaSession.cpp index e55fa28d96..9120aa5379 100644 --- a/dom/media/mediasession/MediaSession.cpp +++ b/dom/media/mediasession/MediaSession.cpp @@ -20,6 +20,29 @@ namespace mozilla::dom { +double PositionState::CurrentPlaybackPosition(TimeStamp aNow) const { + // https://w3c.github.io/mediasession/#current-playback-position + + // Set time elapsed to the system time in seconds minus the last position + // updated time. + auto timeElapsed = aNow - mPositionUpdatedTime; + // Mutliply time elapsed with actual playback rate. + timeElapsed = timeElapsed.MultDouble(mPlaybackRate); + // Set position to time elapsed added to last reported playback position. + auto position = timeElapsed.ToSeconds() + mLastReportedPlaybackPosition; + + // If position is less than zero, return zero. + if (position < 0.0) { + return 0.0; + } + // If position is greater than duration, return duration. + if (position > mDuration) { + return mDuration; + } + // Return position. + return position; +} + // We don't use NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE because we need to // unregister MediaSession from document's activity listeners. NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(MediaSession) @@ -138,6 +161,7 @@ void MediaSession::SetPositionState(const MediaPositionState& aState, // If the state is an empty dictionary then clear the position state. if (!aState.IsAnyMemberPresent()) { mPositionState.reset(); + NotifyPositionStateChanged(); return; } @@ -175,8 +199,8 @@ void MediaSession::SetPositionState(const MediaPositionState& aState, // Update the position state and last position updated time. MOZ_ASSERT(aState.mDuration.WasPassed()); - mPositionState = - Some(PositionState(aState.mDuration.Value(), playbackRate, position)); + mPositionState = Some(PositionState(aState.mDuration.Value(), playbackRate, + position, TimeStamp::Now())); NotifyPositionStateChanged(); } @@ -328,7 +352,7 @@ void MediaSession::NotifyPositionStateChanged() { RefPtr currentBC = GetParentObject()->GetBrowsingContext(); MOZ_ASSERT(currentBC, "Update action after context destroyed!"); if (RefPtr updater = ContentMediaAgent::Get(currentBC)) { - updater->UpdatePositionState(currentBC->Id(), *mPositionState); + updater->UpdatePositionState(currentBC->Id(), mPositionState); } } diff --git a/dom/media/mediasession/MediaSession.h b/dom/media/mediasession/MediaSession.h index db6864c842..6784fb531f 100644 --- a/dom/media/mediasession/MediaSession.h +++ b/dom/media/mediasession/MediaSession.h @@ -11,6 +11,7 @@ #include "mozilla/Attributes.h" #include "mozilla/dom/MediaSessionBinding.h" #include "mozilla/EnumeratedArray.h" +#include "mozilla/TimeStamp.h" #include "nsCycleCollectionParticipant.h" #include "nsIDocumentActivity.h" #include "nsWrapperCache.h" @@ -29,13 +30,21 @@ class MediaMetadata; struct PositionState { PositionState() = default; PositionState(double aDuration, double aPlaybackRate, - double aLastReportedTime) + double aLastReportedTime, TimeStamp aPositionUpdatedTime) : mDuration(aDuration), mPlaybackRate(aPlaybackRate), - mLastReportedPlaybackPosition(aLastReportedTime) {} - double mDuration; - double mPlaybackRate; - double mLastReportedPlaybackPosition; + mLastReportedPlaybackPosition(aLastReportedTime), + mPositionUpdatedTime(aPositionUpdatedTime) {} + + double mDuration = 0.0; + double mPlaybackRate = 0.0; + double mLastReportedPlaybackPosition = 0.0; + TimeStamp mPositionUpdatedTime; + + // Returns the playback position in seconds (from 0 to mDuration) + // at the current time (aNow). + // https://w3c.github.io/mediasession/#current-playback-position + double CurrentPlaybackPosition(TimeStamp aNow = TimeStamp::Now()) const; }; class MediaSession final : public nsIDocumentActivity, public nsWrapperCache { @@ -111,8 +120,7 @@ class MediaSession final : public nsIDocumentActivity, public nsWrapperCache { RefPtr mMediaMetadata; - EnumeratedArray> + EnumeratedArray> mActionHandlers; // This is used as is a hint for the user agent to determine whether the diff --git a/dom/media/mediasession/MediaSessionIPCUtils.h b/dom/media/mediasession/MediaSessionIPCUtils.h index c44f4b4553..610b123b6c 100644 --- a/dom/media/mediasession/MediaSessionIPCUtils.h +++ b/dom/media/mediasession/MediaSessionIPCUtils.h @@ -6,6 +6,7 @@ #define DOM_MEDIA_MEDIASESSION_MEDIASESSIONIPCUTILS_H_ #include "ipc/EnumSerializer.h" +#include "mozilla/dom/BindingIPCUtils.h" #include "MediaMetadata.h" #include "mozilla/dom/MediaSession.h" #include "mozilla/dom/MediaSessionBinding.h" @@ -71,12 +72,14 @@ struct ParamTraits { WriteParam(aWriter, aParam.mDuration); WriteParam(aWriter, aParam.mPlaybackRate); WriteParam(aWriter, aParam.mLastReportedPlaybackPosition); + WriteParam(aWriter, aParam.mPositionUpdatedTime); } static bool Read(MessageReader* aReader, paramType* aResult) { if (!ReadParam(aReader, &(aResult->mDuration)) || !ReadParam(aReader, &(aResult->mPlaybackRate)) || - !ReadParam(aReader, &(aResult->mLastReportedPlaybackPosition))) { + !ReadParam(aReader, &(aResult->mLastReportedPlaybackPosition)) || + !ReadParam(aReader, &(aResult->mPositionUpdatedTime))) { return false; } return true; @@ -85,17 +88,13 @@ struct ParamTraits { template <> struct ParamTraits - : public ContiguousEnumSerializer< - mozilla::dom::MediaSessionPlaybackState, - mozilla::dom::MediaSessionPlaybackState::None, - mozilla::dom::MediaSessionPlaybackState::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer< + mozilla::dom::MediaSessionPlaybackState> {}; template <> struct ParamTraits - : public ContiguousEnumSerializer< - mozilla::dom::MediaSessionAction, - mozilla::dom::MediaSessionAction::Play, - mozilla::dom::MediaSessionAction::EndGuard_> {}; + : public mozilla::dom::WebIDLEnumSerializer< + mozilla::dom::MediaSessionAction> {}; } // namespace IPC diff --git a/dom/media/mediasession/moz.build b/dom/media/mediasession/moz.build index 805bf67971..70f69fc14f 100644 --- a/dom/media/mediasession/moz.build +++ b/dom/media/mediasession/moz.build @@ -6,6 +6,10 @@ MOCHITEST_MANIFESTS += ["test/mochitest.toml"] +TEST_DIRS += [ + "test/gtest", +] + EXPORTS.mozilla.dom += [ "MediaMetadata.h", "MediaSession.h", diff --git a/dom/media/mediasession/test/gtest/TestPositionState.cpp b/dom/media/mediasession/test/gtest/TestPositionState.cpp new file mode 100644 index 0000000000..5fa38d0294 --- /dev/null +++ b/dom/media/mediasession/test/gtest/TestPositionState.cpp @@ -0,0 +1,127 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" +#include "MediaSession.h" + +using namespace mozilla; +using namespace mozilla::dom; + +struct CurrentPlaybackPositionFixture { + double mDuration = 0.0; + double mPlaybackRate = 0.0; + double mLastReportedTime = 0.0; + + TimeDuration mDelta; + double mExpectedPosition = -1; +}; + +class CurrentPlaybackPositionTest + : public testing::TestWithParam {}; + +static const std::initializer_list kFixtures = { + // position must be positive + { + 10.0, + 1.0, + 0.0, + TimeDuration::FromSeconds(-1.0), + 0.0, + }, + // no time elapsed + { + 10.0, + 1.0, + 0.0, + TimeDuration::FromSeconds(0.0), + 0.0, + }, + { + 10.0, + 1.0, + 0.0, + TimeDuration::FromSeconds(3.0), + 3.0, + }, + { + 10.0, + 1.0, + 0.0, + TimeDuration::FromSeconds(10.0), + 10.0, + }, + // position is clamped to the duration + { + 10.0, + 1.0, + 0.0, + TimeDuration::FromSeconds(20.0), + 10.0, + }, + { + 10.0, + 1.0, + 5.0, + TimeDuration::FromSeconds(-1.0), + 4.0, + }, + { + 10.0, + 1.0, + 5.0, + TimeDuration::FromSeconds(-6.0), + 0.0, + }, + { + 10.0, + 1.0, + 5.0, + TimeDuration::FromSeconds(6.0), + 10.0, + }, + // expected: 5s + 2 * 2s + { + 10.0, + 2.0, + 5.0, + TimeDuration::FromSeconds(2.0), + 9.0, + }, + // expected: 5s + 0.5 * 2s + { + 10.0, + 0.5, + 5.0, + TimeDuration::FromSeconds(2.0), + 6.0, + }, + { + 5.0, + 4.0, + 10.0, + TimeDuration::FromSeconds(20.0), + 5.0, + }, + // empty media (0s) + { + 0.0, + 4.0, + 5.0, + TimeDuration::FromSeconds(20.0), + 0.0, + }, +}; + +TEST_P(CurrentPlaybackPositionTest, Run) { + const auto& fixture = GetParam(); + PositionState state(fixture.mDuration, fixture.mPlaybackRate, + fixture.mLastReportedTime, TimeStamp::Now()); + + ASSERT_DOUBLE_EQ(state.CurrentPlaybackPosition(state.mPositionUpdatedTime + + fixture.mDelta), + fixture.mExpectedPosition); +} + +INSTANTIATE_TEST_SUITE_P(PositionState, CurrentPlaybackPositionTest, + testing::ValuesIn(kFixtures)); diff --git a/dom/media/mediasession/test/gtest/moz.build b/dom/media/mediasession/test/gtest/moz.build new file mode 100644 index 0000000000..fd2a9a6e30 --- /dev/null +++ b/dom/media/mediasession/test/gtest/moz.build @@ -0,0 +1,15 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +UNIFIED_SOURCES += [ + "TestPositionState.cpp", +] + +LOCAL_INCLUDES += [ + "/dom/media/mediasession", +] + +FINAL_LIBRARY = "xul-gtest" diff --git a/dom/media/mediasource/MediaSourceDecoder.cpp b/dom/media/mediasource/MediaSourceDecoder.cpp index 24a74e261b..3a1308a9ca 100644 --- a/dom/media/mediasource/MediaSourceDecoder.cpp +++ b/dom/media/mediasource/MediaSourceDecoder.cpp @@ -57,8 +57,13 @@ MediaDecoderStateMachineBase* MediaSourceDecoder::CreateStateMachine( TrackingId::TrackAcrossProcesses::Yes); mReader = new MediaFormatReader(init, mDemuxer); #ifdef MOZ_WMF_MEDIA_ENGINE - // TODO : Only for testing development for now. In the future this should be - // used for encrypted content only. + // Our main purpose is to only using this state machine for encrypted playback + // (unless explicitly set the pref to allow non-encrypted playback), but we + // can't determine if playback is encrypted or not at the moment. Therefore, + // we will handle that in ExternalEngineStateMachine, and report special + // errors, such as NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR or + // NS_ERROR_DOM_MEDIA_CDM_PROXY_NOT_SUPPORTED_ERR, to switch the state + // machine if necessary. if (StaticPrefs::media_wmf_media_engine_enabled() && !aDisableExternalEngine) { return new ExternalEngineStateMachine(this, mReader); @@ -319,12 +324,13 @@ bool MediaSourceDecoder::CanPlayThroughImpl() { } // If we have data up to the mediasource's duration or 3s ahead, we can // assume that we can play without interruption. - dom::SourceBufferList* sourceBuffers = mMediaSource->ActiveSourceBuffers(); - TimeUnit bufferedEnd = sourceBuffers->GetHighestBufferedEndTime(); + TimeIntervals buffered = GetBuffered(); + buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2); TimeUnit timeAhead = std::min(duration, currentPosition + TimeUnit::FromSeconds(3)); TimeInterval interval(currentPosition, timeAhead); - return bufferedEnd >= timeAhead; + return buffered.ToMicrosecondResolution().ContainsWithStrictEnd( + ClampIntervalToEnd(interval)); } TimeInterval MediaSourceDecoder::ClampIntervalToEnd( @@ -366,6 +372,23 @@ bool MediaSourceDecoder::HadCrossOriginRedirects() { return false; } +#ifdef MOZ_WMF_MEDIA_ENGINE +void MediaSourceDecoder::MetadataLoaded( + UniquePtr aInfo, UniquePtr aTags, + MediaDecoderEventVisibility aEventVisibility) { + // If the previous state machine has loaded the metadata, then we don't need + // to load it again. This can happen when the media format or key system is + // not supported by previous state machine. + if (mFiredMetadataLoaded && mStateMachineRecreated) { + MSE_DEBUG( + "Metadata already loaded and being informed by previous state machine"); + return; + } + MediaDecoder::MetadataLoaded(std::move(aInfo), std::move(aTags), + aEventVisibility); +} +#endif + #undef MSE_DEBUG #undef MSE_DEBUGV diff --git a/dom/media/mediasource/MediaSourceDecoder.h b/dom/media/mediasource/MediaSourceDecoder.h index ff312cb6cf..2ebba67a17 100644 --- a/dom/media/mediasource/MediaSourceDecoder.h +++ b/dom/media/mediasource/MediaSourceDecoder.h @@ -85,6 +85,11 @@ class MediaSourceDecoder : public MediaDecoder, media::TimeInterval ClampIntervalToEnd(const media::TimeInterval& aInterval); bool CanPlayThroughImpl() override; +#ifdef MOZ_WMF_MEDIA_ENGINE + void MetadataLoaded(UniquePtr aInfo, UniquePtr aTags, + MediaDecoderEventVisibility aEventVisibility) override; +#endif + RefPtr mPrincipal; // The owning MediaSource holds a strong reference to this decoder, and diff --git a/dom/media/mediasource/test/mochitest.toml b/dom/media/mediasource/test/mochitest.toml index b9ed95b2bb..7e60af8929 100644 --- a/dom/media/mediasource/test/mochitest.toml +++ b/dom/media/mediasource/test/mochitest.toml @@ -270,3 +270,5 @@ skip-if = [ ["test_WaitingOnMissingData_mp4.html"] ["test_WaitingToEndedTransition_mp4.html"] + +["test_BufferedSeekCanPlayThrough.html"] diff --git a/dom/media/mediasource/test/test_BufferedSeekCanPlayThrough.html b/dom/media/mediasource/test/test_BufferedSeekCanPlayThrough.html new file mode 100644 index 0000000000..5c789a7b6e --- /dev/null +++ b/dom/media/mediasource/test/test_BufferedSeekCanPlayThrough.html @@ -0,0 +1,46 @@ + + + + MSE: Don't get stuck buffering for too long when we have frames to show + + + + + +

+
+ + diff --git a/dom/media/mediasource/test/test_EndOfStream.html b/dom/media/mediasource/test/test_EndOfStream.html index b926869f1f..bcaf2ee54d 100644 --- a/dom/media/mediasource/test/test_EndOfStream.html +++ b/dom/media/mediasource/test/test_EndOfStream.html @@ -12,7 +12,7 @@ SimpleTest.waitForExplicitFinish(); -runWithMSE(async (ms, v) => { +runWithMSE(async (ms) => { await once(ms, "sourceopen"); const sb = ms.addSourceBuffer("video/webm"); diff --git a/dom/media/mediasource/test/test_EndOfStream_mp4.html b/dom/media/mediasource/test/test_EndOfStream_mp4.html index 9319b80390..140641565d 100644 --- a/dom/media/mediasource/test/test_EndOfStream_mp4.html +++ b/dom/media/mediasource/test/test_EndOfStream_mp4.html @@ -12,7 +12,7 @@ SimpleTest.waitForExplicitFinish(); -runWithMSE(async (ms, v) => { +runWithMSE(async (ms) => { await once(ms, "sourceopen"); const sb = ms.addSourceBuffer("video/mp4"); diff --git a/dom/media/mediasource/test/test_ExperimentalAsync.html b/dom/media/mediasource/test/test_ExperimentalAsync.html index 6617716f26..e64a9befeb 100644 --- a/dom/media/mediasource/test/test_ExperimentalAsync.html +++ b/dom/media/mediasource/test/test_ExperimentalAsync.html @@ -61,7 +61,7 @@ runWithMSE(async function(ms, el) { await once(el, "seeked"); dump("dump: seeked to " + seekTime); is(el.currentTime, seekTime, "correctly seeked to " + seekTime); - await audiosb.appendBufferAsync(audioBuffer).catch(async function(ex2) { + await audiosb.appendBufferAsync(audioBuffer).catch(async function() { ok(false, "Shouldn't throw another time when data can be evicted"); dump(JSON.stringify(await SpecialPowers.wrap(el).mozRequestDebugInfo())); SimpleTest.finish(); @@ -73,7 +73,7 @@ runWithMSE(async function(ms, el) { await audiosb.removeAsync(ms.duration + 1, Infinity).catch(async function(ex4) { ok(true, "remove promise got rejected with start > duration"); is(ex4.name, "TypeError"); - await audiosb.removeAsync(0, Infinity).catch(function(ex5) { + await audiosb.removeAsync(0, Infinity).catch(function() { ok(false, "shouldn't throw"); }); ok(true, "remove succeeded"); diff --git a/dom/media/mediasource/test/test_SetModeThrows.html b/dom/media/mediasource/test/test_SetModeThrows.html index c715854b41..c81cb1b70f 100644 --- a/dom/media/mediasource/test/test_SetModeThrows.html +++ b/dom/media/mediasource/test/test_SetModeThrows.html @@ -13,7 +13,7 @@ SimpleTest.waitForExplicitFinish(); // MSE supports setting mode now. make sure it does not throw. -runWithMSE(function(ms, v) { +runWithMSE(function(ms) { ms.addEventListener("sourceopen", () => { const sb = ms.addSourceBuffer("video/webm"); diff --git a/dom/media/metrics.yaml b/dom/media/metrics.yaml index 46801ebaf4..fe2ed5ff6a 100644 --- a/dom/media/metrics.yaml +++ b/dom/media/metrics.yaml @@ -96,3 +96,47 @@ media.audio: notification_emails: - media-alerts@mozilla.com expires: never + +media.playback: + first_frame_loaded: + type: event + description: + The time that the media pipeline takes to load the first video frame. + metadata: + tags: + - 'Core :: Audio/Video: Playback' + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1882205 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1882205 + data_sensitivity: + - technical + notification_emails: + - media-alerts@mozilla.com + extra_keys: + first_frame_loaded_time: + description: + How long (in milliseconds) does the our media pipeline take to load + the first video frame. + type: quantity + playback_type: + description: + The type of the playback. The value could be one of following + (1) Non-MSE playback + (2) MSE playback + (3) EME playback + // Following are Windows-only + (4) Non-MSE media-engine playback + (5) MSE media-engine playback + (6) EME media-engine playback + type: string + video_codec: + description: The video codec used for playback + type: string + resolution: + description: The video resolution used for playback + type: string + key_system: + description: The key system used for the EME playback if exists + type: string + expires: never diff --git a/dom/media/nsIMediaManager.idl b/dom/media/nsIMediaManager.idl index 9cc39d04f4..68334b2e6a 100644 --- a/dom/media/nsIMediaManager.idl +++ b/dom/media/nsIMediaManager.idl @@ -8,11 +8,6 @@ interface nsIArray; interface nsIDOMWindow; interface nsIMediaDevice; -%{C++ -#define NS_MEDIAMANAGERSERVICE_CID {0xabc622ea, 0x9655, 0x4123, {0x80, 0xd9, 0x22, 0x62, 0x1b, 0xdd, 0x54, 0x65}} -#define MEDIAMANAGERSERVICE_CONTRACTID "@mozilla.org/mediaManagerService;1" -%} - [scriptable, builtinclass, uuid(24b23e01-33fd-401f-ba25-6e52658750b0)] interface nsIMediaManagerService : nsISupports { diff --git a/dom/media/platforms/PDMFactory.cpp b/dom/media/platforms/PDMFactory.cpp index 2964527e07..00f46385e2 100644 --- a/dom/media/platforms/PDMFactory.cpp +++ b/dom/media/platforms/PDMFactory.cpp @@ -45,9 +45,6 @@ # include "mozilla/CDMProxy.h" # endif #endif -#ifdef MOZ_FFVPX -# include "FFVPXRuntimeLinker.h" -#endif #ifdef MOZ_FFMPEG # include "FFmpegRuntimeLinker.h" #endif @@ -60,6 +57,7 @@ #ifdef MOZ_OMX # include "OmxDecoderModule.h" #endif +#include "FFVPXRuntimeLinker.h" #include @@ -99,14 +97,12 @@ class PDMInitializer final { #ifdef MOZ_APPLEMEDIA AppleDecoderModule::Init(); #endif -#ifdef MOZ_FFVPX - FFVPXRuntimeLinker::Init(); -#endif #ifdef MOZ_FFMPEG if (StaticPrefs::media_rdd_ffmpeg_enabled()) { FFmpegRuntimeLinker::Init(); } #endif + FFVPXRuntimeLinker::Init(); } static void InitUtilityPDMs() { @@ -127,11 +123,9 @@ class PDMInitializer final { AppleDecoderModule::Init(); } #endif -#ifdef MOZ_FFVPX if (kind == ipc::SandboxingKind::GENERIC_UTILITY) { FFVPXRuntimeLinker::Init(); } -#endif #ifdef MOZ_FFMPEG if (StaticPrefs::media_utility_ffmpeg_enabled() && kind == ipc::SandboxingKind::GENERIC_UTILITY) { @@ -160,9 +154,7 @@ class PDMInitializer final { #ifdef MOZ_OMX OmxDecoderModule::Init(); #endif -#ifdef MOZ_FFVPX FFVPXRuntimeLinker::Init(); -#endif #ifdef MOZ_FFMPEG FFmpegRuntimeLinker::Init(); #endif @@ -183,9 +175,7 @@ class PDMInitializer final { #ifdef MOZ_OMX OmxDecoderModule::Init(); #endif -#ifdef MOZ_FFVPX FFVPXRuntimeLinker::Init(); -#endif #ifdef MOZ_FFMPEG FFmpegRuntimeLinker::Init(); #endif @@ -547,12 +537,7 @@ void PDMFactory::CreateRddPDMs() { CreateAndStartupPDM(); } #endif -#ifdef MOZ_FFVPX - if (StaticPrefs::media_ffvpx_enabled() && - StaticPrefs::media_rdd_ffvpx_enabled()) { - StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); - } -#endif + StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); #ifdef MOZ_FFMPEG if (StaticPrefs::media_ffmpeg_enabled() && StaticPrefs::media_rdd_ffmpeg_enabled() && @@ -580,12 +565,9 @@ void PDMFactory::CreateUtilityPDMs() { } #endif if (aKind == ipc::SandboxingKind::GENERIC_UTILITY) { -#ifdef MOZ_FFVPX - if (StaticPrefs::media_ffvpx_enabled() && - StaticPrefs::media_utility_ffvpx_enabled()) { + if (StaticPrefs::media_utility_ffvpx_enabled()) { StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); } -#endif #ifdef MOZ_FFMPEG if (StaticPrefs::media_ffmpeg_enabled() && StaticPrefs::media_utility_ffmpeg_enabled() && @@ -667,11 +649,7 @@ void PDMFactory::CreateContentPDMs() { CreateAndStartupPDM(); } #endif -#ifdef MOZ_FFVPX - if (StaticPrefs::media_ffvpx_enabled()) { - StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); - } -#endif + StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); #ifdef MOZ_FFMPEG if (StaticPrefs::media_ffmpeg_enabled() && !StartupPDM(FFmpegRuntimeLinker::CreateDecoder())) { @@ -719,11 +697,7 @@ void PDMFactory::CreateDefaultPDMs() { CreateAndStartupPDM(); } #endif -#ifdef MOZ_FFVPX - if (StaticPrefs::media_ffvpx_enabled()) { - StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); - } -#endif + StartupPDM(FFVPXRuntimeLinker::CreateDecoder()); #ifdef MOZ_FFMPEG if (StaticPrefs::media_ffmpeg_enabled() && !StartupPDM(FFmpegRuntimeLinker::CreateDecoder())) { @@ -898,9 +872,6 @@ DecodeSupportSet PDMFactory::SupportsMimeType( /* static */ bool PDMFactory::AllDecodersAreRemote() { return StaticPrefs::media_rdd_process_enabled() && -#if defined(MOZ_FFVPX) - StaticPrefs::media_rdd_ffvpx_enabled() && -#endif StaticPrefs::media_rdd_opus_enabled() && StaticPrefs::media_rdd_theora_enabled() && StaticPrefs::media_rdd_vorbis_enabled() && diff --git a/dom/media/platforms/PEMFactory.cpp b/dom/media/platforms/PEMFactory.cpp index 9647c5b079..a5b42914eb 100644 --- a/dom/media/platforms/PEMFactory.cpp +++ b/dom/media/platforms/PEMFactory.cpp @@ -20,13 +20,12 @@ # include "WMFEncoderModule.h" #endif -#ifdef MOZ_FFVPX -# include "FFVPXRuntimeLinker.h" -#endif #ifdef MOZ_FFMPEG # include "FFmpegRuntimeLinker.h" #endif +#include "FFVPXRuntimeLinker.h" + #include "mozilla/StaticPrefs_media.h" #include "mozilla/gfx/gfxVars.h" @@ -56,15 +55,12 @@ PEMFactory::PEMFactory() { mCurrentPEMs.AppendElement(new WMFEncoderModule()); #endif -#ifdef MOZ_FFVPX - if (StaticPrefs::media_ffvpx_enabled() && - StaticPrefs::media_ffmpeg_encoder_enabled()) { + if (StaticPrefs::media_ffmpeg_encoder_enabled()) { if (RefPtr pem = FFVPXRuntimeLinker::CreateEncoder()) { mCurrentPEMs.AppendElement(pem); } } -#endif #ifdef MOZ_FFMPEG if (StaticPrefs::media_ffmpeg_enabled() && diff --git a/dom/media/platforms/agnostic/bytestreams/Adts.cpp b/dom/media/platforms/agnostic/bytestreams/Adts.cpp index 5f31904d9c..71c9f15308 100644 --- a/dom/media/platforms/agnostic/bytestreams/Adts.cpp +++ b/dom/media/platforms/agnostic/bytestreams/Adts.cpp @@ -4,37 +4,56 @@ #include "Adts.h" #include "MediaData.h" +#include "PlatformDecoderModule.h" #include "mozilla/Array.h" #include "mozilla/ArrayUtils.h" +#include "mozilla/Logging.h" +#include "ADTSDemuxer.h" + +extern mozilla::LazyLogModule gMediaDemuxerLog; +#define LOG(msg, ...) \ + MOZ_LOG(gMediaDemuxerLog, LogLevel::Debug, msg, ##__VA_ARGS__) +#define ADTSLOG(msg, ...) \ + DDMOZ_LOG(gMediaDemuxerLog, LogLevel::Debug, msg, ##__VA_ARGS__) +#define ADTSLOGV(msg, ...) \ + DDMOZ_LOG(gMediaDemuxerLog, LogLevel::Verbose, msg, ##__VA_ARGS__) namespace mozilla { +namespace ADTS { static const int kADTSHeaderSize = 7; -int8_t Adts::GetFrequencyIndex(uint32_t aSamplesPerSecond) { - static const uint32_t freq_lookup[] = {96000, 88200, 64000, 48000, 44100, - 32000, 24000, 22050, 16000, 12000, - 11025, 8000, 7350, 0}; +constexpr std::array FREQ_LOOKUP{96000, 88200, 64000, 48000, 44100, + 32000, 24000, 22050, 16000, 12000, + 11025, 8000, 7350, 0}; - int8_t i = 0; - while (freq_lookup[i] && aSamplesPerSecond < freq_lookup[i]) { - i++; - } +Result GetFrequencyIndex(uint32_t aSamplesPerSecond) { + auto found = + std::find(FREQ_LOOKUP.begin(), FREQ_LOOKUP.end(), aSamplesPerSecond); - if (!freq_lookup[i]) { - return -1; + if (found == FREQ_LOOKUP.end()) { + return Err(false); } - return i; + return std::distance(FREQ_LOOKUP.begin(), found); } -bool Adts::ConvertSample(uint16_t aChannelCount, int8_t aFrequencyIndex, - int8_t aProfile, MediaRawData* aSample) { +bool ConvertSample(uint16_t aChannelCount, uint8_t aFrequencyIndex, + uint8_t aProfile, MediaRawData* aSample) { size_t newSize = aSample->Size() + kADTSHeaderSize; + MOZ_LOG(sPDMLog, LogLevel::Debug, + ("Converting sample to ADTS format: newSize: %zu, ch: %u, " + "profile: %u, freq index: %d", + newSize, aChannelCount, aProfile, aFrequencyIndex)); + // ADTS header uses 13 bits for packet size. - if (newSize >= (1 << 13) || aChannelCount > 15 || aFrequencyIndex < 0 || - aProfile < 1 || aProfile > 4) { + if (newSize >= (1 << 13) || aChannelCount > 15 || aProfile < 1 || + aProfile > 4 || aFrequencyIndex >= FREQ_LOOKUP.size()) { + MOZ_LOG(sPDMLog, LogLevel::Debug, + ("Couldn't convert sample to ADTS format: newSize: %zu, ch: %u, " + "profile: %u, freq index: %d", + newSize, aChannelCount, aProfile, aFrequencyIndex)); return false; } @@ -66,7 +85,36 @@ bool Adts::ConvertSample(uint16_t aChannelCount, int8_t aFrequencyIndex, return true; } -bool Adts::RevertSample(MediaRawData* aSample) { +bool StripHeader(MediaRawData* aSample) { + if (aSample->Size() < kADTSHeaderSize) { + return false; + } + + FrameHeader header; + auto data = Span{aSample->Data(), aSample->Size()}; + MOZ_ASSERT(FrameHeader::MatchesSync(data), + "Don't attempt to strip the ADTS header of a raw AAC packet."); + + bool crcPresent = header.mHaveCrc; + + LOG(("Stripping ADTS, crc %spresent", crcPresent ? "" : "not ")); + + size_t toStrip = crcPresent ? kADTSHeaderSize + 2 : kADTSHeaderSize; + + UniquePtr writer(aSample->CreateWriter()); + writer->PopFront(toStrip); + + if (aSample->mCrypto.IsEncrypted()) { + if (aSample->mCrypto.mPlainSizes.Length() > 0 && + writer->mCrypto.mPlainSizes[0] >= kADTSHeaderSize) { + writer->mCrypto.mPlainSizes[0] -= kADTSHeaderSize; + } + } + + return true; +} + +bool RevertSample(MediaRawData* aSample) { if (aSample->Size() < kADTSHeaderSize) { return false; } @@ -91,4 +139,156 @@ bool Adts::RevertSample(MediaRawData* aSample) { return true; } -} // namespace mozilla + +bool FrameHeader::MatchesSync(const Span& aData) { + return aData.Length() >= 2 && aData[0] == 0xFF && (aData[1] & 0xF6) == 0xF0; +} + +FrameHeader::FrameHeader() { Reset(); } + +// Header size +uint64_t FrameHeader::HeaderSize() const { return (mHaveCrc) ? 9 : 7; } + +bool FrameHeader::IsValid() const { return mFrameLength > 0; } + +// Resets the state to allow for a new parsing session. +void FrameHeader::Reset() { PodZero(this); } + +// Returns whether the byte creates a valid sequence up to this point. +bool FrameHeader::Parse(const Span& aData) { + if (!MatchesSync(aData)) { + return false; + } + + // AAC has 1024 samples per frame per channel. + mSamples = 1024; + + mHaveCrc = !(aData[1] & 0x01); + mObjectType = ((aData[2] & 0xC0) >> 6) + 1; + mSamplingIndex = (aData[2] & 0x3C) >> 2; + mChannelConfig = (aData[2] & 0x01) << 2 | (aData[3] & 0xC0) >> 6; + mFrameLength = + static_cast((aData[3] & 0x03) << 11 | (aData[4] & 0xFF) << 3 | + (aData[5] & 0xE0) >> 5); + mNumAACFrames = (aData[6] & 0x03) + 1; + + static const uint32_t SAMPLE_RATES[] = {96000, 88200, 64000, 48000, 44100, + 32000, 24000, 22050, 16000, 12000, + 11025, 8000, 7350}; + if (mSamplingIndex >= ArrayLength(SAMPLE_RATES)) { + LOG(("ADTS: Init() failure: invalid sample-rate index value: %" PRIu32 ".", + mSamplingIndex)); + // This marks the header as invalid. + mFrameLength = 0; + return false; + } + mSampleRate = SAMPLE_RATES[mSamplingIndex]; + + MOZ_ASSERT(mChannelConfig < 8); + mChannels = (mChannelConfig == 7) ? 8 : mChannelConfig; + + return true; +} + +Frame::Frame() : mOffset(0), mHeader() {} +uint64_t Frame::Offset() const { return mOffset; } +size_t Frame::Length() const { + // TODO: If fields are zero'd when invalid, this check wouldn't be + // necessary. + if (!mHeader.IsValid()) { + return 0; + } + + return mHeader.mFrameLength; +} + +// Returns the offset to the start of frame's raw data. +uint64_t Frame::PayloadOffset() const { return mOffset + mHeader.HeaderSize(); } + +// Returns the length of the frame's raw data (excluding the header) in bytes. +size_t Frame::PayloadLength() const { + // TODO: If fields are zero'd when invalid, this check wouldn't be + // necessary. + if (!mHeader.IsValid()) { + return 0; + } + + return mHeader.mFrameLength - mHeader.HeaderSize(); +} + +// Returns the parsed frame header. +const FrameHeader& Frame::Header() const { return mHeader; } + +bool Frame::IsValid() const { return mHeader.IsValid(); } + +// Resets the frame header and data. +void Frame::Reset() { + mHeader.Reset(); + mOffset = 0; +} + +// Returns whether the valid +bool Frame::Parse(uint64_t aOffset, const uint8_t* aStart, + const uint8_t* aEnd) { + MOZ_ASSERT(aStart && aEnd && aStart <= aEnd); + + bool found = false; + const uint8_t* ptr = aStart; + // Require at least 7 bytes of data at the end of the buffer for the minimum + // ADTS frame header. + while (ptr < aEnd - 7 && !found) { + found = mHeader.Parse(Span(ptr, aEnd)); + ptr++; + } + + mOffset = aOffset + (static_cast(ptr - aStart)) - 1u; + + return found; +} + +const Frame& FrameParser::CurrentFrame() { return mFrame; } + +const Frame& FrameParser::FirstFrame() const { return mFirstFrame; } + +void FrameParser::Reset() { + EndFrameSession(); + mFirstFrame.Reset(); +} + +void FrameParser::EndFrameSession() { mFrame.Reset(); } + +bool FrameParser::Parse(uint64_t aOffset, const uint8_t* aStart, + const uint8_t* aEnd) { + const bool found = mFrame.Parse(aOffset, aStart, aEnd); + + if (mFrame.Length() && !mFirstFrame.Length()) { + mFirstFrame = mFrame; + } + + return found; +} + +// Initialize the AAC AudioSpecificConfig. +// Only handles two-byte version for AAC-LC. +void InitAudioSpecificConfig(const ADTS::Frame& frame, + MediaByteBuffer* aBuffer) { + const ADTS::FrameHeader& header = frame.Header(); + MOZ_ASSERT(header.IsValid()); + + int audioObjectType = header.mObjectType; + int samplingFrequencyIndex = header.mSamplingIndex; + int channelConfig = header.mChannelConfig; + + uint8_t asc[2]; + asc[0] = (audioObjectType & 0x1F) << 3 | (samplingFrequencyIndex & 0x0E) >> 1; + asc[1] = (samplingFrequencyIndex & 0x01) << 7 | (channelConfig & 0x0F) << 3; + + aBuffer->AppendElements(asc, 2); +} + +}; // namespace ADTS +}; // namespace mozilla + +#undef LOG +#undef ADTSLOG +#undef ADTSLOGV diff --git a/dom/media/platforms/agnostic/bytestreams/Adts.h b/dom/media/platforms/agnostic/bytestreams/Adts.h index c2b6b558b6..e6d20806ab 100644 --- a/dom/media/platforms/agnostic/bytestreams/Adts.h +++ b/dom/media/platforms/agnostic/bytestreams/Adts.h @@ -6,17 +6,124 @@ #define ADTS_H_ #include +#include "MediaData.h" +#include "mozilla/Result.h" namespace mozilla { class MediaRawData; -class Adts { +namespace ADTS { + +// adts::FrameHeader - Holds the ADTS frame header and its parsing +// state. +// +// ADTS Frame Structure +// +// 11111111 1111BCCD EEFFFFGH HHIJKLMM MMMMMMMM MMMOOOOO OOOOOOPP(QQQQQQQQ +// QQQQQQQQ) +// +// Header consists of 7 or 9 bytes(without or with CRC). +// Letter Length(bits) Description +// { sync } 12 syncword 0xFFF, all bits must be 1 +// B 1 MPEG Version: 0 for MPEG-4, 1 for MPEG-2 +// C 2 Layer: always 0 +// D 1 protection absent, Warning, set to 1 if there is no +// CRC and 0 if there is CRC +// E 2 profile, the MPEG-4 Audio Object Type minus 1 +// F 4 MPEG-4 Sampling Frequency Index (15 is forbidden) +// H 3 MPEG-4 Channel Configuration (in the case of 0, the +// channel configuration is sent via an in-band PCE) +// M 13 frame length, this value must include 7 or 9 bytes of +// header length: FrameLength = +// (ProtectionAbsent == 1 ? 7 : 9) + size(AACFrame) +// O 11 Buffer fullness +// P 2 Number of AAC frames(RDBs) in ADTS frame minus 1, for +// maximum compatibility always use 1 AAC frame per ADTS +// frame +// Q 16 CRC if protection absent is 0 +class FrameHeader { public: - static int8_t GetFrequencyIndex(uint32_t aSamplesPerSecond); - static bool ConvertSample(uint16_t aChannelCount, int8_t aFrequencyIndex, - int8_t aProfile, mozilla::MediaRawData* aSample); - static bool RevertSample(MediaRawData* aSample); + uint32_t mFrameLength{}; + uint32_t mSampleRate{}; + uint32_t mSamples{}; + uint32_t mChannels{}; + uint8_t mObjectType{}; + uint8_t mSamplingIndex{}; + uint8_t mChannelConfig{}; + uint8_t mNumAACFrames{}; + bool mHaveCrc{}; + + // Returns whether aPtr matches a valid ADTS header sync marker + static bool MatchesSync(const Span& aData); + FrameHeader(); + // Header size + uint64_t HeaderSize() const; + bool IsValid() const; + // Resets the state to allow for a new parsing session. + void Reset(); + + // Returns whether the byte creates a valid sequence up to this point. + bool Parse(const Span& aData); }; +class Frame { + public: + Frame(); + + uint64_t Offset() const; + size_t Length() const; + // Returns the offset to the start of frame's raw data. + uint64_t PayloadOffset() const; + + size_t PayloadLength() const; + // Returns the parsed frame header. + const FrameHeader& Header() const; + bool IsValid() const; + // Resets the frame header and data. + void Reset(); + // Returns whether the valid + bool Parse(uint64_t aOffset, const uint8_t* aStart, const uint8_t* aEnd); + + private: + // The offset to the start of the header. + uint64_t mOffset; + // The currently parsed frame header. + FrameHeader mHeader; +}; + +class FrameParser { + public: + // Returns the currently parsed frame. Reset via Reset or EndFrameSession. + const Frame& CurrentFrame(); + // Returns the first parsed frame. Reset via Reset. + const Frame& FirstFrame() const; + // Resets the parser. Don't use between frames as first frame data is reset. + void Reset(); + // Clear the last parsed frame to allow for next frame parsing, i.e.: + // - sets PrevFrame to CurrentFrame + // - resets the CurrentFrame + // - resets ID3Header if no valid header was parsed yet + void EndFrameSession(); + // Parses contents of given ByteReader for a valid frame header and returns + // true if one was found. After returning, the variable passed to + // 'aBytesToSkip' holds the amount of bytes to be skipped (if any) in order to + // jump across a large ID3v2 tag spanning multiple buffers. + bool Parse(uint64_t aOffset, const uint8_t* aStart, const uint8_t* aEnd); + + private: + // We keep the first parsed frame around for static info access, the + // previously parsed frame for debugging and the currently parsed frame. + Frame mFirstFrame; + Frame mFrame; +}; + +// Extract the audiospecificconfig from an ADTS header +void InitAudioSpecificConfig(const Frame& aFrame, MediaByteBuffer* aBuffer); +bool StripHeader(MediaRawData* aSample); +Result GetFrequencyIndex(uint32_t aSamplesPerSecond); +bool ConvertSample(uint16_t aChannelCount, uint8_t aFrequencyIndex, + uint8_t aProfile, mozilla::MediaRawData* aSample); +bool RevertSample(MediaRawData* aSample); +} // namespace ADTS } // namespace mozilla #endif diff --git a/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp b/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp index 086936dcc6..4721ddefc3 100644 --- a/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp +++ b/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp @@ -256,21 +256,21 @@ static Result FindStartCodeInternal(BufferReader& aBr) { while (aBr.Remaining() >= 6) { uint32_t x32; MOZ_TRY_VAR(x32, aBr.PeekU32()); - if ((x32 - 0x01010101) & (~x32) & 0x80808080) { - if ((x32 >> 8) == 0x000001) { + if ((x32 - 0x01010101) & (~x32) & 0x80808080) { // Has 0x00 byte(s). + if ((x32 >> 8) == 0x000001) { // 0x000001?? return Ok(); } - if (x32 == 0x000001) { + if ((x32 & 0xffffff) == 0x000001) { // 0x??000001 mozilla::Unused << aBr.Read(1); return Ok(); } - if ((x32 & 0xff) == 0) { + if ((x32 & 0xff) == 0) { // 0x??????00 const uint8_t* p = aBr.Peek(1); - if ((x32 & 0xff00) == 0 && p[4] == 1) { + if ((x32 & 0xff00) == 0 && p[4] == 1) { // 0x????0000,01 mozilla::Unused << aBr.Read(2); return Ok(); } - if (p[4] == 0 && p[5] == 1) { + if (p[4] == 0 && p[5] == 1) { // 0x??????00,00,01 mozilla::Unused << aBr.Read(3); return Ok(); } diff --git a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp index e71632e6d3..4c74fa8723 100644 --- a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp +++ b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp @@ -11,6 +11,9 @@ #include "GMPVideoDecoder.h" #include "MP4Decoder.h" #include "VPXDecoder.h" +#ifdef MOZ_AV1 +# include "AOMDecoder.h" +#endif namespace mozilla { @@ -45,6 +48,21 @@ static uint32_t ToCDMH264Profile(uint8_t aProfile) { return cdm::VideoCodecProfile::kUnknownVideoCodecProfile; } +#ifdef MOZ_AV1 +static uint32_t ToCDMAV1Profile(uint8_t aProfile) { + switch (aProfile) { + case 0: + return cdm::VideoCodecProfile::kAv1ProfileMain; + case 1: + return cdm::VideoCodecProfile::kAv1ProfileHigh; + case 2: + return cdm::VideoCodecProfile::kAv1ProfilePro; + default: + return cdm::VideoCodecProfile::kUnknownVideoCodecProfile; + } +} +#endif + RefPtr ChromiumCDMVideoDecoder::Init() { if (!mCDMParent) { // Must have failed to get the CDMParent from the ChromiumCDMProxy @@ -60,6 +78,16 @@ RefPtr ChromiumCDMVideoDecoder::Init() { ToCDMH264Profile(mConfig.mExtraData->SafeElementAt(1, 0)); config.mExtraData() = mConfig.mExtraData->Clone(); mConvertToAnnexB = true; +#ifdef MOZ_AV1 + } else if (AOMDecoder::IsAV1(mConfig.mMimeType)) { + AOMDecoder::AV1SequenceInfo seqInfo; + MediaResult seqHdrResult; + AOMDecoder::TryReadAV1CBox(mConfig.mExtraData, seqInfo, seqHdrResult); + config.mCodec() = cdm::VideoCodec::kCodecAv1; + config.mProfile() = NS_SUCCEEDED(seqHdrResult.Code()) + ? ToCDMAV1Profile(seqInfo.mProfile) + : cdm::VideoCodecProfile::kUnknownVideoCodecProfile; +#endif } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) { config.mCodec() = cdm::VideoCodec::kCodecVp8; config.mProfile() = cdm::VideoCodecProfile::kProfileNotNeeded; @@ -105,9 +133,16 @@ nsCString ChromiumCDMVideoDecoder::GetDescriptionName() const { nsCString ChromiumCDMVideoDecoder::GetCodecName() const { if (MP4Decoder::IsH264(mConfig.mMimeType)) { return "h264"_ns; - } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) { + } +#ifdef MOZ_AV1 + if (AOMDecoder::IsAV1(mConfig.mMimeType)) { + return "av1"_ns; + } +#endif + if (VPXDecoder::IsVP8(mConfig.mMimeType)) { return "vp8"_ns; - } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) { + } + if (VPXDecoder::IsVP9(mConfig.mMimeType)) { return "vp9"_ns; } return "unknown"_ns; diff --git a/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp b/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp index c143172073..a06dd30f89 100644 --- a/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp +++ b/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp @@ -28,7 +28,7 @@ namespace mozilla { -typedef MozPromiseRequestHolder DecryptPromiseRequestHolder; +using DecryptPromiseRequestHolder = MozPromiseRequestHolder; DDLoggedTypeDeclNameAndBase(EMEDecryptor, MediaDataDecoder); @@ -45,7 +45,7 @@ class ADTSSampleConverter { // doesn't care what is set. , mProfile(aInfo.mProfile < 1 || aInfo.mProfile > 4 ? 2 : aInfo.mProfile), - mFrequencyIndex(Adts::GetFrequencyIndex(aInfo.mRate)) { + mFrequencyIndex(ADTS::GetFrequencyIndex(aInfo.mRate).unwrapOr(255)) { EME_LOG("ADTSSampleConvertor(): aInfo.mProfile=%" PRIi8 " aInfo.mExtendedProfile=%" PRIi8, aInfo.mProfile, aInfo.mExtendedProfile); @@ -56,17 +56,17 @@ class ADTSSampleConverter { } } bool Convert(MediaRawData* aSample) const { - return Adts::ConvertSample(mNumChannels, mFrequencyIndex, mProfile, + return ADTS::ConvertSample(mNumChannels, mFrequencyIndex, mProfile, aSample); } bool Revert(MediaRawData* aSample) const { - return Adts::RevertSample(aSample); + return ADTS::RevertSample(aSample); } private: const uint32_t mNumChannels; const uint8_t mProfile; - const uint8_t mFrequencyIndex; + const uint8_t mFrequencyIndex{}; }; class EMEDecryptor final : public MediaDataDecoder, @@ -124,7 +124,7 @@ class EMEDecryptor final : public MediaDataDecoder, mThroughputLimiter->Throttle(aSample) ->Then( mThread, __func__, - [self](RefPtr aSample) { + [self](const RefPtr& aSample) { self->mThrottleRequest.Complete(); self->AttemptDecode(aSample); }, @@ -223,7 +223,7 @@ class EMEDecryptor final : public MediaDataDecoder, mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); mThroughputLimiter->Flush(); for (auto iter = mDecrypts.Iter(); !iter.Done(); iter.Next()) { - auto holder = iter.UserData(); + auto* holder = iter.UserData(); holder->DisconnectIfExists(); iter.Remove(); } @@ -240,7 +240,7 @@ class EMEDecryptor final : public MediaDataDecoder, MOZ_ASSERT(mDecodePromise.IsEmpty() && !mDecodeRequest.Exists(), "Must wait for decoding to complete"); for (auto iter = mDecrypts.Iter(); !iter.Done(); iter.Next()) { - auto holder = iter.UserData(); + auto* holder = iter.UserData(); holder->DisconnectIfExists(); iter.Remove(); } @@ -323,7 +323,7 @@ RefPtr EMEMediaDataDecoderProxy::Decode( mSamplesWaitingForKey->WaitIfKeyNotUsable(sample) ->Then( mThread, __func__, - [self, this](RefPtr aSample) { + [self, this](const RefPtr& aSample) { mKeyRequest.Complete(); MediaDataDecoderProxy::Decode(aSample) diff --git a/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp index f01c7e94e4..e9c41be1f0 100644 --- a/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp +++ b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp @@ -6,6 +6,9 @@ #include "GMPDecoderModule.h" +#ifdef MOZ_AV1 +# include "AOMDecoder.h" +#endif #include "DecoderDoctorDiagnostics.h" #include "GMPService.h" #include "GMPUtils.h" @@ -43,6 +46,9 @@ static already_AddRefed CreateDecoderWrapper( already_AddRefed GMPDecoderModule::CreateVideoDecoder( const CreateDecoderParams& aParams) { if (!MP4Decoder::IsH264(aParams.mConfig.mMimeType) && +#ifdef MOZ_AV1 + !AOMDecoder::IsAV1(aParams.mConfig.mMimeType) && +#endif !VPXDecoder::IsVP8(aParams.mConfig.mMimeType) && !VPXDecoder::IsVP9(aParams.mConfig.mMimeType)) { return nullptr; @@ -63,6 +69,10 @@ media::DecodeSupportSet GMPDecoderModule::SupportsMimeType( AutoTArray tags; if (MP4Decoder::IsH264(aMimeType)) { tags.AppendElement("h264"_ns); +#ifdef MOZ_AV1 + } else if (AOMDecoder::IsAV1(aMimeType)) { + tags.AppendElement("av1"_ns); +#endif } else if (VPXDecoder::IsVP9(aMimeType)) { tags.AppendElement("vp9"_ns); } else if (VPXDecoder::IsVP8(aMimeType)) { diff --git a/dom/media/platforms/apple/AppleATDecoder.cpp b/dom/media/platforms/apple/AppleATDecoder.cpp index ed64b62d60..3065ac0c27 100644 --- a/dom/media/platforms/apple/AppleATDecoder.cpp +++ b/dom/media/platforms/apple/AppleATDecoder.cpp @@ -14,6 +14,9 @@ #include "mozilla/SyncRunnable.h" #include "mozilla/UniquePtr.h" #include "nsTArray.h" +#include "ADTSDemuxer.h" + +#include #define LOG(...) DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, __VA_ARGS__) #define LOGEX(_this, ...) \ @@ -62,6 +65,7 @@ AppleATDecoder::~AppleATDecoder() { RefPtr AppleATDecoder::Init() { if (!mFormatID) { + LOG("AppleATDecoder::Init failure: unknown format ID"); return InitPromise::CreateAndReject( MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, RESULT_DETAIL("Non recognised format")), @@ -85,6 +89,7 @@ RefPtr AppleATDecoder::Flush() { } } if (mErrored) { + LOG("Flush error"); mParsedFramesForAACMagicCookie = 0; mMagicCookie.Clear(); ProcessShutdown(); @@ -188,18 +193,28 @@ RefPtr AppleATDecoder::Decode( MediaResult rv = NS_OK; if (!mConverter) { + LOG("Lazily initing the decoder"); rv = SetupDecoder(aSample); if (rv != NS_OK && rv != NS_ERROR_NOT_INITIALIZED) { + LOG("Decoder not initialized"); return DecodePromise::CreateAndReject(rv, __func__); } } + if (mIsADTS) { + bool rv = ADTS::StripHeader(aSample); + if (!rv) { + LOG("Stripping the ADTS header in AppleATDecoder failed"); + } + } + mQueuedSamples.AppendElement(aSample); if (rv == NS_OK) { for (size_t i = 0; i < mQueuedSamples.Length(); i++) { rv = DecodeSample(mQueuedSamples[i]); if (NS_FAILED(rv)) { + LOG("Decoding error"); mErrored = true; return DecodePromise::CreateAndReject(rv, __func__); } @@ -277,7 +292,7 @@ MediaResult AppleATDecoder::DecodeSample(MediaRawData* aSample) { } size_t numFrames = outputData.Length() / channels; - int rate = mOutputFormat.mSampleRate; + int rate = AssertedCast(mOutputFormat.mSampleRate); media::TimeUnit duration(numFrames, rate); if (!duration.IsValid()) { NS_WARNING("Invalid count of accumulated audio samples"); @@ -340,8 +355,8 @@ MediaResult AppleATDecoder::GetInputAudioDescription( aDesc.mChannelsPerFrame = mConfig.mChannels; aDesc.mSampleRate = mConfig.mRate; UInt32 inputFormatSize = sizeof(aDesc); - OSStatus rv = AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0, NULL, - &inputFormatSize, &aDesc); + OSStatus rv = AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0, + nullptr, &inputFormatSize, &aDesc); if (NS_WARN_IF(rv)) { return MediaResult( NS_ERROR_FAILURE, @@ -419,7 +434,7 @@ nsresult AppleATDecoder::SetupChannelLayout() { UInt32 propertySize; UInt32 size; OSStatus status = AudioConverterGetPropertyInfo( - mConverter, kAudioConverterOutputChannelLayout, &propertySize, NULL); + mConverter, kAudioConverterOutputChannelLayout, &propertySize, nullptr); if (status || !propertySize) { LOG("Couldn't get channel layout property (%s)", FourCC2Str(status)); return NS_ERROR_FAILURE; @@ -504,15 +519,36 @@ MediaResult AppleATDecoder::SetupDecoder(MediaRawData* aSample) { MOZ_ASSERT(mThread->IsOnCurrentThread()); static const uint32_t MAX_FRAMES = 2; + bool isADTS = + ADTS::FrameHeader::MatchesSync(Span{aSample->Data(), aSample->Size()}); + + if (isADTS) { + ADTS::FrameParser parser; + if (!parser.Parse(0, aSample->Data(), aSample->Data() + aSample->Size())) { + LOG("ADTS frame parsing error"); + return NS_ERROR_NOT_INITIALIZED; + } + + AudioCodecSpecificBinaryBlob blob; + ADTS::InitAudioSpecificConfig(parser.FirstFrame(), blob.mBinaryBlob); + mConfig.mCodecSpecificConfig = AudioCodecSpecificVariant{std::move(blob)}; + mConfig.mProfile = mConfig.mExtendedProfile = + parser.FirstFrame().Header().mObjectType; + mIsADTS = true; + } + if (mFormatID == kAudioFormatMPEG4AAC && mConfig.mExtendedProfile == 2 && mParsedFramesForAACMagicCookie < MAX_FRAMES) { + LOG("Attempting to get implicit AAC magic cookie"); // Check for implicit SBR signalling if stream is AAC-LC // This will provide us with an updated magic cookie for use with // GetInputAudioDescription. if (NS_SUCCEEDED(GetImplicitAACMagicCookie(aSample)) && - !mMagicCookie.Length()) { + !mMagicCookie.Length() && !isADTS) { // nothing found yet, will try again later + LOG("Getting implicit AAC magic cookie failed"); mParsedFramesForAACMagicCookie++; + LOG("Not initialized -- need magic cookie"); return NS_ERROR_NOT_INITIALIZED; } // An error occurred, fallback to using default stream description @@ -538,6 +574,7 @@ MediaResult AppleATDecoder::SetupDecoder(MediaRawData* aSample) { MediaResult rv = GetInputAudioDescription(inputFormat, magicCookie); if (NS_FAILED(rv)) { + LOG("GetInputAudioDescription failure"); return rv; } // Fill in the output format manually. @@ -617,28 +654,41 @@ static void _SampleCallback(void* aSBR, UInt32 aNumBytes, UInt32 aNumPackets, const void* aData, AudioStreamPacketDescription* aPackets) {} -nsresult AppleATDecoder::GetImplicitAACMagicCookie( - const MediaRawData* aSample) { +nsresult AppleATDecoder::GetImplicitAACMagicCookie(MediaRawData* aSample) { MOZ_ASSERT(mThread->IsOnCurrentThread()); - // Prepend ADTS header to AAC audio. - RefPtr adtssample(aSample->Clone()); - if (!adtssample) { - return NS_ERROR_OUT_OF_MEMORY; - } - int8_t frequency_index = Adts::GetFrequencyIndex(mConfig.mRate); + bool isADTS = + ADTS::FrameHeader::MatchesSync(Span{aSample->Data(), aSample->Size()}); - bool rv = Adts::ConvertSample(mConfig.mChannels, frequency_index, - mConfig.mProfile, adtssample); - if (!rv) { - NS_WARNING("Failed to apply ADTS header"); - return NS_ERROR_FAILURE; + RefPtr adtssample = aSample; + + if (!isADTS) { + // Prepend ADTS header to AAC audio. + adtssample = aSample->Clone(); + if (!adtssample) { + return NS_ERROR_OUT_OF_MEMORY; + } + auto frequency_index = ADTS::GetFrequencyIndex(mConfig.mRate); + + if (frequency_index.isErr()) { + LOG("%d isn't a valid rate for AAC", mConfig.mRate); + return NS_ERROR_FAILURE; + } + + // Arbitrarily pick main profile if not specified + int profile = mConfig.mProfile ? mConfig.mProfile : 1; + bool rv = ADTS::ConvertSample(mConfig.mChannels, frequency_index.unwrap(), + profile, adtssample); + if (!rv) { + LOG("Failed to apply ADTS header"); + return NS_ERROR_FAILURE; + } } if (!mStream) { OSStatus rv = AudioFileStreamOpen(this, _MetadataCallback, _SampleCallback, kAudioFileAAC_ADTSType, &mStream); if (rv) { - NS_WARNING("Couldn't open AudioFileStream"); + LOG("Couldn't open AudioFileStream"); return NS_ERROR_FAILURE; } } @@ -646,7 +696,7 @@ nsresult AppleATDecoder::GetImplicitAACMagicCookie( OSStatus status = AudioFileStreamParseBytes( mStream, adtssample->Size(), adtssample->Data(), 0 /* discontinuity */); if (status) { - NS_WARNING("Couldn't parse sample"); + LOG("Couldn't parse sample"); } if (status || mFileStreamError || mMagicCookie.Length()) { diff --git a/dom/media/platforms/apple/AppleATDecoder.h b/dom/media/platforms/apple/AppleATDecoder.h index d7aba2aacb..392b39993f 100644 --- a/dom/media/platforms/apple/AppleATDecoder.h +++ b/dom/media/platforms/apple/AppleATDecoder.h @@ -38,7 +38,7 @@ class AppleATDecoder final : public MediaDataDecoder, nsCString GetCodecName() const override; // Callbacks also need access to the config. - const AudioInfo mConfig; + AudioInfo mConfig; // Use to extract magic cookie for HE-AAC detection. nsTArray mMagicCookie; @@ -67,11 +67,12 @@ class AppleATDecoder final : public MediaDataDecoder, // Setup AudioConverter once all information required has been gathered. // Will return NS_ERROR_NOT_INITIALIZED if more data is required. MediaResult SetupDecoder(MediaRawData* aSample); - nsresult GetImplicitAACMagicCookie(const MediaRawData* aSample); + nsresult GetImplicitAACMagicCookie(MediaRawData* aSample); nsresult SetupChannelLayout(); uint32_t mParsedFramesForAACMagicCookie; uint32_t mEncoderDelay = 0; uint64_t mTotalMediaFrames = 0; + bool mIsADTS = false; bool mErrored; }; diff --git a/dom/media/platforms/apple/AppleDecoderModule.cpp b/dom/media/platforms/apple/AppleDecoderModule.cpp index 520685fff6..c54593a495 100644 --- a/dom/media/platforms/apple/AppleDecoderModule.cpp +++ b/dom/media/platforms/apple/AppleDecoderModule.cpp @@ -124,8 +124,7 @@ DecodeSupportSet AppleDecoderModule::Supports( case MediaCodec::VP8: [[fallthrough]]; case MediaCodec::VP9: - if (StaticPrefs::media_ffvpx_enabled() && - StaticPrefs::media_rdd_vpx_enabled() && + if (StaticPrefs::media_rdd_vpx_enabled() && StaticPrefs::media_utility_ffvpx_enabled()) { dss += DecodeSupport::SoftwareDecode; } @@ -233,6 +232,7 @@ bool AppleDecoderModule::CanCreateHWDecoder(MediaCodec aCodec) { /* static */ bool AppleDecoderModule::RegisterSupplementalVP9Decoder() { +#ifdef XP_MACOSX static bool sRegisterIfAvailable = []() { if (__builtin_available(macos 11.0, *)) { VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9); @@ -241,6 +241,9 @@ bool AppleDecoderModule::RegisterSupplementalVP9Decoder() { return false; }(); return sRegisterIfAvailable; +#else // iOS + return false; +#endif } /* static */ diff --git a/dom/media/platforms/apple/AppleVTDecoder.cpp b/dom/media/platforms/apple/AppleVTDecoder.cpp index aae9c1fc9b..ae34c2d142 100644 --- a/dom/media/platforms/apple/AppleVTDecoder.cpp +++ b/dom/media/platforms/apple/AppleVTDecoder.cpp @@ -7,7 +7,7 @@ #include "AppleVTDecoder.h" #include -#include +#include #include #include "AppleDecoderModule.h" @@ -486,7 +486,6 @@ void AppleVTDecoder::OutputFrame(CVPixelBufferRef aImage, // Unlock the returned image data. CVPixelBufferUnlockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly); } else { -#ifndef MOZ_WIDGET_UIKIT // Set pixel buffer properties on aImage before we extract its surface. // This ensures that we can use defined enums to set values instead // of later setting magic CFSTR values on the surface itself. @@ -535,9 +534,6 @@ void AppleVTDecoder::OutputFrame(CVPixelBufferRef aImage, info.mDisplay, aFrameRef.byte_offset, aFrameRef.composition_timestamp, aFrameRef.duration, image.forget(), aFrameRef.is_sync_point, aFrameRef.decode_timestamp); -#else - MOZ_ASSERT_UNREACHABLE("No MacIOSurface on iOS"); -#endif } if (!data) { @@ -719,7 +715,6 @@ CFDictionaryRef AppleVTDecoder::CreateOutputConfiguration() { &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); } -#ifndef MOZ_WIDGET_UIKIT // Output format type: bool is10Bit = (gfx::BitDepthForColorDepth(mColorDepth) == 10); @@ -754,9 +749,6 @@ CFDictionaryRef AppleVTDecoder::CreateOutputConfiguration() { return CFDictionaryCreate( kCFAllocatorDefault, outputKeys, outputValues, ArrayLength(outputKeys), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); -#else - MOZ_ASSERT_UNREACHABLE("No MacIOSurface on iOS"); -#endif } } // namespace mozilla diff --git a/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp b/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp index 43041f81ea..1e8e488e25 100644 --- a/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp @@ -164,7 +164,7 @@ static AlignedAudioBuffer CopyAndPackAudio(AVFrame* aFrame, int16_t* data = reinterpret_cast(aFrame->data)[0]; for (uint32_t frame = 0; frame < aNumAFrames; frame++) { for (uint32_t channel = 0; channel < aNumChannels; channel++) { - *tmp++ = AudioSampleToFloat(*data++); + *tmp++ = ConvertAudioSample(*data++); } } } else if (aFrame->format == AV_SAMPLE_FMT_S16P) { @@ -174,7 +174,7 @@ static AlignedAudioBuffer CopyAndPackAudio(AVFrame* aFrame, int16_t** data = reinterpret_cast(aFrame->data); for (uint32_t frame = 0; frame < aNumAFrames; frame++) { for (uint32_t channel = 0; channel < aNumChannels; channel++) { - *tmp++ = AudioSampleToFloat(data[channel][frame]); + *tmp++ = ConvertAudioSample(data[channel][frame]); } } } else if (aFrame->format == AV_SAMPLE_FMT_S32) { @@ -183,7 +183,7 @@ static AlignedAudioBuffer CopyAndPackAudio(AVFrame* aFrame, int32_t* data = reinterpret_cast(aFrame->data)[0]; for (uint32_t frame = 0; frame < aNumAFrames; frame++) { for (uint32_t channel = 0; channel < aNumChannels; channel++) { - *tmp++ = AudioSampleToFloat(*data++); + *tmp++ = ConvertAudioSample(*data++); } } } else if (aFrame->format == AV_SAMPLE_FMT_S32P) { @@ -193,7 +193,7 @@ static AlignedAudioBuffer CopyAndPackAudio(AVFrame* aFrame, int32_t** data = reinterpret_cast(aFrame->data); for (uint32_t frame = 0; frame < aNumAFrames; frame++) { for (uint32_t channel = 0; channel < aNumChannels; channel++) { - *tmp++ = AudioSampleToFloat(data[channel][frame]); + *tmp++ = ConvertAudioSample(data[channel][frame]); } } } else if (aFrame->format == AV_SAMPLE_FMT_U8) { @@ -202,7 +202,7 @@ static AlignedAudioBuffer CopyAndPackAudio(AVFrame* aFrame, uint8_t* data = reinterpret_cast(aFrame->data)[0]; for (uint32_t frame = 0; frame < aNumAFrames; frame++) { for (uint32_t channel = 0; channel < aNumChannels; channel++) { - *tmp++ = UInt8bitToAudioSample(*data++); + *tmp++ = ConvertAudioSample(*data++); } } } else if (aFrame->format == AV_SAMPLE_FMT_U8P) { @@ -212,7 +212,7 @@ static AlignedAudioBuffer CopyAndPackAudio(AVFrame* aFrame, uint8_t** data = reinterpret_cast(aFrame->data); for (uint32_t frame = 0; frame < aNumAFrames; frame++) { for (uint32_t channel = 0; channel < aNumChannels; channel++) { - *tmp++ = UInt8bitToAudioSample(data[channel][frame]); + *tmp++ = ConvertAudioSample(data[channel][frame]); } } } diff --git a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp index 4a30f2dd2d..bfb3105a57 100644 --- a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp @@ -177,9 +177,6 @@ FFmpegLibWrapper::LinkResult FFmpegLibWrapper::Link() { AV_FUNC(av_packet_alloc, (AV_FUNC_57 | AV_FUNC_58 | AV_FUNC_59 | AV_FUNC_60)) AV_FUNC(av_packet_unref, (AV_FUNC_57 | AV_FUNC_58 | AV_FUNC_59 | AV_FUNC_60)) AV_FUNC(av_packet_free, (AV_FUNC_57 | AV_FUNC_58 | AV_FUNC_59 | AV_FUNC_60)) - AV_FUNC_OPTION(av_rdft_init, AV_FUNC_AVCODEC_ALL) - AV_FUNC_OPTION(av_rdft_calc, AV_FUNC_AVCODEC_ALL) - AV_FUNC_OPTION(av_rdft_end, AV_FUNC_AVCODEC_ALL) AV_FUNC(avcodec_descriptor_get, AV_FUNC_AVCODEC_ALL) AV_FUNC(av_log_set_level, AV_FUNC_AVUTIL_ALL) AV_FUNC(av_malloc, AV_FUNC_AVUTIL_ALL) @@ -254,6 +251,10 @@ FFmpegLibWrapper::LinkResult FFmpegLibWrapper::Link() { AV_FUNC_AVUTIL_59 | AV_FUNC_AVUTIL_60) #endif + + AV_FUNC_OPTION(av_tx_init, AV_FUNC_AVUTIL_ALL) + AV_FUNC_OPTION(av_tx_uninit, AV_FUNC_AVUTIL_ALL) + #undef AV_FUNC #undef AV_FUNC_OPTION diff --git a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h index 98ab2f7930..eacbba286a 100644 --- a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h +++ b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h @@ -5,9 +5,9 @@ #ifndef __FFmpegLibWrapper_h__ #define __FFmpegLibWrapper_h__ -#include "FFmpegRDFTTypes.h" // for AvRdftInitFn, etc. #include "mozilla/Attributes.h" #include "mozilla/Types.h" +#include "ffvpx/tx.h" struct AVCodec; struct AVCodecContext; @@ -148,11 +148,6 @@ struct MOZ_ONLY_USED_TO_AVOID_STATIC_CONSTRUCTORS FFmpegLibWrapper { int (*avcodec_send_frame)(AVCodecContext* avctx, const AVFrame* frame); int (*avcodec_receive_frame)(AVCodecContext* avctx, AVFrame* frame); - // libavcodec optional - AvRdftInitFn av_rdft_init; - AvRdftCalcFn av_rdft_calc; - AvRdftEndFn av_rdft_end; - // libavutil void (*av_log_set_level)(int level); void* (*av_malloc)(size_t size); @@ -216,6 +211,10 @@ struct MOZ_ONLY_USED_TO_AVOID_STATIC_CONSTRUCTORS FFmpegLibWrapper { void* (*vaGetDisplayDRM)(int fd); #endif + // Only ever used with ffvpx + decltype(::av_tx_init)* av_tx_init; + decltype(::av_tx_uninit)* av_tx_uninit; + PRLibrary* mAVCodecLib; PRLibrary* mAVUtilLib; #ifdef MOZ_WIDGET_GTK diff --git a/dom/media/platforms/ffmpeg/FFmpegRDFTTypes.h b/dom/media/platforms/ffmpeg/FFmpegRDFTTypes.h deleted file mode 100644 index cb3e2476fb..0000000000 --- a/dom/media/platforms/ffmpeg/FFmpegRDFTTypes.h +++ /dev/null @@ -1,34 +0,0 @@ -/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=8 sts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ - -#ifndef FFmpegRDFTTypes_h -#define FFmpegRDFTTypes_h - -struct RDFTContext; - -typedef float FFTSample; - -enum RDFTransformType { - DFT_R2C, - IDFT_C2R, - IDFT_R2C, - DFT_C2R, -}; - -extern "C" { - -typedef RDFTContext* (*AvRdftInitFn)(int nbits, enum RDFTransformType trans); -typedef void (*AvRdftCalcFn)(RDFTContext* s, FFTSample* data); -typedef void (*AvRdftEndFn)(RDFTContext* s); -} - -struct FFmpegRDFTFuncs { - AvRdftInitFn init; - AvRdftCalcFn calc; - AvRdftEndFn end; -}; - -#endif // FFmpegRDFTTypes_h diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp b/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp index dcc3d9a88d..a3cfdf1b1d 100644 --- a/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp @@ -504,128 +504,64 @@ MediaResult FFmpegVideoEncoder::InitInternal() { mCodecContext->flags |= AV_CODEC_FLAG_FRAME_DURATION; #endif mCodecContext->gop_size = static_cast(mConfig.mKeyframeInterval); - // TODO (bug 1872871): Move the following extra settings to some helpers - // instead. + if (mConfig.mUsage == MediaDataEncoder::Usage::Realtime) { mLib->av_opt_set(mCodecContext->priv_data, "deadline", "realtime", 0); // Explicitly ask encoder do not keep in flight at any one time for // lookahead purposes. mLib->av_opt_set(mCodecContext->priv_data, "lag-in-frames", "0", 0); } - // Apply SVC settings. - if (Maybe svc = - GetVPXSVCSetting(mConfig.mScalabilityMode, mConfig.mBitrate)) { - // For libvpx. - if (mCodecName == "libvpx" || mCodecName == "libvpx-vp9") { - // Show a warning if mScalabilityMode mismatches mNumTemporalLayers - if (mConfig.mCodecSpecific) { - if (mConfig.mCodecSpecific->is() || - mConfig.mCodecSpecific->is()) { - const uint8_t numTemporalLayers = - mConfig.mCodecSpecific->is() - ? mConfig.mCodecSpecific->as().mNumTemporalLayers - : mConfig.mCodecSpecific->as() - .mNumTemporalLayers; - if (numTemporalLayers != svc->mNumberLayers) { - FFMPEGV_LOG( - "Force using %zu layers defined in scalability mode instead of " - "the %u layers defined in VP8/9Specific", - svc->mNumberLayers, numTemporalLayers); - } - } - } - // Set ts_layering_mode. - nsPrintfCString parameters("ts_layering_mode=%u", svc->mLayeringMode); - // Set ts_target_bitrate. - parameters.Append(":ts_target_bitrate="); - for (size_t i = 0; i < svc->mTargetBitrates.Length(); ++i) { - if (i > 0) { - parameters.Append(","); - } - parameters.AppendPrintf("%d", svc->mTargetBitrates[i]); - } - // TODO: Set ts_number_layers, ts_periodicity, ts_layer_id and - // ts_rate_decimator if they are different from the preset values in - // ts_layering_mode. - - // Set parameters into ts-parameters. - mLib->av_opt_set(mCodecContext->priv_data, "ts-parameters", - parameters.get(), 0); - - // FFmpegVideoEncoder would be reset after Drain(), so mSVCInfo should be - // reset() before emplace(). - mSVCInfo.reset(); - mSVCInfo.emplace(std::move(svc->mLayerIds)); - - // TODO: layer settings should be changed dynamically when the frame's - // color space changed. - } else { - FFMPEGV_LOG("SVC setting is not implemented for %s codec", - mCodecName.get()); - } + if (Maybe settings = GetSVCSettings()) { + SVCSettings s = settings.extract(); + mLib->av_opt_set(mCodecContext->priv_data, s.mSettingKeyValue.first.get(), + s.mSettingKeyValue.second.get(), 0); + + // FFmpegVideoEncoder is reset after Drain(), so mSVCInfo should be reset() + // before emplace(). + mSVCInfo.reset(); + mSVCInfo.emplace(std::move(s.mTemporalLayerIds)); + + // TODO: layer settings should be changed dynamically when the frame's + // color space changed. } - // Apply codec specific settings. - nsAutoCString codecSpecificLog; - if (mConfig.mCodecSpecific) { - if (mConfig.mCodecSpecific->is()) { - // For libx264. - if (mCodecName == "libx264") { - codecSpecificLog.Append(", H264:"); - - const H264Specific& specific = - mConfig.mCodecSpecific->as(); - - // Set profile. - Maybe profile = GetH264Profile(specific.mProfile); - if (!profile) { - FFMPEGV_LOG("failed to get h264 profile"); - return MediaResult(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, - RESULT_DETAIL("H264 profile is unknown")); - } - codecSpecificLog.Append( - nsPrintfCString(" profile - %d", profile->mValue)); - mCodecContext->profile = profile->mValue; - if (!profile->mString.IsEmpty()) { - codecSpecificLog.AppendPrintf(" (%s)", profile->mString.get()); - mLib->av_opt_set(mCodecContext->priv_data, "profile", - profile->mString.get(), 0); - } - - // Set level. - Maybe level = GetH264Level(specific.mLevel); - if (!level) { - FFMPEGV_LOG("failed to get h264 level"); - return MediaResult(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, - RESULT_DETAIL("H264 level is unknown")); - } - codecSpecificLog.AppendPrintf(", level %d (%s)", level->mValue, - level->mString.get()); - mCodecContext->level = level->mValue; - MOZ_ASSERT(!level->mString.IsEmpty()); - mLib->av_opt_set(mCodecContext->priv_data, "level", - level->mString.get(), 0); - - // Set format: libx264's default format is annexb - if (specific.mFormat == H264BitStreamFormat::AVC) { - codecSpecificLog.Append(", AVCC"); - mLib->av_opt_set(mCodecContext->priv_data, "x264-params", "annexb=0", - 0); - // mCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER - // if we don't want to append SPS/PPS data in all keyframe - // (LIBAVCODEC_VERSION_MAJOR >= 57 only). - } else { - codecSpecificLog.Append(", AnnexB"); - // Set annexb explicitly even if it's default format. - mLib->av_opt_set(mCodecContext->priv_data, "x264-params", "annexb=1", - 0); - } - } else { - FFMPEGV_LOG("H264 settings is not implemented for codec %s ", - mCodecName.get()); + + nsAutoCString h264Log; + if (mConfig.mCodecSpecific && mConfig.mCodecSpecific->is()) { + // TODO: Set profile, level, avcc/annexb for openh264 and others. + if (mCodecName == "libx264") { + const H264Specific& h264Specific = + mConfig.mCodecSpecific->as(); + H264Settings s = GetH264Settings(h264Specific); + mCodecContext->profile = s.mProfile; + mCodecContext->level = s.mLevel; + for (const auto& pair : s.mSettingKeyValuePairs) { + mLib->av_opt_set(mCodecContext->priv_data, pair.first.get(), + pair.second.get(), 0); } + + // Log the settings. + // When using profile other than EXTENDED, the profile string is in the + // first element of mSettingKeyValuePairs, while EXTENDED profile has no + // profile string. + + MOZ_ASSERT_IF( + s.mSettingKeyValuePairs.Length() != 3, + h264Specific.mProfile == H264_PROFILE::H264_PROFILE_EXTENDED); + const char* profileStr = s.mSettingKeyValuePairs.Length() == 3 + ? s.mSettingKeyValuePairs[0].second.get() + : "extended"; + const char* levelStr = s.mSettingKeyValuePairs.Length() == 3 + ? s.mSettingKeyValuePairs[1].second.get() + : s.mSettingKeyValuePairs[0].second.get(); + const char* formatStr = + h264Specific.mFormat == H264BitStreamFormat::AVC ? "AVCC" : "AnnexB"; + h264Log.AppendPrintf(", H264: profile - %d (%s), level %d (%s), %s", + mCodecContext->profile, profileStr, + mCodecContext->level, levelStr, formatStr); } } + // TODO: keyint_min, max_b_frame? // - if mConfig.mDenoising is set: av_opt_set_int(mCodecContext->priv_data, // "noise_sensitivity", x, 0), where the x is from 0(disabled) to 6. @@ -657,7 +593,7 @@ MediaResult FFmpegVideoEncoder::InitInternal() { static_cast(mCodecContext->bit_rate), mCodecContext->width, mCodecContext->height, mCodecContext->time_base.num, mCodecContext->time_base.den, - codecSpecificLog.IsEmpty() ? "" : codecSpecificLog.get()); + h264Log.IsEmpty() ? "" : h264Log.get()); return MediaResult(NS_OK); } @@ -1152,4 +1088,99 @@ void FFmpegVideoEncoder::ForceEnablingFFmpegDebugLogs() { #endif // DEBUG } +Maybe::SVCSettings> +FFmpegVideoEncoder::GetSVCSettings() { + MOZ_ASSERT(!mCodecName.IsEmpty()); + + // TODO: Add support for AV1 and H264. + if (mCodecName != "libvpx" && mCodecName != "libvpx-vp9") { + FFMPEGV_LOG("SVC setting is not implemented for %s codec", + mCodecName.get()); + return Nothing(); + } + + Maybe svc = + GetVPXSVCSetting(mConfig.mScalabilityMode, mConfig.mBitrate); + if (!svc) { + FFMPEGV_LOG("No SVC settings obtained. Skip"); + return Nothing(); + } + + // Check if the number of temporal layers in codec specific settings matches + // the number of layers for the given scalability mode. + + auto GetNumTemporalLayers = [&]() -> uint8_t { + uint8_t layers = 0; + if (mConfig.mCodecSpecific) { + if (mConfig.mCodecSpecific->is()) { + layers = mConfig.mCodecSpecific->as().mNumTemporalLayers; + MOZ_ASSERT(layers > 0); + } else if (mConfig.mCodecSpecific->is()) { + layers = mConfig.mCodecSpecific->as().mNumTemporalLayers; + MOZ_ASSERT(layers > 0); + } + } + return layers; + }; + + DebugOnly numTemporalLayers = GetNumTemporalLayers(); + MOZ_ASSERT_IF(numTemporalLayers > 0, numTemporalLayers == svc->mNumberLayers); + + // Form an SVC setting string for libvpx. + + nsPrintfCString parameters("ts_layering_mode=%u", svc->mLayeringMode); + parameters.Append(":ts_target_bitrate="); + for (size_t i = 0; i < svc->mTargetBitrates.Length(); ++i) { + if (i > 0) { + parameters.Append(","); + } + parameters.AppendPrintf("%d", svc->mTargetBitrates[i]); + } + + // TODO: Set ts_number_layers, ts_periodicity, ts_layer_id and + // ts_rate_decimator if they are different from the preset values in + // ts_layering_mode. + + return Some( + SVCSettings{std::move(svc->mLayerIds), + std::make_pair("ts-parameters"_ns, std::move(parameters))}); +} + +FFmpegVideoEncoder::H264Settings FFmpegVideoEncoder< + LIBAV_VER>::GetH264Settings(const H264Specific& aH264Specific) { + MOZ_ASSERT(mCodecName == "libx264", + "GetH264Settings is libx264-only for now"); + + nsTArray> keyValuePairs; + + Maybe profile = GetH264Profile(aH264Specific.mProfile); + MOZ_RELEASE_ASSERT(profile.isSome()); + if (!profile->mString.IsEmpty()) { + keyValuePairs.AppendElement(std::make_pair("profile"_ns, profile->mString)); + } else { + MOZ_RELEASE_ASSERT(aH264Specific.mProfile == + H264_PROFILE::H264_PROFILE_EXTENDED); + } + + Maybe level = GetH264Level(aH264Specific.mLevel); + MOZ_RELEASE_ASSERT(level.isSome()); + MOZ_RELEASE_ASSERT(!level->mString.IsEmpty()); + keyValuePairs.AppendElement(std::make_pair("level"_ns, level->mString)); + + // Set format: libx264's default format is annexb. + if (aH264Specific.mFormat == H264BitStreamFormat::AVC) { + keyValuePairs.AppendElement(std::make_pair("x264-params"_ns, "annexb=0")); + // mCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER + // if we don't want to append SPS/PPS data in all keyframe + // (LIBAVCODEC_VERSION_MAJOR >= 57 only). + } else { + // Set annexb explicitly even if it's default format. + keyValuePairs.AppendElement(std::make_pair("x264-params"_ns, "annexb=1")); + } + + return H264Settings{.mProfile = profile->mValue, + .mLevel = level->mValue, + .mSettingKeyValuePairs = std::move(keyValuePairs)}; +} + } // namespace mozilla diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h b/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h index 1bcdd3eaf9..07c433ddd7 100644 --- a/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h +++ b/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h @@ -75,6 +75,19 @@ class FFmpegVideoEncoder final : public MediaDataEncoder { Result, nsresult> GetExtraData( AVPacket* aPacket); void ForceEnablingFFmpegDebugLogs(); + struct SVCSettings { + nsTArray mTemporalLayerIds; + // A key-value pair for av_opt_set. + std::pair mSettingKeyValue; + }; + Maybe GetSVCSettings(); + struct H264Settings { + int mProfile; + int mLevel; + // A list of key-value pairs for av_opt_set. + nsTArray> mSettingKeyValuePairs; + }; + H264Settings GetH264Settings(const H264Specific& aH264Specific); // This refers to a static FFmpegLibWrapper, so raw pointer is adequate. const FFmpegLibWrapper* mLib; diff --git a/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp b/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp index ba9ca4834e..dfc8244f1d 100644 --- a/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp +++ b/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp @@ -145,19 +145,13 @@ already_AddRefed FFVPXRuntimeLinker::CreateEncoder() { } /* static */ -void FFVPXRuntimeLinker::GetRDFTFuncs(FFmpegRDFTFuncs* aOutFuncs) { +void FFVPXRuntimeLinker::GetFFTFuncs(FFmpegFFTFuncs* aOutFuncs) { []() MOZ_NO_THREAD_SAFETY_ANALYSIS { MOZ_ASSERT(sLinkStatus != LinkStatus_INIT); }(); - if (sFFVPXLib.av_rdft_init && sFFVPXLib.av_rdft_calc && - sFFVPXLib.av_rdft_end) { - aOutFuncs->init = sFFVPXLib.av_rdft_init; - aOutFuncs->calc = sFFVPXLib.av_rdft_calc; - aOutFuncs->end = sFFVPXLib.av_rdft_end; - } else { - NS_WARNING("RDFT functions expected but not found"); - *aOutFuncs = FFmpegRDFTFuncs(); // zero - } + MOZ_ASSERT(sFFVPXLib.av_tx_init && sFFVPXLib.av_tx_uninit); + aOutFuncs->init = sFFVPXLib.av_tx_init; + aOutFuncs->uninit = sFFVPXLib.av_tx_uninit; } } // namespace mozilla diff --git a/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.h b/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.h index e52f108272..dccd37c7da 100644 --- a/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.h +++ b/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.h @@ -11,8 +11,12 @@ #include "PlatformEncoderModule.h" #include "mozilla/StaticMutex.h" #include "mozilla/ThreadSafety.h" +#include "ffvpx/tx.h" -struct FFmpegRDFTFuncs; +struct FFmpegFFTFuncs { + decltype(av_tx_init)* init; + decltype(av_tx_uninit)* uninit; +}; namespace mozilla { @@ -23,7 +27,7 @@ class FFVPXRuntimeLinker { static already_AddRefed CreateEncoder(); // Call (on any thread) after Init(). - static void GetRDFTFuncs(FFmpegRDFTFuncs* aOutFuncs); + static void GetFFTFuncs(FFmpegFFTFuncs* aOutFuncs); private: // Provide critical-section for Init() and sLinkStatus. diff --git a/dom/media/platforms/moz.build b/dom/media/platforms/moz.build index 34acf4e8d1..6f71c5cc12 100644 --- a/dom/media/platforms/moz.build +++ b/dom/media/platforms/moz.build @@ -47,19 +47,13 @@ DIRS += ["agnostic/bytestreams", "agnostic/eme", "agnostic/gmp", "omx"] if CONFIG["MOZ_WMF"]: DIRS += ["wmf"] -if CONFIG["MOZ_FFVPX"] or CONFIG["MOZ_FFMPEG"]: - # common code to either FFmpeg or FFVPX - EXPORTS += [ - "ffmpeg/FFmpegRDFTTypes.h", - ] - UNIFIED_SOURCES += [ - "ffmpeg/FFmpegLibWrapper.cpp", - ] +UNIFIED_SOURCES += [ + "ffmpeg/FFmpegLibWrapper.cpp", +] -if CONFIG["MOZ_FFVPX"]: - DIRS += [ - "ffmpeg/ffvpx", - ] +DIRS += [ + "ffmpeg/ffvpx", +] if CONFIG["MOZ_FFMPEG"]: DIRS += [ diff --git a/dom/media/platforms/wmf/MFCDMSession.cpp b/dom/media/platforms/wmf/MFCDMSession.cpp index cec783cbc6..b797898abb 100644 --- a/dom/media/platforms/wmf/MFCDMSession.cpp +++ b/dom/media/platforms/wmf/MFCDMSession.cpp @@ -304,8 +304,7 @@ void MFCDMSession::OnSessionKeyMessage( case MF_MEDIAKEYSESSION_MESSAGETYPE_INDIVIDUALIZATION_REQUEST: return dom::MediaKeyMessageType::Individualization_request; default: - MOZ_ASSERT_UNREACHABLE("Unknown session message type"); - return dom::MediaKeyMessageType::EndGuard_; + MOZ_CRASH("Unknown session message type"); } }; LOG("Notify 'keymessage' for %s", NS_ConvertUTF16toUTF8(*mSessionId).get()); diff --git a/dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp b/dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp index 4acf26e041..969f817882 100644 --- a/dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp +++ b/dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp @@ -93,7 +93,7 @@ HRESULT MFMediaEngineAudioStream::CreateMediaType(const TrackInfo& aInfo, bool MFMediaEngineAudioStream::HasEnoughRawData() const { // If more than this much raw audio is queued, we'll hold off request more // audio. - return mRawDataQueueForFeedingEngine.Duration() >= + return mRawDataQueueForFeedingEngine.PreciseDuration() >= StaticPrefs::media_wmf_media_engine_raw_data_threshold_audio(); } diff --git a/dom/media/platforms/wmf/MFMediaEngineDecoderModule.cpp b/dom/media/platforms/wmf/MFMediaEngineDecoderModule.cpp index 5b99fb0f2c..e291ab6a54 100644 --- a/dom/media/platforms/wmf/MFMediaEngineDecoderModule.cpp +++ b/dom/media/platforms/wmf/MFMediaEngineDecoderModule.cpp @@ -6,6 +6,7 @@ #include "MFTDecoder.h" #include "VideoUtils.h" +#include "mozilla/gfx/gfxVars.h" #include "mozilla/MFMediaEngineParent.h" #include "mozilla/MFMediaEngineUtils.h" #include "mozilla/RemoteDecoderManagerChild.h" @@ -99,6 +100,11 @@ media::DecodeSupportSet MFMediaEngineDecoderModule::SupportInternal( if (!StaticPrefs::media_wmf_media_engine_enabled()) { return media::DecodeSupportSet{}; } + // Only support hardware decoding. + if (!gfx::gfxVars::CanUseHardwareVideoDecoding() && + !StaticPrefs::media_wmf_media_engine_bypass_gfx_blocklist()) { + return media::DecodeSupportSet{}; + } bool supports = false; WMFStreamType type = GetStreamTypeFromMimeType(aParams.MimeType()); if (type != WMFStreamType::Unknown) { @@ -107,13 +113,11 @@ media::DecodeSupportSet MFMediaEngineDecoderModule::SupportInternal( MOZ_LOG(sPDMLog, LogLevel::Debug, ("MFMediaEngine decoder %s requested type '%s'", supports ? "supports" : "rejects", aParams.MimeType().get())); - // We only support HEVC hardware decoding. - if (supports && type == WMFStreamType::HEVC) { - return media::DecodeSupport::HardwareDecode; + if (!supports) { + return media::DecodeSupportSet{}; } - // TODO : find a way to report accurate result. - return supports ? media::DecodeSupport::SoftwareDecode - : media::DecodeSupportSet{}; + return StreamTypeIsVideo(type) ? media::DecodeSupport::HardwareDecode + : media::DecodeSupport::SoftwareDecode; } static bool CreateMFTDecoderOnMTA(const WMFStreamType& aType) { diff --git a/dom/media/platforms/wmf/MFMediaEngineDecoderModule.h b/dom/media/platforms/wmf/MFMediaEngineDecoderModule.h index c23b9010cc..1c8de5a161 100644 --- a/dom/media/platforms/wmf/MFMediaEngineDecoderModule.h +++ b/dom/media/platforms/wmf/MFMediaEngineDecoderModule.h @@ -10,6 +10,8 @@ namespace mozilla { +// MFMediaEngineDecoderModule is used for the media engine playback, which only +// supports hardware decoding. class MFMediaEngineDecoderModule final : public PlatformDecoderModule { public: static void Init(); diff --git a/dom/media/platforms/wmf/MFMediaEngineStream.cpp b/dom/media/platforms/wmf/MFMediaEngineStream.cpp index 6dce37ee35..70ffa50142 100644 --- a/dom/media/platforms/wmf/MFMediaEngineStream.cpp +++ b/dom/media/platforms/wmf/MFMediaEngineStream.cpp @@ -107,7 +107,11 @@ MFMediaEngineStreamWrapper::NeedsConversion() const { } MFMediaEngineStream::MFMediaEngineStream() - : mIsShutdown(false), mIsSelected(false), mReceivedEOS(false) { + : mIsShutdown(false), + mIsSelected(false), + mRawDataQueueForFeedingEngine(true /* aEnablePreciseDuration */), + mRawDataQueueForGeneratingOutput(true /* aEnablePreciseDuration */), + mReceivedEOS(false) { MOZ_COUNT_CTOR(MFMediaEngineStream); } @@ -282,17 +286,8 @@ void MFMediaEngineStream::ReplySampleRequestIfPossible() { while (!mSampleRequestTokens.empty()) { mSampleRequestTokens.pop(); } - - SLOG("Notify end events"); - MOZ_ASSERT(mRawDataQueueForFeedingEngine.GetSize() == 0); MOZ_ASSERT(mSampleRequestTokens.empty()); - RETURN_VOID_IF_FAILED(mMediaEventQueue->QueueEventParamUnk( - MEEndOfStream, GUID_NULL, S_OK, nullptr)); - mEndedEvent.Notify(TrackType()); - PROFILER_MARKER_TEXT( - "MFMediaEngineStream:NotifyEnd", MEDIA_PLAYBACK, {}, - nsPrintfCString("stream=%s, id=%" PRIu64, GetDescriptionName().get(), - mStreamId)); + NotifyEndEvent(); return; } @@ -318,6 +313,18 @@ void MFMediaEngineStream::ReplySampleRequestIfPossible() { MEMediaSample, GUID_NULL, S_OK, inputSample.Get())); } +void MFMediaEngineStream::NotifyEndEvent() { + AssertOnTaskQueue(); + SLOG("Notify end event"); + MOZ_ASSERT(mRawDataQueueForFeedingEngine.GetSize() == 0); + RETURN_VOID_IF_FAILED(mMediaEventQueue->QueueEventParamUnk( + MEEndOfStream, GUID_NULL, S_OK, nullptr)); + mEndedEvent.Notify(TrackType()); + PROFILER_MARKER_TEXT("MFMediaEngineStream:NotifyEnd", MEDIA_PLAYBACK, {}, + nsPrintfCString("stream=%s, id=%" PRIu64, + GetDescriptionName().get(), mStreamId)); +} + bool MFMediaEngineStream::ShouldServeSamples() const { AssertOnTaskQueue(); return mParentSource && @@ -486,7 +493,7 @@ void MFMediaEngineStream::NotifyNewData(MediaRawData* aSample) { "], queue size=%zu, queue duration=%" PRId64, aSample->mTime.ToMicroseconds(), aSample->GetEndTime().ToMicroseconds(), mRawDataQueueForFeedingEngine.GetSize(), - mRawDataQueueForFeedingEngine.Duration()); + mRawDataQueueForFeedingEngine.PreciseDuration()); if (mReceivedEOS) { SLOG("Receive a new data, cancel old EOS flag"); mReceivedEOS = false; @@ -501,7 +508,7 @@ void MFMediaEngineStream::SendRequestSampleEvent(bool aIsEnough) { AssertOnTaskQueue(); SLOGV("data is %s, queue duration=%" PRId64, aIsEnough ? "enough" : "not enough", - mRawDataQueueForFeedingEngine.Duration()); + mRawDataQueueForFeedingEngine.PreciseDuration()); mParentSource->mRequestSampleEvent.Notify( SampleRequest{TrackType(), aIsEnough}); } diff --git a/dom/media/platforms/wmf/MFMediaEngineStream.h b/dom/media/platforms/wmf/MFMediaEngineStream.h index aa3bf7e65d..e11d900498 100644 --- a/dom/media/platforms/wmf/MFMediaEngineStream.h +++ b/dom/media/platforms/wmf/MFMediaEngineStream.h @@ -84,7 +84,7 @@ class MFMediaEngineStream // Return the type of the track, the result should be either audio or video. virtual TrackInfo::TrackType TrackType() = 0; - RefPtr Flush(); + virtual RefPtr Flush(); MediaEventProducer& EndedEvent() { return mEndedEvent; } @@ -93,7 +93,7 @@ class MFMediaEngineStream virtual MFMediaEngineVideoStream* AsVideoStream() { return nullptr; } - RefPtr OutputData( + virtual RefPtr OutputData( RefPtr aSample); virtual RefPtr Drain(); @@ -133,11 +133,13 @@ class MFMediaEngineStream // should uses `mRawDataQueueForGeneratingOutput` to generate output. virtual already_AddRefed OutputDataInternal() = 0; - void SendRequestSampleEvent(bool aIsEnough); + virtual void SendRequestSampleEvent(bool aIsEnough); HRESULT AddEncryptAttributes(IMFSample* aSample, const CryptoSample& aCryptoConfig); + void NotifyEndEvent(); + void AssertOnTaskQueue() const; void AssertOnMFThreadPool() const; diff --git a/dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp b/dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp index ca043478f0..0fedcd31b9 100644 --- a/dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp +++ b/dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp @@ -49,7 +49,7 @@ void MFMediaEngineVideoStream::SetKnowsCompositor( this]() { mKnowsCompositor = knowCompositor; LOG("Set SetKnowsCompositor=%p", mKnowsCompositor.get()); - ResolvePendingDrainPromiseIfNeeded(); + ResolvePendingPromisesIfNeeded(); })); } @@ -74,7 +74,7 @@ void MFMediaEngineVideoStream::SetDCompSurfaceHandle(HANDLE aDCompSurfaceHandle, } } LOG("Set DCompSurfaceHandle, handle=%p", mDCompSurfaceHandle); - ResolvePendingDrainPromiseIfNeeded(); + ResolvePendingPromisesIfNeeded(); })); } @@ -209,7 +209,7 @@ HRESULT MFMediaEngineVideoStream::CreateMediaType(const TrackInfo& aInfo, bool MFMediaEngineVideoStream::HasEnoughRawData() const { // If more than this much raw video is queued, we'll hold off request more // video. - return mRawDataQueueForFeedingEngine.Duration() >= + return mRawDataQueueForFeedingEngine.PreciseDuration() >= StaticPrefs::media_wmf_media_engine_raw_data_threshold_video(); } @@ -240,6 +240,32 @@ bool MFMediaEngineVideoStream::IsDCompImageReady() { return true; } +RefPtr MFMediaEngineVideoStream::OutputData( + RefPtr aSample) { + if (IsShutdown()) { + return MediaDataDecoder::DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_FAILURE, + RESULT_DETAIL("MFMediaEngineStream is shutdown")), + __func__); + } + AssertOnTaskQueue(); + NotifyNewData(aSample); + MediaDataDecoder::DecodedData outputs; + if (RefPtr outputData = OutputDataInternal()) { + outputs.AppendElement(outputData); + LOGV("Output data [%" PRId64 ",%" PRId64 "]", + outputData->mTime.ToMicroseconds(), + outputData->GetEndTime().ToMicroseconds()); + } + if (ShouldDelayVideoDecodeBeforeDcompReady()) { + LOG("Dcomp isn't ready and we already have enough video data. We will send " + "them back together at one when Dcomp is ready"); + return mVideoDecodeBeforeDcompPromise.Ensure(__func__); + } + return MediaDataDecoder::DecodePromise::CreateAndResolve(std::move(outputs), + __func__); +} + already_AddRefed MFMediaEngineVideoStream::OutputDataInternal() { AssertOnTaskQueue(); if (mRawDataQueueForGeneratingOutput.GetSize() == 0 || !IsDCompImageReady()) { @@ -261,28 +287,62 @@ RefPtr MFMediaEngineVideoStream::Drain() { MediaDataDecoder::DecodedData outputs; if (!IsDCompImageReady()) { LOGV("Waiting for dcomp image for draining"); + // A workaround for a special case where we have sent all input data to the + // media engine, and waiting for an output. Sometime media engine would + // never return the first frame to us, unless we notify it the end event, + // which happens on the case where the video only contains one frame. If we + // don't send end event to the media engine, the drain promise would be + // pending forever. + if (!mSampleRequestTokens.empty() && + mRawDataQueueForFeedingEngine.GetSize() == 0) { + NotifyEndEvent(); + } return mPendingDrainPromise.Ensure(__func__); } return MFMediaEngineStream::Drain(); } -void MFMediaEngineVideoStream::ResolvePendingDrainPromiseIfNeeded() { +RefPtr MFMediaEngineVideoStream::Flush() { + AssertOnTaskQueue(); + auto promise = MFMediaEngineStream::Flush(); + mPendingDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mVideoDecodeBeforeDcompPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, + __func__); + return promise; +} + +void MFMediaEngineVideoStream::ResolvePendingPromisesIfNeeded() { AssertOnTaskQueue(); - if (mPendingDrainPromise.IsEmpty()) { - return; - } if (!IsDCompImageReady()) { return; } - MediaDataDecoder::DecodedData outputs; - while (RefPtr outputData = OutputDataInternal()) { - outputs.AppendElement(outputData); - LOGV("Output data [%" PRId64 ",%" PRId64 "]", - outputData->mTime.ToMicroseconds(), - outputData->GetEndTime().ToMicroseconds()); + + // Resolve decoding promise first, then drain promise + if (!mVideoDecodeBeforeDcompPromise.IsEmpty()) { + MediaDataDecoder::DecodedData outputs; + while (RefPtr outputData = OutputDataInternal()) { + outputs.AppendElement(outputData); + LOGV("Output data [%" PRId64 ",%" PRId64 "]", + outputData->mTime.ToMicroseconds(), + outputData->GetEndTime().ToMicroseconds()); + } + mVideoDecodeBeforeDcompPromise.Resolve(std::move(outputs), __func__); + LOG("Resolved video decode before Dcomp promise"); + } + + // This drain promise could return no data, if all data has been processed in + // the decoding promise. + if (!mPendingDrainPromise.IsEmpty()) { + MediaDataDecoder::DecodedData outputs; + while (RefPtr outputData = OutputDataInternal()) { + outputs.AppendElement(outputData); + LOGV("Output data [%" PRId64 ",%" PRId64 "]", + outputData->mTime.ToMicroseconds(), + outputData->GetEndTime().ToMicroseconds()); + } + mPendingDrainPromise.Resolve(std::move(outputs), __func__); + LOG("Resolved pending drain promise"); } - mPendingDrainPromise.Resolve(std::move(outputs), __func__); - LOG("Resolved pending drain promise"); } MediaDataDecoder::ConversionRequired MFMediaEngineVideoStream::NeedsConversion() @@ -336,6 +396,20 @@ void MFMediaEngineVideoStream::UpdateConfig(const VideoInfo& aInfo) { void MFMediaEngineVideoStream::ShutdownCleanUpOnTaskQueue() { AssertOnTaskQueue(); mPendingDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mVideoDecodeBeforeDcompPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, + __func__); +} + +void MFMediaEngineVideoStream::SendRequestSampleEvent(bool aIsEnough) { + AssertOnTaskQueue(); + MFMediaEngineStream::SendRequestSampleEvent(aIsEnough); + // We need more data to be sent in, we should resolve the promise to allow + // more input data to be sent. + if (!aIsEnough && !mVideoDecodeBeforeDcompPromise.IsEmpty()) { + LOG("Resolved pending input promise to allow more input be sent in"); + mVideoDecodeBeforeDcompPromise.Resolve(MediaDataDecoder::DecodedData{}, + __func__); + } } bool MFMediaEngineVideoStream::IsEnded() const { @@ -352,6 +426,10 @@ bool MFMediaEngineVideoStream::IsEnded() const { bool MFMediaEngineVideoStream::IsEncrypted() const { return mIsEncrypted; } +bool MFMediaEngineVideoStream::ShouldDelayVideoDecodeBeforeDcompReady() { + return HasEnoughRawData() && !IsDCompImageReady(); +} + nsCString MFMediaEngineVideoStream::GetCodecName() const { switch (mStreamType) { case WMFStreamType::H264: diff --git a/dom/media/platforms/wmf/MFMediaEngineVideoStream.h b/dom/media/platforms/wmf/MFMediaEngineVideoStream.h index df17c264e4..51fbe4876b 100644 --- a/dom/media/platforms/wmf/MFMediaEngineVideoStream.h +++ b/dom/media/platforms/wmf/MFMediaEngineVideoStream.h @@ -19,6 +19,7 @@ class DcompSurfaceImage; } // namespace layers class MFMediaSource; +class MediaRawData; class MFMediaEngineVideoStream final : public MFMediaEngineStream { public: @@ -50,8 +51,13 @@ class MFMediaEngineVideoStream final : public MFMediaEngineStream { // change happens during playback. void SetConfig(const TrackInfo& aConfig); + RefPtr OutputData( + RefPtr aSample) override; + RefPtr Drain() override; + RefPtr Flush() override; + bool IsEncrypted() const override; private: @@ -66,12 +72,25 @@ class MFMediaEngineVideoStream final : public MFMediaEngineStream { bool IsDCompImageReady(); - void ResolvePendingDrainPromiseIfNeeded(); + // Those promises are used to handle decode/drain which happens before the + // Dcomp surface is ready. + void ResolvePendingPromisesIfNeeded(); void ShutdownCleanUpOnTaskQueue() override; bool IsEnded() const override; + // Before Dcomp surface is ready, we can't return any video data due to + // lacking of the image, which should only happen on the beginning of the + // video playback. In that situation, once we have enough video raw data, we + // can stop delaying the decode promise by waiting the Dcomp surface and + // resolveing the promise when Dcomp surface is ready. Doing so helps to keep + // the decode promise pending, so that the MFR won't keep sending more input + // data, which we actually don't need that many. + bool ShouldDelayVideoDecodeBeforeDcompReady(); + + void SendRequestSampleEvent(bool aIsEnough) override; + // Task queue only members. HANDLE mDCompSurfaceHandle; bool mNeedRecreateImage; @@ -98,6 +117,12 @@ class MFMediaEngineVideoStream final : public MFMediaEngineStream { // have dcomp image. MozPromiseHolder mPendingDrainPromise; + // The promise used to return all video output which are requested before the + // Dcomp surface is ready. This should only be used once in entire playback, + // typically happening around the beginning of the playback. + MozPromiseHolder + mVideoDecodeBeforeDcompPromise; + // Set when `CreateMediaType()` is called. bool mIsEncrypted = false; }; diff --git a/dom/media/platforms/wmf/MFMediaSource.h b/dom/media/platforms/wmf/MFMediaSource.h index 735d53579e..0e44ef12aa 100644 --- a/dom/media/platforms/wmf/MFMediaSource.h +++ b/dom/media/platforms/wmf/MFMediaSource.h @@ -132,8 +132,6 @@ class MFMediaSource : public Microsoft::WRL::RuntimeClass< void AssertOnManagerThread() const; void AssertOnMFThreadPool() const; - void NotifyEndOfStreamInternal(TrackInfo::TrackType aType); - bool IsSeekable() const; // A thread-safe event queue. diff --git a/dom/media/platforms/wmf/WMFAudioMFTManager.cpp b/dom/media/platforms/wmf/WMFAudioMFTManager.cpp index 6ebcf9a80a..63db5efae8 100644 --- a/dom/media/platforms/wmf/WMFAudioMFTManager.cpp +++ b/dom/media/platforms/wmf/WMFAudioMFTManager.cpp @@ -55,6 +55,9 @@ WMFAudioMFTManager::WMFAudioMFTManager(const AudioInfo& aConfig) audioSpecConfig = audioCodecSpecificBinaryBlob->Elements(); configLength = audioCodecSpecificBinaryBlob->Length(); } + // If no extradata has been provided, assume this is ADTS. Otherwise, + // assume raw AAC packets. + mIsADTS = !configLength; AACAudioSpecificConfigToUserData(aConfig.mExtendedProfile, audioSpecConfig, configLength, mUserData); } @@ -104,7 +107,8 @@ bool WMFAudioMFTManager::Init() { NS_ENSURE_TRUE(SUCCEEDED(hr), false); if (mStreamType == WMFStreamType::AAC) { - hr = inputType->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, 0x0); // Raw AAC packet + UINT32 payloadType = mIsADTS ? 1 : 0; + hr = inputType->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, payloadType); NS_ENSURE_TRUE(SUCCEEDED(hr), false); hr = inputType->SetBlob(MF_MT_USER_DATA, mUserData.Elements(), @@ -144,7 +148,8 @@ WMFAudioMFTManager::Input(MediaRawData* aSample) { nsCString WMFAudioMFTManager::GetCodecName() const { if (mStreamType == WMFStreamType::AAC) { return "aac"_ns; - } else if (mStreamType == WMFStreamType::MP3) { + } + if (mStreamType == WMFStreamType::MP3) { return "mp3"_ns; } return "unknown"_ns; @@ -177,8 +182,8 @@ WMFAudioMFTManager::UpdateOutputType() { } HRESULT -WMFAudioMFTManager::Output(int64_t aStreamOffset, RefPtr& aOutData) { - aOutData = nullptr; +WMFAudioMFTManager::Output(int64_t aStreamOffset, RefPtr& aOutput) { + aOutput = nullptr; RefPtr sample; HRESULT hr; int typeChangeCount = 0; @@ -242,8 +247,8 @@ WMFAudioMFTManager::Output(int64_t aStreamOffset, RefPtr& aOutData) { NS_ENSURE_TRUE(SUCCEEDED(hr), hr); // Output is made of floats. - int32_t numSamples = currentLength / sizeof(float); - int32_t numFrames = numSamples / mAudioChannels; + uint32_t numSamples = currentLength / sizeof(float); + uint32_t numFrames = numSamples / mAudioChannels; MOZ_ASSERT(numFrames >= 0); MOZ_ASSERT(numSamples >= 0); if (numFrames == 0) { @@ -275,10 +280,10 @@ WMFAudioMFTManager::Output(int64_t aStreamOffset, RefPtr& aOutData) { return MF_E_TRANSFORM_NEED_MORE_INPUT; } - aOutData = new AudioData(aStreamOffset, pts, std::move(audioData), - mAudioChannels, mAudioRate, mChannelsMap); - MOZ_DIAGNOSTIC_ASSERT(duration == aOutData->mDuration, "must be equal"); - mLastOutputDuration = aOutData->mDuration; + aOutput = new AudioData(aStreamOffset, pts, std::move(audioData), + mAudioChannels, mAudioRate, mChannelsMap); + MOZ_DIAGNOSTIC_ASSERT(duration == aOutput->mDuration, "must be equal"); + mLastOutputDuration = aOutput->mDuration; #ifdef LOG_SAMPLE_DECODE LOG("Decoded audio sample! timestamp=%lld duration=%lld currentLength=%u", diff --git a/dom/media/platforms/wmf/WMFAudioMFTManager.h b/dom/media/platforms/wmf/WMFAudioMFTManager.h index b5dc379396..f772593545 100644 --- a/dom/media/platforms/wmf/WMFAudioMFTManager.h +++ b/dom/media/platforms/wmf/WMFAudioMFTManager.h @@ -58,6 +58,7 @@ class WMFAudioMFTManager : public MFTManager { media::TimeUnit mLastOutputDuration = media::TimeUnit::Zero(); bool mFirstFrame = true; + bool mIsADTS = false; uint64_t mTotalMediaFrames = 0; uint32_t mEncoderDelay = 0; diff --git a/dom/media/platforms/wmf/WMFMediaDataEncoder.h b/dom/media/platforms/wmf/WMFMediaDataEncoder.h index 13848b47ad..31a63c8347 100644 --- a/dom/media/platforms/wmf/WMFMediaDataEncoder.h +++ b/dom/media/platforms/wmf/WMFMediaDataEncoder.h @@ -202,7 +202,9 @@ class WMFMediaDataEncoder final : public MediaDataEncoder { MOZ_ASSERT(mEncoder); const layers::PlanarYCbCrImage* image = aData->mImage->AsPlanarYCbCrImage(); - MOZ_ASSERT(image); + // TODO: Take care non planar Y-Cb-Cr image (Bug 1881647). + NS_ENSURE_TRUE(image, nullptr); + const layers::PlanarYCbCrData* yuv = image->GetData(); auto ySize = yuv->YDataSize(); auto cbcrSize = yuv->CbCrDataSize(); @@ -223,6 +225,7 @@ class WMFMediaDataEncoder final : public MediaDataEncoder { LockBuffer lockBuffer(buffer); NS_ENSURE_TRUE(SUCCEEDED(lockBuffer.Result()), nullptr); + // TODO: Take care non I420 image (Bug 1881647). bool ok = libyuv::I420ToNV12( yuv->mYChannel, yuv->mYStride, yuv->mCbChannel, yuv->mCbCrStride, yuv->mCrChannel, yuv->mCbCrStride, diff --git a/dom/media/platforms/wmf/WMFUtils.cpp b/dom/media/platforms/wmf/WMFUtils.cpp index d096979919..dda9df808e 100644 --- a/dom/media/platforms/wmf/WMFUtils.cpp +++ b/dom/media/platforms/wmf/WMFUtils.cpp @@ -177,7 +177,8 @@ Maybe GetYUVColorSpace(IMFMediaType* aType) { } int32_t MFOffsetToInt32(const MFOffset& aOffset) { - return int32_t(aOffset.value + (aOffset.fract / 65536.0f)); + return AssertedCast(AssertedCast(aOffset.value) + + (AssertedCast(aOffset.fract) / 65536.0f)); } TimeUnit GetSampleDuration(IMFSample* aSample) { @@ -204,7 +205,7 @@ GetPictureRegion(IMFMediaType* aMediaType, gfx::IntRect& aOutPictureRegion) { // Determine if "pan and scan" is enabled for this media. If it is, we // only display a region of the video frame, not the entire frame. BOOL panScan = - MFGetAttributeUINT32(aMediaType, MF_MT_PAN_SCAN_ENABLED, FALSE); + !!MFGetAttributeUINT32(aMediaType, MF_MT_PAN_SCAN_ENABLED, FALSE); // If pan and scan mode is enabled. Try to get the display region. HRESULT hr = E_FAIL; @@ -300,11 +301,14 @@ const char* MFTMessageTypeToStr(MFT_MESSAGE_TYPE aMsg) { GUID AudioMimeTypeToMediaFoundationSubtype(const nsACString& aMimeType) { if (aMimeType.EqualsLiteral("audio/mpeg")) { return MFAudioFormat_MP3; - } else if (MP4Decoder::IsAAC(aMimeType)) { + } + if (MP4Decoder::IsAAC(aMimeType)) { return MFAudioFormat_AAC; - } else if (aMimeType.EqualsLiteral("audio/vorbis")) { + } + if (aMimeType.EqualsLiteral("audio/vorbis")) { return MFAudioFormat_Vorbis; - } else if (aMimeType.EqualsLiteral("audio/opus")) { + } + if (aMimeType.EqualsLiteral("audio/opus")) { return MFAudioFormat_Opus; } NS_WARNING("Unsupport audio mimetype"); @@ -314,17 +318,19 @@ GUID AudioMimeTypeToMediaFoundationSubtype(const nsACString& aMimeType) { GUID VideoMimeTypeToMediaFoundationSubtype(const nsACString& aMimeType) { if (MP4Decoder::IsH264(aMimeType)) { return MFVideoFormat_H264; - } else if (VPXDecoder::IsVP8(aMimeType)) { + } + if (VPXDecoder::IsVP8(aMimeType)) { return MFVideoFormat_VP80; - } else if (VPXDecoder::IsVP9(aMimeType)) { + } + if (VPXDecoder::IsVP9(aMimeType)) { return MFVideoFormat_VP90; } #ifdef MOZ_AV1 - else if (AOMDecoder::IsAV1(aMimeType)) { + if (AOMDecoder::IsAV1(aMimeType)) { return MFVideoFormat_AV1; } #endif - else if (MP4Decoder::IsHEVC(aMimeType)) { + if (MP4Decoder::IsHEVC(aMimeType)) { return MFVideoFormat_HEVC; } NS_WARNING("Unsupport video mimetype"); @@ -368,7 +374,9 @@ void AACAudioSpecificConfigToUserData(uint8_t aAACProfileLevelIndication, // the rest can be all 0x00. BYTE heeInfo[heeInfoLen] = {0}; WORD* w = (WORD*)heeInfo; - w[0] = 0x0; // Payload type raw AAC packet + // If extradata has been provided, assume raw AAC packets (0). Otherwise, + // assume ADTS (1) + w[0] = aConfigLength ? 0 : 1; w[1] = aAACProfileLevelIndication; aOutUserData.AppendElements(heeInfo, heeInfoLen); @@ -377,10 +385,10 @@ void AACAudioSpecificConfigToUserData(uint8_t aAACProfileLevelIndication, // The AudioSpecificConfig is TTTTTFFF|FCCCCGGG // (T=ObjectType, F=Frequency, C=Channel, G=GASpecificConfig) // If frequency = 0xf, then the frequency is explicitly defined on 24 bits. - int8_t frequency = + uint8_t frequency = (aAudioSpecConfig[0] & 0x7) << 1 | (aAudioSpecConfig[1] & 0x80) >> 7; - int8_t channels = (aAudioSpecConfig[1] & 0x78) >> 3; - int8_t gasc = aAudioSpecConfig[1] & 0x7; + uint8_t channels = (aAudioSpecConfig[1] & 0x78) >> 3; + uint8_t gasc = aAudioSpecConfig[1] & 0x7; if (frequency != 0xf && channels && !gasc) { // We enter this condition if the AudioSpecificConfig should theorically // be 2 bytes long but it's not. diff --git a/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java b/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java index cc54009a7b..b4cc65ec9c 100644 --- a/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java +++ b/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java @@ -17,6 +17,7 @@ import android.content.Context; import android.util.Log; import android.view.Surface; import android.view.WindowManager; +import androidx.annotation.NonNull; import java.util.concurrent.CountDownLatch; @@ -37,10 +38,8 @@ public class VideoCaptureAndroid implements CameraVideoCapturer.CameraEventsHand private final String deviceName; private volatile long native_capturer; // |VideoCaptureAndroid*| in C++. - private Context context; + private final Context context; private CameraVideoCapturer cameraVideoCapturer; - private EglBase eglBase; - private SurfaceTextureHelper surfaceTextureHelper; // This class is recreated everytime we start/stop capture, so we // can safely create the CountDownLatches here. @@ -48,8 +47,16 @@ public class VideoCaptureAndroid implements CameraVideoCapturer.CameraEventsHand private boolean capturerStartedSucceeded = false; private final CountDownLatch capturerStopped = new CountDownLatch(1); - @WebRTCJNITarget - public VideoCaptureAndroid(String deviceName) { + @WebRTCJNITarget + public static VideoCaptureAndroid create(@NonNull final String deviceName) { + final Context context = GetContext(); + return new VideoCaptureAndroid(context, deviceName, + Camera2Enumerator.isSupported(context) + ? new Camera2Enumerator(context) + : new Camera1Enumerator()); + } + + private VideoCaptureAndroid(@NonNull final Context context, @NonNull final String deviceName, @NonNull final CameraEnumerator enumerator) { // Remove the camera facing information from the name. String[] parts = deviceName.split("Facing (front|back):"); if (parts.length == 2) { @@ -58,20 +65,14 @@ public class VideoCaptureAndroid implements CameraVideoCapturer.CameraEventsHand Log.e(TAG, "VideoCaptureAndroid: Expected facing mode as part of name: " + deviceName); this.deviceName = deviceName; } - this.context = GetContext(); + this.context = context; - CameraEnumerator enumerator; - if (Camera2Enumerator.isSupported(context)) { - enumerator = new Camera2Enumerator(context); - } else { - enumerator = new Camera1Enumerator(); - } try { cameraVideoCapturer = enumerator.createCapturer(this.deviceName, this); - eglBase = EglBase.create(); - surfaceTextureHelper = SurfaceTextureHelper.create("VideoCaptureAndroidSurfaceTextureHelper", eglBase.getEglBaseContext()); + final EglBase eglBase = EglBase.create(); + final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("VideoCaptureAndroidSurfaceTextureHelper", eglBase.getEglBaseContext()); cameraVideoCapturer.initialize(surfaceTextureHelper, context, this); - } catch (java.lang.IllegalArgumentException e) { + } catch (java.lang.RuntimeException e) { Log.e(TAG, "VideoCaptureAndroid: Exception while creating capturer: " + e); } } @@ -80,6 +81,10 @@ public class VideoCaptureAndroid implements CameraVideoCapturer.CameraEventsHand @WebRTCJNITarget private static native Context GetContext(); + public boolean canCapture() { + return cameraVideoCapturer != null; + } + // Called by native code. Returns true if capturer is started. // // Note that this actually opens the camera, and Camera callbacks run on the diff --git a/dom/media/systemservices/android_video_capture/video_capture_android.cc b/dom/media/systemservices/android_video_capture/video_capture_android.cc index da0715db27..cd5a73ceb6 100644 --- a/dom/media/systemservices/android_video_capture/video_capture_android.cc +++ b/dom/media/systemservices/android_video_capture/video_capture_android.cc @@ -177,12 +177,14 @@ int32_t VideoCaptureAndroid::Init(const char* deviceUniqueIdUTF8) { AttachThreadScoped ats(g_jvm_capture); JNIEnv* env = ats.env(); - jmethodID ctor = env->GetMethodID(g_java_capturer_class, "", - "(Ljava/lang/String;)V"); - assert(ctor); + jmethodID factory = + env->GetStaticMethodID(g_java_capturer_class, "create", + "(Ljava/lang/String;)" + "Lorg/webrtc/videoengine/VideoCaptureAndroid;"); + assert(factory); jstring j_deviceName = env->NewStringUTF(_deviceUniqueId); - _jCapturer = env->NewGlobalRef( - env->NewObject(g_java_capturer_class, ctor, j_deviceName)); + _jCapturer = env->NewGlobalRef(env->CallStaticObjectMethod( + g_java_capturer_class, factory, j_deviceName)); assert(_jCapturer); return 0; } diff --git a/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm b/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm index 36d5f56b16..ca7bf4db9b 100644 --- a/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm +++ b/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm @@ -88,10 +88,11 @@ AVCaptureDeviceFormat* _Nullable FindFormat( - (void)capturer:(RTCVideoCapturer* _Nonnull)capturer didCaptureVideoFrame:(RTCVideoFrame* _Nonnull)frame { - rtc::scoped_refptr cap; + webrtc::scoped_refptr + cap; { webrtc::MutexLock lock(&_mutex); - cap = rtc::scoped_refptr(_capturer); + cap = webrtc::scoped_refptr(_capturer); } if (!cap) return; cap->OnFrame(frame); diff --git a/dom/media/test/background_video.js b/dom/media/test/background_video.js index 508f8fd89a..6cce797474 100644 --- a/dom/media/test/background_video.js +++ b/dom/media/test/background_video.js @@ -72,7 +72,7 @@ function appendVideoToDocWithoutLoad(token, width, height) { } function loadAndWaitUntilLoadedmetadata(video, url, preloadType = "metadata") { - return new Promise((resolve, reject) => { + return new Promise(resolve => { video.preload = preloadType; video.addEventListener( "loadedmetadata", diff --git a/dom/media/test/bipbop-clearkey-video-av1.mp4 b/dom/media/test/bipbop-clearkey-video-av1.mp4 new file mode 100644 index 0000000000..78c1e56b2b Binary files /dev/null and b/dom/media/test/bipbop-clearkey-video-av1.mp4 differ diff --git a/dom/media/test/bipbop-clearkey-video-av1.mp4^headers^ b/dom/media/test/bipbop-clearkey-video-av1.mp4^headers^ new file mode 100644 index 0000000000..4030ea1d3d --- /dev/null +++ b/dom/media/test/bipbop-clearkey-video-av1.mp4^headers^ @@ -0,0 +1 @@ +Cache-Control: no-store diff --git a/dom/media/test/bipbop-clearkey-video-av1.webm b/dom/media/test/bipbop-clearkey-video-av1.webm new file mode 100644 index 0000000000..840888a8b1 Binary files /dev/null and b/dom/media/test/bipbop-clearkey-video-av1.webm differ diff --git a/dom/media/test/bipbop-clearkey-video-av1.webm^headers^ b/dom/media/test/bipbop-clearkey-video-av1.webm^headers^ new file mode 100644 index 0000000000..4030ea1d3d --- /dev/null +++ b/dom/media/test/bipbop-clearkey-video-av1.webm^headers^ @@ -0,0 +1 @@ +Cache-Control: no-store diff --git a/dom/media/test/browser/browser.toml b/dom/media/test/browser/browser.toml index cf25369576..3e83448d47 100644 --- a/dom/media/test/browser/browser.toml +++ b/dom/media/test/browser/browser.toml @@ -2,6 +2,7 @@ subsuite = "media-bc" prefs = ["gfx.font_loader.delay=0"] support-files = [ + "head.js", "file_empty_page.html", "file_media.html", "../av1.mp4", @@ -16,11 +17,14 @@ support-files = [ "../small-shot.mp3", "../small-shot.ogg", "../TestPatternHDR.mp4", + "../../mediasource/test/bipbop/bipbop2s.mp4", ] ["browser_encrypted_play_time_telemetry.js"] skip-if = ["apple_silicon"] # Disabled due to bleedover with other tests when run in regular suites; passes in "failures" jobs +["browser_glean_first_frame_loaded_time.js"] + ["browser_tab_visibility_and_play_time.js"] ["browser_telemetry_video_hardware_decoding_support.js"] diff --git a/dom/media/test/browser/browser_encrypted_play_time_telemetry.js b/dom/media/test/browser/browser_encrypted_play_time_telemetry.js index ff4f2753ec..1a64717419 100644 --- a/dom/media/test/browser/browser_encrypted_play_time_telemetry.js +++ b/dom/media/test/browser/browser_encrypted_play_time_telemetry.js @@ -29,76 +29,6 @@ async function clearTelemetry() { }); } -// Opens a tab containing a blank page, returns a promise that will resolve -// to that tab. -async function openTab() { - const emptyPageUri = - "https://example.com/browser/dom/media/test/browser/file_empty_page.html"; - return BrowserTestUtils.openNewForegroundTab(window.gBrowser, emptyPageUri); -} - -// Creates and configures a video element for EME playback in `tab`. Does not -// start playback for the element. Returns a promise that will resolve once -// the element is setup and ready for playback. -async function loadEmeVideo(tab) { - const emeHelperUri = - gTestPath.substr(0, gTestPath.lastIndexOf("/")) + "/eme_standalone.js"; - return SpecialPowers.spawn( - tab.linkedBrowser, - [emeHelperUri], - async _emeHelperUri => { - // Begin helper functions. - async function once(target, name) { - return new Promise(r => - target.addEventListener(name, r, { once: true }) - ); - } - - // Helper to clone data into content so the EME helper can use the data. - function cloneIntoContent(data) { - return Cu.cloneInto(data, content.wrappedJSObject); - } - // End helper functions. - - // Load the EME helper into content. - Services.scriptloader.loadSubScript(_emeHelperUri, content); - // Setup EME with the helper. - let video = content.document.createElement("video"); - video.id = "media"; - content.document.body.appendChild(video); - let emeHelper = new content.wrappedJSObject.EmeHelper(); - emeHelper.SetKeySystem( - content.wrappedJSObject.EmeHelper.GetClearkeyKeySystemString() - ); - emeHelper.SetInitDataTypes(cloneIntoContent(["webm"])); - emeHelper.SetVideoCapabilities( - cloneIntoContent([{ contentType: 'video/webm; codecs="vp9"' }]) - ); - emeHelper.AddKeyIdAndKey( - "2cdb0ed6119853e7850671c3e9906c3c", - "808b9adac384de1e4f56140f4ad76194" - ); - emeHelper.onerror = error => { - is(false, `Got unexpected error from EME helper: ${error}`); - }; - await emeHelper.ConfigureEme(video); - // Done setting up EME. - - // Setup MSE. - const ms = new content.wrappedJSObject.MediaSource(); - video.src = content.wrappedJSObject.URL.createObjectURL(ms); - await once(ms, "sourceopen"); - const sb = ms.addSourceBuffer("video/webm"); - const videoFile = "sintel-short-clearkey-subsample-encrypted-video.webm"; - let fetchResponse = await content.fetch(videoFile); - sb.appendBuffer(await fetchResponse.arrayBuffer()); - await once(sb, "updateend"); - ms.endOfStream(); - await once(ms, "sourceended"); - } - ); -} - // Plays the media in `tab` until the 'ended' event is fire. Returns a promise // that resolves once that state has been reached. async function playMediaThrough(tab) { diff --git a/dom/media/test/browser/browser_glean_first_frame_loaded_time.js b/dom/media/test/browser/browser_glean_first_frame_loaded_time.js new file mode 100644 index 0000000000..1acfa9957e --- /dev/null +++ b/dom/media/test/browser/browser_glean_first_frame_loaded_time.js @@ -0,0 +1,81 @@ +"use strict"; + +// Disabling undef warning because in `run()` we use functions from head.js +/* eslint-disable no-undef */ + +/** + * This test is used to ensure that Glean probe 'first_frame_loaded' can be + * recorded correctly in different situations. + */ + +const testCases = [ + { + expected: { + playback_type: "Non-MSE playback", + video_codec: "video/avc", + resolution: "AV,240 { element.addEventListener( "timeupdate", - e => { + () => { resolve(); }, { once: true } diff --git a/dom/media/test/browser/head.js b/dom/media/test/browser/head.js new file mode 100644 index 0000000000..7ef578a804 --- /dev/null +++ b/dom/media/test/browser/head.js @@ -0,0 +1,119 @@ +"use strict"; + +/* import-globals-from ../eme_standalone.js */ + +// Opens a tab containing a blank page, returns a promise that will resolve +// to that tab. +function openTab() { + const emptyPageUri = + "https://example.com/browser/dom/media/test/browser/file_empty_page.html"; + return BrowserTestUtils.openNewForegroundTab(window.gBrowser, emptyPageUri); +} + +// Creates and configures a video element for non-MSE playback in `tab`. Does not +// start playback for the element. Returns a promise that will resolve once +// the element is setup and ready for playback. +function loadVideo(tab) { + return SpecialPowers.spawn(tab.linkedBrowser, [], async _ => { + let video = content.document.createElement("video"); + video.id = "media"; + content.document.body.appendChild(video); + + video.src = "gizmo.mp4"; + video.load(); + + info(`waiting 'loadeddata' event to ensure playback is ready`); + await new Promise(r => (video.onloadeddata = r)); + }); +} + +// Creates and configures a video element for MSE playback in `tab`. Does not +// start playback for the element. Returns a promise that will resolve once +// the element is setup and ready for playback. +function loadMseVideo(tab) { + return SpecialPowers.spawn(tab.linkedBrowser, [], async _ => { + async function once(target, name) { + return new Promise(r => target.addEventListener(name, r, { once: true })); + } + + let video = content.document.createElement("video"); + video.id = "media"; + content.document.body.appendChild(video); + + info(`starting setup MSE`); + const ms = new content.wrappedJSObject.MediaSource(); + video.src = content.wrappedJSObject.URL.createObjectURL(ms); + await once(ms, "sourceopen"); + const sb = ms.addSourceBuffer("video/mp4"); + const videoFile = "bipbop2s.mp4"; + let fetchResponse = await content.fetch(videoFile); + sb.appendBuffer(await fetchResponse.arrayBuffer()); + await once(sb, "updateend"); + ms.endOfStream(); + await once(ms, "sourceended"); + + info(`waiting 'loadeddata' event to ensure playback is ready`); + await once(video, "loadeddata"); + }); +} + +// Creates and configures a video element for EME playback in `tab`. Does not +// start playback for the element. Returns a promise that will resolve once +// the element is setup and ready for playback. +function loadEmeVideo(tab) { + const emeHelperUri = + gTestPath.substr(0, gTestPath.lastIndexOf("/")) + "/eme_standalone.js"; + return SpecialPowers.spawn( + tab.linkedBrowser, + [emeHelperUri], + async _emeHelperUri => { + async function once(target, name) { + return new Promise(r => + target.addEventListener(name, r, { once: true }) + ); + } + + // Helper to clone data into content so the EME helper can use the data. + function cloneIntoContent(data) { + return Cu.cloneInto(data, content.wrappedJSObject); + } + + info(`starting setup EME`); + Services.scriptloader.loadSubScript(_emeHelperUri, content); + let video = content.document.createElement("video"); + video.id = "media"; + content.document.body.appendChild(video); + let emeHelper = new content.wrappedJSObject.EmeHelper(); + emeHelper.SetKeySystem( + content.wrappedJSObject.EmeHelper.GetClearkeyKeySystemString() + ); + emeHelper.SetInitDataTypes(cloneIntoContent(["webm"])); + emeHelper.SetVideoCapabilities( + cloneIntoContent([{ contentType: 'video/webm; codecs="vp9"' }]) + ); + emeHelper.AddKeyIdAndKey( + "2cdb0ed6119853e7850671c3e9906c3c", + "808b9adac384de1e4f56140f4ad76194" + ); + emeHelper.onerror = error => { + is(false, `Got unexpected error from EME helper: ${error}`); + }; + await emeHelper.ConfigureEme(video); + + info(`starting setup MSE`); + const ms = new content.wrappedJSObject.MediaSource(); + video.src = content.wrappedJSObject.URL.createObjectURL(ms); + await once(ms, "sourceopen"); + const sb = ms.addSourceBuffer("video/webm"); + const videoFile = "sintel-short-clearkey-subsample-encrypted-video.webm"; + let fetchResponse = await content.fetch(videoFile); + sb.appendBuffer(await fetchResponse.arrayBuffer()); + await once(sb, "updateend"); + ms.endOfStream(); + await once(ms, "sourceended"); + + info(`waiting 'loadeddata' event to ensure playback is ready`); + await once(video, "loadeddata"); + } + ); +} diff --git a/dom/media/test/browser/wmfme/browser.toml b/dom/media/test/browser/wmfme/browser.toml index 422ea29228..7449d232d9 100644 --- a/dom/media/test/browser/wmfme/browser.toml +++ b/dom/media/test/browser/wmfme/browser.toml @@ -3,11 +3,17 @@ subsuite = "media-bc" tags = "media-engine-compatible" run-if = ["wmfme"] support-files = [ - "head.js", "file_video.html", + "head.js", + "../head.js", + "../../eme_standalone.js", "../../gizmo.mp4", + "../../sintel-short-clearkey-subsample-encrypted-video.webm", + "../../../mediasource/test/bipbop/bipbop2s.mp4", ] ["browser_wmfme_crash.js"] +["browser_wmfme_glean_first_frame_loaded_time.js"] + ["browser_wmfme_max_crashes.js"] diff --git a/dom/media/test/browser/wmfme/browser_wmfme_glean_first_frame_loaded_time.js b/dom/media/test/browser/wmfme/browser_wmfme_glean_first_frame_loaded_time.js new file mode 100644 index 0000000000..f9e97aaa49 --- /dev/null +++ b/dom/media/test/browser/wmfme/browser_wmfme_glean_first_frame_loaded_time.js @@ -0,0 +1,98 @@ +"use strict"; + +// Disabling undef warning because in `run()` we use functions from head.js +/* eslint-disable no-undef */ + +/** + * This test is used to ensure that Glean probe 'first_frame_loaded' can be + * recorded correctly in different situations. + */ + +/* import-globals-from ../head.js */ +Services.scriptloader.loadSubScript( + "chrome://mochitests/content/browser/dom/media/test/browser/head.js", + this +); + +add_task(async function setupTestingPref() { + await SpecialPowers.pushPrefEnv({ + set: [ + ["media.wmf.media-engine.enabled", 1], + ["media.wmf.media-engine.channel-decoder.enabled", true], + ["media.eme.wmf.clearkey.enabled", true], + ], + }); +}); + +const testCases = [ + { + expected: { + playback_type: "Non-MSE media-engine playback", + video_codec: "video/avc", + resolution: "AV,240 { + return new Promise(resolve => { script.addMessageListener("media-test:cwd", cwd => { if (!cwd) { ok(false, "Failed to find path to test files"); @@ -1891,6 +1891,46 @@ var gEMETests = [ sessionCount: 1, duration: 1.98, }, + // ffmpeg -i bipbop.mp4 -t 00:00:02 -c:v libaom-av1 bipbop_av1.mp4 + // packager-linux-x64 in=bipbop_av1.mp4,stream=video,out=bipbop-clearkey-video-av1.mp4 --enable_raw_key_encryption --keys label=:key_id=8b5df745ad84145b5617c33116e35a67:key=bddfd35dd9be033ee73bc18bc1885056 --clear_lead 0 + { + name: "MP4 av1 video clearkey", + tracks: [ + { + name: "video", + type: 'video/mp4; codecs="av1"', + fragments: ["bipbop-clearkey-video-av1.mp4"], + }, + ], + keys: { + // "keyid" : "key" + "8b5df745ad84145b5617c33116e35a67": "bddfd35dd9be033ee73bc18bc1885056", + }, + sessionType: "temporary", + sessionCount: 1, + duration: 2.0, + skipTests: ["waitingforkey"], + }, + // ffmpeg -i bipbop.mp4 -t 00:00:02 -c:v libaom-av1 bipbop_av1.webm + // packager-linux-x64 in=bipbop_av1.webm,stream=video,out=bipbop-clearkey-video-av1.webm --enable_raw_key_encryption --keys label=:key_id=8b5df745ad84145b5617c33116e35a67:key=bddfd35dd9be033ee73bc18bc1885056 --clear_lead 0 + { + name: "WebM av1 video clearkey", + tracks: [ + { + name: "video", + type: 'video/webm; codecs="av1"', + fragments: ["bipbop-clearkey-video-av1.webm"], + }, + ], + keys: { + // "keyid" : "key" + "8b5df745ad84145b5617c33116e35a67": "bddfd35dd9be033ee73bc18bc1885056", + }, + sessionType: "temporary", + sessionCount: 1, + duration: 2.0, + skipTests: ["waitingforkey"], + }, { name: "WebM vorbis audio & vp8 video clearkey", tracks: [ @@ -2241,7 +2281,7 @@ function removeNodeAndSource(n) { } function once(target, name, cb) { - var p = new Promise(function (resolve, reject) { + var p = new Promise(function (resolve) { target.addEventListener( name, function () { @@ -2262,7 +2302,7 @@ function once(target, name, cb) { * @returns {Promise} A promise that is resolved when event happens. */ function nextEvent(video, eventName) { - return new Promise(function (resolve, reject) { + return new Promise(function (resolve) { let f = function (event) { video.removeEventListener(eventName, f); resolve(event); diff --git a/dom/media/test/mochitest.toml b/dom/media/test/mochitest.toml index de8c690255..3c8a382766 100644 --- a/dom/media/test/mochitest.toml +++ b/dom/media/test/mochitest.toml @@ -88,6 +88,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", @@ -798,6 +802,7 @@ skip-if = [ "os == 'mac'", # 1517199 (timeout-on-osx) "os == 'linux'", # 1713397, 1719881 (high intermittent failure on linux tsan), 1776937 "apple_silicon", # Disabled due to bleedover with other tests when run in regular suites; passes in "failures" jobs + "mda_gpu", # Disabled because high intermittent failure ] ["test_chaining.html"] diff --git a/dom/media/test/mochitest_background_video.toml b/dom/media/test/mochitest_background_video.toml index 762a9a8648..e1bc542264 100644 --- a/dom/media/test/mochitest_background_video.toml +++ b/dom/media/test/mochitest_background_video.toml @@ -84,6 +84,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", diff --git a/dom/media/test/mochitest_bugs.toml b/dom/media/test/mochitest_bugs.toml index 9d4a0dea2f..5c68d0e795 100644 --- a/dom/media/test/mochitest_bugs.toml +++ b/dom/media/test/mochitest_bugs.toml @@ -84,6 +84,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", @@ -769,7 +773,10 @@ support-files = [ ["test_bug465498.html"] ["test_bug495145.html"] -skip-if = ["os == 'win'"] #Bug 1404373 +skip-if = [ + "win10_2009", #Bug 1404373 + "win11_2009", #Bug 1404373 +] ["test_bug495300.html"] @@ -786,15 +793,15 @@ skip-if = ["os == 'android'"] # bug 1285441, android(bug 1232305) tags = "capturestream" ["test_bug895305.html"] -skip-if = ["android_version == '25' && debug"] # android(bug 1232305) ["test_bug919265.html"] -skip-if = ["android_version == '25' && debug"] # android(bug 1232305) ["test_bug1113600.html"] skip-if = [ - "os == 'win' && os_version == '10.0' && debug", # Bug 1713410 - "os == 'mac'", # Bug 1198168 + "win10_2009 && debug", # Bug 1713410 + "win11_2009 && debug", # Bug 1713410 + "apple_catalina", # Bug 1198168 + "apple_silicon", # Bug 1198168 ] ["test_bug1120222.html"] diff --git a/dom/media/test/mochitest_compat.toml b/dom/media/test/mochitest_compat.toml index c477661a19..86f76f1464 100644 --- a/dom/media/test/mochitest_compat.toml +++ b/dom/media/test/mochitest_compat.toml @@ -90,6 +90,10 @@ support-files = [ "bipbop-cenc-videoinit.mp4^headers^", "bipbop-cenc-video-10s.mp4", "bipbop-cenc-video-10s.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop-clearkey-keyrotation-clear-lead-audio.mp4", "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", @@ -807,15 +811,12 @@ skip-if = ["true"] # bug 475110 - disabled since we don't play Wave files standa ["test_bug1431810_opus_downmix_to_mono.html"] ["test_can_play_type.html"] -skip-if = ["(android_version == '25' && debug)"] # android(bug 1232305) ["test_can_play_type_mpeg.html"] ["test_can_play_type_no_ogg.html"] -skip-if = ["(android_version == '25' && debug)"] # android(bug 1232305) ["test_can_play_type_ogg.html"] -skip-if = ["(android_version == '25' && debug)"] # android(bug 1232305) ["test_can_play_type_wave.html"] @@ -1000,10 +1001,12 @@ skip-if = ["true"] # see bug 1319725 ["test_readyState.html"] ["test_referer.html"] -skip-if = ["android_version == '25' && debug"] # android(bug 1232305) ["test_reset_src.html"] -skip-if = ["verify && debug && os == 'win'"] +skip-if = [ + "win10_2009 && debug && verify", + "win11_2009 && debug && verify", +] ["test_source.html"] diff --git a/dom/media/test/mochitest_eme.toml b/dom/media/test/mochitest_eme.toml index 696e0e947e..d7f39c3eb8 100644 --- a/dom/media/test/mochitest_eme.toml +++ b/dom/media/test/mochitest_eme.toml @@ -84,6 +84,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", diff --git a/dom/media/test/mochitest_media_recorder.toml b/dom/media/test/mochitest_media_recorder.toml index 428f99601d..a4893d9cf4 100644 --- a/dom/media/test/mochitest_media_recorder.toml +++ b/dom/media/test/mochitest_media_recorder.toml @@ -84,6 +84,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", diff --git a/dom/media/test/mochitest_seek.toml b/dom/media/test/mochitest_seek.toml index 0c90bd1bfb..d71aac775a 100644 --- a/dom/media/test/mochitest_seek.toml +++ b/dom/media/test/mochitest_seek.toml @@ -84,6 +84,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", diff --git a/dom/media/test/mochitest_stream.toml b/dom/media/test/mochitest_stream.toml index 043f8471d4..0badfc52ab 100644 --- a/dom/media/test/mochitest_stream.toml +++ b/dom/media/test/mochitest_stream.toml @@ -84,6 +84,10 @@ support-files = [ "bipbop-clearkey-keyrotation-clear-lead-audio.mp4^headers^", "bipbop-clearkey-keyrotation-clear-lead-video.mp4", "bipbop-clearkey-keyrotation-clear-lead-video.mp4^headers^", + "bipbop-clearkey-video-av1.mp4", + "bipbop-clearkey-video-av1.mp4^headers^", + "bipbop-clearkey-video-av1.webm", + "bipbop-clearkey-video-av1.webm^headers^", "bipbop_225w_175kbps.mp4", "bipbop_225w_175kbps.mp4^headers^", "bipbop_225w_175kbps-cenc-audio-key1-1.m4s", diff --git a/dom/media/test/reftest/color_quads/reftest.list b/dom/media/test/reftest/color_quads/reftest.list index 6071733f76..63a538b78a 100644 --- a/dom/media/test/reftest/color_quads/reftest.list +++ b/dom/media/test/reftest/color_quads/reftest.list @@ -14,35 +14,35 @@ defaults pref(media.av1.enabled,true) # - # yuv420p -fuzzy(16-51,5234-5622) fuzzy-if(swgl,32-38,1600-91746) fuzzy-if(useDrawSnapshot,16-16,11600-11600) fuzzy-if(OSX,16-73,5212-5622) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm ../reftest_img.html?src=color_quads/720p.png -fuzzy-if(winWidget&&swgl,0-20,0-5620) fuzzy-if(winWidget&&!swgl,0-1,0-78) fuzzy-if(Android,254-255,273680-273807) fuzzy-if(OSX,0-35,0-1947) fuzzy-if(OSX&&swgl,0-67,0-5451) fuzzy-if(appleSilicon,30-48,1760-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm +fuzzy(16-51,5234-5622) fuzzy-if(swgl,32-38,1600-91746) fuzzy-if(useDrawSnapshot,16-16,11600-11600) fuzzy-if(cocoaWidget,16-73,5212-5622) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm ../reftest_img.html?src=color_quads/720p.png +fuzzy-if(winWidget&&swgl,0-20,0-5620) fuzzy-if(winWidget&&!swgl,0-1,0-78) fuzzy-if(Android,254-255,273680-273807) fuzzy-if(cocoaWidget,0-35,0-1947) fuzzy-if(cocoaWidget&&swgl,0-67,0-5451) fuzzy-if(appleSilicon,30-48,1760-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm fuzzy-if(winWidget,0-1,0-78) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm -skip-if(winWidget&&isCoverageBuild) fuzzy(0-16,75-1941) fuzzy-if(Android,28-255,273680-359920) fuzzy-if(OSX,30-32,187326-187407) fuzzy-if(appleSilicon,30-48,1835-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm -fuzzy-if(winWidget&&swgl,0-20,0-5620) fuzzy-if(winWidget&&!swgl,0-1,0-78) fuzzy-if(Android,254-255,273680-273807) fuzzy-if(OSX,0-35,0-1947) fuzzy-if(OSX&&swgl,0-67,0-5451) fuzzy-if(appleSilicon,30-48,1760-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm +skip-if(winWidget&&isCoverageBuild) fuzzy(0-16,75-1941) fuzzy-if(Android,28-255,273680-359920) fuzzy-if(cocoaWidget,30-32,187326-187407) fuzzy-if(appleSilicon,30-48,1835-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm +fuzzy-if(winWidget&&swgl,0-20,0-5620) fuzzy-if(winWidget&&!swgl,0-1,0-78) fuzzy-if(Android,254-255,273680-273807) fuzzy-if(cocoaWidget,0-35,0-1947) fuzzy-if(cocoaWidget&&swgl,0-67,0-5451) fuzzy-if(appleSilicon,30-48,1760-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm skip-if(Android) fuzzy(16-48,8107-8818) fuzzy-if(winWidget&&swgl,31-38,8240-184080) fuzzy-if(appleSilicon,33-38,8819-11705) fuzzy-if(useDrawSnapshot,20-20,187200-187200) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm ../reftest_img.html?src=color_quads/720p.png skip-if(Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm # On Windows & sw render, we noticed that the comparison image captured from AV1 is not equal to its displayed video frame, so we would need to compare other codecs directly to PNG file. That should be fixed in bug 1748540. -skip-if(Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(OSX,0-16,0-1718) fuzzy-if(OSX&&swgl,0-20,0-2423) fuzzy-if(appleSilicon,0-16,0-1874) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm -skip-if(Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(OSX,2-36,184281-187407) fuzzy-if(winWidget,0-21,0-360000) fuzzy-if(appleSilicon,36-49,187329-187407) fuzzy-if(useDrawSnapshot,0-1,0-10) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm -skip-if(Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(OSX,0-16,0-1718) fuzzy-if(OSX&&swgl,0-20,0-2423) fuzzy-if(appleSilicon,0-16,0-1874) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm +skip-if(Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(cocoaWidget,0-16,0-1718) fuzzy-if(cocoaWidget&&swgl,0-20,0-2423) fuzzy-if(appleSilicon,0-16,0-1874) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm +skip-if(Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(cocoaWidget,2-36,184281-187407) fuzzy-if(winWidget,0-21,0-360000) fuzzy-if(appleSilicon,36-49,187329-187407) fuzzy-if(useDrawSnapshot,0-1,0-10) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm +skip-if(Android) skip-if(winWidget&&swgl) fuzzy-if(Android,255-255,273726-273726) fuzzy-if(cocoaWidget,0-16,0-1718) fuzzy-if(cocoaWidget&&swgl,0-20,0-2423) fuzzy-if(appleSilicon,0-16,0-1874) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.av1.webm skip-if(Android) skip-if(!(winWidget&&swgl)) fuzzy(0-35,0-8506) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.webm ../reftest_img.html?src=color_quads/720p.png skip-if(Android) skip-if(!(winWidget&&swgl)) fuzzy(0-35,0-8506) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p.vp9.mp4 ../reftest_img.html?src=color_quads/720p.png # - # yuv420p10 -skip-if(Android) fuzzy(33-49,1870-2579) fuzzy-if(swgl,34-52,180421-270528) fuzzy-if(useDrawSnapshot,16-16,183840-183840) fuzzy-if(OSX,60-74,270329-271024) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm ../reftest_img.html?src=color_quads/720p.png -skip-if(Android) fuzzy-if(OSX,0-12,0-187770) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm +skip-if(Android) fuzzy(33-49,1870-2579) fuzzy-if(swgl,34-52,180421-270528) fuzzy-if(useDrawSnapshot,16-16,183840-183840) fuzzy-if(cocoaWidget,60-74,270329-271024) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm ../reftest_img.html?src=color_quads/720p.png +skip-if(Android) fuzzy-if(cocoaWidget,0-12,0-187770) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm skip-if(Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm #[2] skip-if(Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm -skip-if(Android) fuzzy-if(OSX,0-12,0-187770) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm +skip-if(Android) fuzzy-if(cocoaWidget,0-12,0-187770) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p10.av1.webm -skip-if(Android) fuzzy(33-49,174620-270059) fuzzy-if(swgl&&!winWidget,36-52,11553-11555) fuzzy-if(swgl&&winWidget,36-52,11554-187200) fuzzy-if(swgl&&OSX,34-50,11465-270059) fuzzy-if(useDrawSnapshot,20-20,186800-186800) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm ../reftest_img.html?src=color_quads/720p.png -skip-if(Android) fuzzy-if(OSX,0-12,0-274122) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm +skip-if(Android) fuzzy(33-49,174620-270059) fuzzy-if(!winWidget&&swgl,36-52,11553-11555) fuzzy-if(winWidget&&swgl,36-52,11554-187200) fuzzy-if(swgl&&cocoaWidget,34-50,11465-270059) fuzzy-if(useDrawSnapshot,20-20,186800-186800) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm ../reftest_img.html?src=color_quads/720p.png +skip-if(Android) fuzzy-if(cocoaWidget,0-12,0-274122) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.vp9.webm ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm skip-if(Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm #[2] skip-if(Android) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm -skip-if(Android) fuzzy-if(OSX,0-12,0-274122) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm +skip-if(Android) fuzzy-if(cocoaWidget,0-12,0-274122) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.vp9.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.pc.yuv420p10.av1.webm # Android is really broken in a variety of ways for p10. #[2]: yuv420p10 broken in h264.mp4: https://bugzilla.mozilla.org/show_bug.cgi?id=1711812 diff --git a/dom/media/test/reftest/reftest.list b/dom/media/test/reftest/reftest.list index 6bbd7b9a53..0f709a35ee 100644 --- a/dom/media/test/reftest/reftest.list +++ b/dom/media/test/reftest/reftest.list @@ -1,12 +1,12 @@ -skip-if(Android) fuzzy-if(OSX,0-80,0-76800) fuzzy-if(appleSilicon,0-80,0-76800) fuzzy-if(winWidget,0-63,0-76799) fuzzy-if(gtkWidget&&layersGPUAccelerated,0-70,0-2032) HTTP(..) == short.mp4.firstframe.html short.mp4.firstframe-ref.html -skip-if(Android) fuzzy-if(OSX,0-87,0-76797) fuzzy-if(appleSilicon,0-87,0-76797) fuzzy-if(winWidget,0-60,0-76797) fuzzy-if(gtkWidget&&layersGPUAccelerated,0-60,0-6070) HTTP(..) == short.mp4.lastframe.html short.mp4.lastframe-ref.html -skip-if(Android) skip-if(cocoaWidget) skip-if(winWidget) fuzzy-if(gtkWidget&&layersGPUAccelerated,0-57,0-4282) fuzzy-if(OSX,55-80,4173-4417) HTTP(..) == bipbop_300_215kbps.mp4.lastframe.html bipbop_300_215kbps.mp4.lastframe-ref.html -skip-if(Android) fuzzy-if(OSX,0-25,0-175921) fuzzy-if(appleSilicon,34-34,40100-40100) fuzzy-if(winWidget,0-71,0-179198) HTTP(..) == gizmo.mp4.seek.html gizmo.mp4.55thframe-ref.html +skip-if(Android) fuzzy-if(cocoaWidget,0-80,0-76800) fuzzy-if(appleSilicon,0-80,0-76800) fuzzy-if(winWidget,0-63,0-76799) fuzzy-if(gtkWidget,0-70,0-2032) HTTP(..) == short.mp4.firstframe.html short.mp4.firstframe-ref.html +skip-if(Android) fuzzy-if(cocoaWidget,0-87,0-76797) fuzzy-if(appleSilicon,0-87,0-76797) fuzzy-if(winWidget,0-60,0-76797) fuzzy-if(gtkWidget,0-60,0-6070) HTTP(..) == short.mp4.lastframe.html short.mp4.lastframe-ref.html +skip-if(Android) skip-if(cocoaWidget) skip-if(winWidget) fuzzy-if(gtkWidget,0-57,0-4282) fuzzy-if(cocoaWidget,55-80,4173-4417) HTTP(..) == bipbop_300_215kbps.mp4.lastframe.html bipbop_300_215kbps.mp4.lastframe-ref.html +skip-if(Android) fuzzy-if(cocoaWidget,0-25,0-175921) fuzzy-if(appleSilicon,34-34,40100-40100) fuzzy-if(winWidget,0-71,0-179198) HTTP(..) == gizmo.mp4.seek.html gizmo.mp4.55thframe-ref.html # Bug 1758718 -skip-if(Android) skip-if(MinGW) skip-if(OSX) fuzzy(0-10,0-778236) == image-10bits-rendering-video.html image-10bits-rendering-ref.html -skip-if(Android) skip-if(MinGW) fuzzy(0-10,0-778536) fuzzy-if(appleSilicon,0-37,0-699614) == image-10bits-rendering-90-video.html image-10bits-rendering-90-ref.html +skip-if(Android) skip-if(cocoaWidget) fuzzy(0-10,0-778236) == image-10bits-rendering-video.html image-10bits-rendering-ref.html +skip-if(Android) fuzzy(0-10,0-778536) fuzzy-if(appleSilicon,0-37,0-699614) == image-10bits-rendering-90-video.html image-10bits-rendering-90-ref.html # Bug 1758718 -skip-if(Android) fuzzy(0-27,0-573106) skip-if(OSX) == image-10bits-rendering-720-video.html image-10bits-rendering-720-ref.html +skip-if(Android) fuzzy(0-27,0-573106) skip-if(cocoaWidget) == image-10bits-rendering-720-video.html image-10bits-rendering-720-ref.html skip-if(Android) fuzzy(0-31,0-573249) fuzzy-if(appleSilicon,0-37,0-543189) == image-10bits-rendering-720-90-video.html image-10bits-rendering-720-90-ref.html skip-if(Android) fuzzy(0-84,0-771156) fails-if(useDrawSnapshot) == uneven_frame_duration_video.html uneven_frame_duration_video-ref.html # Skip on Windows 7 as the resolution of the video is too high for test machines and will fail in the decoder. # Set media.dormant-on-pause-timeout-ms to avoid decoders becoming dormant and busting test, skip on android as test is too noisy and unstable diff --git a/dom/media/test/test_arraybuffer.html b/dom/media/test/test_arraybuffer.html index 9ef84c53dc..146d3ea5ac 100644 --- a/dom/media/test/test_arraybuffer.html +++ b/dom/media/test/test_arraybuffer.html @@ -49,14 +49,14 @@ function startTest(test, token) { events.forEach(function(e) { v.addEventListener(e, logEvent); }); - once(v, "stalled", function(e) { + once(v, "stalled", function() { // Resource fetch algorithm in local mode should never fire stalled event. // https://html.spec.whatwg.org/multipage/media.html#concept-media-load-resource ok(false, test.name + ": got stalled"); removeNodeAndSource(v); manager.finished(token); }); - once(v, "canplaythrough", function(e) { + once(v, "canplaythrough", function() { ok(true, test.name + ": got canplaythrough"); is(v.readyState, v.HAVE_ENOUGH_DATA, test.name + ": readyState is HAVE_ENOUGH_DATA"); removeNodeAndSource(v); diff --git a/dom/media/test/test_aspectratio_mp4.html b/dom/media/test/test_aspectratio_mp4.html index 5e01875439..959096e902 100644 --- a/dom/media/test/test_aspectratio_mp4.html +++ b/dom/media/test/test_aspectratio_mp4.html @@ -28,7 +28,7 @@ v.onloadedmetadata = function() { is(v.videoHeight, resource.height, "Intrinsic height should match video height"); SimpleTest.finish(); } -v.addEventListener("error", function(ev) { +v.addEventListener("error", function() { if (v.readyState < v.HAVE_METADATA) { info("Video element returns with readyState " + v.readyState + " error.code " + v.error.code); todo(false, "This platform doesn't support to retrieve MP4 metadata."); diff --git a/dom/media/test/test_background_video_resume_after_end_show_last_frame.html b/dom/media/test/test_background_video_resume_after_end_show_last_frame.html index 68f7179e12..991de9ac98 100644 --- a/dom/media/test/test_background_video_resume_after_end_show_last_frame.html +++ b/dom/media/test/test_background_video_resume_after_end_show_last_frame.html @@ -79,7 +79,7 @@ function waitUntilSeekToLastFrame(video) { ); } - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { video.seenEnded = false; video.addEventListener("ended", () => { video.seenEnded = true; diff --git a/dom/media/test/test_buffered.html b/dom/media/test/test_buffered.html index 86d8eec28a..85bbef2189 100644 --- a/dom/media/test/test_buffered.html +++ b/dom/media/test/test_buffered.html @@ -74,7 +74,7 @@ function fetch(url, fetched_callback) { xhr.open("GET", url, true); xhr.responseType = "blob"; - var loaded = function (event) { + var loaded = function () { if (xhr.status == 200 || xhr.status == 206) { ok(true, `${url}: Fetch succeeded, status=${xhr.status}`); // Request fulfilled. Note sometimes we get 206... Presumably because either diff --git a/dom/media/test/test_bug1431810_opus_downmix_to_mono.html b/dom/media/test/test_bug1431810_opus_downmix_to_mono.html index 647ddf0489..2d81fba1c7 100644 --- a/dom/media/test/test_bug1431810_opus_downmix_to_mono.html +++ b/dom/media/test/test_bug1431810_opus_downmix_to_mono.html @@ -116,14 +116,14 @@ function mediaElementWithPhaseInversionDisabled(audioContext, mediaElement, succ let ac = new AudioContext(); -function testPhaseInversion(mediaElement) { - return new Promise((accept, reject) => { +function testPhaseInversion() { + return new Promise((accept) => { mediaElementWithPhaseInversion(ac, a, accept); }); } -function testPhaseInversionDisabled(mediaElement) { - return new Promise((accept, reject) => { +function testPhaseInversionDisabled() { + return new Promise((accept) => { mediaElementWithPhaseInversionDisabled(ac, b, accept); }); } diff --git a/dom/media/test/test_cueless_webm_seek-1.html b/dom/media/test/test_cueless_webm_seek-1.html index db58a89665..dbc2933b92 100644 --- a/dom/media/test/test_cueless_webm_seek-1.html +++ b/dom/media/test/test_cueless_webm_seek-1.html @@ -101,7 +101,7 @@ function fetch(url, fetched_callback) { xhr.open("GET", url, true); xhr.responseType = "blob"; - var loaded = function (event) { + var loaded = function () { if (xhr.status == 200 || xhr.status == 206) { // Request fulfilled. Note sometimes we get 206... Presumably because either // httpd.js or Necko cached the result. diff --git a/dom/media/test/test_cueless_webm_seek-2.html b/dom/media/test/test_cueless_webm_seek-2.html index 720cc18399..f9fd29fcfb 100644 --- a/dom/media/test/test_cueless_webm_seek-2.html +++ b/dom/media/test/test_cueless_webm_seek-2.html @@ -91,7 +91,7 @@ function fetch(url, fetched_callback) { xhr.open("GET", url, true); xhr.responseType = "blob"; - var loaded = function (event) { + var loaded = function () { if (xhr.status == 200 || xhr.status == 206) { // Request fulfilled. Note sometimes we get 206... Presumably because either // httpd.js or Necko cached the result. diff --git a/dom/media/test/test_cueless_webm_seek-3.html b/dom/media/test/test_cueless_webm_seek-3.html index d6e3e50d7d..235066e6eb 100644 --- a/dom/media/test/test_cueless_webm_seek-3.html +++ b/dom/media/test/test_cueless_webm_seek-3.html @@ -85,7 +85,7 @@ function fetch(url, fetched_callback) { xhr.open("GET", url, true); xhr.responseType = "blob"; - var loaded = function (event) { + var loaded = function () { if (xhr.status == 200 || xhr.status == 206) { // Request fulfilled. Note sometimes we get 206... Presumably because either // httpd.js or Necko cached the result. diff --git a/dom/media/test/test_decode_error_crossorigin.html b/dom/media/test/test_decode_error_crossorigin.html index 24c1430a5b..8fa3db6c38 100644 --- a/dom/media/test/test_decode_error_crossorigin.html +++ b/dom/media/test/test_decode_error_crossorigin.html @@ -19,7 +19,7 @@ function startTest(test, token) { }; const v = document.createElement("video"); manager.started(token); - v.addEventListener("error", event => { + v.addEventListener("error", () => { if (v.readyState == v.HAVE_NOTHING) { is(v.error.code, MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED, "Expected code for a load error"); diff --git a/dom/media/test/test_delay_load.html b/dom/media/test/test_delay_load.html index 05877aa911..6c10e5b957 100644 --- a/dom/media/test/test_delay_load.html +++ b/dom/media/test/test_delay_load.html @@ -87,7 +87,7 @@ var w = window.open("", "testWindow", "width=400,height=400"); testWindows.push(w); v = createVideo(test.name, test.type, "4"); -v.onloadstart = function(e) { +v.onloadstart = function() { // Using a new window to do this is a bit annoying, but if we use an iframe here, // delaying of the iframe's load event might interfere with the firing of our load event // in some confusing way. So it's simpler just to use another window. diff --git a/dom/media/test/test_eme_detach_media_keys.html b/dom/media/test/test_eme_detach_media_keys.html index 69b812032a..5faa68d9c5 100644 --- a/dom/media/test/test_eme_detach_media_keys.html +++ b/dom/media/test/test_eme_detach_media_keys.html @@ -15,7 +15,7 @@ SimpleTest.waitForExplicitFinish(); function createAndSet() { - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var m; navigator.requestMediaKeySystemAccess(CLEARKEY_KEYSYSTEM, gCencMediaKeySystemConfig) .then(function (access) { diff --git a/dom/media/test/test_eme_detach_reattach_same_mediakeys_during_playback.html b/dom/media/test/test_eme_detach_reattach_same_mediakeys_during_playback.html index e47fae5891..6911ac2da8 100644 --- a/dom/media/test/test_eme_detach_reattach_same_mediakeys_during_playback.html +++ b/dom/media/test/test_eme_detach_reattach_same_mediakeys_during_playback.html @@ -72,7 +72,7 @@ function startTest(test, token) ok(false, " Failed to set original mediakeys back."); } - function onCanPlayAgain(ev) { + function onCanPlayAgain() { Promise.all([closeSessions()]) .then(() => { ok(true, " (ENCRYPTED) Playback can be resumed."); @@ -91,7 +91,7 @@ function startTest(test, token) v.currentTime = v.duration / 2; } - function onCanPlay(ev) { + function onCanPlay() { function onSetMediaKeysToNullOK() { ok(true, TimeStamp(token) + " Set MediaKeys to null. OK!"); diff --git a/dom/media/test/test_eme_getstatusforpolicy.html b/dom/media/test/test_eme_getstatusforpolicy.html index 2031d04df5..6e9503e33a 100644 --- a/dom/media/test/test_eme_getstatusforpolicy.html +++ b/dom/media/test/test_eme_getstatusforpolicy.html @@ -68,7 +68,7 @@ add_task(async function testGetStatusForPolicy() { "Should have set MediaKeys on media element"); let keyStatus = await video.mediaKeys.getStatusForPolicy({minHdcpVersion : result.minHdcpVersion}) - .catch(e => ok(false, "getStatusForPolicy failed!")); + .catch(() => ok(false, "getStatusForPolicy failed!")); info(`getStatusForPolicy for HDCP ${result.minHdcpVersion} : ${keyStatus}`); is(keyStatus, result.expectedResult, `Expected ${result.expectedResult}, got ${keyStatus}`); diff --git a/dom/media/test/test_eme_initDataTypes.html b/dom/media/test/test_eme_initDataTypes.html index 587e6fc161..a1e0c15182 100644 --- a/dom/media/test/test_eme_initDataTypes.html +++ b/dom/media/test/test_eme_initDataTypes.html @@ -93,7 +93,7 @@ function PrepareInitData(initDataType, initData) } function Test(test) { - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var configs = [{ initDataTypes: [test.initDataType], videoCapabilities: [{contentType: 'video/mp4' }], @@ -113,7 +113,7 @@ function Test(test) { var initData = PrepareInitData(test.initDataType, test.initData); return session.generateRequest(test.initDataType, initData); } - ).catch((x) => { + ).catch(() => { ok(!test.expectPass, "'" + test.name + "' expected to fail."); resolve(); }); diff --git a/dom/media/test/test_eme_mfcdm_generate_request.html b/dom/media/test/test_eme_mfcdm_generate_request.html index 57be325e36..b1b8396c0f 100644 --- a/dom/media/test/test_eme_mfcdm_generate_request.html +++ b/dom/media/test/test_eme_mfcdm_generate_request.html @@ -65,11 +65,11 @@ async function testKeySystemRequest(keySystem) { }]; info(`requestMediaKeySystemAccess for ${keySystem}`); let access = await navigator.requestMediaKeySystemAccess(keySystem, configs) - .catch(e => ok(false, `failed to create key system access`)); + .catch(() => ok(false, `failed to create key system access`)); info('creating media key'); let mediaKeys = await access.createMediaKeys() - .catch(e => ok(false, `failed to create media key`));; + .catch(() => ok(false, `failed to create media key`));; info(`creating a temporary media key session`); let session = mediaKeys.createSession(sessionType); @@ -84,7 +84,7 @@ async function testKeySystemRequest(keySystem) { await session.generateRequest( 'keyids', new TextEncoder().encode(fakeKID)) - .catch(e => ok(false, `failed to generate request`)); + .catch(() => ok(false, `failed to generate request`)); await messagePromise; } diff --git a/dom/media/test/test_eme_mfcdm_getstatusforpolicy.html b/dom/media/test/test_eme_mfcdm_getstatusforpolicy.html index 7e22be25d9..83cd740570 100644 --- a/dom/media/test/test_eme_mfcdm_getstatusforpolicy.html +++ b/dom/media/test/test_eme_mfcdm_getstatusforpolicy.html @@ -89,7 +89,7 @@ add_task(async function testGetStatusForPolicy() { "Should have set MediaKeys on media element"); let keyStatus = await video.mediaKeys.getStatusForPolicy({minHdcpVersion : result.minHdcpVersion}) - .catch(e => ok(false, "getStatusForPolicy failed!")); + .catch(() => ok(false, "getStatusForPolicy failed!")); info(`getStatusForPolicy for HDCP ${result.minHdcpVersion} : ${keyStatus}`); is(keyStatus, result.expectedResult, `Expected ${result.expectedResult}, got ${keyStatus}`); diff --git a/dom/media/test/test_eme_missing_pssh.html b/dom/media/test/test_eme_missing_pssh.html index 29f77d021a..cb94ffbf23 100644 --- a/dom/media/test/test_eme_missing_pssh.html +++ b/dom/media/test/test_eme_missing_pssh.html @@ -63,7 +63,7 @@ } function DownloadMedia(url, type, mediaSource) { - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var sourceBuffer = mediaSource.addSourceBuffer(type); fetchWithXHR(url, (response) => { once(sourceBuffer, "updateend", resolve); diff --git a/dom/media/test/test_eme_non_mse_fails.html b/dom/media/test/test_eme_non_mse_fails.html index 6ff17d59ff..e2719f244c 100644 --- a/dom/media/test/test_eme_non_mse_fails.html +++ b/dom/media/test/test_eme_non_mse_fails.html @@ -65,7 +65,7 @@ function TestSetSrc(test, token) manager.started(token); var v = document.createElement("video"); - v.addEventListener("error", function(err) { + v.addEventListener("error", function() { ok(true, token + " got error setting src on video element, as expected"); manager.finished(token); }); diff --git a/dom/media/test/test_eme_playback.html b/dom/media/test/test_eme_playback.html index bcfa058e34..478b7b0a25 100644 --- a/dom/media/test/test_eme_playback.html +++ b/dom/media/test/test_eme_playback.html @@ -109,7 +109,7 @@ function startTest(test, token) var gotEncrypted = 0; let finish = new EMEPromise; - v.addEventListener("encrypted", function(ev) { + v.addEventListener("encrypted", function() { gotEncrypted += 1; }); @@ -119,7 +119,7 @@ function startTest(test, token) is(v.isEncrypted, undefined, "isEncrypted should not be accessible from content"); }); - v.addEventListener("ended", function(ev) { + v.addEventListener("ended", function() { ok(true, TimeStamp(token) + " got ended event"); is(gotEncrypted, test.sessionCount, diff --git a/dom/media/test/test_eme_protection_query.html b/dom/media/test/test_eme_protection_query.html index 8bf97d8100..a8a1c7b376 100644 --- a/dom/media/test/test_eme_protection_query.html +++ b/dom/media/test/test_eme_protection_query.html @@ -53,7 +53,7 @@ async function setupEme(video) { let session = video.mediaKeys.createSession(); video.onencrypted = async encryptedEvent => { - session.onmessage = messageEvent => { + session.onmessage = () => { // Handle license messages. Hard code the license because we always test // with the same file and we know what the license should be. const license = { diff --git a/dom/media/test/test_eme_pssh_in_moof.html b/dom/media/test/test_eme_pssh_in_moof.html index d1965be844..406600056e 100644 --- a/dom/media/test/test_eme_pssh_in_moof.html +++ b/dom/media/test/test_eme_pssh_in_moof.html @@ -61,7 +61,7 @@ // Specialized create media keys function, since the one in eme.js relies // on listening for encrypted events, and we want to manage those // ourselves within this test. - async function createAndSetMediaKeys(video, test, token) { + async function createAndSetMediaKeys(video, test) { function streamType(type) { var x = test.tracks.find(o => o.name == type); return x ? x.type : undefined; diff --git a/dom/media/test/test_eme_requestKeySystemAccess.html b/dom/media/test/test_eme_requestKeySystemAccess.html index b044fe8c84..4014ef937a 100644 --- a/dom/media/test/test_eme_requestKeySystemAccess.html +++ b/dom/media/test/test_eme_requestKeySystemAccess.html @@ -20,7 +20,7 @@ function ValidateConfig(name, expected, observed) { is(observed.label, expected.label, name + " label should match"); if (expected.initDataTypes) { - ok(expected.initDataTypes.every((element, index, array) => observed.initDataTypes.includes(element)), name + " initDataTypes should match."); + ok(expected.initDataTypes.every((element) => observed.initDataTypes.includes(element)), name + " initDataTypes should match."); } if (expected.audioCapabilities) { @@ -45,7 +45,7 @@ function ValidateConfig(name, expected, observed) { function Test(test) { var name = "'" + test.name + "'"; - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var p; if (test.options) { var keySystem = (test.keySystem !== undefined) ? test.keySystem : CLEARKEY_KEYSYSTEM; diff --git a/dom/media/test/test_eme_request_notifications.html b/dom/media/test/test_eme_request_notifications.html index 6c44f892a6..62ef28e57d 100644 --- a/dom/media/test/test_eme_request_notifications.html +++ b/dom/media/test/test_eme_request_notifications.html @@ -17,7 +17,7 @@ function SetPrefs(prefs) { } function observe() { - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var observer = function(subject, topic, data) { SpecialPowers.Services.obs.removeObserver(observer, "mediakeys-request"); resolve(JSON.parse(data).status); @@ -70,7 +70,7 @@ var tests = [ SimpleTest.waitForExplicitFinish(); tests - .reduce(function(p, c, i, array) { + .reduce(function(p, c) { return p.then(function() { return Test(c); }); diff --git a/dom/media/test/test_eme_sample_groups_playback.html b/dom/media/test/test_eme_sample_groups_playback.html index cef1e26b33..e1e09ad73d 100644 --- a/dom/media/test/test_eme_sample_groups_playback.html +++ b/dom/media/test/test_eme_sample_groups_playback.html @@ -81,7 +81,7 @@ }); var request = new TextEncoder().encode(json); session.generateRequest("keyids", request) - .then(e => { + .then(() => { Log(test.name, "Request license success"); }, reason => { Log("Request license failed! " + reason); @@ -90,7 +90,7 @@ } function DownloadMedia(url, type, mediaSource) { - return new Promise((resolve, reject) => { + return new Promise((resolve) => { var sourceBuffer = mediaSource.addSourceBuffer(type); fetchWithXHR(url, (response) => { once(sourceBuffer, "updateend", resolve); diff --git a/dom/media/test/test_eme_stream_capture_blocked_case1.html b/dom/media/test/test_eme_stream_capture_blocked_case1.html index 0fc8d28364..e6743b4f8a 100644 --- a/dom/media/test/test_eme_stream_capture_blocked_case1.html +++ b/dom/media/test/test_eme_stream_capture_blocked_case1.html @@ -35,7 +35,7 @@ function startTest(test, token) var context = new AudioContext(); context.createMediaElementSource(v1); - v1.addEventListener("loadeddata", function(ev) { + v1.addEventListener("loadeddata", function() { ok(false, TimeStamp(case1token) + " should never reach loadeddata, as setMediaKeys should fail"); }); diff --git a/dom/media/test/test_eme_stream_capture_blocked_case2.html b/dom/media/test/test_eme_stream_capture_blocked_case2.html index b60538caf0..6ee415c124 100644 --- a/dom/media/test/test_eme_stream_capture_blocked_case2.html +++ b/dom/media/test/test_eme_stream_capture_blocked_case2.html @@ -20,7 +20,7 @@ function startTest(test, token) var case2token = token + "_case2"; let v2 = document.createElement("video"); - v2.addEventListener("loadeddata", function(ev) { + v2.addEventListener("loadeddata", function() { ok(true, case2token + " should reach loadeddata"); var threw = false; try { diff --git a/dom/media/test/test_eme_stream_capture_blocked_case3.html b/dom/media/test/test_eme_stream_capture_blocked_case3.html index e25a900956..23a23542e9 100644 --- a/dom/media/test/test_eme_stream_capture_blocked_case3.html +++ b/dom/media/test/test_eme_stream_capture_blocked_case3.html @@ -19,7 +19,7 @@ function startTest(test, token) var case3token = token + "_case3"; let v3 = document.createElement("video"); - v3.addEventListener("loadeddata", function(ev) { + v3.addEventListener("loadeddata", function() { ok(true, TimeStamp(case3token) + " should reach loadeddata"); var threw = false; try { diff --git a/dom/media/test/test_eme_unsetMediaKeys_then_capture.html b/dom/media/test/test_eme_unsetMediaKeys_then_capture.html index 3ecdc79dbf..eb20f2bee2 100644 --- a/dom/media/test/test_eme_unsetMediaKeys_then_capture.html +++ b/dom/media/test/test_eme_unsetMediaKeys_then_capture.html @@ -36,11 +36,11 @@ function startTest(test, token) let finish = new EMEPromise; - function onVideoEnded(ev) { + function onVideoEnded() { ok(true, TimeStamp(token) + " (ENCRYPTED) content playback ended."); function playClearVideo() { - var p1 = once(v, 'loadeddata', (e) => { + var p1 = once(v, 'loadeddata', () => { ok(true, TimeStamp(token) + " Receiving event 'loadeddata' for (CLEAR) content."); let canvasElem = document.createElement('canvas'); document.body.appendChild(canvasElem); diff --git a/dom/media/test/test_eme_wideinve_l1_installation.html b/dom/media/test/test_eme_wideinve_l1_installation.html index fa6a0c350b..da1f1ab0f9 100644 --- a/dom/media/test/test_eme_wideinve_l1_installation.html +++ b/dom/media/test/test_eme_wideinve_l1_installation.html @@ -19,7 +19,7 @@ function SetPrefs(prefs) { } function observe() { - return new Promise(function(resolve, reject) { + return new Promise(function(resolve) { var observer = function(subject, topic, data) { SpecialPowers.Services.obs.removeObserver(observer, "mediakeys-request"); resolve(JSON.parse(data).status); @@ -74,7 +74,7 @@ const tests = [ }, ]; -tests.reduce(function(p, c, i, array) { +tests.reduce(function(p, c) { return p.then(function() { return Test(c); }); diff --git a/dom/media/test/test_eme_wv_privacy.html b/dom/media/test/test_eme_wv_privacy.html index a050d2a528..e18d31a36f 100644 --- a/dom/media/test/test_eme_wv_privacy.html +++ b/dom/media/test/test_eme_wv_privacy.html @@ -12,7 +12,7 @@ + + + +
+
+
+ + diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelay.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelay.html index 180abc075a..7f3f9f57ad 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelay.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelay.html @@ -24,6 +24,8 @@ if (!("mediaDevices" in navigator)) { ['media.peerconnection.nat_simulator.filtering_type', 'PORT_DEPENDENT'], ['media.peerconnection.nat_simulator.mapping_type', 'PORT_DEPENDENT'], ['media.peerconnection.nat_simulator.block_tcp', true], + // The above triggers warning about 5 ICE servers + ['media.peerconnection.treat_warnings_as_errors', false], ['media.getusermedia.insecure.enabled', true]); options.expectedLocalCandidateType = "srflx"; options.expectedRemoteCandidateType = "relay"; diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTCP.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTCP.html index 7bb51764bd..387341151f 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTCP.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTCP.html @@ -26,6 +26,8 @@ if (!("mediaDevices" in navigator)) { ['media.peerconnection.nat_simulator.block_tcp', false], ['media.peerconnection.nat_simulator.block_tls', true], ['media.peerconnection.ice.loopback', true], + // The above sets up 5+ ICE servers which triggers a warning + ['media.peerconnection.treat_warnings_as_errors', false], ['media.getusermedia.insecure.enabled', true]); options.expectedLocalCandidateType = "relay-tcp"; options.expectedRemoteCandidateType = "relay-tcp"; diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTLS.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTLS.html index 7446401f87..2ac886c156 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTLS.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATRelayTLS.html @@ -25,6 +25,8 @@ if (!("mediaDevices" in navigator)) { ['media.peerconnection.nat_simulator.block_udp', true], ['media.peerconnection.nat_simulator.block_tcp', true], ['media.peerconnection.ice.loopback', true], + // The above triggers warning about 5 ICE servers + ['media.peerconnection.treat_warnings_as_errors', false], ['media.getusermedia.insecure.enabled', true]); options.expectedLocalCandidateType = "relay-tls"; options.expectedRemoteCandidateType = "relay-tls"; diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATSrflx.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATSrflx.html index 78fa8bcb2c..69fc2af80a 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATSrflx.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNATSrflx.html @@ -24,6 +24,8 @@ if (!("mediaDevices" in navigator)) { ['media.peerconnection.nat_simulator.filtering_type', 'ENDPOINT_INDEPENDENT'], ['media.peerconnection.nat_simulator.mapping_type', 'ENDPOINT_INDEPENDENT'], ['media.peerconnection.nat_simulator.block_tcp', true], + // The above triggers warning about 5 ICE servers + ['media.peerconnection.treat_warnings_as_errors', false], ['media.getusermedia.insecure.enabled', true]); options.expectedLocalCandidateType = "srflx"; options.expectedRemoteCandidateType = "srflx"; diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNoisyUDPBlock.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNoisyUDPBlock.html index 297121cd94..4175419d44 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNoisyUDPBlock.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioNoisyUDPBlock.html @@ -25,6 +25,8 @@ if (!("mediaDevices" in navigator)) { ['media.peerconnection.nat_simulator.block_udp', true], ['media.peerconnection.nat_simulator.error_code_for_drop', 3 /*R_INTERNAL*/], ['media.peerconnection.nat_simulator.block_tls', true], + // The above triggers warning about 5 ICE servers + ['media.peerconnection.treat_warnings_as_errors', false], ['media.getusermedia.insecure.enabled', true]); options.expectedLocalCandidateType = "relay-tcp"; options.expectedRemoteCandidateType = "relay-tcp"; diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioRelayPolicy.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioRelayPolicy.html index ced57ff8a3..313d76441b 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioRelayPolicy.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_basicAudioRelayPolicy.html @@ -14,7 +14,10 @@ createHTML({ runNetworkTest(async () => { await pushPrefs( // Enable mDNS, since there are some checks we want to run with that - ['media.peerconnection.ice.obfuscate_host_addresses', true]); + ['media.peerconnection.ice.obfuscate_host_addresses', true], + // The above triggers warning about 5 ICE servers + ['media.peerconnection.treat_warnings_as_errors', false], + ); const offerer = new RTCPeerConnection({iceServers: iceServersArray, iceTransportPolicy: 'relay'}); const answerer = new RTCPeerConnection({iceServers: iceServersArray}); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_bug825703.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_bug825703.html index 5cd168af8a..5efb4a17d2 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_bug825703.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_bug825703.html @@ -25,7 +25,7 @@ var makePC = (config, expected_error) => { exception = e; } is((exception? exception.name : "success"), expected_error || "success", - "RTCPeerConnection(" + JSON.stringify(config) + ")"); + "RTCPeerConnection(" + JSON.stringify(config) + ") " + exception?.message); }; // The order of properties in objects is not guaranteed in JavaScript, so this @@ -39,8 +39,8 @@ var toComparable = o => }, {}); // This is a test of the iceServers parsing code + readable errors -runNetworkTest(() => { - var exception = null; +runNetworkTest(async () => { + let exception = null; try { new RTCPeerConnection().close(); @@ -56,26 +56,37 @@ runNetworkTest(() => { { urls:"stun:127.0.0.1" }, { urls:"stun:localhost", foo:"" }, { urls: ["stun:127.0.0.1", "stun:localhost"] }, - { urls:"stuns:localhost", foo:"" }, + ]}); + makePC({ iceServers: [ { urls:"turn:[::1]:3478", username:"p", credential:"p" }, { urls:"turn:[::1]:3478", username:"", credential:"" }, { urls:"turns:[::1]:3478", username:"", credential:"" }, + ]}); + makePC({ iceServers: [ { urls:"turn:localhost:3478?transport=udp", username:"p", credential:"p" }, { urls: ["turn:[::1]:3478", "turn:localhost"], username:"p", credential:"p" }, { urls:"turns:localhost:3478?transport=udp", username:"p", credential:"p" }, - { url:"stun:localhost", foo:"" }, - { url:"turn:localhost", username:"p", credential:"p" } ]}); - makePC({ iceServers: [{ urls:"http:0.0.0.0" }] }, "SyntaxError"); try { - new RTCPeerConnection({ iceServers: [{ url:"http:0.0.0.0" }] }).close(); + new RTCPeerConnection({ iceServers: [{ urls:"http:0.0.0.0" }] }).close(); } catch (e) { ok(e.message.indexOf("http") > 0, "RTCPeerConnection() constructor has readable exceptions"); } + const push = prefs => SpecialPowers.pushPrefEnv(prefs); + + // Remaining tests trigger warnings + await push({ set: [['media.peerconnection.treat_warnings_as_errors', false]] }); + + makePC({ iceServers: [ + { urls:"stuns:localhost", foo:"" }, + { url:"stun:localhost", foo:"" }, + { url:"turn:localhost", username:"p", credential:"p" } + ]}); + // Test getConfiguration const config = { bundlePolicy: "max-bundle", @@ -98,41 +109,35 @@ runNetworkTest(() => { JSON.stringify(toComparable(config)), "getConfiguration"); pc.close(); - var push = prefs => SpecialPowers.pushPrefEnv(prefs); - - return Promise.resolve() // This set of tests are setting the about:config User preferences for default // ice servers and checking the outputs when RTCPeerConnection() is // invoked. See Bug 1167922 for more information. - .then(() => push({ set: [['media.peerconnection.default_iceservers', ""]] }) - .then(() => makePC()) - .then(() => push({ set: [['media.peerconnection.default_iceservers', "k"]] })) - .then(() => makePC()) - .then(() => push({ set: [['media.peerconnection.default_iceservers', "[{\"urls\": [\"stun:stun.services.mozilla.com\"]}]"]] })) - .then(() => makePC())) + await push({ set: [['media.peerconnection.default_iceservers', ""]] }); + makePC(); + await push({ set: [['media.peerconnection.default_iceservers', "k"]] }); + makePC(); + await push({ set: [['media.peerconnection.default_iceservers', + "[{\"urls\": [\"stun:stun.services.mozilla.com\"]}]"]]}); + makePC(); // This set of tests check that warnings work. See Bug 1254839 for more. - .then(() => { - let promise = new Promise(resolve => { - SpecialPowers.registerConsoleListener(msg => { - if (msg.message.includes("onaddstream")) { - SpecialPowers.postConsoleSentinel(); - resolve(msg.message); - } - }); + const warning = await new Promise(resolve => { + SpecialPowers.registerConsoleListener(msg => { + if (msg.message.includes("onaddstream")) { + SpecialPowers.postConsoleSentinel(); + resolve(msg.message); + } }); lineNumberAndFunction.func(); - return promise; - }).then(warning => { - is(warning.split('"')[1], - "WebRTC: onaddstream is deprecated! Use peerConnection.ontrack instead.", - "warning logged"); - var remainder = warning.split('"').slice(2).join('"'); - info(remainder); - ok(remainder.includes('file: "' + window.location + '"'), - "warning has this file"); - ok(remainder.includes('line: ' + lineNumberAndFunction.line), - "warning has correct line number"); - }); + }); + is(warning.split('"')[1], + "WebRTC: onaddstream is deprecated! Use peerConnection.ontrack instead.", + "warning logged"); + const remainder = warning.split('"').slice(2).join('"'); + info(remainder); + ok(remainder.includes('file: "' + window.location + '"'), + "warning has this file"); + ok(remainder.includes('line: ' + lineNumberAndFunction.line), + "warning has correct line number"); }); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_callbacks.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_callbacks.html index 4c890e4400..ffd7a1b0c1 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_callbacks.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_callbacks.html @@ -1,4 +1,4 @@ - + @@ -50,22 +50,27 @@ pc2.onicecandidate = e => { .catch(generateErrorCallback()); }; -var v1, v2; -var delivered = new Promise(resolve => { - pc2.onaddstream = e => { - v2.srcObject = e.stream; - resolve(e.stream); - }; -}); +runNetworkTest(async function() { + // Tests trigger warnings + await SpecialPowers.pushPrefEnv({ + set: [['media.peerconnection.treat_warnings_as_errors', false]] + }); + + const v1 = createMediaElement('video', 'v1'); + const v2 = createMediaElement('video', 'v2'); + + const delivered = new Promise(resolve => { + pc2.onaddstream = e => { + v2.srcObject = e.stream; + resolve(e.stream); + }; + }); -runNetworkTest(function() { - v1 = createMediaElement('video', 'v1'); - v2 = createMediaElement('video', 'v2'); var canPlayThrough = new Promise(resolve => v2.canplaythrough = resolve); is(v2.currentTime, 0, "v2.currentTime is zero at outset"); // not testing legacy gUM here - return navigator.mediaDevices.getUserMedia({ video: true, audio: true }) + await navigator.mediaDevices.getUserMedia({ video: true, audio: true }) .then(stream => pc1.addStream(v1.srcObject = stream)) .then(() => pcall(pc1, pc1.createOffer)) .then(offer => pcall(pc1, pc1.setLocalDescription, offer)) diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_localRollback.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_localRollback.html index 5bdc8cc029..8c9470e7f2 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_localRollback.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_localRollback.html @@ -32,7 +32,7 @@ // Rolling back should shut down gathering function PC_REMOTE_WAIT_FOR_END_OF_TRICKLE(test) { - return test.pcRemote.endOfTrickleIce; + is(test.pcRemote._pc.iceGatheringState, "new"); }, function PC_REMOTE_SETUP_ICE_HANDLER(test) { diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_portRestrictions.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_portRestrictions.html index 7cd695ff54..589d8ed6ad 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_portRestrictions.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_portRestrictions.html @@ -46,10 +46,14 @@ runNetworkTest(() => { { urls:"turn:[::1]:5349", username:"p", credential:"p" }, { urls:"turn:[::1]:3478", username:"p", credential:"p" }, { urls:"turn:[::1]", username:"p", credential:"p" }, + ]}); + makePC({ iceServers: [ { urls:"turn:localhost:53?transport=udp", username:"p", credential:"p" }, { urls:"turn:localhost:3478?transport=udp", username:"p", credential:"p" }, { urls:"turn:localhost:53?transport=tcp", username:"p", credential:"p" }, { urls:"turn:localhost:3478?transport=tcp", username:"p", credential:"p" }, + ]}); + makePC({ iceServers: [ { urls:"turns:localhost:3478?transport=udp", username:"p", credential:"p" }, { urls:"stun:localhost", foo:"" } ]}); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIce.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIce.html index d94bb084b7..add3882a4d 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIce.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIce.html @@ -21,12 +21,14 @@ function PC_LOCAL_SET_OFFER_OPTION(test) { test.setOfferOptions({ iceRestart: true }); }, - function PC_LOCAL_EXPECT_ICE_CHECKING(test) { - test.pcLocal.expectIceChecking(); + // Make sure we don't get the end of gathering racing against the + // setting of the new offer + function PC_LOCAL_WAIT_FOR_END_OF_TRICKLE(test) { + return test.pcLocal.endOfTrickleIce; + }, + function PC_LOCAL_SETUP_ICE_HANDLER(test) { + test.pcLocal.setupIceCandidateHandler(test); }, - function PC_REMOTE_EXPECT_ICE_CHECKING(test) { - test.pcRemote.expectIceChecking(); - } ] ); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalAndRemoteRollback.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalAndRemoteRollback.html index 6bbf9440fc..3307258ef9 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalAndRemoteRollback.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalAndRemoteRollback.html @@ -31,10 +31,6 @@ test.chain.replaceAfter('PC_REMOTE_CREATE_ANSWER', [ - function PC_LOCAL_EXPECT_ICE_CONNECTED(test) { - test.pcLocal.iceCheckingIceRollbackExpected = true; - }, - function PC_REMOTE_ROLLBACK(test) { return test.setRemoteDescription(test.pcRemote, { type: "rollback" }, STABLE); @@ -58,12 +54,6 @@ return test.pcLocal.endOfTrickleIce; }, - function PC_LOCAL_EXPECT_ICE_CHECKING(test) { - test.pcLocal.expectIceChecking(); - }, - function PC_REMOTE_EXPECT_ICE_CHECKING(test) { - test.pcRemote.expectIceChecking(); - } ], 1 // Replaces after second PC_REMOTE_CREATE_ANSWER ); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalRollback.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalRollback.html index f5f9a1f220..a7adb7c3e9 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalRollback.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceLocalRollback.html @@ -33,9 +33,6 @@ HAVE_LOCAL_OFFER); }); }, - function PC_LOCAL_EXPECT_ICE_CONNECTED(test) { - test.pcLocal.iceCheckingIceRollbackExpected = true; - }, function PC_LOCAL_WAIT_FOR_GATHERING(test) { return new Promise(r => { test.pcLocal._pc.addEventListener("icegatheringstatechange", () => { @@ -54,12 +51,6 @@ function PC_LOCAL_WAIT_FOR_END_OF_TRICKLE(test) { return test.pcLocal.endOfTrickleIce; }, - function PC_LOCAL_EXPECT_ICE_CHECKING(test) { - test.pcLocal.expectIceChecking(); - }, - function PC_REMOTE_EXPECT_ICE_CHECKING(test) { - test.pcRemote.expectIceChecking(); - } ] ); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundle.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundle.html index 134fa97cc0..ef40d4039c 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundle.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundle.html @@ -23,12 +23,14 @@ function PC_LOCAL_SET_OFFER_OPTION(test) { test.setOfferOptions({ iceRestart: true }); }, - function PC_LOCAL_EXPECT_ICE_CHECKING(test) { - test.pcLocal.expectIceChecking(); + // Make sure we don't get the end of gathering racing against the + // setting of the new offer + function PC_LOCAL_WAIT_FOR_END_OF_TRICKLE(test) { + return test.pcLocal.endOfTrickleIce; + }, + function PC_LOCAL_SETUP_ICE_HANDLER(test) { + test.pcLocal.setupIceCandidateHandler(test); }, - function PC_REMOTE_EXPECT_ICE_CHECKING(test) { - test.pcRemote.expectIceChecking(); - } ] ); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundleNoRtcpMux.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundleNoRtcpMux.html index 06a3a3c980..85a3fcc45d 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundleNoRtcpMux.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoBundleNoRtcpMux.html @@ -24,12 +24,14 @@ function PC_LOCAL_SET_OFFER_OPTION(test) { test.setOfferOptions({ iceRestart: true }); }, - function PC_LOCAL_EXPECT_ICE_CHECKING(test) { - test.pcLocal.expectIceChecking(); + // Make sure we don't get the end of gathering racing against the + // setting of the new offer + function PC_LOCAL_WAIT_FOR_END_OF_TRICKLE(test) { + return test.pcLocal.endOfTrickleIce; + }, + function PC_LOCAL_SETUP_ICE_HANDLER(test) { + test.pcLocal.setupIceCandidateHandler(test); }, - function PC_REMOTE_EXPECT_ICE_CHECKING(test) { - test.pcRemote.expectIceChecking(); - } ] ); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoRtcpMux.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoRtcpMux.html index 5d4780211a..80fd1090bc 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoRtcpMux.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_restartIceNoRtcpMux.html @@ -23,12 +23,15 @@ function PC_LOCAL_SET_OFFER_OPTION(test) { test.setOfferOptions({ iceRestart: true }); }, - function PC_LOCAL_EXPECT_ICE_CHECKING(test) { - test.pcLocal.expectIceChecking(); + // Make sure we don't get the end of gathering racing against the + // setting of the new offer + function PC_LOCAL_WAIT_FOR_END_OF_TRICKLE(test) { + return test.pcLocal.endOfTrickleIce; + }, + // Expect gathering to start again + function PC_LOCAL_SETUP_ICE_HANDLER(test) { + test.pcLocal.setupIceCandidateHandler(test); }, - function PC_REMOTE_EXPECT_ICE_CHECKING(test) { - test.pcRemote.expectIceChecking(); - } ] ); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_sillyCodecPriorities.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_sillyCodecPriorities.html index c8354356b0..aa0644ebbb 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_sillyCodecPriorities.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_sillyCodecPriorities.html @@ -15,12 +15,12 @@ visible: true }); - function makeCodecTopPriority(sdp, codec) { + function makeCodecTopPriority({type, sdp}, codec) { const ptToMove = sdputils.findCodecId(sdp, codec); - return sdp.replace( + return {type, sdp: sdp.replace( // m=video port type pts ptToMove more-pts? new RegExp(`(m=video [^ ]+ [^ ]+)(.*)( ${ptToMove})( [^ ]+)?`, "g"), - '$1$3$2$4'); + '$1$3$2$4')}; } function isCodecFirst(sdp, codec) { @@ -34,8 +34,7 @@ const stream = await navigator.mediaDevices.getUserMedia({ video: true }); const sender = pc1.addTrack(stream.getTracks()[0]); await pc1.setLocalDescription(); - let mungedOffer = pc1.localDescription; - mungedOffer.sdp = makeCodecTopPriority(mungedOffer.sdp, codec); + const mungedOffer = makeCodecTopPriority(pc1.localDescription, codec); await pc2.setRemoteDescription(mungedOffer); await pc2.setLocalDescription(); await pc1.setRemoteDescription(pc2.localDescription); @@ -51,8 +50,7 @@ await pc1.setLocalDescription(); await pc2.setRemoteDescription(pc1.localDescription); await pc2.setLocalDescription(); - let mungedAnswer = pc2.localDescription; - mungedAnswer.sdp = makeCodecTopPriority(mungedAnswer.sdp, codec); + const mungedAnswer = makeCodecTopPriority(pc2.localDescription, codec); await pc1.setRemoteDescription(mungedAnswer); is(isCodecFirst((await pc1.createOffer()).sdp, codec), !isPseudoCodec, "Top-priority codecs should come first in reoffers, unless they are pseudo codecs (eg; ulpfec)"); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_stats_relayProtocol.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_stats_relayProtocol.html index cdc328fd2b..5146d423a6 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_stats_relayProtocol.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_stats_relayProtocol.html @@ -23,6 +23,8 @@ if (!("mediaDevices" in navigator)) { await pushPrefs( ['media.peerconnection.nat_simulator.filtering_type', 'ENDPOINT_INDEPENDENT'], ['media.peerconnection.nat_simulator.mapping_type', 'ENDPOINT_INDEPENDENT'], + // The above triggers warning about 5 ICE servers + ['media.peerconnection.treat_warnings_as_errors', false], ['media.getusermedia.insecure.enabled', true]); const test = new PeerConnectionTest(options); makeOffererNonTrickle(test.chain); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_threeUnbundledConnections.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_threeUnbundledConnections.html index 75f0d12463..8627590b65 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_threeUnbundledConnections.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_threeUnbundledConnections.html @@ -98,10 +98,10 @@ runNetworkTest(async () => { //info("Original OFFER: " + JSON.stringify(offer)); offer.sdp = sdputils.removeBundle(offer.sdp); //info("OFFER w/o BUNDLE: " + JSON.stringify(offer)); - const offerAudio = new RTCSessionDescription(JSON.parse(JSON.stringify(offer))); + const offerAudio = JSON.parse(JSON.stringify(offer)); offerAudio.sdp = offerAudio.sdp.replace('m=video 9', 'm=video 0'); //info("offerAudio: " + JSON.stringify(offerAudio)); - const offerVideo = new RTCSessionDescription(JSON.parse(JSON.stringify(offer))); + const offerVideo = JSON.parse(JSON.stringify(offer)); offerVideo.sdp = offerVideo.sdp.replace('m=audio 9', 'm=audio 0'); //info("offerVideo: " + JSON.stringify(offerVideo)); diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_throwInCallbacks.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_throwInCallbacks.html index c476c60161..342b72a5e1 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_throwInCallbacks.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_throwInCallbacks.html @@ -11,7 +11,12 @@ title: "Throw in PeerConnection callbacks" }); -runNetworkTest(function () { +runNetworkTest(async function () { + // Tests trigger warnings + await SpecialPowers.pushPrefEnv({ + set: [['media.peerconnection.treat_warnings_as_errors', false]] + }); + let finish; const onfinished = new Promise(r => finish = async () => { window.onerror = oldOnError; diff --git a/dom/media/webrtc/tests/mochitests/test_peerConnection_verifyDescriptions.html b/dom/media/webrtc/tests/mochitests/test_peerConnection_verifyDescriptions.html index f685f7c99a..8c12a50942 100644 --- a/dom/media/webrtc/tests/mochitests/test_peerConnection_verifyDescriptions.html +++ b/dom/media/webrtc/tests/mochitests/test_peerConnection_verifyDescriptions.html @@ -8,49 +8,70 @@ diff --git a/dom/media/webrtc/third_party_build/default_config_env b/dom/media/webrtc/third_party_build/default_config_env index af8b77bba6..7013520a30 100644 --- a/dom/media/webrtc/third_party_build/default_config_env +++ b/dom/media/webrtc/third_party_build/default_config_env @@ -5,41 +5,41 @@ export MOZ_LIBWEBRTC_SRC=$STATE_DIR/moz-libwebrtc # The previous fast-forward bug number is used for some error messaging. -export MOZ_PRIOR_FASTFORWARD_BUG="1867099" +export MOZ_PRIOR_FASTFORWARD_BUG="1871981" # Fast-forwarding each Chromium version of libwebrtc should be done # under a separate bugzilla bug. This bug number is used when crafting # the commit summary as each upstream commit is vendored into the # mercurial repository. The bug used for the v106 fast-forward was # 1800920. -export MOZ_FASTFORWARD_BUG="1871981" +export MOZ_FASTFORWARD_BUG="1876843" # MOZ_NEXT_LIBWEBRTC_MILESTONE and MOZ_NEXT_FIREFOX_REL_TARGET are # not used during fast-forward processing, but facilitate generating this # default config. To generate an default config for the next update, run # bash dom/media/webrtc/third_party_build/update_default_config_env.sh -export MOZ_NEXT_LIBWEBRTC_MILESTONE=120 -export MOZ_NEXT_FIREFOX_REL_TARGET=124 +export MOZ_NEXT_LIBWEBRTC_MILESTONE=121 +export MOZ_NEXT_FIREFOX_REL_TARGET=125 # For Chromium release branches, see: # https://chromiumdash.appspot.com/branches -# Chromium's v119 release branch was 6045. This is used to pre-stack +# Chromium's v120 release branch was 6099. This is used to pre-stack # the previous release branch's commits onto the appropriate base commit # (the first common commit between trunk and the release branch). -export MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM="6045" +export MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM="6099" -# New target release branch for v120 is branch-heads/6099. This is used +# New target release branch for v121 is branch-heads/6167. This is used # to calculate the next upstream commit. -export MOZ_TARGET_UPSTREAM_BRANCH_HEAD="branch-heads/6099" +export MOZ_TARGET_UPSTREAM_BRANCH_HEAD="branch-heads/6167" # For local development 'mozpatches' is fine for a branch name, but when # pushing the patch stack to github, it should be named something like -# 'moz-mods-chr120-for-rel124'. +# 'moz-mods-chr121-for-rel125'. export MOZ_LIBWEBRTC_BRANCH="mozpatches" # After elm has been merged to mozilla-central, the patch stack in # moz-libwebrtc should be pushed to github. The script # push_official_branch.sh uses this branch name when pushing to the # public repo. -export MOZ_LIBWEBRTC_OFFICIAL_BRANCH="moz-mods-chr120-for-rel124" +export MOZ_LIBWEBRTC_OFFICIAL_BRANCH="moz-mods-chr121-for-rel125" diff --git a/dom/media/webrtc/third_party_build/filter_git_changes.py b/dom/media/webrtc/third_party_build/filter_git_changes.py index ee6cdbcbd9..86358b0b01 100644 --- a/dom/media/webrtc/third_party_build/filter_git_changes.py +++ b/dom/media/webrtc/third_party_build/filter_git_changes.py @@ -48,7 +48,7 @@ def filter_git_changes(github_path, commit_sha, diff_filter): # out the excluded directory paths (note the lack of trailing '$' # in the regex). regex_excludes = "|".join( - ["^(M|A|D|R\d\d\d)\t{}".format(i) for i in exclude_dir_list] + ["^(M|A|D|R\\d\\d\\d)\t{}".format(i) for i in exclude_dir_list] ) files_not_excluded = [ path for path in changed_files if not re.findall(regex_excludes, path) diff --git a/dom/media/webrtc/third_party_build/prep_repo.sh b/dom/media/webrtc/third_party_build/prep_repo.sh index b601ebc808..8cd9ff6816 100644 --- a/dom/media/webrtc/third_party_build/prep_repo.sh +++ b/dom/media/webrtc/third_party_build/prep_repo.sh @@ -69,6 +69,12 @@ rm -f *.patch CHERRY_PICK_BASE=`git merge-base branch-heads/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM master` echo "common commit: $CHERRY_PICK_BASE" +# find the last upstream commit used by the previous update, so we don't +# accidentally grab release branch commits that were added after we started +# the previous update. +LAST_UPSTREAM_COMMIT_SHA=`tail -1 $CURRENT_DIR/third_party/libwebrtc/README.moz-ff-commit` +echo "previous update's last commit: $LAST_UPSTREAM_COMMIT_SHA" + # create a new branch at the common commit and checkout the new branch ERROR_HELP=$" Unable to create branch '$MOZ_LIBWEBRTC_BRANCH'. This probably means @@ -89,7 +95,7 @@ git checkout $MOZ_LIBWEBRTC_BRANCH rm -f $TMP_DIR/*.patch $TMP_DIR/*.patch.bak # grab the patches for all the commits in chrome's release branch for libwebrtc -git format-patch -o $TMP_DIR -k $CHERRY_PICK_BASE..branch-heads/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM +git format-patch -o $TMP_DIR -k $CHERRY_PICK_BASE..$LAST_UPSTREAM_COMMIT_SHA # tweak the release branch commit summaries to show they were cherry picked sed -i.bak -e "/^Subject: / s/^Subject: /Subject: (cherry-pick-branch-heads\/$MOZ_PRIOR_UPSTREAM_BRANCH_HEAD_NUM) /" $TMP_DIR/*.patch git am $TMP_DIR/*.patch # applies to branch mozpatches diff --git a/dom/media/webrtc/third_party_build/save_patch_stack.py b/dom/media/webrtc/third_party_build/save_patch_stack.py index 9b080ac0ff..907b90f593 100644 --- a/dom/media/webrtc/third_party_build/save_patch_stack.py +++ b/dom/media/webrtc/third_party_build/save_patch_stack.py @@ -52,7 +52,7 @@ def save_patch_stack( # remove the commit summary from the file name patches_to_rename = os.listdir(patch_directory) for file in patches_to_rename: - shortened_name = re.sub("^(\d\d\d\d)-.*\.patch", "\\1.patch", file) + shortened_name = re.sub(r"^(\d\d\d\d)-.*\.patch", "\\1.patch", file) os.rename( os.path.join(patch_directory, file), os.path.join(patch_directory, shortened_name), diff --git a/dom/media/webrtc/third_party_build/verify_vendoring.sh b/dom/media/webrtc/third_party_build/verify_vendoring.sh index 869a2c8c67..008ab0d5db 100644 --- a/dom/media/webrtc/third_party_build/verify_vendoring.sh +++ b/dom/media/webrtc/third_party_build/verify_vendoring.sh @@ -17,12 +17,12 @@ echo "MOZ_LIBWEBRTC_SRC: $MOZ_LIBWEBRTC_SRC" echo "MOZ_LIBWEBRTC_BRANCH: $MOZ_LIBWEBRTC_BRANCH" echo "MOZ_FASTFORWARD_BUG: $MOZ_FASTFORWARD_BUG" -TIP_SHA=`hg id -r tip | awk '{ print $1; }'` -echo "TIP_SHA: $TIP_SHA" +CURRENT_SHA=`hg id -r . | awk '{ print $1; }'` +echo "CURRENT_SHA: $CURRENT_SHA" # we grab the entire firstline description for convenient logging -LAST_PATCHSTACK_UPDATE_COMMIT=`hg log --template "{node|short} {desc|firstline}\n" \ - --include "third_party/libwebrtc/moz-patch-stack/*.patch" | head -1` +LAST_PATCHSTACK_UPDATE_COMMIT=`hg log -r ::. --template "{node|short} {desc|firstline}\n" \ + --include "third_party/libwebrtc/moz-patch-stack/*.patch" | tail -1` echo "LAST_PATCHSTACK_UPDATE_COMMIT: $LAST_PATCHSTACK_UPDATE_COMMIT" LAST_PATCHSTACK_UPDATE_COMMIT_SHA=`echo $LAST_PATCHSTACK_UPDATE_COMMIT \ @@ -31,7 +31,7 @@ echo "LAST_PATCHSTACK_UPDATE_COMMIT_SHA: $LAST_PATCHSTACK_UPDATE_COMMIT_SHA" # grab the oldest, non "Vendor from libwebrtc" line OLDEST_CANDIDATE_COMMIT=`hg log --template "{node|short} {desc|firstline}\n" \ - -r $LAST_PATCHSTACK_UPDATE_COMMIT_SHA::tip \ + -r $LAST_PATCHSTACK_UPDATE_COMMIT_SHA::. \ | grep -v "Vendor libwebrtc from" | head -1` echo "OLDEST_CANDIDATE_COMMIT: $OLDEST_CANDIDATE_COMMIT" @@ -39,9 +39,9 @@ OLDEST_CANDIDATE_SHA=`echo $OLDEST_CANDIDATE_COMMIT \ | awk '{ print $1; }'` echo "OLDEST_CANDIDATE_SHA: $OLDEST_CANDIDATE_SHA" -EXTRACT_COMMIT_RANGE="{start-commit-sha}::tip" -if [ "x$TIP_SHA" != "x$OLDEST_CANDIDATE_SHA" ]; then - EXTRACT_COMMIT_RANGE="$OLDEST_CANDIDATE_SHA::tip" +EXTRACT_COMMIT_RANGE="{start-commit-sha}::." +if [ "x$CURRENT_SHA" != "x$OLDEST_CANDIDATE_SHA" ]; then + EXTRACT_COMMIT_RANGE="$OLDEST_CANDIDATE_SHA::." echo "EXTRACT_COMMIT_RANGE: $EXTRACT_COMMIT_RANGE" fi diff --git a/dom/media/webrtc/transport/nricectx.cpp b/dom/media/webrtc/transport/nricectx.cpp index f30c2734c2..7c71c0ab06 100644 --- a/dom/media/webrtc/transport/nricectx.cpp +++ b/dom/media/webrtc/transport/nricectx.cpp @@ -257,14 +257,14 @@ nsresult NrIceTurnServer::ToNicerTurnStruct(nr_ice_turn_server* server) const { } NrIceCtx::NrIceCtx(const std::string& name) - : connection_state_(ICE_CTX_INIT), - gathering_state_(ICE_CTX_GATHER_INIT), - name_(name), + : name_(name), ice_controlling_set_(false), ctx_(nullptr), peer_(nullptr), ice_handler_vtbl_(nullptr), ice_handler_(nullptr), + ice_gather_handler_vtbl_(nullptr), + ice_gather_handler_(nullptr), trickle_(true), config_(), nat_(nullptr), @@ -343,13 +343,10 @@ void NrIceCtx::DestroyStream(const std::string& id) { auto it = streams_.find(id); if (it != streams_.end()) { auto preexisting_stream = it->second; + SignalConnectionStateChange(preexisting_stream, ICE_CTX_CLOSED); streams_.erase(it); preexisting_stream->Close(); } - - if (streams_.empty()) { - SetGatheringState(ICE_CTX_GATHER_INIT); - } } // Handler callbacks @@ -377,6 +374,7 @@ int NrIceCtx::stream_ready(void* obj, nr_ice_media_stream* stream) { MOZ_ASSERT(s); s->Ready(stream); + ctx->SignalConnectionStateChange(s, ICE_CTX_CONNECTED); return 0; } @@ -393,44 +391,101 @@ int NrIceCtx::stream_failed(void* obj, nr_ice_media_stream* stream) { // Streams which do not exist should never fail. MOZ_ASSERT(s); - ctx->SetConnectionState(ICE_CTX_FAILED); + if (!ctx->dumped_rlog_) { + // Do this at most once per ctx + ctx->dumped_rlog_ = true; + MOZ_MTLOG(ML_INFO, + "NrIceCtx(" << ctx->name_ << "): dumping r_log ringbuffer... "); + std::deque logs; + RLogConnector::GetInstance()->GetAny(0, &logs); + for (auto& log : logs) { + MOZ_MTLOG(ML_INFO, log); + } + } + s->Failed(); + ctx->SignalConnectionStateChange(s, ICE_CTX_FAILED); return 0; } -int NrIceCtx::ice_checking(void* obj, nr_ice_peer_ctx* pctx) { - MOZ_MTLOG(ML_DEBUG, "ice_checking called"); +int NrIceCtx::stream_checking(void* obj, nr_ice_media_stream* stream) { + MOZ_MTLOG(ML_DEBUG, "stream_checking called"); + MOZ_ASSERT(!stream->local_stream); + MOZ_ASSERT(!stream->obsolete); // Get the ICE ctx NrIceCtx* ctx = static_cast(obj); + RefPtr s = ctx->FindStream(stream); - ctx->SetConnectionState(ICE_CTX_CHECKING); + MOZ_ASSERT(s); + if (!s->AnyGenerationIsConnected()) { + // the checking state only applies if we aren't connected + ctx->SignalConnectionStateChange(s, ICE_CTX_CHECKING); + } return 0; } -int NrIceCtx::ice_connected(void* obj, nr_ice_peer_ctx* pctx) { - MOZ_MTLOG(ML_DEBUG, "ice_connected called"); +int NrIceCtx::stream_disconnected(void* obj, nr_ice_media_stream* stream) { + MOZ_MTLOG(ML_DEBUG, "stream_disconnected called"); + MOZ_ASSERT(!stream->local_stream); + MOZ_ASSERT(!stream->obsolete); // Get the ICE ctx NrIceCtx* ctx = static_cast(obj); + RefPtr s = ctx->FindStream(stream); - // This is called even on failed contexts. - if (ctx->connection_state() != ICE_CTX_FAILED) { - ctx->SetConnectionState(ICE_CTX_CONNECTED); - } + MOZ_ASSERT(s); + ctx->SignalConnectionStateChange(s, ICE_CTX_DISCONNECTED); return 0; } -int NrIceCtx::ice_disconnected(void* obj, nr_ice_peer_ctx* pctx) { - MOZ_MTLOG(ML_DEBUG, "ice_disconnected called"); +int NrIceCtx::stream_gathering(void* obj, nr_ice_media_stream* stream) { + MOZ_MTLOG(ML_DEBUG, "stream_gathering called"); + MOZ_ASSERT(!stream->local_stream); + MOZ_ASSERT(!stream->obsolete); + + // Get the ICE ctx + NrIceCtx* ctx = static_cast(obj); + RefPtr s = ctx->FindStream(stream); + + MOZ_ASSERT(s); + + s->OnGatheringStarted(stream); + return 0; +} + +int NrIceCtx::stream_gathered(void* obj, nr_ice_media_stream* stream) { + MOZ_MTLOG(ML_DEBUG, "stream_gathered called"); + MOZ_ASSERT(!stream->local_stream); // Get the ICE ctx NrIceCtx* ctx = static_cast(obj); + RefPtr s = ctx->FindStream(stream); - ctx->SetConnectionState(ICE_CTX_DISCONNECTED); + // We get this callback for destroyed streams in some cases + if (s) { + s->OnGatheringComplete(stream); + } + return 0; +} + +int NrIceCtx::ice_checking(void* obj, nr_ice_peer_ctx* pctx) { + MOZ_MTLOG(ML_DEBUG, "ice_checking called"); + // We don't use this; we react to the stream-specific callbacks instead + return 0; +} + +int NrIceCtx::ice_connected(void* obj, nr_ice_peer_ctx* pctx) { + MOZ_MTLOG(ML_DEBUG, "ice_connected called"); + // We don't use this; we react to the stream-specific callbacks instead + return 0; +} +int NrIceCtx::ice_disconnected(void* obj, nr_ice_peer_ctx* pctx) { + MOZ_MTLOG(ML_DEBUG, "ice_disconnected called"); + // We don't use this; we react to the stream-specific callbacks instead return 0; } @@ -466,7 +521,6 @@ void NrIceCtx::trickle_cb(void* arg, nr_ice_ctx* ice_ctx, } if (!candidate) { - s->SignalCandidate(s, "", stream->ufrag, "", ""); return; } @@ -587,11 +641,20 @@ void NrIceCtx::SetStunAddrs(const nsTArray& addrs) { } bool NrIceCtx::Initialize() { + // Create the gather handler objects + ice_gather_handler_vtbl_ = new nr_ice_gather_handler_vtbl(); + ice_gather_handler_vtbl_->stream_gathering = &NrIceCtx::stream_gathering; + ice_gather_handler_vtbl_->stream_gathered = &NrIceCtx::stream_gathered; + ice_gather_handler_ = new nr_ice_gather_handler(); + ice_gather_handler_->vtbl = ice_gather_handler_vtbl_; + ice_gather_handler_->obj = this; + // Create the ICE context int r; UINT4 flags = NR_ICE_CTX_FLAGS_AGGRESSIVE_NOMINATION; - r = nr_ice_ctx_create(const_cast(name_.c_str()), flags, &ctx_); + r = nr_ice_ctx_create(const_cast(name_.c_str()), flags, + ice_gather_handler_, &ctx_); if (r) { MOZ_MTLOG(ML_ERROR, "Couldn't create ICE ctx for '" << name_ << "'"); @@ -634,6 +697,16 @@ bool NrIceCtx::Initialize() { ice_handler_vtbl_->select_pair = &NrIceCtx::select_pair; ice_handler_vtbl_->stream_ready = &NrIceCtx::stream_ready; ice_handler_vtbl_->stream_failed = &NrIceCtx::stream_failed; + ice_handler_vtbl_->stream_checking = &NrIceCtx::stream_checking; + ice_handler_vtbl_->stream_disconnected = &NrIceCtx::stream_disconnected; + // stream_gathering and stream_gathered do not go here, since those are tied + // to the _local_ nr_ice_media_stream in nICEr. nICEr allows a local + // nr_ice_media_stream (which has a single set of candidates, and therefore a + // single gathering state) to be associated with multiple remote + // nr_ice_media_streams (which each have their own ICE connection state) + // because it allows forking. We never encounter forking, so these will be + // one-to-one in practice, but the architecture in nICEr means we have to set + // up these callbacks on the nr_ice_ctx, not the nr_ice_peer_ctx. ice_handler_vtbl_->ice_connected = &NrIceCtx::ice_connected; ice_handler_vtbl_->msg_recvd = &NrIceCtx::msg_recvd; ice_handler_vtbl_->ice_checking = &NrIceCtx::ice_checking; @@ -720,8 +793,13 @@ NrIceStats NrIceCtx::Destroy() { delete ice_handler_vtbl_; delete ice_handler_; + delete ice_gather_handler_vtbl_; + delete ice_gather_handler_; + ice_handler_vtbl_ = nullptr; ice_handler_ = nullptr; + ice_gather_handler_vtbl_ = nullptr; + ice_gather_handler_ = nullptr; proxy_config_ = nullptr; streams_.clear(); @@ -854,15 +932,10 @@ nsresult NrIceCtx::StartGathering(bool default_route_only, // finished. int r = nr_ice_gather(ctx_, &NrIceCtx::gather_cb, this); - if (!r) { - SetGatheringState(ICE_CTX_GATHER_COMPLETE); - } else if (r == R_WOULDBLOCK) { - SetGatheringState(ICE_CTX_GATHER_STARTED); - } else { - SetGatheringState(ICE_CTX_GATHER_COMPLETE); + if (r && r != R_WOULDBLOCK) { MOZ_MTLOG(ML_ERROR, "ICE FAILED: Couldn't gather ICE candidates for '" << name_ << "', error=" << r); - SetConnectionState(ICE_CTX_FAILED); + SignalAllStreamsFailed(); return NS_ERROR_FAILURE; } @@ -940,7 +1013,7 @@ nsresult NrIceCtx::StartChecks() { r = nr_ice_peer_ctx_pair_candidates(peer_); if (r) { MOZ_MTLOG(ML_ERROR, "ICE FAILED: Couldn't pair candidates on " << name_); - SetConnectionState(ICE_CTX_FAILED); + SignalAllStreamsFailed(); return NS_ERROR_FAILURE; } @@ -952,7 +1025,7 @@ nsresult NrIceCtx::StartChecks() { } else { MOZ_MTLOG(ML_ERROR, "ICE FAILED: Couldn't start peer checks on " << name_); - SetConnectionState(ICE_CTX_FAILED); + SignalAllStreamsFailed(); return NS_ERROR_FAILURE; } } @@ -961,18 +1034,21 @@ nsresult NrIceCtx::StartChecks() { } void NrIceCtx::gather_cb(NR_SOCKET s, int h, void* arg) { - NrIceCtx* ctx = static_cast(arg); + MOZ_MTLOG(ML_DEBUG, "gather_cb called"); + // We don't use this; we react to the stream-specific callbacks instead +} - ctx->SetGatheringState(ICE_CTX_GATHER_COMPLETE); +void NrIceCtx::SignalAllStreamsFailed() { + for (auto& [id, stream] : streams_) { + Unused << id; + stream->Failed(); + SignalConnectionStateChange(stream, ICE_CTX_FAILED); + } } void NrIceCtx::UpdateNetworkState(bool online) { MOZ_MTLOG(ML_NOTICE, "NrIceCtx(" << name_ << "): updating network state to " << (online ? "online" : "offline")); - if (connection_state_ == ICE_CTX_CLOSED) { - return; - } - if (online) { nr_ice_peer_ctx_refresh_consent_all_streams(peer_); } else { @@ -980,36 +1056,6 @@ void NrIceCtx::UpdateNetworkState(bool online) { } } -void NrIceCtx::SetConnectionState(ConnectionState state) { - if (state == connection_state_) return; - - MOZ_MTLOG(ML_INFO, "NrIceCtx(" << name_ << "): state " << connection_state_ - << "->" << state); - connection_state_ = state; - - if (connection_state_ == ICE_CTX_FAILED) { - MOZ_MTLOG(ML_INFO, - "NrIceCtx(" << name_ << "): dumping r_log ringbuffer... "); - std::deque logs; - RLogConnector::GetInstance()->GetAny(0, &logs); - for (auto& log : logs) { - MOZ_MTLOG(ML_INFO, log); - } - } - - SignalConnectionStateChange(this, state); -} - -void NrIceCtx::SetGatheringState(GatheringState state) { - if (state == gathering_state_) return; - - MOZ_MTLOG(ML_DEBUG, "NrIceCtx(" << name_ << "): gathering state " - << gathering_state_ << "->" << state); - gathering_state_ = state; - - SignalGatheringStateChange(this, state); -} - void NrIceCtx::GenerateObfuscatedAddress(nr_ice_candidate* candidate, std::string* mdns_address, std::string* actual_address) { diff --git a/dom/media/webrtc/transport/nricectx.h b/dom/media/webrtc/transport/nricectx.h index a0a0b5b772..01ad6b5dbd 100644 --- a/dom/media/webrtc/transport/nricectx.h +++ b/dom/media/webrtc/transport/nricectx.h @@ -74,6 +74,8 @@ typedef struct nr_ice_peer_ctx_ nr_ice_peer_ctx; typedef struct nr_ice_media_stream_ nr_ice_media_stream; typedef struct nr_ice_handler_ nr_ice_handler; typedef struct nr_ice_handler_vtbl_ nr_ice_handler_vtbl; +typedef struct nr_ice_gather_handler_ nr_ice_gather_handler; +typedef struct nr_ice_gather_handler_vtbl_ nr_ice_gather_handler_vtbl; typedef struct nr_ice_candidate_ nr_ice_candidate; typedef struct nr_ice_cand_pair_ nr_ice_cand_pair; typedef struct nr_ice_stun_server_ nr_ice_stun_server; @@ -200,12 +202,6 @@ class NrIceCtx { ICE_CTX_CLOSED }; - enum GatheringState { - ICE_CTX_GATHER_INIT, - ICE_CTX_GATHER_STARTED, - ICE_CTX_GATHER_COMPLETE - }; - enum Controlling { ICE_CONTROLLING, ICE_CONTROLLED }; enum Policy { ICE_POLICY_RELAY, ICE_POLICY_NO_HOST, ICE_POLICY_ALL }; @@ -294,12 +290,6 @@ class NrIceCtx { // The name of the ctx const std::string& name() const { return name_; } - // Current state - ConnectionState connection_state() const { return connection_state_; } - - // Current state - GatheringState gathering_state() const { return gathering_state_; } - // Get the global attributes std::vector GetGlobalAttributes(); @@ -351,9 +341,7 @@ class NrIceCtx { // Signals to indicate events. API users can (and should) // register for these. - sigslot::signal2 - SignalGatheringStateChange; - sigslot::signal2 + sigslot::signal2 SignalConnectionStateChange; // The thread to direct method calls to @@ -375,7 +363,11 @@ class NrIceCtx { static int select_pair(void* obj, nr_ice_media_stream* stream, int component_id, nr_ice_cand_pair** potentials, int potential_ct); + static int stream_gathering(void* obj, nr_ice_media_stream* stream); + static int stream_gathered(void* obj, nr_ice_media_stream* stream); + static int stream_checking(void* obj, nr_ice_media_stream* stream); static int stream_ready(void* obj, nr_ice_media_stream* stream); + static int stream_disconnected(void* obj, nr_ice_media_stream* stream); static int stream_failed(void* obj, nr_ice_media_stream* stream); static int ice_checking(void* obj, nr_ice_peer_ctx* pctx); static int ice_connected(void* obj, nr_ice_peer_ctx* pctx); @@ -387,28 +379,25 @@ class NrIceCtx { nr_ice_media_stream* stream, int component_id, nr_ice_candidate* candidate); + void SignalAllStreamsFailed(); + // Find a media stream by stream ptr. Gross RefPtr FindStream(nr_ice_media_stream* stream); - // Set the state - void SetConnectionState(ConnectionState state); - - // Set the state - void SetGatheringState(GatheringState state); - void GenerateObfuscatedAddress(nr_ice_candidate* candidate, std::string* mdns_address, std::string* actual_address); - ConnectionState connection_state_; - GatheringState gathering_state_; + bool dumped_rlog_ = false; const std::string name_; bool ice_controlling_set_; std::map> streams_; nr_ice_ctx* ctx_; nr_ice_peer_ctx* peer_; - nr_ice_handler_vtbl* ice_handler_vtbl_; // Must be pointer - nr_ice_handler* ice_handler_; // Must be pointer + nr_ice_handler_vtbl* ice_handler_vtbl_; // Must be pointer + nr_ice_handler* ice_handler_; // Must be pointer + nr_ice_gather_handler_vtbl* ice_gather_handler_vtbl_; // Must be pointer + nr_ice_gather_handler* ice_gather_handler_; // Must be pointer bool trickle_; nsCOMPtr sts_target_; // The thread to run on Config config_; diff --git a/dom/media/webrtc/transport/nricemediastream.cpp b/dom/media/webrtc/transport/nricemediastream.cpp index 426aee230e..a3a0c147c9 100644 --- a/dom/media/webrtc/transport/nricemediastream.cpp +++ b/dom/media/webrtc/transport/nricemediastream.cpp @@ -204,11 +204,17 @@ nsresult NrIceMediaStream::ConnectToPeer( MOZ_ASSERT(stream_); if (Matches(old_stream_, ufrag, pwd)) { + bool wasGathering = !AllGenerationsDoneGathering(); // (We swap before we close so we never have stream_ == nullptr) MOZ_MTLOG(ML_DEBUG, "Rolling back to old stream ufrag=" << ufrag << " " << name_); std::swap(stream_, old_stream_); CloseStream(&old_stream_); + if (wasGathering && AllGenerationsDoneGathering()) { + // Special case; we do not need to send another empty candidate, but we + // do need to handle the transition from gathering to complete. + SignalGatheringStateChange(GetId(), ICE_STREAM_GATHER_COMPLETE); + } } else if (old_stream_) { // Right now we wait for ICE to complete before closing the old stream. // It might be worth it to close it sooner, but we don't want to close it @@ -273,6 +279,10 @@ nsresult NrIceMediaStream::SetIceCredentials(const std::string& ufrag, } state_ = ICE_CONNECTING; + + MOZ_MTLOG(ML_WARNING, + "SetIceCredentials new=" << stream_ << " old=" << old_stream_); + return NS_OK; } @@ -661,6 +671,21 @@ void NrIceMediaStream::Failed() { } } +void NrIceMediaStream::OnGatheringStarted(nr_ice_media_stream* stream) { + MOZ_MTLOG(ML_WARNING, "OnGatheringStarted called for " << stream); + SignalGatheringStateChange(GetId(), ICE_STREAM_GATHER_STARTED); +} + +void NrIceMediaStream::OnGatheringComplete(nr_ice_media_stream* stream) { + MOZ_MTLOG(ML_WARNING, "OnGatheringComplete called for " << stream); + // Spec says to queue two separate tasks; one for the empty candidate, and + // the next for the state change. + SignalCandidate(this, "", stream->ufrag, "", ""); + if (AllGenerationsDoneGathering()) { + SignalGatheringStateChange(GetId(), ICE_STREAM_GATHER_COMPLETE); + } +} + void NrIceMediaStream::Close() { MOZ_MTLOG(ML_DEBUG, "Marking stream closed '" << name_ << "'"); state_ = ICE_CLOSED; @@ -709,4 +734,34 @@ nr_ice_media_stream* NrIceMediaStream::GetStreamForRemoteUfrag( return nullptr; } +bool NrIceMediaStream::AllGenerationsDoneGathering() const { + if (stream_ && !nr_ice_media_stream_is_done_gathering(stream_)) { + return false; + } + if (old_stream_ && !nr_ice_media_stream_is_done_gathering(old_stream_)) { + return false; + } + return true; +} + +bool NrIceMediaStream::AnyGenerationIsConnected() const { + nr_ice_media_stream* peer_stream = nullptr; + + if (stream_ && + !nr_ice_peer_ctx_find_pstream(ctx_->peer(), stream_, &peer_stream)) { + if (peer_stream->ice_state == NR_ICE_MEDIA_STREAM_CHECKS_CONNECTED && + !peer_stream->disconnected) { + return true; + } + } + + if (old_stream_ && + !nr_ice_peer_ctx_find_pstream(ctx_->peer(), old_stream_, &peer_stream)) { + if (peer_stream->ice_state == NR_ICE_MEDIA_STREAM_CHECKS_CONNECTED && + !peer_stream->disconnected) { + return true; + } + } + return false; +} } // namespace mozilla diff --git a/dom/media/webrtc/transport/nricemediastream.h b/dom/media/webrtc/transport/nricemediastream.h index 406373573c..d434dd34dd 100644 --- a/dom/media/webrtc/transport/nricemediastream.h +++ b/dom/media/webrtc/transport/nricemediastream.h @@ -126,6 +126,12 @@ struct NrIceCandidatePair { class NrIceMediaStream { public: + enum GatheringState { + ICE_STREAM_GATHER_INIT, + ICE_STREAM_GATHER_STARTED, + ICE_STREAM_GATHER_COMPLETE + }; + NrIceMediaStream(NrIceCtx* ctx, const std::string& id, const std::string& name, size_t components); @@ -182,6 +188,9 @@ class NrIceMediaStream { void Ready(nr_ice_media_stream* stream); void Failed(); + void OnGatheringStarted(nr_ice_media_stream* stream); + void OnGatheringComplete(nr_ice_media_stream* stream); + // Close the stream. Called by the NrIceCtx. // Different from the destructor because other people // might be holding RefPtrs but we want those writes to fail once @@ -192,9 +201,14 @@ class NrIceMediaStream { // the candidate belongs to. const std::string& GetId() const { return id_; } + bool AllGenerationsDoneGathering() const; + bool AnyGenerationIsConnected() const; + sigslot::signal5 SignalCandidate; // A new ICE candidate: + sigslot::signal2 + SignalGatheringStateChange; sigslot::signal1 SignalReady; // Candidate pair ready. sigslot::signal1 SignalFailed; // Candidate pair failed. diff --git a/dom/media/webrtc/transport/test/ice_unittest.cpp b/dom/media/webrtc/transport/test/ice_unittest.cpp index 4d097fafa3..50febb3cdd 100644 --- a/dom/media/webrtc/transport/test/ice_unittest.cpp +++ b/dom/media/webrtc/transport/test/ice_unittest.cpp @@ -304,6 +304,8 @@ class SchedulableTrickleCandidate { } void Schedule(unsigned int ms) { + std::cerr << "Scheduling " << Candidate() << " in " << ms << "ms" + << std::endl; test_utils_->SyncDispatchToSTS( WrapRunnable(this, &SchedulableTrickleCandidate::Schedule_s, ms)); } @@ -355,10 +357,7 @@ class IceTestPeer : public sigslot::has_slots<> { offerer_(offerer), stream_counter_(0), shutting_down_(false), - gathering_complete_(false), ready_ct_(0), - ice_connected_(false), - ice_failed_(false), ice_reached_checking_(false), received_(0), sent_(0), @@ -372,8 +371,6 @@ class IceTestPeer : public sigslot::has_slots<> { simulate_ice_lite_(false), nat_(new TestNat), test_utils_(utils) { - ice_ctx_->SignalGatheringStateChange.connect( - this, &IceTestPeer::GatheringStateChange); ice_ctx_->SignalConnectionStateChange.connect( this, &IceTestPeer::ConnectionStateChange); @@ -426,6 +423,10 @@ class IceTestPeer : public sigslot::has_slots<> { stream->SignalReady.connect(this, &IceTestPeer::StreamReady); stream->SignalFailed.connect(this, &IceTestPeer::StreamFailed); stream->SignalPacketReceived.connect(this, &IceTestPeer::PacketReceived); + stream->SignalGatheringStateChange.connect( + this, &IceTestPeer::GatheringStateChange); + mConnectionStates[id] = NrIceCtx::ICE_CTX_INIT; + mGatheringStates[id] = NrIceMediaStream::ICE_STREAM_GATHER_INIT; } void AddStream(int components) { @@ -434,7 +435,10 @@ class IceTestPeer : public sigslot::has_slots<> { } void RemoveStream_s(size_t index) { - ice_ctx_->DestroyStream(MakeTransportId(index)); + const std::string id = MakeTransportId(index); + ice_ctx_->DestroyStream(id); + mConnectionStates.erase(id); + mGatheringStates.erase(id); } void RemoveStream(size_t index) { @@ -650,7 +654,15 @@ class IceTestPeer : public sigslot::has_slots<> { return host_net; } - bool gathering_complete() { return gathering_complete_; } + bool gathering_complete() { + for (const auto& [id, state] : mGatheringStates) { + Unused << id; + if (state != NrIceMediaStream::ICE_STREAM_GATHER_COMPLETE) { + return false; + } + } + return true; + } int ready_ct() { return ready_ct_; } bool is_ready_s(size_t index) { auto media_stream = GetStream_s(index); @@ -666,8 +678,33 @@ class IceTestPeer : public sigslot::has_slots<> { WrapRunnableRet(&result, this, &IceTestPeer::is_ready_s, stream)); return result; } - bool ice_connected() { return ice_connected_; } - bool ice_failed() { return ice_failed_; } + bool ice_connected() { + for (const auto& [id, state] : mConnectionStates) { + if (state != NrIceCtx::ICE_CTX_CONNECTED) { + return false; + } + } + return true; + } + bool ice_failed() { + for (const auto& [id, state] : mConnectionStates) { + if (state == NrIceCtx::ICE_CTX_FAILED) { + return true; + } + } + return false; + } + bool ice_checking() { + if (ice_failed() || ice_connected()) { + return false; + } + for (const auto& [id, state] : mConnectionStates) { + if (state == NrIceCtx::ICE_CTX_CHECKING) { + return true; + } + } + return false; + } bool ice_reached_checking() { return ice_reached_checking_; } size_t received() { return received_; } size_t sent() { return sent_; } @@ -680,13 +717,16 @@ class IceTestPeer : public sigslot::has_slots<> { void RestartIce_s() { for (auto& stream : ice_ctx_->GetStreams()) { SetIceCredentials_s(*stream); + mConnectionStates[stream->GetId()] = NrIceCtx::ICE_CTX_INIT; + mGatheringStates[stream->GetId()] = + NrIceMediaStream::ICE_STREAM_GATHER_INIT; } // take care of some local bookkeeping ready_ct_ = 0; - gathering_complete_ = false; - ice_connected_ = false; - ice_failed_ = false; - ice_reached_checking_ = false; + // We do not unset ice_reached_checking_ here, since we do not expect + // ICE to return to checking in an ICE restart, because the ICE stack + // continues using the old streams (which are probably connected) until the + // new ones are connected. remote_ = nullptr; } @@ -709,9 +749,6 @@ class IceTestPeer : public sigslot::has_slots<> { remote_ = remote; trickle_mode_ = trickle_mode; - ice_connected_ = false; - ice_failed_ = false; - ice_reached_checking_ = false; res = ice_ctx_->ParseGlobalAttributes(remote->GetGlobalAttributes()); ASSERT_FALSE(remote->simulate_ice_lite_ && (ice_ctx_->GetControlling() == NrIceCtx::ICE_CONTROLLED)); @@ -793,8 +830,11 @@ class IceTestPeer : public sigslot::has_slots<> { auto stream = GetStream_s(index); if (!stream) { // stream might have gone away before the trickle timer popped + std::cerr << "Trickle candidate has no stream: " << index << std::endl; return NS_OK; } + std::cerr << "Trickle candidate for " << index << " (" << stream->GetId() + << "):" << candidate << std::endl; return stream->ParseTrickleCandidate(candidate, ufrag, ""); } @@ -940,16 +980,18 @@ class IceTestPeer : public sigslot::has_slots<> { } // Handle events - void GatheringStateChange(NrIceCtx* ctx, NrIceCtx::GatheringState state) { + void GatheringStateChange(const std::string& aTransportId, + NrIceMediaStream::GatheringState state) { if (shutting_down_) { return; } - if (state != NrIceCtx::ICE_CTX_GATHER_COMPLETE) { + mGatheringStates[aTransportId] = state; + + if (!gathering_complete()) { return; } std::cerr << name_ << " Gathering complete" << std::endl; - gathering_complete_ = true; std::cerr << name_ << " ATTRIBUTES:" << std::endl; for (const auto& stream : ice_ctx_->GetStreams()) { @@ -973,9 +1015,9 @@ class IceTestPeer : public sigslot::has_slots<> { if (candidate.empty()) { return; } - std::cerr << "Candidate for stream " << stream->name() + std::cerr << "Candidate for stream " << stream->GetId() << " initialized: " << candidate << std::endl; - candidates_[stream->name()].push_back(candidate); + candidates_[stream->GetId()].push_back(candidate); // If we are connected, then try to trickle to the other side. if (remote_ && remote_->remote_ && (trickle_mode_ != TRICKLE_SIMULATE)) { @@ -990,7 +1032,7 @@ class IceTestPeer : public sigslot::has_slots<> { return; } } - ADD_FAILURE() << "No matching stream found for " << stream; + ADD_FAILURE() << "No matching stream found for " << stream->GetId(); } } @@ -1133,32 +1175,45 @@ class IceTestPeer : public sigslot::has_slots<> { DumpCandidatePairs_s(stream); } - void ConnectionStateChange(NrIceCtx* ctx, NrIceCtx::ConnectionState state) { - (void)ctx; + void ConnectionStateChange(NrIceMediaStream* stream, + NrIceCtx::ConnectionState state) { + mConnectionStates[stream->GetId()] = state; + if (ice_checking()) { + ice_reached_checking_ = true; + } + switch (state) { case NrIceCtx::ICE_CTX_INIT: break; case NrIceCtx::ICE_CTX_CHECKING: - std::cerr << name_ << " ICE reached checking" << std::endl; - ice_reached_checking_ = true; + std::cerr << name_ << " ICE reached checking (" << stream->GetId() + << ")" << std::endl; + MOZ_ASSERT(ice_reached_checking_); break; case NrIceCtx::ICE_CTX_CONNECTED: - std::cerr << name_ << " ICE connected" << std::endl; - ice_connected_ = true; + std::cerr << name_ << " ICE reached connected (" << stream->GetId() + << ")" << std::endl; + MOZ_ASSERT(ice_reached_checking_); break; case NrIceCtx::ICE_CTX_COMPLETED: - std::cerr << name_ << " ICE completed" << std::endl; + std::cerr << name_ << " ICE reached completed (" << stream->GetId() + << ")" << std::endl; + MOZ_ASSERT(ice_reached_checking_); break; case NrIceCtx::ICE_CTX_FAILED: - std::cerr << name_ << " ICE failed" << std::endl; - ice_failed_ = true; + std::cerr << name_ << " ICE reached failed (" << stream->GetId() << ")" + << std::endl; + MOZ_ASSERT(ice_reached_checking_); break; case NrIceCtx::ICE_CTX_DISCONNECTED: - std::cerr << name_ << " ICE disconnected" << std::endl; - ice_connected_ = false; + std::cerr << name_ << " ICE reached disconnected (" << stream->GetId() + << ")" << std::endl; + MOZ_ASSERT(ice_reached_checking_); + break; + case NrIceCtx::ICE_CTX_CLOSED: + std::cerr << name_ << " ICE reached closed (" << stream->GetId() << ")" + << std::endl; break; - default: - MOZ_CRASH(); } } @@ -1326,10 +1381,9 @@ class IceTestPeer : public sigslot::has_slots<> { std::map> mOldIceCredentials; size_t stream_counter_; bool shutting_down_; - bool gathering_complete_; + std::map mConnectionStates; + std::map mGatheringStates; int ready_ct_; - bool ice_connected_; - bool ice_failed_; bool ice_reached_checking_; size_t received_; size_t sent_; @@ -1686,10 +1740,8 @@ class WebRtcIceConnectTest : public StunTest { TrickleMode mode = TRICKLE_NONE) { ASSERT_TRUE(caller->ready_ct() == 0); ASSERT_TRUE(caller->ice_connected() == 0); - ASSERT_TRUE(caller->ice_reached_checking() == 0); ASSERT_TRUE(callee->ready_ct() == 0); ASSERT_TRUE(callee->ice_connected() == 0); - ASSERT_TRUE(callee->ice_reached_checking() == 0); // IceTestPeer::Connect grabs attributes from the first arg, and // gives them to |this|, meaning that callee->Connect(caller, ...) @@ -3361,6 +3413,8 @@ TEST_F(WebRtcIceConnectTest, TestConnectTrickleAddStreamDuringICE) { RealisticTrickleDelay(p1_->ControlTrickle(0)); RealisticTrickleDelay(p2_->ControlTrickle(0)); AddStream(1); + ASSERT_TRUE(Gather()); + ConnectTrickle(); RealisticTrickleDelay(p1_->ControlTrickle(1)); RealisticTrickleDelay(p2_->ControlTrickle(1)); WaitForConnected(1000); diff --git a/dom/media/webrtc/transport/test/test_nr_socket_ice_unittest.cpp b/dom/media/webrtc/transport/test/test_nr_socket_ice_unittest.cpp index b55b05f10c..20754f033b 100644 --- a/dom/media/webrtc/transport/test/test_nr_socket_ice_unittest.cpp +++ b/dom/media/webrtc/transport/test/test_nr_socket_ice_unittest.cpp @@ -73,7 +73,8 @@ class IcePeer { peer_ctx_(nullptr), nat_(nat), test_utils_(test_utils) { - nr_ice_ctx_create(const_cast(name_.c_str()), flags, &ice_ctx_); + nr_ice_ctx_create(const_cast(name_.c_str()), flags, nullptr, + &ice_ctx_); if (nat_) { nr_socket_factory* factory; diff --git a/dom/media/webrtc/transport/test/transport_unittests.cpp b/dom/media/webrtc/transport/test/transport_unittests.cpp index 7729151ade..0e9702ced8 100644 --- a/dom/media/webrtc/transport/test/transport_unittests.cpp +++ b/dom/media/webrtc/transport/test/transport_unittests.cpp @@ -586,16 +586,15 @@ class TransportTestPeer : public sigslot::has_slots<> { void InitIce() { nsresult res; - // Attach our slots - ice_ctx_->SignalGatheringStateChange.connect( - this, &TransportTestPeer::GatheringStateChange); - char name[100]; snprintf(name, sizeof(name), "%s:stream%d", name_.c_str(), (int)streams_.size()); // Create the media stream RefPtr stream = ice_ctx_->CreateStream(name, name, 1); + // Attach our slots + stream->SignalGatheringStateChange.connect( + this, &TransportTestPeer::GatheringStateChange); ASSERT_TRUE(stream != nullptr); stream->SetIceCredentials("ufrag", "pass"); @@ -639,9 +638,12 @@ class TransportTestPeer : public sigslot::has_slots<> { << std::endl; } - void GatheringStateChange(NrIceCtx* ctx, NrIceCtx::GatheringState state) { - (void)ctx; - if (state == NrIceCtx::ICE_CTX_GATHER_COMPLETE) { + void GatheringStateChange(const std::string& aTransportId, + NrIceMediaStream::GatheringState state) { + // We only use one stream, no need to check whether all streams are done + // gathering. + Unused << aTransportId; + if (state == NrIceMediaStream::ICE_STREAM_GATHER_COMPLETE) { GatheringComplete(); } } diff --git a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.c b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.c index 0d498845a4..b428264e5a 100644 --- a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.c +++ b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.c @@ -325,8 +325,9 @@ int nr_ice_fetch_turn_servers(int ct, nr_ice_turn_server **out) #endif /* USE_TURN */ #define MAXADDRS 100 /* Ridiculously high */ -int nr_ice_ctx_create(char *label, UINT4 flags, nr_ice_ctx **ctxp) - { + int nr_ice_ctx_create(char* label, UINT4 flags, + nr_ice_gather_handler* gather_handler, + nr_ice_ctx** ctxp) { nr_ice_ctx *ctx=0; int r,_status; @@ -341,6 +342,8 @@ int nr_ice_ctx_create(char *label, UINT4 flags, nr_ice_ctx **ctxp) if(!(ctx->label=r_strdup(label))) ABORT(R_NO_MEMORY); + ctx->gather_handler = gather_handler; + /* Get the STUN servers */ if(r=NR_reg_get_child_count(NR_ICE_REG_STUN_SRV_PRFX, (unsigned int *)&ctx->stun_server_ct_cfg)||ctx->stun_server_ct_cfg==0) { @@ -442,7 +445,7 @@ int nr_ice_ctx_create(char *label, UINT4 flags, nr_ice_ctx **ctxp) int i; nr_ice_stun_id *id1,*id2; - ctx->done_cb = 0; + ctx->gather_done_cb = 0; ctx->trickle_cb = 0; STAILQ_FOREACH_SAFE(s1, &ctx->streams, entry, s2){ @@ -452,6 +455,8 @@ int nr_ice_ctx_create(char *label, UINT4 flags, nr_ice_ctx **ctxp) RFREE(ctx->label); + ctx->gather_handler = 0; + RFREE(ctx->stun_servers_cfg); RFREE(ctx->local_addrs); @@ -539,20 +544,26 @@ void nr_ice_gather_finished_cb(NR_SOCKET s, int h, void *cb_arg) } } - if (nr_ice_media_stream_is_done_gathering(stream) && - ctx->trickle_cb) { - ctx->trickle_cb(ctx->trickle_cb_arg, ctx, stream, component_id, NULL); + if (nr_ice_media_stream_is_done_gathering(stream)) { + if (ctx->gather_handler && ctx->gather_handler->vtbl->stream_gathered) { + ctx->gather_handler->vtbl->stream_gathered(ctx->gather_handler->obj, + stream); + } + if (ctx->trickle_cb) { + ctx->trickle_cb(ctx->trickle_cb_arg, ctx, stream, component_id, NULL); + } } if(ctx->uninitialized_candidates==0){ + assert(nr_ice_media_stream_is_done_gathering(stream)); r_log(LOG_ICE, LOG_INFO, "ICE(%s): All candidates initialized", ctx->label); - if (ctx->done_cb) { - ctx->done_cb(0,0,ctx->cb_arg); - } - else { + if (ctx->gather_done_cb) { + ctx->gather_done_cb(0, 0, ctx->cb_arg); + } else { r_log(LOG_ICE, LOG_INFO, - "ICE(%s): No done_cb. We were probably destroyed.", ctx->label); + "ICE(%s): No gather_done_cb. We were probably destroyed.", + ctx->label); } } else { @@ -850,8 +861,7 @@ int nr_ice_set_target_for_default_local_address_lookup(nr_ice_ctx *ctx, const ch return(_status); } -int nr_ice_gather(nr_ice_ctx *ctx, NR_async_cb done_cb, void *cb_arg) - { + int nr_ice_gather(nr_ice_ctx* ctx, NR_async_cb gather_done_cb, void* cb_arg) { int r,_status; nr_ice_media_stream *stream; nr_local_addr stun_addrs[MAXADDRS]; @@ -872,7 +882,7 @@ int nr_ice_gather(nr_ice_ctx *ctx, NR_async_cb done_cb, void *cb_arg) } r_log(LOG_ICE,LOG_DEBUG,"ICE(%s): Initializing candidates",ctx->label); - ctx->done_cb=done_cb; + ctx->gather_done_cb = gather_done_cb; ctx->cb_arg=cb_arg; /* Initialize all the media stream/component pairs */ diff --git a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.h b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.h index 8b3081f567..4039c741ec 100644 --- a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.h +++ b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_ctx.h @@ -97,9 +97,23 @@ typedef struct nr_ice_stats_ { UINT2 turn_438s; } nr_ice_stats; +typedef struct nr_ice_gather_handler_vtbl_ { + /* This media stream is gathering */ + int (*stream_gathering)(void* obj, nr_ice_media_stream* stream); + + /* This media stream has finished gathering */ + int (*stream_gathered)(void* obj, nr_ice_media_stream* stream); +} nr_ice_gather_handler_vtbl; + +typedef struct nr_ice_gather_handler_ { + void* obj; + nr_ice_gather_handler_vtbl* vtbl; +} nr_ice_gather_handler; + struct nr_ice_ctx_ { UINT4 flags; char *label; + nr_ice_gather_handler* gather_handler; UINT4 Ta; @@ -129,7 +143,7 @@ struct nr_ice_ctx_ { nr_ice_peer_ctx_head peers; nr_ice_stun_id_head ids; - NR_async_cb done_cb; + NR_async_cb gather_done_cb; void *cb_arg; nr_ice_trickle_candidate_cb trickle_cb; @@ -141,7 +155,8 @@ struct nr_ice_ctx_ { nr_transport_addr *target_for_default_local_address_lookup; }; -int nr_ice_ctx_create(char *label, UINT4 flags, nr_ice_ctx **ctxp); +int nr_ice_ctx_create(char* label, UINT4 flags, + nr_ice_gather_handler* gather_handler, nr_ice_ctx** ctxp); int nr_ice_ctx_create_with_credentials(char *label, UINT4 flags, char* ufrag, char* pwd, nr_ice_ctx **ctxp); #define NR_ICE_CTX_FLAGS_AGGRESSIVE_NOMINATION (1) #define NR_ICE_CTX_FLAGS_LITE (1<<1) @@ -156,7 +171,7 @@ void nr_ice_ctx_remove_flags(nr_ice_ctx *ctx, UINT4 flags); void nr_ice_ctx_destroy(nr_ice_ctx** ctxp); int nr_ice_set_local_addresses(nr_ice_ctx *ctx, nr_local_addr* stun_addrs, int stun_addr_ct); int nr_ice_set_target_for_default_local_address_lookup(nr_ice_ctx *ctx, const char *target_ip, UINT2 target_port); -int nr_ice_gather(nr_ice_ctx *ctx, NR_async_cb done_cb, void *cb_arg); +int nr_ice_gather(nr_ice_ctx* ctx, NR_async_cb gather_done_cb, void* cb_arg); int nr_ice_add_candidate(nr_ice_ctx *ctx, nr_ice_candidate *cand); void nr_ice_gather_finished_cb(NR_SOCKET s, int h, void *cb_arg); int nr_ice_add_media_stream(nr_ice_ctx *ctx,const char *label,const char *ufrag,const char *pwd,int components, nr_ice_media_stream **streamp); diff --git a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_handler.h b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_handler.h index 5a0690adad..ab3e41ef2d 100644 --- a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_handler.h +++ b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_handler.h @@ -56,9 +56,15 @@ int component_id, nr_ice_cand_pair **potentials,int potential_ct); */ int (*stream_ready)(void *obj, nr_ice_media_stream *stream); + /* This media stream is checking */ + int (*stream_checking)(void* obj, nr_ice_media_stream* stream); + /* This media stream has failed */ int (*stream_failed)(void *obj, nr_ice_media_stream *stream); + /* This media stream has disconnected */ + int (*stream_disconnected)(void* obj, nr_ice_media_stream* stream); + /* ICE is connected for this peer ctx */ int (*ice_connected)(void *obj, nr_ice_peer_ctx *pctx); diff --git a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.c b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.c index 62bfbad629..90e278bedb 100644 --- a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.c +++ b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.c @@ -80,6 +80,7 @@ int nr_ice_media_stream_create(nr_ice_ctx *ctx,const char *label,const char *ufr stream->component_ct=components; stream->ice_state = NR_ICE_MEDIA_STREAM_UNPAIRED; stream->obsolete = 0; + stream->actually_started_checking = 0; stream->r2l_user = 0; stream->l2r_user = 0; stream->flags = ctx->flags; @@ -177,8 +178,20 @@ int nr_ice_media_stream_initialize(nr_ice_ctx *ctx, nr_ice_media_stream *stream) comp=STAILQ_NEXT(comp,entry); } + if (!nr_ice_media_stream_is_done_gathering(stream) && ctx->gather_handler && + ctx->gather_handler->vtbl->stream_gathering) { + ctx->gather_handler->vtbl->stream_gathering(ctx->gather_handler->obj, + stream); + } + _status=0; abort: + if (_status) { + if (ctx->gather_handler && ctx->gather_handler->vtbl->stream_gathered) { + ctx->gather_handler->vtbl->stream_gathered(ctx->gather_handler->obj, + stream); + } + } return(_status); } @@ -413,6 +426,19 @@ static void nr_ice_media_stream_check_timer_cb(NR_SOCKET s, int h, void *cb_arg) if(pair){ nr_ice_candidate_pair_start(pair->pctx,pair); /* Ignore failures */ + + /* stream->ice_state goes to checking when we decide that it is ok to + * start checking, which can happen before we get remote candidates. We + * want to fire this event when we _actually_ start sending checks. */ + if (!stream->actually_started_checking) { + stream->actually_started_checking = 1; + if (stream->pctx->handler && + stream->pctx->handler->vtbl->stream_checking) { + stream->pctx->handler->vtbl->stream_checking( + stream->pctx->handler->obj, stream->local_stream); + } + } + NR_ASYNC_TIMER_SET(timer_val,nr_ice_media_stream_check_timer_cb,cb_arg,&stream->timer); } else { @@ -729,9 +755,21 @@ void nr_ice_media_stream_set_disconnected(nr_ice_media_stream *stream, int disco if (disconnected == NR_ICE_MEDIA_STREAM_DISCONNECTED) { if (!stream->local_stream->obsolete) { + if (stream->pctx->handler && + stream->pctx->handler->vtbl->stream_disconnected) { + stream->pctx->handler->vtbl->stream_disconnected( + stream->pctx->handler->obj, stream->local_stream); + } nr_ice_peer_ctx_disconnected(stream->pctx); } } else { + if (!stream->local_stream->obsolete) { + if (stream->pctx->handler && + stream->pctx->handler->vtbl->stream_ready) { + stream->pctx->handler->vtbl->stream_ready(stream->pctx->handler->obj, + stream->local_stream); + } + } nr_ice_peer_ctx_check_if_connected(stream->pctx); } } diff --git a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.h b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.h index 99f906c100..3da20e3b5e 100644 --- a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.h +++ b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_media_stream.h @@ -72,6 +72,7 @@ struct nr_ice_media_stream_ { * processing. If this stream is connected already, traffic can continue to * flow for a limited time while the new stream gets ready. */ int obsolete; + int actually_started_checking; #define NR_ICE_MEDIA_STREAM_UNPAIRED 1 #define NR_ICE_MEDIA_STREAM_CHECKS_FROZEN 2 diff --git a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_peer_ctx.c b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_peer_ctx.c index 0bf97eb984..bfa1fcf44b 100644 --- a/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_peer_ctx.c +++ b/dom/media/webrtc/transport/third_party/nICEr/src/ice/ice_peer_ctx.c @@ -671,6 +671,12 @@ void nr_ice_peer_ctx_refresh_consent_all_streams(nr_ice_peer_ctx *pctx) void nr_ice_peer_ctx_disconnected(nr_ice_peer_ctx *pctx) { + if (pctx->connected_cb_timer) { + /* Whoops, never mind */ + NR_async_timer_cancel(pctx->connected_cb_timer); + pctx->connected_cb_timer = 0; + } + if (pctx->reported_connected && pctx->handler && pctx->handler->vtbl->ice_disconnected) { diff --git a/dom/media/webrtc/transportbridge/MediaPipelineFilter.cpp b/dom/media/webrtc/transportbridge/MediaPipelineFilter.cpp index fc1a51dd0d..275e1bf45a 100644 --- a/dom/media/webrtc/transportbridge/MediaPipelineFilter.cpp +++ b/dom/media/webrtc/transportbridge/MediaPipelineFilter.cpp @@ -85,23 +85,27 @@ bool MediaPipelineFilter::Filter(const webrtc::RTPHeader& header) { // Remote SSRC based filtering // - if (remote_ssrc_set_.count(header.ssrc)) { + if (!remote_ssrc_set_.empty()) { + if (remote_ssrc_set_.count(header.ssrc)) { + DEBUG_LOG( + ("MediaPipelineFilter SSRC: %u matched remote SSRC set." + " passing packet", + header.ssrc)); + return true; + } DEBUG_LOG( - ("MediaPipelineFilter SSRC: %u matched remote SSRC set." - " passing packet", - header.ssrc)); - return true; + ("MediaPipelineFilter SSRC: %u did not match any of %zu" + " remote SSRCS.", + header.ssrc, remote_ssrc_set_.size())); + return false; } - DEBUG_LOG( - ("MediaPipelineFilter SSRC: %u did not match any of %zu" - " remote SSRCS.", - header.ssrc, remote_ssrc_set_.size())); // - // PT, payload type, last ditch effort filtering + // PT, payload type, last ditch effort filtering. We only try this if we do + // not have any ssrcs configured (either by learning them, or negotiation). // - if (payload_type_set_.count(header.payloadType)) { + if (receive_payload_type_set_.count(header.payloadType)) { DEBUG_LOG( ("MediaPipelineFilter payload-type: %u matched %zu" " unique payload type. learning ssrc. passing packet", @@ -114,7 +118,7 @@ bool MediaPipelineFilter::Filter(const webrtc::RTPHeader& header) { DEBUG_LOG( ("MediaPipelineFilter payload-type: %u did not match any of %zu" " unique payload-types.", - header.payloadType, payload_type_set_.size())); + header.payloadType, receive_payload_type_set_.size())); DEBUG_LOG( ("MediaPipelineFilter packet failed to match any criteria." " ignoring packet")); @@ -125,8 +129,8 @@ void MediaPipelineFilter::AddRemoteSSRC(uint32_t ssrc) { remote_ssrc_set_.insert(ssrc); } -void MediaPipelineFilter::AddUniquePT(uint8_t payload_type) { - payload_type_set_.insert(payload_type); +void MediaPipelineFilter::AddUniqueReceivePT(uint8_t payload_type) { + receive_payload_type_set_.insert(payload_type); } void MediaPipelineFilter::Update(const MediaPipelineFilter& filter_update) { @@ -143,7 +147,7 @@ void MediaPipelineFilter::Update(const MediaPipelineFilter& filter_update) { mRemoteMid = filter_update.mRemoteMid; mRemoteMidBindings = filter_update.mRemoteMidBindings; } - payload_type_set_ = filter_update.payload_type_set_; + receive_payload_type_set_ = filter_update.receive_payload_type_set_; // Use extmapping from new filter mExtMap = filter_update.mExtMap; diff --git a/dom/media/webrtc/transportbridge/MediaPipelineFilter.h b/dom/media/webrtc/transportbridge/MediaPipelineFilter.h index d6bb7abd43..62d4b63ab8 100644 --- a/dom/media/webrtc/transportbridge/MediaPipelineFilter.h +++ b/dom/media/webrtc/transportbridge/MediaPipelineFilter.h @@ -66,7 +66,7 @@ class MediaPipelineFilter { void SetRemoteMediaStreamId(const Maybe& aMid); // When a payload type id is unique to our media section, add it here. - void AddUniquePT(uint8_t payload_type); + void AddUniqueReceivePT(uint8_t payload_type); void Update(const MediaPipelineFilter& filter_update); @@ -76,7 +76,7 @@ class MediaPipelineFilter { // The number of filters we manage here is quite small, so I am optimizing // for readability. std::set remote_ssrc_set_; - std::set payload_type_set_; + std::set receive_payload_type_set_; Maybe mRemoteMid; std::set mRemoteMidBindings; // RID extension can be set by tests and is sticky, the rest of diff --git a/dom/media/webspeech/recognition/SpeechRecognition.cpp b/dom/media/webspeech/recognition/SpeechRecognition.cpp index 75d1ba7709..7239a88237 100644 --- a/dom/media/webspeech/recognition/SpeechRecognition.cpp +++ b/dom/media/webspeech/recognition/SpeechRecognition.cpp @@ -437,12 +437,13 @@ uint32_t SpeechRecognition::ProcessAudioSegment(AudioSegment* aSegment, // we need to call the nsISpeechRecognitionService::ProcessAudioSegment // in a separate thread so that any eventual encoding or pre-processing // of the audio does not block the main thread - nsresult rv = mEncodeTaskQueue->Dispatch( - NewRunnableMethod, TrackRate>( - "nsISpeechRecognitionService::ProcessAudioSegment", - mRecognitionService, - &nsISpeechRecognitionService::ProcessAudioSegment, - std::move(*aSegment), aTrackRate)); + nsresult rv = mEncodeTaskQueue->Dispatch(NS_NewRunnableFunction( + "nsISpeechRecognitionService::ProcessAudioSegment", + [=, service = mRecognitionService, + segment = std::move(*aSegment)]() mutable { + service->ProcessAudioSegment(&segment, aTrackRate); + })); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; return samples; diff --git a/dom/media/webspeech/synth/nsISynthVoiceRegistry.idl b/dom/media/webspeech/synth/nsISynthVoiceRegistry.idl index 1898bf68c1..8192eff045 100644 --- a/dom/media/webspeech/synth/nsISynthVoiceRegistry.idl +++ b/dom/media/webspeech/synth/nsISynthVoiceRegistry.idl @@ -65,18 +65,6 @@ interface nsISynthVoiceRegistry : nsISupports }; %{C++ -#define NS_SYNTHVOICEREGISTRY_CID \ - { /* {7090524d-5574-4492-a77f-d8d558ced59d} */ \ - 0x7090524d, \ - 0x5574, \ - 0x4492, \ - { 0xa7, 0x7f, 0xd8, 0xd5, 0x58, 0xce, 0xd5, 0x9d } \ - } - #define NS_SYNTHVOICEREGISTRY_CONTRACTID \ "@mozilla.org/synth-voice-registry;1" - -#define NS_SYNTHVOICEREGISTRY_CLASSNAME \ - "Speech Synthesis Voice Registry" - %} diff --git a/dom/media/webvtt/WebVTTListener.cpp b/dom/media/webvtt/WebVTTListener.cpp index 3f8d99a8f3..b0691491e3 100644 --- a/dom/media/webvtt/WebVTTListener.cpp +++ b/dom/media/webvtt/WebVTTListener.cpp @@ -65,7 +65,7 @@ nsresult WebVTTListener::LoadResource() { if (IsCanceled()) { return NS_OK; } - // Exit if we failed to create the WebVTTParserWrapper (vtt.jsm) + // Exit if we failed to create the WebVTTParserWrapper (vtt.sys.mjs) NS_ENSURE_SUCCESS(mParserWrapperError, mParserWrapperError); mElement->SetReadyState(TextTrackReadyState::Loading); diff --git a/dom/media/webvtt/test/mochitest/manifest.js b/dom/media/webvtt/test/mochitest/manifest.js index 91c481feb9..dc72b55ffa 100644 --- a/dom/media/webvtt/test/mochitest/manifest.js +++ b/dom/media/webvtt/test/mochitest/manifest.js @@ -11,7 +11,7 @@ function removeNodeAndSource(n) { } function once(target, name, cb) { - var p = new Promise(function (resolve, reject) { + var p = new Promise(function (resolve) { target.addEventListener( name, function () { diff --git a/dom/media/webvtt/test/mochitest/mochitest.toml b/dom/media/webvtt/test/mochitest/mochitest.toml index 6ec6042407..6fd4c9e7ba 100644 --- a/dom/media/webvtt/test/mochitest/mochitest.toml +++ b/dom/media/webvtt/test/mochitest/mochitest.toml @@ -18,15 +18,14 @@ support-files = [ ["test_bug883173.html"] skip-if = [ - "android_version == '25' && debug", # android(bug 1232305) - "os == 'linux' && (asan || tsan || debug)", # Bug 1829511 + "os == 'linux' && os_version == '18.04' && asan", # Bug 1829511 + "os == 'linux' && os_version == '18.04' && tsan", # Bug 1829511 + "os == 'linux' && os_version == '18.04' && debug", # Bug 1829511 ] ["test_bug895091.html"] -skip-if = ["android_version == '25' && debug"] # android(bug 1232305) ["test_bug957847.html"] -skip-if = ["android_version == '25' && debug"] # android(bug 1232305) ["test_bug1018933.html"] @@ -39,10 +38,7 @@ skip-if = ["android_version == '25' && debug"] # android(bug 1232305) ["test_texttrack_cors_preload_none.html"] ["test_texttrack_mode_change_during_loading.html"] -skip-if = [ - "true", - "os == 'android'", # Bug 1636572, android(bug 1562021) -] +skip-if = ["true"] ["test_texttrack_moz.html"] diff --git a/dom/media/webvtt/update-webvtt.js b/dom/media/webvtt/update-webvtt.js index 20a3e2669f..8350cff5dc 100644 --- a/dom/media/webvtt/update-webvtt.js +++ b/dom/media/webvtt/update-webvtt.js @@ -6,7 +6,7 @@ var gift = require("gift"), fs = require("fs"), argv = require("optimist") .usage( - "Update vtt.jsm with the latest from a vtt.js directory.\nUsage:" + + "Update vtt.sys.mjs with the latest from a vtt.js directory.\nUsage:" + " $0 -d [dir]" ) .demand("d") @@ -22,7 +22,7 @@ var gift = require("gift"), .options("w", { alias: "write", describe: "Path to file to write to.", - default: "./vtt.jsm", + default: "./vtt.sys.mjs", }).argv; var repo = gift(argv.d); -- cgit v1.2.3