/* * Copyright (C) 2005-2018 Team Kodi * This file is part of Kodi - https://kodi.tv * * SPDX-License-Identifier: GPL-2.0-or-later * See LICENSES/README.md for more information. */ #include "VideoPlayerAudio.h" #include "DVDCodecs/Audio/DVDAudioCodec.h" #include "DVDCodecs/DVDFactoryCodec.h" #include "ServiceBroker.h" #include "cores/AudioEngine/Interfaces/AE.h" #include "cores/AudioEngine/Utils/AEUtil.h" #include "cores/VideoPlayer/Interface/DemuxPacket.h" #include "settings/Settings.h" #include "settings/SettingsComponent.h" #include "utils/MathUtils.h" #include "utils/log.h" #include #ifdef TARGET_RASPBERRY_PI #include "platform/linux/RBP.h" #endif #include #include #include using namespace std::chrono_literals; class CDVDMsgAudioCodecChange : public CDVDMsg { public: CDVDMsgAudioCodecChange(const CDVDStreamInfo& hints, std::unique_ptr codec) : CDVDMsg(GENERAL_STREAMCHANGE), m_codec(std::move(codec)), m_hints(hints) {} ~CDVDMsgAudioCodecChange() override = default; std::unique_ptr m_codec; CDVDStreamInfo m_hints; }; CVideoPlayerAudio::CVideoPlayerAudio(CDVDClock* pClock, CDVDMessageQueue& parent, CProcessInfo &processInfo) : CThread("VideoPlayerAudio"), IDVDStreamPlayerAudio(processInfo) , m_messageQueue("audio") , m_messageParent(parent) , m_audioSink(pClock) { m_pClock = pClock; m_audioClock = 0; m_speed = DVD_PLAYSPEED_NORMAL; m_stalled = true; m_paused = false; m_syncState = IDVDStreamPlayer::SYNC_STARTING; m_synctype = SYNC_DISCON; m_prevsynctype = -1; m_prevskipped = false; m_maxspeedadjust = 0.0; m_messageQueue.SetMaxDataSize(6 * 1024 * 1024); m_messageQueue.SetMaxTimeSize(8.0); m_disconAdjustTimeMs = processInfo.GetMaxPassthroughOffSyncDuration(); } CVideoPlayerAudio::~CVideoPlayerAudio() { StopThread(); // close the stream, and don't wait for the audio to be finished // CloseStream(true); } bool CVideoPlayerAudio::OpenStream(CDVDStreamInfo hints) { CLog::Log(LOGINFO, "Finding audio codec for: {}", hints.codec); bool allowpassthrough = !CServiceBroker::GetSettingsComponent()->GetSettings()->GetBool(CSettings::SETTING_VIDEOPLAYER_USEDISPLAYASCLOCK); if (m_processInfo.IsRealtimeStream()) allowpassthrough = false; CAEStreamInfo::DataType streamType = m_audioSink.GetPassthroughStreamType(hints.codec, hints.samplerate, hints.profile); std::unique_ptr codec = CDVDFactoryCodec::CreateAudioCodec( hints, m_processInfo, allowpassthrough, m_processInfo.AllowDTSHDDecode(), streamType); if(!codec) { CLog::Log(LOGERROR, "Unsupported audio codec"); return false; } if(m_messageQueue.IsInited()) m_messageQueue.Put(std::make_shared(hints, std::move(codec)), 0); else { OpenStream(hints, std::move(codec)); m_messageQueue.Init(); CLog::Log(LOGINFO, "Creating audio thread"); Create(); } return true; } void CVideoPlayerAudio::OpenStream(CDVDStreamInfo& hints, std::unique_ptr codec) { m_pAudioCodec = std::move(codec); m_processInfo.ResetAudioCodecInfo(); /* store our stream hints */ m_streaminfo = hints; /* update codec information from what codec gave out, if any */ int channelsFromCodec = m_pAudioCodec->GetFormat().m_channelLayout.Count(); int samplerateFromCodec = m_pAudioCodec->GetFormat().m_sampleRate; if (channelsFromCodec > 0) m_streaminfo.channels = channelsFromCodec; if (samplerateFromCodec > 0) m_streaminfo.samplerate = samplerateFromCodec; /* check if we only just got sample rate, in which case the previous call * to CreateAudioCodec() couldn't have started passthrough */ if (hints.samplerate != m_streaminfo.samplerate) SwitchCodecIfNeeded(); m_audioClock = 0; m_stalled = m_messageQueue.GetPacketCount(CDVDMsg::DEMUXER_PACKET) == 0; m_prevsynctype = -1; m_synctype = SYNC_DISCON; if (CServiceBroker::GetSettingsComponent()->GetSettings()->GetBool(CSettings::SETTING_VIDEOPLAYER_USEDISPLAYASCLOCK)) m_synctype = SYNC_RESAMPLE; else if (m_processInfo.IsRealtimeStream()) m_synctype = SYNC_RESAMPLE; if (m_synctype == SYNC_DISCON) CLog::LogF(LOGINFO, "Allowing max Out-Of-Sync Value of {} ms", m_disconAdjustTimeMs); m_prevskipped = false; m_maxspeedadjust = 5.0; m_messageParent.Put(std::make_shared(CDVDMsg::PLAYER_AVCHANGE)); m_syncState = IDVDStreamPlayer::SYNC_STARTING; } void CVideoPlayerAudio::CloseStream(bool bWaitForBuffers) { bool bWait = bWaitForBuffers && m_speed > 0 && !CServiceBroker::GetActiveAE()->IsSuspended(); // wait until buffers are empty if (bWait) m_messageQueue.WaitUntilEmpty(); // send abort message to the audio queue m_messageQueue.Abort(); CLog::Log(LOGINFO, "Waiting for audio thread to exit"); // shut down the adio_decode thread and wait for it StopThread(); // will set this->m_bStop to true // destroy audio device CLog::Log(LOGINFO, "Closing audio device"); if (bWait) { m_bStop = false; m_audioSink.Drain(); m_bStop = true; } else { m_audioSink.Flush(); } m_audioSink.Destroy(true); // uninit queue m_messageQueue.End(); CLog::Log(LOGINFO, "Deleting audio codec"); if (m_pAudioCodec) { m_pAudioCodec->Dispose(); m_pAudioCodec.reset(); } } void CVideoPlayerAudio::OnStartup() { } void CVideoPlayerAudio::UpdatePlayerInfo() { std::ostringstream s; s << "aq:" << std::setw(2) << std::min(99,m_messageQueue.GetLevel()) << "%"; s << ", Kb/s:" << std::fixed << std::setprecision(2) << m_audioStats.GetBitrate() / 1024.0; // print a/v discontinuity adjustments counter when audio is not resampled (passthrough mode) if (m_synctype == SYNC_DISCON) s << ", a/v corrections (" << m_disconAdjustTimeMs << "ms): " << m_disconAdjustCounter; //print the inverse of the resample ratio, since that makes more sense //if the resample ratio is 0.5, then we're playing twice as fast else if (m_synctype == SYNC_RESAMPLE) s << ", rr:" << std::fixed << std::setprecision(5) << 1.0 / m_audioSink.GetResampleRatio(); SInfo info; info.info = s.str(); info.pts = m_audioSink.GetPlayingPts(); info.passthrough = m_pAudioCodec && m_pAudioCodec->NeedPassthrough(); { std::unique_lock lock(m_info_section); m_info = info; } } void CVideoPlayerAudio::Process() { CLog::Log(LOGINFO, "running thread: CVideoPlayerAudio::Process()"); DVDAudioFrame audioframe; audioframe.nb_frames = 0; audioframe.framesOut = 0; m_audioStats.Start(); m_disconAdjustCounter = 0; // Only enable "learning" if advancedsettings m_maxPassthroughOffSyncDuration // not exists or has it's default 10 ms value, otherwise use advancedsettings value if (m_disconAdjustTimeMs == 10) { m_disconTimer.Set(30s); m_disconLearning = true; } else { m_disconLearning = false; } bool onlyPrioMsgs = false; while (!m_bStop) { std::shared_ptr pMsg; int timeout = (int)(1000 * m_audioSink.GetCacheTime()); // read next packet and return -1 on error int priority = 1; //Do we want a new audio frame? if (m_syncState == IDVDStreamPlayer::SYNC_STARTING || /* when not started */ m_processInfo.IsTempoAllowed(static_cast(m_speed)/DVD_PLAYSPEED_NORMAL) || m_speed < DVD_PLAYSPEED_PAUSE || /* when rewinding */ (m_speed > DVD_PLAYSPEED_NORMAL && m_audioClock < m_pClock->GetClock())) /* when behind clock in ff */ priority = 0; if (m_syncState == IDVDStreamPlayer::SYNC_WAITSYNC) priority = 1; if (m_paused) priority = 1; if (onlyPrioMsgs) { priority = 1; timeout = 0; } MsgQueueReturnCode ret = m_messageQueue.Get(pMsg, timeout, priority); onlyPrioMsgs = false; if (MSGQ_IS_ERROR(ret)) { if (!m_messageQueue.ReceivedAbortRequest()) CLog::Log(LOGERROR, "MSGQ_IS_ERROR returned true ({})", ret); break; } else if (ret == MSGQ_TIMEOUT) { if (ProcessDecoderOutput(audioframe)) { onlyPrioMsgs = true; continue; } // if we only wanted priority messages, this isn't a stall if (priority) continue; if (m_processInfo.IsTempoAllowed(static_cast(m_speed)/DVD_PLAYSPEED_NORMAL) && !m_stalled && m_syncState == IDVDStreamPlayer::SYNC_INSYNC) { // while AE sync is active, we still have time to fill buffers if (m_syncTimer.IsTimePast()) { CLog::Log(LOGINFO, "CVideoPlayerAudio::Process - stream stalled"); m_stalled = true; } } if (timeout == 0) CThread::Sleep(10ms); continue; } // handle messages if (pMsg->IsType(CDVDMsg::GENERAL_SYNCHRONIZE)) { if (std::static_pointer_cast(pMsg)->Wait(100ms, SYNCSOURCE_AUDIO)) CLog::Log(LOGDEBUG, "CVideoPlayerAudio - CDVDMsg::GENERAL_SYNCHRONIZE"); else m_messageQueue.Put(pMsg, 1); // push back as prio message, to process other prio messages } else if (pMsg->IsType(CDVDMsg::GENERAL_RESYNC)) { //player asked us to set internal clock double pts = std::static_pointer_cast(pMsg)->m_value; CLog::Log(LOGDEBUG, "CVideoPlayerAudio - CDVDMsg::GENERAL_RESYNC({:f}), level: {}, cache: {:f}", pts, m_messageQueue.GetLevel(), m_audioSink.GetDelay()); double delay = m_audioSink.GetDelay(); if (pts > m_audioClock - delay + 0.5 * DVD_TIME_BASE) { m_audioSink.Flush(); } m_audioClock = pts + delay; if (m_speed != DVD_PLAYSPEED_PAUSE) m_audioSink.Resume(); m_syncState = IDVDStreamPlayer::SYNC_INSYNC; m_syncTimer.Set(3000ms); } else if (pMsg->IsType(CDVDMsg::GENERAL_RESET)) { if (m_pAudioCodec) m_pAudioCodec->Reset(); m_audioSink.Flush(); m_stalled = true; m_audioClock = 0; audioframe.nb_frames = 0; m_syncState = IDVDStreamPlayer::SYNC_STARTING; } else if (pMsg->IsType(CDVDMsg::GENERAL_FLUSH)) { bool sync = std::static_pointer_cast(pMsg)->m_value; m_audioSink.Flush(); m_stalled = true; m_audioClock = 0; audioframe.nb_frames = 0; if (sync) { m_syncState = IDVDStreamPlayer::SYNC_STARTING; m_audioSink.Pause(); } if (m_pAudioCodec) m_pAudioCodec->Reset(); } else if (pMsg->IsType(CDVDMsg::GENERAL_EOF)) { CLog::Log(LOGDEBUG, "CVideoPlayerAudio - CDVDMsg::GENERAL_EOF"); } else if (pMsg->IsType(CDVDMsg::PLAYER_SETSPEED)) { double speed = std::static_pointer_cast(pMsg)->m_value; if (m_processInfo.IsTempoAllowed(static_cast(speed)/DVD_PLAYSPEED_NORMAL)) { if (speed != m_speed) { if (m_syncState == IDVDStreamPlayer::SYNC_INSYNC) { m_audioSink.Resume(); m_stalled = false; } } } else { m_audioSink.Pause(); } m_speed = (int)speed; } else if (pMsg->IsType(CDVDMsg::GENERAL_STREAMCHANGE)) { auto msg = std::static_pointer_cast(pMsg); OpenStream(msg->m_hints, std::move(msg->m_codec)); msg->m_codec = NULL; } else if (pMsg->IsType(CDVDMsg::GENERAL_PAUSE)) { m_paused = std::static_pointer_cast(pMsg)->m_value; CLog::Log(LOGDEBUG, "CVideoPlayerAudio - CDVDMsg::GENERAL_PAUSE: {}", m_paused); } else if (pMsg->IsType(CDVDMsg::PLAYER_REQUEST_STATE)) { SStateMsg msg; msg.player = VideoPlayer_AUDIO; msg.syncState = m_syncState; m_messageParent.Put( std::make_shared>(CDVDMsg::PLAYER_REPORT_STATE, msg)); } else if (pMsg->IsType(CDVDMsg::DEMUXER_PACKET)) { DemuxPacket* pPacket = std::static_pointer_cast(pMsg)->GetPacket(); bool bPacketDrop = std::static_pointer_cast(pMsg)->GetPacketDrop(); if (bPacketDrop) { if (m_syncState != IDVDStreamPlayer::SYNC_STARTING) { m_audioSink.Drain(); m_audioSink.Flush(); audioframe.nb_frames = 0; } m_syncState = IDVDStreamPlayer::SYNC_STARTING; continue; } if (!m_processInfo.IsTempoAllowed(static_cast(m_speed) / DVD_PLAYSPEED_NORMAL) && m_syncState == IDVDStreamPlayer::SYNC_INSYNC) { continue; } if (!m_pAudioCodec->AddData(*pPacket)) { m_messageQueue.PutBack(pMsg); onlyPrioMsgs = true; continue; } m_audioStats.AddSampleBytes(pPacket->iSize); UpdatePlayerInfo(); if (ProcessDecoderOutput(audioframe)) { onlyPrioMsgs = true; } } else if (pMsg->IsType(CDVDMsg::PLAYER_DISPLAY_RESET)) { m_displayReset = true; } } } bool CVideoPlayerAudio::ProcessDecoderOutput(DVDAudioFrame &audioframe) { if (audioframe.nb_frames <= audioframe.framesOut) { audioframe.hasDownmix = false; m_pAudioCodec->GetData(audioframe); if (audioframe.nb_frames == 0) { return false; } audioframe.hasTimestamp = true; if (audioframe.pts == DVD_NOPTS_VALUE) { audioframe.pts = m_audioClock; audioframe.hasTimestamp = false; } else { m_audioClock = audioframe.pts; } if (audioframe.format.m_sampleRate && m_streaminfo.samplerate != (int) audioframe.format.m_sampleRate) { // The sample rate has changed or we just got it for the first time // for this stream. See if we should enable/disable passthrough due // to it. m_streaminfo.samplerate = audioframe.format.m_sampleRate; if (SwitchCodecIfNeeded()) { audioframe.nb_frames = 0; return false; } } // if stream switches to realtime, disable pass through // or switch to resample if (m_processInfo.IsRealtimeStream() && m_synctype != SYNC_RESAMPLE) { m_synctype = SYNC_RESAMPLE; if (SwitchCodecIfNeeded()) { audioframe.nb_frames = 0; return false; } } // Display reset event has occurred // See if we should enable passthrough if (m_displayReset) { if (SwitchCodecIfNeeded()) { audioframe.nb_frames = 0; return false; } } // demuxer reads metatags that influence channel layout if (m_streaminfo.codec == AV_CODEC_ID_FLAC && m_streaminfo.channellayout) audioframe.format.m_channelLayout = CAEUtil::GetAEChannelLayout(m_streaminfo.channellayout); // we have successfully decoded an audio frame, setup renderer to match if (!m_audioSink.IsValidFormat(audioframe)) { if (m_speed) m_audioSink.Drain(); m_audioSink.Destroy(false); if (!m_audioSink.Create(audioframe, m_streaminfo.codec, m_synctype == SYNC_RESAMPLE)) CLog::Log(LOGERROR, "{} - failed to create audio renderer", __FUNCTION__); m_prevsynctype = -1; if (m_syncState == IDVDStreamPlayer::SYNC_INSYNC) m_audioSink.Resume(); } m_audioSink.SetDynamicRangeCompression( static_cast(m_processInfo.GetVideoSettings().m_VolumeAmplification * 100)); SetSyncType(audioframe.passthrough); // downmix double clev = audioframe.hasDownmix ? audioframe.centerMixLevel : M_SQRT1_2; double curDB = 20 * log10(clev); audioframe.centerMixLevel = pow(10, (curDB + m_processInfo.GetVideoSettings().m_CenterMixLevel) / 20); audioframe.hasDownmix = true; } if (m_synctype == SYNC_DISCON) { double syncerror = m_audioSink.GetSyncError(); if (m_disconLearning) { const double syncErr = std::abs(syncerror); if (syncErr > DVD_MSEC_TO_TIME(m_disconAdjustTimeMs)) m_disconAdjustTimeMs = DVD_TIME_TO_MSEC(syncErr); if (m_disconTimer.IsTimePast()) { m_disconLearning = false; m_disconAdjustTimeMs = (static_cast(m_disconAdjustTimeMs) * 1.15) + 5.0; if (m_disconAdjustTimeMs > 100) // sanity check m_disconAdjustTimeMs = 100; CLog::LogF(LOGINFO, "Changed max allowed Out-Of-Sync value to {} ms due self-learning", m_disconAdjustTimeMs); } } else if (std::abs(syncerror) > DVD_MSEC_TO_TIME(m_disconAdjustTimeMs)) { double correction = m_pClock->ErrorAdjust(syncerror, "CVideoPlayerAudio::OutputPacket"); if (correction != 0) { m_audioSink.SetSyncErrorCorrection(-correction); m_disconAdjustCounter++; } } } int framesOutput = m_audioSink.AddPackets(audioframe); // guess next pts m_audioClock += audioframe.duration * ((double)framesOutput / audioframe.nb_frames); audioframe.framesOut += framesOutput; // signal to our parent that we have initialized if (m_syncState == IDVDStreamPlayer::SYNC_STARTING) { double cachetotal = m_audioSink.GetCacheTotal(); double cachetime = m_audioSink.GetCacheTime(); if (cachetime >= cachetotal * 0.75) { m_syncState = IDVDStreamPlayer::SYNC_WAITSYNC; m_stalled = false; SStartMsg msg; msg.player = VideoPlayer_AUDIO; msg.cachetotal = m_audioSink.GetMaxDelay() * DVD_TIME_BASE; msg.cachetime = m_audioSink.GetDelay(); msg.timestamp = audioframe.hasTimestamp ? audioframe.pts : DVD_NOPTS_VALUE; m_messageParent.Put(std::make_shared>(CDVDMsg::PLAYER_STARTED, msg)); m_streaminfo.channels = audioframe.format.m_channelLayout.Count(); m_processInfo.SetAudioChannels(audioframe.format.m_channelLayout); m_processInfo.SetAudioSampleRate(audioframe.format.m_sampleRate); m_processInfo.SetAudioBitsPerSample(audioframe.bits_per_sample); m_processInfo.SetAudioDecoderName(m_pAudioCodec->GetName()); m_messageParent.Put(std::make_shared(CDVDMsg::PLAYER_AVCHANGE)); } } return true; } void CVideoPlayerAudio::SetSyncType(bool passthrough) { if (passthrough && m_synctype == SYNC_RESAMPLE) m_synctype = SYNC_DISCON; //if SetMaxSpeedAdjust returns false, it means no video is played and we need to use clock feedback double maxspeedadjust = 0.0; if (m_synctype == SYNC_RESAMPLE) maxspeedadjust = m_maxspeedadjust; m_pClock->SetMaxSpeedAdjust(maxspeedadjust); if (m_synctype != m_prevsynctype) { const char *synctypes[] = {"clock feedback", "resample", "invalid"}; int synctype = (m_synctype >= 0 && m_synctype <= 1) ? m_synctype : 2; CLog::Log(LOGDEBUG, "CVideoPlayerAudio:: synctype set to {}: {}", m_synctype, synctypes[synctype]); m_prevsynctype = m_synctype; if (m_synctype == SYNC_RESAMPLE) m_audioSink.SetResampleMode(1); else m_audioSink.SetResampleMode(0); } } void CVideoPlayerAudio::OnExit() { #ifdef TARGET_WINDOWS CoUninitialize(); #endif CLog::Log(LOGINFO, "thread end: CVideoPlayerAudio::OnExit()"); } void CVideoPlayerAudio::SetSpeed(int speed) { if(m_messageQueue.IsInited()) m_messageQueue.Put(std::make_shared(CDVDMsg::PLAYER_SETSPEED, speed), 1); else m_speed = speed; } void CVideoPlayerAudio::Flush(bool sync) { m_messageQueue.Flush(); m_messageQueue.Put(std::make_shared(CDVDMsg::GENERAL_FLUSH, sync), 1); m_audioSink.AbortAddPackets(); } bool CVideoPlayerAudio::AcceptsData() const { bool full = m_messageQueue.IsFull(); return !full; } bool CVideoPlayerAudio::SwitchCodecIfNeeded() { if (m_displayReset) CLog::Log(LOGINFO, "CVideoPlayerAudio: display reset occurred, checking for passthrough"); else CLog::Log(LOGDEBUG, "CVideoPlayerAudio: stream props changed, checking for passthrough"); m_displayReset = false; bool allowpassthrough = !CServiceBroker::GetSettingsComponent()->GetSettings()->GetBool(CSettings::SETTING_VIDEOPLAYER_USEDISPLAYASCLOCK); if (m_processInfo.IsRealtimeStream() || m_synctype == SYNC_RESAMPLE) allowpassthrough = false; CAEStreamInfo::DataType streamType = m_audioSink.GetPassthroughStreamType( m_streaminfo.codec, m_streaminfo.samplerate, m_streaminfo.profile); std::unique_ptr codec = CDVDFactoryCodec::CreateAudioCodec( m_streaminfo, m_processInfo, allowpassthrough, m_processInfo.AllowDTSHDDecode(), streamType); if (!codec || codec->NeedPassthrough() == m_pAudioCodec->NeedPassthrough()) { // passthrough state has not changed return false; } m_pAudioCodec = std::move(codec); return true; } std::string CVideoPlayerAudio::GetPlayerInfo() { std::unique_lock lock(m_info_section); return m_info.info; } int CVideoPlayerAudio::GetAudioChannels() { return m_streaminfo.channels; } bool CVideoPlayerAudio::IsPassthrough() const { std::unique_lock lock(m_info_section); return m_info.passthrough; }