diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-10 18:07:22 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-10 18:07:22 +0000 |
commit | c04dcc2e7d834218ef2d4194331e383402495ae1 (patch) | |
tree | 7333e38d10d75386e60f336b80c2443c1166031d /xbmc/interfaces/python/PythonInvoker.cpp | |
parent | Initial commit. (diff) | |
download | kodi-c04dcc2e7d834218ef2d4194331e383402495ae1.tar.xz kodi-c04dcc2e7d834218ef2d4194331e383402495ae1.zip |
Adding upstream version 2:20.4+dfsg.upstream/2%20.4+dfsg
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'xbmc/interfaces/python/PythonInvoker.cpp')
-rw-r--r-- | xbmc/interfaces/python/PythonInvoker.cpp | 724 |
1 files changed, 724 insertions, 0 deletions
diff --git a/xbmc/interfaces/python/PythonInvoker.cpp b/xbmc/interfaces/python/PythonInvoker.cpp new file mode 100644 index 0000000..1e9d344 --- /dev/null +++ b/xbmc/interfaces/python/PythonInvoker.cpp @@ -0,0 +1,724 @@ +/* + * Copyright (C) 2013-2018 Team Kodi + * This file is part of Kodi - https://kodi.tv + * + * SPDX-License-Identifier: GPL-2.0-or-later + * See LICENSES/README.md for more information. + */ + +// clang-format off +// python.h should always be included first before any other includes +#include <mutex> +#include <Python.h> +// clang-format on + +#include "PythonInvoker.h" + +#include "ServiceBroker.h" +#include "addons/AddonManager.h" +#include "addons/addoninfo/AddonInfo.h" +#include "addons/addoninfo/AddonType.h" +#include "dialogs/GUIDialogKaiToast.h" +#include "filesystem/SpecialProtocol.h" +#include "guilib/GUIComponent.h" +#include "guilib/GUIWindowManager.h" +#include "guilib/LocalizeStrings.h" +#include "interfaces/python/PyContext.h" +#include "interfaces/python/pythreadstate.h" +#include "interfaces/python/swig.h" +#include "messaging/ApplicationMessenger.h" +#include "threads/SingleLock.h" +#include "threads/SystemClock.h" +#include "utils/CharsetConverter.h" +#include "utils/FileUtils.h" +#include "utils/StringUtils.h" +#include "utils/URIUtils.h" +#include "utils/XTimeUtils.h" +#include "utils/log.h" +#include "windowing/GraphicContext.h" + +// clang-format off +// This breaks fmt because of SEP define, don't include +// before anything that includes logging +#include <osdefs.h> +// clang-format on + +#include <cassert> +#include <iterator> + +#ifdef TARGET_WINDOWS +extern "C" FILE* fopen_utf8(const char* _Filename, const char* _Mode); +#else +#define fopen_utf8 fopen +#endif + +#define GC_SCRIPT \ + "import gc\n" \ + "gc.collect(2)\n" + +#define PY_PATH_SEP DELIM + +// Time before ill-behaved scripts are terminated +#define PYTHON_SCRIPT_TIMEOUT 5000ms // ms + +using namespace XFILE; +using namespace std::chrono_literals; + +#define PythonModulesSize sizeof(PythonModules) / sizeof(PythonModule) + +CCriticalSection CPythonInvoker::s_critical; + +static const std::string getListOfAddonClassesAsString( + XBMCAddon::AddonClass::Ref<XBMCAddon::Python::PythonLanguageHook>& languageHook) +{ + std::string message; + std::unique_lock<CCriticalSection> l(*(languageHook.get())); + const std::set<XBMCAddon::AddonClass*>& acs = languageHook->GetRegisteredAddonClasses(); + bool firstTime = true; + for (const auto& iter : acs) + { + if (!firstTime) + message += ","; + else + firstTime = false; + message += iter->GetClassname(); + } + + return message; +} + +CPythonInvoker::CPythonInvoker(ILanguageInvocationHandler* invocationHandler) + : ILanguageInvoker(invocationHandler), m_threadState(NULL), m_stop(false) +{ +} + +CPythonInvoker::~CPythonInvoker() +{ + // nothing to do for the default invoker used for registration with the + // CScriptInvocationManager + if (GetId() < 0) + return; + + if (GetState() < InvokerStateExecutionDone) + CLog::Log(LOGDEBUG, "CPythonInvoker({}): waiting for python thread \"{}\" to stop", GetId(), + (!m_sourceFile.empty() ? m_sourceFile : "unknown script")); + Stop(true); + pulseGlobalEvent(); + + onExecutionFinalized(); +} + +bool CPythonInvoker::Execute( + const std::string& script, + const std::vector<std::string>& arguments /* = std::vector<std::string>() */) +{ + if (script.empty()) + return false; + + if (!CFileUtils::Exists(script)) + { + CLog::Log(LOGERROR, "CPythonInvoker({}): python script \"{}\" does not exist", GetId(), + CSpecialProtocol::TranslatePath(script)); + return false; + } + + if (!onExecutionInitialized()) + return false; + + return ILanguageInvoker::Execute(script, arguments); +} + +bool CPythonInvoker::execute(const std::string& script, const std::vector<std::string>& arguments) +{ + std::vector<std::wstring> w_arguments; + for (const auto& argument : arguments) + { + std::wstring w_argument; + g_charsetConverter.utf8ToW(argument, w_argument); + w_arguments.push_back(w_argument); + } + return execute(script, w_arguments); +} + +bool CPythonInvoker::execute(const std::string& script, std::vector<std::wstring>& arguments) +{ + // copy the code/script into a local string buffer + m_sourceFile = script; + std::set<std::string> pythonPath; + + CLog::Log(LOGDEBUG, "CPythonInvoker({}, {}): start processing", GetId(), m_sourceFile); + + std::string realFilename(CSpecialProtocol::TranslatePath(m_sourceFile)); + std::string scriptDir = URIUtils::GetDirectory(realFilename); + URIUtils::RemoveSlashAtEnd(scriptDir); + + // set m_threadState if it's not set. + PyThreadState* l_threadState = nullptr; + bool newInterp = false; + { + if (!m_threadState) + { +#if PY_VERSION_HEX < 0x03070000 + // this is a TOTAL hack. We need the GIL but we need to borrow a PyThreadState in order to get it + // as of Python 3.2 since PyEval_AcquireLock is deprecated + extern PyThreadState* savestate; + PyEval_RestoreThread(savestate); +#else + PyThreadState* ts = PyInterpreterState_ThreadHead(PyInterpreterState_Main()); + PyEval_RestoreThread(ts); +#endif + l_threadState = Py_NewInterpreter(); + PyEval_ReleaseThread(l_threadState); + if (l_threadState == NULL) + { + CLog::Log(LOGERROR, "CPythonInvoker({}, {}): FAILED to get thread m_threadState!", GetId(), + m_sourceFile); + return false; + } + newInterp = true; + } + else + l_threadState = m_threadState; + } + + // get the GIL + PyEval_RestoreThread(l_threadState); + if (newInterp) + { + m_languageHook = new XBMCAddon::Python::PythonLanguageHook(l_threadState->interp); + m_languageHook->RegisterMe(); + + onInitialization(); + setState(InvokerStateInitialized); + + if (realFilename == m_sourceFile) + CLog::Log(LOGDEBUG, "CPythonInvoker({}, {}): the source file to load is \"{}\"", GetId(), + m_sourceFile, m_sourceFile); + else + CLog::Log(LOGDEBUG, "CPythonInvoker({}, {}): the source file to load is \"{}\" (\"{}\")", + GetId(), m_sourceFile, m_sourceFile, realFilename); + + // get path from script file name and add python path's + // this is used for python so it will search modules from script path first + pythonPath.emplace(scriptDir); + + // add all addon module dependencies to path + if (m_addon) + { + std::set<std::string> paths; + getAddonModuleDeps(m_addon, paths); + for (const auto& it : paths) + pythonPath.emplace(it); + } + else + { // for backwards compatibility. + // we don't have any addon so just add all addon modules installed + CLog::Log( + LOGWARNING, + "CPythonInvoker({}): Script invoked without an addon. Adding all addon " + "modules installed to python path as fallback. This behaviour will be removed in future " + "version.", + GetId()); + ADDON::VECADDONS addons; + CServiceBroker::GetAddonMgr().GetAddons(addons, ADDON::AddonType::SCRIPT_MODULE); + for (unsigned int i = 0; i < addons.size(); ++i) + pythonPath.emplace(CSpecialProtocol::TranslatePath(addons[i]->LibPath())); + } + + PyObject* sysPath = PySys_GetObject("path"); + + std::for_each(pythonPath.crbegin(), pythonPath.crend(), + [&sysPath](const auto& path) + { + PyObject* pyPath = PyUnicode_FromString(path.c_str()); + PyList_Insert(sysPath, 0, pyPath); + + Py_DECREF(pyPath); + }); + + CLog::Log(LOGDEBUG, "CPythonInvoker({}): full python path:", GetId()); + + Py_ssize_t pathListSize = PyList_Size(sysPath); + + for (Py_ssize_t index = 0; index < pathListSize; index++) + { + if (index == 0 && !pythonPath.empty()) + CLog::Log(LOGDEBUG, "CPythonInvoker({}): custom python path:", GetId()); + + if (index == static_cast<ssize_t>(pythonPath.size())) + CLog::Log(LOGDEBUG, "CPythonInvoker({}): default python path:", GetId()); + + PyObject* pyPath = PyList_GetItem(sysPath, index); + CLog::Log(LOGDEBUG, "CPythonInvoker({}): {}", GetId(), PyUnicode_AsUTF8(pyPath)); + } + + { // set the m_threadState to this new interp + std::unique_lock<CCriticalSection> lockMe(m_critical); + m_threadState = l_threadState; + } + } + else + // swap in my thread m_threadState + PyThreadState_Swap(m_threadState); + + PyObject* sysArgv = PyList_New(0); + + if (arguments.empty()) + arguments.emplace_back(L""); + + CLog::Log(LOGDEBUG, "CPythonInvoker({}): adding args:", GetId()); + + for (const auto& arg : arguments) + { + PyObject* pyArg = PyUnicode_FromWideChar(arg.c_str(), arg.length()); + PyList_Append(sysArgv, pyArg); + CLog::Log(LOGDEBUG, "CPythonInvoker({}): {}", GetId(), PyUnicode_AsUTF8(pyArg)); + + Py_DECREF(pyArg); + } + + PySys_SetObject("argv", sysArgv); + + CLog::Log(LOGDEBUG, "CPythonInvoker({}, {}): entering source directory {}", GetId(), m_sourceFile, + scriptDir); + PyObject* module = PyImport_AddModule("__main__"); + PyObject* moduleDict = PyModule_GetDict(module); + + // we need to check if we was asked to abort before we had inited + bool stopping = false; + { + GilSafeSingleLock lock(m_critical); + stopping = m_stop; + } + + bool failed = false; + std::string exceptionType, exceptionValue, exceptionTraceback; + if (!stopping) + { + try + { + // run script from file + // We need to have python open the file because on Windows the DLL that python + // is linked against may not be the DLL that xbmc is linked against so + // passing a FILE* to python from an fopen has the potential to crash. + + PyObject* pyRealFilename = Py_BuildValue("s", realFilename.c_str()); + FILE* fp = _Py_fopen_obj(pyRealFilename, "rb"); + Py_DECREF(pyRealFilename); + + if (fp != NULL) + { + PyObject* f = PyUnicode_FromString(realFilename.c_str()); + PyDict_SetItemString(moduleDict, "__file__", f); + + onPythonModuleInitialization(moduleDict); + + Py_DECREF(f); + setState(InvokerStateRunning); + XBMCAddon::Python::PyContext + pycontext; // this is a guard class that marks this callstack as being in a python context + executeScript(fp, realFilename, moduleDict); + } + else + CLog::Log(LOGERROR, "CPythonInvoker({}, {}): {} not found!", GetId(), m_sourceFile, + m_sourceFile); + } + catch (const XbmcCommons::Exception& e) + { + setState(InvokerStateFailed); + e.LogThrowMessage(); + failed = true; + } + catch (...) + { + setState(InvokerStateFailed); + CLog::Log(LOGERROR, "CPythonInvoker({}, {}): failure in script", GetId(), m_sourceFile); + failed = true; + } + } + + m_systemExitThrown = false; + InvokerState stateToSet; + if (!failed && !PyErr_Occurred()) + { + CLog::Log(LOGDEBUG, "CPythonInvoker({}, {}): script successfully run", GetId(), m_sourceFile); + stateToSet = InvokerStateScriptDone; + onSuccess(); + } + else if (PyErr_ExceptionMatches(PyExc_SystemExit)) + { + m_systemExitThrown = true; + CLog::Log(LOGDEBUG, "CPythonInvoker({}, {}): script aborted", GetId(), m_sourceFile); + stateToSet = InvokerStateFailed; + onAbort(); + } + else + { + stateToSet = InvokerStateFailed; + + // if it failed with an exception we already logged the details + if (!failed) + { + PythonBindings::PythonToCppException* e = NULL; + if (PythonBindings::PythonToCppException::ParsePythonException(exceptionType, exceptionValue, + exceptionTraceback)) + e = new PythonBindings::PythonToCppException(exceptionType, exceptionValue, + exceptionTraceback); + else + e = new PythonBindings::PythonToCppException(); + + e->LogThrowMessage(); + delete e; + } + + onError(exceptionType, exceptionValue, exceptionTraceback); + } + + std::unique_lock<CCriticalSection> lock(m_critical); + // no need to do anything else because the script has already stopped + if (failed) + { + setState(stateToSet); + return true; + } + + if (m_threadState) + { + // make sure all sub threads have finished + for (PyThreadState* old = nullptr; m_threadState != nullptr;) + { + PyThreadState* s = PyInterpreterState_ThreadHead(m_threadState->interp); + for (; s && s == m_threadState;) + s = PyThreadState_Next(s); + + if (!s) + break; + + if (old != s) + { + CLog::Log(LOGINFO, "CPythonInvoker({}, {}): waiting on thread {}", GetId(), m_sourceFile, + (uint64_t)s->thread_id); + old = s; + } + + lock.unlock(); + CPyThreadState pyState; + KODI::TIME::Sleep(100ms); + pyState.Restore(); + lock.lock(); + } + } + + // pending calls must be cleared out + XBMCAddon::RetardedAsyncCallbackHandler::clearPendingCalls(m_threadState); + + assert(m_threadState != nullptr); + PyEval_ReleaseThread(m_threadState); + + setState(stateToSet); + + return true; +} + +void CPythonInvoker::executeScript(FILE* fp, const std::string& script, PyObject* moduleDict) +{ + if (fp == NULL || script.empty() || moduleDict == NULL) + return; + + int m_Py_file_input = Py_file_input; + PyRun_FileExFlags(fp, script.c_str(), m_Py_file_input, moduleDict, moduleDict, 1, NULL); +} + +FILE* CPythonInvoker::PyFile_AsFileWithMode(PyObject* py_file, const char* mode) +{ + PyObject* ret = PyObject_CallMethod(py_file, "flush", ""); + if (ret == NULL) + return NULL; + Py_DECREF(ret); + + int fd = PyObject_AsFileDescriptor(py_file); + if (fd == -1) + return NULL; + + FILE* f = fdopen(fd, mode); + if (f == NULL) + { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return f; +} + +bool CPythonInvoker::stop(bool abort) +{ + std::unique_lock<CCriticalSection> lock(m_critical); + m_stop = true; + + if (!IsRunning() && !m_threadState) + return false; + + if (m_threadState != NULL) + { + if (IsRunning()) + { + setState(InvokerStateStopping); + lock.unlock(); + + PyEval_RestoreThread((PyThreadState*)m_threadState); + + //tell xbmc.Monitor to call onAbortRequested() + if (m_addon) + { + CLog::Log(LOGDEBUG, "CPythonInvoker({}, {}): trigger Monitor abort request", GetId(), + m_sourceFile); + AbortNotification(); + } + + PyEval_ReleaseThread(m_threadState); + } + else + //Release the lock while waiting for threads to finish + lock.unlock(); + + XbmcThreads::EndTime<> timeout(PYTHON_SCRIPT_TIMEOUT); + while (!m_stoppedEvent.Wait(15ms)) + { + if (timeout.IsTimePast()) + { + CLog::Log(LOGERROR, + "CPythonInvoker({}, {}): script didn't stop in {} seconds - let's kill it", + GetId(), m_sourceFile, + std::chrono::duration_cast<std::chrono::seconds>(PYTHON_SCRIPT_TIMEOUT).count()); + break; + } + + // We can't empty-spin in the main thread and expect scripts to be able to + // dismantle themselves. Python dialogs aren't normal XBMC dialogs, they rely + // on TMSG_GUI_PYTHON_DIALOG messages, so pump the message loop. + if (CServiceBroker::GetAppMessenger()->IsProcessThread()) + { + CServiceBroker::GetAppMessenger()->ProcessMessages(); + } + } + + lock.lock(); + + setState(InvokerStateExecutionDone); + + // Useful for add-on performance metrics + if (!timeout.IsTimePast()) + CLog::Log(LOGDEBUG, "CPythonInvoker({}, {}): script termination took {}ms", GetId(), + m_sourceFile, (PYTHON_SCRIPT_TIMEOUT - timeout.GetTimeLeft()).count()); + + // Since we released the m_critical it's possible that the state is cleaned up + // so we need to recheck for m_threadState == NULL + if (m_threadState != NULL) + { + { + // grabbing the PyLock while holding the m_critical is asking for a deadlock + CSingleExit ex2(m_critical); + PyEval_RestoreThread((PyThreadState*)m_threadState); + } + + + PyThreadState* state = PyInterpreterState_ThreadHead(m_threadState->interp); + while (state) + { + // Raise a SystemExit exception in python threads + Py_XDECREF(state->async_exc); + state->async_exc = PyExc_SystemExit; + Py_XINCREF(state->async_exc); + state = PyThreadState_Next(state); + } + + // If a dialog entered its doModal(), we need to wake it to see the exception + pulseGlobalEvent(); + + PyEval_ReleaseThread(m_threadState); + } + lock.unlock(); + + setState(InvokerStateFailed); + } + + return true; +} + +// Always called from Invoker thread +void CPythonInvoker::onExecutionDone() +{ + std::unique_lock<CCriticalSection> lock(m_critical); + if (m_threadState != NULL) + { + CLog::Log(LOGDEBUG, "{}({}, {})", __FUNCTION__, GetId(), m_sourceFile); + + PyEval_RestoreThread(m_threadState); + + onDeinitialization(); + + // run the gc before finishing + // + // if the script exited by throwing a SystemExit exception then going back + // into the interpreter causes this python bug to get hit: + // http://bugs.python.org/issue10582 + // and that causes major failures. So we are not going to go back in + // to run the GC if that's the case. + if (!m_stop && m_languageHook->HasRegisteredAddonClasses() && !m_systemExitThrown && + PyRun_SimpleString(GC_SCRIPT) == -1) + CLog::Log(LOGERROR, + "CPythonInvoker({}, {}): failed to run the gc to clean up after running prior to " + "shutting down the Interpreter", + GetId(), m_sourceFile); + + Py_EndInterpreter(m_threadState); + + // If we still have objects left around, produce an error message detailing what's been left behind + if (m_languageHook->HasRegisteredAddonClasses()) + CLog::Log(LOGWARNING, + "CPythonInvoker({}, {}): the python script \"{}\" has left several " + "classes in memory that we couldn't clean up. The classes include: {}", + GetId(), m_sourceFile, m_sourceFile, getListOfAddonClassesAsString(m_languageHook)); + + // unregister the language hook + m_languageHook->UnregisterMe(); + +#if PY_VERSION_HEX < 0x03070000 + PyEval_ReleaseLock(); +#else + PyThreadState_Swap(PyInterpreterState_ThreadHead(PyInterpreterState_Main())); + PyEval_SaveThread(); +#endif + + // set stopped event - this allows ::stop to run and kill remaining threads + // this event has to be fired without holding m_critical + // also the GIL (PyEval_AcquireLock) must not be held + // if not obeyed there is still no deadlock because ::stop waits with timeout (smart one!) + m_stoppedEvent.Set(); + + m_threadState = nullptr; + + setState(InvokerStateExecutionDone); + } + ILanguageInvoker::onExecutionDone(); +} + +void CPythonInvoker::onExecutionFailed() +{ + PyEval_SaveThread(); + + setState(InvokerStateFailed); + CLog::Log(LOGERROR, "CPythonInvoker({}, {}): abnormally terminating python thread", GetId(), + m_sourceFile); + + std::unique_lock<CCriticalSection> lock(m_critical); + m_threadState = NULL; + + ILanguageInvoker::onExecutionFailed(); +} + +void CPythonInvoker::onInitialization() +{ + XBMC_TRACE; + { + GilSafeSingleLock lock(s_critical); + initializeModules(getModules()); + } + + // get a possible initialization script + const char* runscript = getInitializationScript(); + if (runscript != NULL && strlen(runscript) > 0) + { + // redirecting default output to debug console + if (PyRun_SimpleString(runscript) == -1) + CLog::Log(LOGFATAL, "CPythonInvoker({}, {}): initialize error", GetId(), m_sourceFile); + } +} + +void CPythonInvoker::onPythonModuleInitialization(void* moduleDict) +{ + if (m_addon.get() == NULL || moduleDict == NULL) + return; + + PyObject* moduleDictionary = (PyObject*)moduleDict; + + PyObject* pyaddonid = PyUnicode_FromString(m_addon->ID().c_str()); + PyDict_SetItemString(moduleDictionary, "__xbmcaddonid__", pyaddonid); + + ADDON::CAddonVersion version = m_addon->GetDependencyVersion("xbmc.python"); + PyObject* pyxbmcapiversion = PyUnicode_FromString(version.asString().c_str()); + PyDict_SetItemString(moduleDictionary, "__xbmcapiversion__", pyxbmcapiversion); + + PyObject* pyinvokerid = PyLong_FromLong(GetId()); + PyDict_SetItemString(moduleDictionary, "__xbmcinvokerid__", pyinvokerid); + + CLog::Log(LOGDEBUG, + "CPythonInvoker({}, {}): instantiating addon using automatically obtained id of \"{}\" " + "dependent on version {} of the xbmc.python api", + GetId(), m_sourceFile, m_addon->ID(), version.asString()); +} + +void CPythonInvoker::onDeinitialization() +{ + XBMC_TRACE; +} + +void CPythonInvoker::onError(const std::string& exceptionType /* = "" */, + const std::string& exceptionValue /* = "" */, + const std::string& exceptionTraceback /* = "" */) +{ + CPyThreadState releaseGil; + std::unique_lock<CCriticalSection> gc(CServiceBroker::GetWinSystem()->GetGfxContext()); + + CGUIDialogKaiToast* pDlgToast = + CServiceBroker::GetGUI()->GetWindowManager().GetWindow<CGUIDialogKaiToast>( + WINDOW_DIALOG_KAI_TOAST); + if (pDlgToast != NULL) + { + std::string message; + if (m_addon && !m_addon->Name().empty()) + message = StringUtils::Format(g_localizeStrings.Get(2102), m_addon->Name()); + else + message = g_localizeStrings.Get(2103); + pDlgToast->QueueNotification(CGUIDialogKaiToast::Error, message, g_localizeStrings.Get(2104)); + } +} + +void CPythonInvoker::initializeModules( + const std::map<std::string, PythonModuleInitialization>& modules) +{ + for (const auto& module : modules) + { + if (!initializeModule(module.second)) + CLog::Log(LOGWARNING, "CPythonInvoker({}, {}): unable to initialize python module \"{}\"", + GetId(), m_sourceFile, module.first); + } +} + +bool CPythonInvoker::initializeModule(PythonModuleInitialization module) +{ + if (module == NULL) + return false; + + return module() != nullptr; +} + +void CPythonInvoker::getAddonModuleDeps(const ADDON::AddonPtr& addon, std::set<std::string>& paths) +{ + for (const auto& it : addon->GetDependencies()) + { + //Check if dependency is a module addon + ADDON::AddonPtr dependency; + if (CServiceBroker::GetAddonMgr().GetAddon(it.id, dependency, ADDON::AddonType::SCRIPT_MODULE, + ADDON::OnlyEnabled::CHOICE_YES)) + { + std::string path = CSpecialProtocol::TranslatePath(dependency->LibPath()); + if (paths.find(path) == paths.end()) + { + // add it and its dependencies + paths.insert(path); + getAddonModuleDeps(dependency, paths); + } + } + } +} |