diff options
Diffstat (limited to 'apt-pkg')
135 files changed, 58244 insertions, 0 deletions
diff --git a/apt-pkg/CMakeLists.txt b/apt-pkg/CMakeLists.txt new file mode 100644 index 0000000..d13aed9 --- /dev/null +++ b/apt-pkg/CMakeLists.txt @@ -0,0 +1,86 @@ +# Include apt-pkg directly, as some files have #include <system.h> +include_directories(${PROJECT_BINARY_DIR}/include/apt-pkg) + +file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/include/apt-pkg/) +execute_process(COMMAND grep -v "^#" "${CMAKE_CURRENT_SOURCE_DIR}/tagfile-keys.list" + OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/tagfile-keys.clean.list") +execute_process(COMMAND ${TRIEHASH_EXECUTABLE} + --ignore-case + --header ${PROJECT_BINARY_DIR}/include/apt-pkg/tagfile-keys.h + --code ${CMAKE_CURRENT_BINARY_DIR}/tagfile-keys.cc + --enum-class + --enum-name pkgTagSection::Key + --function-name pkgTagHash + --include "<apt-pkg/tagfile.h>" + --include "<apt-pkg/header-is-private.h>" + "${CMAKE_CURRENT_BINARY_DIR}/tagfile-keys.clean.list") +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "tagfile-keys.list") +set_source_files_properties("${CMAKE_CURRENT_BINARY_DIR}/tagfile-keys.cc" PROPERTIES COMPILE_DEFINITIONS APT_COMPILING_APT) + + +# Set the version of the library +execute_process(COMMAND awk -v ORS=. "/^\#define APT_PKG_M/ {print \$3}" + COMMAND sed "s/\\.\$//" + INPUT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/contrib/macros.h + OUTPUT_VARIABLE MAJOR OUTPUT_STRIP_TRAILING_WHITESPACE) +execute_process(COMMAND grep "^#define APT_PKG_RELEASE" + COMMAND cut -d " " -f 3 + INPUT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/contrib/macros.h + OUTPUT_VARIABLE MINOR OUTPUT_STRIP_TRAILING_WHITESPACE) + +message(STATUS "Building libapt-pkg ${MAJOR} (release ${MINOR})") +set(APT_PKG_MAJOR ${MAJOR} PARENT_SCOPE) # exporting for methods/CMakeLists.txt + +configure_file(apt-pkg.pc.in ${CMAKE_CURRENT_BINARY_DIR}/apt-pkg.pc @ONLY) + +# Definition of the C++ files used to build the library - note that this +# is expanded at CMake time, so you have to rerun cmake if you add or remove +# a file (you can just run cmake . in the build directory) +file(GLOB_RECURSE library "*.cc" "${CMAKE_CURRENT_BINARY_DIR}/tagfile-keys.cc") +file(GLOB_RECURSE headers "*.h") + +# Create a library using the C++ files +add_library(apt-pkg SHARED ${library}) +add_dependencies(apt-pkg apt-pkg-versionscript) +# Link the library and set the SONAME +target_include_directories(apt-pkg + PRIVATE ${ZLIB_INCLUDE_DIRS} + ${BZIP2_INCLUDE_DIR} + ${LZMA_INCLUDE_DIRS} + ${LZ4_INCLUDE_DIRS} + $<$<BOOL:${ZSTD_FOUND}>:${ZSTD_INCLUDE_DIRS}> + $<$<BOOL:${UDEV_FOUND}>:${UDEV_INCLUDE_DIRS}> + $<$<BOOL:${SYSTEMD_FOUND}>:${SYSTEMD_INCLUDE_DIRS}> + ${ICONV_INCLUDE_DIRS} + $<$<BOOL:${GCRYPT_FOUND}>:${GCRYPT_INCLUDE_DIRS}> + $<$<BOOL:${XXHASH_FOUND}>:${XXHASH_INCLUDE_DIRS}> +) + +target_link_libraries(apt-pkg + PRIVATE -lutil ${CMAKE_DL_LIBS} ${RESOLV_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} + ${ZLIB_LIBRARIES} + ${BZIP2_LIBRARIES} + ${LZMA_LIBRARIES} + ${LZ4_LIBRARIES} + $<$<BOOL:${ZSTD_FOUND}>:${ZSTD_LIBRARIES}> + $<$<BOOL:${UDEV_FOUND}>:${UDEV_LIBRARIES}> + $<$<BOOL:${SYSTEMD_FOUND}>:${SYSTEMD_LIBRARIES}> + ${ICONV_LIBRARIES} + $<$<BOOL:${GCRYPT_FOUND}>:${GCRYPT_LIBRARIES}> + $<$<BOOL:${XXHASH_FOUND}>:${XXHASH_LIBRARIES}> +) +set_target_properties(apt-pkg PROPERTIES VERSION ${MAJOR}.${MINOR}) +set_target_properties(apt-pkg PROPERTIES SOVERSION ${MAJOR}) +set_target_properties(apt-pkg PROPERTIES CXX_VISIBILITY_PRESET hidden) +add_version_script(apt-pkg) + +# Install the library and the header files +install(TARGETS apt-pkg LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) +install(FILES ${headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/apt-pkg) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/apt-pkg.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +flatify(${PROJECT_BINARY_DIR}/include/apt-pkg/ "${headers}") + +if(CMAKE_BUILD_TYPE STREQUAL "Coverage") + target_link_libraries(apt-pkg PUBLIC noprofile) +endif() diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc new file mode 100644 index 0000000..7df6483 --- /dev/null +++ b/apt-pkg/acquire-item.cc @@ -0,0 +1,4115 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire Item - Item to acquire + + Each item can download to exactly one file at a time. This means you + cannot create an item that fetches two uri's to two files at the same + time. The pkgAcqIndex class creates a second class upon instantiation + to fetch the other index files because of this. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/acquire-item.h> +#include <apt-pkg/acquire-worker.h> +#include <apt-pkg/acquire.h> +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/gpgv.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/indexfile.h> +#include <apt-pkg/metaindex.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgrecords.h> +#include <apt-pkg/sourcelist.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile.h> + +#include <algorithm> +#include <ctime> +#include <chrono> +#include <iostream> +#include <memory> +#include <numeric> +#include <random> +#include <sstream> +#include <string> +#include <unordered_set> +#include <vector> +#include <errno.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +static std::string GetPartialFileName(std::string const &file) /*{{{*/ +{ + std::string DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += file; + return DestFile; +} + /*}}}*/ +static std::string GetPartialFileNameFromURI(std::string const &uri) /*{{{*/ +{ + return GetPartialFileName(URItoFileName(uri)); +} + /*}}}*/ +static std::string GetFinalFileNameFromURI(std::string const &uri) /*{{{*/ +{ + return _config->FindDir("Dir::State::lists") + URItoFileName(uri); +} + /*}}}*/ +static std::string GetKeepCompressedFileName(std::string file, IndexTarget const &Target)/*{{{*/ +{ + if (Target.KeepCompressed == false) + return file; + + std::string const KeepCompressedAs = Target.Option(IndexTarget::KEEPCOMPRESSEDAS); + if (KeepCompressedAs.empty() == false) + { + std::string const ext = KeepCompressedAs.substr(0, KeepCompressedAs.find(' ')); + if (ext != "uncompressed") + file.append(".").append(ext); + } + return file; +} + /*}}}*/ +static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)/*{{{*/ +{ + // rred expects the patch as $FinalFile.ed.$patchname.gz + return Final + ".ed." + Patch + ".gz"; +} + /*}}}*/ +static std::string GetDiffsPatchFileName(std::string const &Final) /*{{{*/ +{ + // rred expects the patch as $FinalFile.ed + return Final + ".ed"; +} + /*}}}*/ +static std::string GetExistingFilename(std::string const &File) /*{{{*/ +{ + if (RealFileExists(File)) + return File; + for (auto const &type : APT::Configuration::getCompressorExtensions()) + { + std::string const Final = File + type; + if (RealFileExists(Final)) + return Final; + } + return ""; +} + /*}}}*/ +static std::string GetDiffIndexFileName(std::string const &Name) /*{{{*/ +{ + return Name + ".diff/Index"; +} + /*}}}*/ +static std::string GetDiffIndexURI(IndexTarget const &Target) /*{{{*/ +{ + return Target.URI + ".diff/Index"; +} + /*}}}*/ + +static void ReportMirrorFailureToCentral(pkgAcquire::Item const &I, std::string const &FailCode, std::string const &Details)/*{{{*/ +{ + // we only act if a mirror was used at all + if(I.UsedMirror.empty()) + return; +#if 0 + std::cerr << "\nReportMirrorFailure: " + << UsedMirror + << " Uri: " << DescURI() + << " FailCode: " + << FailCode << std::endl; +#endif + string const report = _config->Find("Methods::Mirror::ProblemReporting", + LIBEXEC_DIR "/apt-report-mirror-failure"); + if(!FileExists(report)) + return; + + std::vector<char const*> const Args = { + report.c_str(), + I.UsedMirror.c_str(), + I.DescURI().c_str(), + FailCode.c_str(), + Details.c_str(), + NULL + }; + + pid_t pid = ExecFork(); + if(pid < 0) + { + _error->Error("ReportMirrorFailure Fork failed"); + return; + } + else if(pid == 0) + { + execvp(Args[0], (char**)Args.data()); + std::cerr << "Could not exec " << Args[0] << std::endl; + _exit(100); + } + if(!ExecWait(pid, "report-mirror-failure")) + _error->Warning("Couldn't report problem to '%s'", report.c_str()); +} + /*}}}*/ + +static APT_NONNULL(2) bool MessageInsecureRepository(bool const isError, char const * const msg, std::string const &repo)/*{{{*/ +{ + std::string m; + strprintf(m, msg, repo.c_str()); + if (isError) + { + _error->Error("%s", m.c_str()); + _error->Notice("%s", _("Updating from such a repository can't be done securely, and is therefore disabled by default.")); + } + else + { + _error->Warning("%s", m.c_str()); + _error->Notice("%s", _("Data from such a repository can't be authenticated and is therefore potentially dangerous to use.")); + } + _error->Notice("%s", _("See apt-secure(8) manpage for repository creation and user configuration details.")); + return false; +} + /*}}}*/ +// AllowInsecureRepositories /*{{{*/ +enum class InsecureType { UNSIGNED, WEAK, NORELEASE }; +static bool TargetIsAllowedToBe(IndexTarget const &Target, InsecureType const type) +{ + if (_config->FindB("Acquire::AllowInsecureRepositories")) + return true; + + if (Target.OptionBool(IndexTarget::ALLOW_INSECURE)) + return true; + + switch (type) + { + case InsecureType::UNSIGNED: break; + case InsecureType::NORELEASE: break; + case InsecureType::WEAK: + if (_config->FindB("Acquire::AllowWeakRepositories")) + return true; + if (Target.OptionBool(IndexTarget::ALLOW_WEAK)) + return true; + break; + } + return false; +} +static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType const msg, std::string const &repo, + metaIndex const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I) +{ + // we skip weak downgrades as its unlikely that a repository gets really weaker – + // its more realistic that apt got pickier in a newer version + if (msg != InsecureType::WEAK) + { + std::string const FinalInRelease = TransactionManager->GetFinalFilename(); + std::string const FinalReleasegpg = FinalInRelease.substr(0, FinalInRelease.length() - strlen("InRelease")) + "Release.gpg"; + if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease)) + { + char const * msgstr = nullptr; + switch (msg) + { + case InsecureType::UNSIGNED: msgstr = _("The repository '%s' is no longer signed."); break; + case InsecureType::NORELEASE: msgstr = _("The repository '%s' no longer has a Release file."); break; + case InsecureType::WEAK: /* unreachable */ break; + } + if (_config->FindB("Acquire::AllowDowngradeToInsecureRepositories") || + TransactionManager->Target.OptionBool(IndexTarget::ALLOW_DOWNGRADE_TO_INSECURE)) + { + // meh, the users wants to take risks (we still mark the packages + // from this repository as unauthenticated) + _error->Warning(msgstr, repo.c_str()); + _error->Warning(_("This is normally not allowed, but the option " + "Acquire::AllowDowngradeToInsecureRepositories was " + "given to override it.")); + } else { + MessageInsecureRepository(true, msgstr, repo); + TransactionManager->AbortTransaction(); + I->Status = pkgAcquire::Item::StatError; + return false; + } + } + } + + if(MetaIndexParser->GetTrusted() == metaIndex::TRI_YES) + return true; + + char const * msgstr = nullptr; + switch (msg) + { + case InsecureType::UNSIGNED: msgstr = _("The repository '%s' is not signed."); break; + case InsecureType::NORELEASE: msgstr = _("The repository '%s' does not have a Release file."); break; + case InsecureType::WEAK: msgstr = _("The repository '%s' provides only weak security information."); break; + } + + if (TargetIsAllowedToBe(TransactionManager->Target, msg) == true) + { + MessageInsecureRepository(false, msgstr, repo); + return true; + } + + MessageInsecureRepository(true, msgstr, repo); + TransactionManager->AbortTransaction(); + I->Status = pkgAcquire::Item::StatError; + return false; +} + /*}}}*/ +static HashStringList GetExpectedHashesFromFor(metaIndex * const Parser, std::string const &MetaKey)/*{{{*/ +{ + if (Parser == NULL) + return HashStringList(); + metaIndex::checkSum * const R = Parser->Lookup(MetaKey); + if (R == NULL) + return HashStringList(); + return R->Hashes; +} + /*}}}*/ + +class pkgAcquire::Item::Private /*{{{*/ +{ +public: + struct AlternateURI + { + std::string URI; + std::unordered_map<std::string, std::string> changefields; + AlternateURI(std::string &&u, decltype(changefields) &&cf) : URI(u), changefields(cf) {} + }; + std::list<AlternateURI> AlternativeURIs; + std::vector<std::string> BadAlternativeSites; + std::vector<std::string> PastRedirections; + std::unordered_map<std::string, std::string> CustomFields; + time_point FetchAfter = {}; + + Private() + { + } +}; + /*}}}*/ + +// all ::HashesRequired and ::GetExpectedHashes implementations /*{{{*/ +/* ::GetExpectedHashes is abstract and has to be implemented by all subclasses. + It is best to implement it as broadly as possible, while ::HashesRequired defaults + to true and should be as restrictive as possible for false cases. Note that if + a hash is returned by ::GetExpectedHashes it must match. Only if it doesn't + ::HashesRequired is called to evaluate if its okay to have no hashes. */ +APT_PURE bool pkgAcqTransactionItem::HashesRequired() const +{ + /* signed repositories obviously have a parser and good hashes. + unsigned repositories, too, as even if we can't trust them for security, + we can at least trust them for integrity of the download itself. + Only repositories without a Release file can (obviously) not have + hashes – and they are very uncommon and strongly discouraged */ + if (TransactionManager->MetaIndexParser->GetLoadedSuccessfully() != metaIndex::TRI_YES) + return false; + if (TargetIsAllowedToBe(Target, InsecureType::WEAK)) + { + /* If we allow weak hashes, we check that we have some (weak) and then + declare hashes not needed. That will tip us in the right direction + as if hashes exist, they will be used, even if not required */ + auto const hsl = GetExpectedHashes(); + if (hsl.usable()) + return true; + if (hsl.empty() == false) + return false; + } + return true; +} +HashStringList pkgAcqTransactionItem::GetExpectedHashes() const +{ + return GetExpectedHashesFor(GetMetaKey()); +} + +APT_PURE bool pkgAcqMetaBase::HashesRequired() const +{ + // Release and co have no hashes 'by design'. + return false; +} +HashStringList pkgAcqMetaBase::GetExpectedHashes() const +{ + return HashStringList(); +} + +APT_PURE bool pkgAcqIndexDiffs::HashesRequired() const +{ + /* We can't check hashes of rred result as we don't know what the + hash of the file will be. We just know the hash of the patch(es), + the hash of the file they will apply on and the hash of the resulting + file. */ + if (State == StateFetchDiff) + return true; + return false; +} +HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const +{ + if (State == StateFetchDiff) + return available_patches[0].download_hashes; + return HashStringList(); +} + +APT_PURE bool pkgAcqIndexMergeDiffs::HashesRequired() const +{ + /* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that + we can check the rred result after all patches are applied as + we know the expected result rather than potentially apply more patches */ + if (State == StateFetchDiff) + return true; + return State == StateApplyDiff; +} +HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const +{ + if (State == StateFetchDiff) + return patch.download_hashes; + else if (State == StateApplyDiff) + return GetExpectedHashesFor(Target.MetaKey); + return HashStringList(); +} + +APT_PURE bool pkgAcqArchive::HashesRequired() const +{ + return LocalSource == false; +} +HashStringList pkgAcqArchive::GetExpectedHashes() const +{ + // figured out while parsing the records + return ExpectedHashes; +} + +APT_PURE bool pkgAcqFile::HashesRequired() const +{ + // supplied as parameter at creation time, so the caller decides + return ExpectedHashes.usable(); +} +HashStringList pkgAcqFile::GetExpectedHashes() const +{ + return ExpectedHashes; +} + /*}}}*/ +// Acquire::Item::QueueURI and specialisations from child classes /*{{{*/ +bool pkgAcquire::Item::QueueURI(pkgAcquire::ItemDesc &Item) +{ + Owner->Enqueue(Item); + return true; +} +/* The idea here is that an item isn't queued if it exists on disk and the + transition manager was a hit as this means that the files it contains + the checksums for can't be updated either (or they are and we are asking + for a hashsum mismatch to happen which helps nobody) */ +bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item) +{ + if (TransactionManager->State != TransactionStarted) + { + if (_config->FindB("Debug::Acquire::Transaction", false)) + std::clog << "Skip " << Target.URI << " as transaction was already dealt with!" << std::endl; + return false; + } + std::string const FinalFile = GetFinalFilename(); + if (TransactionManager->IMSHit == true && FileExists(FinalFile) == true) + { + PartialFile = DestFile = FinalFile; + Status = StatDone; + return false; + } + // this ensures we rewrite only once and only the first step + auto const OldBaseURI = Target.Option(IndexTarget::BASE_URI); + if (OldBaseURI.empty() || APT::String::Startswith(Item.URI, OldBaseURI) == false) + return pkgAcquire::Item::QueueURI(Item); + // the given URI is our last resort + PushAlternativeURI(std::string(Item.URI), {}, false); + // If we got the InRelease file via a mirror, pick all indexes directly from this mirror, too + std::string SameMirrorURI; + if (TransactionManager->BaseURI.empty() == false && TransactionManager->UsedMirror.empty() == false && + URI::SiteOnly(Item.URI) != URI::SiteOnly(TransactionManager->BaseURI)) + { + auto ExtraPath = Item.URI.substr(OldBaseURI.length()); + auto newURI = flCombine(TransactionManager->BaseURI, std::move(ExtraPath)); + if (IsGoodAlternativeURI(newURI)) + { + SameMirrorURI = std::move(newURI); + PushAlternativeURI(std::string(SameMirrorURI), {}, false); + } + } + // add URI and by-hash based on it + if (AcquireByHash()) + { + // if we use the mirror transport, ask it for by-hash uris + // we need to stick to the same mirror only for non-unique filenames + auto const sameMirrorException = [&]() { + if (Item.URI.find("mirror") == std::string::npos) + return false; + ::URI uri(Item.URI); + return uri.Access == "mirror" || APT::String::Startswith(uri.Access, "mirror+") || + APT::String::Endswith(uri.Access, "+mirror") || uri.Access.find("+mirror+") != std::string::npos; + }(); + if (sameMirrorException) + SameMirrorURI.clear(); + // now add the actual by-hash uris + auto const Expected = GetExpectedHashes(); + auto const TargetHash = Expected.find(nullptr); + auto const PushByHashURI = [&](std::string U) { + if (unlikely(TargetHash == nullptr)) + return false; + auto const trailing_slash = U.find_last_of("/"); + if (unlikely(trailing_slash == std::string::npos)) + return false; + auto byhashSuffix = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); + U.replace(trailing_slash, U.length() - trailing_slash, std::move(byhashSuffix)); + PushAlternativeURI(std::move(U), {}, false); + return true; + }; + PushByHashURI(Item.URI); + if (SameMirrorURI.empty() == false && PushByHashURI(SameMirrorURI) == false) + SameMirrorURI.clear(); + } + // the last URI added is the first one tried + if (unlikely(PopAlternativeURI(Item.URI) == false)) + return false; + if (SameMirrorURI.empty() == false) + { + UsedMirror = TransactionManager->UsedMirror; + if (Item.Description.find(" ") != string::npos) + Item.Description.replace(0, Item.Description.find(" "), UsedMirror); + } + return pkgAcquire::Item::QueueURI(Item); +} +/* The transition manager InRelease itself (or its older sisters-in-law + Release & Release.gpg) is always queued as this allows us to rerun gpgv + on it to verify that we aren't stalled with old files */ +bool pkgAcqMetaBase::QueueURI(pkgAcquire::ItemDesc &Item) +{ + return pkgAcquire::Item::QueueURI(Item); +} +/* the Diff/Index needs to queue also the up-to-date complete index file + to ensure that the list cleaner isn't eating it */ +bool pkgAcqDiffIndex::QueueURI(pkgAcquire::ItemDesc &Item) +{ + if (pkgAcqTransactionItem::QueueURI(Item) == true) + return true; + QueueOnIMSHit(); + return false; +} + /*}}}*/ +// Acquire::Item::GetFinalFilename and specialisations for child classes /*{{{*/ +std::string pkgAcquire::Item::GetFinalFilename() const +{ + // Beware: Desc.URI is modified by redirections + return GetFinalFileNameFromURI(Desc.URI); +} +std::string pkgAcqDiffIndex::GetFinalFilename() const +{ + std::string const FinalFile = GetFinalFileNameFromURI(GetDiffIndexURI(Target)); + // we don't want recompress, so lets keep whatever we got + if (CurrentCompressionExtension == "uncompressed") + return FinalFile; + return FinalFile + "." + CurrentCompressionExtension; +} +std::string pkgAcqIndex::GetFinalFilename() const +{ + std::string const FinalFile = GetFinalFileNameFromURI(Target.URI); + return GetKeepCompressedFileName(FinalFile, Target); +} +std::string pkgAcqMetaSig::GetFinalFilename() const +{ + return GetFinalFileNameFromURI(Target.URI); +} +std::string pkgAcqBaseIndex::GetFinalFilename() const +{ + return GetFinalFileNameFromURI(Target.URI); +} +std::string pkgAcqMetaBase::GetFinalFilename() const +{ + return GetFinalFileNameFromURI(Target.URI); +} +std::string pkgAcqArchive::GetFinalFilename() const +{ + return _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename); +} + /*}}}*/ +// pkgAcqTransactionItem::GetMetaKey and specialisations for child classes /*{{{*/ +std::string pkgAcqTransactionItem::GetMetaKey() const +{ + return Target.MetaKey; +} +std::string pkgAcqIndex::GetMetaKey() const +{ + if (Stage == STAGE_DECOMPRESS_AND_VERIFY || CurrentCompressionExtension == "uncompressed") + return Target.MetaKey; + return Target.MetaKey + "." + CurrentCompressionExtension; +} +std::string pkgAcqDiffIndex::GetMetaKey() const +{ + auto const metakey = GetDiffIndexFileName(Target.MetaKey); + if (CurrentCompressionExtension == "uncompressed") + return metakey; + return metakey + "." + CurrentCompressionExtension; +} + /*}}}*/ +//pkgAcqTransactionItem::TransactionState and specialisations for child classes /*{{{*/ +bool pkgAcqTransactionItem::TransactionState(TransactionStates const state) +{ + bool const Debug = _config->FindB("Debug::Acquire::Transaction", false); + switch(state) + { + case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break; + case TransactionAbort: + if(Debug == true) + std::clog << " Cancel: " << DestFile << std::endl; + if (Status == pkgAcquire::Item::StatIdle) + { + Status = pkgAcquire::Item::StatDone; + Dequeue(); + } + break; + case TransactionCommit: + if(PartialFile.empty() == false) + { + bool sameFile = (PartialFile == DestFile); + // we use symlinks on IMS-Hit to avoid copies + if (RealFileExists(DestFile)) + { + struct stat Buf; + if (lstat(PartialFile.c_str(), &Buf) != -1) + { + if (S_ISLNK(Buf.st_mode) && Buf.st_size > 0) + { + char partial[Buf.st_size + 1]; + ssize_t const sp = readlink(PartialFile.c_str(), partial, Buf.st_size); + if (sp == -1) + _error->Errno("pkgAcqTransactionItem::TransactionState-sp", _("Failed to readlink %s"), PartialFile.c_str()); + else + { + partial[sp] = '\0'; + sameFile = (DestFile == partial); + } + } + } + else + _error->Errno("pkgAcqTransactionItem::TransactionState-stat", _("Failed to stat %s"), PartialFile.c_str()); + } + if (sameFile == false) + { + // ensure that even without lists-cleanup all compressions are nuked + std::string FinalFile = GetFinalFileNameFromURI(Target.URI); + if (FileExists(FinalFile)) + { + if(Debug == true) + std::clog << "rm " << FinalFile << " # " << DescURI() << std::endl; + if (RemoveFile("TransactionStates-Cleanup", FinalFile) == false) + return false; + } + for (auto const &ext: APT::Configuration::getCompressorExtensions()) + { + auto const Final = FinalFile + ext; + if (FileExists(Final)) + { + if(Debug == true) + std::clog << "rm " << Final << " # " << DescURI() << std::endl; + if (RemoveFile("TransactionStates-Cleanup", Final) == false) + return false; + } + } + if(Debug == true) + std::clog << "mv " << PartialFile << " -> "<< DestFile << " # " << DescURI() << std::endl; + if (Rename(PartialFile, DestFile) == false) + return false; + } + else if(Debug == true) + std::clog << "keep " << PartialFile << " # " << DescURI() << std::endl; + + } else { + if(Debug == true) + std::clog << "rm " << DestFile << " # " << DescURI() << std::endl; + if (RemoveFile("TransItem::TransactionCommit", DestFile) == false) + return false; + } + break; + } + return true; +} +bool pkgAcqMetaBase::TransactionState(TransactionStates const state) +{ + // Do not remove InRelease on IMSHit of Release.gpg [yes, this is very edgecasey] + if (TransactionManager->IMSHit == false) + return pkgAcqTransactionItem::TransactionState(state); + return true; +} +bool pkgAcqIndex::TransactionState(TransactionStates const state) +{ + if (pkgAcqTransactionItem::TransactionState(state) == false) + return false; + + switch (state) + { + case TransactionStarted: _error->Fatal("AcqIndex %s changed to invalid transaction start state!", Target.URI.c_str()); break; + case TransactionAbort: + if (Stage == STAGE_DECOMPRESS_AND_VERIFY) + { + // keep the compressed file, but drop the decompressed + EraseFileName.clear(); + if (PartialFile.empty() == false && flExtension(PartialFile) != CurrentCompressionExtension) + RemoveFile("TransactionAbort", PartialFile); + } + break; + case TransactionCommit: + if (EraseFileName.empty() == false) + RemoveFile("AcqIndex::TransactionCommit", EraseFileName); + break; + } + return true; +} +bool pkgAcqDiffIndex::TransactionState(TransactionStates const state) +{ + if (pkgAcqTransactionItem::TransactionState(state) == false) + return false; + + switch (state) + { + case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break; + case TransactionCommit: + break; + case TransactionAbort: + std::string const Partial = GetPartialFileNameFromURI(Target.URI); + RemoveFile("TransactionAbort", Partial); + break; + } + + return true; +} + /*}}}*/ +// pkgAcqTransactionItem::AcquireByHash and specialisations for child classes /*{{{*/ +bool pkgAcqTransactionItem::AcquireByHash() const +{ + if (TransactionManager->MetaIndexParser == nullptr) + return false; + auto const useByHashConf = Target.Option(IndexTarget::BY_HASH); + if (useByHashConf == "force") + return true; + return StringToBool(useByHashConf) == true && TransactionManager->MetaIndexParser->GetSupportsAcquireByHash(); +} +// pdiff patches have a unique name already, no need for by-hash +bool pkgAcqIndexMergeDiffs::AcquireByHash() const +{ + return false; +} +bool pkgAcqIndexDiffs::AcquireByHash() const +{ + return false; +} + /*}}}*/ + +class APT_HIDDEN NoActionItem : public pkgAcquire::Item /*{{{*/ +/* The sole purpose of this class is having an item which does nothing to + reach its done state to prevent cleanup deleting the mentioned file. + Handy in cases in which we know we have the file already, like IMS-Hits. */ +{ + IndexTarget const Target; + public: + virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;}; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE {return HashStringList();}; + + NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target) : + pkgAcquire::Item(Owner), Target(Target) + { + Status = StatDone; + DestFile = GetFinalFileNameFromURI(Target.URI); + } + NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target, std::string const &FinalFile) : + pkgAcquire::Item(Owner), Target(Target) + { + Status = StatDone; + DestFile = FinalFile; + } +}; + /*}}}*/ +class APT_HIDDEN CleanupItem : public pkgAcqTransactionItem /*{{{*/ +/* This class ensures that a file which was configured but isn't downloaded + for various reasons isn't kept in an old version in the lists directory. + In a way its the reverse of NoActionItem as it helps with removing files + even if the lists-cleanup is deactivated. */ +{ + public: + virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;}; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE {return HashStringList();}; + + CleanupItem(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target) : + pkgAcqTransactionItem(Owner, TransactionManager, Target) + { + Status = StatDone; + DestFile = GetFinalFileNameFromURI(Target.URI); + } + bool TransactionState(TransactionStates const state) APT_OVERRIDE + { + switch (state) + { + case TransactionStarted: + break; + case TransactionAbort: + break; + case TransactionCommit: + if (_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "rm " << DestFile << " # " << DescURI() << std::endl; + if (RemoveFile("TransItem::TransactionCommit", DestFile) == false) + return false; + break; + } + return true; + } +}; + /*}}}*/ + +// Acquire::Item::Item - Constructor /*{{{*/ +pkgAcquire::Item::Item(pkgAcquire * const owner) : + FileSize(0), PartialSize(0), ID(0), Complete(false), Local(false), + QueueCounter(0), ExpectedAdditionalItems(0), Retries(_config->FindI("Acquire::Retries", 3)), Owner(owner), d(new Private()) +{ + Owner->Add(this); + Status = StatIdle; +} + /*}}}*/ +// Acquire::Item::~Item - Destructor /*{{{*/ +pkgAcquire::Item::~Item() +{ + Owner->Remove(this); + delete d; +} + /*}}}*/ +std::string pkgAcquire::Item::Custom600Headers() const /*{{{*/ +{ + std::ostringstream header; + for (auto const &f : d->CustomFields) + if (f.second.empty() == false) + header << '\n' + << f.first << ": " << f.second; + return header.str(); +} + /*}}}*/ +std::unordered_map<std::string, std::string> &pkgAcquire::Item::ModifyCustomFields() /*{{{*/ +{ + return d->CustomFields; +} + /*}}}*/ +bool pkgAcquire::Item::PopAlternativeURI(std::string &NewURI) /*{{{*/ +{ + if (d->AlternativeURIs.empty()) + return false; + auto const AltUri = d->AlternativeURIs.front(); + d->AlternativeURIs.pop_front(); + NewURI = AltUri.URI; + auto &CustomFields = ModifyCustomFields(); + for (auto const &f : AltUri.changefields) + { + if (f.second.empty()) + CustomFields.erase(f.first); + else + CustomFields[f.first] = f.second; + } + return true; +} + /*}}}*/ +bool pkgAcquire::Item::IsGoodAlternativeURI(std::string const &AltUri) const/*{{{*/ +{ + return std::find(d->PastRedirections.cbegin(), d->PastRedirections.cend(), AltUri) == d->PastRedirections.cend() && + std::find(d->BadAlternativeSites.cbegin(), d->BadAlternativeSites.cend(), URI::SiteOnly(AltUri)) == d->BadAlternativeSites.cend(); +} + /*}}}*/ +void pkgAcquire::Item::PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back) /*{{{*/ +{ + if (IsGoodAlternativeURI(NewURI) == false) + return; + if (at_the_back) + d->AlternativeURIs.emplace_back(std::move(NewURI), std::move(fields)); + else + d->AlternativeURIs.emplace_front(std::move(NewURI), std::move(fields)); +} + /*}}}*/ +void pkgAcquire::Item::RemoveAlternativeSite(std::string &&OldSite) /*{{{*/ +{ + d->AlternativeURIs.erase(std::remove_if(d->AlternativeURIs.begin(), d->AlternativeURIs.end(), + [&](decltype(*d->AlternativeURIs.cbegin()) AltUri) { + return URI::SiteOnly(AltUri.URI) == OldSite; + }), + d->AlternativeURIs.end()); + d->BadAlternativeSites.push_back(std::move(OldSite)); +} + /*}}}*/ +std::string pkgAcquire::Item::ShortDesc() const /*{{{*/ +{ + return DescURI(); +} + /*}}}*/ +void pkgAcquire::Item::Finished() /*{{{*/ +{ +} + /*}}}*/ +APT_PURE pkgAcquire * pkgAcquire::Item::GetOwner() const /*{{{*/ +{ + return Owner; +} + /*}}}*/ +APT_PURE pkgAcquire::ItemDesc &pkgAcquire::Item::GetItemDesc() /*{{{*/ +{ + return Desc; +} + /*}}}*/ +APT_PURE bool pkgAcquire::Item::IsTrusted() const /*{{{*/ +{ + return false; +} + /*}}}*/ +// Acquire::Item::Failed - Item failed to download /*{{{*/ +// --------------------------------------------------------------------- +/* We return to an idle state if there are still other queues that could + fetch this object */ +static void formatHashsum(std::ostream &out, HashString const &hs) +{ + auto const type = hs.HashType(); + if (type == "Checksum-FileSize") + out << " - Filesize"; + else + out << " - " << type; + out << ':' << hs.HashValue(); + if (hs.usable() == false) + out << " [weak]"; + out << std::endl; +} +void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) +{ + if (QueueCounter <= 1) + { + /* This indicates that the file is not available right now but might + be sometime later. If we do a retry cycle then this should be + retried [CDROMs] */ + if (Cnf != NULL && Cnf->LocalOnly == true && + StringToBool(LookupTag(Message,"Transient-Failure"),false) == true) + { + Status = StatIdle; + Dequeue(); + return; + } + + switch (Status) + { + case StatIdle: + case StatFetching: + case StatDone: + Status = StatError; + break; + case StatAuthError: + case StatError: + case StatTransientNetworkError: + break; + } + Complete = false; + Dequeue(); + } + + FailMessage(Message); + + if (QueueCounter > 1) + Status = StatIdle; +} +void pkgAcquire::Item::FailMessage(string const &Message) +{ + string const FailReason = LookupTag(Message, "FailReason"); + enum { MAXIMUM_SIZE_EXCEEDED, HASHSUM_MISMATCH, WEAK_HASHSUMS, REDIRECTION_LOOP, OTHER } failreason = OTHER; + if ( FailReason == "MaximumSizeExceeded") + failreason = MAXIMUM_SIZE_EXCEEDED; + else if ( FailReason == "WeakHashSums") + failreason = WEAK_HASHSUMS; + else if (FailReason == "RedirectionLoop") + failreason = REDIRECTION_LOOP; + else if (Status == StatAuthError) + failreason = HASHSUM_MISMATCH; + + if(ErrorText.empty()) + { + std::ostringstream out; + switch (failreason) + { + case HASHSUM_MISMATCH: + out << _("Hash Sum mismatch") << std::endl; + break; + case WEAK_HASHSUMS: + out << _("Insufficient information available to perform this download securely") << std::endl; + break; + case REDIRECTION_LOOP: + out << "Redirection loop encountered" << std::endl; + break; + case MAXIMUM_SIZE_EXCEEDED: + out << LookupTag(Message, "Message") << std::endl; + break; + case OTHER: + out << LookupTag(Message, "Message"); + break; + } + + if (Status == StatAuthError) + { + auto const ExpectedHashes = GetExpectedHashes(); + if (ExpectedHashes.empty() == false) + { + out << "Hashes of expected file:" << std::endl; + for (auto const &hs: ExpectedHashes) + formatHashsum(out, hs); + } + if (failreason == HASHSUM_MISMATCH) + { + out << "Hashes of received file:" << std::endl; + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + { + std::string const tagname = std::string(*type) + "-Hash"; + std::string const hashsum = LookupTag(Message, tagname.c_str()); + if (hashsum.empty() == false) + formatHashsum(out, HashString(*type, hashsum)); + } + } + auto const lastmod = LookupTag(Message, "Last-Modified", ""); + if (lastmod.empty() == false) + out << "Last modification reported: " << lastmod << std::endl; + } + ErrorText = out.str(); + } + + switch (failreason) + { + case MAXIMUM_SIZE_EXCEEDED: RenameOnError(MaximumSizeExceeded); break; + case HASHSUM_MISMATCH: RenameOnError(HashSumMismatch); break; + case WEAK_HASHSUMS: break; + case REDIRECTION_LOOP: break; + case OTHER: break; + } + + if (FailReason.empty() == false) + ReportMirrorFailureToCentral(*this, FailReason, ErrorText); + else + ReportMirrorFailureToCentral(*this, ErrorText, ErrorText); +} + /*}}}*/ +// Acquire::Item::Start - Item has begun to download /*{{{*/ +// --------------------------------------------------------------------- +/* Stash status and the file size. Note that setting Complete means + sub-phases of the acquire process such as decompression are operating */ +void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const Size) +{ + Status = StatFetching; + ErrorText.clear(); + if (FileSize == 0 && Complete == false) + FileSize = Size; +} + /*}}}*/ +// Acquire::Item::VerifyDone - check if Item was downloaded OK /*{{{*/ +/* Note that hash-verification is 'hardcoded' in acquire-worker and has + * already passed if this method is called. */ +bool pkgAcquire::Item::VerifyDone(std::string const &Message, + pkgAcquire::MethodConfig const * const /*Cnf*/) +{ + std::string const FileName = LookupTag(Message,"Filename"); + if (FileName.empty() == true) + { + Status = StatError; + ErrorText = "Method gave a blank filename"; + return false; + } + + return true; +} + /*}}}*/ +// Acquire::Item::Done - Item downloaded OK /*{{{*/ +void pkgAcquire::Item::Done(string const &/*Message*/, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const /*Cnf*/) +{ + // We just downloaded something.. + if (FileSize == 0) + { + unsigned long long const downloadedSize = Hashes.FileSize(); + if (downloadedSize != 0) + { + FileSize = downloadedSize; + } + } + Status = StatDone; + ErrorText.clear(); + Dequeue(); +} + /*}}}*/ +// Acquire::Item::Rename - Rename a file /*{{{*/ +// --------------------------------------------------------------------- +/* This helper function is used by a lot of item methods as their final + step */ +bool pkgAcquire::Item::Rename(string const &From,string const &To) +{ + if (From == To || rename(From.c_str(),To.c_str()) == 0) + return true; + + std::string S; + strprintf(S, _("rename failed, %s (%s -> %s)."), strerror(errno), + From.c_str(),To.c_str()); + Status = StatError; + if (ErrorText.empty()) + ErrorText = S; + else + ErrorText = ErrorText + ": " + S; + return false; +} + /*}}}*/ +void pkgAcquire::Item::Dequeue() /*{{{*/ +{ + d->AlternativeURIs.clear(); + Owner->Dequeue(this); +} + /*}}}*/ +bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/ +{ + if (RealFileExists(DestFile)) + Rename(DestFile, DestFile + ".FAILED"); + + std::string errtext; + switch (error) + { + case HashSumMismatch: + errtext = _("Hash Sum mismatch"); + break; + case SizeMismatch: + errtext = _("Size mismatch"); + Status = StatAuthError; + break; + case InvalidFormat: + errtext = _("Invalid file format"); + Status = StatError; + // do not report as usually its not the mirrors fault, but Portal/Proxy + break; + case SignatureError: + errtext = _("Signature error"); + Status = StatError; + break; + case NotClearsigned: + strprintf(errtext, _("Clearsigned file isn't valid, got '%s' (does the network require authentication?)"), "NOSPLIT"); + Status = StatAuthError; + break; + case MaximumSizeExceeded: + // the method is expected to report a good error for this + break; + case PDiffError: + // no handling here, done by callers + break; + } + if (ErrorText.empty()) + ErrorText = errtext; + return false; +} + /*}}}*/ +void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess)/*{{{*/ +{ + ActiveSubprocess = subprocess; +} + /*}}}*/ +std::string pkgAcquire::Item::HashSum() const /*{{{*/ +{ + HashStringList const hashes = GetExpectedHashes(); + HashString const * const hs = hashes.find(NULL); + return hs != NULL ? hs->toStr() : ""; +} + /*}}}*/ +void pkgAcquire::Item::FetchAfter(time_point FetchAfter) /*{{{*/ +{ + d->FetchAfter = FetchAfter; +} +pkgAcquire::time_point pkgAcquire::Item::FetchAfter() +{ + return d->FetchAfter; +} + /*}}}*/ +bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI) /*{{{*/ +{ + // store can fail due to permission errors and the item will "loop" then + if (APT::String::Startswith(NewURI, "store:")) + return false; + if (d->PastRedirections.empty()) + { + d->PastRedirections.push_back(NewURI); + return false; + } + auto const LastURI = std::prev(d->PastRedirections.end()); + // redirections to the same file are a way of restarting/resheduling, + // individual methods will have to make sure that they aren't looping this way + if (*LastURI == NewURI) + return false; + if (std::find(d->PastRedirections.begin(), LastURI, NewURI) != LastURI) + return true; + d->PastRedirections.push_back(NewURI); + return false; +} + /*}}}*/ +int pkgAcquire::Item::Priority() /*{{{*/ +{ + // Stage 0: Files requested by methods + // - they will usually not end up here, but if they do we make sure + // to get them as soon as possible as they are probably blocking + // the processing of files by the requesting method + if (dynamic_cast<pkgAcqAuxFile *>(this) != nullptr) + return 5000; + // Stage 1: Meta indices and diff indices + // - those need to be fetched first to have progress reporting working + // for the rest + if (dynamic_cast<pkgAcqMetaSig*>(this) != nullptr + || dynamic_cast<pkgAcqMetaBase*>(this) != nullptr + || dynamic_cast<pkgAcqDiffIndex*>(this) != nullptr) + return 1000; + // Stage 2: Diff files + // - fetch before complete indexes so we can apply the diffs while fetching + // larger files. + if (dynamic_cast<pkgAcqIndexDiffs*>(this) != nullptr || + dynamic_cast<pkgAcqIndexMergeDiffs*>(this) != nullptr) + return 800; + + // Stage 3: The rest - complete index files and other stuff + return 500; +} + /*}}}*/ + +pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner, /*{{{*/ + pkgAcqMetaClearSig * const transactionManager, IndexTarget const &target) : + pkgAcquire::Item(Owner), d(NULL), Target(target), TransactionManager(transactionManager) +{ + if (TransactionManager != this) + TransactionManager->Add(this); + ModifyCustomFields() = { + {"Target-Site", Target.Option(IndexTarget::SITE)}, + {"Target-Repo-URI", Target.Option(IndexTarget::REPO_URI)}, + {"Target-Base-URI", Target.Option(IndexTarget::BASE_URI)}, + {"Target-Component", Target.Option(IndexTarget::COMPONENT)}, + {"Target-Release", Target.Option(IndexTarget::RELEASE)}, + {"Target-Architecture", Target.Option(IndexTarget::ARCHITECTURE)}, + {"Target-Language", Target.Option(IndexTarget::LANGUAGE)}, + {"Target-Type", "index"}, + }; +} + /*}}}*/ +pkgAcqTransactionItem::~pkgAcqTransactionItem() /*{{{*/ +{ +} + /*}}}*/ +HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const &MetaKey) const /*{{{*/ +{ + return GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, MetaKey); +} + /*}}}*/ + +static void LoadLastMetaIndexParser(pkgAcqMetaClearSig * const TransactionManager, std::string const &FinalRelease, std::string const &FinalInRelease)/*{{{*/ +{ + if (TransactionManager->IMSHit == true) + return; + if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease)) + { + TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone(); + if (TransactionManager->LastMetaIndexParser != NULL) + { + _error->PushToStack(); + if (RealFileExists(FinalInRelease)) + TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL); + else + TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL); + // its unlikely to happen, but if what we have is bad ignore it + if (_error->PendingError()) + { + delete TransactionManager->LastMetaIndexParser; + TransactionManager->LastMetaIndexParser = NULL; + } + _error->RevertToStack(); + } + } +} + /*}}}*/ + +// AcqMetaBase - Constructor /*{{{*/ +pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire * const Owner, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &DataTarget) +: pkgAcqTransactionItem(Owner, TransactionManager, DataTarget), d(NULL), + AuthPass(false), IMSHit(false), State(TransactionStarted) +{ +} + /*}}}*/ +// AcqMetaBase::Add - Add a item to the current Transaction /*{{{*/ +void pkgAcqMetaBase::Add(pkgAcqTransactionItem * const I) +{ + Transaction.push_back(I); +} + /*}}}*/ +// AcqMetaBase::AbortTransaction - Abort the current Transaction /*{{{*/ +void pkgAcqMetaBase::AbortTransaction() +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "AbortTransaction: " << TransactionManager << std::endl; + + switch (TransactionManager->State) + { + case TransactionStarted: break; + case TransactionAbort: _error->Fatal("Transaction %s was already aborted and is aborted again", TransactionManager->Target.URI.c_str()); return; + case TransactionCommit: _error->Fatal("Transaction %s was already aborted and is now committed", TransactionManager->Target.URI.c_str()); return; + } + TransactionManager->State = TransactionAbort; + TransactionManager->ExpectedAdditionalItems = 0; + + // ensure the toplevel is in error state too + for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin(); + I != Transaction.end(); ++I) + { + (*I)->ExpectedAdditionalItems = 0; + if ((*I)->Status != pkgAcquire::Item::StatFetching) + (*I)->Dequeue(); + (*I)->TransactionState(TransactionAbort); + } + Transaction.clear(); +} + /*}}}*/ +// AcqMetaBase::TransactionHasError - Check for errors in Transaction /*{{{*/ +APT_PURE bool pkgAcqMetaBase::TransactionHasError() const +{ + for (std::vector<pkgAcqTransactionItem*>::const_iterator I = Transaction.begin(); + I != Transaction.end(); ++I) + { + switch((*I)->Status) { + case StatDone: break; + case StatIdle: break; + case StatAuthError: return true; + case StatError: return true; + case StatTransientNetworkError: return true; + case StatFetching: break; + } + } + return false; +} + /*}}}*/ +// AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/ +void pkgAcqMetaBase::CommitTransaction() +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "CommitTransaction: " << this << std::endl; + + switch (TransactionManager->State) + { + case TransactionStarted: break; + case TransactionAbort: _error->Fatal("Transaction %s was already committed and is now aborted", TransactionManager->Target.URI.c_str()); return; + case TransactionCommit: _error->Fatal("Transaction %s was already committed and is again committed", TransactionManager->Target.URI.c_str()); return; + } + TransactionManager->State = TransactionCommit; + + // move new files into place *and* remove files that are not + // part of the transaction but are still on disk + for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin(); + I != Transaction.end(); ++I) + { + (*I)->TransactionState(TransactionCommit); + } + Transaction.clear(); +} + /*}}}*/ +// AcqMetaBase::TransactionStageCopy - Stage a file for copying /*{{{*/ +void pkgAcqMetaBase::TransactionStageCopy(pkgAcqTransactionItem * const I, + const std::string &From, + const std::string &To) +{ + I->PartialFile = From; + I->DestFile = To; +} + /*}}}*/ +// AcqMetaBase::TransactionStageRemoval - Stage a file for removal /*{{{*/ +void pkgAcqMetaBase::TransactionStageRemoval(pkgAcqTransactionItem * const I, + const std::string &FinalFile) +{ + I->PartialFile = ""; + I->DestFile = FinalFile; +} + /*}}}*/ +// AcqMetaBase::GenerateAuthWarning - Check gpg authentication error /*{{{*/ +/* This method is called from ::Failed handlers. If it returns true, + no fallback to other files or modi is performed */ +bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message) +{ + string const Final = I->GetFinalFilename(); + std::string const GPGError = LookupTag(Message, "Message"); + if (FileExists(Final)) + { + I->Status = StatTransientNetworkError; + _error->Warning(_("An error occurred during the signature verification. " + "The repository is not updated and the previous index files will be used. " + "GPG error: %s: %s"), + Desc.Description.c_str(), + GPGError.c_str()); + RunScripts("APT::Update::Auth-Failure"); + return true; + } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) { + /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */ + _error->Error(_("GPG error: %s: %s"), + Desc.Description.c_str(), + GPGError.c_str()); + I->Status = StatAuthError; + return true; + } else { + _error->Warning(_("GPG error: %s: %s"), + Desc.Description.c_str(), + GPGError.c_str()); + } + // gpgv method failed + ReportMirrorFailureToCentral(*this, "GPGFailure", GPGError); + return false; +} + /*}}}*/ +// AcqMetaBase::Custom600Headers - Get header for AcqMetaBase /*{{{*/ +// --------------------------------------------------------------------- +string pkgAcqMetaBase::Custom600Headers() const +{ + std::string Header = pkgAcqTransactionItem::Custom600Headers(); + Header.append("\nIndex-File: true"); + std::string MaximumSize; + strprintf(MaximumSize, "\nMaximum-Size: %i", + _config->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000)); + Header += MaximumSize; + + string const FinalFile = GetFinalFilename(); + struct stat Buf; + if (stat(FinalFile.c_str(),&Buf) == 0) + Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false); + + return Header; +} + /*}}}*/ +// AcqMetaBase::QueueForSignatureVerify /*{{{*/ +void pkgAcqMetaBase::QueueForSignatureVerify(pkgAcqTransactionItem * const I, std::string const &File, std::string const &Signature) +{ + AuthPass = true; + I->Desc.URI = "gpgv:" + pkgAcquire::URIEncode(Signature); + I->DestFile = File; + QueueURI(I->Desc); + I->SetActiveSubprocess("gpgv"); +} + /*}}}*/ +// AcqMetaBase::CheckDownloadDone /*{{{*/ +bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const std::string &Message, HashStringList const &Hashes) const +{ + // We have just finished downloading a Release file (it is not + // verified yet) + + // Save the final base URI we got this Release file from + if (I->UsedMirror.empty() == false && _config->FindB("Acquire::SameMirrorForAllIndexes", true)) + { + auto InReleasePath = Target.Option(IndexTarget::INRELEASE_PATH); + if (InReleasePath.empty()) + InReleasePath = "InRelease"; + + if (APT::String::Endswith(I->Desc.URI, InReleasePath)) + { + TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - InReleasePath.length()); + TransactionManager->UsedMirror = I->UsedMirror; + } + else if (APT::String::Endswith(I->Desc.URI, "Release")) + { + TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - strlen("Release")); + TransactionManager->UsedMirror = I->UsedMirror; + } + } + + std::string const FileName = LookupTag(Message,"Filename"); + if (FileName != I->DestFile && RealFileExists(I->DestFile) == false) + { + I->Local = true; + I->Desc.URI = "copy:" + pkgAcquire::URIEncode(FileName); + I->QueueURI(I->Desc); + return false; + } + + // make sure to verify against the right file on I-M-S hit + bool IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"), false); + if (IMSHit == false && Hashes.usable()) + { + // detect IMS-Hits servers haven't detected by Hash comparison + std::string const FinalFile = I->GetFinalFilename(); + if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true) + { + IMSHit = true; + RemoveFile("CheckDownloadDone", I->DestFile); + } + } + + if(IMSHit == true) + { + // for simplicity, the transaction manager is always InRelease + // even if it doesn't exist. + TransactionManager->IMSHit = true; + I->PartialFile = I->DestFile = I->GetFinalFilename(); + } + + // set Item to complete as the remaining work is all local (verify etc) + I->Complete = true; + + return true; +} + /*}}}*/ +bool pkgAcqMetaBase::CheckAuthDone(string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/ +{ + /* If we work with a recent version of our gpgv method, we expect that it tells us + which key(s) have signed the file so stuff like CVE-2018-0501 is harder in the future */ + if (Cnf->Version != "1.0" && LookupTag(Message, "Signed-By").empty()) + { + std::string errmsg; + strprintf(errmsg, "Internal Error: Signature on %s seems good, but expected details are missing! (%s)", Target.URI.c_str(), "Signed-By"); + if (ErrorText.empty()) + ErrorText = errmsg; + Status = StatAuthError; + return _error->Error("%s", errmsg.c_str()); + } + + // At this point, the gpgv method has succeeded, so there is a + // valid signature from a key in the trusted keyring. We + // perform additional verification of its contents, and use them + // to verify the indexes we are about to download + if (_config->FindB("Debug::pkgAcquire::Auth", false)) + std::cerr << "Signature verification succeeded: " << DestFile << std::endl; + + if (TransactionManager->IMSHit == false) + { + // open the last (In)Release if we have it + std::string const FinalFile = GetFinalFilename(); + std::string FinalRelease; + std::string FinalInRelease; + if (APT::String::Endswith(FinalFile, "InRelease")) + { + FinalInRelease = FinalFile; + FinalRelease = FinalFile.substr(0, FinalFile.length() - strlen("InRelease")) + "Release"; + } + else + { + FinalInRelease = FinalFile.substr(0, FinalFile.length() - strlen("Release")) + "InRelease"; + FinalRelease = FinalFile; + } + LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease); + } + + bool const GoodAuth = TransactionManager->MetaIndexParser->Load(DestFile, &ErrorText); + if (GoodAuth == false && AllowInsecureRepositories(InsecureType::WEAK, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == false) + { + Status = StatAuthError; + return false; + } + + if (!VerifyVendor(Message)) + { + Status = StatAuthError; + return false; + } + + // Download further indexes with verification + TransactionManager->QueueIndexes(GoodAuth); + + return GoodAuth; +} + /*}}}*/ +void pkgAcqMetaClearSig::QueueIndexes(bool const verify) /*{{{*/ +{ + // at this point the real Items are loaded in the fetcher + ExpectedAdditionalItems = 0; + + std::unordered_set<std::string> targetsSeen, componentsSeen; + bool const hasReleaseFile = TransactionManager->MetaIndexParser != NULL; + bool hasHashes = true; + auto IndexTargets = TransactionManager->MetaIndexParser->GetIndexTargets(); + if (hasReleaseFile && verify == false) + hasHashes = std::any_of(IndexTargets.begin(), IndexTargets.end(), + [&](IndexTarget const &Target) { return TransactionManager->MetaIndexParser->Exists(Target.MetaKey); }); + if (_config->FindB("Acquire::IndexTargets::Randomized", true) && likely(IndexTargets.empty() == false)) + { + /* For fallback handling and to have some reasonable progress information + we can't randomize everything, but at least the order in the same type + can be as we shouldn't be telling the mirrors (and everyone else watching) + which is native/foreign arch, specific order of preference of translations, … */ + auto range_start = IndexTargets.begin(); + auto seed = (std::chrono::high_resolution_clock::now().time_since_epoch() / std::chrono::nanoseconds(1)) ^ getpid(); + std::default_random_engine g(seed); + do { + auto const type = range_start->Option(IndexTarget::CREATED_BY); + auto const range_end = std::find_if_not(range_start, IndexTargets.end(), + [&type](IndexTarget const &T) { return type == T.Option(IndexTarget::CREATED_BY); }); + std::shuffle(range_start, range_end, g); + range_start = range_end; + } while (range_start != IndexTargets.end()); + } + /* Collect all components for which files exist to prevent apt from warning users + about "hidden" components for which not all files exist like main/debian-installer + and Translation files */ + if (hasReleaseFile == true) + for (auto const &Target : IndexTargets) + if (TransactionManager->MetaIndexParser->Exists(Target.MetaKey)) + { + auto component = Target.Option(IndexTarget::COMPONENT); + if (component.empty() == false) + componentsSeen.emplace(std::move(component)); + } + + for (auto&& Target: IndexTargets) + { + // if we have seen a target which is created-by a target this one here is declared a + // fallback to, we skip acquiring the fallback (but we make sure we clean up) + if (targetsSeen.find(Target.Option(IndexTarget::FALLBACK_OF)) != targetsSeen.end()) + { + targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY)); + new CleanupItem(Owner, TransactionManager, Target); + continue; + } + // all is an implementation detail. Users shouldn't use this as arch + // We need this support trickery here as e.g. Debian has binary-all files already, + // but arch:all packages are still in the arch:any files, so we would waste precious + // download time, bandwidth and diskspace for nothing, BUT Debian doesn't feature all + // in the set of supported architectures, so we can filter based on this property rather + // than invent an entirely new flag we would need to carry for all of eternity. + if (hasReleaseFile && Target.Option(IndexTarget::ARCHITECTURE) == "all") + { + if (TransactionManager->MetaIndexParser->IsArchitectureAllSupportedFor(Target) == false) + { + new CleanupItem(Owner, TransactionManager, Target); + continue; + } + } + + bool trypdiff = Target.OptionBool(IndexTarget::PDIFFS); + if (hasReleaseFile == true) + { + if (TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false) + { + auto const component = Target.Option(IndexTarget::COMPONENT); + if (component.empty() == false && + componentsSeen.find(component) == std::end(componentsSeen) && + TransactionManager->MetaIndexParser->HasSupportForComponent(component) == false) + { + new CleanupItem(Owner, TransactionManager, Target); + _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' doesn't have the component '%s' (component misspelt in sources.list?)"), + Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str(), component.c_str()); + continue; + + } + + // optional targets that we do not have in the Release file are skipped + if (hasHashes == true && Target.IsOptional) + { + new CleanupItem(Owner, TransactionManager, Target); + continue; + } + + std::string const &arch = Target.Option(IndexTarget::ARCHITECTURE); + if (arch.empty() == false) + { + if (TransactionManager->MetaIndexParser->IsArchitectureSupported(arch) == false) + { + new CleanupItem(Owner, TransactionManager, Target); + _error->Notice(_("Skipping acquire of configured file '%s' as repository '%s' doesn't support architecture '%s'"), + Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str(), arch.c_str()); + continue; + } + // if the architecture is officially supported but currently no packages for it available, + // ignore silently as this is pretty much the same as just shipping an empty file. + // if we don't know which architectures are supported, we do NOT ignore it to notify user about this + if (hasHashes == true && TransactionManager->MetaIndexParser->IsArchitectureSupported("*undefined*") == false) + { + new CleanupItem(Owner, TransactionManager, Target); + continue; + } + } + + if (hasHashes == true) + { + new CleanupItem(Owner, TransactionManager, Target); + _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' does not seem to provide it (sources.list entry misspelt?)"), + Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str()); + continue; + } + else + { + new pkgAcqIndex(Owner, TransactionManager, Target); + continue; + } + } + else if (verify) + { + auto const hashes = GetExpectedHashesFor(Target.MetaKey); + if (hashes.empty() == false) + { + if (hashes.usable() == false && TargetIsAllowedToBe(TransactionManager->Target, InsecureType::WEAK) == false) + { + new CleanupItem(Owner, TransactionManager, Target); + _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"), + Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str()); + continue; + } + // empty files are skipped as acquiring the very small compressed files is a waste of time + else if (hashes.FileSize() == 0) + { + new CleanupItem(Owner, TransactionManager, Target); + targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY)); + continue; + } + } + } + + // autoselect the compression method + std::vector<std::string> types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' '); + types.erase(std::remove_if(types.begin(), types.end(), [&](std::string const &t) { + if (t == "uncompressed") + return TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false; + std::string const MetaKey = Target.MetaKey + "." + t; + return TransactionManager->MetaIndexParser->Exists(MetaKey) == false; + }), types.end()); + if (types.empty() == false) + { + std::ostringstream os; + std::copy(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " ")); + os << *types.rbegin(); + Target.Options["COMPRESSIONTYPES"] = os.str(); + } + else + Target.Options["COMPRESSIONTYPES"].clear(); + + std::string filename = GetExistingFilename(GetFinalFileNameFromURI(Target.URI)); + if (filename.empty() == false) + { + // if the Release file is a hit and we have an index it must be the current one + if (TransactionManager->IMSHit == true) + ; + else if (TransactionManager->LastMetaIndexParser != NULL) + { + // see if the file changed since the last Release file + // we use the uncompressed files as we might compress differently compared to the server, + // so the hashes might not match, even if they contain the same data. + HashStringList const newFile = GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, Target.MetaKey); + HashStringList const oldFile = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey); + if (newFile != oldFile) + filename.clear(); + } + else + filename.clear(); + } + else + trypdiff = false; // no file to patch + + if (filename.empty() == false) + { + new NoActionItem(Owner, Target, filename); + std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(Target)); + if (FileExists(idxfilename)) + new NoActionItem(Owner, Target, idxfilename); + targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY)); + continue; + } + + // check if we have patches available + trypdiff &= TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey)); + } + else + { + // if we have no file to patch, no point in trying + trypdiff &= (GetExistingFilename(GetFinalFileNameFromURI(Target.URI)).empty() == false); + } + + // no point in patching from local sources + if (trypdiff) + { + std::string const proto = Target.URI.substr(0, strlen("file:/")); + if (proto == "file:/" || proto == "copy:/" || proto == "cdrom:") + trypdiff = false; + } + + // Queue the Index file (Packages, Sources, Translation-$foo, …) + targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY)); + if (trypdiff) + new pkgAcqDiffIndex(Owner, TransactionManager, Target); + else + new pkgAcqIndex(Owner, TransactionManager, Target); + } +} + /*}}}*/ +bool pkgAcqMetaBase::VerifyVendor(string const &) /*{{{*/ +{ + if (TransactionManager->MetaIndexParser->GetValidUntil() > 0) + { + time_t const invalid_since = time(NULL) - TransactionManager->MetaIndexParser->GetValidUntil(); + if (invalid_since > 0) + { + std::string errmsg; + strprintf(errmsg, + // TRANSLATOR: The first %s is the URL of the bad Release file, the second is + // the time since then the file is invalid - formatted in the same way as in + // the download progress display (e.g. 7d 3h 42min 1s) + _("Release file for %s is expired (invalid since %s). " + "Updates for this repository will not be applied."), + Target.URI.c_str(), TimeToStr(invalid_since).c_str()); + if (ErrorText.empty()) + ErrorText = errmsg; + return _error->Error("%s", errmsg.c_str()); + } + } + + if (TransactionManager->MetaIndexParser->GetNotBefore() > 0) + { + time_t const invalid_for = TransactionManager->MetaIndexParser->GetNotBefore() - time(nullptr); + if (invalid_for > 0) + { + std::string errmsg; + strprintf(errmsg, + // TRANSLATOR: The first %s is the URL of the bad Release file, the second is + // the time until the file will be valid - formatted in the same way as in + // the download progress display (e.g. 7d 3h 42min 1s) + _("Release file for %s is not valid yet (invalid for another %s). " + "Updates for this repository will not be applied."), + Target.URI.c_str(), TimeToStr(invalid_for).c_str()); + if (ErrorText.empty()) + ErrorText = errmsg; + return _error->Error("%s", errmsg.c_str()); + } + } + + /* Did we get a file older than what we have? This is a last minute IMS hit and doubles + as a prevention of downgrading us to older (still valid) files */ + if (TransactionManager->IMSHit == false && TransactionManager->LastMetaIndexParser != NULL && + TransactionManager->LastMetaIndexParser->GetDate() > TransactionManager->MetaIndexParser->GetDate()) + { + TransactionManager->IMSHit = true; + RemoveFile("VerifyVendor", DestFile); + PartialFile = DestFile = GetFinalFilename(); + // load the 'old' file in the 'new' one instead of flipping pointers as + // the new one isn't owned by us, while the old one is so cleanup would be confused. + TransactionManager->MetaIndexParser->swapLoad(TransactionManager->LastMetaIndexParser); + delete TransactionManager->LastMetaIndexParser; + TransactionManager->LastMetaIndexParser = NULL; + } + + if (_config->FindB("Debug::pkgAcquire::Auth", false)) + { + std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetCodename() << std::endl; + std::cerr << "Got Suite: " << TransactionManager->MetaIndexParser->GetSuite() << std::endl; + std::cerr << "Expecting Dist: " << TransactionManager->MetaIndexParser->GetExpectedDist() << std::endl; + } + + // One day that might become fatal… + auto const ExpectedDist = TransactionManager->MetaIndexParser->GetExpectedDist(); + auto const NowCodename = TransactionManager->MetaIndexParser->GetCodename(); + if (TransactionManager->MetaIndexParser->CheckDist(ExpectedDist) == false) + _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"), + Desc.Description.c_str(), ExpectedDist.c_str(), NowCodename.c_str()); + + // changed info potentially breaks user config like pinning + if (TransactionManager->LastMetaIndexParser != nullptr) + { + std::vector<pkgAcquireStatus::ReleaseInfoChange> Changes; + auto const AllowInfoChange = _config->FindB("Acquire::AllowReleaseInfoChange", false); + auto const quietInfoChange = _config->FindB("quiet::ReleaseInfoChange", false); + struct { + char const * const Type; + bool const Allowed; + decltype(&metaIndex::GetOrigin) const Getter; + } checkers[] = { + { "Origin", AllowInfoChange, &metaIndex::GetOrigin }, + { "Label", AllowInfoChange, &metaIndex::GetLabel }, + { "Version", true, &metaIndex::GetVersion }, // numbers change all the time, that is okay + { "Suite", true, &metaIndex::GetSuite }, + { "Codename", AllowInfoChange, &metaIndex::GetCodename }, + { nullptr, false, nullptr } + }; + auto const CheckReleaseInfo = [&](char const * const Type, bool const AllowChange, decltype(checkers[0].Getter) const Getter) { + std::string const Last = (TransactionManager->LastMetaIndexParser->*Getter)(); + std::string const Now = (TransactionManager->MetaIndexParser->*Getter)(); + if (Last == Now) + return; + auto const Allow = _config->FindB(std::string("Acquire::AllowReleaseInfoChange::").append(Type), AllowChange); + if (Allow == true && _config->FindB(std::string("quiet::ReleaseInfoChange::").append(Type), quietInfoChange) == true) + return; + std::string msg; + strprintf(msg, _("Repository '%s' changed its '%s' value from '%s' to '%s'"), + Desc.Description.c_str(), Type, Last.c_str(), Now.c_str()); + Changes.push_back({Type, std::move(Last), std::move(Now), std::move(msg), Allow}); + }; + for (short i = 0; checkers[i].Type != nullptr; ++i) + CheckReleaseInfo(checkers[i].Type, checkers[i].Allowed, checkers[i].Getter); + + { + auto const Last = TransactionManager->LastMetaIndexParser->GetDefaultPin(); + auto const Now = TransactionManager->MetaIndexParser->GetDefaultPin(); + if (Last != Now) + { + auto const Allow = _config->FindB("Acquire::AllowReleaseInfoChange::DefaultPin", AllowInfoChange); + if (Allow == false || _config->FindB("quiet::ReleaseInfoChange::DefaultPin", quietInfoChange) == false) + { + std::string msg; + strprintf(msg, _("Repository '%s' changed its default priority for %s from %hi to %hi."), + Desc.Description.c_str(), "apt_preferences(5)", Last, Now); + Changes.push_back({"DefaultPin", std::to_string(Last), std::to_string(Now), std::move(msg), Allow}); + } + } + } + if (Changes.empty() == false) + { + auto const notes = TransactionManager->MetaIndexParser->GetReleaseNotes(); + if (notes.empty() == false) + { + std::string msg; + // TRANSLATOR: the "this" refers to changes in the repository like a new release or owner change + strprintf(msg, _("More information about this can be found online in the Release notes at: %s"), notes.c_str()); + Changes.push_back({"Release-Notes", "", std::move(notes), std::move(msg), true}); + } + if (std::any_of(Changes.begin(),Changes.end(),[](pkgAcquireStatus::ReleaseInfoChange const &c) { return c.DefaultAction == false; })) + { + std::string msg; + // TRANSLATOR: %s is the name of the manpage in question, e.g. apt-secure(8) + strprintf(msg, _("This must be accepted explicitly before updates for " + "this repository can be applied. See %s manpage for details."), "apt-secure(8)"); + Changes.push_back({"Confirmation", "", "", std::move(msg), true}); + } + + } + if (Owner->Log == nullptr) + return pkgAcquireStatus::ReleaseInfoChangesAsGlobalErrors(std::move(Changes)); + return Owner->Log->ReleaseInfoChanges(TransactionManager->LastMetaIndexParser, TransactionManager->MetaIndexParser, std::move(Changes)); + } + return true; +} + /*}}}*/ +pkgAcqMetaBase::~pkgAcqMetaBase() +{ +} + +pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner, /*{{{*/ + IndexTarget const &ClearsignedTarget, + IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget, + metaIndex * const MetaIndexParser) : + pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget), + d(NULL), DetachedDataTarget(DetachedDataTarget), + MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL) +{ + // index targets + (worst case:) Release/Release.gpg + ExpectedAdditionalItems = std::numeric_limits<decltype(ExpectedAdditionalItems)>::max(); + TransactionManager->Add(this); +} + /*}}}*/ +pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/ +{ + if (LastMetaIndexParser != NULL) + delete LastMetaIndexParser; +} + /*}}}*/ +// pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/ +string pkgAcqMetaClearSig::Custom600Headers() const +{ + string Header = pkgAcqMetaBase::Custom600Headers(); + Header += "\nFail-Ignore: true"; + std::string const key = TransactionManager->MetaIndexParser->GetSignedBy(); + if (key.empty() == false) + Header += "\nSigned-By: " + QuoteString(key, ""); + + return Header; +} + /*}}}*/ +void pkgAcqMetaClearSig::Finished() /*{{{*/ +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "Finished: " << DestFile <<std::endl; + if(TransactionManager->State == TransactionStarted && + TransactionManager->TransactionHasError() == false) + TransactionManager->CommitTransaction(); +} + /*}}}*/ +bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message, /*{{{*/ + pkgAcquire::MethodConfig const * const Cnf) +{ + if (Item::VerifyDone(Message, Cnf) == false) + return false; + + if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile)) + return RenameOnError(NotClearsigned); + + return true; +} + /*}}}*/ +// pkgAcqMetaClearSig::Done - We got a file /*{{{*/ +void pkgAcqMetaClearSig::Done(std::string const &Message, + HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) +{ + Item::Done(Message, Hashes, Cnf); + + if(AuthPass == false) + { + if(CheckDownloadDone(this, Message, Hashes) == true) + QueueForSignatureVerify(this, DestFile, DestFile); + return; + } + else if (CheckAuthDone(Message, Cnf) == true) + { + if (TransactionManager->IMSHit == false) + TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); + else if (RealFileExists(GetFinalFilename()) == false) + { + // We got an InRelease file IMSHit, but we haven't one, which means + // we had a valid Release/Release.gpg combo stepping in, which we have + // to 'acquire' now to ensure list cleanup isn't removing them + new NoActionItem(Owner, DetachedDataTarget); + new NoActionItem(Owner, DetachedSigTarget); + } + } + else if (Status != StatAuthError) + { + string const FinalFile = GetFinalFileNameFromURI(DetachedDataTarget.URI); + string const OldFile = GetFinalFilename(); + if (TransactionManager->IMSHit == false) + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); + else if (RealFileExists(OldFile) == false) + new NoActionItem(Owner, DetachedDataTarget); + else + TransactionManager->TransactionStageCopy(this, OldFile, FinalFile); + } +} + /*}}}*/ +void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) /*{{{*/ +{ + Item::Failed(Message, Cnf); + + if (AuthPass == false) + { + if (Status == StatTransientNetworkError) + { + TransactionManager->AbortTransaction(); + return; + } + auto const failreason = LookupTag(Message, "FailReason"); + auto const httperror = "HttpError"; + if (Status == StatAuthError || + Target.Option(IndexTarget::INRELEASE_PATH).empty() == false || /* do not fallback if InRelease was requested */ + (strncmp(failreason.c_str(), httperror, strlen(httperror)) == 0 && + failreason != "HttpError404")) + { + // if we expected a ClearTextSignature (InRelease) but got a network + // error or got a file, but it wasn't valid, we end up here (see VerifyDone). + // As these is usually called by web-portals we do not try Release/Release.gpg + // as this is going to fail anyway and instead abort our try (LP#346386) + _error->PushToStack(); + _error->Error(_("Failed to fetch %s %s"), Target.URI.c_str(), ErrorText.c_str()); + if (Target.Option(IndexTarget::INRELEASE_PATH).empty() == true && AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true) + _error->RevertToStack(); + else + return; + } + + // Queue the 'old' InRelease file for removal if we try Release.gpg + // as otherwise the file will stay around and gives a false-auth + // impression (CVE-2012-0214) + TransactionManager->TransactionStageRemoval(this, GetFinalFilename()); + Status = StatDone; + + new pkgAcqMetaIndex(Owner, TransactionManager, DetachedDataTarget, DetachedSigTarget); + } + else + { + if(CheckStopAuthentication(this, Message)) + return; + + if(AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true) + { + Status = StatDone; + + /* InRelease files become Release files, otherwise + * they would be considered as trusted later on */ + string const FinalRelease = GetFinalFileNameFromURI(DetachedDataTarget.URI); + string const PartialRelease = GetPartialFileNameFromURI(DetachedDataTarget.URI); + string const FinalInRelease = GetFinalFilename(); + Rename(DestFile, PartialRelease); + LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease); + + // we parse the indexes here because at this point the user wanted + // a repository that may potentially harm him + if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false) + /* expired Release files are still a problem you need extra force for */; + else + { + TransactionManager->TransactionStageCopy(this, PartialRelease, FinalRelease); + TransactionManager->QueueIndexes(true); + } + } + } +} + /*}}}*/ + +pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire * const Owner, /*{{{*/ + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &DataTarget, + IndexTarget const &DetachedSigTarget) : + pkgAcqMetaBase(Owner, TransactionManager, DataTarget), d(NULL), + DetachedSigTarget(DetachedSigTarget) +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgAcqMetaIndex with TransactionManager " + << this->TransactionManager << std::endl; + + DestFile = GetPartialFileNameFromURI(DataTarget.URI); + + // Create the item + Desc.Description = DataTarget.Description; + Desc.Owner = this; + Desc.ShortDesc = DataTarget.ShortDesc; + + // Rewrite the description URI if INRELEASE_PATH was specified so + // we download the specified file instead. + auto InReleasePath = DataTarget.Option(IndexTarget::INRELEASE_PATH); + if (InReleasePath.empty() == false && APT::String::Endswith(DataTarget.URI, "/InRelease")) + { + Desc.URI = DataTarget.URI.substr(0, DataTarget.URI.size() - strlen("InRelease")) + InReleasePath; + } + else + { + Desc.URI = DataTarget.URI; + } + + QueueURI(Desc); +} + /*}}}*/ +void pkgAcqMetaIndex::Done(string const &Message, /*{{{*/ + HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cfg) +{ + Item::Done(Message,Hashes,Cfg); + + if(CheckDownloadDone(this, Message, Hashes)) + { + // we have a Release file, now download the Signature, all further + // verify/queue for additional downloads will be done in the + // pkgAcqMetaSig::Done() code + new pkgAcqMetaSig(Owner, TransactionManager, DetachedSigTarget, this); + } +} + /*}}}*/ +// pkgAcqMetaIndex::Failed - no Release file present /*{{{*/ +void pkgAcqMetaIndex::Failed(string const &Message, + pkgAcquire::MethodConfig const * const Cnf) +{ + pkgAcquire::Item::Failed(Message, Cnf); + Status = StatDone; + + // No Release file was present so fall + // back to queueing Packages files without verification + // only allow going further if the user explicitly wants it + if(AllowInsecureRepositories(InsecureType::NORELEASE, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true) + { + // ensure old Release files are removed + TransactionManager->TransactionStageRemoval(this, GetFinalFilename()); + + // queue without any kind of hashsum support + TransactionManager->QueueIndexes(false); + } +} + /*}}}*/ +std::string pkgAcqMetaIndex::DescURI() const /*{{{*/ +{ + return Target.URI; +} + /*}}}*/ +pkgAcqMetaIndex::~pkgAcqMetaIndex() {} + +// AcqMetaSig::AcqMetaSig - Constructor /*{{{*/ +pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, + pkgAcqMetaIndex * const MetaIndex) : + pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL), MetaIndex(MetaIndex) +{ + DestFile = GetPartialFileNameFromURI(Target.URI); + + // remove any partial downloaded sig-file in partial/. + // it may confuse proxies and is too small to warrant a + // partial download anyway + RemoveFile("pkgAcqMetaSig", DestFile); + + // set the TransactionManager + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgAcqMetaSig with TransactionManager " + << TransactionManager << std::endl; + + // Create the item + Desc.Description = Target.Description; + Desc.Owner = this; + Desc.ShortDesc = Target.ShortDesc; + Desc.URI = Target.URI; + + // If we got a hit for Release, we will get one for Release.gpg too (or obscure errors), + // so we skip the download step and go instantly to verification + if (TransactionManager->IMSHit == true && RealFileExists(GetFinalFilename())) + { + Complete = true; + Status = StatDone; + PartialFile = DestFile = GetFinalFilename(); + MetaIndexFileSignature = DestFile; + MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile); + } + else + QueueURI(Desc); +} + /*}}}*/ +pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/ +{ +} + /*}}}*/ +// pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/ +std::string pkgAcqMetaSig::Custom600Headers() const +{ + std::string Header = pkgAcqTransactionItem::Custom600Headers(); + std::string const key = TransactionManager->MetaIndexParser->GetSignedBy(); + if (key.empty() == false) + Header += "\nSigned-By: " + QuoteString(key, ""); + return Header; +} + /*}}}*/ +// AcqMetaSig::Done - The signature was downloaded/verified /*{{{*/ +void pkgAcqMetaSig::Done(string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cfg) +{ + if (MetaIndexFileSignature.empty() == false) + { + DestFile = MetaIndexFileSignature; + MetaIndexFileSignature.clear(); + } + Item::Done(Message, Hashes, Cfg); + + if(MetaIndex->AuthPass == false) + { + if(MetaIndex->CheckDownloadDone(this, Message, Hashes) == true) + { + // destfile will be modified to point to MetaIndexFile for the + // gpgv method, so we need to save it here + MetaIndexFileSignature = DestFile; + MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile); + } + return; + } + else if (MetaIndex->CheckAuthDone(Message, Cfg) == true) + { + auto const Releasegpg = GetFinalFilename(); + auto const Release = MetaIndex->GetFinalFilename(); + // if this is an IMS-Hit on Release ensure we also have the Release.gpg file stored + // (previously an unknown pubkey) – but only if the Release file exists locally (unlikely + // event of InRelease removed from the mirror causing fallback but still an IMS-Hit) + if (TransactionManager->IMSHit == false || + (FileExists(Releasegpg) == false && FileExists(Release) == true)) + { + TransactionManager->TransactionStageCopy(this, DestFile, Releasegpg); + TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, Release); + } + } + else if (MetaIndex->Status != StatAuthError) + { + std::string const FinalFile = MetaIndex->GetFinalFilename(); + if (TransactionManager->IMSHit == false) + TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalFile); + else + TransactionManager->TransactionStageCopy(MetaIndex, FinalFile, FinalFile); + } +} + /*}}}*/ +void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/ +{ + Item::Failed(Message,Cnf); + + // check if we need to fail at this point + if (MetaIndex->AuthPass == true && MetaIndex->CheckStopAuthentication(this, Message)) + return; + + // ensures that a Release.gpg file in the lists/ is removed by the transaction + if (not MetaIndexFileSignature.empty()) + { + DestFile = MetaIndexFileSignature; + MetaIndexFileSignature.clear(); + } + TransactionManager->TransactionStageRemoval(this, DestFile); + + // only allow going further if the user explicitly wants it + if (AllowInsecureRepositories(InsecureType::UNSIGNED, MetaIndex->Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true) + { + string const FinalRelease = MetaIndex->GetFinalFilename(); + string const FinalInRelease = TransactionManager->GetFinalFilename(); + LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease); + + // we parse the indexes here because at this point the user wanted + // a repository that may potentially harm him + bool const GoodLoad = TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile, &ErrorText); + if (MetaIndex->VerifyVendor(Message) == false) + /* expired Release files are still a problem you need extra force for */; + else + { + TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalRelease); + TransactionManager->QueueIndexes(GoodLoad); + } + } + else if (TransactionManager->IMSHit == false) + Rename(MetaIndex->DestFile, MetaIndex->DestFile + ".FAILED"); + + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor + if (Cnf->LocalOnly == true || + StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) + { + // Ignore this + Status = StatDone; + } +} + /*}}}*/ + + +// AcqBaseIndex - Constructor /*{{{*/ +pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire * const Owner, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target) +: pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL) +{ +} + /*}}}*/ +void pkgAcqBaseIndex::Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/ +{ + pkgAcquire::Item::Failed(Message, Cnf); + if (Status != StatAuthError) + return; + + ErrorText.append("Release file created at: "); + auto const timespec = TransactionManager->MetaIndexParser->GetDate(); + if (timespec == 0) + ErrorText.append("<unknown>"); + else + ErrorText.append(TimeRFC1123(timespec, true)); + ErrorText.append("\n"); +} + /*}}}*/ +pkgAcqBaseIndex::~pkgAcqBaseIndex() {} + +// AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Get the DiffIndex file first and see if there are patches available + * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the + * patches. If anything goes wrong in that process, it will fall back to + * the original packages file + */ +pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target) + : pkgAcqIndex(Owner, TransactionManager, Target, true), d(NULL), diffs(NULL) +{ + // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire + ExpectedAdditionalItems = 40; + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + CompressionExtensions.clear(); + { + std::vector<std::string> types = APT::Configuration::getCompressionTypes(); + if (types.empty() == false) + { + std::ostringstream os; + std::copy_if(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "), [&](std::string const type) { + if (type == "uncompressed") + return true; + return TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey) + '.' + type); + }); + os << *types.rbegin(); + CompressionExtensions = os.str(); + } + } + Init(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Target.ShortDesc); + + if(Debug) + std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl; +} + /*}}}*/ +void pkgAcqDiffIndex::QueueOnIMSHit() const /*{{{*/ +{ + // list cleanup needs to know that this file as well as the already + // present index is ours, so we create an empty diff to save it for us + new pkgAcqIndexDiffs(Owner, TransactionManager, Target); +} + /*}}}*/ +static bool RemoveFileForBootstrapLinking(std::string &ErrorText, std::string const &For, std::string const &Boot)/*{{{*/ +{ + if (FileExists(Boot) && RemoveFile("Bootstrap-linking", Boot) == false) + { + strprintf(ErrorText, "Bootstrap for patching %s by removing stale %s failed!", For.c_str(), Boot.c_str()); + return false; + } + return true; +} + /*}}}*/ +bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ +{ + available_patches.clear(); + ExpectedAdditionalItems = 0; + // failing here is fine: our caller will take care of trying to + // get the complete file if patching fails + if(Debug) + std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile + << std::endl; + + FileFd Fd(IndexDiffFile, FileFd::ReadOnly, FileFd::Extension); + pkgTagFile TF(&Fd); + if (Fd.IsOpen() == false || Fd.Failed()) + return false; + + pkgTagSection Tags; + if(unlikely(TF.Step(Tags) == false)) + return false; + + HashStringList ServerHashes; + unsigned long long ServerSize = 0; + + auto const &posix = std::locale::classic(); + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + { + std::string tagname = *type; + tagname.append("-Current"); + auto const tmp = Tags.Find(tagname); + if (tmp.empty() == true) + continue; + + string hash; + unsigned long long size; + std::stringstream ss(tmp.to_string()); + ss.imbue(posix); + ss >> hash >> size; + if (unlikely(hash.empty() == true)) + continue; + if (unlikely(ServerSize != 0 && ServerSize != size)) + continue; + ServerHashes.push_back(HashString(*type, hash)); + ServerSize = size; + } + + if (ServerHashes.usable() == false) + { + ErrorText = "Did not find a good hashsum in the index"; + return false; + } + + std::string const CurrentPackagesFile = GetFinalFileNameFromURI(Target.URI); + HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey); + if (TargetFileHashes.usable() == false || ServerHashes != TargetFileHashes) + { + ErrorText = "Index has different hashes than parser (probably older)"; + return false; + } + + HashStringList LocalHashes; + // try avoiding calculating the hash here as this is costly + if (TransactionManager->LastMetaIndexParser != NULL) + LocalHashes = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey); + if (LocalHashes.usable() == false) + { + FileFd fd(CurrentPackagesFile, FileFd::ReadOnly, FileFd::Auto); + Hashes LocalHashesCalc(ServerHashes); + LocalHashesCalc.AddFD(fd); + LocalHashes = LocalHashesCalc.GetHashStringList(); + } + + if (ServerHashes == LocalHashes) + { + available_patches.clear(); + return true; + } + + if(Debug) + std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at " + << CurrentPackagesFile << " " << LocalHashes.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl; + + // historically, older hashes have more info than newer ones, so start + // collecting with older ones first to avoid implementing complicated + // information merging techniques… a failure is after all always + // recoverable with a complete file and hashes aren't changed that often. + std::vector<char const *> types; + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + types.push_back(*type); + + // parse all of (provided) history + bool firstAcceptedHashes = true; + for (auto type = types.crbegin(); type != types.crend(); ++type) + { + if (LocalHashes.find(*type) == NULL) + continue; + + std::string tagname = *type; + tagname.append("-History"); + auto const tmp = Tags.Find(tagname); + if (tmp.empty() == true) + continue; + + string hash, filename; + unsigned long long size; + std::stringstream ss(tmp.to_string()); + ss.imbue(posix); + + while (ss >> hash >> size >> filename) + { + if (unlikely(hash.empty() == true || filename.empty() == true)) + continue; + + // see if we have a record for this file already + std::vector<DiffInfo>::iterator cur = available_patches.begin(); + for (; cur != available_patches.end(); ++cur) + { + if (cur->file != filename) + continue; + cur->result_hashes.push_back(HashString(*type, hash)); + break; + } + if (cur != available_patches.end()) + continue; + if (firstAcceptedHashes == true) + { + DiffInfo next; + next.file = filename; + next.result_hashes.push_back(HashString(*type, hash)); + next.result_hashes.FileSize(size); + available_patches.push_back(next); + } + else + { + if (Debug == true) + std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename + << " wasn't in the list for the first parsed hash! (history)" << std::endl; + break; + } + } + firstAcceptedHashes = false; + } + + if (unlikely(available_patches.empty() == true)) + { + ErrorText = "Couldn't find any patches for the patch series"; + return false; + } + + for (auto type = types.crbegin(); type != types.crend(); ++type) + { + if (LocalHashes.find(*type) == NULL) + continue; + + std::string tagname = *type; + tagname.append("-Patches"); + auto const tmp = Tags.Find(tagname); + if (tmp.empty() == true) + continue; + + string hash, filename; + unsigned long long size; + std::stringstream ss(tmp.to_string()); + ss.imbue(posix); + + while (ss >> hash >> size >> filename) + { + if (unlikely(hash.empty() == true || filename.empty() == true)) + continue; + + // see if we have a record for this file already + std::vector<DiffInfo>::iterator cur = available_patches.begin(); + for (; cur != available_patches.end(); ++cur) + { + if (cur->file != filename) + continue; + if (cur->patch_hashes.empty()) + cur->patch_hashes.FileSize(size); + cur->patch_hashes.push_back(HashString(*type, hash)); + break; + } + if (cur != available_patches.end()) + continue; + if (Debug == true) + std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename + << " wasn't in the list for the first parsed hash! (patches)" << std::endl; + break; + } + } + + for (auto type = types.crbegin(); type != types.crend(); ++type) + { + std::string tagname = *type; + tagname.append("-Download"); + auto const tmp = Tags.Find(tagname); + if (tmp.empty() == true) + continue; + + string hash, filename; + unsigned long long size; + std::stringstream ss(tmp.to_string()); + ss.imbue(posix); + + // FIXME: all of pdiff supports only .gz compressed patches + while (ss >> hash >> size >> filename) + { + if (unlikely(hash.empty() == true || filename.empty() == true)) + continue; + if (unlikely(APT::String::Endswith(filename, ".gz") == false)) + continue; + filename.erase(filename.length() - 3); + + // see if we have a record for this file already + std::vector<DiffInfo>::iterator cur = available_patches.begin(); + for (; cur != available_patches.end(); ++cur) + { + if (cur->file != filename) + continue; + if (cur->download_hashes.empty()) + cur->download_hashes.FileSize(size); + cur->download_hashes.push_back(HashString(*type, hash)); + break; + } + if (cur != available_patches.end()) + continue; + if (Debug == true) + std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename + << " wasn't in the list for the first parsed hash! (download)" << std::endl; + break; + } + } + + { + auto const foundStart = std::find_if(available_patches.rbegin(), available_patches.rend(), + [&](auto const &cur) { return LocalHashes == cur.result_hashes; }); + if (foundStart == available_patches.rend() || unlikely(available_patches.empty())) + { + ErrorText = "Couldn't find the start of the patch series"; + return false; + } + available_patches.erase(available_patches.begin(), std::prev(foundStart.base())); + + auto const patch = std::find_if(available_patches.cbegin(), available_patches.cend(), [](auto const &patch) { + return not patch.result_hashes.usable() || + not patch.patch_hashes.usable() || + not patch.download_hashes.usable(); + }); + if (patch != available_patches.cend()) + { + strprintf(ErrorText, "Provides no usable hashes for %s", patch->file.c_str()); + return false; + } + } + + // patching with too many files is rather slow compared to a fast download + unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0); + if (fileLimit != 0 && fileLimit < available_patches.size()) + { + strprintf(ErrorText, "Need %lu diffs, but limit is %lu", available_patches.size(), fileLimit); + return false; + } + + /* decide if we should download patches one by one or in one go: + The first is good if the server merges patches, but many don't so client + based merging can be attempt in which case the second is better. + "bad things" will happen if patches are merged on the server, + but client side merging is attempt as well */ + pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true); + if (pdiff_merge == true) + { + // reprepro and dak add this flag if they merge patches on the server + auto const precedence = Tags.Find("X-Patch-Precedence"); + pdiff_merge = (precedence != "merged"); + } + + // calculate the size of all patches we have to get + unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100); + if (sizeLimitPercent > 0) + { + unsigned long long downloadSize = 0; + if (pdiff_merge) + downloadSize = std::accumulate(available_patches.begin(), available_patches.end(), 0llu, + [](unsigned long long const T, DiffInfo const &I) { + return T + I.download_hashes.FileSize(); + }); + // if server-side merging, assume we will need only the first patch + else if (not available_patches.empty()) + downloadSize = available_patches.front().download_hashes.FileSize(); + if (downloadSize != 0) + { + unsigned long long downloadSizeIdx = 0; + auto const types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' '); + for (auto const &t : types) + { + std::string MetaKey = Target.MetaKey; + if (t != "uncompressed") + MetaKey += '.' + t; + HashStringList const hsl = GetExpectedHashesFor(MetaKey); + if (unlikely(hsl.usable() == false)) + continue; + downloadSizeIdx = hsl.FileSize(); + break; + } + unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent; + if ((sizeLimit/100) < downloadSize) + { + strprintf(ErrorText, "Need %llu compressed bytes, but limit is %llu and original is %llu", downloadSize, (sizeLimit/100), downloadSizeIdx); + return false; + } + } + } + + // clean the plate + { + std::string const Final = GetExistingFilename(CurrentPackagesFile); + if (unlikely(Final.empty())) // because we wouldn't be called in such a case + return false; + std::string const PartialFile = GetPartialFileNameFromURI(Target.URI); + std::string const PatchedFile = GetKeepCompressedFileName(PartialFile + "-patched", Target); + if (not RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PartialFile) || + not RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PatchedFile)) + return false; + { + auto const exts = APT::Configuration::getCompressorExtensions(); + if (not std::all_of(exts.cbegin(), exts.cend(), [&](auto const &ext) { + return RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PartialFile + ext) && + RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PatchedFile + ext); + })) + return false; + } + std::string const Ext = Final.substr(CurrentPackagesFile.length()); + std::string const Partial = PartialFile + Ext; + if (symlink(Final.c_str(), Partial.c_str()) != 0) + { + strprintf(ErrorText, "Bootstrap for patching by linking %s to %s failed!", Final.c_str(), Partial.c_str()); + return false; + } + } + + return true; +} + /*}}}*/ +void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/ +{ + if (CommonFailed(GetDiffIndexURI(Target), Message, Cnf)) + return; + + RenameOnError(PDiffError); + Status = StatDone; + ExpectedAdditionalItems = 0; + + if(Debug) + std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl + << "Falling back to normal index file acquire" << std::endl; + + new pkgAcqIndex(Owner, TransactionManager, Target); +} + /*}}}*/ +bool pkgAcqDiffIndex::VerifyDone(std::string const &Message, pkgAcquire::MethodConfig const * const)/*{{{*/ +{ + string const FinalFile = GetFinalFilename(); + if(StringToBool(LookupTag(Message,"IMS-Hit"),false)) + DestFile = FinalFile; + + if (ParseDiffIndex(DestFile)) + return true; + + Status = StatError; + if (ErrorText.empty()) + ErrorText = "Couldn't parse pdiff index"; + return false; +} + /*}}}*/ +void pkgAcqDiffIndex::Done(string const &Message,HashStringList const &Hashes, /*{{{*/ + pkgAcquire::MethodConfig const * const Cnf) +{ + if(Debug) + std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl; + + Item::Done(Message, Hashes, Cnf); + + if (available_patches.empty()) + { + // we have the same sha1 as the server so we are done here + if(Debug) + std::clog << "pkgAcqDiffIndex: Package file is up-to-date" << std::endl; + QueueOnIMSHit(); + } + else + { + if (pdiff_merge == false) + new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches); + else + { + diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size()); + for(size_t i = 0; i < available_patches.size(); ++i) + (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager, + Target, + available_patches[i], + diffs); + } + } + + TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); + + Complete = true; + Status = StatDone; + Dequeue(); + + return; +} + /*}}}*/ +pkgAcqDiffIndex::~pkgAcqDiffIndex() +{ + if (diffs != NULL) + delete diffs; +} + +// AcqIndexDiffs::AcqIndexDiffs - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* The package diff is added to the queue. one object is constructed + * for each diff and the index + */ +pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *const Owner, + pkgAcqMetaClearSig *const TransactionManager, + IndexTarget const &Target, + vector<DiffInfo> const &diffs) + : pkgAcqBaseIndex(Owner, TransactionManager, Target), + available_patches(diffs) +{ + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); + + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Desc.Owner = this; + Desc.ShortDesc = Target.ShortDesc; + + if(available_patches.empty() == true) + { + // we are done (yeah!), check hashes against the final file + DestFile = GetKeepCompressedFileName(GetFinalFileNameFromURI(Target.URI), Target); + Finish(true); + } + else + { + State = StateFetchDiff; + QueueNextDiff(); + } +} + /*}}}*/ +void pkgAcqIndexDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/ +{ + pkgAcqBaseIndex::Failed(Message,Cnf); + Status = StatDone; + + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); + if(Debug) + std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl + << "Falling back to normal index file acquire " << std::endl; + RenameOnError(PDiffError); + std::string const patchname = GetDiffsPatchFileName(DestFile); + if (RealFileExists(patchname)) + Rename(patchname, patchname + ".FAILED"); + std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI)); + if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile)) + Rename(UnpatchedFile, UnpatchedFile + ".FAILED"); + new pkgAcqIndex(Owner, TransactionManager, Target); + Finish(); +} + /*}}}*/ +// Finish - helper that cleans the item out of the fetcher queue /*{{{*/ +void pkgAcqIndexDiffs::Finish(bool allDone) +{ + if(Debug) + std::clog << "pkgAcqIndexDiffs::Finish(): " + << allDone << " " + << Desc.URI << std::endl; + + // we restore the original name, this is required, otherwise + // the file will be cleaned + if(allDone) + { + std::string const Final = GetKeepCompressedFileName(GetFinalFilename(), Target); + TransactionManager->TransactionStageCopy(this, DestFile, Final); + + // this is for the "real" finish + Complete = true; + Status = StatDone; + Dequeue(); + if(Debug) + std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl; + return; + } + else + DestFile.clear(); + + if(Debug) + std::clog << "Finishing: " << Desc.URI << std::endl; + Complete = false; + Status = StatDone; + Dequeue(); + return; +} + /*}}}*/ +bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ +{ + // calc sha1 of the just patched file + std::string const PartialFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI)); + if(unlikely(PartialFile.empty())) + { + Failed("Message: The file " + GetPartialFileNameFromURI(Target.URI) + " isn't available", NULL); + return false; + } + + FileFd fd(PartialFile, FileFd::ReadOnly, FileFd::Extension); + Hashes LocalHashesCalc; + LocalHashesCalc.AddFD(fd); + HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList(); + + if(Debug) + std::clog << "QueueNextDiff: " << PartialFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl; + + HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey); + if (unlikely(LocalHashes.usable() == false || TargetFileHashes.usable() == false)) + { + Failed("Local/Expected hashes are not usable for " + PartialFile, NULL); + return false; + } + + // final file reached before all patches are applied + if(LocalHashes == TargetFileHashes) + { + Finish(true); + return true; + } + + // remove all patches until the next matching patch is found + // this requires the Index file to be ordered + available_patches.erase(available_patches.begin(), + std::find_if(available_patches.begin(), available_patches.end(), [&](DiffInfo const &I) { + return I.result_hashes == LocalHashes; + })); + + // error checking and falling back if no patch was found + if(available_patches.empty() == true) + { + Failed("No patches left to reach target for " + PartialFile, NULL); + return false; + } + + // queue the right diff + auto const BaseFileURI = Target.URI + ".diff/" + pkgAcquire::URIEncode(available_patches[0].file); + Desc.URI = BaseFileURI + ".gz"; + Desc.Description = Target.Description + " " + available_patches[0].file + string(".pdiff"); + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(BaseFileURI), Target); + + if(Debug) + std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl; + + QueueURI(Desc); + + return true; +} + /*}}}*/ +void pkgAcqIndexDiffs::Done(string const &Message, HashStringList const &Hashes, /*{{{*/ + pkgAcquire::MethodConfig const * const Cnf) +{ + if (Debug) + std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl; + + Item::Done(Message, Hashes, Cnf); + + std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI); + std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile); + std::string const PatchFile = GetDiffsPatchFileName(UnpatchedFile); + std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target); + + switch (State) + { + // success in downloading a diff, enter ApplyDiff state + case StateFetchDiff: + Rename(DestFile, PatchFile); + DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target); + if(Debug) + std::clog << "Sending to rred method: " << UnpatchedFile << std::endl; + State = StateApplyDiff; + Local = true; + Desc.URI = "rred:" + pkgAcquire::URIEncode(UnpatchedFile); + QueueURI(Desc); + SetActiveSubprocess("rred"); + return; + // success in download/apply a diff, queue next (if needed) + case StateApplyDiff: + // remove the just applied patch and base file + available_patches.erase(available_patches.begin()); + RemoveFile("pkgAcqIndexDiffs::Done", PatchFile); + RemoveFile("pkgAcqIndexDiffs::Done", UnpatchedFile); + if(Debug) + std::clog << "Moving patched file in place: " << std::endl + << DestFile << " -> " << PatchedFile << std::endl; + Rename(DestFile, PatchedFile); + + // see if there is more to download + if(available_patches.empty() == false) + { + new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches); + Finish(); + } else { + DestFile = PatchedFile; + Finish(true); + } + return; + } +} + /*}}}*/ +std::string pkgAcqIndexDiffs::Custom600Headers() const /*{{{*/ +{ + if(State != StateApplyDiff) + return pkgAcqBaseIndex::Custom600Headers(); + std::ostringstream patchhashes; + for (auto && hs : available_patches[0].result_hashes) + patchhashes << "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue(); + for (auto && hs : available_patches[0].patch_hashes) + patchhashes << "\nPatch-0-" << hs.HashType() << "-Hash: " << hs.HashValue(); + patchhashes << pkgAcqBaseIndex::Custom600Headers(); + return patchhashes.str(); +} + /*}}}*/ +pkgAcqIndexDiffs::~pkgAcqIndexDiffs() {} + +// AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ +pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *const Owner, + pkgAcqMetaClearSig *const TransactionManager, + IndexTarget const &Target, + DiffInfo const &patch, + std::vector<pkgAcqIndexMergeDiffs *> const *const allPatches) + : pkgAcqBaseIndex(Owner, TransactionManager, Target), + patch(patch), allPatches(allPatches), State(StateFetchDiff) +{ + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Desc.Owner = this; + Desc.ShortDesc = Target.ShortDesc; + Desc.URI = Target.URI + ".diff/" + pkgAcquire::URIEncode(patch.file) + ".gz"; + Desc.Description = Target.Description + " " + patch.file + ".pdiff"; + DestFile = GetPartialFileNameFromURI(Desc.URI); + + if(Debug) + std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl; + + QueueURI(Desc); +} + /*}}}*/ +void pkgAcqIndexMergeDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/ +{ + if(Debug) + std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl; + + pkgAcqBaseIndex::Failed(Message,Cnf); + Status = StatDone; + + // check if we are the first to fail, otherwise we are done here + State = StateDoneDiff; + for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + if ((*I)->State == StateErrorDiff) + { + State = StateErrorDiff; + return; + } + + // first failure means we should fallback + State = StateErrorDiff; + if (Debug) + std::clog << "Falling back to normal index file acquire" << std::endl; + RenameOnError(PDiffError); + std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI)); + if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile)) + Rename(UnpatchedFile, UnpatchedFile + ".FAILED"); + DestFile.clear(); + new pkgAcqIndex(Owner, TransactionManager, Target); +} + /*}}}*/ +void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Hashes, /*{{{*/ + pkgAcquire::MethodConfig const * const Cnf) +{ + if(Debug) + std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl; + + Item::Done(Message, Hashes, Cnf); + + if (std::any_of(allPatches->begin(), allPatches->end(), + [](pkgAcqIndexMergeDiffs const * const P) { return P->State == StateErrorDiff; })) + { + if(Debug) + std::clog << "Another patch failed already, no point in processing this one." << std::endl; + State = StateErrorDiff; + return; + } + + std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI); + std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile); + if (UnpatchedFile.empty()) + { + _error->Fatal("Unpatched file %s doesn't exist (anymore)!", UncompressedUnpatchedFile.c_str()); + State = StateErrorDiff; + return; + } + std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target); + + switch (State) + { + case StateFetchDiff: + // check if this is the last completed diff + State = StateDoneDiff; + for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + if ((*I)->State != StateDoneDiff) + { + if(Debug) + std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl; + return; + } + for (auto * diff : *allPatches) + Rename(diff->DestFile, GetMergeDiffsPatchFileName(UnpatchedFile, diff->patch.file)); + // this is the last completed diff, so we are ready to apply now + DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target); + if(Debug) + std::clog << "Sending to rred method: " << UnpatchedFile << std::endl; + State = StateApplyDiff; + Local = true; + Desc.URI = "rred:" + pkgAcquire::URIEncode(UnpatchedFile); + QueueURI(Desc); + SetActiveSubprocess("rred"); + return; + case StateApplyDiff: + // success in download & apply all diffs, finialize and clean up + if(Debug) + std::clog << "Queue patched file in place: " << std::endl + << DestFile << " -> " << PatchedFile << std::endl; + + // queue for copy by the transaction manager + TransactionManager->TransactionStageCopy(this, DestFile, GetKeepCompressedFileName(GetFinalFilename(), Target)); + + // ensure the ed's are gone regardless of list-cleanup + for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + RemoveFile("pkgAcqIndexMergeDiffs::Done", GetMergeDiffsPatchFileName(UnpatchedFile, (*I)->patch.file)); + RemoveFile("pkgAcqIndexMergeDiffs::Done", UnpatchedFile); + + // all set and done + Complete = true; + if(Debug) + std::clog << "allDone: " << DestFile << "\n" << std::endl; + return; + case StateDoneDiff: _error->Fatal("Done called for %s which is in an invalid Done state", patch.file.c_str()); break; + case StateErrorDiff: _error->Fatal("Done called for %s which is in an invalid Error state", patch.file.c_str()); break; + } +} + /*}}}*/ +std::string pkgAcqIndexMergeDiffs::Custom600Headers() const /*{{{*/ +{ + if(State != StateApplyDiff) + return pkgAcqBaseIndex::Custom600Headers(); + std::ostringstream patchhashes; + unsigned int seen_patches = 0; + for (auto && hs : (*allPatches)[0]->patch.result_hashes) + patchhashes << "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue(); + for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + { + HashStringList const ExpectedHashes = (*I)->patch.patch_hashes; + for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs) + patchhashes << "\nPatch-" << std::to_string(seen_patches) << "-" << hs->HashType() << "-Hash: " << hs->HashValue(); + ++seen_patches; + } + patchhashes << pkgAcqBaseIndex::Custom600Headers(); + return patchhashes.str(); +} + /*}}}*/ +pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {} + +// AcqIndex::AcqIndex - Constructor /*{{{*/ +pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, bool const Derived) + : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD), + CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES)) +{ + if (Derived) + return; + Init(Target.URI, Target.Description, Target.ShortDesc); + + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgIndex with TransactionManager " + << TransactionManager << std::endl; +} + /*}}}*/ +// AcqIndex::Init - deferred Constructor /*{{{*/ +void pkgAcqIndex::Init(string const &URI, string const &URIDesc, + string const &ShortDesc) +{ + Stage = STAGE_DOWNLOAD; + + DestFile = GetPartialFileNameFromURI(URI); + size_t const nextExt = CompressionExtensions.find(' '); + if (nextExt == std::string::npos) + { + CurrentCompressionExtension = CompressionExtensions; + CompressionExtensions.clear(); + } + else + { + CurrentCompressionExtension = CompressionExtensions.substr(0, nextExt); + CompressionExtensions = CompressionExtensions.substr(nextExt+1); + } + + if (CurrentCompressionExtension == "uncompressed") + { + Desc.URI = URI; + } + else if (unlikely(CurrentCompressionExtension.empty())) + return; + else + { + Desc.URI = URI + '.' + CurrentCompressionExtension; + DestFile = DestFile + '.' + CurrentCompressionExtension; + } + + // store file size of the download to ensure the fetcher gives + // accurate progress reporting + FileSize = GetExpectedHashes().FileSize(); + + Desc.Description = URIDesc; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + + QueueURI(Desc); +} + /*}}}*/ +// AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ +// --------------------------------------------------------------------- +/* The only header we use is the last-modified header. */ +string pkgAcqIndex::Custom600Headers() const +{ + std::string msg = pkgAcqBaseIndex::Custom600Headers(); + msg.append("\nIndex-File: true"); + + if (TransactionManager->LastMetaIndexParser == NULL) + { + std::string const Final = GetFinalFilename(); + + struct stat Buf; + if (stat(Final.c_str(),&Buf) == 0) + msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false); + } + + if(Target.IsOptional) + msg += "\nFail-Ignore: true"; + + return msg; +} + /*}}}*/ +// AcqIndex::Failed - getting the indexfile failed /*{{{*/ +bool pkgAcqIndex::CommonFailed(std::string const &TargetURI, + std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) +{ + pkgAcqBaseIndex::Failed(Message,Cnf); + // authorisation matches will not be fixed by other compression types + if (Status != StatAuthError) + { + if (CompressionExtensions.empty() == false) + { + Status = StatIdle; + Init(TargetURI, Desc.Description, Desc.ShortDesc); + return true; + } + } + return false; +} +void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) +{ + if (CommonFailed(Target.URI, Message, Cnf)) + return; + + if(Target.IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD) + Status = StatDone; + else + TransactionManager->AbortTransaction(); +} + /*}}}*/ +// AcqIndex::Done - Finished a fetch /*{{{*/ +// --------------------------------------------------------------------- +/* This goes through a number of states.. On the initial fetch the + method could possibly return an alternate filename which points + to the uncompressed version of the file. If this is so the file + is copied into the partial directory. In all other cases the file + is decompressed with a compressed uri. */ +void pkgAcqIndex::Done(string const &Message, + HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cfg) +{ + Item::Done(Message,Hashes,Cfg); + + switch(Stage) + { + case STAGE_DOWNLOAD: + StageDownloadDone(Message); + break; + case STAGE_DECOMPRESS_AND_VERIFY: + StageDecompressDone(); + break; + } +} + /*}}}*/ +// AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/ +void pkgAcqIndex::StageDownloadDone(string const &Message) +{ + Local = true; + Complete = true; + + std::string const AltFilename = LookupTag(Message,"Alt-Filename"); + std::string Filename = LookupTag(Message,"Filename"); + + // we need to verify the file against the current Release file again + // on if-modified-since hit to avoid a stale attack against us + if (StringToBool(LookupTag(Message, "IMS-Hit"), false)) + { + Filename = GetExistingFilename(GetFinalFileNameFromURI(Target.URI)); + EraseFileName = DestFile = flCombine(flNotFile(DestFile), flNotDir(Filename)); + if (symlink(Filename.c_str(), DestFile.c_str()) != 0) + _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str()); + Stage = STAGE_DECOMPRESS_AND_VERIFY; + Desc.URI = "store:" + pkgAcquire::URIEncode(DestFile); + QueueURI(Desc); + SetActiveSubprocess(::URI(Desc.URI).Access); + return; + } + // methods like file:// give us an alternative (uncompressed) file + else if (Target.KeepCompressed == false && AltFilename.empty() == false) + { + Filename = AltFilename; + EraseFileName.clear(); + } + // Methods like e.g. "file:" will give us a (compressed) FileName that is + // not the "DestFile" we set, in this case we uncompress from the local file + else if (Filename != DestFile && RealFileExists(DestFile) == false) + { + // symlinking ensures that the filename can be used for compression detection + // that is e.g. needed for by-hash which has no extension over file + if (symlink(Filename.c_str(),DestFile.c_str()) != 0) + _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str()); + else + { + EraseFileName = DestFile; + Filename = DestFile; + } + } + + Stage = STAGE_DECOMPRESS_AND_VERIFY; + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); + if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile)) + Desc.URI = "copy:" + pkgAcquire::URIEncode(Filename); + else + Desc.URI = "store:" + pkgAcquire::URIEncode(Filename); + if (DestFile == Filename) + { + if (CurrentCompressionExtension == "uncompressed") + return StageDecompressDone(); + DestFile = "/dev/null"; + } + + if (EraseFileName.empty() && Filename != AltFilename) + EraseFileName = Filename; + + // queue uri for the next stage + QueueURI(Desc); + SetActiveSubprocess(::URI(Desc.URI).Access); +} + /*}}}*/ +// AcqIndex::StageDecompressDone - Final verification /*{{{*/ +void pkgAcqIndex::StageDecompressDone() +{ + if (DestFile == "/dev/null") + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); + + // Done, queue for rename on transaction finished + TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); +} + /*}}}*/ +pkgAcqIndex::~pkgAcqIndex() {} + +// AcqArchive::AcqArchive - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* This just sets up the initial fetch environment and queues the first + possibilitiy */ +pkgAcqArchive::pkgAcqArchive(pkgAcquire *const Owner, pkgSourceList *const Sources, + pkgRecords *const Recs, pkgCache::VerIterator const &Version, + string &StoreFilename) : Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs), + StoreFilename(StoreFilename), + Trusted(false) +{ + if (Version.Arch() == 0) + { + _error->Error(_("I wasn't able to locate a file for the %s package. " + "This might mean you need to manually fix this package. " + "(due to missing arch)"), + Version.ParentPkg().FullName().c_str()); + return; + } + + // check if we have one trusted source for the package. if so, switch + // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode + bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false); + bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false); + bool seenUntrusted = false; + for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i) + { + pkgIndexFile *Index; + if (Sources->FindIndex(i.File(),Index) == false) + continue; + + if (debugAuth == true) + std::cerr << "Checking index: " << Index->Describe() + << "(Trusted=" << Index->IsTrusted() << ")" << std::endl; + + if (Index->IsTrusted() == true) + { + Trusted = true; + if (allowUnauth == false) + break; + } + else + seenUntrusted = true; + } + + // "allow-unauthenticated" restores apts old fetching behaviour + // that means that e.g. unauthenticated file:// uris are higher + // priority than authenticated http:// uris + if (allowUnauth == true && seenUntrusted == true) + Trusted = false; + + StoreFilename.clear(); + for (auto Vf = Version.FileList(); Vf.end() == false; ++Vf) + { + auto const PkgF = Vf.File(); + if (unlikely(PkgF.end())) + continue; + if (PkgF.Flagged(pkgCache::Flag::NotSource)) + continue; + pkgIndexFile *Index; + if (Sources->FindIndex(PkgF, Index) == false) + continue; + if (Trusted && Index->IsTrusted() == false) + continue; + + pkgRecords::Parser &Parse = Recs->Lookup(Vf); + // collect the hashes from the indexes + auto hsl = Parse.Hashes(); + if (ExpectedHashes.empty()) + ExpectedHashes = hsl; + else + { + // bad things will likely happen, but the user might be "lucky" still + // if the sources provide the same hashtypes (so that they aren't mixed) + for (auto const &hs : hsl) + if (ExpectedHashes.push_back(hs) == false) + { + _error->Warning("Sources disagree on hashes for supposedly identical version '%s' of '%s'.", + Version.VerStr(), Version.ParentPkg().FullName(false).c_str()); + break; + } + } + // only allow local volatile sources to have no hashes + if (PkgF.Flagged(pkgCache::Flag::LocalSource)) + LocalSource = true; + else if (hsl.empty()) + continue; + + std::string poolfilename = Parse.FileName(); + if (poolfilename.empty()) + continue; + + std::remove_reference<decltype(ModifyCustomFields())>::type fields; + { + auto const debIndex = dynamic_cast<pkgDebianIndexTargetFile const *const>(Index); + if (debIndex != nullptr) + { + auto const IT = debIndex->GetIndexTarget(); + fields.emplace("Target-Repo-URI", IT.Option(IndexTarget::REPO_URI)); + fields.emplace("Target-Release", IT.Option(IndexTarget::RELEASE)); + fields.emplace("Target-Site", IT.Option(IndexTarget::SITE)); + } + } + fields.emplace("Target-Base-URI", Index->ArchiveURI("")); + if (PkgF->Component != 0) + fields.emplace("Target-Component", PkgF.Component()); + auto const RelF = PkgF.ReleaseFile(); + if (RelF.end() == false) + { + if (RelF->Codename != 0) + fields.emplace("Target-Codename", RelF.Codename()); + if (RelF->Archive != 0) + fields.emplace("Target-Suite", RelF.Archive()); + } + fields.emplace("Target-Architecture", Version.Arch()); + fields.emplace("Target-Type", flExtension(poolfilename)); + + if (StoreFilename.empty()) + { + /* We pick a filename based on the information we have for the version, + but we don't know what extension such a file should have, so we look + at the filenames used online and assume that they are the same for + all repositories containing this file */ + StoreFilename = QuoteString(Version.ParentPkg().Name(), "_:") + '_' + + QuoteString(Version.VerStr(), "_:") + '_' + + QuoteString(Version.Arch(), "_:.") + + "." + flExtension(poolfilename); + + Desc.URI = Index->ArchiveURI(poolfilename); + Desc.Description = Index->ArchiveInfo(Version); + Desc.Owner = this; + Desc.ShortDesc = Version.ParentPkg().FullName(true); + auto &customfields = ModifyCustomFields(); + for (auto const &f : fields) + customfields[f.first] = f.second; + FileSize = Version->Size; + } + else + PushAlternativeURI(Index->ArchiveURI(poolfilename), std::move(fields), true); + } + if (StoreFilename.empty()) + { + _error->Error(_("Can't find a source to download version '%s' of '%s'"), + Version.VerStr(), Version.ParentPkg().FullName(false).c_str()); + return; + } + if (FileSize == 0 && not _config->FindB("Acquire::AllowUnsizedPackages", false)) + { + _error->Error("Repository is broken: %s has no Size information", + Desc.Description.c_str()); + return; + } + + // Check if we already downloaded the file + struct stat Buf; + auto FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename); + if (stat(FinalFile.c_str(), &Buf) == 0) + { + // Make sure the size matches + if ((unsigned long long)Buf.st_size == Version->Size) + { + Complete = true; + Local = true; + Status = StatDone; + StoreFilename = DestFile = FinalFile; + return; + } + + /* Hmm, we have a file and its size does not match, this shouldn't + happen.. */ + RemoveFile("pkgAcqArchive::QueueNext", FinalFile); + } + + // Check the destination file + DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename); + if (stat(DestFile.c_str(), &Buf) == 0) + { + // Hmm, the partial file is too big, erase it + if ((unsigned long long)Buf.st_size > Version->Size) + RemoveFile("pkgAcqArchive::QueueNext", DestFile); + else + PartialSize = Buf.st_size; + } + + // Disables download of archives - useful if no real installation follows, + // e.g. if we are just interested in proposed installation order + if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true) + { + Complete = true; + Local = true; + Status = StatDone; + StoreFilename = DestFile = FinalFile; + return; + } + + // Create the item + Local = false; + QueueURI(Desc); +} + /*}}}*/ +bool pkgAcqArchive::QueueNext() /*{{{*/ +{ + return false; +} + /*}}}*/ +// AcqArchive::Done - Finished fetching /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cfg) +{ + Item::Done(Message, Hashes, Cfg); + + // Grab the output filename + std::string const FileName = LookupTag(Message,"Filename"); + if (DestFile != FileName && RealFileExists(DestFile) == false) + { + StoreFilename = DestFile = FileName; + Local = true; + Complete = true; + return; + } + + // Done, move it into position + string const FinalFile = GetFinalFilename(); + Rename(DestFile,FinalFile); + StoreFilename = DestFile = FinalFile; + Complete = true; +} + /*}}}*/ +// AcqArchive::Failed - Failure handler /*{{{*/ +// --------------------------------------------------------------------- +/* Here we try other sources */ +void pkgAcqArchive::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) +{ + Item::Failed(Message,Cnf); +} + /*}}}*/ +APT_PURE bool pkgAcqArchive::IsTrusted() const /*{{{*/ +{ + return Trusted; +} + /*}}}*/ +void pkgAcqArchive::Finished() /*{{{*/ +{ + if (Status == pkgAcquire::Item::StatDone && + Complete == true) + return; + StoreFilename = string(); +} + /*}}}*/ +std::string pkgAcqArchive::DescURI() const /*{{{*/ +{ + return Desc.URI; +} + /*}}}*/ +std::string pkgAcqArchive::ShortDesc() const /*{{{*/ +{ + return Desc.ShortDesc; +} + /*}}}*/ +pkgAcqArchive::~pkgAcqArchive() {} + +// AcqChangelog::pkgAcqChangelog - Constructors /*{{{*/ +class pkgAcqChangelog::Private +{ + public: + std::string FinalFile; +}; +pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver, + std::string const &DestDir, std::string const &DestFilename) : + pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(Ver.SourcePkgName()), SrcVersion(Ver.SourceVerStr()) +{ + Desc.URI = URI(Ver); + Init(DestDir, DestFilename); +} +// some parameters are char* here as they come likely from char* interfaces – which can also return NULL +pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &RlsFile, + char const * const Component, char const * const SrcName, char const * const SrcVersion, + const string &DestDir, const string &DestFilename) : + pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion) +{ + Desc.URI = URI(RlsFile, Component, SrcName, SrcVersion); + Init(DestDir, DestFilename); +} +pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, + std::string const &URI, char const * const SrcName, char const * const SrcVersion, + const string &DestDir, const string &DestFilename) : + pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion) +{ + Desc.URI = URI; + Init(DestDir, DestFilename); +} +void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFilename) +{ + if (Desc.URI.empty()) + { + Status = StatError; + // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1 + strprintf(ErrorText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str()); + // Let the error message print something sensible rather than "Failed to fetch /" + if (DestFilename.empty()) + DestFile = SrcName + ".changelog"; + else + DestFile = DestFilename; + Desc.URI = "changelog:/" + pkgAcquire::URIEncode(DestFile); + return; + } + + std::string DestFileName; + if (DestFilename.empty()) + DestFileName = flCombine(DestFile, SrcName + ".changelog"); + else + DestFileName = flCombine(DestFile, DestFilename); + + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + std::string const systemTemp = GetTempDir(SandboxUser); + char tmpname[1000]; + snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", systemTemp.c_str()); + if (NULL == mkdtemp(tmpname)) + { + _error->Errno("mkdtemp", "mkdtemp failed in changelog acquire of %s %s", SrcName.c_str(), SrcVersion.c_str()); + Status = StatError; + return; + } + TemporaryDirectory = tmpname; + + ChangeOwnerAndPermissionOfFile("pkgAcqChangelog::Init", TemporaryDirectory.c_str(), + SandboxUser.c_str(), ROOT_GROUP, 0700); + + DestFile = flCombine(TemporaryDirectory, DestFileName); + if (DestDir.empty() == false) + { + d->FinalFile = flCombine(DestDir, DestFileName); + if (RealFileExists(d->FinalFile)) + { + FileFd file1, file2; + if (file1.Open(DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive) && + file2.Open(d->FinalFile, FileFd::ReadOnly) && CopyFile(file2, file1)) + { + ChangeOwnerAndPermissionOfFile("pkgAcqChangelog::Init", DestFile.c_str(), "root", ROOT_GROUP, 0644); + struct timeval times[2]; + times[0].tv_sec = times[1].tv_sec = file2.ModificationTime(); + times[0].tv_usec = times[1].tv_usec = 0; + utimes(DestFile.c_str(), times); + } + } + } + + Desc.ShortDesc = "Changelog"; + strprintf(Desc.Description, "%s %s %s Changelog", URI::SiteOnly(Desc.URI).c_str(), SrcName.c_str(), SrcVersion.c_str()); + Desc.Owner = this; + QueueURI(Desc); +} + /*}}}*/ +std::string pkgAcqChangelog::URI(pkgCache::VerIterator const &Ver) /*{{{*/ +{ + std::string const confOnline = "Acquire::Changelogs::AlwaysOnline"; + bool AlwaysOnline = _config->FindB(confOnline, false); + if (AlwaysOnline == false) + for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF) + { + pkgCache::PkgFileIterator const PF = VF.File(); + if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0) + continue; + pkgCache::RlsFileIterator const RF = PF.ReleaseFile(); + if (RF->Origin != 0 && _config->FindB(confOnline + "::Origin::" + RF.Origin(), false)) + { + AlwaysOnline = true; + break; + } + } + if (AlwaysOnline == false) + { + pkgCache::PkgIterator const Pkg = Ver.ParentPkg(); + if (Pkg->CurrentVer != 0 && Pkg.CurrentVer() == Ver) + { + auto const LocalFile = [](pkgCache::PkgIterator const &Pkg) -> std::string { + std::string const root = _config->FindDir("Dir"); + std::string const basename = root + std::string("usr/share/doc/") + Pkg.Name() + "/changelog"; + std::string const debianname = basename + ".Debian"; + auto const exts = APT::Configuration::getCompressorExtensions(); // likely we encounter only .gz + for (auto file : { debianname, basename }) + { + if (FileExists(file)) + return "copy://" + file; + for (auto const& ext : exts) + { + auto const compressedfile = file + ext; + if (FileExists(compressedfile)) + return "store://" + compressedfile; + } + } + return ""; + }(Pkg); + if (not LocalFile.empty()) + { + _error->PushToStack(); + FileFd trimmed; + if (APT::String::Startswith(LocalFile, "copy://")) + trimmed.Open(LocalFile.substr(7), FileFd::ReadOnly, FileFd::None); + else + trimmed.Open(LocalFile.substr(8), FileFd::ReadOnly, FileFd::Extension); + + bool trimmedFile = false; + if (trimmed.IsOpen()) + { + /* We want to look at the last line… in a (likely) compressed file, + which means we more or less have to uncompress the entire file. + So we skip ahead the filesize minus our choosen line size in + the hope that changelogs don't grow by being compressed to + avoid doing this costly dance on at least a bit of the file. */ + char buffer[150]; + if (auto const filesize = trimmed.FileSize(); filesize > sizeof(buffer)) + trimmed.Skip(filesize - sizeof(buffer)); + std::string_view giveaways[] = { + "# To read the complete changelog use", // Debian + "# For older changelog entries, run", // Ubuntu + }; + while (trimmed.ReadLine(buffer, sizeof(buffer)) != nullptr) + { + std::string_view const line{buffer}; + if (std::any_of(std::begin(giveaways), std::end(giveaways), [=](auto const gw) { return line.compare(0, gw.size(), gw) == 0; })) + { + trimmedFile = true; + break; + } + } + } + _error->RevertToStack(); + if (not trimmedFile) + return LocalFile; + } + } + } + + char const * const SrcName = Ver.SourcePkgName(); + char const * const SrcVersion = Ver.SourceVerStr(); + // find the first source for this version which promises a changelog + for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF) + { + pkgCache::PkgFileIterator const PF = VF.File(); + if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0) + continue; + pkgCache::RlsFileIterator const RF = PF.ReleaseFile(); + std::string const uri = URI(RF, PF.Component(), SrcName, SrcVersion); + if (uri.empty()) + continue; + return uri; + } + return ""; +} +std::string pkgAcqChangelog::URITemplate(pkgCache::RlsFileIterator const &Rls) +{ + if (Rls.end() == true || (Rls->Label == 0 && Rls->Origin == 0)) + return ""; + std::string const serverConfig = "Acquire::Changelogs::URI"; + std::string server; +#define APT_EMPTY_SERVER \ + if (server.empty() == false) \ + { \ + if (server != "no") \ + return server; \ + return ""; \ + } +#define APT_CHECK_SERVER(X, Y) \ + if (Rls->X != 0) \ + { \ + std::string const specialServerConfig = serverConfig + "::" + Y + #X + "::" + Rls.X(); \ + server = _config->Find(specialServerConfig); \ + APT_EMPTY_SERVER \ + } + // this way e.g. Debian-Security can fallback to Debian + APT_CHECK_SERVER(Label, "Override::") + APT_CHECK_SERVER(Origin, "Override::") + + if (RealFileExists(Rls.FileName())) + { + _error->PushToStack(); + FileFd rf; + /* This can be costly. A caller wanting to get millions of URIs might + want to do this on its own once and use Override settings. + We don't do this here as Origin/Label are not as unique as they + should be so this could produce request order-dependent anomalies */ + if (OpenMaybeClearSignedFile(Rls.FileName(), rf) == true) + { + pkgTagFile TagFile(&rf, rf.Size()); + pkgTagSection Section; + if (TagFile.Step(Section) == true) + server = Section.FindS("Changelogs"); + } + _error->RevertToStack(); + APT_EMPTY_SERVER + } + + APT_CHECK_SERVER(Label, "") + APT_CHECK_SERVER(Origin, "") +#undef APT_CHECK_SERVER +#undef APT_EMPTY_SERVER + return ""; +} +std::string pkgAcqChangelog::URI(pkgCache::RlsFileIterator const &Rls, + char const * const Component, char const * const SrcName, + char const * const SrcVersion) +{ + return URI(URITemplate(Rls), Component, SrcName, SrcVersion); +} +std::string pkgAcqChangelog::URI(std::string const &Template, + char const * const Component, char const * const SrcName, + char const * const SrcVersion) +{ + if (Template.find("@CHANGEPATH@") == std::string::npos) + return ""; + + // the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. main/a/apt/apt_1.1 or contrib/liba/libapt/libapt_2.0 + std::string const Src{SrcName}; + std::string path = pkgAcquire::URIEncode(APT::String::Startswith(SrcName, "lib") ? Src.substr(0, 4) : Src.substr(0,1)); + path.append("/").append(pkgAcquire::URIEncode(Src)).append("/"); + path.append(pkgAcquire::URIEncode(Src)).append("_").append(pkgAcquire::URIEncode(StripEpoch(SrcVersion))); + // we omit component for releases without one (= flat-style repositories) + if (Component != NULL && strlen(Component) != 0) + path = pkgAcquire::URIEncode(Component) + "/" + path; + + return SubstVar(Template, "@CHANGEPATH@", path); +} + /*}}}*/ +// AcqChangelog::Failed - Failure handler /*{{{*/ +void pkgAcqChangelog::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf) +{ + Item::Failed(Message,Cnf); + + std::string errText; + // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1 + strprintf(errText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str()); + + // Error is probably something techy like 404 Not Found + if (ErrorText.empty()) + ErrorText = errText; + else + ErrorText = errText + " (" + ErrorText + ")"; +} + /*}}}*/ +// AcqChangelog::Done - Item downloaded OK /*{{{*/ +void pkgAcqChangelog::Done(string const &Message,HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const * const Cnf) +{ + Item::Done(Message,CalcHashes,Cnf); + if (d->FinalFile.empty() == false) + { + if (RemoveFile("pkgAcqChangelog::Done", d->FinalFile) == false || + Rename(DestFile, d->FinalFile) == false) + Status = StatError; + } + + Complete = true; +} + /*}}}*/ +pkgAcqChangelog::~pkgAcqChangelog() /*{{{*/ +{ + if (TemporaryDirectory.empty() == false) + { + RemoveFile("~pkgAcqChangelog", DestFile); + rmdir(TemporaryDirectory.c_str()); + } + delete d; +} + /*}}}*/ + +// AcqFile::pkgAcqFile - Constructor /*{{{*/ +pkgAcqFile::pkgAcqFile(pkgAcquire *const Owner, string const &URI, HashStringList const &Hashes, + unsigned long long const Size, string const &Dsc, string const &ShortDesc, + const string &DestDir, const string &DestFilename, + bool const IsIndexFile) : Item(Owner), d(NULL), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes) +{ + ::URI url{URI}; + if (url.Path.find(' ') != std::string::npos || url.Path.find('%') == std::string::npos) + url.Path = pkgAcquire::URIEncode(url.Path); + + if(!DestFilename.empty()) + DestFile = DestFilename; + else if(!DestDir.empty()) + DestFile = DestDir + "/" + DeQuoteString(flNotDir(url.Path)); + else + DestFile = DeQuoteString(flNotDir(url.Path)); + + // Create the item + Desc.URI = std::string(url); + Desc.Description = Dsc; + Desc.Owner = this; + + // Set the short description to the archive component + Desc.ShortDesc = ShortDesc; + + // Get the transfer sizes + FileSize = Size; + struct stat Buf; + if (stat(DestFile.c_str(),&Buf) == 0) + { + // Hmm, the partial file is too big, erase it + if ((Size > 0) && (unsigned long long)Buf.st_size > Size) + RemoveFile("pkgAcqFile", DestFile); + else + PartialSize = Buf.st_size; + } + + QueueURI(Desc); +} + /*}}}*/ +// AcqFile::Done - Item downloaded OK /*{{{*/ +void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const * const Cnf) +{ + Item::Done(Message,CalcHashes,Cnf); + + std::string const FileName = LookupTag(Message,"Filename"); + Complete = true; + + // The files timestamp matches + if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + return; + + // We have to copy it into place + if (RealFileExists(DestFile.c_str()) == false) + { + Local = true; + if (_config->FindB("Acquire::Source-Symlinks",true) == false || + Cnf->Removable == true) + { + Desc.URI = "copy:" + pkgAcquire::URIEncode(FileName); + QueueURI(Desc); + return; + } + + // Erase the file if it is a symlink so we can overwrite it + struct stat St; + if (lstat(DestFile.c_str(),&St) == 0) + { + if (S_ISLNK(St.st_mode) != 0) + RemoveFile("pkgAcqFile::Done", DestFile); + } + + // Symlink the file + if (symlink(FileName.c_str(),DestFile.c_str()) != 0) + { + _error->PushToStack(); + _error->Errno("pkgAcqFile::Done", "Symlinking file %s failed", DestFile.c_str()); + std::stringstream msg; + _error->DumpErrors(msg, GlobalError::DEBUG, false); + _error->RevertToStack(); + ErrorText = msg.str(); + Status = StatError; + Complete = false; + } + } +} + /*}}}*/ +string pkgAcqFile::Custom600Headers() const /*{{{*/ +{ + string Header = pkgAcquire::Item::Custom600Headers(); + if (not IsIndexFile) + return Header; + return Header + "\nIndex-File: true"; +} + /*}}}*/ +pkgAcqFile::~pkgAcqFile() {} + +void pkgAcqAuxFile::Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/ +{ + pkgAcqFile::Failed(Message, Cnf); + if (Status == StatIdle) + return; + if (RealFileExists(DestFile)) + Rename(DestFile, DestFile + ".FAILED"); + Worker->ReplyAux(Desc); +} + /*}}}*/ +void pkgAcqAuxFile::Done(std::string const &Message, HashStringList const &CalcHashes, /*{{{*/ + pkgAcquire::MethodConfig const *const Cnf) +{ + pkgAcqFile::Done(Message, CalcHashes, Cnf); + if (Status == StatDone) + Worker->ReplyAux(Desc); + else if (Status == StatAuthError || Status == StatError) + Worker->ReplyAux(Desc); +} + /*}}}*/ +std::string pkgAcqAuxFile::Custom600Headers() const /*{{{*/ +{ + if (MaximumSize == 0) + return pkgAcqFile::Custom600Headers(); + std::string maxsize; + strprintf(maxsize, "\nMaximum-Size: %llu", MaximumSize); + return pkgAcqFile::Custom600Headers().append(maxsize); +} + /*}}}*/ +void pkgAcqAuxFile::Finished() /*{{{*/ +{ + auto dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"); + if (APT::String::Startswith(DestFile, dirname)) + { + // the file is never returned by method requesting it, so fix up the permission now + if (FileExists(DestFile)) + { + ChangeOwnerAndPermissionOfFile("pkgAcqAuxFile", DestFile.c_str(), "root", ROOT_GROUP, 0644); + if (Status == StatDone) + return; + } + } + else + { + dirname = flNotFile(DestFile); + RemoveFile("pkgAcqAuxFile::Finished", DestFile); + RemoveFile("pkgAcqAuxFile::Finished", DestFile + ".FAILED"); + rmdir(dirname.c_str()); + } + DestFile.clear(); +} + /*}}}*/ +// GetAuxFileNameFromURI /*{{{*/ +static std::string GetAuxFileNameFromURIInLists(std::string const &uri) +{ + // check if we have write permission for our usual location. + auto const dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"); + char const * const filetag = ".apt-acquire-privs-test.XXXXXX"; + std::string const tmpfile_tpl = flCombine(dirname, filetag); + std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free }; + int const fd = mkstemp(tmpfile.get()); + if (fd == -1) + return ""; + RemoveFile("GetAuxFileNameFromURI", tmpfile.get()); + close(fd); + return flCombine(dirname, URItoFileName(uri)); +} +static std::string GetAuxFileNameFromURI(std::string const &uri) +{ + auto const lists = GetAuxFileNameFromURIInLists(uri); + if (lists.empty() == false) + return lists; + + std::string tmpdir_tpl; + strprintf(tmpdir_tpl, "%s/apt-auxfiles-XXXXXX", GetTempDir().c_str()); + std::unique_ptr<char, decltype(std::free) *> tmpdir { strndup(tmpdir_tpl.data(), tmpdir_tpl.length()), std::free }; + if (mkdtemp(tmpdir.get()) == nullptr) + { + _error->Errno("GetAuxFileNameFromURI", "mkdtemp of %s failed", tmpdir.get()); + return flCombine("/nonexistent/auxfiles/", URItoFileName(uri)); + } + chmod(tmpdir.get(), 0755); + auto const filename = flCombine(tmpdir.get(), URItoFileName(uri)); + _error->PushToStack(); + FileFd in(flCombine(flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"), URItoFileName(uri)), FileFd::ReadOnly); + if (in.IsOpen()) + { + FileFd out(filename, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive); + CopyFile(in, out); + ChangeOwnerAndPermissionOfFile("GetAuxFileNameFromURI", filename.c_str(), "root", ROOT_GROUP, 0644); + } + _error->RevertToStack(); + return filename; +} + /*}}}*/ +pkgAcqAuxFile::pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker, + std::string const &ShortDesc, std::string const &Desc, std::string const &URI, + HashStringList const &Hashes, unsigned long long const MaximumSize) : pkgAcqFile(Owner->GetOwner(), URI, Hashes, Hashes.FileSize(), Desc, ShortDesc, "", GetAuxFileNameFromURI(URI), false), + Owner(Owner), Worker(Worker), MaximumSize(MaximumSize) +{ + /* very bad failures can happen while constructing which causes + us to hang as the aux request is never answered (e.g. method not available) + Ideally we catch failures earlier, but a safe guard can't hurt. */ + if (Status == pkgAcquire::Item::StatIdle || Status == pkgAcquire::Item::StatFetching) + return; + Failed(std::string("400 URI Failure\n") + + "URI: " + URI + "\n" + + "Filename: " + DestFile, + nullptr); +} +pkgAcqAuxFile::~pkgAcqAuxFile() {} diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h new file mode 100644 index 0000000..22da981 --- /dev/null +++ b/apt-pkg/acquire-item.h @@ -0,0 +1,1218 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire Item - Item to acquire + + When an item is instantiated it will add it self to the local list in + the Owner Acquire class. Derived classes will then call QueueURI to + register all the URI's they wish to fetch at the initial moment. + + Three item classes are provided to provide functionality for + downloading of Index, Translation and Packages files. + + A Archive class is provided for downloading .deb files. It does Hash + checking and source location as well as a retry algorithm. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_ACQUIRE_ITEM_H +#define PKGLIB_ACQUIRE_ITEM_H + +#include <apt-pkg/acquire.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/indexfile.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/weakptr.h> + +#include <map> +#include <string> +#include <unordered_map> +#include <vector> + + +/** \addtogroup acquire + * @{ + * + * \file acquire-item.h + */ + +class pkgRecords; +class pkgSourceList; +class pkgAcqMetaClearSig; +class pkgAcqIndexMergeDiffs; +class metaIndex; + +class APT_PUBLIC pkgAcquire::Item : public WeakPointable /*{{{*/ +/** \brief Represents the process by which a pkgAcquire object should + * retrieve a file or a collection of files. + * + * By convention, Item subclasses should insert themselves into the + * acquire queue when they are created by calling QueueURI(), and + * remove themselves by calling Dequeue() when either Done() or + * Failed() is invoked. Item objects are also responsible for + * notifying the download progress indicator (accessible via + * #Owner->Log) of their status. + * + * \see pkgAcquire + */ +{ + public: + + /** \brief The current status of this item. */ + enum ItemState + { + /** \brief The item is waiting to be downloaded. */ + StatIdle, + + /** \brief The item is currently being downloaded. */ + StatFetching, + + /** \brief The item has been successfully downloaded. */ + StatDone, + + /** \brief An error was encountered while downloading this + * item. + */ + StatError, + + /** \brief The item was downloaded but its authenticity could + * not be verified. + */ + StatAuthError, + + /** \brief The item was could not be downloaded because of + * a transient network error (e.g. network down) + */ + StatTransientNetworkError, + } Status; + + /** \brief Contains a textual description of the error encountered + * if #ItemState is #StatError or #StatAuthError. + */ + std::string ErrorText; + + /** \brief The size of the object to fetch. */ + unsigned long long FileSize; + + /** \brief How much of the object was already fetched. */ + unsigned long long PartialSize; + + /** \brief contains the name of the subprocess that is operating on this object + * (for instance, "gzip", "rred" or "gpgv"). This is obsoleting #Mode from above + * as it can manage the lifetime of included string properly. */ + std::string ActiveSubprocess; + + /** \brief A client-supplied unique identifier. + * + * This field is initialized to 0; it is meant to be filled in by + * clients that wish to use it to uniquely identify items. + * + * APT progress reporting will store an ID there as shown in "Get:42 …" + */ + unsigned long ID; + + /** \brief If \b true, the entire object has been successfully fetched. + * + * Subclasses should set this to \b true when appropriate. + */ + bool Complete; + + /** \brief If \b true, the URI of this object is "local". + * + * The only effect of this field is to exclude the object from the + * download progress indicator's overall statistics. + */ + bool Local; + + std::string UsedMirror; + + /** \brief The number of fetch queues into which this item has been + * inserted. + * + * There is one queue for each source from which an item could be + * downloaded. + * + * \sa pkgAcquire + */ + unsigned int QueueCounter; + + /** \brief The number of additional fetch items that are expected + * once this item is done. + * + * Some items like pkgAcqMeta{Index,Sig} will queue additional + * items. This variable can be set by the methods if it knows + * in advance how many items to expect to get a more accurate + * progress. + */ + unsigned int ExpectedAdditionalItems; + + /** \brief The name of the file into which the retrieved object + * will be written. + */ + std::string DestFile; + + /** \brief Number of retries */ + unsigned int Retries; + + /** \brief Invoked by the acquire worker when the object couldn't + * be fetched. + * + * This is a branch of the continuation of the fetch process. + * + * \param Message An RFC822-formatted message from the acquire + * method describing what went wrong. Use LookupTag() to parse + * it. + * + * \param Cnf The method via which the worker tried to fetch this object. + * + * \sa pkgAcqMethod + */ + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf); + APT_HIDDEN void FailMessage(std::string const &Message); + + /** \brief Invoked by the acquire worker to check if the successfully + * fetched object is also the objected we wanted to have. + * + * Note that the object might \e not have been written to + * DestFile; check for the presence of an Alt-Filename entry in + * Message to find the file to which it was really written. + * + * This is called before Done is called and can prevent it by returning + * \b false which will result in Failed being called instead. + * + * You should prefer to use this method over calling Failed() from Done() + * as this has e.g. the wrong progress reporting. + * + * \param Message Data from the acquire method. Use LookupTag() + * to parse it. + * \param Cnf The method via which the object was fetched. + * + * \sa pkgAcqMethod + */ + virtual bool VerifyDone(std::string const &Message, + pkgAcquire::MethodConfig const * const Cnf); + + /** \brief Invoked by the acquire worker when the object was + * fetched successfully. + * + * Note that the object might \e not have been written to + * DestFile; check for the presence of an Alt-Filename entry in + * Message to find the file to which it was really written. + * + * Done is often used to switch from one stage of the processing + * to the next (e.g. fetching, unpacking, copying). It is one + * branch of the continuation of the fetch process. + * + * \param Message Data from the acquire method. Use LookupTag() + * to parse it. + * \param Hashes The HashSums of the object that was fetched. + * \param Cnf The method via which the object was fetched. + * + * \sa pkgAcqMethod + */ + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf); + + /** \brief Invoked when the worker starts to fetch this object. + * + * \param Message RFC822-formatted data from the worker process. + * Use LookupTag() to parse it. + * + * \param Hashes The expected hashes of the object being fetched. + * + * \sa pkgAcqMethod + */ + virtual void Start(std::string const &Message, unsigned long long const Size); + + /** \brief Custom headers to be sent to the fetch process. + * + * \return a string containing RFC822-style headers that are to be + * inserted into the 600 URI Acquire message sent to the fetch + * subprocess. The headers are inserted after a newline-less + * line, so they should (if nonempty) have a leading newline and + * no trailing newline. + */ + virtual std::string Custom600Headers() const; + // this is more a hack than a proper external interface, hence hidden + APT_HIDDEN std::unordered_map<std::string, std::string> &ModifyCustomFields(); + // this isn't the super nicest interface either… + APT_HIDDEN bool PopAlternativeURI(std::string &NewURI); + APT_HIDDEN bool IsGoodAlternativeURI(std::string const &AltUri) const; + APT_HIDDEN void PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back); + APT_HIDDEN void RemoveAlternativeSite(std::string &&OldSite); + + /** \brief A "descriptive" URI-like string. + * + * \return a URI that should be used to describe what is being fetched. + */ + virtual std::string DescURI() const = 0; + /** \brief Short item description. + * + * \return a brief description of the object being fetched. + */ + virtual std::string ShortDesc() const; + + /** \brief Invoked by the worker when the download is completely done. */ + virtual void Finished(); + + /** \return HashSums the DestFile is supposed to have in this stage */ + virtual HashStringList GetExpectedHashes() const = 0; + /** \return the 'best' hash for display proposes like --print-uris */ + std::string HashSum() const; + + /** \return if having no hashes is a hard failure or not + * + * Idealy this is always \b true for every subclass, but thanks to + * historical grow we don't have hashes for all files in all cases + * in all steps, so it is slightly more complicated than it should be. + */ + virtual bool HashesRequired() const { return true; } + + /** \return the acquire process with which this item is associated. */ + pkgAcquire *GetOwner() const; + pkgAcquire::ItemDesc &GetItemDesc(); + + /** \return \b true if this object is being fetched from a trusted source. */ + virtual bool IsTrusted() const; + + /** \brief Set the name of the current active subprocess + * + * See also #ActiveSubprocess + */ + void SetActiveSubprocess(std::string const &subprocess); + + /** \brief Initialize an item. + * + * Adds the item to the list of items known to the acquire + * process, but does not place it into any fetch queues (you must + * manually invoke QueueURI() to do so). + * + * \param Owner The new owner of this item. + */ + explicit Item(pkgAcquire * const Owner); + + /** \brief Remove this item from its owner's queue by invoking + * pkgAcquire::Remove. + */ + virtual ~Item(); + + bool APT_HIDDEN IsRedirectionLoop(std::string const &NewURI); + /** \brief The priority of the item, used for queuing */ + int APT_HIDDEN Priority(); + + /** \brief internal clock definitions to avoid typing all that all over the place */ + void APT_HIDDEN FetchAfter(time_point FetchAfter); + time_point APT_HIDDEN FetchAfter(); + + protected: + /** \brief The acquire object with which this item is associated. */ + pkgAcquire * const Owner; + + /** \brief The item that is currently being downloaded. */ + pkgAcquire::ItemDesc Desc; + + enum RenameOnErrorState { + HashSumMismatch, + SizeMismatch, + InvalidFormat, + SignatureError, + NotClearsigned, + MaximumSizeExceeded, + PDiffError, + }; + + /** \brief Rename failed file and set error + * + * \param state respresenting the error we encountered + */ + bool RenameOnError(RenameOnErrorState const state); + + /** \brief Insert this item into its owner's queue. + * + * The method is designed to check if the request would end + * in an IMSHit and if it determines that it would, it isn't + * queueing the Item and instead sets it to completion instantly. + * + * \param Item Metadata about this item (its URI and + * description). + * \return true if the item was inserted, false if IMSHit was detected + */ + virtual bool QueueURI(ItemDesc &Item); + + /** \brief Remove this item from its owner's queue. */ + void Dequeue(); + + /** \brief Rename a file without modifying its timestamp. + * + * Many item methods call this as their final action. + * + * \param From The file to be renamed. + * + * \param To The new name of \a From. If \a To exists it will be + * overwritten. If \a From and \a To are equal nothing happens. + */ + bool Rename(std::string const &From, std::string const &To); + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const; + + private: + class Private; + Private * const d; + + friend class pkgAcqMetaBase; + friend class pkgAcqMetaClearSig; +}; + /*}}}*/ +class APT_HIDDEN pkgAcqTransactionItem: public pkgAcquire::Item /*{{{*/ +/** \brief baseclass for the indexes files to manage them all together */ +{ + void * const d; + protected: + HashStringList GetExpectedHashesFor(std::string const &MetaKey) const; + + bool QueueURI(pkgAcquire::ItemDesc &Item) APT_OVERRIDE; + + public: + IndexTarget const Target; + + /** \brief storge name until a transaction is finished */ + std::string PartialFile; + + /** \brief TransactionManager */ + pkgAcqMetaClearSig * const TransactionManager; + + enum TransactionStates { + TransactionStarted, + TransactionCommit, + TransactionAbort, + }; + virtual bool TransactionState(TransactionStates const state); + + virtual std::string DescURI() const APT_OVERRIDE { return Target.URI; } + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE; + virtual std::string GetMetaKey() const; + virtual bool HashesRequired() const APT_OVERRIDE; + virtual bool AcquireByHash() const; + + pkgAcqTransactionItem(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target) APT_NONNULL(2, 3); + virtual ~pkgAcqTransactionItem(); + + friend class pkgAcqMetaBase; + friend class pkgAcqMetaClearSig; +}; + /*}}}*/ +class APT_HIDDEN pkgAcqMetaBase : public pkgAcqTransactionItem /*{{{*/ +/** \brief the manager of a transaction */ +{ + void * const d; + protected: + std::vector<pkgAcqTransactionItem*> Transaction; + + /** \brief If \b true, the index's signature is currently being verified. + */ + bool AuthPass; + + /** \brief Called when a file is finished being retrieved. + * + * If the file was not downloaded to DestFile, a copy process is + * set up to copy it to DestFile; otherwise, Complete is set to \b + * true and the file is moved to its final location. + * + * \param Message The message block received from the fetch + * subprocess. + */ + bool CheckDownloadDone(pkgAcqTransactionItem * const I, const std::string &Message, HashStringList const &Hashes) const; + + /** \brief Queue the downloaded Signature for verification */ + void QueueForSignatureVerify(pkgAcqTransactionItem * const I, std::string const &File, std::string const &Signature); + + virtual std::string Custom600Headers() const APT_OVERRIDE; + + /** \brief Called when authentication succeeded. + * + * Sanity-checks the authenticated file, queues up the individual + * index files for download, and saves the signature in the lists + * directory next to the authenticated list file. + * + * \param Message The message block received from the fetch + * subprocess. + * \param Cnf The method and its configuration which handled the request + */ + bool CheckAuthDone(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf); + + /** Check if the current item should fail at this point */ + bool CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message); + + /** \brief Check that the release file is a release file for the + * correct distribution. + * + * \return \b true if no fatal errors were encountered. + */ + bool VerifyVendor(std::string const &Message); + + virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE; + + public: + // This refers more to the Transaction-Manager than the actual file + bool IMSHit; + TransactionStates State; + std::string BaseURI; + + virtual bool QueueURI(pkgAcquire::ItemDesc &Item) APT_OVERRIDE; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE; + virtual bool HashesRequired() const APT_OVERRIDE; + + // transaction code + void Add(pkgAcqTransactionItem * const I); + void AbortTransaction(); + bool TransactionHasError() const; + void CommitTransaction(); + + /** \brief Stage (queue) a copy action when the transaction is committed + */ + void TransactionStageCopy(pkgAcqTransactionItem * const I, + const std::string &From, + const std::string &To); + /** \brief Stage (queue) a removal action when the transaction is committed + */ + void TransactionStageRemoval(pkgAcqTransactionItem * const I, const std::string &FinalFile); + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + + pkgAcqMetaBase(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &DataTarget) APT_NONNULL(2, 3); + virtual ~pkgAcqMetaBase(); +}; + /*}}}*/ +/** \brief An item that is responsible for downloading the meta-index {{{ + * file (i.e., Release) itself and verifying its signature. + * + * Once the download and verification are complete, the downloads of + * the individual index files are queued up using pkgAcqDiffIndex. + * If the meta-index file had a valid signature, the expected hashsums + * of the index files will be the md5sums listed in the meta-index; + * otherwise, the expected hashsums will be "" (causing the + * authentication of the index files to be bypassed). + */ +class APT_HIDDEN pkgAcqMetaIndex : public pkgAcqMetaBase +{ + void * const d; + protected: + IndexTarget const DetachedSigTarget; + + /** \brief delayed constructor */ + void Init(std::string const &URIDesc, std::string const &ShortDesc); + + public: + virtual std::string DescURI() const APT_OVERRIDE; + + // Specialized action members + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + + /** \brief Create a new pkgAcqMetaIndex. */ + pkgAcqMetaIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &DataTarget, IndexTarget const &DetachedSigTarget) APT_NONNULL(2, 3); + virtual ~pkgAcqMetaIndex(); + + friend class pkgAcqMetaSig; +}; + /*}}}*/ +/** \brief An acquire item that downloads the detached signature {{{ + * of a meta-index (Release) file, then queues up the release + * file itself. + * + * \todo Why protected members? + * + * \sa pkgAcqMetaIndex + */ +class APT_HIDDEN pkgAcqMetaSig : public pkgAcqTransactionItem +{ + void * const d; + + pkgAcqMetaIndex * const MetaIndex; + + /** \brief The file we use to verify the MetaIndexFile with (not always set!) */ + std::string MetaIndexFileSignature; + + protected: + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + + public: + virtual bool HashesRequired() const APT_OVERRIDE { return false; } + + // Specialized action members + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + + /** \brief Create a new pkgAcqMetaSig. */ + pkgAcqMetaSig(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, pkgAcqMetaIndex * const MetaIndex) APT_NONNULL(2, 3, 5); + virtual ~pkgAcqMetaSig(); +}; + /*}}}*/ +/** \brief An item responsible for downloading clearsigned metaindexes {{{*/ +class APT_HIDDEN pkgAcqMetaClearSig : public pkgAcqMetaIndex +{ + void * const d; + IndexTarget const DetachedDataTarget; + + public: + /** \brief A package-system-specific parser for the meta-index file. */ + metaIndex *MetaIndexParser; + metaIndex *LastMetaIndexParser; + + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual bool VerifyDone(std::string const &Message, pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Finished() APT_OVERRIDE; + + /** \brief Starts downloading the individual index files. + * + * \param verify If \b true, only indices whose expected hashsum + * can be determined from the meta-index will be downloaded, and + * the hashsums of indices will be checked (reporting + * #StatAuthError if there is a mismatch). If verify is \b false, + * no hashsum checking will be performed. + */ + void QueueIndexes(bool const verify); + + /** \brief Create a new pkgAcqMetaClearSig. */ + pkgAcqMetaClearSig(pkgAcquire * const Owner, + IndexTarget const &ClearsignedTarget, + IndexTarget const &DetachedDataTarget, + IndexTarget const &DetachedSigTarget, + metaIndex * const MetaIndexParser); + virtual ~pkgAcqMetaClearSig(); +}; + /*}}}*/ +/** \brief Common base class for all classes that deal with fetching indexes {{{*/ +class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem +{ + void * const d; + + public: + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + + pkgAcqBaseIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target) APT_NONNULL(2, 3); + virtual ~pkgAcqBaseIndex(); +}; + /*}}}*/ +/** \brief An acquire item that is responsible for fetching an index {{{ + * file (e.g., Packages or Sources). + * + * \sa pkgAcqDiffIndex, pkgAcqIndexDiffs, pkgAcqIndexTrans + * + * \todo Why does pkgAcqIndex have protected members? + */ +class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex +{ + void * const d; + + protected: + + /** \brief The stages the method goes through + * + * The method first downloads the indexfile, then its decompressed (or + * copied) and verified + */ + enum AllStages { + STAGE_DOWNLOAD, + STAGE_DECOMPRESS_AND_VERIFY, + }; + AllStages Stage; + + /** \brief Handle what needs to be done when the download is done */ + void StageDownloadDone(std::string const &Message); + + /** \brief Handle what needs to be done when the decompression/copy is + * done + */ + void StageDecompressDone(); + + /** \brief If \b set, this partially downloaded file will be + * removed when the download completes. + */ + std::string EraseFileName; + + /** \brief The compression-related file extensions that are being + * added to the downloaded file one by one if first fails (e.g., "gz bz2"). + */ + std::string CompressionExtensions; + + /** \brief The actual compression extension currently used */ + std::string CurrentCompressionExtension; + + /** \brief Do the changes needed to fetch via AptByHash (if needed) */ + void InitByHashIfNeeded(); + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + + virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE; + + public: + // Specialized action members + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;}; + virtual std::string GetMetaKey() const APT_OVERRIDE; + + pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, bool const Derived = false) APT_NONNULL(2, 3); + virtual ~pkgAcqIndex(); + + protected: + APT_HIDDEN void Init(std::string const &URI, std::string const &URIDesc, + std::string const &ShortDesc); + APT_HIDDEN bool CommonFailed(std::string const &TargetURI, + std::string const &Message, pkgAcquire::MethodConfig const *const Cnf); +}; + /*}}}*/ +struct APT_HIDDEN DiffInfo { /*{{{*/ + /** The filename of the diff. */ + std::string file; + + /** The hashes of the file after the diff is applied */ + HashStringList result_hashes; + + /** The hashes of the diff */ + HashStringList patch_hashes; + + /** The hashes of the compressed diff */ + HashStringList download_hashes; +}; + /*}}}*/ +/** \brief An item that is responsible for fetching an index file of {{{ + * package list diffs and starting the package list's download. + * + * This item downloads the Index file and parses it, then enqueues + * additional downloads of either the individual patches (using + * pkgAcqIndexDiffs) or the entire Packages file (using pkgAcqIndex). + * + * \sa pkgAcqIndexDiffs, pkgAcqIndex + */ +class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqIndex +{ + void * const d; + std::vector<pkgAcqIndexMergeDiffs*> * diffs; + std::vector<DiffInfo> available_patches; + bool pdiff_merge; + + protected: + /** \brief If \b true, debugging information will be written to std::clog. */ + bool Debug; + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + + virtual bool QueueURI(pkgAcquire::ItemDesc &Item) APT_OVERRIDE; + + virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE; + public: + // Specialized action members + virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual bool VerifyDone(std::string const &Message, pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Target.URI + "Index";}; + virtual std::string GetMetaKey() const APT_OVERRIDE; + + /** \brief Parse the Index file for a set of Packages diffs. + * + * Parses the Index file and creates additional download items as + * necessary. + * + * \param IndexDiffFile The name of the Index file. + * + * \return \b true if the Index file was successfully parsed, \b + * false otherwise. + */ + bool ParseDiffIndex(std::string const &IndexDiffFile); + + /** \brief Create a new pkgAcqDiffIndex. + * + * \param Owner The Acquire object that owns this item. + * + * \param URI The URI of the list file to download. + * + * \param URIDesc A long description of the list file to download. + * + * \param ShortDesc A short description of the list file to download. + */ + pkgAcqDiffIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target) APT_NONNULL(2, 3); + virtual ~pkgAcqDiffIndex(); + private: + APT_HIDDEN void QueueOnIMSHit() const; +}; + /*}}}*/ +/** \brief An item that is responsible for fetching client-merge patches {{{ + * that need to be applied to a given package index file. + * + * Instead of downloading and applying each patch one by one like its + * sister #pkgAcqIndexDiffs this class will download all patches at once + * and call rred with all the patches downloaded once. Rred will then + * merge and apply them in one go, which should be a lot faster – but is + * incompatible with server-based merges of patches like reprepro can do. + * + * \sa pkgAcqDiffIndex, pkgAcqIndex + */ +class APT_HIDDEN pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex +{ + protected: + + /** \brief If \b true, debugging output will be written to + * std::clog. + */ + bool Debug; + + /** \brief information about the current patch */ + struct DiffInfo const patch; + + /** \brief list of all download items for the patches */ + std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches; + + /** The current status of this patch. */ + enum DiffState + { + /** \brief The diff is currently being fetched. */ + StateFetchDiff, + + /** \brief The diff is currently being applied. */ + StateApplyDiff, + + /** \brief the work with this diff is done */ + StateDoneDiff, + + /** \brief something bad happened and fallback was triggered */ + StateErrorDiff + } State; + + public: + /** \brief Called when the patch file failed to be downloaded. + * + * This method will fall back to downloading the whole index file + * outright; its arguments are ignored. + */ + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Target.URI + "Index";}; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE; + virtual bool HashesRequired() const APT_OVERRIDE; + virtual bool AcquireByHash() const APT_OVERRIDE; + + /** \brief Create an index merge-diff item. + * + * \param Owner The pkgAcquire object that owns this item. + * \param TransactionManager responsible for this item + * \param Target we intend to built via pdiff patching + * \param baseURI is the URI used for the Index, but stripped down to Target + * \param DiffInfo of the patch in question + * \param patch contains infos about the patch this item is supposed + * to download which were read from the index + * \param allPatches contains all related items so that each item can + * check if it was the last one to complete the download step + */ + pkgAcqIndexMergeDiffs(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, + IndexTarget const &Target, DiffInfo const &patch, + std::vector<pkgAcqIndexMergeDiffs *> const *const allPatches) APT_NONNULL(2, 3, 6); + virtual ~pkgAcqIndexMergeDiffs(); +}; + /*}}}*/ +/** \brief An item that is responsible for fetching server-merge patches {{{ + * that need to be applied to a given package index file. + * + * After downloading and applying a single patch, this item will + * enqueue a new pkgAcqIndexDiffs to download and apply the remaining + * patches. If no patch can be found that applies to an intermediate + * file or if one of the patches cannot be downloaded, falls back to + * downloading the entire package index file using pkgAcqIndex. + * + * \sa pkgAcqDiffIndex, pkgAcqIndex + */ +class APT_HIDDEN pkgAcqIndexDiffs : public pkgAcqBaseIndex +{ + private: + + /** \brief Queue up the next diff download. + * + * Search for the next available diff that applies to the file + * that currently exists on disk, and enqueue it by calling + * QueueURI(). + * + * \return \b true if an applicable diff was found, \b false + * otherwise. + */ + APT_HIDDEN bool QueueNextDiff(); + + /** \brief Handle tasks that must be performed after the item + * finishes downloading. + * + * Dequeues the item and checks the resulting file's hashsums + * against ExpectedHashes after the last patch was applied. + * There is no need to check the md5/sha1 after a "normal" + * patch because QueueNextDiff() will check the sha1 later. + * + * \param allDone If \b true, the file was entirely reconstructed, + * and its md5sum is verified. + */ + APT_HIDDEN void Finish(bool const allDone=false); + + protected: + + /** \brief If \b true, debugging output will be written to + * std::clog. + */ + bool Debug; + + /** The patches that remain to be downloaded, including the patch + * being downloaded right now. This list should be ordered so + * that each diff appears before any diff that depends on it. + * + * \todo These are indexed by sha1sum; why not use some sort of + * dictionary instead of relying on ordering and stripping them + * off the front? + */ + std::vector<DiffInfo> available_patches; + + /** The current status of this patch. */ + enum DiffState + { + /** \brief The diff is currently being fetched. */ + StateFetchDiff, + + /** \brief The diff is currently being applied. */ + StateApplyDiff + } State; + + public: + + /** \brief Called when the patch file failed to be downloaded. + * + * This method will fall back to downloading the whole index file + * outright; its arguments are ignored. + */ + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Target.URI + "IndexDiffs";}; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE; + virtual bool HashesRequired() const APT_OVERRIDE; + virtual bool AcquireByHash() const APT_OVERRIDE; + + /** \brief Create an index diff item. + * + * After filling in its basic fields, this invokes Finish(true) if + * \a diffs is empty, or QueueNextDiff() otherwise. + * + * \param Owner The pkgAcquire object that owns this item. + * \param TransactionManager responsible for this item + * \param Target we want to built via pdiff patching + * \param baseURI is the URI used for the Index, but stripped down to Target + * \param diffs The remaining diffs from the index of diffs. They + * should be ordered so that each diff appears before any diff + * that depends on it. + */ + pkgAcqIndexDiffs(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, + IndexTarget const &Target, + std::vector<DiffInfo> const &diffs = std::vector<DiffInfo>()) APT_NONNULL(2, 3); + virtual ~pkgAcqIndexDiffs(); +}; + /*}}}*/ +/** \brief An item that is responsible for fetching a package file. {{{ + * + * If the package file already exists in the cache, nothing will be + * done. + */ +class APT_PUBLIC pkgAcqArchive : public pkgAcquire::Item +{ + void * const d; + + bool LocalSource; + HashStringList ExpectedHashes; + + protected: + /** \brief The package version being fetched. */ + pkgCache::VerIterator Version; + + /** \brief The list of sources from which to pick archives to + * download this package from. + */ + pkgSourceList *Sources; + + /** \brief A package records object, used to look up the file + * corresponding to each version of the package. + */ + pkgRecords *Recs; + + /** \brief A location in which the actual filename of the package + * should be stored. + */ + std::string &StoreFilename; + + /** \brief \b true if this version file is being downloaded from a + * trusted source. + */ + bool Trusted; + + /** \brief Queue up the next available file for this version. */ + bool QueueNext(); + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + + public: + + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE; + virtual std::string ShortDesc() const APT_OVERRIDE; + virtual void Finished() APT_OVERRIDE; + virtual bool IsTrusted() const APT_OVERRIDE; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE; + virtual bool HashesRequired() const APT_OVERRIDE; + + /** \brief Create a new pkgAcqArchive. + * + * \param Owner The pkgAcquire object with which this item is + * associated. + * + * \param Sources The sources from which to download version + * files. + * + * \param Recs A package records object, used to look up the file + * corresponding to each version of the package. + * + * \param Version The package version to download. + * + * \param[out] StoreFilename A location in which the actual filename of + * the package should be stored. It will be set to a guessed + * basename in the constructor, and filled in with a fully + * qualified filename once the download finishes. + */ + pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sources, + pkgRecords * const Recs,pkgCache::VerIterator const &Version, + std::string &StoreFilename); + virtual ~pkgAcqArchive(); +}; + /*}}}*/ +/** \brief Retrieve the changelog for the given version {{{ + * + * Downloads the changelog to a temporary file it will also remove again + * while it is deconstructed or downloads it to a named location. + */ +class APT_PUBLIC pkgAcqChangelog : public pkgAcquire::Item +{ + class Private; + Private * const d; + std::string TemporaryDirectory; + std::string const SrcName; + std::string const SrcVersion; + + public: + // we will never have hashes for changelogs. + // If you need verified ones, download the deb and extract the changelog. + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE { return HashStringList(); } + virtual bool HashesRequired() const APT_OVERRIDE { return false; } + + // Specialized action members + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;}; + + /** returns the URI to the changelog of this version + * + * @param Ver is the version to get the changelog for + * @return the URI which will be used to acquire the changelog + */ + static std::string URI(pkgCache::VerIterator const &Ver); + + /** returns the URI to the changelog of this version + * + * \param Rls is the Release file the package comes from + * \param Component in which the package resides, can be empty + * \param SrcName is the source package name + * \param SrcVersion is the source package version + * @return the URI which will be used to acquire the changelog + */ + static std::string URI(pkgCache::RlsFileIterator const &Rls, + char const * const Component, char const * const SrcName, + char const * const SrcVersion); + + /** returns the URI to the changelog of this version + * + * \param Template URI where @CHANGEPATH@ has to be filled in + * \param Component in which the package resides, can be empty + * \param SrcName is the source package name + * \param SrcVersion is the source package version + * @return the URI which will be used to acquire the changelog + */ + static std::string URI(std::string const &Template, + char const * const Component, char const * const SrcName, + char const * const SrcVersion); + + /** returns the URI template for this release file + * + * \param Rls is a Release file + * @return the URI template to use for this release file + */ + static std::string URITemplate(pkgCache::RlsFileIterator const &Rls); + + /** \brief Create a new pkgAcqChangelog object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * \param Ver is the version to get the changelog for + * \param DestDir The directory the file should be downloaded into. + * Will be an autocreated (and cleaned up) temporary directory if not set. + * \param DestFilename The filename the file should have in #DestDir + * Defaults to sourcepackagename.changelog if not set. + */ + pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver, + std::string const &DestDir="", std::string const &DestFilename=""); + + /** \brief Create a new pkgAcqChangelog object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * \param Rls is the Release file the package comes from + * \param Component in which the package resides, can be empty + * \param SrcName is the source package name + * \param SrcVersion is the source package version + * \param DestDir The directory the file should be downloaded into. + * Will be an autocreated (and cleaned up) temporary directory if not set. + * \param DestFilename The filename the file should have in #DestDir + * Defaults to sourcepackagename.changelog if not set. + */ + pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &Rls, + char const * const Component, char const * const SrcName, char const * const SrcVersion, + std::string const &DestDir="", std::string const &DestFilename=""); + + /** \brief Create a new pkgAcqChangelog object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * \param URI is to be used to get the changelog + * \param SrcName is the source package name + * \param SrcVersion is the source package version + * \param DestDir The directory the file should be downloaded into. + * Will be an autocreated (and cleaned up) temporary directory if not set. + * \param DestFilename The filename the file should have in #DestDir + * Defaults to sourcepackagename.changelog if not set. + */ + pkgAcqChangelog(pkgAcquire * const Owner, std::string const &URI, + char const * const SrcName, char const * const SrcVersion, + std::string const &DestDir="", std::string const &DestFilename=""); + + virtual ~pkgAcqChangelog(); + +private: + APT_HIDDEN void Init(std::string const &DestDir, std::string const &DestFilename); +}; + /*}}}*/ +/** \brief Retrieve an arbitrary file to the current directory. {{{ + * + * The file is retrieved even if it is accessed via a URL type that + * normally is a NOP, such as "file". If the download fails, the + * partial file is renamed to get a ".FAILED" extension. + */ +class APT_PUBLIC pkgAcqFile : public pkgAcquire::Item +{ + void * const d; + + /** \brief Should this file be considered a index file */ + bool IsIndexFile; + + HashStringList const ExpectedHashes; + public: + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE; + virtual bool HashesRequired() const APT_OVERRIDE; + + // Specialized action members + virtual void Done(std::string const &Message, HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;}; + virtual std::string Custom600Headers() const APT_OVERRIDE; + + /** \brief Create a new pkgAcqFile object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * + * \param URI The URI to download. + * + * \param Hashes The hashsums of the file to download, if they are known; + * otherwise empty list. + * + * \param Size The size of the file to download, if it is known; + * otherwise 0. + * + * \param Desc A description of the file being downloaded. + * + * \param ShortDesc A brief description of the file being + * downloaded. + * + * \param DestDir The directory the file should be downloaded into. + * + * \param DestFilename The filename+path the file is downloaded to. + * + * \param IsIndexFile The file is considered a IndexFile and cache-control + * headers like "cache-control: max-age=0" are send + * + * If DestFilename is empty, download to DestDir/\<basename\> if + * DestDir is non-empty, $CWD/\<basename\> otherwise. If + * DestFilename is NOT empty, DestDir is ignored and DestFilename + * is the absolute name to which the file should be downloaded. + */ + + pkgAcqFile(pkgAcquire * const Owner, std::string const &URI, HashStringList const &Hashes, unsigned long long const Size, + std::string const &Desc, std::string const &ShortDesc, + std::string const &DestDir="", std::string const &DestFilename="", + bool const IsIndexFile=false); + virtual ~pkgAcqFile(); +}; + /*}}}*/ +class APT_HIDDEN pkgAcqAuxFile : public pkgAcqFile /*{{{*/ +{ + pkgAcquire::Item *const Owner; + pkgAcquire::Worker *const Worker; + unsigned long long MaximumSize; + + public: + virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual void Finished() APT_OVERRIDE; + + pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker, + std::string const &ShortDesc, std::string const &Desc, std::string const &URI, + HashStringList const &Hashes, unsigned long long const MaximumSize); + virtual ~pkgAcqAuxFile(); +}; + /*}}}*/ +/** @} */ + +#endif diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc new file mode 100644 index 0000000..0895825 --- /dev/null +++ b/apt-pkg/acquire-method.cc @@ -0,0 +1,600 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire Method + + This is a skeleton class that implements most of the functionality + of a method and some useful functions to make method implementation + simpler. The methods all derive this and specialize it. The most + complex implementation is the http method which needs to provide + pipelining, it runs the message engine at the same time it is + downloading files.. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/acquire-method.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <iostream> +#include <iterator> +#include <sstream> +#include <string> +#include <vector> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + /*}}}*/ + +using namespace std; + +// poor mans unordered_map::try_emplace for C++11 as it is a C++17 feature /*{{{*/ +template <typename Arg> +static void try_emplace(std::unordered_map<std::string, std::string> &fields, std::string &&name, Arg &&value) +{ + if (fields.find(name) == fields.end()) + fields.emplace(std::move(name), std::forward<Arg>(value)); +} + /*}}}*/ + +// AcqMethod::pkgAcqMethod - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* This constructs the initialization text */ +pkgAcqMethod::pkgAcqMethod(const char *Ver,unsigned long Flags) +{ + std::unordered_map<std::string, std::string> fields; + try_emplace(fields, "Version", Ver); + if ((Flags & SingleInstance) == SingleInstance) + try_emplace(fields, "Single-Instance", "true"); + + if ((Flags & Pipeline) == Pipeline) + try_emplace(fields, "Pipeline", "true"); + + if ((Flags & SendConfig) == SendConfig) + try_emplace(fields, "Send-Config", "true"); + + if ((Flags & LocalOnly) == LocalOnly) + try_emplace(fields, "Local-Only", "true"); + + if ((Flags & NeedsCleanup) == NeedsCleanup) + try_emplace(fields, "Needs-Cleanup", "true"); + + if ((Flags & Removable) == Removable) + try_emplace(fields, "Removable", "true"); + + if ((Flags & AuxRequests) == AuxRequests) + try_emplace(fields, "AuxRequests", "true"); + + if ((Flags & SendURIEncoded) == SendURIEncoded) + try_emplace(fields, "Send-URI-Encoded", "true"); + + SendMessage("100 Capabilities", std::move(fields)); + + SetNonBlock(STDIN_FILENO,true); + + Queue = 0; + QueueBack = 0; +} + /*}}}*/ +void pkgAcqMethod::SendMessage(std::string const &header, std::unordered_map<std::string, std::string> &&fields) /*{{{*/ +{ + auto CheckKey = [](std::string const &str) { + // Space, hyphen-minus, and alphanum are allowed for keys/headers. + return str.find_first_not_of(" -0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz") == std::string::npos; + }; + + auto CheckValue = [](std::string const &str) { + return std::all_of(str.begin(), str.end(), [](unsigned char c) -> bool { + return c > 127 // unicode + || (c > 31 && c < 127) // printable chars + || c == '\n' || c == '\t'; // special whitespace + }); + }; + + auto Error = [this]() { + _error->Error("SECURITY: Message contains control characters, rejecting."); + _error->DumpErrors(); + SendMessage("400 URI Failure", {{"URI", "<UNKNOWN>"}, {"Message", "SECURITY: Message contains control characters, rejecting."}}); + abort(); + }; + + if (!CheckKey(header)) + return Error(); + + for (auto const &f : fields) + { + if (!CheckKey(f.first)) + return Error(); + if (!CheckValue(f.second)) + return Error(); + } + + std::cout << header << '\n'; + for (auto const &f : fields) + { + if (f.second.empty()) + continue; + std::cout << f.first << ": "; + auto const lines = VectorizeString(f.second, '\n'); + if (likely(lines.empty() == false)) + { + std::copy(lines.begin(), std::prev(lines.end()), std::ostream_iterator<std::string>(std::cout, "\n ")); + std::cout << *lines.rbegin(); + } + std::cout << '\n'; + } + std::cout << '\n' + << std::flush; +} + /*}}}*/ +// AcqMethod::Fail - A fetch has failed /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMethod::Fail(bool Transient) +{ + + Fail("", Transient); +} + /*}}}*/ +// AcqMethod::Fail - A fetch has failed /*{{{*/ +void pkgAcqMethod::Fail(string Err, bool Transient) +{ + + if (not _error->empty()) + { + while (not _error->empty()) + { + std::string msg; + if (_error->PopMessage(msg)) + { + if (not Err.empty()) + Err.append("\n"); + Err.append(msg); + } + } + } + if (Err.empty()) + Err = "Undetermined Error"; + + // Strip out junk from the error messages + std::transform(Err.begin(), Err.end(), Err.begin(), [](char const c) { + if (c == '\r' || c == '\n') + return ' '; + return c; + }); + if (IP.empty() == false && _config->FindB("Acquire::Failure::ShowIP", true) == true) + Err.append(" ").append(IP); + + std::unordered_map<std::string, std::string> fields; + if (Queue != nullptr) + try_emplace(fields, "URI", Queue->Uri); + else + try_emplace(fields, "URI", "<UNKNOWN>"); + try_emplace(fields, "Message", Err); + + if(FailReason.empty() == false) + try_emplace(fields, "FailReason", FailReason); + if (UsedMirror.empty() == false) + try_emplace(fields, "UsedMirror", UsedMirror); + if (Transient == true) + try_emplace(fields, "Transient-Failure", "true"); + + SendMessage("400 URI Failure", std::move(fields)); + + if (Queue != nullptr) + Dequeue(); +} + /*}}}*/ +// AcqMethod::DropPrivsOrDie - Drop privileges or die /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMethod::DropPrivsOrDie() +{ + if (!DropPrivileges()) { + Fail(false); + exit(112); /* call the european emergency number */ + } +} + + /*}}}*/ +// AcqMethod::URIStart - Indicate a download is starting /*{{{*/ +void pkgAcqMethod::URIStart(FetchResult &Res) +{ + if (Queue == 0) + abort(); + + std::unordered_map<std::string, std::string> fields; + try_emplace(fields, "URI", Queue->Uri); + if (Res.Size != 0) + try_emplace(fields, "Size", std::to_string(Res.Size)); + if (Res.LastModified != 0) + try_emplace(fields, "Last-Modified", TimeRFC1123(Res.LastModified, true)); + if (Res.ResumePoint != 0) + try_emplace(fields, "Resume-Point", std::to_string(Res.ResumePoint)); + if (UsedMirror.empty() == false) + try_emplace(fields, "UsedMirror", UsedMirror); + + SendMessage("200 URI Start", std::move(fields)); +} + /*}}}*/ +// AcqMethod::URIDone - A URI is finished /*{{{*/ +static void printHashStringList(std::unordered_map<std::string, std::string> &fields, std::string const &Prefix, HashStringList const &list) +{ + for (auto const &hash : list) + { + // very old compatibility name for MD5Sum + if (hash.HashType() == "MD5Sum") + try_emplace(fields, Prefix + "MD5-Hash", hash.HashValue()); + try_emplace(fields, Prefix + hash.HashType() + "-Hash", hash.HashValue()); + } +} +void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt) +{ + if (Queue == 0) + abort(); + + std::unordered_map<std::string, std::string> fields; + try_emplace(fields, "URI", Queue->Uri); + if (Res.Filename.empty() == false) + try_emplace(fields, "Filename", Res.Filename); + if (Res.Size != 0) + try_emplace(fields, "Size", std::to_string(Res.Size)); + if (Res.LastModified != 0) + try_emplace(fields, "Last-Modified", TimeRFC1123(Res.LastModified, true)); + printHashStringList(fields, "", Res.Hashes); + + if (UsedMirror.empty() == false) + try_emplace(fields, "UsedMirror", UsedMirror); + if (Res.GPGVOutput.empty() == false) + { + std::ostringstream os; + std::copy(Res.GPGVOutput.begin(), Res.GPGVOutput.end() - 1, std::ostream_iterator<std::string>(os, "\n")); + os << *Res.GPGVOutput.rbegin(); + try_emplace(fields, "GPGVOutput", os.str()); + } + if (Res.ResumePoint != 0) + try_emplace(fields, "Resume-Point", std::to_string(Res.ResumePoint)); + if (Res.IMSHit == true) + try_emplace(fields, "IMS-Hit", "true"); + + if (Alt != nullptr) + { + if (Alt->Filename.empty() == false) + try_emplace(fields, "Alt-Filename", Alt->Filename); + if (Alt->Size != 0) + try_emplace(fields, "Alt-Size", std::to_string(Alt->Size)); + if (Alt->LastModified != 0) + try_emplace(fields, "Alt-Last-Modified", TimeRFC1123(Alt->LastModified, true)); + if (Alt->IMSHit == true) + try_emplace(fields, "Alt-IMS-Hit", "true"); + printHashStringList(fields, "Alt-", Alt->Hashes); + } + + SendMessage("201 URI Done", std::move(fields)); + Dequeue(); +} + /*}}}*/ +// AcqMethod::MediaFail - Synchronous request for new media /*{{{*/ +// --------------------------------------------------------------------- +/* This sends a 403 Media Failure message to the APT and waits for it + to be ackd */ +bool pkgAcqMethod::MediaFail(string Required,string Drive) +{ + fprintf(stdout, "403 Media Failure\nMedia: %s\nDrive: %s\n", + Required.c_str(),Drive.c_str()); + std::cout << "\n" << std::flush; + + vector<string> MyMessages; + + /* Here we read messages until we find a 603, each non 603 message is + appended to the main message list for later processing */ + while (1) + { + if (WaitFd(STDIN_FILENO) == false) + return false; + + if (ReadMessages(STDIN_FILENO,MyMessages) == false) + return false; + + string Message = MyMessages.front(); + MyMessages.erase(MyMessages.begin()); + + // Fetch the message number + char *End; + int Number = strtol(Message.c_str(),&End,10); + if (End == Message.c_str()) + { + cerr << "Malformed message!" << endl; + exit(100); + } + + // Change ack + if (Number == 603) + { + while (MyMessages.empty() == false) + { + Messages.push_back(MyMessages.front()); + MyMessages.erase(MyMessages.begin()); + } + + return !StringToBool(LookupTag(Message,"Failed"),false); + } + + Messages.push_back(Message); + } +} + /*}}}*/ +// AcqMethod::Configuration - Handle the configuration message /*{{{*/ +// --------------------------------------------------------------------- +/* This parses each configuration entry and puts it into the _config + Configuration class. */ +bool pkgAcqMethod::Configuration(string Message) +{ + ::Configuration &Cnf = *_config; + + const char *I = Message.c_str(); + const char *MsgEnd = I + Message.length(); + + unsigned int Length = strlen("Config-Item"); + for (; I + Length < MsgEnd; I++) + { + // Not a config item + if (I[Length] != ':' || stringcasecmp(I,I+Length,"Config-Item") != 0) + continue; + + I += Length + 1; + + for (; I < MsgEnd && *I == ' '; I++); + const char *Equals = (const char*) memchr(I, '=', MsgEnd - I); + if (Equals == NULL) + return false; + const char *End = (const char*) memchr(Equals, '\n', MsgEnd - Equals); + if (End == NULL) + End = MsgEnd; + + Cnf.Set(DeQuoteString(string(I,Equals-I)), + DeQuoteString(string(Equals+1,End-Equals-1))); + I = End; + } + + return true; +} + /*}}}*/ +// AcqMethod::Run - Run the message engine /*{{{*/ +// --------------------------------------------------------------------- +/* Fetch any messages and execute them. In single mode it returns 1 if + there are no more available messages - any other result is a + fatal failure code! */ +int pkgAcqMethod::Run(bool Single) +{ + while (1) + { + // Block if the message queue is empty + if (Messages.empty() == true) + { + if (Single == false) + if (WaitFd(STDIN_FILENO) == false) + break; + if (ReadMessages(STDIN_FILENO,Messages) == false) + break; + } + + // Single mode exits if the message queue is empty + if (Single == true && Messages.empty() == true) + return -1; + + string Message = Messages.front(); + Messages.erase(Messages.begin()); + + // Fetch the message number + char *End; + int Number = strtol(Message.c_str(),&End,10); + if (End == Message.c_str()) + { + cerr << "Malformed message!" << endl; + return 100; + } + + switch (Number) + { + case 601: + if (Configuration(Message) == false) + return 100; + break; + + case 600: + { + FetchItem *Tmp = new FetchItem; + + Tmp->Uri = LookupTag(Message,"URI"); + Tmp->Proxy(LookupTag(Message, "Proxy")); + Tmp->DestFile = LookupTag(Message,"FileName"); + if (RFC1123StrToTime(LookupTag(Message,"Last-Modified"),Tmp->LastModified) == false) + Tmp->LastModified = 0; + Tmp->IndexFile = StringToBool(LookupTag(Message,"Index-File"),false); + Tmp->FailIgnore = StringToBool(LookupTag(Message,"Fail-Ignore"),false); + Tmp->ExpectedHashes = HashStringList(); + for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t) + { + std::string tag = "Expected-"; + tag.append(*t); + std::string const hash = LookupTag(Message, tag.c_str()); + if (hash.empty() == false) + Tmp->ExpectedHashes.push_back(HashString(*t, hash)); + } + char *End; + if (Tmp->ExpectedHashes.FileSize() > 0) + Tmp->MaximumSize = Tmp->ExpectedHashes.FileSize(); + else + Tmp->MaximumSize = strtoll(LookupTag(Message, "Maximum-Size", "0").c_str(), &End, 10); + Tmp->Next = 0; + + // Append it to the list + FetchItem **I = &Queue; + for (; *I != 0; I = &(*I)->Next); + *I = Tmp; + if (QueueBack == 0) + QueueBack = Tmp; + + // Notify that this item is to be fetched. + if (URIAcquire(Message, Tmp) == false) + Fail(); + + break; + } + } + } + + Exit(); + return 0; +} + /*}}}*/ +// AcqMethod::PrintStatus - privately really send a log/status message /*{{{*/ +void pkgAcqMethod::PrintStatus(char const * const header, const char* Format, + va_list &args) const +{ + string CurrentURI = "<UNKNOWN>"; + if (Queue != 0) + CurrentURI = Queue->Uri; + if (UsedMirror.empty() == true) + fprintf(stdout, "%s\nURI: %s\nMessage: ", + header, CurrentURI.c_str()); + else + fprintf(stdout, "%s\nURI: %s\nUsedMirror: %s\nMessage: ", + header, CurrentURI.c_str(), UsedMirror.c_str()); + vfprintf(stdout,Format,args); + std::cout << "\n\n" << std::flush; +} + /*}}}*/ +// AcqMethod::Log - Send a log message /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMethod::Log(const char *Format,...) +{ + va_list args; + ssize_t size = 400; + std::ostringstream outstr; + while (true) { + bool ret; + va_start(args,Format); + ret = iovprintf(outstr, Format, args, size); + va_end(args); + if (ret == true) + break; + } + std::unordered_map<std::string, std::string> fields; + if (Queue != 0) + try_emplace(fields, "URI", Queue->Uri); + else + try_emplace(fields, "URI", "<UNKNOWN>"); + if (not UsedMirror.empty()) + try_emplace(fields, "UsedMirror", UsedMirror); + try_emplace(fields, "Message", outstr.str()); + SendMessage("101 Log", std::move(fields)); +} + /*}}}*/ +// AcqMethod::Status - Send a status message /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMethod::Status(const char *Format,...) +{ + va_list args; + ssize_t size = 400; + std::ostringstream outstr; + while (true) { + bool ret; + va_start(args,Format); + ret = iovprintf(outstr, Format, args, size); + va_end(args); + if (ret == true) + break; + } + std::unordered_map<std::string, std::string> fields; + if (Queue != 0) + try_emplace(fields, "URI", Queue->Uri); + else + try_emplace(fields, "URI", "<UNKNOWN>"); + if (not UsedMirror.empty()) + try_emplace(fields, "UsedMirror", UsedMirror); + try_emplace(fields, "Message", outstr.str()); + SendMessage("102 Status", std::move(fields)); +} + /*}}}*/ +// AcqMethod::Redirect - Send a redirect message /*{{{*/ +// --------------------------------------------------------------------- +/* This method sends the redirect message and dequeues the item as + * the worker will enqueue again later on to the right queue */ +void pkgAcqMethod::Redirect(const string &NewURI) +{ + if (NewURI.find_first_not_of(" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~") != std::string::npos) + { + _error->Error("SECURITY: URL redirect target contains control characters, rejecting."); + Fail(); + return; + } + std::unordered_map<std::string, std::string> fields; + try_emplace(fields, "URI", Queue->Uri); + try_emplace(fields, "New-URI", NewURI); + SendMessage("103 Redirect", std::move(fields)); + Dequeue(); +} + /*}}}*/ +// AcqMethod::FetchResult::FetchResult - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgAcqMethod::FetchResult::FetchResult() : LastModified(0), + IMSHit(false), Size(0), ResumePoint(0), d(NULL) +{ +} + /*}}}*/ +// AcqMethod::FetchResult::TakeHashes - Load hashes /*{{{*/ +// --------------------------------------------------------------------- +/* This hides the number of hashes we are supporting from the caller. + It just deals with the hash class. */ +void pkgAcqMethod::FetchResult::TakeHashes(class Hashes &Hash) +{ + Hashes = Hash.GetHashStringList(); +} + /*}}}*/ +void pkgAcqMethod::Dequeue() { /*{{{*/ + FetchItem const * const Tmp = Queue; + Queue = Queue->Next; + if (Tmp == QueueBack) + QueueBack = Queue; + delete Tmp; +} + /*}}}*/ +pkgAcqMethod::~pkgAcqMethod() {} + +struct pkgAcqMethod::FetchItem::Private +{ + std::string Proxy; +}; + +pkgAcqMethod::FetchItem::FetchItem() : Next(nullptr), DestFileFd(-1), LastModified(0), IndexFile(false), + FailIgnore(false), MaximumSize(0), d(new Private) +{} + +std::string pkgAcqMethod::FetchItem::Proxy() +{ + return d->Proxy; +} + +void pkgAcqMethod::FetchItem::Proxy(std::string const &Proxy) +{ + d->Proxy = Proxy; +} + +pkgAcqMethod::FetchItem::~FetchItem() { delete d; } + +pkgAcqMethod::FetchResult::~FetchResult() {} diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h new file mode 100644 index 0000000..b4b238c --- /dev/null +++ b/apt-pkg/acquire-method.h @@ -0,0 +1,137 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire Method - Method helper class + functions + + These functions are designed to be used within the method task to + ease communication with APT. + + ##################################################################### */ + /*}}}*/ + +/** \addtogroup acquire + * @{ + * + * \file acquire-method.h + */ + +#ifndef PKGLIB_ACQUIRE_METHOD_H +#define PKGLIB_ACQUIRE_METHOD_H + +#include <apt-pkg/hashes.h> +#include <apt-pkg/macros.h> + +#include <stdarg.h> +#include <time.h> + +#include <string> +#include <unordered_map> +#include <vector> + + +class APT_PUBLIC pkgAcqMethod +{ + protected: + + struct FetchItem + { + FetchItem *Next; + + std::string Uri; + std::string DestFile; + int DestFileFd; + time_t LastModified; + bool IndexFile; + bool FailIgnore; + HashStringList ExpectedHashes; + // a maximum size we will download, this can be the exact filesize + // for when we know it or a arbitrary limit when we don't know the + // filesize (like a InRelease file) + unsigned long long MaximumSize; + + FetchItem(); + virtual ~FetchItem(); + std::string Proxy(); // For internal use only. + void Proxy(std::string const &Proxy) APT_HIDDEN; + + private: + struct Private; + Private *const d; + }; + + struct FetchResult + { + HashStringList Hashes; + std::vector<std::string> GPGVOutput; + time_t LastModified; + bool IMSHit; + std::string Filename; + unsigned long long Size; + unsigned long long ResumePoint; + + void TakeHashes(class Hashes &Hash); + FetchResult(); + virtual ~FetchResult(); + private: + void * const d; + }; + + // State + std::vector<std::string> Messages; + FetchItem *Queue; + FetchItem *QueueBack; + std::string FailReason; + std::string UsedMirror; + std::string IP; + + // Handlers for messages + virtual bool Configuration(std::string Message); + virtual bool Fetch(FetchItem * /*Item*/) {return true;}; + virtual bool URIAcquire(std::string const &/*Message*/, FetchItem *Itm) { return Fetch(Itm); }; + + // Outgoing messages + void Fail(bool Transient = false); + inline void Fail(const char *Why, bool Transient = false) {Fail(std::string(Why),Transient);}; + virtual void Fail(std::string Why, bool Transient = false); + virtual void URIStart(FetchResult &Res); + virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + void SendMessage(std::string const &header, std::unordered_map<std::string, std::string> &&fields); + + bool MediaFail(std::string Required,std::string Drive); + virtual void Exit() {}; + + APT_DEPRECATED_MSG("Use SendMessage instead") void PrintStatus(char const * const header, const char* Format, va_list &args) const; + + public: + enum CnfFlags + { + SingleInstance = (1 << 0), + Pipeline = (1 << 1), + SendConfig = (1 << 2), + LocalOnly = (1 << 3), + NeedsCleanup = (1 << 4), + Removable = (1 << 5), + AuxRequests = (1 << 6), + SendURIEncoded = (1 << 7), + }; + + void Log(const char *Format,...); + void Status(const char *Format,...); + + void Redirect(const std::string &NewURI); + + int Run(bool Single = false); + inline void SetFailReason(std::string Msg) {FailReason = Msg;}; + inline void SetIP(std::string aIP) {IP = aIP;}; + + pkgAcqMethod(const char *Ver,unsigned long Flags = 0); + virtual ~pkgAcqMethod(); + void DropPrivsOrDie(); + private: + APT_HIDDEN void Dequeue(); +}; + +/** @} */ + +#endif diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc new file mode 100644 index 0000000..1bf07e8 --- /dev/null +++ b/apt-pkg/acquire-worker.cc @@ -0,0 +1,1019 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire Worker + + The worker process can startup either as a Configuration prober + or as a queue runner. As a configuration prober it only reads the + configuration message and + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/acquire-item.h> +#include <apt-pkg/acquire-worker.h> +#include <apt-pkg/acquire.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/proxy.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <iostream> +#include <string> +#include <vector> + +#include <sstream> +#include <errno.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +// Worker::Worker - Constructor for Queue startup /*{{{*/ +pkgAcquire::Worker::Worker(Queue *Q, MethodConfig *Cnf, pkgAcquireStatus *log) : + d(NULL), OwnerQ(Q), Log(log), Config(Cnf), Access(Cnf->Access), + CurrentItem(nullptr) +{ + Construct(); +} + /*}}}*/ +// Worker::Worker - Constructor for method config startup /*{{{*/ +pkgAcquire::Worker::Worker(MethodConfig *Cnf) : Worker(nullptr, Cnf, nullptr) +{ +} + /*}}}*/ +// Worker::Construct - Constructor helper /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcquire::Worker::Construct() +{ + NextQueue = 0; + NextAcquire = 0; + Process = -1; + InFd = -1; + OutFd = -1; + OutReady = false; + InReady = false; + Debug = _config->FindB("Debug::pkgAcquire::Worker",false); +} + /*}}}*/ +// Worker::~Worker - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgAcquire::Worker::~Worker() +{ + close(InFd); + close(OutFd); + + if (Process > 0) + { + /* Closing of stdin is the signal to exit and die when the process + indicates it needs cleanup */ + if (Config->NeedsCleanup == false) + kill(Process,SIGINT); + ExecWait(Process,Access.c_str(),true); + } +} + /*}}}*/ +// Worker::Start - Start the worker process /*{{{*/ +// --------------------------------------------------------------------- +/* This forks the method and inits the communication channel */ +bool pkgAcquire::Worker::Start() +{ + // Get the method path + constexpr char const * const methodsDir = "Dir::Bin::Methods"; + std::string const confItem = std::string(methodsDir) + "::" + Access; + std::string Method; + if (_config->Exists(confItem)) + Method = _config->FindFile(confItem.c_str()); + else if (Access == "ftp" || Access == "rsh" || Access == "ssh") + return _error->Error(_("The method '%s' is unsupported and disabled by default. Consider switching to http(s). Set Dir::Bin::Methods::%s to \"%s\" to enable it again."), Access.c_str(), Access.c_str(), Access.c_str()); + else + Method = _config->FindDir(methodsDir) + Access; + if (FileExists(Method) == false) + { + if (flNotDir(Method) == "false") + { + _error->Error(_("The method '%s' is explicitly disabled via configuration."), Access.c_str()); + if (Access == "http" || Access == "https") + _error->Notice(_("If you meant to use Tor remember to use %s instead of %s."), ("tor+" + Access).c_str(), Access.c_str()); + return false; + } + _error->Error(_("The method driver %s could not be found."),Method.c_str()); + std::string const A(Access.cbegin(), std::find(Access.cbegin(), Access.cend(), '+')); + std::string pkg; + strprintf(pkg, "apt-transport-%s", A.c_str()); + _error->Notice(_("Is the package %s installed?"), pkg.c_str()); + return false; + } + std::string const Calling = _config->FindDir(methodsDir) + Access; + + if (Debug == true) + { + std::clog << "Starting method '" << Calling << "'"; + if (Calling != Method) + std::clog << " ( via " << Method << " )"; + std::clog << endl; + } + + // Create the pipes + int Pipes[4] = {-1,-1,-1,-1}; + if (pipe(Pipes) != 0 || pipe(Pipes+2) != 0) + { + _error->Errno("pipe","Failed to create IPC pipe to subprocess"); + for (int I = 0; I != 4; I++) + close(Pipes[I]); + return false; + } + for (int I = 0; I != 4; I++) + SetCloseExec(Pipes[I],true); + + // Fork off the process + Process = ExecFork(); + if (Process == 0) + { + // Setup the FDs + dup2(Pipes[1],STDOUT_FILENO); + dup2(Pipes[2],STDIN_FILENO); + SetCloseExec(STDOUT_FILENO,false); + SetCloseExec(STDIN_FILENO,false); + SetCloseExec(STDERR_FILENO,false); + + const char * const Args[] = { Calling.c_str(), nullptr }; + execv(Method.c_str() ,const_cast<char **>(Args)); + std::cerr << "Failed to exec method " << Calling << " ( via " << Method << ")" << endl; + _exit(100); + } + + // Fix up our FDs + InFd = Pipes[0]; + OutFd = Pipes[3]; + SetNonBlock(Pipes[0],true); + SetNonBlock(Pipes[3],true); + close(Pipes[1]); + close(Pipes[2]); + OutReady = false; + InReady = true; + + // Read the configuration data + if (WaitFd(InFd) == false || + ReadMessages() == false) + return _error->Error(_("Method %s did not start correctly"),Method.c_str()); + + RunMessages(); + if (OwnerQ != 0) + SendConfiguration(); + + return true; +} + /*}}}*/ +// Worker::ReadMessages - Read all pending messages into the list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgAcquire::Worker::ReadMessages() +{ + if (::ReadMessages(InFd,MessageQueue) == false) + return MethodFailure(); + return true; +} + /*}}}*/ +// Worker::RunMessage - Empty the message queue /*{{{*/ +// --------------------------------------------------------------------- +/* This takes the messages from the message queue and runs them through + the parsers in order. */ +enum class APT_HIDDEN MessageType +{ + CAPABILITIES = 100, + LOG = 101, + STATUS = 102, + REDIRECT = 103, + WARNING = 104, + URI_START = 200, + URI_DONE = 201, + AUX_REQUEST = 351, + URI_FAILURE = 400, + GENERAL_FAILURE = 401, + MEDIA_CHANGE = 403 +}; +static bool isDoomedItem(pkgAcquire::Item const * const Itm) +{ + auto const TransItm = dynamic_cast<pkgAcqTransactionItem const * const>(Itm); + if (TransItm == nullptr) + return false; + return TransItm->TransactionManager->State != pkgAcqTransactionItem::TransactionStarted; +} +static HashStringList GetHashesFromMessage(std::string const &Prefix, std::string const &Message) +{ + HashStringList hsl; + for (char const *const *type = HashString::SupportedHashes(); *type != NULL; ++type) + { + std::string const tagname = Prefix + *type + "-Hash"; + std::string const hashsum = LookupTag(Message, tagname.c_str()); + if (hashsum.empty() == false) + hsl.push_back(HashString(*type, hashsum)); + } + return hsl; +} +static void APT_NONNULL(3) ChangeSiteIsMirrorChange(std::string const &NewURI, pkgAcquire::ItemDesc &desc, pkgAcquire::Item *const Owner) +{ + if (URI::SiteOnly(NewURI) == URI::SiteOnly(desc.URI)) + return; + + auto const firstSpace = desc.Description.find(" "); + if (firstSpace != std::string::npos) + { + std::string const OldSite = desc.Description.substr(0, firstSpace); + if (likely(APT::String::Startswith(desc.URI, OldSite))) + { + std::string const OldExtra = desc.URI.substr(OldSite.length() + 1); + if (likely(APT::String::Endswith(NewURI, OldExtra))) + { + std::string const NewSite = NewURI.substr(0, NewURI.length() - OldExtra.length()); + Owner->UsedMirror = URI::ArchiveOnly(NewSite); + desc.Description.replace(0, firstSpace, Owner->UsedMirror); + } + } + } +} +bool pkgAcquire::Worker::RunMessages() +{ + while (MessageQueue.empty() == false) + { + string Message = MessageQueue.front(); + MessageQueue.erase(MessageQueue.begin()); + + if (Debug == true) + clog << " <- " << Access << ':' << QuoteString(Message,"\n") << endl; + + // Fetch the message number + char *End; + MessageType const Number = static_cast<MessageType>(strtoul(Message.c_str(),&End,10)); + if (End == Message.c_str()) + return _error->Error("Invalid message from method %s: %s",Access.c_str(),Message.c_str()); + + string URI = LookupTag(Message,"URI"); + pkgAcquire::Queue::QItem *Itm = NULL; + if (URI.empty() == false) + Itm = OwnerQ->FindItem(URI,this); + + if (Itm != NULL) + { + // update used mirror + string UsedMirror = LookupTag(Message,"UsedMirror", ""); + if (UsedMirror.empty() == false) + { + for (pkgAcquire::Queue::QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O) + (*O)->UsedMirror = UsedMirror; + + if (Itm->Description.find(" ") != string::npos) + Itm->Description.replace(0, Itm->Description.find(" "), UsedMirror); + } + } + + // Determine the message number and dispatch + switch (Number) + { + case MessageType::CAPABILITIES: + if (Capabilities(Message) == false) + return _error->Error("Unable to process Capabilities message from %s",Access.c_str()); + break; + + case MessageType::LOG: + if (Debug == true) + clog << " <- (log) " << LookupTag(Message,"Message") << endl; + break; + + case MessageType::STATUS: + Status = LookupTag(Message,"Message"); + break; + + case MessageType::REDIRECT: + { + if (Itm == nullptr) + { + _error->Error("Method gave invalid 103 Redirect message"); + break; + } + + std::string const GotNewURI = LookupTag(Message,"New-URI",URI.c_str()); + if (Config->GetSendURIEncoded()) + Itm->URI = GotNewURI; + else + { + ::URI tmpuri{GotNewURI}; + tmpuri.Path = pkgAcquire::URIEncode(tmpuri.Path); + Itm->URI = tmpuri; + } + auto NewURI = Itm->URI; + + auto const AltUris = VectorizeString(LookupTag(Message, "Alternate-URIs"), '\n'); + + ItemDone(); + + // Change the status so that it can be dequeued + for (auto const &O: Itm->Owners) + O->Status = pkgAcquire::Item::StatIdle; + // Mark the item as done (taking care of all queues) + // and then put it in the main queue again + std::vector<Item*> const ItmOwners = Itm->Owners; + OwnerQ->ItemDone(Itm); + Itm = nullptr; + for (auto const &Owner: ItmOwners) + { + pkgAcquire::ItemDesc &desc = Owner->GetItemDesc(); + + // for a simplified retry a method might redirect without URI change + // see also IsRedirectionLoop implementation + bool simpleRetry = false; + if (Config->GetSendURIEncoded()) + { + for (auto alt = AltUris.crbegin(); alt != AltUris.crend(); ++alt) + Owner->PushAlternativeURI(std::string(*alt), {}, false); + if (desc.URI == GotNewURI) + simpleRetry = true; + } + else + { + for (auto alt = AltUris.crbegin(); alt != AltUris.crend(); ++alt) + { + ::URI tmpuri{*alt}; + tmpuri.Path = pkgAcquire::URIEncode(tmpuri.Path); + Owner->PushAlternativeURI(std::string(tmpuri), {}, false); + } + ::URI tmpuri{desc.URI}; + tmpuri.Path = DeQuoteString(tmpuri.Path); + if (GotNewURI == std::string(tmpuri)) + simpleRetry = true; + } + + if (not simpleRetry) + { + if (Owner->IsGoodAlternativeURI(NewURI) == false && Owner->PopAlternativeURI(NewURI) == false) + NewURI.clear(); + if (NewURI.empty() || Owner->IsRedirectionLoop(NewURI)) + { + std::string msg = Message; + msg.append("\nFailReason: RedirectionLoop"); + Owner->Failed(msg, Config); + if (Log != nullptr) + Log->Fail(Owner->GetItemDesc()); + continue; + } + + if (Log != nullptr) + Log->Done(desc); + + ChangeSiteIsMirrorChange(NewURI, desc, Owner); + desc.URI = NewURI; + } + if (isDoomedItem(Owner) == false) + OwnerQ->Owner->Enqueue(desc); + } + break; + } + + case MessageType::WARNING: + _error->Warning("%s: %s", Itm ? Itm->Owner ? Itm->Owner->DescURI().c_str() : Access.c_str() : Access.c_str(), LookupTag(Message, "Message").c_str()); + break; + + case MessageType::URI_START: + { + if (Itm == nullptr) + { + _error->Error("Method gave invalid 200 URI Start message"); + break; + } + + CurrentItem = Itm; + Itm->CurrentSize = 0; + Itm->TotalSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10); + Itm->ResumePoint = strtoull(LookupTag(Message,"Resume-Point","0").c_str(), NULL, 10); + for (auto const Owner: Itm->Owners) + { + Owner->Start(Message, Itm->TotalSize); + // Display update before completion + if (Log != nullptr) + { + if (Log->MorePulses == true) + Log->Pulse(Owner->GetOwner()); + Log->Fetch(Owner->GetItemDesc()); + } + } + + break; + } + + case MessageType::URI_DONE: + { + if (Itm == nullptr) + { + _error->Error("Method gave invalid 201 URI Done message"); + break; + } + + PrepareFiles("201::URIDone", Itm); + + // Display update before completion + if (Log != 0 && Log->MorePulses == true) + for (pkgAcquire::Queue::QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O) + Log->Pulse((*O)->GetOwner()); + + HashStringList ReceivedHashes; + { + std::string const givenfilename = LookupTag(Message, "Filename"); + std::string const filename = givenfilename.empty() ? Itm->Owner->DestFile : givenfilename; + // see if we got hashes to verify + ReceivedHashes = GetHashesFromMessage("", Message); + // not all methods always sent Hashes our way + if (ReceivedHashes.usable() == false) + { + HashStringList const ExpectedHashes = Itm->GetExpectedHashes(); + if (ExpectedHashes.usable() == true && RealFileExists(filename)) + { + Hashes calc(ExpectedHashes); + FileFd file(filename, FileFd::ReadOnly, FileFd::None); + calc.AddFD(file); + ReceivedHashes = calc.GetHashStringList(); + } + } + + // only local files can refer other filenames and counting them as fetched would be unfair + if (Log != NULL && Itm->Owner->Complete == false && Itm->Owner->Local == false && givenfilename == filename) + Log->Fetched(ReceivedHashes.FileSize(),atoi(LookupTag(Message,"Resume-Point","0").c_str())); + } + + std::vector<Item*> const ItmOwners = Itm->Owners; + OwnerQ->ItemDone(Itm); + Itm = NULL; + + bool const isIMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false) || + StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false); + auto const forcedHash = _config->Find("Acquire::ForceHash"); + for (auto const Owner: ItmOwners) + { + HashStringList const ExpectedHashes = Owner->GetExpectedHashes(); + if(_config->FindB("Debug::pkgAcquire::Auth", false) == true) + { + std::clog << "201 URI Done: " << Owner->DescURI() << endl + << "ReceivedHash:" << endl; + for (HashStringList::const_iterator hs = ReceivedHashes.begin(); hs != ReceivedHashes.end(); ++hs) + std::clog << "\t- " << hs->toStr() << std::endl; + std::clog << "ExpectedHash:" << endl; + for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs) + std::clog << "\t- " << hs->toStr() << std::endl; + std::clog << endl; + } + + // decide if what we got is what we expected + bool consideredOkay = false; + if ((forcedHash.empty() && ExpectedHashes.empty() == false) || + (forcedHash.empty() == false && ExpectedHashes.usable())) + { + if (ReceivedHashes.empty()) + { + /* IMS-Hits can't be checked here as we will have uncompressed file, + but the hashes for the compressed file. What we have was good through + so all we have to ensure later is that we are not stalled. */ + consideredOkay = isIMSHit; + } + else if (ReceivedHashes == ExpectedHashes) + consideredOkay = true; + else + consideredOkay = false; + + } + else + consideredOkay = !Owner->HashesRequired(); + + if (consideredOkay == true) + consideredOkay = Owner->VerifyDone(Message, Config); + else // hashsum mismatch + Owner->Status = pkgAcquire::Item::StatAuthError; + + + if (consideredOkay == true) + { + if (isDoomedItem(Owner) == false) + Owner->Done(Message, ReceivedHashes, Config); + if (Log != nullptr) + { + if (isIMSHit) + Log->IMSHit(Owner->GetItemDesc()); + else + Log->Done(Owner->GetItemDesc()); + } + } + else + { + auto SavedDesc = Owner->GetItemDesc(); + if (isDoomedItem(Owner) == false) + { + if (Message.find("\nFailReason:") == std::string::npos) + { + if (ReceivedHashes != ExpectedHashes) + Message.append("\nFailReason: HashSumMismatch"); + else + Message.append("\nFailReason: WeakHashSums"); + } + Owner->Failed(Message,Config); + } + if (Log != nullptr) + Log->Fail(SavedDesc); + } + } + ItemDone(); + break; + } + + case MessageType::AUX_REQUEST: + { + if (Itm == nullptr) + { + _error->Error("Method gave invalid Aux Request message"); + break; + } + else if (Config->GetAuxRequests() == false) + { + std::vector<Item *> const ItmOwners = Itm->Owners; + Message.append("\nMessage: Method tried to make an Aux Request while not being allowed to do them"); + OwnerQ->ItemDone(Itm); + Itm = nullptr; + HandleFailure(ItmOwners, Config, Log, Message, false, false); + ItemDone(); + + std::string Msg = "600 URI Acquire\n"; + Msg.reserve(200); + Msg += "URI: " + LookupTag(Message, "Aux-URI", ""); + Msg += "\nFilename: /nonexistent/auxrequest.blocked"; + Msg += "\n\n"; + if (Debug == true) + clog << " -> " << Access << ':' << QuoteString(Msg, "\n") << endl; + OutQueue += Msg; + OutReady = true; + break; + } + + auto maxsizestr = LookupTag(Message, "MaximumSize", ""); + unsigned long long const MaxSize = maxsizestr.empty() ? 0 : strtoull(maxsizestr.c_str(), nullptr, 10); + new pkgAcqAuxFile(Itm->Owner, this, LookupTag(Message, "Aux-ShortDesc", ""), + LookupTag(Message, "Aux-Description", ""), LookupTag(Message, "Aux-URI", ""), + GetHashesFromMessage("Aux-", Message), MaxSize); + break; + } + + case MessageType::URI_FAILURE: + { + if (Itm == nullptr) + { + std::string const msg = LookupTag(Message,"Message"); + _error->Error("Method gave invalid 400 URI Failure message: %s", msg.c_str()); + break; + } + + PrepareFiles("400::URIFailure", Itm); + + // Display update before completion + if (Log != nullptr && Log->MorePulses == true) + for (pkgAcquire::Queue::QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O) + Log->Pulse((*O)->GetOwner()); + + std::vector<Item*> const ItmOwners = Itm->Owners; + OwnerQ->ItemDone(Itm); + Itm = nullptr; + + bool errTransient = false, errAuthErr = false; + if (StringToBool(LookupTag(Message, "Transient-Failure"), false) == true) + errTransient = true; + else + { + std::string const failReason = LookupTag(Message, "FailReason"); + { + auto const reasons = { "Timeout", "ConnectionRefused", + "ConnectionTimedOut", "ResolveFailure", "TmpResolveFailure" }; + errTransient = std::find(std::begin(reasons), std::end(reasons), failReason) != std::end(reasons); + } + if (errTransient == false) + { + auto const reasons = { "HashSumMismatch", "WeakHashSums", "MaximumSizeExceeded" }; + errAuthErr = std::find(std::begin(reasons), std::end(reasons), failReason) != std::end(reasons); + } + } + HandleFailure(ItmOwners, Config, Log, Message, errTransient, errAuthErr); + ItemDone(); + + break; + } + + case MessageType::GENERAL_FAILURE: + _error->Error("Method %s General failure: %s",Access.c_str(),LookupTag(Message,"Message").c_str()); + break; + + case MessageType::MEDIA_CHANGE: + MediaChange(Message); + break; + } + } + return true; +} + /*}}}*/ +void pkgAcquire::Worker::HandleFailure(std::vector<pkgAcquire::Item *> const &ItmOwners, /*{{{*/ + pkgAcquire::MethodConfig *const Config, pkgAcquireStatus *const Log, + std::string const &Message, bool const errTransient, bool const errAuthErr) +{ + auto currentTime = clock::now(); + for (auto const Owner : ItmOwners) + { + std::string NewURI; + if (errTransient == true && Config->LocalOnly == false && Owner->Retries != 0) + { + --Owner->Retries; + Owner->FailMessage(Message); + auto SavedDesc = Owner->GetItemDesc(); + if (_config->FindB("Acquire::Retries::Delay", true)) + { + auto Iter = _config->FindI("Acquire::Retries", 3) - Owner->Retries - 1; + auto const MaxDur = _config->FindI("Acquire::Retries::Delay::Maximum", 30); + auto Dur = std::chrono::seconds(std::min(1 << Iter, MaxDur)); + if (_config->FindB("Debug::Acquire::Retries")) + std::clog << "Delaying " << SavedDesc.Description << " by " << Dur.count() << " seconds" << std::endl; + Owner->FetchAfter(currentTime + Dur); + } + else + { + Owner->FetchAfter(currentTime); + } + + if (Log != nullptr) + Log->Fail(SavedDesc); + if (isDoomedItem(Owner) == false) + OwnerQ->Owner->Enqueue(SavedDesc); + } + else + { + if (errAuthErr) + Owner->RemoveAlternativeSite(URI::SiteOnly(Owner->GetItemDesc().URI)); + if (Owner->PopAlternativeURI(NewURI)) + { + Owner->FailMessage(Message); + auto &desc = Owner->GetItemDesc(); + if (Log != nullptr) + Log->Fail(desc); + ChangeSiteIsMirrorChange(NewURI, desc, Owner); + desc.URI = NewURI; + if (isDoomedItem(Owner) == false) + OwnerQ->Owner->Enqueue(desc); + } + else + { + if (errAuthErr && Owner->GetExpectedHashes().empty() == false) + Owner->Status = pkgAcquire::Item::StatAuthError; + else if (errTransient) + Owner->Status = pkgAcquire::Item::StatTransientNetworkError; + auto SavedDesc = Owner->GetItemDesc(); + if (isDoomedItem(Owner) == false) + Owner->Failed(Message, Config); + if (Log != nullptr) + Log->Fail(SavedDesc); + } + } + } +} + /*}}}*/ +// Worker::Capabilities - 100 Capabilities handler /*{{{*/ +// --------------------------------------------------------------------- +/* This parses the capabilities message and dumps it into the configuration + structure. */ +bool pkgAcquire::Worker::Capabilities(string Message) +{ + if (Config == 0) + return true; + + Config->Version = LookupTag(Message,"Version"); + Config->SingleInstance = StringToBool(LookupTag(Message,"Single-Instance"),false); + Config->Pipeline = StringToBool(LookupTag(Message,"Pipeline"),false); + Config->SendConfig = StringToBool(LookupTag(Message,"Send-Config"),false); + Config->LocalOnly = StringToBool(LookupTag(Message,"Local-Only"),false); + Config->NeedsCleanup = StringToBool(LookupTag(Message,"Needs-Cleanup"),false); + Config->Removable = StringToBool(LookupTag(Message,"Removable"),false); + Config->SetAuxRequests(StringToBool(LookupTag(Message, "AuxRequests"), false)); + if (_config->FindB("Acquire::Send-URI-Encoded", true)) + Config->SetSendURIEncoded(StringToBool(LookupTag(Message, "Send-URI-Encoded"), false)); + + // Some debug text + if (Debug == true) + { + clog << "Configured access method " << Config->Access << endl; + clog << "Version:" << Config->Version << " SingleInstance:" << Config->SingleInstance + << " Pipeline:" << Config->Pipeline << " SendConfig:" << Config->SendConfig + << " LocalOnly: " << Config->LocalOnly << " NeedsCleanup: " << Config->NeedsCleanup + << " Removable: " << Config->Removable << " AuxRequests: " << Config->GetAuxRequests() + << " SendURIEncoded: " << Config->GetSendURIEncoded() << '\n'; + } + + return true; +} + /*}}}*/ +// Worker::MediaChange - Request a media change /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgAcquire::Worker::MediaChange(string Message) +{ + int status_fd = _config->FindI("APT::Status-Fd",-1); + if(status_fd > 0) + { + string Media = LookupTag(Message,"Media"); + string Drive = LookupTag(Message,"Drive"); + ostringstream msg,status; + ioprintf(msg,_("Please insert the disc labeled: " + "'%s' " + "in the drive '%s' and press [Enter]."), + Media.c_str(),Drive.c_str()); + status << "media-change: " // message + << Media << ":" // media + << Drive << ":" // drive + << msg.str() // l10n message + << endl; + + std::string const dlstatus = status.str(); + FileFd::Write(status_fd, dlstatus.c_str(), dlstatus.size()); + } + + if (Log == 0 || Log->MediaChange(LookupTag(Message,"Media"), + LookupTag(Message,"Drive")) == false) + { + char S[300]; + snprintf(S,sizeof(S),"603 Media Changed\nFailed: true\n\n"); + if (Debug == true) + clog << " -> " << Access << ':' << QuoteString(S,"\n") << endl; + OutQueue += S; + OutReady = true; + return true; + } + + char S[300]; + snprintf(S,sizeof(S),"603 Media Changed\n\n"); + if (Debug == true) + clog << " -> " << Access << ':' << QuoteString(S,"\n") << endl; + OutQueue += S; + OutReady = true; + return true; +} + /*}}}*/ +// Worker::SendConfiguration - Send the config to the method /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgAcquire::Worker::SendConfiguration() +{ + if (Config->SendConfig == false) + return true; + + if (OutFd == -1) + return false; + + /* Write out all of the configuration directives by walking the + configuration tree */ + std::ostringstream Message; + Message << "601 Configuration\n"; + if (not _config->Exists("Acquire::Send-URI-Encoded")) + Message << "Config-Item: Acquire::Send-URI-Encoded=1\n"; + _config->Dump(Message, NULL, "Config-Item: %F=%V\n", false); + Message << '\n'; + + if (Debug == true) + clog << " -> " << Access << ':' << QuoteString(Message.str(),"\n") << endl; + OutQueue += Message.str(); + OutReady = true; + + return true; +} + /*}}}*/ +// Worker::QueueItem - Add an item to the outbound queue /*{{{*/ +// --------------------------------------------------------------------- +/* Send a URI Acquire message to the method */ +bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) +{ + if (OutFd == -1) + return false; + + if (isDoomedItem(Item->Owner)) + return true; + + Item->SyncDestinationFiles(); + + string Message = "600 URI Acquire\n"; + Message.reserve(300); + URI URL(Item->URI); + if (Config->GetSendURIEncoded()) + Message += "URI: " + Item->URI; + else + { + URL.Path = DeQuoteString(URL.Path); + Message += "URI: " + std::string(URL); + } + Message += "\nFilename: " + Item->Owner->DestFile; + + // FIXME: We should not hard code proxy protocols here. + if (URL.Access == "http" || URL.Access == "https") + { + AutoDetectProxy(URL); + if (_config->Exists("Acquire::" + URL.Access + "::proxy::" + URL.Host)) + { + Message += "\nProxy: " + _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host); + } + } + + HashStringList const hsl = Item->GetExpectedHashes(); + for (HashStringList::const_iterator hs = hsl.begin(); hs != hsl.end(); ++hs) + Message += "\nExpected-" + hs->HashType() + ": " + hs->HashValue(); + + Message += Item->Custom600Headers(); + + if (hsl.FileSize() == 0 && Message.find("\nMaximum-Size: ") == std::string::npos) + { + unsigned long long FileSize = Item->GetMaximumSize(); + if(FileSize > 0) + { + string MaximumSize; + strprintf(MaximumSize, "%llu", FileSize); + Message += "\nMaximum-Size: " + MaximumSize; + } + } + + Message += "\n\n"; + + if (RealFileExists(Item->Owner->DestFile)) + { + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + ChangeOwnerAndPermissionOfFile("Item::QueueURI", Item->Owner->DestFile.c_str(), + SandboxUser.c_str(), ROOT_GROUP, 0600); + } + + if (Debug == true) + clog << " -> " << Access << ':' << QuoteString(Message,"\n") << endl; + OutQueue += Message; + OutReady = true; + + return true; +} + /*}}}*/ +// Worker::ReplyAux - reply to an aux request from this worker /*{{{*/ +bool pkgAcquire::Worker::ReplyAux(pkgAcquire::ItemDesc const &Item) +{ + if (OutFd == -1) + return false; + + if (isDoomedItem(Item.Owner)) + return true; + + string Message = "600 URI Acquire\n"; + Message.reserve(200); + Message += "URI: " + Item.URI; + if (RealFileExists(Item.Owner->DestFile)) + { + if (Item.Owner->Status == pkgAcquire::Item::StatDone) + { + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + ChangeOwnerAndPermissionOfFile("Worker::ReplyAux", Item.Owner->DestFile.c_str(), + SandboxUser.c_str(), ROOT_GROUP, 0600); + Message += "\nFilename: " + Item.Owner->DestFile; + } + else + { + // we end up here in case we would need root-rights to delete a file, + // but we run the command as non-root… (yes, it is unlikely) + Message += "\nFilename: " + flCombine("/nonexistent", Item.Owner->DestFile); + } + } + else + Message += "\nFilename: " + Item.Owner->DestFile; + Message += "\n\n"; + + if (Debug == true) + clog << " -> " << Access << ':' << QuoteString(Message, "\n") << endl; + OutQueue += Message; + OutReady = true; + + return true; +} + /*}}}*/ +// Worker::OutFdRead - Out bound FD is ready /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgAcquire::Worker::OutFdReady() +{ + int Res; + do + { + Res = write(OutFd,OutQueue.c_str(),OutQueue.length()); + } + while (Res < 0 && errno == EINTR); + + if (Res <= 0) + return MethodFailure(); + + OutQueue.erase(0,Res); + if (OutQueue.empty() == true) + OutReady = false; + + return true; +} + /*}}}*/ +// Worker::InFdRead - In bound FD is ready /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgAcquire::Worker::InFdReady() +{ + if (ReadMessages() == false) + return false; + RunMessages(); + return true; +} + /*}}}*/ +// Worker::MethodFailure - Called when the method fails /*{{{*/ +// --------------------------------------------------------------------- +/* This is called when the method is believed to have failed, probably because + read returned -1. */ +bool pkgAcquire::Worker::MethodFailure() +{ + _error->Error("Method %s has died unexpectedly!",Access.c_str()); + + // do not reap the child here to show meaningful error to the user + ExecWait(Process,Access.c_str(),false); + Process = -1; + close(InFd); + close(OutFd); + InFd = -1; + OutFd = -1; + OutReady = false; + InReady = false; + OutQueue = string(); + MessageQueue.erase(MessageQueue.begin(),MessageQueue.end()); + + return false; +} + /*}}}*/ +// Worker::Pulse - Called periodically /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcquire::Worker::Pulse() +{ + if (CurrentItem == 0) + return; + + struct stat Buf; + if (stat(CurrentItem->Owner->DestFile.c_str(),&Buf) != 0) + return; + CurrentItem->CurrentSize = Buf.st_size; +} + /*}}}*/ +// Worker::ItemDone - Called when the current item is finished /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcquire::Worker::ItemDone() +{ + CurrentItem = nullptr; + Status = string(); +} + /*}}}*/ +void pkgAcquire::Worker::PrepareFiles(char const * const caller, pkgAcquire::Queue::QItem const * const Itm)/*{{{*/ +{ + if (RealFileExists(Itm->Owner->DestFile)) + { + ChangeOwnerAndPermissionOfFile(caller, Itm->Owner->DestFile.c_str(), "root", ROOT_GROUP, 0644); + std::string const filename = Itm->Owner->DestFile; + for (pkgAcquire::Queue::QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O) + { + pkgAcquire::Item const * const Owner = *O; + if (Owner->DestFile == filename || filename == "/dev/null") + continue; + RemoveFile("PrepareFiles", Owner->DestFile); + if (link(filename.c_str(), Owner->DestFile.c_str()) != 0) + { + // different mounts can't happen for us as we download to lists/ by default, + // but if the system is reused by others the locations can potentially be on + // different disks, so use symlink as poor-men replacement. + // FIXME: Real copying as last fallback, but that is costly, so offload to a method preferable + if (symlink(filename.c_str(), Owner->DestFile.c_str()) != 0) + _error->Error("Can't create (sym)link of file %s to %s", filename.c_str(), Owner->DestFile.c_str()); + } + } + } + else + { + for (pkgAcquire::Queue::QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O) + RemoveFile("PrepareFiles", (*O)->DestFile); + } +} + /*}}}*/ diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h new file mode 100644 index 0000000..f59d659 --- /dev/null +++ b/apt-pkg/acquire-worker.h @@ -0,0 +1,324 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire Worker - Worker process manager + + Each worker class is associated with exactly one subprocess. + + ##################################################################### */ + /*}}}*/ + +/** \addtogroup acquire + * @{ + * + * \file acquire-worker.h + */ + +#ifndef PKGLIB_ACQUIRE_WORKER_H +#define PKGLIB_ACQUIRE_WORKER_H + +#include <apt-pkg/acquire.h> +#include <apt-pkg/weakptr.h> + +#include <string> +#include <vector> +#include <sys/types.h> + +/** \brief A fetch subprocess. + * + * A worker process is responsible for one stage of the fetch. This + * class encapsulates the communications protocol between the master + * process and the worker, from the master end. + * + * Each worker is intrinsically placed on two linked lists. The + * Queue list (maintained in the #NextQueue variable) is maintained + * by the pkgAcquire::Queue class; it represents the set of workers + * assigned to a particular queue. The Acquire list (maintained in + * the #NextAcquire variable) is maintained by the pkgAcquire class; + * it represents the set of active workers for a particular + * pkgAcquire object. + * + * \todo Like everything else in the Acquire system, this has way too + * many protected items. + * + * \sa pkgAcqMethod, pkgAcquire::Item, pkgAcquire + */ +class APT_PUBLIC pkgAcquire::Worker : public WeakPointable +{ + /** \brief dpointer placeholder (for later in case we need it) */ + void * const d; + + friend class pkgAcquire; + + protected: + friend class Queue; + + /** \brief The next link on the Queue list. + * + * \todo This is always NULL; is it just for future use? + */ + Worker *NextQueue; + + /** \brief The next link on the Acquire list. */ + Worker *NextAcquire; + + /** \brief The Queue with which this worker is associated. */ + Queue *OwnerQ; + + /** \brief The download progress indicator to which progress + * messages should be sent. + */ + pkgAcquireStatus *Log; + + /** \brief The configuration of this method. On startup, the + * target of this pointer is filled in with basic data about the + * method, as reported by the worker. + */ + MethodConfig *Config; + + /** \brief The access method to be used by this worker. + * + * \todo Doesn't this duplicate Config->Access? + */ + std::string Access; + + /** \brief The PID of the subprocess. */ + pid_t Process; + + /** \brief A file descriptor connected to the standard output of + * the subprocess. + * + * Used to read messages and data from the subprocess. + */ + int InFd; + + /** \brief A file descriptor connected to the standard input of the + * subprocess. + * + * Used to send commands and configuration data to the subprocess. + */ + int OutFd; + + /** \brief The socket to send SCM_RIGHTS message through + */ + int PrivSepSocketFd; + int PrivSepSocketFdChild; + + /** \brief Set to \b true if the worker is in a state in which it + * might generate data or command responses. + * + * \todo Is this right? It's a guess. + */ + bool InReady; + + /** \brief Set to \b true if the worker is in a state in which it + * is legal to send commands to it. + * + * \todo Is this right? + */ + bool OutReady; + + /** If \b true, debugging output will be sent to std::clog. */ + bool Debug; + + /** \brief The raw text values of messages received from the + * worker, in sequence. + */ + std::vector<std::string> MessageQueue; + + /** \brief Buffers pending writes to the subprocess. + * + * \todo Wouldn't a std::dequeue be more appropriate? + */ + std::string OutQueue; + + /** \brief Common code for the constructor. + * + * Initializes NextQueue and NextAcquire to NULL; Process, InFd, + * and OutFd to -1, OutReady and InReady to \b false, and Debug + * from _config. + */ + void Construct(); + + /** \brief Retrieve any available messages from the subprocess. + * + * The messages are retrieved as in \link strutl.h ReadMessages()\endlink, and + * #MethodFailure() is invoked if an error occurs; in particular, + * if the pipe to the subprocess dies unexpectedly while a message + * is being read. + * + * \return \b true if the messages were successfully read, \b + * false otherwise. + */ + bool ReadMessages(); + + /** \brief Parse and dispatch pending messages. + * + * This dispatches the message in a manner appropriate for its + * type. + * + * \todo Several message types lack separate handlers. + * + * \sa Capabilities(), SendConfiguration(), MediaChange() + */ + bool RunMessages(); + + /** \brief Read and dispatch any pending messages from the + * subprocess. + * + * \return \b false if the subprocess died unexpectedly while a + * message was being transmitted. + */ + bool InFdReady(); + + /** \brief Send any pending commands to the subprocess. + * + * This method will fail if there is no pending output. + * + * \return \b true if all commands were succeeded, \b false if an + * error occurred (in which case MethodFailure() will be invoked). + */ + bool OutFdReady(); + + /** \brief Handle a 100 Capabilities response from the subprocess. + * + * \param Message the raw text of the message from the subprocess. + * + * The message will be parsed and its contents used to fill + * #Config. If #Config is NULL, this routine is a NOP. + * + * \return \b true. + */ + bool Capabilities(std::string Message); + + /** \brief Send a 601 Configuration message (containing the APT + * configuration) to the subprocess. + * + * The APT configuration will be send to the subprocess in a + * message of the following form: + * + * <pre> + * 601 Configuration + * Config-Item: Fully-Qualified-Item=Val + * Config-Item: Fully-Qualified-Item=Val + * ... + * </pre> + * + * \return \b true if the command was successfully sent, \b false + * otherwise. + */ + bool SendConfiguration(); + + /** \brief Handle a 403 Media Change message. + * + * \param Message the raw text of the message; the Media field + * indicates what type of media should be changed, and the Drive + * field indicates where the media is located. + * + * Invokes pkgAcquireStatus::MediaChange(Media, Drive) to ask the + * user to swap disks; informs the subprocess of the result (via + * 603 Media Changed, with the Failed field set to \b true if the + * user cancelled the media change). + */ + bool MediaChange(std::string Message); + + /** \brief Invoked when the worked process dies unexpectedly. + * + * Waits for the subprocess to terminate and generates an error if + * it terminated abnormally, then closes and blanks out all file + * descriptors. Discards all pending messages from the + * subprocess. + * + * \return \b false. + */ + bool MethodFailure(); + + /** \brief Invoked when a fetch job is completed, either + * successfully or unsuccessfully. + * + * Resets the status information for the worker process. + */ + void ItemDone(); + + public: + + /** \brief The queue entry that is currently being downloaded. */ + pkgAcquire::Queue::QItem *CurrentItem; + + /** \brief The most recent status string received from the + * subprocess. + */ + std::string Status; + + /** \brief Tell the subprocess to download the given item. + * + * \param Item the item to queue up. + * \return \b true if the item was successfully enqueued. + * + * Queues up a 600 URI Acquire message for the given item to be + * sent at the next possible moment. Does \e not flush the output + * queue. + */ + bool QueueItem(pkgAcquire::Queue::QItem *Item); + APT_HIDDEN bool ReplyAux(pkgAcquire::ItemDesc const &Item); + + /** \brief Start up the worker and fill in #Config. + * + * Reads the first message from the worker, which is assumed to be + * a 100 Capabilities message. + * + * \return \b true if all operations completed successfully. + */ + bool Start(); + + /** \brief Update the worker statistics (CurrentSize, TotalSize, + * etc). + */ + void Pulse(); + + /** \return The fetch method configuration. */ + inline const MethodConfig *GetConf() const {return Config;}; + + /** \brief Create a new Worker to download files. + * + * \param OwnerQ The queue into which this worker should be + * placed. + * + * \param Config A location in which to store information about + * the fetch method. + * + * \param Log The download progress indicator that should be used + * to report the progress of this worker. + */ + Worker(Queue *OwnerQ,MethodConfig *Config,pkgAcquireStatus *Log); + + /** \brief Create a new Worker that should just retrieve + * information about the fetch method. + * + * Nothing in particular forces you to refrain from actually + * downloading stuff, but the various status callbacks won't be + * invoked. + * + * \param Config A location in which to store information about + * the fetch method. + */ + explicit Worker(MethodConfig *Config); + + /** \brief Clean up this worker. + * + * Closes the file descriptors; if MethodConfig::NeedsCleanup is + * \b false, also rudely interrupts the worker with a SIGINT. + */ + virtual ~Worker(); + +private: + APT_HIDDEN void PrepareFiles(char const * const caller, pkgAcquire::Queue::QItem const * const Itm); + APT_HIDDEN void HandleFailure(std::vector<pkgAcquire::Item *> const &ItmOwners, + pkgAcquire::MethodConfig *const Config, pkgAcquireStatus *const Log, + std::string const &Message, bool const errTransient, bool const errAuthErr); +}; + +/** @} */ + +#endif diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc new file mode 100644 index 0000000..cdb2c52 --- /dev/null +++ b/apt-pkg/acquire.cc @@ -0,0 +1,1570 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire - File Acquiration + + The core element for the schedule system is the concept of a named + queue. Each queue is unique and each queue has a name derived from the + URI. The degree of paralization can be controlled by how the queue + name is derived from the URI. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/acquire-item.h> +#include <apt-pkg/acquire-worker.h> +#include <apt-pkg/acquire.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <chrono> +#include <cmath> +#include <iomanip> +#include <iostream> +#include <memory> +#include <numeric> +#include <sstream> +#include <string> +#include <tuple> +#include <vector> + +#include <dirent.h> +#include <errno.h> +#include <fcntl.h> +#include <grp.h> +#include <pwd.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/select.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +// helper to convert time_point to a timeval +constexpr struct timeval SteadyDurationToTimeVal(std::chrono::steady_clock::duration Time) +{ + auto const Time_sec = std::chrono::duration_cast<std::chrono::seconds>(Time); + auto const Time_usec = std::chrono::duration_cast<std::chrono::microseconds>(Time - Time_sec); + return timeval{static_cast<time_t>(Time_sec.count()), static_cast<suseconds_t>(Time_usec.count())}; +} + +std::string pkgAcquire::URIEncode(std::string const &part) /*{{{*/ +{ + // The "+" is encoded as a workaround for an S3 bug (LP#1003633 and LP#1086997) + return QuoteString(part, _config->Find("Acquire::URIEncode", "+~ ").c_str()); +} + /*}}}*/ +// Acquire::pkgAcquire - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* We grab some runtime state from the configuration space */ +pkgAcquire::pkgAcquire() : LockFD(-1), d(NULL), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0), + Debug(_config->FindB("Debug::pkgAcquire",false)), + Running(false) +{ + Initialize(); +} +pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), d(NULL), Queues(0), Workers(0), + Configs(0), Log(NULL), ToFetch(0), + Debug(_config->FindB("Debug::pkgAcquire",false)), + Running(false) +{ + Initialize(); + SetLog(Progress); +} +void pkgAcquire::Initialize() +{ + string const Mode = _config->Find("Acquire::Queue-Mode","host"); + if (strcasecmp(Mode.c_str(),"host") == 0) + QueueMode = QueueHost; + if (strcasecmp(Mode.c_str(),"access") == 0) + QueueMode = QueueAccess; +} + /*}}}*/ +// Acquire::GetLock - lock directory and prepare for action /*{{{*/ +static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent, std::string const &postfix, mode_t const mode) +{ + if (_config->FindB("Debug::SetupAPTPartialDirectory::AssumeGood", false)) + return true; + std::string const partial = parent + postfix; + bool const partialExists = DirectoryExists(partial); + if (partialExists == false) + { + mode_t const old_umask = umask(S_IWGRP | S_IWOTH); + bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false && + CreateAPTDirectoryIfNeeded(parent, partial) == false); + umask(old_umask); + if (creation_fail == true) + return false; + } + + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + if (getuid() == 0) + { + if (SandboxUser.empty() == false && SandboxUser != "root") // if we aren't root, we can't chown, so don't try it + { + struct passwd const * const pw = getpwnam(SandboxUser.c_str()); + struct group const * const gr = getgrnam(ROOT_GROUP); + if (pw != NULL && gr != NULL) + { + // chown the partial dir + if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chown to %s:%s of directory %s failed", SandboxUser.c_str(), ROOT_GROUP, partial.c_str()); + } + } + if (chmod(partial.c_str(), mode) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chmod 0%03o of directory %s failed", mode, partial.c_str()); + + } + else if (chmod(partial.c_str(), mode) != 0) + { + // if we haven't created the dir and aren't root, it is kinda expected that chmod doesn't work + if (partialExists == false) + _error->WarningE("SetupAPTPartialDirectory", "chmod 0%03o of directory %s failed", mode, partial.c_str()); + } + + _error->PushToStack(); + // remove 'old' FAILED files to stop us from collecting them for no reason + for (auto const &Failed: GetListOfFilesInDir(partial, "FAILED", false, false)) + RemoveFile("SetupAPTPartialDirectory", Failed); + _error->RevertToStack(); + + return true; +} +bool pkgAcquire::GetLock(std::string const &Lock) +{ + if (Lock.empty() == true) + return false; + + // check for existence and possibly create auxiliary directories + string const listDir = _config->FindDir("Dir::State::lists"); + string const archivesDir = _config->FindDir("Dir::Cache::Archives"); + + if (Lock == listDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "partial", 0700) == false) + return _error->Errno("Acquire", _("List directory %s is missing."), (listDir + "partial").c_str()); + } + if (Lock == archivesDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir, "partial", 0700) == false) + return _error->Errno("Acquire", _("Archives directory %s is missing."), (archivesDir + "partial").c_str()); + } + if (Lock == listDir || Lock == archivesDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false) + { + // not being able to create lists/auxfiles isn't critical as we will use a tmpdir then + } + } + + if (_config->FindB("Debug::NoLocking", false) == true) + return true; + + // Lock the directory this acquire object will work in + if (LockFD != -1) + close(LockFD); + LockFD = ::GetLock(flCombine(Lock, "lock")); + if (LockFD == -1) + return _error->Error(_("Unable to lock directory %s"), Lock.c_str()); + + return true; +} + /*}}}*/ +// Acquire::~pkgAcquire - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* Free our memory, clean up the queues (destroy the workers) */ +pkgAcquire::~pkgAcquire() +{ + Shutdown(); + + if (LockFD != -1) + close(LockFD); + + while (Configs != 0) + { + MethodConfig *Jnk = Configs; + Configs = Configs->Next; + delete Jnk; + } +} + /*}}}*/ +// Acquire::Shutdown - Clean out the acquire object /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcquire::Shutdown() +{ + while (Items.empty() == false) + { + if (Items[0]->Status == Item::StatFetching) + Items[0]->Status = Item::StatError; + delete Items[0]; + } + + while (Queues != 0) + { + Queue *Jnk = Queues; + Queues = Queues->Next; + delete Jnk; + } +} + /*}}}*/ +// Acquire::Add - Add a new item /*{{{*/ +// --------------------------------------------------------------------- +/* This puts an item on the acquire list. This list is mainly for tracking + item status */ +void pkgAcquire::Add(Item *Itm) +{ + Items.push_back(Itm); +} + /*}}}*/ +// Acquire::Remove - Remove a item /*{{{*/ +// --------------------------------------------------------------------- +/* Remove an item from the acquire list. This is usually not used.. */ +void pkgAcquire::Remove(Item *Itm) +{ + Dequeue(Itm); + + for (ItemIterator I = Items.begin(); I != Items.end();) + { + if (*I == Itm) + { + Items.erase(I); + I = Items.begin(); + } + else + ++I; + } +} + /*}}}*/ +// Acquire::Add - Add a worker /*{{{*/ +// --------------------------------------------------------------------- +/* A list of workers is kept so that the select loop can direct their FD + usage. */ +void pkgAcquire::Add(Worker *Work) +{ + Work->NextAcquire = Workers; + Workers = Work; +} + /*}}}*/ +// Acquire::Remove - Remove a worker /*{{{*/ +// --------------------------------------------------------------------- +/* A worker has died. This can not be done while the select loop is running + as it would require that RunFds could handling a changing list state and + it can't.. */ +void pkgAcquire::Remove(Worker *Work) +{ + if (Running == true) + abort(); + + Worker **I = &Workers; + for (; *I != 0;) + { + if (*I == Work) + *I = (*I)->NextAcquire; + else + I = &(*I)->NextAcquire; + } +} + /*}}}*/ +// Acquire::Enqueue - Queue an URI for fetching /*{{{*/ +// --------------------------------------------------------------------- +/* This is the entry point for an item. An item calls this function when + it is constructed which creates a queue (based on the current queue + mode) and puts the item in that queue. If the system is running then + the queue might be started. */ +static bool CheckForBadItemAndFailIt(pkgAcquire::Item * const Item, + pkgAcquire::MethodConfig const * const Config, pkgAcquireStatus * const Log) +{ + auto SavedDesc = Item->GetItemDesc(); + if (Item->IsRedirectionLoop(SavedDesc.URI)) + { + std::string const Message = "400 URI Failure" + "\nURI: " + SavedDesc.URI + + "\nFilename: " + Item->DestFile + + "\nFailReason: RedirectionLoop"; + + Item->Status = pkgAcquire::Item::StatError; + Item->Failed(Message, Config); + if (Log != nullptr) + Log->Fail(SavedDesc); + return true; + } + + HashStringList const hsl = Item->GetExpectedHashes(); + if (hsl.usable() == false && Item->HashesRequired() && + _config->Exists("Acquire::ForceHash") == false) + { + std::string const Message = "400 URI Failure" + "\nURI: " + SavedDesc.URI + + "\nFilename: " + Item->DestFile + + "\nFailReason: WeakHashSums"; + + Item->Status = pkgAcquire::Item::StatAuthError; + Item->Failed(Message, Config); + if (Log != nullptr) + Log->Fail(SavedDesc); + return true; + } + return false; +} +void pkgAcquire::Enqueue(ItemDesc &Item) +{ + // Determine which queue to put the item in + const MethodConfig *Config = nullptr; + string Name = QueueName(Item.URI,Config); + if (Name.empty() == true) + { + Item.Owner->Status = pkgAcquire::Item::StatError; + return; + } + + /* the check for running avoids that we produce errors + in logging before we actually have started, which would + be easier to implement but would confuse users/implementations + so we check the items skipped here in #Startup */ + if (Running && CheckForBadItemAndFailIt(Item.Owner, Config, Log)) + return; + + // Find the queue structure + Queue *I = Queues; + for (; I != 0 && I->Name != Name; I = I->Next); + if (I == 0) + { + I = new Queue(Name,this); + I->Next = Queues; + Queues = I; + + if (Running == true) + I->Startup(); + } + + // See if this is a local only URI + if (Config->LocalOnly == true && Item.Owner->Complete == false) + Item.Owner->Local = true; + Item.Owner->Status = Item::StatIdle; + + // Queue it into the named queue + if(I->Enqueue(Item)) + ToFetch++; + + // Some trace stuff + if (Debug == true) + { + clog << "Fetching " << Item.URI << endl; + clog << " to " << Item.Owner->DestFile << endl; + clog << " Queue is: " << Name << endl; + } +} + /*}}}*/ +// Acquire::Dequeue - Remove an item from all queues /*{{{*/ +// --------------------------------------------------------------------- +/* This is called when an item is finished being fetched. It removes it + from all the queues */ +void pkgAcquire::Dequeue(Item *Itm) +{ + Queue *I = Queues; + bool Res = false; + if (Debug == true) + clog << "Dequeuing " << Itm->DestFile << endl; + + for (; I != 0; I = I->Next) + { + if (I->Dequeue(Itm)) + { + Res = true; + if (Debug == true) + clog << "Dequeued from " << I->Name << endl; + } + } + + if (Res == true) + ToFetch--; +} + /*}}}*/ +// Acquire::QueueName - Return the name of the queue for this URI /*{{{*/ +// --------------------------------------------------------------------- +/* The string returned depends on the configuration settings and the + method parameters. Given something like http://foo.org/bar it can + return http://foo.org or http */ +string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config) +{ + constexpr int DEFAULT_HOST_LIMIT = 10; + URI U(Uri); + + // Note that this gets written through the reference to the caller. + Config = GetConfig(U.Access); + if (Config == nullptr) + return {}; + + // Access mode forces all methods to be Single-Instance + if (QueueMode == QueueAccess) + return U.Access; + + // Single-Instance methods get exactly one queue per URI + if (Config->SingleInstance == true) + return U.Access; + + // Host-less methods like rred, store, … + if (U.Host.empty()) + { + int existing = 0; + // check how many queues exist already and reuse empty ones + auto const AccessSchema = U.Access + ':'; + for (Queue const *I = Queues; I != 0; I = I->Next) + if (APT::String::Startswith(I->Name, AccessSchema)) + { + if (I->Items == nullptr) + return I->Name; + ++existing; + } + + int const Limit = _config->FindI("Acquire::QueueHost::Limit", +#ifdef _SC_NPROCESSORS_ONLN + sysconf(_SC_NPROCESSORS_ONLN) * 2 +#else + DEFAULT_HOST_LIMIT +#endif + ); + + // create a new worker if we don't have too many yet + if (Limit <= 0 || existing < Limit) + return AccessSchema + std::to_string(existing); + + // find the worker with the least to do + // we already established that there are no empty and we can't spawn new + Queue const *selected = nullptr; + auto selected_backlog = std::numeric_limits<decltype(HashStringList().FileSize())>::max(); + for (Queue const *Q = Queues; Q != nullptr; Q = Q->Next) + if (APT::String::Startswith(Q->Name, AccessSchema)) + { + decltype(selected_backlog) current_backlog = 0; + for (auto const *I = Q->Items; I != nullptr; I = I->Next) + { + auto const hashes = I->Owner->GetExpectedHashes(); + if (not hashes.empty()) + current_backlog += hashes.FileSize(); + else + current_backlog += I->Owner->FileSize; + } + if (current_backlog < selected_backlog) + { + selected = Q; + selected_backlog = current_backlog; + } + } + + if (unlikely(selected == nullptr)) + return AccessSchema + "0"; + return selected->Name; + } + // most methods talking to remotes like http + else + { + auto const FullQueueName = U.Access + ':' + U.Host; + // if the queue already exists, re-use it + for (Queue const *Q = Queues; Q != nullptr; Q = Q->Next) + if (Q->Name == FullQueueName) + return FullQueueName; + + int existing = 0; + // check how many queues exist already and reuse empty ones + auto const AccessSchema = U.Access + ':'; + for (Queue const *Q = Queues; Q != nullptr; Q = Q->Next) + if (APT::String::Startswith(Q->Name, AccessSchema)) + ++existing; + + int const Limit = _config->FindI("Acquire::QueueHost::Limit", DEFAULT_HOST_LIMIT); + // if we have too many hosts open use a single generic for the rest + if (existing >= Limit) + return U.Access; + + // we can still create new named queues + return FullQueueName; + } +} + /*}}}*/ +// Acquire::GetConfig - Fetch the configuration information /*{{{*/ +// --------------------------------------------------------------------- +/* This locates the configuration structure for an access method. If + a config structure cannot be found a Worker will be created to + retrieve it */ +pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access) +{ + // Search for an existing config + MethodConfig *Conf; + for (Conf = Configs; Conf != 0; Conf = Conf->Next) + if (Conf->Access == Access) + return Conf; + + // Create the new config class + Conf = new MethodConfig; + Conf->Access = Access; + + // Create the worker to fetch the configuration + Worker Work(Conf); + if (Work.Start() == false) + { + delete Conf; + return nullptr; + } + Conf->Next = Configs; + Configs = Conf; + + /* if a method uses DownloadLimit, we switch to SingleInstance mode */ + if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0) + Conf->SingleInstance = true; + + return Conf; +} + /*}}}*/ +// Acquire::SetFds - Deal with readable FDs /*{{{*/ +// --------------------------------------------------------------------- +/* Collect FDs that have activity monitors into the fd sets */ +void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet) +{ + for (Worker *I = Workers; I != 0; I = I->NextAcquire) + { + if (I->InReady == true && I->InFd >= 0) + { + if (Fd < I->InFd) + Fd = I->InFd; + FD_SET(I->InFd,RSet); + } + if (I->OutReady == true && I->OutFd >= 0) + { + if (Fd < I->OutFd) + Fd = I->OutFd; + FD_SET(I->OutFd,WSet); + } + } +} + /*}}}*/ +// Acquire::RunFds - Deal with active FDs /*{{{*/ +// --------------------------------------------------------------------- +/* Dispatch active FDs over to the proper workers. It is very important + that a worker never be erased while this is running! The queue class + should never erase a worker except during shutdown processing. */ +bool pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet) +{ + bool Res = true; + + for (Worker *I = Workers; I != 0; I = I->NextAcquire) + { + if (I->InFd >= 0 && FD_ISSET(I->InFd,RSet) != 0) + Res &= I->InFdReady(); + if (I->OutFd >= 0 && FD_ISSET(I->OutFd,WSet) != 0) + Res &= I->OutFdReady(); + } + + return Res; +} + /*}}}*/ +// Acquire::Run - Run the fetch sequence /*{{{*/ +// --------------------------------------------------------------------- +/* This runs the queues. It manages a select loop for all of the + Worker tasks. The workers interact with the queues and items to + manage the actual fetch. */ +static bool IsAccessibleBySandboxUser(std::string const &filename, bool const ReadWrite) +{ + // you would think this is easily to answer with faccessat, right? Wrong! + // It e.g. gets groups wrong, so the only thing which works reliable is trying + // to open the file we want to open later on… + if (unlikely(filename.empty())) + return true; + + if (ReadWrite == false) + { + errno = 0; + // can we read a file? Note that non-existing files are "fine" + int const fd = open(filename.c_str(), O_RDONLY | O_CLOEXEC); + if (fd == -1 && errno == EACCES) + return false; + close(fd); + return true; + } + else + { + // the file might not exist yet and even if it does we will fix permissions, + // so important is here just that the directory it is in allows that + std::string const dirname = flNotFile(filename); + if (unlikely(dirname.empty())) + return true; + + char const * const filetag = ".apt-acquire-privs-test.XXXXXX"; + std::string const tmpfile_tpl = flCombine(dirname, filetag); + std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free }; + int const fd = mkstemp(tmpfile.get()); + if (fd == -1 && errno == EACCES) + return false; + RemoveFile("IsAccessibleBySandboxUser", tmpfile.get()); + close(fd); + return true; + } +} +static void CheckDropPrivsMustBeDisabled(pkgAcquire const &Fetcher) +{ + if(getuid() != 0) + return; + + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + if (SandboxUser.empty() || SandboxUser == "root") + return; + + struct passwd const * const pw = getpwnam(SandboxUser.c_str()); + if (pw == NULL) + { + _error->Warning(_("No sandbox user '%s' on the system, can not drop privileges"), SandboxUser.c_str()); + _config->Set("APT::Sandbox::User", ""); + return; + } + + gid_t const old_euid = geteuid(); + gid_t const old_egid = getegid(); + + long const ngroups_max = sysconf(_SC_NGROUPS_MAX); + std::unique_ptr<gid_t[]> old_gidlist(new gid_t[ngroups_max]); + if (unlikely(old_gidlist == NULL)) + return; + ssize_t old_gidlist_nr; + if ((old_gidlist_nr = getgroups(ngroups_max, old_gidlist.get())) < 0) + { + _error->FatalE("getgroups", "getgroups %lu failed", ngroups_max); + old_gidlist[0] = 0; + old_gidlist_nr = 1; + } + if (setgroups(1, &pw->pw_gid)) + _error->FatalE("setgroups", "setgroups %u failed", pw->pw_gid); + + if (setegid(pw->pw_gid) != 0) + _error->FatalE("setegid", "setegid %u failed", pw->pw_gid); + if (seteuid(pw->pw_uid) != 0) + _error->FatalE("seteuid", "seteuid %u failed", pw->pw_uid); + + for (pkgAcquire::ItemCIterator I = Fetcher.ItemsBegin(); + I != Fetcher.ItemsEnd(); ++I) + { + // no need to drop privileges for a complete file + if ((*I)->Complete == true || (*I)->Status != pkgAcquire::Item::StatIdle) + continue; + + // if destination file is inaccessible all hope is lost for privilege dropping + if (IsAccessibleBySandboxUser((*I)->DestFile, true) == false) + { + _error->WarningE("pkgAcquire::Run", _("Download is performed unsandboxed as root as file '%s' couldn't be accessed by user '%s'."), + (*I)->DestFile.c_str(), SandboxUser.c_str()); + _config->Set("APT::Sandbox::User", ""); + break; + } + + // if its the source file (e.g. local sources) we might be lucky + // by dropping the dropping only for some methods. + URI const source((*I)->DescURI()); + if (source.Access == "file" || source.Access == "copy") + { + std::string const conf = "Binary::" + source.Access + "::APT::Sandbox::User"; + if (_config->Exists(conf) == true) + continue; + + auto const filepath = DeQuoteString(source.Path); + if (not IsAccessibleBySandboxUser(filepath, false)) + { + _error->NoticeE("pkgAcquire::Run", _("Download is performed unsandboxed as root as file '%s' couldn't be accessed by user '%s'."), + filepath.c_str(), SandboxUser.c_str()); + _config->CndSet("Binary::file::APT::Sandbox::User", "root"); + _config->CndSet("Binary::copy::APT::Sandbox::User", "root"); + } + } + } + + if (seteuid(old_euid) != 0) + _error->FatalE("seteuid", "seteuid %u failed", old_euid); + if (setegid(old_egid) != 0) + _error->FatalE("setegid", "setegid %u failed", old_egid); + if (setgroups(old_gidlist_nr, old_gidlist.get())) + _error->FatalE("setgroups", "setgroups %u failed", 0); +} +pkgAcquire::RunResult pkgAcquire::Run(int PulseInterval) +{ + _error->PushToStack(); + CheckDropPrivsMustBeDisabled(*this); + + Running = true; + + if (Log != 0) + Log->Start(); + + for (Queue *I = Queues; I != 0; I = I->Next) + I->Startup(); + + bool WasCancelled = false; + + // Run till all things have been acquired + struct timeval tv = SteadyDurationToTimeVal(std::chrono::microseconds(PulseInterval)); + while (ToFetch > 0) + { + fd_set RFds; + fd_set WFds; + int Highest = 0; + FD_ZERO(&RFds); + FD_ZERO(&WFds); + SetFds(Highest,&RFds,&WFds); + + // Shorten the select() cycle in case we have items about to become ready + auto now = clock::now(); + auto fetchAfter = time_point{}; + for (Queue *I = Queues; I != nullptr; I = I->Next) + { + if (I->Items == nullptr) + continue; + + auto f = I->Items->GetFetchAfter(); + + if (f == time_point() || I->Items->Owner->Status != pkgAcquire::Item::StatIdle) + continue; + + if (f <= now) + { + if (not I->Cycle()) // Queue got stuck, unstuck it. + goto stop; + fetchAfter = now; // need to time out in select() below + if (I->Items->Owner->Status == pkgAcquire::Item::StatIdle) + { + _error->Warning("Tried to start delayed item %s, but failed", I->Items->Description.c_str()); + } + } + else if (f < fetchAfter || fetchAfter == time_point{}) + { + fetchAfter = f; + } + } + + if (fetchAfter != time_point{} && (fetchAfter - now) < std::chrono::seconds(tv.tv_sec) + std::chrono::microseconds(tv.tv_usec)) + { + tv = SteadyDurationToTimeVal(fetchAfter - now); + } + + int Res; + do + { + Res = select(Highest+1,&RFds,&WFds,0,&tv); + } + while (Res < 0 && errno == EINTR); + + if (Res < 0) + { + _error->Errno("select","Select has failed"); + break; + } + + if(RunFds(&RFds,&WFds) == false) + break; + + // Timeout, notify the log class + if (Res == 0 || (Log != 0 && Log->Update == true)) + { + tv = SteadyDurationToTimeVal(std::chrono::microseconds(PulseInterval)); + + for (Worker *I = Workers; I != 0; I = I->NextAcquire) + I->Pulse(); + if (Log != 0 && Log->Pulse(this) == false) + { + WasCancelled = true; + break; + } + } + } +stop: + if (Log != 0) + Log->Stop(); + + // Shut down the acquire bits + Running = false; + for (Queue *I = Queues; I != 0; I = I->Next) + I->Shutdown(false); + + // Shut down the items + for (ItemIterator I = Items.begin(); I != Items.end(); ++I) + (*I)->Finished(); + + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + if (newError) + return Failed; + if (WasCancelled) + return Cancelled; + return Continue; +} + /*}}}*/ +// Acquire::Bump - Called when an item is dequeued /*{{{*/ +// --------------------------------------------------------------------- +/* This routine bumps idle queues in hopes that they will be able to fetch + the dequeued item */ +void pkgAcquire::Bump() +{ + for (Queue *I = Queues; I != 0; I = I->Next) + I->Bump(); +} + /*}}}*/ +// Acquire::WorkerStep - Step to the next worker /*{{{*/ +// --------------------------------------------------------------------- +/* Not inlined to advoid including acquire-worker.h */ +pkgAcquire::Worker *pkgAcquire::WorkerStep(Worker *I) +{ + return I->NextAcquire; +} + /*}}}*/ +// Acquire::Clean - Cleans a directory /*{{{*/ +// --------------------------------------------------------------------- +/* This is a bit simplistic, it looks at every file in the dir and sees + if it is part of the download set. */ +bool pkgAcquire::Clean(string Dir) +{ + // non-existing directories are by definition clean… + if (DirectoryExists(Dir) == false) + return true; + + if(Dir == "/") + return _error->Error(_("Clean of %s is not supported"), Dir.c_str()); + + int const dirfd = open(Dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (dirfd == -1) + return _error->Errno("open",_("Unable to read %s"),Dir.c_str()); + DIR * const D = fdopendir(dirfd); + if (D == nullptr) + return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + + for (struct dirent *E = readdir(D); E != nullptr; E = readdir(D)) + { + // Skip some entries + if (strcmp(E->d_name, "lock") == 0 || + strcmp(E->d_name, "partial") == 0 || + strcmp(E->d_name, "auxfiles") == 0 || + strcmp(E->d_name, "lost+found") == 0 || + strcmp(E->d_name, ".") == 0 || + strcmp(E->d_name, "..") == 0) + continue; + + // Look in the get list and if not found nuke + if (std::any_of(Items.cbegin(), Items.cend(), + [&E](pkgAcquire::Item const * const I) { + return flNotDir(I->DestFile) == E->d_name; + }) == false) + { + RemoveFileAt("pkgAcquire::Clean", dirfd, E->d_name); + } + } + closedir(D); + return true; +} + /*}}}*/ +// Acquire::TotalNeeded - Number of bytes to fetch /*{{{*/ +// --------------------------------------------------------------------- +/* This is the total number of bytes needed */ +APT_PURE unsigned long long pkgAcquire::TotalNeeded() +{ + return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu, + [](unsigned long long const T, Item const * const I) { + return T + I->FileSize; + }); +} + /*}}}*/ +// Acquire::FetchNeeded - Number of bytes needed to get /*{{{*/ +// --------------------------------------------------------------------- +/* This is the number of bytes that is not local */ +APT_PURE unsigned long long pkgAcquire::FetchNeeded() +{ + return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu, + [](unsigned long long const T, Item const * const I) { + if (I->Local == false) + return T + I->FileSize; + else + return T; + }); +} + /*}}}*/ +// Acquire::PartialPresent - Number of partial bytes we already have /*{{{*/ +// --------------------------------------------------------------------- +/* This is the number of bytes that is not local */ +APT_PURE unsigned long long pkgAcquire::PartialPresent() +{ + return std::accumulate(ItemsBegin(), ItemsEnd(), 0llu, + [](unsigned long long const T, Item const * const I) { + if (I->Local == false) + return T + I->PartialSize; + else + return T; + }); +} + /*}}}*/ +// Acquire::UriBegin - Start iterator for the uri list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgAcquire::UriIterator pkgAcquire::UriBegin() +{ + return UriIterator(Queues); +} + /*}}}*/ +// Acquire::UriEnd - End iterator for the uri list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgAcquire::UriIterator pkgAcquire::UriEnd() +{ + return UriIterator(0); +} + /*}}}*/ +// Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/ +class pkgAcquire::MethodConfig::Private +{ + public: + bool AuxRequests = false; + bool SendURIEncoded = false; +}; +pkgAcquire::MethodConfig::MethodConfig() : d(new Private()), Next(0), SingleInstance(false), + Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false), + Removable(false) +{ +} + /*}}}*/ +bool pkgAcquire::MethodConfig::GetAuxRequests() const /*{{{*/ +{ + return d->AuxRequests; +} + /*}}}*/ +void pkgAcquire::MethodConfig::SetAuxRequests(bool const value) /*{{{*/ +{ + d->AuxRequests = value; +} + /*}}}*/ +bool pkgAcquire::MethodConfig::GetSendURIEncoded() const /*{{{*/ +{ + return d->SendURIEncoded; +} + /*}}}*/ +void pkgAcquire::MethodConfig::SetSendURIEncoded(bool const value) /*{{{*/ +{ + d->SendURIEncoded = value; +} + /*}}}*/ + +// Queue::Queue - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgAcquire::Queue::Queue(string const &name,pkgAcquire * const owner) : d(NULL), Next(0), + Name(name), Items(0), Workers(0), Owner(owner), PipeDepth(0), MaxPipeDepth(1) +{ +} + /*}}}*/ +// Queue::~Queue - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgAcquire::Queue::~Queue() +{ + Shutdown(true); + + while (Items != 0) + { + QItem *Jnk = Items; + Items = Items->Next; + delete Jnk; + } +} + /*}}}*/ +// Queue::Enqueue - Queue an item to the queue /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgAcquire::Queue::Enqueue(ItemDesc &Item) +{ + // MetaKeysMatch checks whether the two items have no non-matching + // meta-keys. If the items are not transaction items, it returns + // true, so other items can still be merged. + auto MetaKeysMatch = [](pkgAcquire::ItemDesc const &A, pkgAcquire::Queue::QItem const *B) { + auto OwnerA = dynamic_cast<pkgAcqTransactionItem*>(A.Owner); + if (OwnerA == nullptr) + return true; + + for (auto const & OwnerBUncast : B->Owners) { + auto OwnerB = dynamic_cast<pkgAcqTransactionItem*>(OwnerBUncast); + + if (OwnerB != nullptr && OwnerA->GetMetaKey() != OwnerB->GetMetaKey()) + return false; + } + return true; + }; + QItem **OptimalI = &Items; + QItem **I = &Items; + auto insertLocation = std::make_tuple(Item.Owner->FetchAfter(), -Item.Owner->Priority()); + // move to the end of the queue and check for duplicates here + for (; *I != 0; ) { + if (Item.URI == (*I)->URI && MetaKeysMatch(Item, *I)) + { + if (_config->FindB("Debug::pkgAcquire::Worker",false) == true) + std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl; + (*I)->Owners.push_back(Item.Owner); + Item.Owner->Status = (*I)->Owner->Status; + return false; + } + // Determine the optimal position to insert: before anything with a + // higher priority. + auto queueLocation = std::make_tuple((*I)->GetFetchAfter(), + -(*I)->GetPriority()); + + I = &(*I)->Next; + if (queueLocation <= insertLocation) + { + OptimalI = I; + } + } + + + // Create a new item + QItem *Itm = new QItem; + *Itm = Item; + Itm->Next = *OptimalI; + *OptimalI = Itm; + + Item.Owner->QueueCounter++; + if (Items->Next == 0) + Cycle(); + return true; +} + /*}}}*/ +// Queue::Dequeue - Remove an item from the queue /*{{{*/ +// --------------------------------------------------------------------- +/* We return true if we hit something */ +bool pkgAcquire::Queue::Dequeue(Item *Owner) +{ + if (Owner->Status == pkgAcquire::Item::StatFetching) + return _error->Error("Tried to dequeue a fetching object"); + + bool Res = false; + + QItem **I = &Items; + for (; *I != 0;) + { + if (Owner == (*I)->Owner) + { + QItem *Jnk= *I; + *I = (*I)->Next; + Owner->QueueCounter--; + delete Jnk; + Res = true; + } + else + I = &(*I)->Next; + } + + return Res; +} + /*}}}*/ +// Queue::Startup - Start the worker processes /*{{{*/ +// --------------------------------------------------------------------- +/* It is possible for this to be called with a pre-existing set of + workers. */ +bool pkgAcquire::Queue::Startup() +{ + if (Workers == 0) + { + URI U(Name); + pkgAcquire::MethodConfig * const Cnf = Owner->GetConfig(U.Access); + if (unlikely(Cnf == nullptr)) + return false; + + // now-running twin of the pkgAcquire::Enqueue call + for (QItem *I = Items; I != 0; ) + { + auto const INext = I->Next; + for (auto &&O: I->Owners) + CheckForBadItemAndFailIt(O, Cnf, Owner->Log); + // if an item failed, it will be auto-dequeued invalidation our I here + I = INext; + } + + Workers = new Worker(this,Cnf,Owner->Log); + Owner->Add(Workers); + if (Workers->Start() == false) + return false; + + /* When pipelining we commit 10 items. This needs to change when we + added other source retry to have cycle maintain a pipeline depth + on its own. */ + if (Cnf->Pipeline == true) + MaxPipeDepth = _config->FindI("Acquire::Max-Pipeline-Depth",10); + else + MaxPipeDepth = 1; + } + + return Cycle(); +} + /*}}}*/ +// Queue::Shutdown - Shutdown the worker processes /*{{{*/ +// --------------------------------------------------------------------- +/* If final is true then all workers are eliminated, otherwise only workers + that do not need cleanup are removed */ +bool pkgAcquire::Queue::Shutdown(bool Final) +{ + // Delete all of the workers + pkgAcquire::Worker **Cur = &Workers; + while (*Cur != 0) + { + pkgAcquire::Worker *Jnk = *Cur; + if (Final == true || Jnk->GetConf()->NeedsCleanup == false) + { + *Cur = Jnk->NextQueue; + Owner->Remove(Jnk); + delete Jnk; + } + else + Cur = &(*Cur)->NextQueue; + } + + return true; +} + /*}}}*/ +// Queue::FindItem - Find a URI in the item list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Worker *Owner) +{ + if (Owner->Config->GetSendURIEncoded()) + { + for (QItem *I = Items; I != nullptr; I = I->Next) + if (I->URI == URI && I->Worker == Owner) + return I; + } + else + { + for (QItem *I = Items; I != nullptr; I = I->Next) + { + if (I->Worker != Owner) + continue; + ::URI tmpuri{I->URI}; + tmpuri.Path = DeQuoteString(tmpuri.Path); + if (URI == std::string(tmpuri)) + return I; + } + } + return nullptr; +} + /*}}}*/ +// Queue::ItemDone - Item has been completed /*{{{*/ +// --------------------------------------------------------------------- +/* The worker signals this which causes the item to be removed from the + queue. If this is the last queue instance then it is removed from the + main queue too.*/ +bool pkgAcquire::Queue::ItemDone(QItem *Itm) +{ + PipeDepth--; + for (QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O) + { + if ((*O)->Status == pkgAcquire::Item::StatFetching) + (*O)->Status = pkgAcquire::Item::StatDone; + } + + if (Itm->Owner->QueueCounter <= 1) + Owner->Dequeue(Itm->Owner); + else + { + Dequeue(Itm->Owner); + Owner->Bump(); + } + + return Cycle(); +} + /*}}}*/ +// Queue::Cycle - Queue new items into the method /*{{{*/ +// --------------------------------------------------------------------- +/* This locates a new idle item and sends it to the worker. If pipelining + is enabled then it keeps the pipe full. */ +bool pkgAcquire::Queue::Cycle() +{ + if (Items == 0 || Workers == 0) + return true; + + if (PipeDepth < 0) + return _error->Error("Pipedepth failure"); + + // Look for a queable item + QItem *I = Items; + int ActivePriority = 0; + auto currentTime = clock::now(); + while (PipeDepth < static_cast<decltype(PipeDepth)>(MaxPipeDepth)) + { + for (; I != 0; I = I->Next) { + if (I->Owner->Status == pkgAcquire::Item::StatFetching) + ActivePriority = std::max(ActivePriority, I->GetPriority()); + if (I->Owner->Status == pkgAcquire::Item::StatIdle) + break; + } + + // Nothing to do, queue is idle. + if (I == 0) + return true; + + // This item has a lower priority than stuff in the pipeline, pretend + // the queue is idle + if (I->GetPriority() < ActivePriority) + return true; + + // Item is not ready yet, delay + if (I->GetFetchAfter() > currentTime) + return true; + + I->Worker = Workers; + for (auto const &O: I->Owners) + O->Status = pkgAcquire::Item::StatFetching; + PipeDepth++; + if (Workers->QueueItem(I) == false) + return false; + } + + return true; +} + /*}}}*/ +// Queue::Bump - Fetch any pending objects if we are idle /*{{{*/ +// --------------------------------------------------------------------- +/* This is called when an item in multiple queues is dequeued */ +void pkgAcquire::Queue::Bump() +{ + Cycle(); +} + /*}}}*/ +HashStringList pkgAcquire::Queue::QItem::GetExpectedHashes() const /*{{{*/ +{ + /* each Item can have multiple owners and each owner might have different + hashes, even if that is unlikely in practice and if so at least some + owners will later fail. There is one situation through which is not a + failure and still needs this handling: Two owners who expect the same + file, but one owner only knows the SHA1 while the other only knows SHA256. */ + HashStringList superhsl; + for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O) + { + HashStringList const hsl = (*O)->GetExpectedHashes(); + // we merge both lists - if we find disagreement send no hashes + HashStringList::const_iterator hs = hsl.begin(); + for (; hs != hsl.end(); ++hs) + if (superhsl.push_back(*hs) == false) + break; + if (hs != hsl.end()) + { + superhsl.clear(); + break; + } + } + return superhsl; +} + /*}}}*/ +APT_PURE unsigned long long pkgAcquire::Queue::QItem::GetMaximumSize() const /*{{{*/ +{ + unsigned long long Maximum = std::numeric_limits<unsigned long long>::max(); + for (auto const &O: Owners) + { + if (O->FileSize == 0) + continue; + Maximum = std::min(Maximum, O->FileSize); + } + if (Maximum == std::numeric_limits<unsigned long long>::max()) + return 0; + return Maximum; +} + /*}}}*/ +APT_PURE int pkgAcquire::Queue::QItem::GetPriority() const /*{{{*/ +{ + int Priority = 0; + for (auto const &O: Owners) + Priority = std::max(Priority, O->Priority()); + + return Priority; +} + /*}}}*/ +APT_PURE pkgAcquire::time_point pkgAcquire::Queue::QItem::GetFetchAfter() const /*{{{*/ +{ + time_point FetchAfter{}; + for (auto const &O : Owners) + FetchAfter = std::max(FetchAfter, O->FetchAfter()); + + return FetchAfter; +} + /*}}}*/ +void pkgAcquire::Queue::QItem::SyncDestinationFiles() const /*{{{*/ +{ + /* ensure that the first owner has the best partial file of all and + the rest have (potentially dangling) symlinks to it so that + everything (like progress reporting) finds it easily */ + std::string superfile = Owner->DestFile; + off_t supersize = 0; + for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O) + { + if ((*O)->DestFile == superfile) + continue; + struct stat file; + if (lstat((*O)->DestFile.c_str(),&file) == 0) + { + if ((file.st_mode & S_IFREG) == 0) + RemoveFile("SyncDestinationFiles", (*O)->DestFile); + else if (supersize < file.st_size) + { + supersize = file.st_size; + RemoveFile("SyncDestinationFiles", superfile); + rename((*O)->DestFile.c_str(), superfile.c_str()); + } + else + RemoveFile("SyncDestinationFiles", (*O)->DestFile); + if (symlink(superfile.c_str(), (*O)->DestFile.c_str()) != 0) + { + ; // not a problem per-se and no real alternative + } + } + } +} + /*}}}*/ +std::string pkgAcquire::Queue::QItem::Custom600Headers() const /*{{{*/ +{ + /* The others are relatively easy to merge, but this one? + Lets not merge and see how far we can run with it… + Likely, nobody will ever notice as all the items will + be of the same class and hence generate the same headers. */ + return Owner->Custom600Headers(); +} + /*}}}*/ + +// AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(-1), Update(true), MorePulses(false) +{ + Start(); +} + /*}}}*/ +// AcquireStatus::Pulse - Called periodically /*{{{*/ +// --------------------------------------------------------------------- +/* This computes some internal state variables for the derived classes to + use. It generates the current downloaded bytes and total bytes to download + as well as the current CPS estimate. */ +static struct timeval GetTimevalFromSteadyClock() +{ + return SteadyDurationToTimeVal(std::chrono::steady_clock::now().time_since_epoch()); +} +bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) +{ + TotalBytes = 0; + CurrentBytes = 0; + TotalItems = 0; + CurrentItems = 0; + + // Compute the total number of bytes to fetch + unsigned int Unknown = 0; + unsigned int Count = 0; + bool ExpectAdditionalItems = false; + for (pkgAcquire::ItemCIterator I = Owner->ItemsBegin(); + I != Owner->ItemsEnd(); + ++I, ++Count) + { + TotalItems++; + if ((*I)->Status == pkgAcquire::Item::StatDone) + ++CurrentItems; + + // do we expect to acquire more files than we know of yet? + if ((*I)->ExpectedAdditionalItems > 0) + ExpectAdditionalItems = true; + + TotalBytes += (*I)->FileSize; + if ((*I)->Complete == true) + CurrentBytes += (*I)->FileSize; + if ((*I)->FileSize == 0 && (*I)->Complete == false) + ++Unknown; + } + + // Compute the current completion + unsigned long long ResumeSize = 0; + for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0; + I = Owner->WorkerStep(I)) + { + if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false) + { + CurrentBytes += I->CurrentItem->CurrentSize; + ResumeSize += I->CurrentItem->ResumePoint; + + // Files with unknown size always have 100% completion + if (I->CurrentItem->Owner->FileSize == 0 && + I->CurrentItem->Owner->Complete == false) + TotalBytes += I->CurrentItem->CurrentSize; + } + } + + // Normalize the figures and account for unknown size downloads + if (TotalBytes <= 0) + TotalBytes = 1; + if (Unknown == Count) + TotalBytes = Unknown; + + // Wha?! Is not supposed to happen. + if (CurrentBytes > TotalBytes) + CurrentBytes = TotalBytes; + + // Compute the CPS + struct timeval NewTime = GetTimevalFromSteadyClock(); + + if ((NewTime.tv_sec - Time.tv_sec == 6 && NewTime.tv_usec > Time.tv_usec) || + NewTime.tv_sec - Time.tv_sec > 6) + { + std::chrono::duration<double> Delta = + std::chrono::seconds(NewTime.tv_sec - Time.tv_sec) + + std::chrono::microseconds(NewTime.tv_usec - Time.tv_usec); + + // Compute the CPS value + if (Delta < std::chrono::milliseconds(10)) + CurrentCPS = 0; + else + CurrentCPS = ((CurrentBytes - ResumeSize) - LastBytes)/ Delta.count(); + LastBytes = CurrentBytes - ResumeSize; + ElapsedTime = llround(Delta.count()); + Time = NewTime; + } + + double const OldPercent = Percent; + // calculate the percentage, if we have too little data assume 1% + if (ExpectAdditionalItems) + Percent = 0; + else + // use both files and bytes because bytes can be unreliable + Percent = (0.8 * (CurrentBytes/double(TotalBytes)*100.0) + + 0.2 * (CurrentItems/double(TotalItems)*100.0)); + + // debug + if (_config->FindB("Debug::acquire::progress", false) == true) + { + std::clog + << "[" + << std::setw(5) << std::setprecision(4) << std::showpoint << Percent + << "]" + << " Bytes: " + << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes) + << " # Files: " + << CurrentItems << " / " << TotalItems + << std::endl; + } + + double const DiffPercent = Percent - OldPercent; + if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true) + return true; + + int fd = _config->FindI("APT::Status-Fd",-1); + if(fd > 0) + { + unsigned long long ETA = 0; + if(CurrentCPS > 0 && TotalBytes > CurrentBytes) + ETA = (TotalBytes - CurrentBytes) / CurrentCPS; + + std::string msg; + long i = CurrentItems < TotalItems ? CurrentItems + 1 : CurrentItems; + // only show the ETA if it makes sense + auto const twodays = std::chrono::seconds(std::chrono::hours(24 * 2)).count(); + if (ETA > 0 && ETA < static_cast<decltype(ETA)>(twodays)) + strprintf(msg, _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str()); + else + strprintf(msg, _("Retrieving file %li of %li"), i, TotalItems); + + // build the status str + std::ostringstream str; + str.imbue(std::locale::classic()); + str.precision(4); + str << "dlstatus" << ':' << std::fixed << i << ':' << Percent << ':' << msg << '\n'; + auto const dlstatus = str.str(); + FileFd::Write(fd, dlstatus.data(), dlstatus.size()); + } + + return true; +} + /*}}}*/ +// AcquireStatus::Start - Called when the download is started /*{{{*/ +// --------------------------------------------------------------------- +/* We just reset the counters */ +void pkgAcquireStatus::Start() +{ + Time = StartTime = GetTimevalFromSteadyClock(); + LastBytes = 0; + CurrentCPS = 0; + CurrentBytes = 0; + TotalBytes = 0; + FetchedBytes = 0; + ElapsedTime = 0; + TotalItems = 0; + CurrentItems = 0; +} + /*}}}*/ +// AcquireStatus::Stop - Finished downloading /*{{{*/ +// --------------------------------------------------------------------- +/* This accurately computes the elapsed time and the total overall CPS. */ +void pkgAcquireStatus::Stop() +{ + // Compute the CPS and elapsed time + struct timeval NewTime = GetTimevalFromSteadyClock(); + + std::chrono::duration<double> Delta = + std::chrono::seconds(NewTime.tv_sec - StartTime.tv_sec) + + std::chrono::microseconds(NewTime.tv_usec - StartTime.tv_usec); + + // Compute the CPS value + if (Delta < std::chrono::milliseconds(10)) + CurrentCPS = 0; + else + CurrentCPS = FetchedBytes / Delta.count(); + LastBytes = CurrentBytes; + ElapsedTime = llround(Delta.count()); +} + /*}}}*/ +// AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/ +// --------------------------------------------------------------------- +/* This is used to get accurate final transfer rate reporting. */ +void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume) +{ + FetchedBytes += Size - Resume; +} + /*}}}*/ +bool pkgAcquireStatus::ReleaseInfoChanges(metaIndex const * const LastRelease, metaIndex const * const CurrentRelease, std::vector<ReleaseInfoChange> &&Changes)/*{{{*/ +{ + (void) LastRelease; + (void) CurrentRelease; + return ReleaseInfoChangesAsGlobalErrors(std::move(Changes)); +} + /*}}}*/ +bool pkgAcquireStatus::ReleaseInfoChangesAsGlobalErrors(std::vector<ReleaseInfoChange> &&Changes)/*{{{*/ +{ + bool AllOkay = true; + for (auto const &c: Changes) + if (c.DefaultAction) + _error->Notice("%s", c.Message.c_str()); + else + { + _error->Error("%s", c.Message.c_str()); + AllOkay = false; + } + return AllOkay; +} + /*}}}*/ + + +pkgAcquire::UriIterator::UriIterator(pkgAcquire::Queue *Q) : d(NULL), CurQ(Q), CurItem(0) +{ + while (CurItem == 0 && CurQ != 0) + { + CurItem = CurQ->Items; + CurQ = CurQ->Next; + } +} + +pkgAcquire::UriIterator::~UriIterator() {} +pkgAcquire::MethodConfig::~MethodConfig() { delete d; } +pkgAcquireStatus::~pkgAcquireStatus() {} diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h new file mode 100644 index 0000000..17b6876 --- /dev/null +++ b/apt-pkg/acquire.h @@ -0,0 +1,874 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire - File Acquiration + + This module contains the Acquire system. It is responsible for bringing + files into the local pathname space. It deals with URIs for files and + URI handlers responsible for downloading or finding the URIs. + + Each file to download is represented by an Acquire::Item class subclassed + into a specialization. The Item class can add itself to several URI + acquire queues each prioritized by the download scheduler. When the + system is run the proper URI handlers are spawned and the acquire + queues are fed into the handlers by the schedular until the queues are + empty. This allows for an Item to be downloaded from an alternate source + if the first try turns out to fail. It also allows concurrent downloading + of multiple items from multiple sources as well as dynamic balancing + of load between the sources. + + Scheduling of downloads is done on a first ask first get basis. This + preserves the order of the download as much as possible. And means the + fastest source will tend to process the largest number of files. + + Internal methods and queues for performing gzip decompression, + md5sum hashing and file copying are provided to allow items to apply + a number of transformations to the data files they are working with. + + ##################################################################### */ + /*}}}*/ + +/** \defgroup acquire Acquire system {{{ + * + * \brief The Acquire system is responsible for retrieving files from + * local or remote URIs and postprocessing them (for instance, + * verifying their authenticity). The core class in this system is + * pkgAcquire, which is responsible for managing the download queues + * during the download. There is at least one download queue for + * each supported protocol; protocols such as http may provide one + * queue per host. + * + * Each file to download is represented by a subclass of + * pkgAcquire::Item. The files add themselves to the download + * queue(s) by providing their URI information to + * pkgAcquire::Item::QueueURI, which calls pkgAcquire::Enqueue. + * + * Once the system is set up, the Run method will spawn subprocesses + * to handle the enqueued URIs; the scheduler will then take items + * from the queues and feed them into the handlers until the queues + * are empty. + * + * \todo Acquire supports inserting an object into several queues at + * once, but it is not clear what its behavior in this case is, and + * no subclass of pkgAcquire::Item seems to actually use this + * capability. + */ /*}}}*/ + +/** \addtogroup acquire + * + * @{ + * + * \file acquire.h + */ + +#ifndef PKGLIB_ACQUIRE_H +#define PKGLIB_ACQUIRE_H + +#include <apt-pkg/hashes.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/weakptr.h> + +#include <chrono> +#include <string> +#include <vector> + +#include <stddef.h> +#include <sys/select.h> +#include <sys/time.h> + + + +class pkgAcquireStatus; +class metaIndex; + +/** \brief The core download scheduler. {{{ + * + * This class represents an ongoing download. It manages the lists + * of active and pending downloads and handles setting up and tearing + * down download-related structures. + * + * \todo Why all the protected data items and methods? + */ +class APT_PUBLIC pkgAcquire +{ + private: + /** \brief The monotonic clock used by the Acquire system */ + using clock = std::chrono::steady_clock; + /** \brief Time point on our monotonic clock */ + using time_point = std::chrono::time_point<clock>; + /** \brief FD of the Lock file we acquire in Setup (if any) */ + int LockFD; + /** \brief dpointer placeholder (for later in case we need it) */ + void * const d; + + public: + + class Item; + class Queue; + class Worker; + struct MethodConfig; + struct ItemDesc; + friend class Item; + friend class pkgAcqMetaBase; + friend class Queue; + + typedef std::vector<Item *>::iterator ItemIterator; + typedef std::vector<Item *>::const_iterator ItemCIterator; + + protected: + + /** \brief A list of items to download. + * + * This is built monotonically as items are created and only + * emptied when the download shuts down. + */ + std::vector<Item *> Items; + + /** \brief The head of the list of active queues. + * + * \todo why a hand-managed list of queues instead of std::list or + * std::set? + */ + Queue *Queues; + + /** \brief The head of the list of active workers. + * + * \todo why a hand-managed list of workers instead of std::list + * or std::set? + */ + Worker *Workers; + + /** \brief The head of the list of acquire method configurations. + * + * Each protocol (http, ftp, gzip, etc) via which files can be + * fetched can have a representation in this list. The + * configuration data is filled in by parsing the 100 Capabilities + * string output by a method on startup (see + * pkgAcqMethod::pkgAcqMethod and pkgAcquire::GetConfig). + * + * \todo why a hand-managed config dictionary instead of std::map? + */ + MethodConfig *Configs; + + /** \brief The progress indicator for this download. */ + pkgAcquireStatus *Log; + + /** \brief The number of files which are to be fetched. */ + unsigned long ToFetch; + + // Configurable parameters for the scheduler + + /** \brief Represents the queuing strategy for remote URIs. */ + enum QueueStrategy { + /** \brief Generate one queue for each protocol/host combination; downloads from + * multiple hosts can proceed in parallel. + */ + QueueHost, + /** \brief Generate a single queue for each protocol; serialize + * downloads from multiple hosts. + */ + QueueAccess} QueueMode; + + /** \brief If \b true, debugging information will be dumped to std::clog. */ + bool const Debug; + /** \brief If \b true, a download is currently in progress. */ + bool Running; + + /** \brief Add the given item to the list of items. */ + void Add(Item *Item); + + /** \brief Remove the given item from the list of items. */ + void Remove(Item *Item); + + /** \brief Add the given worker to the list of workers. */ + void Add(Worker *Work); + + /** \brief Remove the given worker from the list of workers. */ + void Remove(Worker *Work); + + /** \brief Insert the given fetch request into the appropriate queue. + * + * \param Item The URI to download and the item to download it + * for. Copied by value into the queue; no reference to Item is + * retained. + */ + void Enqueue(ItemDesc &Item); + + /** \brief Remove all fetch requests for this item from all queues. */ + void Dequeue(Item *Item); + + /** \brief Determine the fetch method and queue of a URI. + * + * \param URI The URI to fetch. + * + * \param[out] Config A location in which to place the method via + * which the URI is to be fetched. + * + * \return the string-name of the queue in which a fetch request + * for the given URI should be placed. + */ + std::string QueueName(std::string URI,MethodConfig const *&Config); + + /** \brief Build up the set of file descriptors upon which select() should + * block. + * + * The default implementation inserts the file descriptors + * corresponding to active downloads. + * + * \param[out] Fd The largest file descriptor in the generated sets. + * + * \param[out] RSet The set of file descriptors that should be + * watched for input. + * + * \param[out] WSet The set of file descriptors that should be + * watched for output. + */ + virtual void SetFds(int &Fd,fd_set *RSet,fd_set *WSet); + + /** Handle input from and output to file descriptors which select() + * has determined are ready. The default implementation + * dispatches to all active downloads. + * + * \param RSet The set of file descriptors that are ready for + * input. + * + * \param WSet The set of file descriptors that are ready for + * output. + * + * \return false if there is an error condition on one of the fds + */ + virtual bool RunFds(fd_set *RSet,fd_set *WSet); + + /** \brief Check for idle queues with ready-to-fetch items. + * + * Called by pkgAcquire::Queue::Done each time an item is dequeued + * but remains on some queues; i.e., another queue should start + * fetching it. + */ + void Bump(); + + public: + + /** \brief Retrieve information about a fetch method by name. + * + * \param Access The name of the method to look up. + * + * \return the method whose name is Access, or \b NULL if no such method exists. + */ + MethodConfig *GetConfig(std::string Access); + + /** \brief Provides information on how a download terminated. */ + enum RunResult { + /** \brief All files were fetched successfully. */ + Continue, + + /** \brief Some files failed to download. */ + Failed, + + /** \brief The download was cancelled by the user (i.e., #Log's + * pkgAcquireStatus::Pulse() method returned \b false). + */ + Cancelled}; + + /** \brief Download all the items that have been Add()ed to this + * download process. + * + * This method will block until the download completes, invoking + * methods on #Log to report on the progress of the download. + * + * \param PulseInterval The method pkgAcquireStatus::Pulse will be + * invoked on #Log at intervals of PulseInterval milliseconds. + * + * \return the result of the download. + */ + RunResult Run(int PulseInterval=500000); + + /** \brief Remove all items from this download process, terminate + * all download workers, and empty all queues. + */ + void Shutdown(); + + /** \brief Get the first Worker object. + * + * \return the first active worker in this download process. + */ + inline Worker *WorkersBegin() {return Workers;}; + + /** \brief Advance to the next Worker object. + * + * \return the worker immediately following I, or \b NULL if none + * exists. + */ + Worker *WorkerStep(Worker *I) APT_PURE; + + /** \brief Get the head of the list of items. */ + inline ItemIterator ItemsBegin() {return Items.begin();}; + inline ItemCIterator ItemsBegin() const {return Items.begin();}; + + /** \brief Get the end iterator of the list of items. */ + inline ItemIterator ItemsEnd() {return Items.end();}; + inline ItemCIterator ItemsEnd() const {return Items.end();}; + + // Iterate over queued Item URIs + class UriIterator; + /** \brief Get the head of the list of enqueued item URIs. + * + * This iterator will step over every element of every active + * queue. + */ + UriIterator UriBegin(); + /** \brief Get the end iterator of the list of enqueued item URIs. */ + UriIterator UriEnd(); + + /** Deletes each entry in the given directory that is not being + * downloaded by this object. For instance, when downloading new + * list files, calling Clean() will delete the old ones. + * + * \param Dir The directory to be cleaned out. + * + * \return \b true if the directory exists and is readable. + */ + bool Clean(std::string Dir); + + /** \return the total size in bytes of all the items included in + * this download. + */ + unsigned long long TotalNeeded(); + + /** \return the size in bytes of all non-local items included in + * this download. + */ + unsigned long long FetchNeeded(); + + /** \return the amount of data to be fetched that is already + * present on the filesystem. + */ + unsigned long long PartialPresent(); + + void SetLog(pkgAcquireStatus *Progress) { Log = Progress; } + + /** \brief acquire lock and perform directory setup + * + * \param Lock defines a lock file that should be acquired to ensure + * only one Acquire class is in action at the time or an empty string + * if no lock file should be used. If set also all needed directories + * will be created and setup. + */ + bool GetLock(std::string const &Lock); + + /** \brief Construct a new pkgAcquire. */ + explicit pkgAcquire(pkgAcquireStatus *Log); + pkgAcquire(); + + /** \brief Destroy this pkgAcquire object. + * + * Destroys all queue, method, and item objects associated with + * this download. + */ + virtual ~pkgAcquire(); + + APT_HIDDEN static std::string URIEncode(std::string const &part); + + private: + APT_HIDDEN void Initialize(); +}; + +/** \brief Represents a single download source from which an item + * should be downloaded. + * + * An item may have several associated ItemDescs over its lifetime. + */ +struct APT_PUBLIC pkgAcquire::ItemDesc : public WeakPointable +{ + /** \brief URI from which to download this item. */ + std::string URI; + /** \brief description of this item. */ + std::string Description; + /** \brief shorter description of this item. */ + std::string ShortDesc; + /** \brief underlying item which is to be downloaded. */ + Item *Owner; +}; + /*}}}*/ +/** \brief A single download queue in a pkgAcquire object. {{{ + * + * \todo Why so many protected values? + */ +class APT_PUBLIC pkgAcquire::Queue +{ + friend class pkgAcquire; + friend class pkgAcquire::UriIterator; + friend class pkgAcquire::Worker; + + /** \brief dpointer placeholder (for later in case we need it) */ + void * const d; + + /** \brief The next queue in the pkgAcquire object's list of queues. */ + Queue *Next; + + protected: + + /** \brief A single item placed in this queue. */ + struct QItem : public ItemDesc + { + /** \brief The next item in the queue. */ + QItem *Next; + /** \brief The worker associated with this item, if any. */ + pkgAcquire::Worker *Worker; + + /** \brief The underlying items interested in the download */ + std::vector<Item*> Owners; + + /** \brief How many bytes of the file have been downloaded. Zero + * if the current progress of the file cannot be determined. + */ + unsigned long long CurrentSize = 0; + + /** \brief The total number of bytes to be downloaded. Zero if the + * total size of the final is unknown. + */ + unsigned long long TotalSize = 0; + + /** \brief How much of the file was already downloaded prior to + * starting this worker. + */ + unsigned long long ResumePoint = 0; + + typedef std::vector<Item*>::const_iterator owner_iterator; + + /** \brief Assign the ItemDesc portion of this QItem from + * another ItemDesc + */ + void operator =(pkgAcquire::ItemDesc const &I) + { + URI = I.URI; + Description = I.Description; + ShortDesc = I.ShortDesc; + Owners.clear(); + Owners.push_back(I.Owner); + Owner = I.Owner; + }; + + /** @return the sum of all expected hashes by all owners */ + HashStringList GetExpectedHashes() const; + + /** @return smallest maximum size of all owners */ + unsigned long long GetMaximumSize() const; + + /** \brief get partial files in order */ + void SyncDestinationFiles() const; + + /** @return the custom headers to use for this item */ + std::string Custom600Headers() const; + /** @return the maximum priority of this item */ + int APT_HIDDEN GetPriority() const; + /** @return the maximum time to fetch this item at */ + time_point APT_HIDDEN GetFetchAfter() const; + }; + + /** \brief The name of this queue. */ + std::string Name; + + /** \brief The head of the list of items contained in this queue. + * + * \todo why a by-hand list instead of an STL structure? + */ + QItem *Items; + + /** \brief The head of the list of workers associated with this queue. + * + * \todo This is plural because support exists in Queue for + * multiple workers. However, it does not appear that there is + * any way to actually associate more than one worker with a + * queue. + * + * \todo Why not just use a std::set? + */ + pkgAcquire::Worker *Workers; + + /** \brief the download scheduler with which this queue is associated. */ + pkgAcquire *Owner; + + /** \brief The number of entries in this queue that are currently + * being downloaded. + */ + signed long PipeDepth; + + /** \brief The maximum number of entries that this queue will + * attempt to download at once. + */ + unsigned long MaxPipeDepth; + + public: + + /** \brief Insert the given fetch request into this queue. + * + * \return \b true if the queuing was successful. May return + * \b false if the Item is already in the queue + */ + bool Enqueue(ItemDesc &Item); + + /** \brief Remove all fetch requests for the given item from this queue. + * + * \return \b true if at least one request was removed from the queue. + */ + bool Dequeue(Item *Owner); + + /** \brief Locate an item in this queue. + * + * \param URI A URI to match against. + * \param Owner A pkgAcquire::Worker to match against. + * + * \return the first item in the queue whose URI is #URI and that + * is being downloaded by #Owner. + */ + QItem *FindItem(std::string URI,pkgAcquire::Worker *Owner) APT_PURE; + + /** Presumably this should start downloading an item? + * + * \todo Unimplemented. Implement it or remove? + */ + bool ItemStart(QItem *Itm,unsigned long long Size); + + /** \brief Remove the given item from this queue and set its state + * to pkgAcquire::Item::StatDone. + * + * If this is the only queue containing the item, the item is also + * removed from the main queue by calling pkgAcquire::Dequeue. + * + * \param Itm The item to remove. + * + * \return \b true if no errors are encountered. + */ + bool ItemDone(QItem *Itm); + + /** \brief Start the worker process associated with this queue. + * + * If a worker process is already associated with this queue, + * this is equivalent to calling Cycle(). + * + * \return \b true if the startup was successful. + */ + bool Startup(); + + /** \brief Shut down the worker process associated with this queue. + * + * \param Final If \b true, then the process is stopped unconditionally. + * Otherwise, it is only stopped if it does not need cleanup + * as indicated by the pkgAcqMethod::NeedsCleanup member of + * its configuration. + * + * \return \b true. + */ + bool Shutdown(bool Final); + + /** \brief Send idle items to the worker process. + * + * Fills up the pipeline by inserting idle items into the worker's queue. + */ + bool Cycle(); + + /** \brief Check for items that could be enqueued. + * + * Call this after an item placed in multiple queues has gone from + * the pkgAcquire::Item::StatFetching state to the + * pkgAcquire::Item::StatIdle state, to possibly refill an empty queue. + * This is an alias for Cycle(). + * + * \todo Why both this and Cycle()? Are they expected to be + * different someday? + */ + void Bump(); + + /** \brief Create a new Queue. + * + * \param Name The name of the new queue. + * \param Owner The download process that owns the new queue. + */ + Queue(std::string const &Name,pkgAcquire * const Owner); + + /** Shut down all the worker processes associated with this queue + * and empty the queue. + */ + virtual ~Queue(); +}; + /*}}}*/ +/** \brief Iterates over all the URIs being fetched by a pkgAcquire object. {{{*/ +class APT_PUBLIC pkgAcquire::UriIterator +{ + /** \brief dpointer placeholder (for later in case we need it) */ + void * const d; + + /** The next queue to iterate over. */ + pkgAcquire::Queue *CurQ; + /** The item that we currently point at. */ + pkgAcquire::Queue::QItem *CurItem; + + public: + + inline void operator ++() {operator ++(0);}; + + void operator ++(int) + { + CurItem = CurItem->Next; + while (CurItem == 0 && CurQ != 0) + { + CurItem = CurQ->Items; + CurQ = CurQ->Next; + } + }; + + inline pkgAcquire::Queue::QItem const *operator ->() const {return CurItem;}; + inline bool operator !=(UriIterator const &rhs) const {return rhs.CurQ != CurQ || rhs.CurItem != CurItem;}; + inline bool operator ==(UriIterator const &rhs) const {return rhs.CurQ == CurQ && rhs.CurItem == CurItem;}; + + /** \brief Create a new UriIterator. + * + * \param Q The queue over which this UriIterator should iterate. + */ + explicit UriIterator(pkgAcquire::Queue *Q); + virtual ~UriIterator(); +}; + /*}}}*/ +/** \brief Information about the properties of a single acquire method. {{{*/ +struct APT_PUBLIC pkgAcquire::MethodConfig +{ + class Private; + /** \brief dpointer placeholder (for later in case we need it) */ + Private *const d; + + /** \brief The next link on the acquire method list. + * + * \todo Why not an STL container? + */ + MethodConfig *Next; + + /** \brief The name of this acquire method (e.g., http). */ + std::string Access; + + /** \brief The implementation version of this acquire method. */ + std::string Version; + + /** \brief If \b true, only one download queue should be created for this + * method. + */ + bool SingleInstance; + + /** \brief If \b true, this method supports pipelined downloading. */ + bool Pipeline; + + /** \brief If \b true, the worker process should send the entire + * APT configuration tree to the fetch subprocess when it starts + * up. + */ + bool SendConfig; + + /** \brief If \b true, this fetch method does not require network access; + * all files are to be acquired from the local disk. + */ + bool LocalOnly; + + /** \brief If \b true, the subprocess has to carry out some cleanup + * actions before shutting down. + * + * For instance, the cdrom method needs to unmount the CD after it + * finishes. + */ + bool NeedsCleanup; + + /** \brief If \b true, this fetch method acquires files from removable media. */ + bool Removable; + + /** \brief Set up the default method parameters. + * + * All fields are initialized to NULL, "", or \b false as + * appropriate. + */ + MethodConfig(); + + APT_HIDDEN bool GetAuxRequests() const; + APT_HIDDEN void SetAuxRequests(bool const value); + APT_HIDDEN bool GetSendURIEncoded() const; + APT_HIDDEN void SetSendURIEncoded(bool const value); + + virtual ~MethodConfig(); +}; + /*}}}*/ +/** \brief A monitor object for downloads controlled by the pkgAcquire class. {{{ + * + * \todo Why protected members? + */ +class APT_PUBLIC pkgAcquireStatus +{ + /** \brief dpointer placeholder (for later in case we need it) */ + void * const d; + + protected: + + /** \brief The last time at which this monitor object was updated. */ + struct timeval Time; + + /** \brief The time at which the download started. */ + struct timeval StartTime; + + /** \brief The number of bytes fetched as of the previous call to + * pkgAcquireStatus::Pulse, including local items. + */ + unsigned long long LastBytes; + + /** \brief The current rate of download as of the most recent call + * to pkgAcquireStatus::Pulse, in bytes per second. + */ + unsigned long long CurrentCPS; + + /** \brief The number of bytes fetched as of the most recent call + * to pkgAcquireStatus::Pulse, including local items. + */ + unsigned long long CurrentBytes; + + /** \brief The total number of bytes that need to be fetched. + * + * \warning This member is inaccurate, as new items might be + * enqueued while the download is in progress! + */ + unsigned long long TotalBytes; + + /** \brief The total number of bytes accounted for by items that + * were successfully fetched. + */ + unsigned long long FetchedBytes; + + /** \brief The amount of time that has elapsed since the download + * started. + */ + unsigned long long ElapsedTime; + + /** \brief The total number of items that need to be fetched. + * + * \warning This member is inaccurate, as new items might be + * enqueued while the download is in progress! + */ + unsigned long TotalItems; + + /** \brief The number of items that have been successfully downloaded. */ + unsigned long CurrentItems; + + /** \brief The estimated percentage of the download (0-100) + */ + double Percent; + + public: + + /** \brief If \b true, the download scheduler should call Pulse() + * at the next available opportunity. + */ + bool Update; + + /** \brief If \b true, extra Pulse() invocations will be performed. + * + * With this option set, Pulse() will be called every time that a + * download item starts downloading, finishes downloading, or + * terminates with an error. + */ + bool MorePulses; + + /** \brief Invoked when a local or remote file has been completely fetched. + * + * \param Size The size of the file fetched. + * + * \param ResumePoint How much of the file was already fetched. + */ + virtual void Fetched(unsigned long long Size,unsigned long long ResumePoint); + + /** \brief Invoked when the user should be prompted to change the + * inserted removable media. + * + * This method should not return until the user has confirmed to + * the user interface that the media change is complete. + * + * \param Media The name of the media type that should be changed. + * + * \param Drive The identifying name of the drive whose media + * should be changed. + * + * \return \b true if the user confirms the media change, \b + * false if it is cancelled. + * + * \todo This is a horrible blocking monster; it should be CPSed + * with prejudice. + */ + virtual bool MediaChange(std::string Media,std::string Drive) = 0; + + struct ReleaseInfoChange + { + std::string Type; /*!< Type of the change like "Origin", "Codename", "Version", … */ + std::string From; /*!< old value */ + std::string To; /*!< new value */ + std::string Message; /*!< translated message describing the change */ + bool DefaultAction; /*!< true if the change is informational, false if it must be explicitly confirmed */ + }; + /** \brief ask the user for confirmation of changes to infos about a repository + * + * This method should present the user with a choice of accepting the change + * or not and indicate the user opinion via the return value. If DefaultAction is true + * it is acceptable to only notify the user about the change, but to accept the change + * automatically on behalf of the user. + * + * The default implementation will fail if any Change has DefaultAction == false. Regardless of + * success it will print for each change the message attached to it via GlobalError either as an + * error (if DefaultAction == false) or as a notice otherwise. + * + * @param LastRelease can be used to extract further information from the previous Release file + * @param CurrentRelease can be used to extract further information from the current Release file + * @param Changes is an array of changes alongside explanatory messages + * which should be presented in some way to the user. + * @return \b true if all changes are accepted by user, otherwise or if user can't be asked \b false + */ + virtual bool ReleaseInfoChanges(metaIndex const * const LastRelease, metaIndex const * const CurrentRelease, std::vector<ReleaseInfoChange> &&Changes); + APT_HIDDEN static bool ReleaseInfoChangesAsGlobalErrors(std::vector<ReleaseInfoChange> &&Changes); + + /** \brief Invoked when an item is confirmed to be up-to-date. + + * For instance, when an HTTP download is informed that the file on + * the server was not modified. + */ + virtual void IMSHit(pkgAcquire::ItemDesc &/*Itm*/) {}; + + /** \brief Invoked when some of an item's data is fetched. */ + virtual void Fetch(pkgAcquire::ItemDesc &/*Itm*/) {}; + + /** \brief Invoked when an item is successfully and completely fetched. */ + virtual void Done(pkgAcquire::ItemDesc &/*Itm*/) {}; + + /** \brief Invoked when the process of fetching an item encounters + * a fatal error. + */ + virtual void Fail(pkgAcquire::ItemDesc &/*Itm*/) {}; + + /** \brief Periodically invoked while the Acquire process is underway. + * + * Subclasses should first call pkgAcquireStatus::Pulse(), then + * update their status output. The download process is blocked + * while Pulse() is being called. + * + * \return \b false if the user asked to cancel the whole Acquire process. + * + * \see pkgAcquire::Run + */ + virtual bool Pulse(pkgAcquire *Owner); + + /** \brief Invoked when the Acquire process starts running. */ + virtual void Start(); + + /** \brief Invoked when the Acquire process stops running. */ + virtual void Stop(); + + /** \brief Initialize all counters to 0 and the time to the current time. */ + pkgAcquireStatus(); + virtual ~pkgAcquireStatus(); +}; + /*}}}*/ +/** @} */ + +#endif diff --git a/apt-pkg/algorithms.cc b/apt-pkg/algorithms.cc new file mode 100644 index 0000000..26d8c71 --- /dev/null +++ b/apt-pkg/algorithms.cc @@ -0,0 +1,1623 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Algorithms - A set of misc algorithms + + The pkgProblemResolver class has become insanely complex and + very sophisticated, it handles every test case I have thrown at it + to my satisfaction. Understanding exactly why all the steps the class + does are required is difficult and changing though not very risky + may result in other cases not working. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/algorithms.h> +#include <apt-pkg/cachefilter.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/depcache.h> +#include <apt-pkg/dpkgpm.h> +#include <apt-pkg/edsp.h> +#include <apt-pkg/error.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/packagemanager.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/string_view.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/version.h> + +#include <apt-pkg/prettyprinters.h> + +#include <cstdlib> +#include <iostream> +#include <map> +#include <set> +#include <sstream> +#include <string> +#include <utility> +#include <vector> +#include <string.h> +#include <sys/utsname.h> + +#include <apti18n.h> + /*}}}*/ +using namespace std; + +class APT_HIDDEN pkgSimulatePrivate +{ +public: + std::vector<pkgDPkgPM::Item> List; +}; +// Simulate::Simulate - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* The legacy translations here of input Pkg iterators is obsolete, + this is not necessary since the pkgCaches are fully shared now. */ +pkgSimulate::pkgSimulate(pkgDepCache *Cache) : pkgPackageManager(Cache), + d(new pkgSimulatePrivate()), iPolicy(Cache), + Sim(&Cache->GetCache(),&iPolicy), + group(Sim) +{ + Sim.Init(0); + auto PackageCount = Cache->Head().PackageCount; + Flags = new unsigned char[PackageCount]; + memset(Flags,0,sizeof(*Flags)*PackageCount); + + // Fake a filename so as not to activate the media swapping + string Jnk = "SIMULATE"; + for (decltype(PackageCount) I = 0; I != PackageCount; ++I) + FileNames[I] = Jnk; + + Cache->CheckConsistency("simulate"); +} + /*}}}*/ +// Simulate::~Simulate - Destructor /*{{{*/ +pkgSimulate::~pkgSimulate() +{ + delete[] Flags; + delete d; +} + /*}}}*/ +// Simulate::Describe - Describe a package /*{{{*/ +// --------------------------------------------------------------------- +/* Parameter Current == true displays the current package version, + Parameter Candidate == true displays the candidate package version */ +void pkgSimulate::Describe(PkgIterator Pkg,ostream &out,bool Current,bool Candidate) +{ + VerIterator Ver(Sim); + + out << Pkg.FullName(true); + + if (Current == true) + { + Ver = Pkg.CurrentVer(); + if (Ver.end() == false) + out << " [" << Ver.VerStr() << ']'; + } + + if (Candidate == true) + { + Ver = Sim[Pkg].CandidateVerIter(Sim); + if (Ver.end() == true) + return; + + out << " (" << Ver.VerStr() << ' ' << Ver.RelStr() << ')'; + } +} + /*}}}*/ +// Simulate::Install - Simulate unpacking of a package /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSimulate::Install(PkgIterator iPkg,string File) +{ + if (iPkg.end() || File.empty()) + return false; + d->List.emplace_back(pkgDPkgPM::Item::Install, iPkg, File); + return true; +} +bool pkgSimulate::RealInstall(PkgIterator iPkg,string /*File*/) +{ + // Adapt the iterator + PkgIterator Pkg = Sim.FindPkg(iPkg.Name(), iPkg.Arch()); + Flags[Pkg->ID] = 1; + + cout << "Inst "; + Describe(Pkg,cout,true,true); + Sim.MarkInstall(Pkg,false); + + // Look for broken conflicts+predepends. + for (PkgIterator I = Sim.PkgBegin(); I.end() == false; ++I) + { + if (Sim[I].InstallVer == 0) + continue; + + for (DepIterator D = Sim[I].InstVerIter(Sim).DependsList(); D.end() == false;) + { + DepIterator Start; + DepIterator End; + D.GlobOr(Start,End); + if (Start.IsNegative() == true || + End->Type == pkgCache::Dep::PreDepends) + { + if ((Sim[End] & pkgDepCache::DepGInstall) == 0) + { + cout << " [" << I.FullName(false) << " on " << Start.TargetPkg().FullName(false) << ']'; + if (Start->Type == pkgCache::Dep::Conflicts) + _error->Error("Fatal, conflicts violated %s",I.FullName(false).c_str()); + } + } + } + } + + if (Sim.BrokenCount() != 0) + ShortBreaks(); + else + cout << endl; + return true; +} + /*}}}*/ +// Simulate::Configure - Simulate configuration of a Package /*{{{*/ +// --------------------------------------------------------------------- +/* This is not an accurate simulation of relatity, we should really not + install the package.. For some investigations it may be necessary + however. */ +bool pkgSimulate::Configure(PkgIterator iPkg) +{ + if (iPkg.end()) + return false; + d->List.emplace_back(pkgDPkgPM::Item::Configure, iPkg); + return true; +} +bool pkgSimulate::RealConfigure(PkgIterator iPkg) +{ + // Adapt the iterator + PkgIterator Pkg = Sim.FindPkg(iPkg.Name(), iPkg.Arch()); + + Flags[Pkg->ID] = 2; + + if (Sim[Pkg].InstBroken() == true) + { + cout << "Conf " << Pkg.FullName(false) << " broken" << endl; + + Sim.Update(); + + // Print out each package and the failed dependencies + for (pkgCache::DepIterator D = Sim[Pkg].InstVerIter(Sim).DependsList(); D.end() == false; ++D) + { + if (Sim.IsImportantDep(D) == false || + (Sim[D] & pkgDepCache::DepInstall) != 0) + continue; + + if (D->Type == pkgCache::Dep::Obsoletes) + cout << " Obsoletes:" << D.TargetPkg().FullName(false); + else if (D->Type == pkgCache::Dep::Conflicts) + cout << " Conflicts:" << D.TargetPkg().FullName(false); + else if (D->Type == pkgCache::Dep::DpkgBreaks) + cout << " Breaks:" << D.TargetPkg().FullName(false); + else + cout << " Depends:" << D.TargetPkg().FullName(false); + } + cout << endl; + + _error->Error("Conf Broken %s",Pkg.FullName(false).c_str()); + } + else + { + cout << "Conf "; + Describe(Pkg,cout,false,true); + } + + if (Sim.BrokenCount() != 0) + ShortBreaks(); + else + cout << endl; + + return true; +} + /*}}}*/ +// Simulate::Remove - Simulate the removal of a package /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSimulate::Remove(PkgIterator iPkg,bool Purge) +{ + if (iPkg.end()) + return false; + d->List.emplace_back(Purge ? pkgDPkgPM::Item::Purge : pkgDPkgPM::Item::Remove, iPkg); + return true; +} +bool pkgSimulate::RealRemove(PkgIterator iPkg,bool Purge) +{ + // Adapt the iterator + PkgIterator Pkg = Sim.FindPkg(iPkg.Name(), iPkg.Arch()); + if (Pkg.end() == true) + { + std::cerr << (Purge ? "Purg" : "Remv") << " invalid package " << iPkg.FullName() << std::endl; + return false; + } + + Flags[Pkg->ID] = 3; + Sim.MarkDelete(Pkg); + + if (Purge == true) + cout << "Purg "; + else + cout << "Remv "; + Describe(Pkg,cout,true,false); + + if (Sim.BrokenCount() != 0) + ShortBreaks(); + else + cout << endl; + + return true; +} + /*}}}*/ +// Simulate::ShortBreaks - Print out a short line describing all breaks /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgSimulate::ShortBreaks() +{ + cout << " ["; + for (PkgIterator I = Sim.PkgBegin(); I.end() == false; ++I) + { + if (Sim[I].InstBroken() == true) + { + if (Flags[I->ID] == 0) + cout << I.FullName(false) << ' '; +/* else + cout << I.Name() << "! ";*/ + } + } + cout << ']' << endl; +} + /*}}}*/ +bool pkgSimulate::Go(APT::Progress::PackageManager *) /*{{{*/ +{ + if (pkgDPkgPM::ExpandPendingCalls(d->List, Cache) == false) + return false; + for (auto && I : d->List) + switch (I.Op) + { + case pkgDPkgPM::Item::Install: + if (RealInstall(I.Pkg, I.File) == false) + return false; + break; + case pkgDPkgPM::Item::Configure: + if (RealConfigure(I.Pkg) == false) + return false; + break; + case pkgDPkgPM::Item::Remove: + if (RealRemove(I.Pkg, false) == false) + return false; + break; + case pkgDPkgPM::Item::Purge: + if (RealRemove(I.Pkg, true) == false) + return false; + break; + case pkgDPkgPM::Item::ConfigurePending: + case pkgDPkgPM::Item::TriggersPending: + case pkgDPkgPM::Item::RemovePending: + case pkgDPkgPM::Item::PurgePending: + return _error->Error("Internal error, simulation encountered unexpected pending item"); + } + return true; +} + /*}}}*/ +// ApplyStatus - Adjust for non-ok packages /*{{{*/ +// --------------------------------------------------------------------- +/* We attempt to change the state of the all packages that have failed + installation toward their real state. The ordering code will perform + the necessary calculations to deal with the problems. */ +bool pkgApplyStatus(pkgDepCache &Cache) +{ + pkgDepCache::ActionGroup group(Cache); + + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + if (I->VersionList == 0) + continue; + + // Only choice for a ReInstReq package is to reinstall + if (I->InstState == pkgCache::State::ReInstReq || + I->InstState == pkgCache::State::HoldReInstReq) + { + if (I->CurrentVer != 0 && I.CurrentVer().Downloadable() == true) + Cache.MarkKeep(I, false, false); + else + { + // Is this right? Will dpkg choke on an upgrade? + if (Cache[I].CandidateVer != 0 && + Cache[I].CandidateVerIter(Cache).Downloadable() == true) + Cache.MarkInstall(I, false, 0, false); + else + return _error->Error(_("The package %s needs to be reinstalled, " + "but I can't find an archive for it."),I.FullName(true).c_str()); + } + + continue; + } + + switch (I->CurrentState) + { + /* This means installation failed somehow - it does not need to be + re-unpacked (probably) */ + case pkgCache::State::UnPacked: + case pkgCache::State::HalfConfigured: + case pkgCache::State::TriggersAwaited: + case pkgCache::State::TriggersPending: + if ((I->CurrentVer != 0 && I.CurrentVer().Downloadable() == true) || + I.State() != pkgCache::PkgIterator::NeedsUnpack) + Cache.MarkKeep(I, false, false); + else + { + if (Cache[I].CandidateVer != 0 && + Cache[I].CandidateVerIter(Cache).Downloadable() == true) + Cache.MarkInstall(I, true, 0, false); + else + Cache.MarkDelete(I, false, 0, false); + } + break; + + // This means removal failed + case pkgCache::State::HalfInstalled: + Cache.MarkDelete(I, false, 0, false); + break; + + default: + if (I->InstState != pkgCache::State::Ok) + return _error->Error("The package %s is not ok and I " + "don't know how to fix it!",I.FullName(false).c_str()); + } + } + return true; +} + /*}}}*/ +// FixBroken - Fix broken packages /*{{{*/ +// --------------------------------------------------------------------- +/* This autoinstalls every broken package and then runs the problem resolver + on the result. */ +bool pkgFixBroken(pkgDepCache &Cache) +{ + pkgDepCache::ActionGroup group(Cache); + + // Auto upgrade all broken packages + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + if (Cache[I].NowBroken() == true) + Cache.MarkInstall(I, true, 0, false); + + /* Fix packages that are in a NeedArchive state but don't have a + downloadable install version */ + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + if (I.State() != pkgCache::PkgIterator::NeedsUnpack || + Cache[I].Delete() == true) + continue; + + if (Cache[I].InstVerIter(Cache).Downloadable() == false) + continue; + + Cache.MarkInstall(I, true, 0, false); + } + + pkgProblemResolver Fix(&Cache); + return Fix.Resolve(true); +} + /*}}}*/ +// ProblemResolver::pkgProblemResolver - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgProblemResolver::pkgProblemResolver(pkgDepCache *pCache) : d(NULL), Cache(*pCache) +{ + // Allocate memory + auto const Size = Cache.Head().PackageCount; + Scores = new int[Size]; + Flags = new unsigned char[Size]; + memset(Flags,0,sizeof(*Flags)*Size); + + // Set debug to true to see its decision logic + Debug = _config->FindB("Debug::pkgProblemResolver",false); +} + /*}}}*/ +// ProblemResolver::~pkgProblemResolver - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgProblemResolver::~pkgProblemResolver() +{ + delete [] Scores; + delete [] Flags; +} + /*}}}*/ +// ProblemResolver::ScoreSort - Sort the list by score /*{{{*/ +// --------------------------------------------------------------------- +/* */ +int pkgProblemResolver::ScoreSort(Package const *A,Package const *B) +{ + if (Scores[A->ID] > Scores[B->ID]) + return -1; + if (Scores[A->ID] < Scores[B->ID]) + return 1; + return 0; +} + /*}}}*/ +// ProblemResolver::MakeScores - Make the score table /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgProblemResolver::MakeScores() +{ + auto const Size = Cache.Head().PackageCount; + memset(Scores,0,sizeof(*Scores)*Size); + + // maps to pkgCache::State::VerPriority: + // Required Important Standard Optional Extra + int PrioMap[] = { + 0, + _config->FindI("pkgProblemResolver::Scores::Required",3), + _config->FindI("pkgProblemResolver::Scores::Important",2), + _config->FindI("pkgProblemResolver::Scores::Standard",1), + _config->FindI("pkgProblemResolver::Scores::Optional",-1), + _config->FindI("pkgProblemResolver::Scores::Extra",-2) + }; + int PrioEssentials = _config->FindI("pkgProblemResolver::Scores::Essentials",100); + int PrioInstalledAndNotObsolete = _config->FindI("pkgProblemResolver::Scores::NotObsolete",1); + int DepMap[] = { + 0, + _config->FindI("pkgProblemResolver::Scores::Depends",1), + _config->FindI("pkgProblemResolver::Scores::PreDepends",1), + _config->FindI("pkgProblemResolver::Scores::Suggests",0), + _config->FindI("pkgProblemResolver::Scores::Recommends",1), + _config->FindI("pkgProblemResolver::Scores::Conflicts",-1), + _config->FindI("pkgProblemResolver::Scores::Replaces",0), + _config->FindI("pkgProblemResolver::Scores::Obsoletes",0), + _config->FindI("pkgProblemResolver::Scores::Breaks",-1), + _config->FindI("pkgProblemResolver::Scores::Enhances",0) + }; + int AddProtected = _config->FindI("pkgProblemResolver::Scores::AddProtected",10000); + int AddEssential = _config->FindI("pkgProblemResolver::Scores::AddEssential",5000); + + if (_config->FindB("Debug::pkgProblemResolver::ShowScores",false) == true) + clog << "Settings used to calculate pkgProblemResolver::Scores::" << endl + << " Required => " << PrioMap[pkgCache::State::Required] << endl + << " Important => " << PrioMap[pkgCache::State::Important] << endl + << " Standard => " << PrioMap[pkgCache::State::Standard] << endl + << " Optional => " << PrioMap[pkgCache::State::Optional] << endl + << " Extra => " << PrioMap[pkgCache::State::Extra] << endl + << " Essentials => " << PrioEssentials << endl + << " InstalledAndNotObsolete => " << PrioInstalledAndNotObsolete << endl + << " Pre-Depends => " << DepMap[pkgCache::Dep::PreDepends] << endl + << " Depends => " << DepMap[pkgCache::Dep::Depends] << endl + << " Recommends => " << DepMap[pkgCache::Dep::Recommends] << endl + << " Suggests => " << DepMap[pkgCache::Dep::Suggests] << endl + << " Conflicts => " << DepMap[pkgCache::Dep::Conflicts] << endl + << " Breaks => " << DepMap[pkgCache::Dep::DpkgBreaks] << endl + << " Replaces => " << DepMap[pkgCache::Dep::Replaces] << endl + << " Obsoletes => " << DepMap[pkgCache::Dep::Obsoletes] << endl + << " Enhances => " << DepMap[pkgCache::Dep::Enhances] << endl + << " AddProtected => " << AddProtected << endl + << " AddEssential => " << AddEssential << endl; + + // Generate the base scores for a package based on its properties + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + if (Cache[I].InstallVer == 0) + continue; + + int &Score = Scores[I->ID]; + + /* This is arbitrary, it should be high enough to elevate an + essantial package above most other packages but low enough + to allow an obsolete essential packages to be removed by + a conflicts on a powerful normal package (ie libc6) */ + if ((I->Flags & pkgCache::Flag::Essential) == pkgCache::Flag::Essential + || (I->Flags & pkgCache::Flag::Important) == pkgCache::Flag::Important) + Score += PrioEssentials; + + pkgCache::VerIterator const InstVer = Cache[I].InstVerIter(Cache); + // We apply priorities only to downloadable packages, all others are prio:extra + // as an obsolete prio:standard package can't be that standard anymore… + if (InstVer->Priority <= pkgCache::State::Extra && InstVer.Downloadable() == true) + Score += PrioMap[InstVer->Priority]; + else + Score += PrioMap[pkgCache::State::Extra]; + + /* This helps to fix oddball problems with conflicting packages + on the same level. We enhance the score of installed packages + if those are not obsolete */ + if (I->CurrentVer != 0 && Cache[I].CandidateVer != 0 && Cache[I].CandidateVerIter(Cache).Downloadable()) + Score += PrioInstalledAndNotObsolete; + + // propagate score points along dependencies + for (pkgCache::DepIterator D = InstVer.DependsList(); not D.end(); ++D) + { + if (DepMap[D->Type] == 0) + continue; + pkgCache::PkgIterator const T = D.TargetPkg(); + if (not D.IsIgnorable(T)) + { + if (D->Version != 0) + { + pkgCache::VerIterator const IV = Cache[T].InstVerIter(Cache); + if (IV.end() || not D.IsSatisfied(IV)) + continue; + } + Scores[T->ID] += DepMap[D->Type]; + } + + std::vector<map_id_t> providers; + for (auto Prv = T.ProvidesList(); not Prv.end(); ++Prv) + { + if (D.IsIgnorable(Prv)) + continue; + auto const PV = Prv.OwnerVer(); + auto const PP = PV.ParentPkg(); + if (PV != Cache[PP].InstVerIter(Cache) || not D.IsSatisfied(Prv)) + continue; + providers.push_back(PP->ID); + } + std::sort(providers.begin(), providers.end()); + providers.erase(std::unique(providers.begin(), providers.end()), providers.end()); + for (auto const prv : providers) + Scores[prv] += DepMap[D->Type]; + } + } + + // Copy the scores to advoid additive looping + std::unique_ptr<int[]> OldScores(new int[Size]); + memcpy(OldScores.get(),Scores,sizeof(*Scores)*Size); + + /* Now we cause 1 level of dependency inheritance, that is we add the + score of the packages that depend on the target Package. This + fortifies high scoring packages */ + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + if (Cache[I].InstallVer == 0) + continue; + + for (pkgCache::DepIterator D = I.RevDependsList(); D.end() == false; ++D) + { + // Only do it for the install version + if ((pkgCache::Version *)D.ParentVer() != Cache[D.ParentPkg()].InstallVer || + (D->Type != pkgCache::Dep::Depends && + D->Type != pkgCache::Dep::PreDepends && + D->Type != pkgCache::Dep::Recommends)) + continue; + + // Do not propagate negative scores otherwise + // an extra (-2) package might score better than an optional (-1) + if (OldScores[D.ParentPkg()->ID] > 0) + Scores[I->ID] += OldScores[D.ParentPkg()->ID]; + } + } + + /* Now we propagate along provides. This makes the packages that + provide important packages extremely important */ + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + auto const transfer = abs(Scores[I->ID] - OldScores[I->ID]); + if (transfer == 0) + continue; + + std::vector<map_id_t> providers; + for (auto Prv = I.ProvidesList(); not Prv.end(); ++Prv) + { + if (Prv.IsMultiArchImplicit()) + continue; + auto const PV = Prv.OwnerVer(); + auto const PP = PV.ParentPkg(); + if (PV != Cache[PP].InstVerIter(Cache)) + continue; + providers.push_back(PP->ID); + } + std::sort(providers.begin(), providers.end()); + providers.erase(std::unique(providers.begin(), providers.end()), providers.end()); + for (auto const prv : providers) + Scores[prv] += transfer; + } + + /* Protected things are pushed really high up. This number should put them + ahead of everything */ + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + if ((Flags[I->ID] & Protected) != 0) + Scores[I->ID] += AddProtected; + if ((I->Flags & pkgCache::Flag::Essential) == pkgCache::Flag::Essential || + (I->Flags & pkgCache::Flag::Important) == pkgCache::Flag::Important) + Scores[I->ID] += AddEssential; + } +} + /*}}}*/ +// ProblemResolver::DoUpgrade - Attempt to upgrade this package /*{{{*/ +// --------------------------------------------------------------------- +/* This goes through and tries to reinstall packages to make this package + installable */ +bool pkgProblemResolver::DoUpgrade(pkgCache::PkgIterator Pkg) +{ + pkgDepCache::ActionGroup group(Cache); + + if ((Flags[Pkg->ID] & Upgradable) == 0 || Cache[Pkg].Upgradable() == false) + return false; + if ((Flags[Pkg->ID] & Protected) == Protected) + return false; + + Flags[Pkg->ID] &= ~Upgradable; + + bool WasKept = Cache[Pkg].Keep(); + if (not Cache.MarkInstall(Pkg, false, 0, false)) + return false; + + // This must be a virtual package or something like that. + if (Cache[Pkg].InstVerIter(Cache).end() == true) + return false; + + // Isolate the problem dependency + bool Fail = false; + for (pkgCache::DepIterator D = Cache[Pkg].InstVerIter(Cache).DependsList(); D.end() == false;) + { + // Compute a single dependency element (glob or) + pkgCache::DepIterator Start = D; + pkgCache::DepIterator End = D; + for (bool LastOR = true; D.end() == false && LastOR == true;) + { + LastOR = (D->CompareOp & pkgCache::Dep::Or) == pkgCache::Dep::Or; + ++D; + if (LastOR == true) + End = D; + } + + // We only worry about critical deps. + if (End.IsCritical() != true) + continue; + + // Iterate over all the members in the or group + while (1) + { + // Dep is ok now + if ((Cache[End] & pkgDepCache::DepGInstall) == pkgDepCache::DepGInstall) + break; + + // Do not change protected packages + PkgIterator P = Start.SmartTargetPkg(); + if (Cache[P].Protect()) + { + if (Debug == true) + clog << " Reinst Failed because of protected " << P.FullName(false) << endl; + Fail = true; + } + else + { + // Upgrade the package if the candidate version will fix the problem. + if ((Cache[Start] & pkgDepCache::DepCVer) == pkgDepCache::DepCVer) + { + if (DoUpgrade(P) == false) + { + if (Debug == true) + clog << " Reinst Failed because of " << P.FullName(false) << endl; + Fail = true; + } + else + { + Fail = false; + break; + } + } + else + { + /* We let the algorithm deal with conflicts on its next iteration, + it is much smarter than us */ + if (Start.IsNegative() == true) + break; + + if (Debug == true) + clog << " Reinst Failed early because of " << Start.TargetPkg().FullName(false) << endl; + Fail = true; + } + } + + if (Start == End) + break; + ++Start; + } + if (Fail == true) + break; + } + + // Undo our operations - it might be smart to undo everything this did.. + if (Fail == true) + { + if (WasKept == true) + Cache.MarkKeep(Pkg, false, false); + else + Cache.MarkDelete(Pkg, false, 0, false); + return false; + } + + if (Debug == true) + clog << " Re-Instated " << Pkg.FullName(false) << endl; + return true; +} + /*}}}*/ +// ProblemResolver::Resolve - calls a resolver to fix the situation /*{{{*/ +bool pkgProblemResolver::Resolve(bool BrokenFix, OpProgress * const Progress) +{ + std::string const solver = _config->Find("APT::Solver", "internal"); + auto const ret = EDSP::ResolveExternal(solver.c_str(), Cache, 0, Progress); + if (solver != "internal") + return ret; + return ResolveInternal(BrokenFix); +} + /*}}}*/ +// ProblemResolver::ResolveInternal - Run the resolution pass /*{{{*/ +// --------------------------------------------------------------------- +/* This routines works by calculating a score for each package. The score + is derived by considering the package's priority and all reverse + dependents giving an integer that reflects the amount of breakage that + adjusting the package will inflict. + + It goes from highest score to lowest and corrects all of the breaks by + keeping or removing the dependent packages. If that fails then it removes + the package itself and goes on. The routine should be able to intelligently + go from any broken state to a fixed state. + + The BrokenFix flag enables a mode where the algorithm tries to + upgrade packages to advoid problems. */ +bool pkgProblemResolver::ResolveInternal(bool const BrokenFix) +{ + pkgDepCache::ActionGroup group(Cache); + + if (Debug) + Cache.CheckConsistency("resolve start"); + + // Record which packages are marked for install + bool Again = false; + do + { + Again = false; + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + if (Cache[I].Install() == true) + Flags[I->ID] |= PreInstalled; + else + { + if (Cache[I].InstBroken() == true && BrokenFix == true) + { + Cache.MarkInstall(I, false, 0, false); + if (Cache[I].Install() == true) + Again = true; + } + + Flags[I->ID] &= ~PreInstalled; + } + Flags[I->ID] |= Upgradable; + } + } + while (Again == true); + + if (Debug == true) { + clog << "Starting pkgProblemResolver with broken count: " + << Cache.BrokenCount() << endl; + } + + MakeScores(); + + auto const Size = Cache.Head().PackageCount; + + /* We have to order the packages so that the broken fixing pass + operates from highest score to lowest. This prevents problems when + high score packages cause the removal of lower score packages that + would cause the removal of even lower score packages. */ + std::unique_ptr<pkgCache::Package *[]> PList(new pkgCache::Package *[Size]); + pkgCache::Package **PEnd = PList.get(); + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + *PEnd++ = I; + + std::sort(PList.get(), PEnd, [this](Package *a, Package *b) { return ScoreSort(a, b) < 0; }); + + if (_config->FindB("Debug::pkgProblemResolver::ShowScores",false) == true) + { + clog << "Show Scores" << endl; + for (pkgCache::Package **K = PList.get(); K != PEnd; K++) + if (Scores[(*K)->ID] != 0) + { + pkgCache::PkgIterator Pkg(Cache,*K); + clog << Scores[(*K)->ID] << ' ' << APT::PrettyPkg(&Cache, Pkg) << std::endl; + } + } + + if (Debug == true) { + clog << "Starting 2 pkgProblemResolver with broken count: " + << Cache.BrokenCount() << endl; + } + + /* Now consider all broken packages. For each broken package we either + remove the package or fix it's problem. We do this once, it should + not be possible for a loop to form (that is a < b < c and fixing b by + changing a breaks c) */ + bool Change = true; + bool const TryFixByInstall = _config->FindB("pkgProblemResolver::FixByInstall", true); + int const MaxCounter = _config->FindI("pkgProblemResolver::MaxCounter", 20); + std::vector<PackageKill> KillList; + for (int Counter = 0; Counter < MaxCounter && Change; ++Counter) + { + Change = false; + for (pkgCache::Package **K = PList.get(); K != PEnd; K++) + { + pkgCache::PkgIterator I(Cache,*K); + + /* We attempt to install this and see if any breaks result, + this takes care of some strange cases */ + if (Cache[I].CandidateVer != Cache[I].InstallVer && + I->CurrentVer != 0 && Cache[I].InstallVer != 0 && + (Flags[I->ID] & PreInstalled) != 0 && + not Cache[I].Protect() && + (Flags[I->ID] & ReInstateTried) == 0) + { + if (Debug == true) + clog << " Try to Re-Instate (" << Counter << ") " << I.FullName(false) << endl; + auto const OldBreaks = Cache.BrokenCount(); + pkgCache::Version *OldVer = Cache[I].InstallVer; + Flags[I->ID] &= ReInstateTried; + + Cache.MarkInstall(I, false, 0, false); + if (Cache[I].InstBroken() == true || + OldBreaks < Cache.BrokenCount()) + { + if (OldVer == 0) + Cache.MarkDelete(I, false, 0, false); + else + Cache.MarkKeep(I, false, false); + } + else + if (Debug == true) + clog << "Re-Instated " << I.FullName(false) << " (" << OldBreaks << " vs " << Cache.BrokenCount() << ')' << endl; + } + + if (Cache[I].InstallVer == 0 || Cache[I].InstBroken() == false) + continue; + + if (Debug == true) + clog << "Investigating (" << Counter << ") " << APT::PrettyPkg(&Cache, I) << endl; + + // Isolate the problem dependency + bool InOr = false; + pkgCache::DepIterator Start; + pkgCache::DepIterator End; + size_t OldSize = 0; + + KillList.clear(); + + enum {OrRemove,OrKeep} OrOp = OrRemove; + for (pkgCache::DepIterator D = Cache[I].InstVerIter(Cache).DependsList(); + D.end() == false || InOr == true;) + { + // Compute a single dependency element (glob or) + if (Start == End) + { + // Decide what to do + if (InOr == true && OldSize == KillList.size()) + { + if (OrOp == OrRemove) + { + if (not Cache[I].Protect()) + { + if (Debug == true) + clog << " Or group remove for " << I.FullName(false) << endl; + Cache.MarkDelete(I, false, 0, false); + Change = true; + } + } + else if (OrOp == OrKeep) + { + if (Debug == true) + clog << " Or group keep for " << I.FullName(false) << endl; + Cache.MarkKeep(I, false, false); + Change = true; + } + } + + /* We do an extra loop (as above) to finalize the or group + processing */ + InOr = false; + OrOp = OrRemove; + D.GlobOr(Start,End); + if (Start.end() == true) + break; + + // We only worry about critical deps. + if (End.IsCritical() != true) + continue; + + InOr = Start != End; + OldSize = KillList.size(); + } + else + { + ++Start; + // We only worry about critical deps. + if (Start.IsCritical() != true) + continue; + } + + // Dep is ok + if ((Cache[End] & pkgDepCache::DepGInstall) == pkgDepCache::DepGInstall) + { + InOr = false; + continue; + } + + if (Debug == true) + clog << "Broken " << APT::PrettyDep(&Cache, Start) << endl; + + /* Look across the version list. If there are no possible + targets then we keep the package and bail. This is necessary + if a package has a dep on another package that can't be found */ + std::unique_ptr<pkgCache::Version *[]> VList(Start.AllTargets()); + if (VList[0] == 0 && not Cache[I].Protect() && + Start.IsNegative() == false && + Cache[I].NowBroken() == false) + { + if (InOr == true) + { + /* No keep choice because the keep being OK could be the + result of another element in the OR group! */ + continue; + } + + Change = true; + Cache.MarkKeep(I, false, false); + break; + } + + bool Done = false; + for (pkgCache::Version **V = VList.get(); *V != 0; V++) + { + pkgCache::VerIterator Ver(Cache,*V); + pkgCache::PkgIterator Pkg = Ver.ParentPkg(); + + /* This is a conflicts, and the version we are looking + at is not the currently selected version of the + package, which means it is not necessary to + remove/keep */ + if (Cache[Pkg].InstallVer != Ver && Start.IsNegative() == true) + { + if (Debug) + clog << " Conflicts//Breaks against version " + << Ver.VerStr() << " for " << Pkg.Name() + << " but that is not InstVer, ignoring" + << endl; + continue; + } + + if (Debug == true) + clog << " Considering " << Pkg.FullName(false) << ' ' << Scores[Pkg->ID] << + " as a solution to " << I.FullName(false) << ' ' << Scores[I->ID] << endl; + + /* Try to fix the package under consideration rather than + fiddle with the VList package */ + if (Scores[I->ID] <= Scores[Pkg->ID] || + ((Cache[Start] & pkgDepCache::DepNow) == 0 && + End.IsNegative() == false)) + { + // Try a little harder to fix protected packages.. + if (Cache[I].Protect()) + { + if (DoUpgrade(Pkg) == true) + { + if (Scores[Pkg->ID] > Scores[I->ID]) + Scores[Pkg->ID] = Scores[I->ID]; + break; + } + + continue; + } + + /* See if a keep will do, unless the package is protected, + then installing it will be necessary */ + bool Installed = Cache[I].Install(); + Cache.MarkKeep(I, false, false); + if (Cache[I].InstBroken() == false) + { + // Unwind operation will be keep now + if (OrOp == OrRemove) + OrOp = OrKeep; + + // Restore + if (InOr == true && Installed == true) + Cache.MarkInstall(I, false, 0, false); + + if (Debug == true) + clog << " Holding Back " << I.FullName(false) << " rather than change " << Start.TargetPkg().FullName(false) << endl; + } + else + { + if (BrokenFix == false || DoUpgrade(I) == false) + { + // Consider other options + if (InOr == false || Cache[I].Garbage == true) + { + if (Debug == true) + clog << " Removing " << I.FullName(false) << " rather than change " << Start.TargetPkg().FullName(false) << endl; + Cache.MarkDelete(I, false, 0, false); + if (Counter > 1 && Scores[Pkg->ID] > Scores[I->ID]) + Scores[I->ID] = Scores[Pkg->ID]; + } + else if (TryFixByInstall == true && + Start.TargetPkg()->CurrentVer == 0 && + Cache[Start.TargetPkg()].Delete() == false && + (Flags[Start.TargetPkg()->ID] & ToRemove) != ToRemove && + Cache.GetCandidateVersion(Start.TargetPkg()).end() == false) + { + /* Before removing or keeping the package with the broken dependency + try instead to install the first not previously installed package + solving this dependency. This helps every time a previous solver + is removed by the resolver because of a conflict or alike but it is + dangerous as it could trigger new breaks/conflicts… */ + if (Debug == true) + clog << " Try Installing " << APT::PrettyPkg(&Cache, Start.TargetPkg()) << " before changing " << I.FullName(false) << std::endl; + auto const OldBroken = Cache.BrokenCount(); + Cache.MarkInstall(Start.TargetPkg(), true, 1, false); + // FIXME: we should undo the complete MarkInstall process here + if (Cache[Start.TargetPkg()].InstBroken() == true || Cache.BrokenCount() > OldBroken) + Cache.MarkDelete(Start.TargetPkg(), false, 1, false); + } + } + } + + Change = true; + Done = true; + break; + } + else + { + if (Start->Type == pkgCache::Dep::DpkgBreaks) + { + // first, try upgradring the package, if that + // does not help, the breaks goes onto the + // kill list + // + // FIXME: use DoUpgrade(Pkg) instead? + if (Cache[End] & pkgDepCache::DepGCVer) + { + if (Debug) + clog << " Upgrading " << Pkg.FullName(false) << " due to Breaks field in " << I.FullName(false) << endl; + Cache.MarkInstall(Pkg, false, 0, false); + continue; + } + } + + // Skip adding to the kill list if it is protected + if (Cache[Pkg].Protect() && Cache[Pkg].Mode != pkgDepCache::ModeDelete) + continue; + + if (Debug == true) + clog << " Added " << Pkg.FullName(false) << " to the remove list" << endl; + + KillList.push_back({Pkg, End}); + + if (Start.IsNegative() == false) + break; + } + } + + // Hm, nothing can possibly satisfy this dep. Nuke it. + if (VList[0] == 0 && + Start.IsNegative() == false && + not Cache[I].Protect()) + { + bool Installed = Cache[I].Install(); + Cache.MarkKeep(I); + if (Cache[I].InstBroken() == false) + { + // Unwind operation will be keep now + if (OrOp == OrRemove) + OrOp = OrKeep; + + // Restore + if (InOr == true && Installed == true) + Cache.MarkInstall(I, false, 0, false); + + if (Debug == true) + clog << " Holding Back " << I.FullName(false) << " because I can't find " << Start.TargetPkg().FullName(false) << endl; + } + else + { + if (Debug == true) + clog << " Removing " << I.FullName(false) << " because I can't find " << Start.TargetPkg().FullName(false) << endl; + if (InOr == false) + Cache.MarkDelete(I, false, 0, false); + } + + Change = true; + Done = true; + } + + // Try some more + if (InOr == true) + continue; + + if (Done == true) + break; + } + + // Apply the kill list now + if (Cache[I].InstallVer != 0) + { + for (auto const &J : KillList) + { + bool foundSomething = false; + if ((Cache[J.Dep] & pkgDepCache::DepGNow) == 0) + { + if (J.Dep.IsNegative() && Cache.MarkDelete(J.Pkg, false, 0, false)) + { + if (Debug) + std::clog << " Fixing " << I.FullName(false) << " via remove of " << J.Pkg.FullName(false) << '\n'; + foundSomething = true; + } + } + else if (Cache.MarkKeep(J.Pkg, false, false)) + { + if (Debug) + std::clog << " Fixing " << I.FullName(false) << " via keep of " << J.Pkg.FullName(false) << '\n'; + foundSomething = true; + } + + if (not foundSomething || Counter > 1) + { + if (Scores[I->ID] > Scores[J.Pkg->ID]) + { + Scores[J.Pkg->ID] = Scores[I->ID]; + Change = true; + } + } + if (foundSomething) + Change = true; + } + } + } + } + + if (Debug == true) + clog << "Done" << endl; + + if (Cache.BrokenCount() != 0) + { + // See if this is the result of a hold + pkgCache::PkgIterator I = Cache.PkgBegin(); + for (;I.end() != true; ++I) + { + if (Cache[I].InstBroken() == false) + continue; + if (not Cache[I].Protect()) + return _error->Error(_("Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages.")); + } + return _error->Error(_("Unable to correct problems, you have held broken packages.")); + } + + // set the auto-flags (mvo: I'm not sure if we _really_ need this) + pkgCache::PkgIterator I = Cache.PkgBegin(); + for (;I.end() != true; ++I) { + if (Cache[I].NewInstall() && !(Flags[I->ID] & PreInstalled)) { + if(_config->FindB("Debug::pkgAutoRemove",false)) { + std::clog << "Resolve installed new pkg: " << I.FullName(false) + << " (now marking it as auto)" << std::endl; + } + Cache[I].Flags |= pkgCache::Flag::Auto; + } + } + + if (Debug) + Cache.CheckConsistency("resolve done"); + + return true; +} + /*}}}*/ +// ProblemResolver::BreaksInstOrPolicy - Check if the given pkg is broken/*{{{*/ +// --------------------------------------------------------------------- +/* This checks if the given package is broken either by a hard dependency + (InstBroken()) or by introducing a new policy breakage e.g. new + unsatisfied recommends for a package that was in "policy-good" state + + Note that this is not perfect as it will ignore further breakage + for already broken policy (recommends) +*/ +bool pkgProblemResolver::InstOrNewPolicyBroken(pkgCache::PkgIterator I) +{ + // a broken install is always a problem + if (Cache[I].InstBroken() == true) + { + if (Debug == true) + std::clog << " Dependencies are not satisfied for " << APT::PrettyPkg(&Cache, I) << std::endl; + return true; + } + + // a newly broken policy (recommends/suggests) is a problem + if (Cache[I].NowPolicyBroken() == false && + Cache[I].InstPolicyBroken() == true) + { + if (Debug == true) + std::clog << " Policy breaks with upgrade of " << APT::PrettyPkg(&Cache, I) << std::endl; + return true; + } + + return false; +} + /*}}}*/ +// ProblemResolver::ResolveByKeep - Resolve problems using keep /*{{{*/ +// --------------------------------------------------------------------- +/* This is the work horse of the soft upgrade routine. It is very gentle + in that it does not install or remove any packages. It is assumed that the + system was non-broken previously. */ +bool pkgProblemResolver::ResolveByKeep(OpProgress * const Progress) +{ + std::string const solver = _config->Find("APT::Solver", "internal"); + constexpr auto flags = EDSP::Request::UPGRADE_ALL | EDSP::Request::FORBID_NEW_INSTALL | EDSP::Request::FORBID_REMOVE; + auto const ret = EDSP::ResolveExternal(solver.c_str(), Cache, flags, Progress); + if (solver != "internal") + return ret; + return ResolveByKeepInternal(); +} + /*}}}*/ +// ProblemResolver::ResolveByKeepInternal - Resolve problems using keep /*{{{*/ +// --------------------------------------------------------------------- +/* This is the work horse of the soft upgrade routine. It is very gentle + in that it does not install or remove any packages. It is assumed that the + system was non-broken previously. */ +bool pkgProblemResolver::ResolveByKeepInternal() +{ + pkgDepCache::ActionGroup group(Cache); + + if (Debug) + Cache.CheckConsistency("keep start"); + + MakeScores(); + + /* We have to order the packages so that the broken fixing pass + operates from highest score to lowest. This prevents problems when + high score packages cause the removal of lower score packages that + would cause the removal of even lower score packages. */ + auto Size = Cache.Head().PackageCount; + pkgCache::Package **PList = new pkgCache::Package *[Size]; + pkgCache::Package **PEnd = PList; + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + *PEnd++ = I; + + std::sort(PList,PEnd,[this](Package *a, Package *b) { return ScoreSort(a, b) < 0; }); + + + if (_config->FindB("Debug::pkgProblemResolver::ShowScores",false) == true) + { + clog << "Show Scores" << endl; + for (pkgCache::Package **K = PList; K != PEnd; K++) + if (Scores[(*K)->ID] != 0) + { + pkgCache::PkgIterator Pkg(Cache,*K); + clog << Scores[(*K)->ID] << ' ' << APT::PrettyPkg(&Cache, Pkg) << std::endl; + } + } + + if (Debug == true) + clog << "Entering ResolveByKeep" << endl; + + // Consider each broken package + pkgCache::Package **LastStop = 0; + for (pkgCache::Package **K = PList; K != PEnd; K++) + { + pkgCache::PkgIterator I(Cache,*K); + + if (Cache[I].InstallVer == 0) + continue; + + if (InstOrNewPolicyBroken(I) == false) + continue; + + /* Keep the package. If this works then great, otherwise we have + to be significantly more aggressive and manipulate its dependencies */ + if (not Cache[I].Protect()) + { + if (Debug == true) + clog << "Keeping package " << I.FullName(false) << endl; + Cache.MarkKeep(I, false, false); + if (InstOrNewPolicyBroken(I) == false) + { + K = PList - 1; + continue; + } + } + + // Isolate the problem dependencies + for (pkgCache::DepIterator D = Cache[I].InstVerIter(Cache).DependsList(); D.end() == false;) + { + DepIterator Start; + DepIterator End; + D.GlobOr(Start,End); + + // We only worry about critical deps. + if (End.IsCritical() != true) + continue; + + // Dep is ok + if ((Cache[End] & pkgDepCache::DepGInstall) == pkgDepCache::DepGInstall) + continue; + + /* Hm, the group is broken.. I suppose the best thing to do is to + is to try every combination of keep/not-keep for the set, but that's + slow, and this never happens, just be conservative and assume the + list of ors is in preference and keep till it starts to work. */ + while (true) + { + if (Debug == true) + clog << "Package " << I.FullName(false) << " " << APT::PrettyDep(&Cache, Start) << endl; + + // Look at all the possible provides on this package + std::unique_ptr<pkgCache::Version *[]> VList(Start.AllTargets()); + for (pkgCache::Version **V = VList.get(); *V != 0; V++) + { + pkgCache::VerIterator Ver(Cache,*V); + pkgCache::PkgIterator Pkg = Ver.ParentPkg(); + + // It is not keepable + if (Cache[Pkg].InstallVer == 0 || + Pkg->CurrentVer == 0) + continue; + + if (not Cache[Pkg].Protect()) + { + if (Debug == true) + clog << " Keeping Package " << Pkg.FullName(false) << " due to " << Start.DepType() << endl; + Cache.MarkKeep(Pkg, false, false); + } + + if (InstOrNewPolicyBroken(I) == false) + break; + } + + if (InstOrNewPolicyBroken(I) == false) + break; + + if (Start == End) + break; + ++Start; + } + + if (InstOrNewPolicyBroken(I) == false) + break; + } + + if (InstOrNewPolicyBroken(I) == true) + continue; + + // Restart again. + if (K == LastStop) { + // I is an iterator based off our temporary package list, + // so copy the name we need before deleting the temporary list + std::string const LoopingPackage = I.FullName(false); + delete[] PList; + return _error->Error("Internal Error, pkgProblemResolver::ResolveByKeep is looping on package %s.", LoopingPackage.c_str()); + } + LastStop = K; + K = PList - 1; + } + + delete[] PList; + + if (Debug) + Cache.CheckConsistency("keep done"); + + return true; +} + /*}}}*/ +// PrioSortList - Sort a list of versions by priority /*{{{*/ +// --------------------------------------------------------------------- +/* This is meant to be used in conjunction with AllTargets to get a list + of versions ordered by preference. */ + +struct PrioComp { + pkgCache &PrioCache; + + explicit PrioComp(pkgCache &PrioCache) : PrioCache(PrioCache) { + } + + bool operator() (pkgCache::Version * const &A, pkgCache::Version * const &B) { + return compare(A, B) < 0; + } + + int compare(pkgCache::Version * const &A, pkgCache::Version * const &B) { + pkgCache::VerIterator L(PrioCache,A); + pkgCache::VerIterator R(PrioCache,B); + + if ((L.ParentPkg()->Flags & pkgCache::Flag::Essential) == pkgCache::Flag::Essential && + (R.ParentPkg()->Flags & pkgCache::Flag::Essential) != pkgCache::Flag::Essential) + return 1; + if ((L.ParentPkg()->Flags & pkgCache::Flag::Essential) != pkgCache::Flag::Essential && + (R.ParentPkg()->Flags & pkgCache::Flag::Essential) == pkgCache::Flag::Essential) + return -1; + + if ((L.ParentPkg()->Flags & pkgCache::Flag::Important) == pkgCache::Flag::Important && + (R.ParentPkg()->Flags & pkgCache::Flag::Important) != pkgCache::Flag::Important) + return 1; + if ((L.ParentPkg()->Flags & pkgCache::Flag::Important) != pkgCache::Flag::Important && + (R.ParentPkg()->Flags & pkgCache::Flag::Important) == pkgCache::Flag::Important) + return -1; + + if (L->Priority != R->Priority) + return R->Priority - L->Priority; + return strcmp(L.ParentPkg().Name(),R.ParentPkg().Name()); + } +}; + +void pkgPrioSortList(pkgCache &Cache,pkgCache::Version **List) +{ + unsigned long Count = 0; + for (pkgCache::Version **I = List; *I != 0; I++) + Count++; + std::sort(List,List+Count,PrioComp(Cache)); +} + /*}}}*/ + +namespace APT +{ + +namespace KernelAutoRemoveHelper +{ + +// \brief Returns the uname from a kernel package name, or "" for non-kernel packages. +std::string getUname(std::string const &packageName) +{ + + static const constexpr char *const prefixes[] = { + "linux-image-", + "kfreebsd-image-", + "gnumach-image-", + }; + + for (auto prefix : prefixes) + { + if (likely(not APT::String::Startswith(packageName, prefix))) + continue; + if (unlikely(APT::String::Endswith(packageName, "-dbgsym"))) + continue; + if (unlikely(APT::String::Endswith(packageName, "-dbg"))) + continue; + + auto aUname = packageName.substr(strlen(prefix)); + + // aUname must start with [0-9]+\. + if (aUname.length() < 2) + continue; + if (strchr("0123456789", aUname[0]) == nullptr) + continue; + auto dot = aUname.find_first_not_of("0123456789"); + if (dot == aUname.npos || aUname[dot] != '.') + continue; + + return aUname; + } + + return ""; +} +std::string GetProtectedKernelsRegex(pkgCache *cache, bool ReturnRemove) +{ + if (_config->FindB("APT::Protect-Kernels", true) == false) + return ""; + + struct CompareKernel + { + pkgCache *cache; + bool operator()(const std::string &a, const std::string &b) const + { + return cache->VS->CmpVersion(a, b) < 0; + } + }; + bool Debug = _config->FindB("Debug::pkgAutoRemove", false); + // kernel version -> list of unames + std::map<std::string, std::vector<std::string>, CompareKernel> version2unames(CompareKernel{cache}); + // needs to be initialized to 0s, might not be set up. + utsname uts{}; + std::string bootedVersion; + + // Get currently booted version, but only when not on reproducible build. + if (getenv("SOURCE_DATE_EPOCH") == 0) + { + if (uname(&uts) != 0) + abort(); + } + + auto VirtualKernelPkg = cache->FindPkg("$kernel", "any"); + if (VirtualKernelPkg.end()) + return ""; + + for (pkgCache::PrvIterator Prv = VirtualKernelPkg.ProvidesList(); Prv.end() == false; ++Prv) + { + auto Pkg = Prv.OwnerPkg(); + if (likely(Pkg->CurrentVer == 0)) + continue; + + auto pkgUname = APT::KernelAutoRemoveHelper::getUname(Pkg.Name()); + auto pkgVersion = Pkg.CurrentVer().VerStr(); + + if (pkgUname.empty()) + continue; + + if (Debug) + std::clog << "Found kernel " << pkgUname << "(" << pkgVersion << ")" << std::endl; + + version2unames[pkgVersion].push_back(pkgUname); + + if (pkgUname == uts.release) + bootedVersion = pkgVersion; + } + + if (version2unames.size() == 0) + return ""; + + auto latest = version2unames.rbegin(); + auto previous = latest; + ++previous; + + std::set<std::string> keep; + + if (not bootedVersion.empty()) + { + if (Debug) + std::clog << "Keeping booted kernel " << bootedVersion << std::endl; + keep.insert(bootedVersion); + } + if (latest != version2unames.rend()) + { + if (Debug) + std::clog << "Keeping latest kernel " << latest->first << std::endl; + keep.insert(latest->first); + } + if (keep.size() < 2 && previous != version2unames.rend()) + { + if (Debug) + std::clog << "Keeping previous kernel " << previous->first << std::endl; + keep.insert(previous->first); + } + // Escape special characters '.' and '+' in version strings so we can build a regular expression + auto escapeSpecial = [](std::string input) -> std::string { + for (size_t pos = 0; (pos = input.find_first_of(".+", pos)) != input.npos; pos += 2) { + input.insert(pos, 1, '\\'); + } + return input; + }; + std::ostringstream ss; + for (auto &pattern : _config->FindVector("APT::VersionedKernelPackages")) + { + // Legacy compatibility: Always protected the booted uname and last installed uname + if (*uts.release) + ss << "|^" << pattern << "-" << escapeSpecial(uts.release) << "$"; + for (auto const &kernel : version2unames) + { + if (ReturnRemove ? keep.find(kernel.first) == keep.end() : keep.find(kernel.first) != keep.end()) + { + for (auto const &uname : kernel.second) + ss << "|^" << pattern << "-" << escapeSpecial(uname) << "$"; + } + } + } + + auto re_with_leading_or = ss.str(); + + if (re_with_leading_or.empty()) + return ""; + + auto re = re_with_leading_or.substr(1); + if (Debug) + std::clog << "Kernel protection regex: " << re << "\n"; + + return re; +} + +std::unique_ptr<APT::CacheFilter::Matcher> GetProtectedKernelsFilter(pkgCache *cache, bool returnRemove) +{ + auto regex = GetProtectedKernelsRegex(cache, returnRemove); + + if (regex.empty()) + return std::make_unique<APT::CacheFilter::FalseMatcher>(); + + return std::make_unique<APT::CacheFilter::PackageNameMatchesRegEx>(regex); +} + +} // namespace KernelAutoRemoveHelper +} // namespace APT diff --git a/apt-pkg/algorithms.h b/apt-pkg/algorithms.h new file mode 100644 index 0000000..12a77d4 --- /dev/null +++ b/apt-pkg/algorithms.h @@ -0,0 +1,164 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Algorithms - A set of misc algorithms + + This simulate class displays what the ordering code has done and + analyses it with a fresh new dependency cache. In this way we can + see all of the effects of an upgrade run. + + pkgDistUpgrade computes an upgrade that causes as many packages as + possible to move to the newest version. + + pkgApplyStatus sets the target state based on the content of the status + field in the status file. It is important to get proper crash recovery. + + pkgFixBroken corrects a broken system so that it is in a sane state. + + pkgAllUpgrade attempts to upgrade as many packages as possible but + without installing new packages. + + The problem resolver class contains a number of complex algorithms + to try to best-guess an upgrade state. It solves the problem of + maximizing the number of install state packages while having no broken + packages. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_ALGORITHMS_H +#define PKGLIB_ALGORITHMS_H + +#include <apt-pkg/cachefilter.h> +#include <apt-pkg/depcache.h> +#include <apt-pkg/packagemanager.h> +#include <apt-pkg/pkgcache.h> + +#include <iostream> +#include <memory> +#include <string> + +#include <apt-pkg/macros.h> + + + + +class pkgSimulatePrivate; +class APT_PUBLIC pkgSimulate : public pkgPackageManager /*{{{*/ +{ + pkgSimulatePrivate * const d; + protected: + + class APT_PUBLIC Policy : public pkgDepCache::Policy + { + pkgDepCache *Cache; + public: + + virtual VerIterator GetCandidateVer(PkgIterator const &Pkg) APT_OVERRIDE + { + return (*Cache)[Pkg].CandidateVerIter(*Cache); + } + + explicit Policy(pkgDepCache *Cache) : Cache(Cache) {}; + }; + + unsigned char *Flags; + + Policy iPolicy; + pkgDepCache Sim; + pkgDepCache::ActionGroup group; + + // The Actual installation implementation + virtual bool Install(PkgIterator Pkg,std::string File) APT_OVERRIDE; + virtual bool Configure(PkgIterator Pkg) APT_OVERRIDE; + virtual bool Remove(PkgIterator Pkg,bool Purge) APT_OVERRIDE; + +public: + bool Go(APT::Progress::PackageManager * progress) override; + +private: + APT_HIDDEN void ShortBreaks(); + APT_HIDDEN void Describe(PkgIterator iPkg,std::ostream &out,bool Current,bool Candidate); + APT_HIDDEN bool RealInstall(PkgIterator Pkg,std::string File); + APT_HIDDEN bool RealConfigure(PkgIterator Pkg); + APT_HIDDEN bool RealRemove(PkgIterator Pkg,bool Purge); + + public: + + explicit pkgSimulate(pkgDepCache *Cache); + virtual ~pkgSimulate(); +}; + /*}}}*/ +class APT_PUBLIC pkgProblemResolver /*{{{*/ +{ + private: + /** \brief dpointer placeholder (for later in case we need it) */ + void * const d; + + pkgDepCache &Cache; + typedef pkgCache::PkgIterator PkgIterator; + typedef pkgCache::VerIterator VerIterator; + typedef pkgCache::DepIterator DepIterator; + typedef pkgCache::PrvIterator PrvIterator; + typedef pkgCache::Version Version; + typedef pkgCache::Package Package; + + enum Flags {Protected = (1 << 0), PreInstalled = (1 << 1), + Upgradable = (1 << 2), ReInstateTried = (1 << 3), + ToRemove = (1 << 4)}; + int *Scores; + unsigned char *Flags; + bool Debug; + + // Sort stuff + APT_HIDDEN int ScoreSort(Package const *A, Package const *B) APT_PURE; + + struct APT_PUBLIC PackageKill + { + PkgIterator Pkg; + DepIterator Dep; + }; + + APT_HIDDEN void MakeScores(); + APT_HIDDEN bool DoUpgrade(pkgCache::PkgIterator Pkg); + + protected: + bool InstOrNewPolicyBroken(pkgCache::PkgIterator Pkg); + + public: + + inline void Protect(pkgCache::PkgIterator Pkg) {Flags[Pkg->ID] |= Protected; Cache.MarkProtected(Pkg);}; + inline void Remove(pkgCache::PkgIterator Pkg) {Flags[Pkg->ID] |= ToRemove;}; + inline void Clear(pkgCache::PkgIterator Pkg) {Flags[Pkg->ID] &= ~(Protected | ToRemove);}; + + // Try to intelligently resolve problems by installing and removing packages + bool Resolve(bool BrokenFix = false, OpProgress * const Progress = NULL); + APT_HIDDEN bool ResolveInternal(bool const BrokenFix = false); + + // Try to resolve problems only by using keep + bool ResolveByKeep(OpProgress * const Progress = NULL); + APT_HIDDEN bool ResolveByKeepInternal(); + + explicit pkgProblemResolver(pkgDepCache *Cache); + virtual ~pkgProblemResolver(); +}; + /*}}}*/ +APT_PUBLIC bool pkgApplyStatus(pkgDepCache &Cache); +APT_PUBLIC bool pkgFixBroken(pkgDepCache &Cache); + +APT_PUBLIC void pkgPrioSortList(pkgCache &Cache,pkgCache::Version **List); + +namespace APT +{ +namespace KernelAutoRemoveHelper +{ +// Public for linking to apt-private, but no A{P,B}I guarantee. +APT_PUBLIC std::unique_ptr<APT::CacheFilter::Matcher> GetProtectedKernelsFilter(pkgCache *cache, bool returnRemove = false); +std::string GetProtectedKernelsRegex(pkgCache *cache, bool ReturnRemove = false); +std::string getUname(std::string const &packageName); + +} // namespace KernelAutoRemoveHelper + +} // namespace APT + +#endif diff --git a/apt-pkg/apt-pkg.pc.in b/apt-pkg/apt-pkg.pc.in new file mode 100644 index 0000000..c6491d9 --- /dev/null +++ b/apt-pkg/apt-pkg.pc.in @@ -0,0 +1,8 @@ +libdir=@CMAKE_INSTALL_FULL_LIBDIR@ +includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ + +Name: apt-pkg +Description: package management runtime library +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -lapt-pkg -pthread +Cflags: -I${includedir} diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc new file mode 100644 index 0000000..00a97a0 --- /dev/null +++ b/apt-pkg/aptconfiguration.cc @@ -0,0 +1,541 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Provide access methods to various configuration settings, + setup defaults and returns validate settings. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgsystem.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <string> +#include <vector> +#include <ctype.h> +#include <dirent.h> +#include <fcntl.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + + /*}}}*/ +namespace APT { +// setDefaultConfigurationForCompressors /*{{{*/ +static void setDefaultConfigurationForCompressors() { + // Set default application paths to check for optional compression types + _config->CndSet("Dir::Bin::gzip", "/bin/gzip"); + _config->CndSet("Dir::Bin::bzip2", "/bin/bzip2"); + _config->CndSet("Dir::Bin::xz", "/usr/bin/xz"); + _config->CndSet("Dir::Bin::lz4", "/usr/bin/lz4"); + _config->CndSet("Dir::Bin::zstd", "/usr/bin/zstd"); + if (FileExists(_config->Find("Dir::Bin::xz")) == true) { + _config->Set("Dir::Bin::lzma", _config->Find("Dir::Bin::xz")); + _config->Set("APT::Compressor::lzma::Binary", "xz"); + if (_config->Exists("APT::Compressor::lzma::CompressArg") == false) { + _config->Set("APT::Compressor::lzma::CompressArg::", "--format=lzma"); + _config->Set("APT::Compressor::lzma::CompressArg::", "-6"); + } + if (_config->Exists("APT::Compressor::lzma::UncompressArg") == false) { + _config->Set("APT::Compressor::lzma::UncompressArg::", "--format=lzma"); + _config->Set("APT::Compressor::lzma::UncompressArg::", "-d"); + } + } else { + _config->CndSet("Dir::Bin::lzma", "/usr/bin/lzma"); + if (_config->Exists("APT::Compressor::lzma::CompressArg") == false) { + _config->Set("APT::Compressor::lzma::CompressArg::", "--suffix="); + _config->Set("APT::Compressor::lzma::CompressArg::", "-6"); + } + if (_config->Exists("APT::Compressor::lzma::UncompressArg") == false) { + _config->Set("APT::Compressor::lzma::UncompressArg::", "--suffix="); + _config->Set("APT::Compressor::lzma::UncompressArg::", "-d"); + } + } + // setup the defaults for the compressiontypes => method mapping + _config->CndSet("Acquire::CompressionTypes::xz","xz"); + _config->CndSet("Acquire::CompressionTypes::bz2","bzip2"); + _config->CndSet("Acquire::CompressionTypes::lzma","lzma"); + _config->CndSet("Acquire::CompressionTypes::gz","gzip"); + _config->CndSet("Acquire::CompressionTypes::lz4","lz4"); + _config->CndSet("Acquire::CompressionTypes::zst", "zstd"); +} + /*}}}*/ +// getCompressionTypes - Return Vector of usable compressiontypes /*{{{*/ +// --------------------------------------------------------------------- +/* return a vector of compression types in the preferred order. */ +std::vector<std::string> +const Configuration::getCompressionTypes(bool const &Cached) { + static std::vector<std::string> types; + if (types.empty() == false) { + if (Cached == true) + return types; + else + types.clear(); + } + + std::vector<APT::Configuration::Compressor> const compressors = getCompressors(); + + // load the order setting into our vector + std::vector<std::string> const order = _config->FindVector("Acquire::CompressionTypes::Order"); + for (std::vector<std::string>::const_iterator o = order.begin(); + o != order.end(); ++o) { + if ((*o).empty() == true) + continue; + // ignore types we have no method ready to use + std::string const method = std::string("Acquire::CompressionTypes::").append(*o); + if (_config->Exists(method) == false) + continue; + // ignore types we have no app ready to use + std::string const app = _config->Find(method); + if (std::find_if(compressors.begin(), compressors.end(), [&app](APT::Configuration::Compressor const &c) { + return c.Name == app; + }) == compressors.end()) + continue; + types.push_back(*o); + } + + // move again over the option tree to add all missing compression types + ::Configuration::Item const *Types = _config->Tree("Acquire::CompressionTypes"); + if (Types != 0) + Types = Types->Child; + + for (; Types != 0; Types = Types->Next) { + if (Types->Tag == "Order" || Types->Tag.empty() == true) + continue; + // ignore types we already have in the vector + if (std::find(types.begin(),types.end(),Types->Tag) != types.end()) + continue; + // ignore types we have no app ready to use + if (std::find_if(compressors.begin(), compressors.end(), [&Types](APT::Configuration::Compressor const &c) { + return c.Name == Types->Value; + }) == compressors.end()) + continue; + types.push_back(Types->Tag); + } + + // add the special "uncompressed" type + if (std::find(types.begin(), types.end(), "uncompressed") == types.end()) + { + std::string const uncompr = _config->Find("Dir::Bin::uncompressed", ""); + if (uncompr.empty() == true || FileExists(uncompr) == true) + types.push_back("uncompressed"); + } + + return types; +} + /*}}}*/ +// GetLanguages - Return Vector of Language Codes /*{{{*/ +// --------------------------------------------------------------------- +/* return a vector of language codes in the preferred order. + the special word "environment" will be replaced with the long and the short + code of the local settings and it will be insured that this will not add + duplicates. So in an german local the setting "environment, de_DE, en, de" + will result in "de_DE, de, en". + The special word "none" is the stopcode for the not-All code vector */ +std::vector<std::string> const Configuration::getLanguages(bool const &All, + bool const &Cached, char const ** const Locale) { + using std::string; + + // The detection is boring and has a lot of cornercases, + // so we cache the results to calculated it only once. + std::vector<string> static allCodes; + std::vector<string> static codes; + + // we have something in the cache + if (codes.empty() == false || allCodes.empty() == false) { + if (Cached == true) { + if(All == true && allCodes.empty() == false) + return allCodes; + else + return codes; + } else { + allCodes.clear(); + codes.clear(); + } + } + + // Include all Language codes we have a Translation file for in /var/lib/apt/lists + // so they will be all included in the Cache. + std::vector<string> builtin; + DIR *D = opendir(_config->FindDir("Dir::State::lists").c_str()); + if (D != NULL) { + builtin.push_back("none"); + for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D)) { + string const name = SubstVar(Ent->d_name, "%5f", "_"); + size_t const foundDash = name.rfind("-"); + size_t const foundUnderscore = name.rfind("_", foundDash); + if (foundDash == string::npos || foundUnderscore == string::npos || + foundDash <= foundUnderscore || + name.substr(foundUnderscore+1, foundDash-(foundUnderscore+1)) != "Translation") + continue; + string const c = name.substr(foundDash+1); + if (unlikely(c.empty() == true) || c == "en") + continue; + // Skip unusual files, like backups or that alike + string::const_iterator s = c.begin(); + for (;s != c.end(); ++s) { + if (isalpha(*s) == 0 && *s != '_') + break; + } + if (s != c.end()) + continue; + if (std::find(builtin.begin(), builtin.end(), c) != builtin.end()) + continue; + builtin.push_back(c); + } + closedir(D); + } + + // FIXME: Remove support for the old APT::Acquire::Translation + // it was undocumented and so it should be not very widthly used + string const oldAcquire = _config->Find("APT::Acquire::Translation",""); + if (oldAcquire.empty() == false && oldAcquire != "environment") { + // TRANSLATORS: the two %s are APT configuration options + _error->Notice("Option '%s' is deprecated. Please use '%s' instead, see 'man 5 apt.conf' for details.", + "APT::Acquire::Translation", "Acquire::Languages"); + if (oldAcquire != "none") + codes.push_back(oldAcquire); + codes.push_back("en"); + allCodes = codes; + for (std::vector<string>::const_iterator b = builtin.begin(); + b != builtin.end(); ++b) + if (std::find(allCodes.begin(), allCodes.end(), *b) == allCodes.end()) + allCodes.push_back(*b); + if (All == true) + return allCodes; + else + return codes; + } + + // get the environment language codes: LC_MESSAGES (and later LANGUAGE) + // we extract both, a long and a short code and then we will + // check if we actually need both (rare) or if the short is enough + string const envMsg = string(Locale == 0 ? ::setlocale(LC_MESSAGES, NULL) : *Locale); + size_t const lenShort = (envMsg.find('_') != string::npos) ? envMsg.find('_') : 2; + size_t const lenLong = (envMsg.find_first_of(".@") != string::npos) ? envMsg.find_first_of(".@") : (lenShort + 3); + + string const envLong = envMsg.substr(0,lenLong); + string const envShort = envLong.substr(0,lenShort); + + // It is very likely we will need the environment codes later, + // so let us generate them now from LC_MESSAGES and LANGUAGE + std::vector<string> environment; + if (envShort != "C") { + // take care of LC_MESSAGES + if (envLong != envShort) + environment.push_back(envLong); + environment.push_back(envShort); + // take care of LANGUAGE + const char *language_env = getenv("LANGUAGE") == 0 ? "" : getenv("LANGUAGE"); + string envLang = Locale == 0 ? language_env : *(Locale+1); + if (envLang.empty() == false) { + std::vector<string> env = VectorizeString(envLang,':'); + short addedLangs = 0; // add a maximum of 3 fallbacks from the environment + for (std::vector<string>::const_iterator e = env.begin(); + e != env.end() && addedLangs < 3; ++e) { + if (unlikely(e->empty() == true) || *e == "en") + continue; + if (*e == envLong || *e == envShort) + continue; + if (std::find(environment.begin(), environment.end(), *e) != environment.end()) + continue; + ++addedLangs; + environment.push_back(*e); + } + } + } else { + // cornercase: LANG=C, so we use only "en" Translation + environment.push_back("en"); + } + + std::vector<string> const lang = _config->FindVector("Acquire::Languages", "environment,en"); + // the configs define the order, so add the environment + // then needed and ensure the codes are not listed twice. + bool noneSeen = false; + for (std::vector<string>::const_iterator l = lang.begin(); + l != lang.end(); ++l) { + if (*l == "environment") { + for (std::vector<string>::const_iterator e = environment.begin(); + e != environment.end(); ++e) { + if (std::find(allCodes.begin(), allCodes.end(), *e) != allCodes.end()) + continue; + if (noneSeen == false) + codes.push_back(*e); + allCodes.push_back(*e); + } + continue; + } else if (*l == "none") { + noneSeen = true; + continue; + } else if (std::find(allCodes.begin(), allCodes.end(), *l) != allCodes.end()) + continue; + + if (noneSeen == false) + codes.push_back(*l); + allCodes.push_back(*l); + } + + if (allCodes.empty() == false) { + for (std::vector<string>::const_iterator b = builtin.begin(); + b != builtin.end(); ++b) + if (std::find(allCodes.begin(), allCodes.end(), *b) == allCodes.end()) + allCodes.push_back(*b); + } else { + // "none" was forced + allCodes.push_back("none"); + } + + if (All == true) + return allCodes; + else + return codes; +} + /*}}}*/ +// checkLanguage - are we interested in the given Language? /*{{{*/ +bool Configuration::checkLanguage(std::string Lang, bool const All) { + // the empty Language is always interesting as it is the original + if (Lang.empty() == true) + return true; + // filenames are encoded, so undo this + Lang = SubstVar(Lang, "%5f", "_"); + std::vector<std::string> const langs = getLanguages(All, true); + return (std::find(langs.begin(), langs.end(), Lang) != langs.end()); +} + /*}}}*/ +// getArchitectures - Return Vector of preferred Architectures /*{{{*/ +std::vector<std::string> const Configuration::getArchitectures(bool const &Cached) { + std::vector<std::string> static archs; + if (likely(Cached == true) && archs.empty() == false) + return archs; + + std::string const arch = _config->Find("APT::Architecture"); + archs = _config->FindVector("APT::Architectures"); + + if (archs.empty() == true && _system != nullptr) + archs = _system->ArchitecturesSupported(); + + if (archs.empty() == true || + std::find(archs.begin(), archs.end(), arch) == archs.end()) + archs.insert(archs.begin(), arch); + + // erase duplicates, empty strings and very foreign architectures + auto newend = std::remove_if(archs.begin(), archs.end(), [](auto const &a) { return a.empty(); }); + for (auto a = archs.begin(); a != newend; ++a) + newend = std::remove(std::next(a), newend, *a); + for (auto const &f : _config->FindVector("APT::BarbarianArchitectures")) + newend = std::remove(archs.begin(), newend, f); + archs.erase(newend, archs.end()); + return archs; +} + /*}}}*/ +// checkArchitecture - are we interested in the given Architecture? /*{{{*/ +bool Configuration::checkArchitecture(std::string const &Arch) { + if (Arch == "all") + return true; + std::vector<std::string> const archs = getArchitectures(true); + return (std::find(archs.begin(), archs.end(), Arch) != archs.end()); +} + /*}}}*/ +// getCompressors - Return Vector of usealbe compressors /*{{{*/ +// --------------------------------------------------------------------- +/* return a vector of compressors used by apt-ftparchive in the + multicompress functionality or to detect data.tar files */ +std::vector<APT::Configuration::Compressor> +const Configuration::getCompressors(bool const Cached) { + static std::vector<APT::Configuration::Compressor> compressors; + if (compressors.empty() == false) { + if (Cached == true) + return compressors; + else + compressors.clear(); + } + + setDefaultConfigurationForCompressors(); + + std::vector<std::string> CompressorsDone; +# define APT_ADD_COMPRESSOR(NAME, EXT, BINARY, ARG, DEARG, COST) \ + { CompressorsDone.push_back(NAME); compressors.emplace_back(NAME, EXT, BINARY, ARG, DEARG, COST); } + APT_ADD_COMPRESSOR(".", "", "", nullptr, nullptr, 0) + if (_config->Exists("Dir::Bin::zstd") == false || FileExists(_config->Find("Dir::Bin::zstd")) == true) + APT_ADD_COMPRESSOR("zstd", ".zst", "zstd", "-19", "-d", 60) +#ifdef HAVE_ZSTD + else + APT_ADD_COMPRESSOR("zstd", ".zst", "false", nullptr, nullptr, 60) +#endif + if (_config->Exists("Dir::Bin::lz4") == false || FileExists(_config->Find("Dir::Bin::lz4")) == true) + APT_ADD_COMPRESSOR("lz4",".lz4","lz4","-1","-d",50) +#ifdef HAVE_LZ4 + else + APT_ADD_COMPRESSOR("lz4",".lz4","false", nullptr, nullptr, 50) +#endif + if (_config->Exists("Dir::Bin::gzip") == false || FileExists(_config->Find("Dir::Bin::gzip")) == true) + APT_ADD_COMPRESSOR("gzip",".gz","gzip","-6n","-d",100) +#ifdef HAVE_ZLIB + else + APT_ADD_COMPRESSOR("gzip",".gz","false", nullptr, nullptr, 100) +#endif + if (_config->Exists("Dir::Bin::xz") == false || FileExists(_config->Find("Dir::Bin::xz")) == true) + APT_ADD_COMPRESSOR("xz",".xz","xz","-6","-d",200) +#ifdef HAVE_LZMA + else + APT_ADD_COMPRESSOR("xz",".xz","false", nullptr, nullptr, 200) +#endif + if (_config->Exists("Dir::Bin::bzip2") == false || FileExists(_config->Find("Dir::Bin::bzip2")) == true) + APT_ADD_COMPRESSOR("bzip2",".bz2","bzip2","-6","-d",300) +#ifdef HAVE_BZ2 + else + APT_ADD_COMPRESSOR("bzip2",".bz2","false", nullptr, nullptr, 300) +#endif + if (_config->Exists("Dir::Bin::lzma") == false || FileExists(_config->Find("Dir::Bin::lzma")) == true) + APT_ADD_COMPRESSOR("lzma",".lzma","lzma","-6","-d",400) +#ifdef HAVE_LZMA + else + APT_ADD_COMPRESSOR("lzma",".lzma","false", nullptr, nullptr, 400) +#endif + + std::vector<std::string> const comp = _config->FindVector("APT::Compressor", "", true); + for (auto const &c: comp) + { + if (c.empty() || std::find(CompressorsDone.begin(), CompressorsDone.end(), c) != CompressorsDone.end()) + continue; + compressors.push_back(Compressor(c.c_str(), std::string(".").append(c).c_str(), c.c_str(), nullptr, nullptr, 1000)); + } + + return compressors; +} + /*}}}*/ +// getCompressorExtensions - supported data.tar extensions /*{{{*/ +// --------------------------------------------------------------------- +/* */ +std::vector<std::string> const Configuration::getCompressorExtensions() { + std::vector<APT::Configuration::Compressor> const compressors = getCompressors(); + std::vector<std::string> ext; + for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressors.begin(); + c != compressors.end(); ++c) + if (c->Extension.empty() == false && c->Extension != ".") + ext.push_back(c->Extension); + return ext; +} + /*}}}*/ +// Compressor constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +Configuration::Compressor::Compressor(char const *name, char const *extension, + char const *binary, + char const *compressArg, char const *uncompressArg, + unsigned short const cost) { + std::string const config = std::string("APT::Compressor::").append(name).append("::"); + Name = _config->Find(std::string(config).append("Name"), name); + Extension = _config->Find(std::string(config).append("Extension"), extension); + Binary = _config->Find(std::string(config).append("Binary"), binary); + Cost = _config->FindI(std::string(config).append("Cost"), cost); + std::string const compConf = std::string(config).append("CompressArg"); + if (_config->Exists(compConf) == true) + CompressArgs = _config->FindVector(compConf); + else if (compressArg != NULL) + CompressArgs.push_back(compressArg); + std::string const uncompConf = std::string(config).append("UncompressArg"); + if (_config->Exists(uncompConf) == true) + UncompressArgs = _config->FindVector(uncompConf); + else if (uncompressArg != NULL) + UncompressArgs.push_back(uncompressArg); +} + /*}}}*/ +// getBuildProfiles - return a vector of enabled build profiles /*{{{*/ +std::vector<std::string> const Configuration::getBuildProfiles() { + // order is: override value (~= commandline), environment variable, list (~= config file) + std::string profiles_env = getenv("DEB_BUILD_PROFILES") == 0 ? "" : getenv("DEB_BUILD_PROFILES"); + if (profiles_env.empty() == false) { + profiles_env = SubstVar(profiles_env, " ", ","); + std::string const bp = _config->Find("APT::Build-Profiles"); + _config->Clear("APT::Build-Profiles"); + if (bp.empty() == false) + _config->Set("APT::Build-Profiles", bp); + } + return _config->FindVector("APT::Build-Profiles", profiles_env); +} +std::string const Configuration::getBuildProfilesString() { + std::vector<std::string> profiles = getBuildProfiles(); + if (profiles.empty() == true) + return ""; + std::vector<std::string>::const_iterator p = profiles.begin(); + std::string list = *p; + for (++p; p != profiles.end(); ++p) + list.append(",").append(*p); + return list; +} + /*}}}*/ + +// getMachineID - read /etc/machine-id into a string /*{{{*/ +// --------------------------------------------------------------------- +/* */ +std::string const Configuration::getMachineID() +{ + std::string id = _config->Find("APT::Machine-ID"); + + if (id.empty()) + { + std::string file = _config->FindFile("Dir::Etc::machine-id"); + + if (file.empty()) + { + file = flCombine(_config->FindDir("Dir::Etc"), "../machine-id"); + } + FileFd fd; + _error->PushToStack(); + + if (not OpenConfigurationFileFd(file, fd) || not fd.ReadLine(id)) + { + if (_config->FindB("Debug::Phasing", false)) + { + _error->DumpErrors(std::clog); + } + } + + _error->RevertToStack(); + } + + return id; +} + /*}}}*/ +// isChroot - whether we are inside a chroot /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool Configuration::isChroot() +{ + static struct once + { + bool res = false; + once() + { + pid_t child = ExecFork(); + if (child == 0) + { + auto binary = _config->FindFile("Dir::Bin::ischroot", "/usr/bin/ischroot"); + const char *const Args[] = { + binary.c_str(), + "-t", + nullptr}; + execvp(Args[0], const_cast<char **>(Args)); + _exit(127); + } + + res = ExecWait(child, "ischroot", true); + } + } once; + + return once.res; +} + /*}}}*/ +} diff --git a/apt-pkg/aptconfiguration.h b/apt-pkg/aptconfiguration.h new file mode 100644 index 0000000..bbeb156 --- /dev/null +++ b/apt-pkg/aptconfiguration.h @@ -0,0 +1,134 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/** \class APT::Configuration + * \brief Provide access methods to various configuration settings + * + * This class and their methods providing a layer around the usual access + * methods with _config to ensure that settings are correct and to be able + * to set defaults without the need to recheck it in every method again. + */ + /*}}}*/ +#ifndef APT_CONFIGURATION_H +#define APT_CONFIGURATION_H +// Include Files /*{{{*/ +#include <apt-pkg/macros.h> +#include <limits> +#include <string> +#include <vector> + /*}}}*/ +namespace APT { +namespace Configuration { /*{{{*/ + /** \brief Returns a vector of usable Compression Types + * + * Files can be compressed in various ways to decrease the size of the + * download. Therefore the Acquiremethods support a few compression types + * and some archives provide also a few different types. This option + * group exists to give the user the choice to prefer one type over the + * other (some compression types are very resource intensive - great if you + * have a limited download, bad if you have a really lowpowered hardware.) + * + * This method ensures that the defaults are set and checks at runtime + * if the type can be used. E.g. the current default is to prefer bzip2 + * over lzma and gz - if the bzip2 binary is not available it has not much + * sense in downloading the bz2 file, therefore we will not return bz2 as + * a usable compression type. The availability is checked with the settings + * in the Dir::Bin group. + * + * \param Cached saves the result so we need to calculated it only once + * this parameter should only be used for testing purposes. + * + * \return a vector of the compression types in the preferred usage order + */ + APT_PUBLIC std::vector<std::string> const getCompressionTypes(bool const &Cached = true); + + /** \brief Returns a vector of Language Codes + * + * Languages can be defined with their two or five chars long code. + * This methods handles the various ways to set the preferred codes, + * honors the environment and ensures that the codes are not listed twice. + * + * The special word "environment" will be replaced with the long and the short + * code of the local settings and it will be insured that this will not add + * duplicates. So in an german local the setting "environment, de_DE, en, de" + * will result in "de_DE, de, en". + * + * Another special word is "none" which separates the preferred from all codes + * in this setting. So setting and method can be used to get codes the user want + * to see or to get all language codes APT (should) have Translations available. + * + * \param All return all codes or only codes for languages we want to use + * \param Cached saves the result so we need to calculated it only once + * this parameter should only be used for testing purposes. + * \param Locale don't get the locale from the system but use this one instead + * this parameter should only be used for testing purposes. + * + * \return a vector of (all) Language Codes in the preferred usage order + */ + APT_PUBLIC std::vector<std::string> const getLanguages(bool const &All = false, + bool const &Cached = true, char const ** const Locale = 0); + + /** \brief Are we interested in the given Language? + * + * \param Lang is the language we want to check + * \param All defines if we check against all codes or only against used codes + * \return true if we are interested, false otherwise + */ + APT_PUBLIC bool checkLanguage(std::string Lang, bool const All = false); + + /** \brief Returns a vector of Architectures we support + * + * \param Cached saves the result so we need to calculated it only once + * this parameter should only be used for testing purposes. + * + * \return a vector of Architectures in preferred order + */ + APT_PUBLIC std::vector<std::string> const getArchitectures(bool const &Cached = true); + + /** \brief Are we interested in the given Architecture? + * + * \param Arch we want to check + * \return true if we are interested, false otherwise + */ + APT_PUBLIC bool checkArchitecture(std::string const &Arch); + + /** \brief Representation of supported compressors */ + struct APT_PUBLIC Compressor { + std::string Name; + std::string Extension; + std::string Binary; + std::vector<std::string> CompressArgs; + std::vector<std::string> UncompressArgs; + unsigned short Cost; + + Compressor(char const *name, char const *extension, char const *binary, + char const *compressArg, char const *uncompressArg, + unsigned short const cost); + Compressor() : Cost(std::numeric_limits<unsigned short>::max()) {}; + }; + + /** \brief Return a vector of Compressors supported for data.tar's + * + * \param Cached saves the result so we need to calculated it only once + * this parameter should only be used for testing purposes. + * + * \return a vector of Compressors + */ + APT_PUBLIC std::vector<Compressor> const getCompressors(bool const Cached = true); + + /** \brief Return a vector of extensions supported for data.tar's */ + APT_PUBLIC std::vector<std::string> const getCompressorExtensions(); + + /** \return Return a vector of enabled build profile specifications */ + APT_PUBLIC std::vector<std::string> const getBuildProfiles(); + /** \return Return a comma-separated list of enabled build profile specifications */ + APT_PUBLIC std::string const getBuildProfilesString(); + + std::string const getMachineID(); + + /** \return Whether we are running in a chroot */ + bool isChroot(); + /*}}}*/ +} + /*}}}*/ +} +#endif diff --git a/apt-pkg/cachefile.cc b/apt-pkg/cachefile.cc new file mode 100644 index 0000000..4c3cc95 --- /dev/null +++ b/apt-pkg/cachefile.cc @@ -0,0 +1,385 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + CacheFile - Simple wrapper class for opening, generating and whatnot + + This class implements a simple 2 line mechanism to open various sorts + of caches. It can operate as root, as not root, show progress and so on, + it transparently handles everything necessary. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/cachefile.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/depcache.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/indexfile.h> +#include <apt-pkg/mmap.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgcachegen.h> +#include <apt-pkg/pkgsystem.h> +#include <apt-pkg/policy.h> +#include <apt-pkg/progress.h> +#include <apt-pkg/sourcelist.h> + +#include <limits> +#include <memory> +#include <string> +#include <vector> +#include <string.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ + +struct pkgCacheFile::Private +{ + bool WithLock = false; + bool InhibitActionGroups = false; +}; + +// CacheFile::CacheFile - Constructor /*{{{*/ +pkgCacheFile::pkgCacheFile() : d(new Private()), ExternOwner(false), Map(NULL), Cache(NULL), + DCache(NULL), SrcList(NULL), Policy(NULL) +{ +} +pkgCacheFile::pkgCacheFile(pkgDepCache * const Owner) : d(new Private()), ExternOwner(true), + Map(&Owner->GetCache().GetMap()), Cache(&Owner->GetCache()), + DCache(Owner), SrcList(NULL), Policy(NULL) +{ +} + /*}}}*/ +// CacheFile::~CacheFile - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgCacheFile::~pkgCacheFile() +{ + if (ExternOwner == false) + { + delete DCache; + delete Cache; + delete Map; + } + delete Policy; + delete SrcList; + if (d->WithLock == true) + _system->UnLock(true); + + delete d; +} + /*}}}*/ +// CacheFile::BuildCaches - Open and build the cache files /*{{{*/ +class APT_HIDDEN ScopedErrorMerge { +public: + ScopedErrorMerge() { _error->PushToStack(); } + ~ScopedErrorMerge() { _error->MergeWithStack(); } +}; + +bool pkgCacheFile::BuildCaches(OpProgress *Progress, bool WithLock) +{ + std::unique_ptr<pkgCache> Cache; + std::unique_ptr<MMap> Map; + + if (this->Cache != NULL) + return true; + + ScopedErrorMerge sem; + if (_config->FindB("pkgCacheFile::Generate", true) == false) + { + FileFd file(_config->FindFile("Dir::Cache::pkgcache"), FileFd::ReadOnly); + if (file.IsOpen() == false || file.Failed()) + return false; + Map.reset(new MMap(file, MMap::Public|MMap::ReadOnly)); + if (unlikely(Map->validData() == false)) + return false; + Cache.reset(new pkgCache(Map.get())); + if (_error->PendingError() == true) + return false; + + this->Cache = Cache.release(); + this->Map = Map.release(); + return true; + } + + if (WithLock == true) + { + if (_system->Lock(Progress) == false) + return false; + d->WithLock = true; + } + + if (_error->PendingError() == true) + return false; + + if (BuildSourceList(Progress) == false) + return false; + + // Read the caches + MMap *TmpMap = nullptr; + pkgCache *TmpCache = nullptr; + bool Res = pkgCacheGenerator::MakeStatusCache(*SrcList,Progress,&TmpMap, &TmpCache, true); + Map.reset(TmpMap); + Cache.reset(TmpCache); + if (Progress != NULL) + Progress->Done(); + if (Res == false) + return _error->Error(_("The package lists or status file could not be parsed or opened.")); + + /* This sux, remove it someday */ + if (_error->PendingError() == true) + _error->Warning(_("You may want to run apt-get update to correct these problems")); + + if (Cache == nullptr) + Cache.reset(new pkgCache(Map.get())); + if (_error->PendingError() == true) + return false; + this->Map = Map.release(); + this->Cache = Cache.release(); + + return true; +} + /*}}}*/ +// CacheFile::BuildSourceList - Open and build all relevant sources.list/*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgCacheFile::BuildSourceList(OpProgress * /*Progress*/) +{ + std::unique_ptr<pkgSourceList> SrcList; + if (this->SrcList != NULL) + return true; + + SrcList.reset(new pkgSourceList()); + if (SrcList->ReadMainList() == false) + return _error->Error(_("The list of sources could not be read.")); + this->SrcList = SrcList.release(); + return true; +} + /*}}}*/ +// CacheFile::BuildPolicy - Open and build all relevant preferences /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgCacheFile::BuildPolicy(OpProgress * /*Progress*/) +{ + std::unique_ptr<pkgPolicy> Policy; + if (this->Policy != NULL) + return true; + + Policy.reset(new pkgPolicy(Cache)); + if (_error->PendingError() == true) + return false; + + ReadPinFile(*Policy); + ReadPinDir(*Policy); + + this->Policy = Policy.release(); + return _error->PendingError() == false; +} + /*}}}*/ +// CacheFile::BuildDepCache - Open and build the dependency cache /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgCacheFile::BuildDepCache(OpProgress *Progress) +{ + if (BuildCaches(Progress, false) == false) + return false; + + std::unique_ptr<pkgDepCache> DCache; + if (this->DCache != NULL) + return true; + + if (BuildPolicy(Progress) == false) + return false; + + DCache.reset(new pkgDepCache(Cache,Policy)); + if (_error->PendingError() == true) + return false; + if (d->InhibitActionGroups) + DCache->IncreaseActionGroupLevel(); + if (DCache->Init(Progress) == false) + return false; + + this->DCache = DCache.release(); + return true; +} + /*}}}*/ +// CacheFile::Open - Open the cache files, creating if necessary /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgCacheFile::Open(OpProgress *Progress, bool WithLock) +{ + if (BuildCaches(Progress,WithLock) == false) + return false; + + if (BuildPolicy(Progress) == false) + return false; + + if (BuildDepCache(Progress) == false) + return false; + + if (Progress != NULL) + Progress->Done(); + if (_error->PendingError() == true) + return false; + + return true; +} + /*}}}*/ +bool pkgCacheFile::AddIndexFile(pkgIndexFile * const File) /*{{{*/ +{ + if (SrcList == NULL) + if (BuildSourceList() == false) + return false; + SrcList->AddVolatileFile(File); + + if (Cache == nullptr || File->HasPackages() == false || File->Exists() == false) + return true; + + if (File->FindInCache(*Cache).end() == false) + return _error->Warning("Duplicate sources.list entry %s", + File->Describe().c_str()); + + if (ExternOwner == false) + { + delete DCache; + delete Cache; + } + delete Policy; + DCache = NULL; + Policy = NULL; + Cache = NULL; + + if (ExternOwner == false) + { + // a dynamic mmap means that we have build at least parts of the cache + // in memory – which we might or might not have written to disk. + // Throwing away would therefore be a very costly operation we want to avoid + DynamicMMap * dynmmap = dynamic_cast<DynamicMMap*>(Map); + if (dynmmap != nullptr) + { + { + pkgCacheGenerator Gen(dynmmap, nullptr); + if (Gen.Start() == false || File->Merge(Gen, nullptr) == false) + return false; + } + Cache = new pkgCache(Map); + if (_error->PendingError() == true) { + delete Cache; + Cache = nullptr; + return false; + } + return true; + } + else + { + delete Map; + Map = NULL; + } + } + else + { + ExternOwner = false; + Map = NULL; + } + _system->UnLock(true); + return true; +} + /*}}}*/ +// CacheFile::RemoveCaches - remove all cache files from disk /*{{{*/ +// --------------------------------------------------------------------- +/* */ +static void SetCacheStartBeforeRemovingCache(std::string const &cache) +{ + if (cache.empty()) + return; + auto const CacheStart = _config->FindI("APT::Cache-Start", 0); + constexpr auto CacheStartDefault = 24 * 1024 * 1024; + struct stat Buf; + if (stat(cache.c_str(), &Buf) == 0 && (Buf.st_mode & S_IFREG) != 0) + { + RemoveFile("RemoveCaches", cache); + if (CacheStart == 0 && std::numeric_limits<decltype(CacheStart)>::max() >= Buf.st_size && Buf.st_size > CacheStartDefault) + _config->Set("APT::Cache-Start", Buf.st_size); + } +} +void pkgCacheFile::RemoveCaches() +{ + std::string const pkgcache = _config->FindFile("Dir::cache::pkgcache"); + SetCacheStartBeforeRemovingCache(pkgcache); + std::string const srcpkgcache = _config->FindFile("Dir::cache::srcpkgcache"); + SetCacheStartBeforeRemovingCache(srcpkgcache); + + if (pkgcache.empty() == false) + { + std::string cachedir = flNotFile(pkgcache); + std::string cachefile = flNotDir(pkgcache); + if (cachedir.empty() != true && cachefile.empty() != true && DirectoryExists(cachedir) == true) + { + cachefile.append("."); + std::vector<std::string> caches = GetListOfFilesInDir(cachedir, false); + for (std::vector<std::string>::const_iterator file = caches.begin(); file != caches.end(); ++file) + { + std::string nuke = flNotDir(*file); + if (strncmp(cachefile.c_str(), nuke.c_str(), cachefile.length()) != 0) + continue; + RemoveFile("RemoveCaches", *file); + } + } + } + + if (srcpkgcache.empty() == true) + return; + + std::string cachedir = flNotFile(srcpkgcache); + std::string cachefile = flNotDir(srcpkgcache); + if (cachedir.empty() == true || cachefile.empty() == true || DirectoryExists(cachedir) == false) + return; + cachefile.append("."); + std::vector<std::string> caches = GetListOfFilesInDir(cachedir, false); + for (std::vector<std::string>::const_iterator file = caches.begin(); file != caches.end(); ++file) + { + std::string nuke = flNotDir(*file); + if (strncmp(cachefile.c_str(), nuke.c_str(), cachefile.length()) != 0) + continue; + RemoveFile("RemoveCaches", *file); + } +} + /*}}}*/ +// CacheFile::Close - close the cache files /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgCacheFile::Close() +{ + if (ExternOwner == false) + { + delete DCache; + delete Cache; + delete Map; + } + else + ExternOwner = false; + delete Policy; + delete SrcList; + if (d->WithLock == true) + { + _system->UnLock(true); + d->WithLock = false; + } + + Map = NULL; + DCache = NULL; + Policy = NULL; + Cache = NULL; + SrcList = NULL; +} + /*}}}*/ +void pkgCacheFile::InhibitActionGroups(bool const yes) +{ + d->InhibitActionGroups = yes; +} diff --git a/apt-pkg/cachefile.h b/apt-pkg/cachefile.h new file mode 100644 index 0000000..4e26e6d --- /dev/null +++ b/apt-pkg/cachefile.h @@ -0,0 +1,90 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + CacheFile - Simple wrapper class for opening, generating and whatnot + + This class implements a simple 2 line mechanism to open various sorts + of caches. It can operate as root, as not root, show progress and so on, + it transparently handles everything necessary. + + This means it can rebuild caches from the source list and instantiates + and prepares the standard policy mechanism. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_CACHEFILE_H +#define PKGLIB_CACHEFILE_H + +#include <stddef.h> + +#include <apt-pkg/depcache.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> + + +class MMap; +class pkgPolicy; +class pkgSourceList; +class pkgIndexFile; +class OpProgress; + +class APT_PUBLIC pkgCacheFile +{ + struct Private; + /** \brief dpointer placeholder (for later in case we need it) */ + Private *const d; + bool ExternOwner; + + protected: + MMap *Map; + pkgCache *Cache; + pkgDepCache *DCache; + pkgSourceList *SrcList; + + public: + pkgPolicy *Policy; + + // We look pretty much exactly like a pointer to a dep cache + inline operator pkgCache &() const {return *Cache;}; + inline operator pkgCache *() const {return Cache;}; + inline operator pkgDepCache &() const {return *DCache;}; + inline operator pkgDepCache *() const {return DCache;}; + inline operator pkgPolicy &() const {return *Policy;}; + inline operator pkgPolicy *() const {return Policy;}; + inline operator pkgSourceList &() const {return *SrcList;}; + inline operator pkgSourceList *() const {return SrcList;}; + inline pkgDepCache *operator ->() const {return DCache;}; + inline pkgDepCache &operator *() const {return *DCache;}; + inline pkgDepCache::StateCache &operator [](pkgCache::PkgIterator const &I) const {return (*DCache)[I];}; + inline unsigned char &operator [](pkgCache::DepIterator const &I) const {return (*DCache)[I];}; + + bool BuildCaches(OpProgress *Progress = NULL,bool WithLock = true); + bool BuildSourceList(OpProgress *Progress = NULL); + bool BuildPolicy(OpProgress *Progress = NULL); + bool BuildDepCache(OpProgress *Progress = NULL); + bool Open(OpProgress *Progress = NULL, bool WithLock = true); + inline bool ReadOnlyOpen(OpProgress *Progress = NULL) { return Open(Progress, false); }; + static void RemoveCaches(); + void Close(); + + bool AddIndexFile(pkgIndexFile * const File); + // Starts DepCache with a claim of one ActionGroup already active + void InhibitActionGroups(bool yes); + + inline pkgCache* GetPkgCache() { BuildCaches(NULL, false); return Cache; }; + inline pkgDepCache* GetDepCache() { BuildDepCache(); return DCache; }; + inline pkgPolicy* GetPolicy() { BuildPolicy(); return Policy; }; + inline pkgSourceList* GetSourceList() { BuildSourceList(); return SrcList; }; + + inline bool IsPkgCacheBuilt() const { return (Cache != NULL); }; + inline bool IsDepCacheBuilt() const { return (DCache != NULL); }; + inline bool IsPolicyBuilt() const { return (Policy != NULL); }; + inline bool IsSrcListBuilt() const { return (SrcList != NULL); }; + + pkgCacheFile(); + explicit pkgCacheFile(pkgDepCache * const Owner); + virtual ~pkgCacheFile(); +}; + +#endif diff --git a/apt-pkg/cachefilter-patterns.cc b/apt-pkg/cachefilter-patterns.cc new file mode 100644 index 0000000..faf5735 --- /dev/null +++ b/apt-pkg/cachefilter-patterns.cc @@ -0,0 +1,614 @@ +/* + * cachefilter-patterns.cc - Parser for aptitude-style patterns + * + * Copyright (c) 2019 Canonical Ltd + * + * SPDX-License-Identifier: GPL-2.0+ + */ + +#include <config.h> + +#include <apt-pkg/cachefilter-patterns.h> + +#include <apti18n.h> + +namespace APT +{ +namespace Internal +{ + +static const constexpr struct +{ + APT::StringView shortName; + APT::StringView longName; + bool takesArgument; +} shortPatterns[] = { + {"r"_sv, "?architecture"_sv, true}, + {"A"_sv, "?archive"_sv, true}, + {"M"_sv, "?automatic"_sv, false}, + {"b"_sv, "?broken"_sv, false}, + {"c"_sv, "?config-files"_sv, false}, + // FIXME: The words after ~D should be case-insensitive + {"DDepends:"_sv, "?depends"_sv, true}, + {"DPre-Depends:"_sv, "?pre-depends"_sv, true}, + {"DSuggests:"_sv, "?suggests"_sv, true}, + {"DRecommends:"_sv, "?recommends"_sv, true}, + {"DConflicts:"_sv, "?conflicts"_sv, true}, + {"DReplaces:"_sv, "?replaces"_sv, true}, + {"DObsoletes:"_sv, "?obsoletes"_sv, true}, + {"DBreaks:"_sv, "?breaks"_sv, true}, + {"DEnhances:"_sv, "?enhances"_sv, true}, + {"D"_sv, "?depends"_sv, true}, + {"RDepends:"_sv, "?reverse-depends"_sv, true}, + {"RPre-Depends:"_sv, "?reverse-pre-depends"_sv, true}, + {"RSuggests:"_sv, "?reverse-suggests"_sv, true}, + {"RRecommends:"_sv, "?reverse-recommends"_sv, true}, + {"RConflicts:"_sv, "?reverse-conflicts"_sv, true}, + {"RReplaces:"_sv, "?reverse-replaces"_sv, true}, + {"RObsoletes:"_sv, "?reverse-obsoletes"_sv, true}, + {"RBreaks:"_sv, "?reverse-breaks"_sv, true}, + {"REnhances:"_sv, "?reverse-enhances"_sv, true}, + {"R"_sv, "?reverse-depends"_sv, true}, + {"E"_sv, "?essential"_sv, false}, + {"F"_sv, "?false"_sv, false}, + {"g"_sv, "?garbage"_sv, false}, + {"i"_sv, "?installed"_sv, false}, + {"n"_sv, "?name"_sv, true}, + {"o"_sv, "?obsolete"_sv, false}, + {"O"_sv, "?origin"_sv, true}, + {"p"_sv, "?priority"_sv, true}, + {"s"_sv, "?section"_sv, true}, + {"e"_sv, "?source-package"_sv, true}, + {"T"_sv, "?true"_sv, false}, + {"U"_sv, "?upgradable"_sv, false}, + {"V"_sv, "?version"_sv, true}, + {"v"_sv, "?virtual"_sv, false}, +}; + +template <class... Args> +std::string rstrprintf(Args... args) +{ + std::string str; + strprintf(str, std::forward<Args>(args)...); + return str; +} + +// Parse a complete pattern, make sure it's the entire input +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parseTop() +{ + skipSpace(); + auto node = parse(); + skipSpace(); + + if (node == nullptr) + throw Error{Node{0, sentence.size()}, "Expected pattern"}; + + if (node->end != sentence.size()) + throw Error{Node{node->end, sentence.size()}, "Expected end of file"}; + + return node; +} + +// Parse any pattern +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parse() +{ + return parseOr(); +} + +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parseOr() +{ + auto start = state.offset; + std::vector<std::unique_ptr<PatternTreeParser::Node>> nodes; + + auto firstNode = parseAnd(); + + if (firstNode == nullptr) + return nullptr; + + nodes.push_back(std::move(firstNode)); + for (skipSpace(); sentence[state.offset] == '|'; skipSpace()) + { + state.offset++; + skipSpace(); + auto node = parseAnd(); + + if (node == nullptr) + throw Error{Node{state.offset, sentence.size()}, "Expected pattern after |"}; + + nodes.push_back(std::move(node)); + } + + if (nodes.size() == 0) + return nullptr; + if (nodes.size() == 1) + return std::move(nodes[0]); + + auto node = std::make_unique<PatternNode>(); + node->start = start; + node->end = nodes[nodes.size() - 1]->end; + node->term = "?or"; + node->arguments = std::move(nodes); + node->haveArgumentList = true; + + return node; +} + +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parseAnd() +{ + auto start = state.offset; + std::vector<std::unique_ptr<PatternTreeParser::Node>> nodes; + + for (skipSpace(); state.offset < sentence.size(); skipSpace()) + { + auto node = parseUnary(); + + if (node == nullptr) + break; + + nodes.push_back(std::move(node)); + } + + if (nodes.size() == 0) + return nullptr; + if (nodes.size() == 1) + return std::move(nodes[0]); + + auto node = std::make_unique<PatternNode>(); + node->start = start; + node->end = nodes[nodes.size() - 1]->end; + node->term = "?and"; + node->arguments = std::move(nodes); + node->haveArgumentList = true; + + return node; +} + +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parseUnary() +{ + + if (sentence[state.offset] != '!') + return parsePrimary(); + + auto start = ++state.offset; + auto primary = parsePrimary(); + + if (primary == nullptr) + throw Error{Node{start, sentence.size()}, "Expected pattern"}; + + auto node = std::make_unique<PatternNode>(); + node->start = start; + node->end = primary->end; + node->term = "?not"; + node->arguments.push_back(std::move(primary)); + node->haveArgumentList = true; + return node; +} + +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parsePrimary() +{ + std::unique_ptr<Node> node; + if ((node = parseShortPattern()) != nullptr) + return node; + if ((node = parsePattern()) != nullptr) + return node; + if ((node = parseGroup()) != nullptr) + return node; + + return nullptr; +} + +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parseGroup() +{ + if (sentence[state.offset] != '(') + return nullptr; + + auto start = state.offset++; + + skipSpace(); + auto node = parse(); + if (node == nullptr) + throw Error{Node{state.offset, sentence.size()}, + "Expected pattern after '('"}; + skipSpace(); + + if (sentence[state.offset] != ')') + throw Error{Node{state.offset, sentence.size()}, + "Expected closing parenthesis"}; + + auto end = ++state.offset; + node->start = start; + node->end = end; + return node; +} + +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parseArgument(bool shrt) +{ + std::unique_ptr<Node> node; + if ((node = parseQuotedWord()) != nullptr) + return node; + if ((node = parseWord(shrt)) != nullptr) + return node; + if ((node = parse()) != nullptr) + return node; + + throw Error{Node{state.offset, sentence.size()}, + "Expected pattern, quoted word, or word"}; +} + +// Parse a short pattern +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parseShortPattern() +{ + if (sentence[state.offset] != '~') + return nullptr; + + for (auto &sp : shortPatterns) + { + if (sentence.substr(state.offset + 1, sp.shortName.size()) != sp.shortName) + continue; + + auto node = std::make_unique<PatternNode>(); + node->end = node->start = state.offset; + node->term = sp.longName; + + state.offset += sp.shortName.size() + 1; + if (sp.takesArgument) + { + node->arguments.push_back(parseArgument(true)); + node->haveArgumentList = true; + } + node->end = state.offset; + + return node; + } + + throw Error{Node{state.offset, sentence.size()}, "Unknown short pattern"}; +} + +// Parse a list pattern (or function call pattern) +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parsePattern() +{ + static constexpr auto CHARS = ("0123456789" + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "-"_sv); + if (sentence[state.offset] != '?') + return nullptr; + + auto node = std::make_unique<PatternNode>(); + node->end = node->start = state.offset; + state.offset++; + + while (CHARS.find(sentence[state.offset]) != APT::StringView::npos) + { + ++state.offset; + } + + node->term = sentence.substr(node->start, state.offset - node->start); + + if (node->term.size() <= 1) + throw Error{*node, "Pattern must have a term/name"}; + + node->end = skipSpace(); + // We don't have any arguments, return node; + if (sentence[state.offset] != '(') + return node; + node->end = ++state.offset; + skipSpace(); + + node->haveArgumentList = true; + + // Empty argument list, return + if (sentence[state.offset] == ')') + { + node->end = ++state.offset; + return node; + } + + node->arguments.push_back(parseArgument(false)); + skipSpace(); + while (sentence[state.offset] == ',') + { + ++state.offset; + skipSpace(); + // This was a trailing comma - allow it and break the loop + if (sentence[state.offset] == ')') + break; + node->arguments.push_back(parseArgument(false)); + skipSpace(); + } + + node->end = state.offset; + if (sentence[state.offset] != ')') + throw Error{node->arguments.empty() ? *node : *node->arguments[node->arguments.size() - 1], + rstrprintf("Expected closing parenthesis or comma after last argument, received %c", sentence[state.offset])}; + + node->end = ++state.offset; + return node; +} + +// Parse a quoted word atom +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parseQuotedWord() +{ + if (sentence[state.offset] != '"') + return nullptr; + + auto node = std::make_unique<WordNode>(); + node->start = state.offset; + + // Eat beginning of string + state.offset++; + + while (sentence[state.offset] != '"' && sentence[state.offset] != '\0') + state.offset++; + + // End of string + if (sentence[state.offset] != '"') + throw Error{*node, "Could not find end of string"}; + state.offset++; + + node->end = state.offset; + node->word = sentence.substr(node->start + 1, node->end - node->start - 2); + + return node; +} + +// Parse a bare word atom +std::unique_ptr<PatternTreeParser::Node> PatternTreeParser::parseWord(bool shrt) +{ + // Characters not allowed at the start of a word (also see ..._SHRT) + static const constexpr auto DISALLOWED_START = "!?~|,() \0"_sv; + // Characters terminating a word inside a long pattern + static const constexpr auto DISALLOWED_LONG = "|,()\0"_sv; + // Characters terminating a word as a short form argument, should contain all of START. + static const constexpr auto DISALLOWED_SHRT = "!?~|,() \0"_sv; + const auto DISALLOWED = shrt ? DISALLOWED_SHRT : DISALLOWED_LONG; + + if (DISALLOWED_START.find(sentence[state.offset]) != APT::StringView::npos) + return nullptr; + + auto node = std::make_unique<WordNode>(); + node->start = state.offset; + + while (DISALLOWED.find(sentence[state.offset]) == APT::StringView::npos) + state.offset++; + + node->end = state.offset; + node->word = sentence.substr(node->start, node->end - node->start); + return node; +} + +// Rendering of the tree in JSON for debugging +std::ostream &PatternTreeParser::PatternNode::render(std::ostream &os) +{ + + os << term.to_string(); + if (haveArgumentList) + { + os << "("; + for (auto &node : arguments) + node->render(os) << ","; + os << ")"; + } + return os; +} + +std::ostream &PatternTreeParser::WordNode::render(std::ostream &os) +{ + return quoted ? os << '"' << word.to_string() << '"' : os << word.to_string(); +} + +std::nullptr_t PatternTreeParser::Node::error(std::string message) +{ + throw Error{*this, message}; +} + +bool PatternTreeParser::PatternNode::matches(APT::StringView name, int min, int max) +{ + if (name != term) + return false; + if (max != 0 && !haveArgumentList) + error(rstrprintf("%s expects an argument list", term.to_string().c_str())); + if (max == 0 && haveArgumentList) + error(rstrprintf("%s does not expect an argument list", term.to_string().c_str())); + if (min >= 0 && min == max && (arguments.size() != size_t(min))) + error(rstrprintf("%s expects %d arguments, but received %d arguments", term.to_string().c_str(), min, arguments.size())); + if (min >= 0 && arguments.size() < size_t(min)) + error(rstrprintf("%s expects at least %d arguments, but received %d arguments", term.to_string().c_str(), min, arguments.size())); + if (max >= 0 && arguments.size() > size_t(max)) + error(rstrprintf("%s expects at most %d arguments, but received %d arguments", term.to_string().c_str(), max, arguments.size())); + return true; +} + +std::unique_ptr<APT::CacheFilter::Matcher> PatternParser::aPattern(std::unique_ptr<PatternTreeParser::Node> &nodeP) +{ + assert(nodeP != nullptr); + auto node = dynamic_cast<PatternTreeParser::PatternNode *>(nodeP.get()); + if (node == nullptr) + nodeP->error("Expected a pattern"); + + if (node->matches("?architecture", 1, 1)) + return std::make_unique<APT::CacheFilter::PackageArchitectureMatchesSpecification>(aWord(node->arguments[0])); + if (node->matches("?archive", 1, 1)) + return std::make_unique<Patterns::VersionIsArchive>(aWord(node->arguments[0])); + if (node->matches("?codename", 1, 1)) + return std::make_unique<Patterns::VersionIsCodename>(aWord(node->arguments[0])); + if (node->matches("?all-versions", 1, 1)) + return std::make_unique<Patterns::VersionIsAllVersions>(aPattern(node->arguments[0])); + if (node->matches("?any-version", 1, 1)) + return std::make_unique<Patterns::VersionIsAnyVersion>(aPattern(node->arguments[0])); + if (node->matches("?automatic", 0, 0)) + return std::make_unique<Patterns::PackageIsAutomatic>(file); + if (node->matches("?broken", 0, 0)) + return std::make_unique<Patterns::PackageIsBroken>(file); + if (node->matches("?config-files", 0, 0)) + return std::make_unique<Patterns::PackageIsConfigFiles>(); + if (node->matches("?depends", 1, 1)) + return std::make_unique<Patterns::VersionDepends>(aPattern(node->arguments[0])); + if (node->matches("?predepends", 1, 1)) + return std::make_unique<Patterns::VersionDepends>(aPattern(node->arguments[0]), pkgCache::Dep::PreDepends); + if (node->matches("?suggests", 1, 1)) + return std::make_unique<Patterns::VersionDepends>(aPattern(node->arguments[0]), pkgCache::Dep::Suggests); + if (node->matches("?recommends", 1, 1)) + return std::make_unique<Patterns::VersionDepends>(aPattern(node->arguments[0]), pkgCache::Dep::Recommends); + if (node->matches("?conflicts", 1, 1)) + return std::make_unique<Patterns::VersionDepends>(aPattern(node->arguments[0]), pkgCache::Dep::Conflicts); + if (node->matches("?replaces", 1, 1)) + return std::make_unique<Patterns::VersionDepends>(aPattern(node->arguments[0]), pkgCache::Dep::Replaces); + if (node->matches("?obsoletes", 1, 1)) + return std::make_unique<Patterns::VersionDepends>(aPattern(node->arguments[0]), pkgCache::Dep::Obsoletes); + if (node->matches("?breaks", 1, 1)) + return std::make_unique<Patterns::VersionDepends>(aPattern(node->arguments[0]), pkgCache::Dep::DpkgBreaks); + if (node->matches("?enhances", 1, 1)) + return std::make_unique<Patterns::VersionDepends>(aPattern(node->arguments[0]), pkgCache::Dep::Enhances); + if (node->matches("?reverse-depends", 1, 1)) + return std::make_unique<Patterns::PackageReverseDepends>(aPattern(node->arguments[0])); + if (node->matches("?reverse-predepends", 1, 1)) + return std::make_unique<Patterns::PackageReverseDepends>(aPattern(node->arguments[0]), pkgCache::Dep::PreDepends); + if (node->matches("?reverse-suggests", 1, 1)) + return std::make_unique<Patterns::PackageReverseDepends>(aPattern(node->arguments[0]), pkgCache::Dep::Suggests); + if (node->matches("?reverse-recommends", 1, 1)) + return std::make_unique<Patterns::PackageReverseDepends>(aPattern(node->arguments[0]), pkgCache::Dep::Recommends); + if (node->matches("?reverse-conflicts", 1, 1)) + return std::make_unique<Patterns::PackageReverseDepends>(aPattern(node->arguments[0]), pkgCache::Dep::Conflicts); + if (node->matches("?reverse-replaces", 1, 1)) + return std::make_unique<Patterns::PackageReverseDepends>(aPattern(node->arguments[0]), pkgCache::Dep::Replaces); + if (node->matches("?reverse-obsoletes", 1, 1)) + return std::make_unique<Patterns::PackageReverseDepends>(aPattern(node->arguments[0]), pkgCache::Dep::Obsoletes); + if (node->matches("?reverse-breaks", 1, 1)) + return std::make_unique<Patterns::PackageReverseDepends>(aPattern(node->arguments[0]), pkgCache::Dep::DpkgBreaks); + if (node->matches("?reverse-enhances", 1, 1)) + return std::make_unique<Patterns::PackageReverseDepends>(aPattern(node->arguments[0]), pkgCache::Dep::Enhances); + if (node->matches("?essential", 0, 0)) + return std::make_unique<Patterns::PackageIsEssential>(); + if (node->matches("?priority", 1, 1)) + return std::make_unique<Patterns::VersionIsPriority>(aWord(node->arguments[0])); + if (node->matches("?exact-name", 1, 1)) + return std::make_unique<Patterns::PackageHasExactName>(aWord(node->arguments[0])); + if (node->matches("?false", 0, 0)) + return std::make_unique<APT::CacheFilter::FalseMatcher>(); + if (node->matches("?garbage", 0, 0)) + return std::make_unique<Patterns::PackageIsGarbage>(file); + if (node->matches("?installed", 0, 0)) + return std::make_unique<Patterns::PackageIsInstalled>(file); + if (node->matches("?name", 1, 1)) + return std::make_unique<APT::CacheFilter::PackageNameMatchesRegEx>(aWord(node->arguments[0])); + if (node->matches("?not", 1, 1)) + return std::make_unique<APT::CacheFilter::NOTMatcher>(aPattern(node->arguments[0]).release()); + if (node->matches("?obsolete", 0, 0)) + return std::make_unique<Patterns::PackageIsObsolete>(); + if (node->matches("?origin", 1, 1)) + return std::make_unique<Patterns::VersionIsOrigin>(aWord(node->arguments[0])); + if (node->matches("?section", 1, 1)) + return std::make_unique<Patterns::VersionIsSection>(aWord(node->arguments[0])); + if (node->matches("?source-package", 1, 1)) + return std::make_unique<Patterns::VersionIsSourcePackage>(aWord(node->arguments[0])); + if (node->matches("?source-version", 1, 1)) + return std::make_unique<Patterns::VersionIsSourceVersion>(aWord(node->arguments[0])); + if (node->matches("?true", 0, 0)) + return std::make_unique<APT::CacheFilter::TrueMatcher>(); + if (node->matches("?upgradable", 0, 0)) + return std::make_unique<Patterns::PackageIsUpgradable>(file); + if (node->matches("?version", 1, 1)) + return std::make_unique<Patterns::VersionIsVersion>(aWord(node->arguments[0])); + if (node->matches("?virtual", 0, 0)) + return std::make_unique<Patterns::PackageIsVirtual>(); + if (node->matches("?x-name-fnmatch", 1, 1)) + return std::make_unique<APT::CacheFilter::PackageNameMatchesFnmatch>(aWord(node->arguments[0])); + + // Variable argument patterns + if (node->matches("?and", 0, -1) || node->matches("?narrow", 0, -1)) + { + auto pattern = std::make_unique<APT::CacheFilter::ANDMatcher>(); + for (auto &arg : node->arguments) + pattern->AND(aPattern(arg).release()); + if (node->term == "?narrow") + return std::make_unique<Patterns::VersionIsAnyVersion>(std::move(pattern)); + return pattern; + } + if (node->matches("?or", 0, -1)) + { + auto pattern = std::make_unique<APT::CacheFilter::ORMatcher>(); + + for (auto &arg : node->arguments) + pattern->OR(aPattern(arg).release()); + return pattern; + } + + node->error(rstrprintf("Unrecognized pattern '%s'", node->term.to_string().c_str())); + + return nullptr; +} + +std::string PatternParser::aWord(std::unique_ptr<PatternTreeParser::Node> &nodeP) +{ + assert(nodeP != nullptr); + auto node = dynamic_cast<PatternTreeParser::WordNode *>(nodeP.get()); + if (node == nullptr) + nodeP->error("Expected a word"); + return node->word.to_string(); +} + +namespace Patterns +{ + +BaseRegexMatcher::BaseRegexMatcher(std::string const &Pattern) +{ + pattern = new regex_t; + int const Res = regcomp(pattern, Pattern.c_str(), REG_EXTENDED | REG_ICASE | REG_NOSUB); + if (Res == 0) + return; + + delete pattern; + pattern = NULL; + char Error[300]; + regerror(Res, pattern, Error, sizeof(Error)); + _error->Error(_("Regex compilation error - %s"), Error); +} +bool BaseRegexMatcher::operator()(const char *string) +{ + if (unlikely(pattern == nullptr) || string == nullptr) + return false; + else + return regexec(pattern, string, 0, 0, 0) == 0; +} +BaseRegexMatcher::~BaseRegexMatcher() +{ + if (pattern == NULL) + return; + regfree(pattern); + delete pattern; +} +} // namespace Patterns + +} // namespace Internal + +// The bridge into the public world +std::unique_ptr<APT::CacheFilter::Matcher> APT::CacheFilter::ParsePattern(APT::StringView pattern, pkgCacheFile *file) +{ + if (file != nullptr && !file->BuildDepCache()) + return nullptr; + + try + { + auto top = APT::Internal::PatternTreeParser(pattern).parseTop(); + APT::Internal::PatternParser parser{file}; + return parser.aPattern(top); + } + catch (APT::Internal::PatternTreeParser::Error &e) + { + std::stringstream ss; + ss << "input:" << e.location.start << "-" << e.location.end << ": error: " << e.message << "\n"; + ss << pattern.to_string() << "\n"; + for (size_t i = 0; i < e.location.start; i++) + ss << " "; + for (size_t i = e.location.start; i < e.location.end; i++) + ss << "^"; + + ss << "\n"; + + _error->Error("%s", ss.str().c_str()); + return nullptr; + } +} + +} // namespace APT diff --git a/apt-pkg/cachefilter-patterns.h b/apt-pkg/cachefilter-patterns.h new file mode 100644 index 0000000..2d48a1c --- /dev/null +++ b/apt-pkg/cachefilter-patterns.h @@ -0,0 +1,456 @@ +/* + * cachefilter-patterns.h - Pattern parser and additional patterns as matchers + * + * Copyright (c) 2019 Canonical Ltd + * + * SPDX-License-Identifier: GPL-2.0+ + */ + +#ifndef APT_CACHEFILTER_PATTERNS_H +#define APT_CACHEFILTER_PATTERNS_H +#include <apt-pkg/cachefile.h> +#include <apt-pkg/cachefilter.h> +#include <apt-pkg/error.h> +#include <apt-pkg/header-is-private.h> +#include <apt-pkg/string_view.h> +#include <apt-pkg/strutl.h> +#include <iostream> +#include <memory> +#include <sstream> +#include <string> +#include <vector> +#include <assert.h> + +namespace APT +{ + +namespace Internal +{ +/** + * \brief PatternTreeParser parses the given sentence into a parse tree. + * + * The parse tree consists of nodes: + * - Word nodes which contains words or quoted words + * - Patterns, which represent ?foo and ?foo(...) patterns + */ +struct APT_PUBLIC PatternTreeParser +{ + + struct Node + { + size_t start = 0; + size_t end = 0; + + explicit Node(size_t start = 0, size_t end = 0) : start(start), end(end) {} + + virtual std::ostream &render(std::ostream &os) { return os; }; + std::nullptr_t error(std::string message); + virtual ~Node() = default; + }; + + struct Error : public std::exception + { + Node location; + std::string message; + + Error(Node location, std::string message) : location(location), message(message) {} + const char *what() const throw() override { return message.c_str(); } + }; + + struct PatternNode : public Node + { + APT::StringView term; + std::vector<std::unique_ptr<Node>> arguments; + bool haveArgumentList = false; + + APT_HIDDEN std::ostream &render(std::ostream &stream) override; + APT_HIDDEN bool matches(APT::StringView name, int min, int max); + }; + + struct WordNode : public Node + { + APT::StringView word; + bool quoted = false; + APT_HIDDEN std::ostream &render(std::ostream &stream) override; + }; + + struct State + { + size_t offset = 0; + }; + + APT::StringView sentence; + State state; + + PatternTreeParser(APT::StringView sentence) : sentence(sentence){}; + off_t skipSpace() + { + while (sentence[state.offset] == ' ' || sentence[state.offset] == '\t' || sentence[state.offset] == '\r' || sentence[state.offset] == '\n') + state.offset++; + return state.offset; + }; + + /// \brief Parse a complete pattern + /// + /// There may not be anything before or after the pattern, except for + /// whitespace. + std::unique_ptr<Node> parseTop(); + std::unique_ptr<Node> parse(); // public for test cases only + + private: + APT_HIDDEN std::unique_ptr<Node> parseOr(); + APT_HIDDEN std::unique_ptr<Node> parseAnd(); + APT_HIDDEN std::unique_ptr<Node> parseUnary(); + APT_HIDDEN std::unique_ptr<Node> parsePrimary(); + APT_HIDDEN std::unique_ptr<Node> parseGroup(); + APT_HIDDEN std::unique_ptr<Node> parsePattern(); + APT_HIDDEN std::unique_ptr<Node> parseShortPattern(); + APT_HIDDEN std::unique_ptr<Node> parseArgument(bool shrt); + APT_HIDDEN std::unique_ptr<Node> parseWord(bool shrt); + APT_HIDDEN std::unique_ptr<Node> parseQuotedWord(); +}; + +/** + * \brief PatternParser parses the given sentence into a parse tree. + * + * The parse tree consists of nodes: + * - Word nodes which contains words or quoted words + * - Patterns, which represent ?foo and ?foo(...) patterns + */ +struct APT_HIDDEN PatternParser +{ + pkgCacheFile *file; + + std::unique_ptr<APT::CacheFilter::Matcher> aPattern(std::unique_ptr<PatternTreeParser::Node> &nodeP); + std::string aWord(std::unique_ptr<PatternTreeParser::Node> &nodeP); +}; + +namespace Patterns +{ +using namespace APT::CacheFilter; + +/** \brief Basic helper class for matching regex */ +class BaseRegexMatcher +{ + regex_t *pattern; + + public: + BaseRegexMatcher(std::string const &string); + ~BaseRegexMatcher(); + bool operator()(const char *cstring); + bool operator()(std::string const &string) + { + return (*this)(string.c_str()); + } +}; + +struct APT_HIDDEN PackageIsAutomatic : public PackageMatcher +{ + pkgCacheFile *Cache; + explicit PackageIsAutomatic(pkgCacheFile *Cache) : Cache(Cache) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + assert(Cache != nullptr); + return ((*Cache)[Pkg].Flags & pkgCache::Flag::Auto) != 0; + } +}; + +struct APT_HIDDEN PackageIsBroken : public PackageMatcher +{ + pkgCacheFile *Cache; + explicit PackageIsBroken(pkgCacheFile *Cache) : Cache(Cache) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + assert(Cache != nullptr); + auto state = (*Cache)[Pkg]; + return state.InstBroken() || state.NowBroken(); + } +}; + +struct APT_HIDDEN PackageIsConfigFiles : public PackageMatcher +{ + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + return Pkg->CurrentState == pkgCache::State::ConfigFiles; + } +}; + +struct APT_HIDDEN PackageIsGarbage : public PackageMatcher +{ + pkgCacheFile *Cache; + explicit PackageIsGarbage(pkgCacheFile *Cache) : Cache(Cache) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + assert(Cache != nullptr); + return (*Cache)[Pkg].Garbage; + } +}; +struct APT_HIDDEN PackageIsEssential : public PackageMatcher +{ + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + return (Pkg->Flags & pkgCache::Flag::Essential) != 0; + } +}; + +struct APT_HIDDEN PackageHasExactName : public PackageMatcher +{ + std::string name; + explicit PackageHasExactName(std::string name) : name(name) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + return Pkg.Name() == name; + } +}; + +struct APT_HIDDEN PackageIsInstalled : public PackageMatcher +{ + pkgCacheFile *Cache; + explicit PackageIsInstalled(pkgCacheFile *Cache) : Cache(Cache) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + assert(Cache != nullptr); + return Pkg->CurrentVer != 0; + } + bool operator()(pkgCache::VerIterator const &Ver) override + { + assert(Cache != nullptr); + return Ver == Ver.ParentPkg().CurrentVer(); + } +}; + +struct APT_HIDDEN PackageIsObsolete : public PackageMatcher +{ + bool operator()(pkgCache::PkgIterator const &pkg) override + { + // This code can be written without loops, as aptitude does, but it + // is far less readable. + if (pkg.CurrentVer().end()) + return false; + + // See if there is any version that exists in a repository, + // if so return false + for (auto ver = pkg.VersionList(); !ver.end(); ver++) + { + for (auto file = ver.FileList(); !file.end(); file++) + { + if ((file.File()->Flags & pkgCache::Flag::NotSource) == 0) + return false; + } + } + + return true; + } +}; + +struct APT_HIDDEN PackageIsUpgradable : public PackageMatcher +{ + pkgCacheFile *Cache; + explicit PackageIsUpgradable(pkgCacheFile *Cache) : Cache(Cache) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + assert(Cache != nullptr); + return Pkg->CurrentVer != 0 && (*Cache)[Pkg].Upgradable(); + } +}; + +struct APT_HIDDEN PackageIsVirtual : public PackageMatcher +{ + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + return Pkg->VersionList == 0; + } +}; + +struct APT_HIDDEN VersionAnyMatcher : public Matcher +{ + bool operator()(pkgCache::GrpIterator const &) override { return false; } + bool operator()(pkgCache::VerIterator const &Ver) override = 0; + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + for (auto Ver = Pkg.VersionList(); not Ver.end(); Ver++) + { + if ((*this)(Ver)) + return true; + } + return false; + } +}; + +struct APT_HIDDEN VersionIsAllVersions : public Matcher +{ + std::unique_ptr<APT::CacheFilter::Matcher> base; + VersionIsAllVersions(std::unique_ptr<APT::CacheFilter::Matcher> base) : base(std::move(base)) {} + bool operator()(pkgCache::GrpIterator const &) override { return false; } + bool operator()(pkgCache::VerIterator const &Ver) override + { + return (*base)(Ver); + } + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + for (auto Ver = Pkg.VersionList(); not Ver.end(); Ver++) + { + if (not(*this)(Ver)) + return false; + } + return true; + } +}; + +struct APT_HIDDEN VersionDepends : public VersionAnyMatcher +{ + std::unique_ptr<APT::CacheFilter::Matcher> base; + pkgCache::Dep::DepType type; + VersionDepends(std::unique_ptr<APT::CacheFilter::Matcher> base, pkgCache::Dep::DepType type = pkgCache::Dep::Depends) : base(std::move(base)), type(type) {} + bool operator()(pkgCache::GrpIterator const &) override { return false; } + bool operator()(pkgCache::VerIterator const &Ver) override + { + for (auto D = Ver.DependsList(); not D.end(); D++) + { + if (D.IsImplicit()) + continue; + if (D->Type != type) + continue; + if ((*base)(D.TargetPkg())) + return true; + } + + return false; + } +}; + +struct APT_HIDDEN PackageReverseDepends : public PackageMatcher +{ + std::unique_ptr<APT::CacheFilter::Matcher> base; + pkgCache::Dep::DepType type; + PackageReverseDepends(std::unique_ptr<APT::CacheFilter::Matcher> base, pkgCache::Dep::DepType type = pkgCache::Dep::Depends) : base(std::move(base)), type(type) {} + bool operator()(pkgCache::PkgIterator const &Pkg) override + { + for (auto D = Pkg.RevDependsList(); not D.end(); D++) + { + if (D.IsImplicit()) + continue; + if (D->Type != type) + continue; + if ((*base)(D.ParentVer())) + return true; + } + + return false; + } +}; + +struct APT_HIDDEN VersionIsAnyVersion : public VersionAnyMatcher +{ + std::unique_ptr<APT::CacheFilter::Matcher> base; + VersionIsAnyVersion(std::unique_ptr<APT::CacheFilter::Matcher> base) : base(std::move(base)) {} + bool operator()(pkgCache::VerIterator const &Ver) override + { + return (*base)(Ver); + } +}; + +struct APT_HIDDEN VersionIsArchive : public VersionAnyMatcher +{ + BaseRegexMatcher matcher; + VersionIsArchive(std::string const &pattern) : matcher(pattern) {} + bool operator()(pkgCache::VerIterator const &Ver) override + { + for (auto VF = Ver.FileList(); not VF.end(); VF++) + { + if (VF.File().Archive() && matcher(VF.File().Archive())) + return true; + } + return false; + } +}; + +struct APT_HIDDEN VersionIsCodename : public VersionAnyMatcher +{ + BaseRegexMatcher matcher; + VersionIsCodename(std::string const &pattern) : matcher(pattern) {} + bool operator()(pkgCache::VerIterator const &Ver) override + { + for (auto VF = Ver.FileList(); not VF.end(); VF++) + { + if (VF.File().Codename() && matcher(VF.File().Codename())) + return true; + } + return false; + } +}; + +struct APT_HIDDEN VersionIsOrigin : public VersionAnyMatcher +{ + BaseRegexMatcher matcher; + VersionIsOrigin(std::string const &pattern) : matcher(pattern) {} + bool operator()(pkgCache::VerIterator const &Ver) override + { + for (auto VF = Ver.FileList(); not VF.end(); VF++) + { + if (VF.File().Origin() && matcher(VF.File().Origin())) + return true; + } + return false; + } +}; + +struct APT_HIDDEN VersionIsSection : public VersionAnyMatcher +{ + BaseRegexMatcher matcher; + VersionIsSection(std::string const &pattern) : matcher(pattern) {} + bool operator()(pkgCache::VerIterator const &Ver) override + { + return matcher(Ver.Section()); + } +}; + +struct APT_HIDDEN VersionIsSourcePackage : public VersionAnyMatcher +{ + BaseRegexMatcher matcher; + VersionIsSourcePackage(std::string const &pattern) : matcher(pattern) {} + bool operator()(pkgCache::VerIterator const &Ver) override + { + return matcher(Ver.SourcePkgName()); + } +}; + +struct APT_HIDDEN VersionIsSourceVersion : public VersionAnyMatcher +{ + BaseRegexMatcher matcher; + VersionIsSourceVersion(std::string const &pattern) : matcher(pattern) {} + bool operator()(pkgCache::VerIterator const &Ver) override + { + return matcher(Ver.SourceVerStr()); + } +}; + +struct APT_HIDDEN VersionIsVersion : public VersionAnyMatcher +{ + BaseRegexMatcher matcher; + VersionIsVersion(std::string const &pattern) : matcher(pattern) {} + bool operator()(pkgCache::VerIterator const &Ver) override + { + return matcher(Ver.VerStr()); + } +}; + +struct APT_HIDDEN VersionIsPriority : public VersionAnyMatcher +{ + std::string name; + explicit VersionIsPriority(std::string name) : name(name) {} + bool operator()(pkgCache::VerIterator const &Ver) override + { + std::string Mapping[] = {"", "required","important","standard", + "optional","extra"}; + if (Ver->Priority > 0 && Ver->Priority < APT_ARRAY_SIZE(Mapping)) { + return name == Mapping[Ver->Priority]; + } + return false; + } +}; + +} // namespace Patterns +} // namespace Internal +} // namespace APT +#endif diff --git a/apt-pkg/cachefilter.cc b/apt-pkg/cachefilter.cc new file mode 100644 index 0000000..eadbb98 --- /dev/null +++ b/apt-pkg/cachefilter.cc @@ -0,0 +1,264 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/** \file cachefilter.h + Collection of functor classes */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/cachefile.h> +#include <apt-pkg/cachefilter.h> +#include <apt-pkg/error.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <string> +#include <unordered_map> +#include <fnmatch.h> +#include <regex.h> +#include <string.h> + +#include <apti18n.h> + /*}}}*/ +namespace APT { + +APT_HIDDEN std::unordered_map<std::string, std::vector<std::string>> ArchToTupleMap; + +namespace CacheFilter { +Matcher::~Matcher() {} +PackageMatcher::~PackageMatcher() {} + +// Name matches RegEx /*{{{*/ +PackageNameMatchesRegEx::PackageNameMatchesRegEx(std::string const &Pattern) { + pattern = new regex_t; + int const Res = regcomp(pattern, Pattern.c_str(), REG_EXTENDED | REG_ICASE | REG_NOSUB); + if (Res == 0) + return; + + delete pattern; + pattern = NULL; + char Error[300]; + regerror(Res, pattern, Error, sizeof(Error)); + _error->Error(_("Regex compilation error - %s"), Error); +} +bool PackageNameMatchesRegEx::operator() (pkgCache::PkgIterator const &Pkg) { + if (unlikely(pattern == NULL)) + return false; + else + return regexec(pattern, Pkg.Name(), 0, 0, 0) == 0; +} +bool PackageNameMatchesRegEx::operator() (pkgCache::GrpIterator const &Grp) { + if (unlikely(pattern == NULL)) + return false; + else + return regexec(pattern, Grp.Name(), 0, 0, 0) == 0; +} +PackageNameMatchesRegEx::~PackageNameMatchesRegEx() { + if (pattern == NULL) + return; + regfree(pattern); + delete pattern; +} + /*}}}*/ +// Name matches Fnmatch /*{{{*/ +PackageNameMatchesFnmatch::PackageNameMatchesFnmatch(std::string const &Pattern) : + Pattern(Pattern) {} +bool PackageNameMatchesFnmatch::operator() (pkgCache::PkgIterator const &Pkg) { + return fnmatch(Pattern.c_str(), Pkg.Name(), FNM_CASEFOLD) == 0; +} +bool PackageNameMatchesFnmatch::operator() (pkgCache::GrpIterator const &Grp) { + return fnmatch(Pattern.c_str(), Grp.Name(), FNM_CASEFOLD) == 0; +} + /*}}}*/ +// Architecture matches <abi>-<libc>-<kernel>-<cpu> specification /*{{{*/ +//---------------------------------------------------------------------- + +static std::vector<std::string> ArchToTuple(std::string arch) { + // Strip leading linux- from arch if present + // dpkg says this may disappear in the future + if (APT::String::Startswith(arch, std::string("linux-"))) + arch = arch.substr(6); + + auto it = ArchToTupleMap.find(arch); + if (it != ArchToTupleMap.end()) + { + std::vector<std::string> result = it->second; + // Hack in support for triplets + if (result.size() == 3) + result.emplace(result.begin(), "base"); + return result; + } else + { + return {}; + } +} + +static std::vector<std::string> PatternToTuple(std::string const &arch) { + std::vector<std::string> tuple = VectorizeString(arch, '-'); + if (std::find(tuple.begin(), tuple.end(), std::string("any")) != tuple.end() || + std::find(arch.begin(), arch.end(), '*') != arch.end()) { + while (tuple.size() < 4) { + tuple.emplace(tuple.begin(), "any"); + } + return tuple; + } else + return ArchToTuple(arch); +} + +/* The complete architecture, consisting of <abi>-<libc>-<kernel>-<cpu>. */ +static std::string CompleteArch(std::string const &arch, bool const isPattern) { + auto tuple = isPattern ? PatternToTuple(arch) : ArchToTuple(arch); + + // Bah, the commandline will try and pass us stuff like amd64- -- we need + // that not to match an architecture, but the code below would turn it into + // a valid tuple. Let's just use an invalid tuple here. + if (APT::String::Endswith(arch, "-") || APT::String::Startswith(arch, "-")) + return "invalid-invalid-invalid-invalid"; + + if (tuple.empty()) { + // Fallback for unknown architectures + // Patterns never fail if they contain wildcards, so by this point, arch + // has no wildcards. + tuple = VectorizeString(arch, '-'); + switch (tuple.size()) { + case 1: + tuple.emplace(tuple.begin(), "linux"); + /* fall through */ + case 2: + tuple.emplace(tuple.begin(), "gnu"); + /* fall through */ + case 3: + tuple.emplace(tuple.begin(), "base"); + /* fall through */ + break; + } + } + + std::replace(tuple.begin(), tuple.end(), std::string("any"), std::string("*")); + return APT::String::Join(tuple, "-"); +} +PackageArchitectureMatchesSpecification::PackageArchitectureMatchesSpecification(std::string const &pattern, bool const pisPattern) : + literal(pattern), complete(CompleteArch(pattern, pisPattern)), isPattern(pisPattern) { +} +bool PackageArchitectureMatchesSpecification::operator() (char const * const &arch) { + if (strcmp(literal.c_str(), arch) == 0 || + strcmp(complete.c_str(), arch) == 0) + return true; + std::string const pkgarch = CompleteArch(arch, !isPattern); + if (isPattern == true) + return fnmatch(complete.c_str(), pkgarch.c_str(), 0) == 0; + return fnmatch(pkgarch.c_str(), complete.c_str(), 0) == 0; +} +bool PackageArchitectureMatchesSpecification::operator() (pkgCache::PkgIterator const &Pkg) { + return (*this)(Pkg.Arch()); +} +PackageArchitectureMatchesSpecification::~PackageArchitectureMatchesSpecification() { +} + /*}}}*/ +// Package is new install /*{{{*/ +PackageIsNewInstall::PackageIsNewInstall(pkgCacheFile * const Cache) : Cache(Cache) {} +APT_PURE bool PackageIsNewInstall::operator() (pkgCache::PkgIterator const &Pkg) { + return (*Cache)[Pkg].NewInstall(); +} +PackageIsNewInstall::~PackageIsNewInstall() {} + /*}}}*/ +// Generica like True, False, NOT, AND, OR /*{{{*/ +APT_PURE bool TrueMatcher::operator() (pkgCache::PkgIterator const &) { return true; } +APT_PURE bool TrueMatcher::operator() (pkgCache::GrpIterator const &) { return true; } +APT_PURE bool TrueMatcher::operator() (pkgCache::VerIterator const &) { return true; } + +APT_PURE bool FalseMatcher::operator() (pkgCache::PkgIterator const &) { return false; } +APT_PURE bool FalseMatcher::operator() (pkgCache::GrpIterator const &) { return false; } +APT_PURE bool FalseMatcher::operator() (pkgCache::VerIterator const &) { return false; } + +NOTMatcher::NOTMatcher(Matcher * const matcher) : matcher(matcher) {} +bool NOTMatcher::operator() (pkgCache::PkgIterator const &Pkg) { return ! (*matcher)(Pkg); } +bool NOTMatcher::operator() (pkgCache::GrpIterator const &Grp) { return ! (*matcher)(Grp); } +bool NOTMatcher::operator() (pkgCache::VerIterator const &Ver) { return ! (*matcher)(Ver); } +NOTMatcher::~NOTMatcher() { delete matcher; } + +ANDMatcher::ANDMatcher() {} +ANDMatcher::ANDMatcher(Matcher * const matcher1) { + AND(matcher1); +} +ANDMatcher::ANDMatcher(Matcher * const matcher1, Matcher * const matcher2) { + AND(matcher1).AND(matcher2); +} +ANDMatcher::ANDMatcher(Matcher * const matcher1, Matcher * const matcher2, Matcher * const matcher3) { + AND(matcher1).AND(matcher2).AND(matcher3); +} +ANDMatcher::ANDMatcher(Matcher * const matcher1, Matcher * const matcher2, Matcher * const matcher3, Matcher * const matcher4) { + AND(matcher1).AND(matcher2).AND(matcher3).AND(matcher4); +} +ANDMatcher::ANDMatcher(Matcher * const matcher1, Matcher * const matcher2, Matcher * const matcher3, Matcher * const matcher4, Matcher * const matcher5) { + AND(matcher1).AND(matcher2).AND(matcher3).AND(matcher4).AND(matcher5); +} +ANDMatcher& ANDMatcher::AND(Matcher * const matcher) { matchers.push_back(matcher); return *this; } +bool ANDMatcher::operator() (pkgCache::PkgIterator const &Pkg) { + for (std::vector<Matcher *>::const_iterator M = matchers.begin(); M != matchers.end(); ++M) + if ((**M)(Pkg) == false) + return false; + return true; +} +bool ANDMatcher::operator() (pkgCache::GrpIterator const &Grp) { + for (std::vector<Matcher *>::const_iterator M = matchers.begin(); M != matchers.end(); ++M) + if ((**M)(Grp) == false) + return false; + return true; +} +bool ANDMatcher::operator() (pkgCache::VerIterator const &Ver) { + for (std::vector<Matcher *>::const_iterator M = matchers.begin(); M != matchers.end(); ++M) + if ((**M)(Ver) == false) + return false; + return true; +} +ANDMatcher::~ANDMatcher() { + for (std::vector<Matcher *>::iterator M = matchers.begin(); M != matchers.end(); ++M) + delete *M; +} + +ORMatcher::ORMatcher() {} +ORMatcher::ORMatcher(Matcher * const matcher1) { + OR(matcher1); +} +ORMatcher::ORMatcher(Matcher * const matcher1, Matcher * const matcher2) { + OR(matcher1).OR(matcher2); +} +ORMatcher::ORMatcher(Matcher * const matcher1, Matcher * const matcher2, Matcher * const matcher3) { + OR(matcher1).OR(matcher2).OR(matcher3); +} +ORMatcher::ORMatcher(Matcher * const matcher1, Matcher * const matcher2, Matcher * const matcher3, Matcher * const matcher4) { + OR(matcher1).OR(matcher2).OR(matcher3).OR(matcher4); +} +ORMatcher::ORMatcher(Matcher * const matcher1, Matcher * const matcher2, Matcher * const matcher3, Matcher * const matcher4, Matcher * const matcher5) { + OR(matcher1).OR(matcher2).OR(matcher3).OR(matcher4).OR(matcher5); +} +ORMatcher& ORMatcher::OR(Matcher * const matcher) { matchers.push_back(matcher); return *this; } +bool ORMatcher::operator() (pkgCache::PkgIterator const &Pkg) { + for (std::vector<Matcher *>::const_iterator M = matchers.begin(); M != matchers.end(); ++M) + if ((**M)(Pkg) == true) + return true; + return false; +} +bool ORMatcher::operator() (pkgCache::GrpIterator const &Grp) { + for (std::vector<Matcher *>::const_iterator M = matchers.begin(); M != matchers.end(); ++M) + if ((**M)(Grp) == true) + return true; + return false; +} +bool ORMatcher::operator() (pkgCache::VerIterator const &Ver) { + for (std::vector<Matcher *>::const_iterator M = matchers.begin(); M != matchers.end(); ++M) + if ((**M)(Ver) == true) + return true; + return false; +} +ORMatcher::~ORMatcher() { + for (std::vector<Matcher *>::iterator M = matchers.begin(); M != matchers.end(); ++M) + delete *M; +} + /*}}}*/ + +} +} diff --git a/apt-pkg/cachefilter.h b/apt-pkg/cachefilter.h new file mode 100644 index 0000000..1b5f9aa --- /dev/null +++ b/apt-pkg/cachefilter.h @@ -0,0 +1,156 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/** \file cachefilter.h + Collection of functor classes */ + /*}}}*/ +#ifndef APT_CACHEFILTER_H +#define APT_CACHEFILTER_H +// Include Files /*{{{*/ +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/string_view.h> + +#include <memory> +#include <string> +#include <vector> + +#include <regex.h> + +class pkgCacheFile; + /*}}}*/ +namespace APT { +namespace CacheFilter { + +class APT_PUBLIC Matcher { +public: + virtual bool operator() (pkgCache::PkgIterator const &/*Pkg*/) = 0; + virtual bool operator() (pkgCache::GrpIterator const &/*Grp*/) = 0; + virtual bool operator() (pkgCache::VerIterator const &/*Ver*/) = 0; + virtual ~Matcher(); +}; + +class APT_PUBLIC PackageMatcher : public Matcher { +public: + bool operator() (pkgCache::PkgIterator const &Pkg) APT_OVERRIDE = 0; + bool operator() (pkgCache::VerIterator const &Ver) APT_OVERRIDE { return (*this)(Ver.ParentPkg()); } + bool operator() (pkgCache::GrpIterator const &/*Grp*/) APT_OVERRIDE { return false; } + ~PackageMatcher() APT_OVERRIDE; +}; + +// Generica like True, False, NOT, AND, OR /*{{{*/ +class APT_PUBLIC TrueMatcher : public Matcher { +public: + bool operator() (pkgCache::PkgIterator const &Pkg) APT_OVERRIDE; + bool operator() (pkgCache::GrpIterator const &Grp) APT_OVERRIDE; + bool operator() (pkgCache::VerIterator const &Ver) APT_OVERRIDE; +}; + +class APT_PUBLIC FalseMatcher : public Matcher { +public: + bool operator() (pkgCache::PkgIterator const &Pkg) APT_OVERRIDE; + bool operator() (pkgCache::GrpIterator const &Grp) APT_OVERRIDE; + bool operator() (pkgCache::VerIterator const &Ver) APT_OVERRIDE; +}; + +class APT_PUBLIC NOTMatcher : public Matcher { + Matcher * const matcher; +public: + explicit NOTMatcher(Matcher * const matcher); + bool operator() (pkgCache::PkgIterator const &Pkg) APT_OVERRIDE; + bool operator() (pkgCache::GrpIterator const &Grp) APT_OVERRIDE; + bool operator() (pkgCache::VerIterator const &Ver) APT_OVERRIDE; + ~NOTMatcher() APT_OVERRIDE; +}; + +class APT_PUBLIC ANDMatcher : public Matcher { + std::vector<Matcher *> matchers; +public: + // 5 ought to be enough for everybody… c++11 variadic templates would be nice + ANDMatcher(); + explicit ANDMatcher(Matcher * const matcher1); + ANDMatcher(Matcher * const matcher1, Matcher * const matcher2); + ANDMatcher(Matcher * const matcher1, Matcher * const matcher2, Matcher * const matcher3); + ANDMatcher(Matcher * const matcher1, Matcher * const matcher2, Matcher * const matcher3, Matcher * const matcher4); + ANDMatcher(Matcher * const matcher1, Matcher * const matcher2, Matcher * const matcher3, Matcher * const matcher4, Matcher * const matcher5); + ANDMatcher& AND(Matcher * const matcher); + bool operator() (pkgCache::PkgIterator const &Pkg) APT_OVERRIDE; + bool operator() (pkgCache::GrpIterator const &Grp) APT_OVERRIDE; + bool operator() (pkgCache::VerIterator const &Ver) APT_OVERRIDE; + ~ANDMatcher() APT_OVERRIDE; +}; +class APT_PUBLIC ORMatcher : public Matcher { + std::vector<Matcher *> matchers; +public: + // 5 ought to be enough for everybody… c++11 variadic templates would be nice + ORMatcher(); + explicit ORMatcher(Matcher * const matcher1); + ORMatcher(Matcher * const matcher1, Matcher * const matcher2); + ORMatcher(Matcher * const matcher1, Matcher * const matcher2, Matcher * const matcher3); + ORMatcher(Matcher * const matcher1, Matcher * const matcher2, Matcher * const matcher3, Matcher * const matcher4); + ORMatcher(Matcher * const matcher1, Matcher * const matcher2, Matcher * const matcher3, Matcher * const matcher4, Matcher * const matcher5); + ORMatcher& OR(Matcher * const matcher); + bool operator() (pkgCache::PkgIterator const &Pkg) APT_OVERRIDE; + bool operator() (pkgCache::GrpIterator const &Grp) APT_OVERRIDE; + bool operator() (pkgCache::VerIterator const &Ver) APT_OVERRIDE; + ~ORMatcher() APT_OVERRIDE; +}; + /*}}}*/ +class APT_PUBLIC PackageNameMatchesRegEx : public PackageMatcher { /*{{{*/ + regex_t* pattern; +public: + explicit PackageNameMatchesRegEx(std::string const &Pattern); + bool operator() (pkgCache::PkgIterator const &Pkg) APT_OVERRIDE; + bool operator() (pkgCache::GrpIterator const &Grp) APT_OVERRIDE; + ~PackageNameMatchesRegEx() APT_OVERRIDE; +}; + /*}}}*/ +class APT_PUBLIC PackageNameMatchesFnmatch : public PackageMatcher { /*{{{*/ + const std::string Pattern; +public: + explicit PackageNameMatchesFnmatch(std::string const &Pattern); + bool operator() (pkgCache::PkgIterator const &Pkg) APT_OVERRIDE; + bool operator() (pkgCache::GrpIterator const &Grp) APT_OVERRIDE; + ~PackageNameMatchesFnmatch() APT_OVERRIDE = default; +}; + /*}}}*/ +class APT_PUBLIC PackageArchitectureMatchesSpecification : public PackageMatcher { /*{{{*/ +/** \class PackageArchitectureMatchesSpecification + \brief matching against architecture specification strings + + The strings are of the format <libc>-<kernel>-<cpu> where either component, + or the whole string, can be the wildcard "any" as defined in + debian-policy §11.1 "Architecture specification strings". + + Examples: i386, mipsel, musl-linux-amd64, linux-any, any-amd64, any */ + std::string literal; + std::string complete; + bool isPattern; +public: + /** \brief matching against architecture specification strings + * + * @param pattern is the architecture specification string + * @param isPattern defines if the given \b pattern is a + * architecture specification pattern to match others against + * or if it is the fixed string and matched against patterns + */ + PackageArchitectureMatchesSpecification(std::string const &pattern, bool const isPattern = true); + bool operator() (char const * const &arch); + using PackageMatcher::operator(); + bool operator() (pkgCache::PkgIterator const &Pkg) APT_OVERRIDE; + ~PackageArchitectureMatchesSpecification() APT_OVERRIDE; +}; + /*}}}*/ +class APT_PUBLIC PackageIsNewInstall : public PackageMatcher { /*{{{*/ + pkgCacheFile * const Cache; +public: + explicit PackageIsNewInstall(pkgCacheFile * const Cache); + using PackageMatcher::operator(); + bool operator() (pkgCache::PkgIterator const &Pkg) APT_OVERRIDE; + ~PackageIsNewInstall() APT_OVERRIDE; +}; + /*}}}*/ + +/// \brief Parse a pattern, return nullptr or pattern +APT_PUBLIC std::unique_ptr<APT::CacheFilter::Matcher> ParsePattern(APT::StringView pattern, pkgCacheFile *file); +} +} +#endif diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h new file mode 100644 index 0000000..9273369 --- /dev/null +++ b/apt-pkg/cacheiterators.h @@ -0,0 +1,548 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Cache Iterators - Iterators for navigating the cache structure + + The iterators all provides ++,==,!=,->,* and end for their type. + The end function can be used to tell if the list has been fully + traversed. + + Unlike STL iterators these contain helper functions to access the data + that is being iterated over. This is because the data structures can't + be formed in a manner that is intuitive to use and also mmapable. + + For each variable in the target structure that would need a translation + to be accessed correctly a translating function of the same name is + present in the iterator. If applicable the translating function will + return an iterator. + + The DepIterator can iterate over two lists, a list of 'version depends' + or a list of 'package reverse depends'. The type is determined by the + structure passed to the constructor, which should be the structure + that has the depends pointer as a member. The provide iterator has the + same system. + + This header is not user includable, please use apt-pkg/pkgcache.h + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_CACHEITERATORS_H +#define PKGLIB_CACHEITERATORS_H +#ifndef __PKGLIB_IN_PKGCACHE_H +#warning apt-pkg/cacheiterators.h should not be included directly, include apt-pkg/pkgcache.h instead +#endif +#include <apt-pkg/macros.h> + +#include <iosfwd> +#include <iterator> +#include <string> +#include <apt-pkg/string_view.h> + +#include <string.h> + +// abstract Iterator template /*{{{*/ +/* This template provides the very basic iterator methods we + need to have for doing some walk-over-the-cache magic */ +template<typename Str, typename Itr> class APT_PUBLIC pkgCache::Iterator { + /** \brief Returns the Pointer for this struct in the owner + * The implementation of this method should be pretty short + * as it will only return the Pointer into the mmap stored + * in the owner but the name of this pointer is different for + * each structure and we want to abstract here at least for the + * basic methods from the actual structure. + * \return Pointer to the first structure of this type + */ + Str* OwnerPointer() const { return static_cast<Itr const*>(this)->OwnerPointer(); } + + protected: + Str *S; + pkgCache *Owner; + + public: + // iterator_traits + using iterator_category = std::forward_iterator_tag; + using value_type = Str; + using difference_type = std::ptrdiff_t; + using pointer = Str*; + using reference = Str&; + // Iteration + inline bool end() const {return Owner == 0 || S == OwnerPointer();} + + // Comparison + inline bool operator ==(const Itr &B) const {return S == B.S;} + inline bool operator !=(const Itr &B) const {return S != B.S;} + + // Accessors + inline Str *operator ->() {return S;} + inline Str const *operator ->() const {return S;} + inline operator Str *() {return S == OwnerPointer() ? 0 : S;} + inline operator Str const *() const {return S == OwnerPointer() ? 0 : S;} + inline Str &operator *() {return *S;} + inline Str const &operator *() const {return *S;} + inline pkgCache *Cache() const {return Owner;} + + // Mixed stuff + inline bool IsGood() const { return S && Owner && ! end();} + inline unsigned long Index() const {return S - OwnerPointer();} + inline map_pointer<Str> MapPointer() const {return map_pointer<Str>(Index()) ;} + + void ReMap(void const * const oldMap, void * const newMap) { + if (Owner == 0 || S == 0) + return; + S = static_cast<Str *>(newMap) + (S - static_cast<Str const *>(oldMap)); + } + + // Constructors - look out for the variable assigning + inline Iterator() : S(0), Owner(0) {} + inline Iterator(pkgCache &Owner,Str *T = 0) : S(T), Owner(&Owner) {} +}; + /*}}}*/ +// Group Iterator /*{{{*/ +/* Packages with the same name are collected in a Group so someone only + interest in package names can iterate easily over the names, so the + different architectures can be treated as of the "same" package + (apt internally treat them as totally different packages) */ +class APT_PUBLIC pkgCache::GrpIterator: public Iterator<Group, GrpIterator> { + long HashIndex; + + public: + inline Group* OwnerPointer() const { + return (Owner != 0) ? Owner->GrpP : 0; + } + + // This constructor is the 'begin' constructor, never use it. + explicit inline GrpIterator(pkgCache &Owner) : Iterator<Group, GrpIterator>(Owner), HashIndex(-1) { + S = OwnerPointer(); + operator++(); + } + + GrpIterator& operator++(); + inline GrpIterator operator++(int) { GrpIterator const tmp(*this); operator++(); return tmp; } + + inline const char *Name() const {return S->Name == 0?0:Owner->StrP + S->Name;} + inline PkgIterator PackageList() const; + inline VerIterator VersionsInSource() const; + PkgIterator FindPkg(APT::StringView Arch = APT::StringView("any", 3)) const; + /** \brief find the package with the "best" architecture + + The best architecture is either the "native" or the first + in the list of Architectures which is not an end-Pointer + + \param PreferNonVirtual tries to respond with a non-virtual package + and only if this fails returns the best virtual package */ + PkgIterator FindPreferredPkg(bool const &PreferNonVirtual = true) const; + PkgIterator NextPkg(PkgIterator const &Pkg) const; + + // Constructors + inline GrpIterator(pkgCache &Owner, Group *Trg) : Iterator<Group, GrpIterator>(Owner, Trg), HashIndex(0) { + if (S == 0) + S = OwnerPointer(); + } + inline GrpIterator() : Iterator<Group, GrpIterator>(), HashIndex(0) {} + +}; + /*}}}*/ +// Package Iterator /*{{{*/ +class APT_PUBLIC pkgCache::PkgIterator: public Iterator<Package, PkgIterator> { + long HashIndex; + + public: + inline Package* OwnerPointer() const { + return (Owner != 0) ? Owner->PkgP : 0; + } + + // This constructor is the 'begin' constructor, never use it. + explicit inline PkgIterator(pkgCache &Owner) : Iterator<Package, PkgIterator>(Owner), HashIndex(-1) { + S = OwnerPointer(); + operator++(); + } + + PkgIterator& operator++(); + inline PkgIterator operator++(int) { PkgIterator const tmp(*this); operator++(); return tmp; } + + enum OkState {NeedsNothing,NeedsUnpack,NeedsConfigure}; + + // Accessors + inline const char *Name() const { return Group().Name(); } + inline bool Purge() const {return S->CurrentState == pkgCache::State::Purge || + (S->CurrentVer == 0 && S->CurrentState == pkgCache::State::NotInstalled);} + inline const char *Arch() const {return S->Arch == 0?0:Owner->StrP + S->Arch;} + inline APT_PURE GrpIterator Group() const { return GrpIterator(*Owner, Owner->GrpP + S->Group);} + + inline VerIterator VersionList() const APT_PURE; + inline VerIterator CurrentVer() const APT_PURE; + inline DepIterator RevDependsList() const APT_PURE; + inline PrvIterator ProvidesList() const APT_PURE; + OkState State() const APT_PURE; + const char *CurVersion() const APT_PURE; + + //Nice printable representation + APT_DEPRECATED_MSG("Use APT::PrettyPkg instead") friend std::ostream& operator <<(std::ostream& out, PkgIterator i); + std::string FullName(bool const &Pretty = false) const; + + // Constructors + inline PkgIterator(pkgCache &Owner,Package *Trg) : Iterator<Package, PkgIterator>(Owner, Trg), HashIndex(0) { + if (S == 0) + S = OwnerPointer(); + } + inline PkgIterator() : Iterator<Package, PkgIterator>(), HashIndex(0) {} +}; + /*}}}*/ +// Version Iterator /*{{{*/ +class APT_PUBLIC pkgCache::VerIterator : public Iterator<Version, VerIterator> { + public: + inline Version* OwnerPointer() const { + return (Owner != 0) ? Owner->VerP : 0; + } + + // Iteration + inline VerIterator& operator++() {if (S != Owner->VerP) S = Owner->VerP + S->NextVer; return *this;} + inline VerIterator operator++(int) { VerIterator const tmp(*this); operator++(); return tmp; } + + inline VerIterator NextInSource() + { + if (S != Owner->VerP) + S = Owner->VerP + S->NextInSource; + return *this; + } + + // Comparison + int CompareVer(const VerIterator &B) const; + /** \brief compares two version and returns if they are similar + + This method should be used to identify if two pseudo versions are + referring to the same "real" version */ + inline bool SimilarVer(const VerIterator &B) const { + return (B.end() == false && S->Hash == B->Hash && strcmp(VerStr(), B.VerStr()) == 0); + } + + // Accessors + inline const char *VerStr() const {return S->VerStr == 0?0:Owner->StrP + S->VerStr;} + inline const char *Section() const {return S->Section == 0?0:Owner->StrP + S->Section;} + /** \brief source package name this version comes from + Always contains the name, even if it is the same as the binary name */ + inline const char *SourcePkgName() const {return Owner->StrP + S->SourcePkgName;} + /** \brief source version this version comes from + Always contains the version string, even if it is the same as the binary version */ + inline const char *SourceVerStr() const {return Owner->StrP + S->SourceVerStr;} + inline const char *Arch() const { + if ((S->MultiArch & pkgCache::Version::All) == pkgCache::Version::All) + return "all"; + return S->ParentPkg == 0?0:Owner->StrP + ParentPkg()->Arch; + } + inline PkgIterator ParentPkg() const {return PkgIterator(*Owner,Owner->PkgP + S->ParentPkg);} + + inline DescIterator DescriptionList() const; + DescIterator TranslatedDescriptionForLanguage(APT::StringView lang) const; + DescIterator TranslatedDescription() const; + inline DepIterator DependsList() const; + inline PrvIterator ProvidesList() const; + inline VerFileIterator FileList() const; + bool Downloadable() const; + inline const char *PriorityType() const {return Owner->Priority(S->Priority);} + const char *MultiArchType() const APT_PURE; + std::string RelStr() const; + + bool Automatic() const; + VerFileIterator NewestFile() const; + +#ifdef APT_COMPILING_APT + inline unsigned int PhasedUpdatePercentage() const + { + return (static_cast<Version::Extra *>(Owner->Map.Data()) + S->d)->PhasedUpdatePercentage; + } + inline bool PhasedUpdatePercentage(unsigned int percentage) + { + if (percentage > 100) + return false; + (static_cast<Version::Extra *>(Owner->Map.Data()) + S->d)->PhasedUpdatePercentage = static_cast<uint8_t>(percentage); + return true; + } +#endif + + inline VerIterator(pkgCache &Owner,Version *Trg = 0) : Iterator<Version, VerIterator>(Owner, Trg) { + if (S == 0) + S = OwnerPointer(); + } + inline VerIterator() : Iterator<Version, VerIterator>() {} +}; + /*}}}*/ +// Description Iterator /*{{{*/ +class APT_PUBLIC pkgCache::DescIterator : public Iterator<Description, DescIterator> { + public: + inline Description* OwnerPointer() const { + return (Owner != 0) ? Owner->DescP : 0; + } + + // Iteration + inline DescIterator& operator++() {if (S != Owner->DescP) S = Owner->DescP + S->NextDesc; return *this;} + inline DescIterator operator++(int) { DescIterator const tmp(*this); operator++(); return tmp; } + + // Comparison + int CompareDesc(const DescIterator &B) const; + + // Accessors + inline const char *LanguageCode() const {return Owner->StrP + S->language_code;} + inline const char *md5() const {return Owner->StrP + S->md5sum;} + inline DescFileIterator FileList() const; + + inline DescIterator() : Iterator<Description, DescIterator>() {} + inline DescIterator(pkgCache &Owner,Description *Trg = 0) : Iterator<Description, DescIterator>(Owner, Trg) { + if (S == 0) + S = Owner.DescP; + } +}; + /*}}}*/ +// Dependency iterator /*{{{*/ +class APT_PUBLIC pkgCache::DepIterator : public Iterator<Dependency, DepIterator> { + enum {DepVer, DepRev} Type; + DependencyData * S2; + + public: + inline Dependency* OwnerPointer() const { + return (Owner != 0) ? Owner->DepP : 0; + } + + // Iteration + DepIterator& operator++(); + inline DepIterator operator++(int) { DepIterator const tmp(*this); operator++(); return tmp; } + + // Accessors + inline const char *TargetVer() const {return S2->Version == 0?0:Owner->StrP + S2->Version;} + inline PkgIterator TargetPkg() const {return PkgIterator(*Owner,Owner->PkgP + S2->Package);} + inline PkgIterator SmartTargetPkg() const {PkgIterator R(*Owner,0);SmartTargetPkg(R);return R;} + inline VerIterator ParentVer() const {return VerIterator(*Owner,Owner->VerP + S->ParentVer);} + inline PkgIterator ParentPkg() const {return PkgIterator(*Owner,Owner->PkgP + Owner->VerP[uint32_t(S->ParentVer)].ParentPkg);} + inline bool Reverse() const {return Type == DepRev;} + bool IsCritical() const APT_PURE; + bool IsNegative() const APT_PURE; + bool IsIgnorable(PrvIterator const &Prv) const APT_PURE; + bool IsIgnorable(PkgIterator const &Pkg) const APT_PURE; + /* MultiArch can be translated to SingleArch for an resolver and we did so, + by adding dependencies to help the resolver understand the problem, but + sometimes it is needed to identify these to ignore them… */ + inline bool IsMultiArchImplicit() const APT_PURE { + return (S2->CompareOp & pkgCache::Dep::MultiArchImplicit) == pkgCache::Dep::MultiArchImplicit; + } + /* This covers additionally negative dependencies, which aren't arch-specific, + but change architecture nonetheless as a Conflicts: foo does applies for all archs */ + bool IsImplicit() const APT_PURE; + + bool IsSatisfied(VerIterator const &Ver) const APT_PURE; + bool IsSatisfied(PrvIterator const &Prv) const APT_PURE; + void GlobOr(DepIterator &Start,DepIterator &End); + Version **AllTargets() const; + bool SmartTargetPkg(PkgIterator &Result) const; + inline const char *CompType() const {return Owner->CompType(S2->CompareOp);} + inline const char *DepType() const {return Owner->DepType(S2->Type);} + + // overrides because we are special + struct DependencyProxy + { + map_stringitem_t &Version; + map_pointer<pkgCache::Package> &Package; + map_id_t &ID; + unsigned char &Type; + unsigned char &CompareOp; + map_pointer<pkgCache::Version> &ParentVer; + map_pointer<pkgCache::DependencyData> &DependencyData; + map_pointer<Dependency> &NextRevDepends; + map_pointer<Dependency> &NextDepends; + map_pointer<pkgCache::DependencyData> &NextData; + DependencyProxy const * operator->() const { return this; } + DependencyProxy * operator->() { return this; } + }; + inline DependencyProxy operator->() const {return (DependencyProxy) { S2->Version, S2->Package, S->ID, S2->Type, S2->CompareOp, S->ParentVer, S->DependencyData, S->NextRevDepends, S->NextDepends, S2->NextData };} + inline DependencyProxy operator->() {return (DependencyProxy) { S2->Version, S2->Package, S->ID, S2->Type, S2->CompareOp, S->ParentVer, S->DependencyData, S->NextRevDepends, S->NextDepends, S2->NextData };} + void ReMap(void const * const oldMap, void * const newMap) + { + Iterator<Dependency, DepIterator>::ReMap(oldMap, newMap); + if (Owner == 0 || S == 0 || S2 == 0) + return; + S2 = static_cast<DependencyData *>(newMap) + (S2 - static_cast<DependencyData const *>(oldMap)); + } + + //Nice printable representation + APT_DEPRECATED_MSG("Use APT::PrettyDep instead") friend std::ostream& operator <<(std::ostream& out, DepIterator D); + + inline DepIterator(pkgCache &Owner, Dependency *Trg, Version* = 0) : + Iterator<Dependency, DepIterator>(Owner, Trg), Type(DepVer), S2(Trg == 0 ? Owner.DepDataP : (Owner.DepDataP + Trg->DependencyData)) { + if (S == 0) + S = Owner.DepP; + } + inline DepIterator(pkgCache &Owner, Dependency *Trg, Package*) : + Iterator<Dependency, DepIterator>(Owner, Trg), Type(DepRev), S2(Trg == 0 ? Owner.DepDataP : (Owner.DepDataP + Trg->DependencyData)) { + if (S == 0) + S = Owner.DepP; + } + inline DepIterator() : Iterator<Dependency, DepIterator>(), Type(DepVer), S2(0) {} +}; + /*}}}*/ +// Provides iterator /*{{{*/ +class APT_PUBLIC pkgCache::PrvIterator : public Iterator<Provides, PrvIterator> { + enum {PrvVer, PrvPkg} Type; + + public: + inline Provides* OwnerPointer() const { + return (Owner != 0) ? Owner->ProvideP : 0; + } + + // Iteration + inline PrvIterator& operator ++() {if (S != Owner->ProvideP) S = Owner->ProvideP + + (Type == PrvVer?S->NextPkgProv:S->NextProvides); return *this;} + inline PrvIterator operator++(int) { PrvIterator const tmp(*this); operator++(); return tmp; } + + // Accessors + inline const char *Name() const {return ParentPkg().Name();} + inline const char *ProvideVersion() const {return S->ProvideVersion == 0?0:Owner->StrP + S->ProvideVersion;} + inline PkgIterator ParentPkg() const {return PkgIterator(*Owner,Owner->PkgP + S->ParentPkg);} + inline VerIterator OwnerVer() const {return VerIterator(*Owner,Owner->VerP + S->Version);} + inline PkgIterator OwnerPkg() const {return PkgIterator(*Owner,Owner->PkgP + Owner->VerP[uint32_t(S->Version)].ParentPkg);} + + /* MultiArch can be translated to SingleArch for an resolver and we did so, + by adding provides to help the resolver understand the problem, but + sometimes it is needed to identify these to ignore them… */ + bool IsMultiArchImplicit() const APT_PURE + { return (S->Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit; } + + + inline PrvIterator() : Iterator<Provides, PrvIterator>(), Type(PrvVer) {} + inline PrvIterator(pkgCache &Owner, Provides *Trg, Version*) : + Iterator<Provides, PrvIterator>(Owner, Trg), Type(PrvVer) { + if (S == 0) + S = Owner.ProvideP; + } + inline PrvIterator(pkgCache &Owner, Provides *Trg, Package*) : + Iterator<Provides, PrvIterator>(Owner, Trg), Type(PrvPkg) { + if (S == 0) + S = Owner.ProvideP; + } +}; + /*}}}*/ +// Release file /*{{{*/ +class APT_PUBLIC pkgCache::RlsFileIterator : public Iterator<ReleaseFile, RlsFileIterator> { + public: + inline ReleaseFile* OwnerPointer() const { + return (Owner != 0) ? Owner->RlsFileP : 0; + } + + // Iteration + inline RlsFileIterator& operator++() {if (S != Owner->RlsFileP) S = Owner->RlsFileP + S->NextFile;return *this;} + inline RlsFileIterator operator++(int) { RlsFileIterator const tmp(*this); operator++(); return tmp; } + + // Accessors + inline const char *FileName() const {return S->FileName == 0?0:Owner->StrP + S->FileName;} + inline const char *Archive() const {return S->Archive == 0?0:Owner->StrP + S->Archive;} + inline const char *Version() const {return S->Version == 0?0:Owner->StrP + S->Version;} + inline const char *Origin() const {return S->Origin == 0?0:Owner->StrP + S->Origin;} + inline const char *Codename() const {return S->Codename ==0?0:Owner->StrP + S->Codename;} + inline const char *Label() const {return S->Label == 0?0:Owner->StrP + S->Label;} + inline const char *Site() const {return S->Site == 0?0:Owner->StrP + S->Site;} + inline bool Flagged(pkgCache::Flag::ReleaseFileFlags const flag) const {return (S->Flags & flag) == flag; } + + std::string RelStr(); + + // Constructors + inline RlsFileIterator() : Iterator<ReleaseFile, RlsFileIterator>() {} + explicit inline RlsFileIterator(pkgCache &Owner) : Iterator<ReleaseFile, RlsFileIterator>(Owner, Owner.RlsFileP) {} + inline RlsFileIterator(pkgCache &Owner,ReleaseFile *Trg) : Iterator<ReleaseFile, RlsFileIterator>(Owner, Trg) {} +}; + /*}}}*/ +// Package file /*{{{*/ +class APT_PUBLIC pkgCache::PkgFileIterator : public Iterator<PackageFile, PkgFileIterator> { + public: + inline PackageFile* OwnerPointer() const { + return (Owner != 0) ? Owner->PkgFileP : 0; + } + + // Iteration + inline PkgFileIterator& operator++() {if (S != Owner->PkgFileP) S = Owner->PkgFileP + S->NextFile; return *this;} + inline PkgFileIterator operator++(int) { PkgFileIterator const tmp(*this); operator++(); return tmp; } + + // Accessors + inline const char *FileName() const {return S->FileName == 0?0:Owner->StrP + S->FileName;} + inline pkgCache::RlsFileIterator ReleaseFile() const {return RlsFileIterator(*Owner, Owner->RlsFileP + S->Release);} + inline const char *Archive() const {return S->Release == 0 ? Component() : ReleaseFile().Archive();} + inline const char *Version() const {return S->Release == 0 ? NULL : ReleaseFile().Version();} + inline const char *Origin() const {return S->Release == 0 ? NULL : ReleaseFile().Origin();} + inline const char *Codename() const {return S->Release == 0 ? NULL : ReleaseFile().Codename();} + inline const char *Label() const {return S->Release == 0 ? NULL : ReleaseFile().Label();} + inline const char *Site() const {return S->Release == 0 ? NULL : ReleaseFile().Site();} + inline bool Flagged(pkgCache::Flag::ReleaseFileFlags const flag) const {return S->Release== 0 ? false : ReleaseFile().Flagged(flag);} + inline bool Flagged(pkgCache::Flag::PkgFFlags const flag) const {return (S->Flags & flag) == flag;} + inline const char *Component() const {return S->Component == 0?0:Owner->StrP + S->Component;} + inline const char *Architecture() const {return S->Architecture == 0?0:Owner->StrP + S->Architecture;} + inline const char *IndexType() const {return S->IndexType == 0?0:Owner->StrP + S->IndexType;} + + std::string RelStr(); + + // Constructors + inline PkgFileIterator() : Iterator<PackageFile, PkgFileIterator>() {} + explicit inline PkgFileIterator(pkgCache &Owner) : Iterator<PackageFile, PkgFileIterator>(Owner, Owner.PkgFileP) {} + inline PkgFileIterator(pkgCache &Owner,PackageFile *Trg) : Iterator<PackageFile, PkgFileIterator>(Owner, Trg) {} +}; + /*}}}*/ +// Version File /*{{{*/ +class APT_PUBLIC pkgCache::VerFileIterator : public pkgCache::Iterator<VerFile, VerFileIterator> { + public: + inline VerFile* OwnerPointer() const { + return (Owner != 0) ? Owner->VerFileP : 0; + } + + // Iteration + inline VerFileIterator& operator++() {if (S != Owner->VerFileP) S = Owner->VerFileP + S->NextFile; return *this;} + inline VerFileIterator operator++(int) { VerFileIterator const tmp(*this); operator++(); return tmp; } + + // Accessors + inline PkgFileIterator File() const {return PkgFileIterator(*Owner, Owner->PkgFileP + S->File);} + + inline VerFileIterator() : Iterator<VerFile, VerFileIterator>() {} + inline VerFileIterator(pkgCache &Owner,VerFile *Trg) : Iterator<VerFile, VerFileIterator>(Owner, Trg) {} +}; + /*}}}*/ +// Description File /*{{{*/ +class APT_PUBLIC pkgCache::DescFileIterator : public Iterator<DescFile, DescFileIterator> { + public: + inline DescFile* OwnerPointer() const { + return (Owner != 0) ? Owner->DescFileP : 0; + } + + // Iteration + inline DescFileIterator& operator++() {if (S != Owner->DescFileP) S = Owner->DescFileP + S->NextFile; return *this;} + inline DescFileIterator operator++(int) { DescFileIterator const tmp(*this); operator++(); return tmp; } + + // Accessors + inline PkgFileIterator File() const {return PkgFileIterator(*Owner, Owner->PkgFileP + S->File);} + + inline DescFileIterator() : Iterator<DescFile, DescFileIterator>() {} + inline DescFileIterator(pkgCache &Owner,DescFile *Trg) : Iterator<DescFile, DescFileIterator>(Owner, Trg) {} +}; + /*}}}*/ +// Inlined Begin functions can't be in the class because of order problems /*{{{*/ +inline pkgCache::PkgIterator pkgCache::GrpIterator::PackageList() const + {return PkgIterator(*Owner,Owner->PkgP + S->FirstPackage);} + inline pkgCache::VerIterator pkgCache::GrpIterator::VersionsInSource() const + { + return VerIterator(*Owner, Owner->VerP + S->VersionsInSource); + } +inline pkgCache::VerIterator pkgCache::PkgIterator::VersionList() const + {return VerIterator(*Owner,Owner->VerP + S->VersionList);} +inline pkgCache::VerIterator pkgCache::PkgIterator::CurrentVer() const + {return VerIterator(*Owner,Owner->VerP + S->CurrentVer);} +inline pkgCache::DepIterator pkgCache::PkgIterator::RevDependsList() const + {return DepIterator(*Owner,Owner->DepP + S->RevDepends,S);} +inline pkgCache::PrvIterator pkgCache::PkgIterator::ProvidesList() const + {return PrvIterator(*Owner,Owner->ProvideP + S->ProvidesList,S);} +inline pkgCache::DescIterator pkgCache::VerIterator::DescriptionList() const + {return DescIterator(*Owner,Owner->DescP + S->DescriptionList);} +inline pkgCache::PrvIterator pkgCache::VerIterator::ProvidesList() const + {return PrvIterator(*Owner,Owner->ProvideP + S->ProvidesList,S);} +inline pkgCache::DepIterator pkgCache::VerIterator::DependsList() const + {return DepIterator(*Owner,Owner->DepP + S->DependsList,S);} +inline pkgCache::VerFileIterator pkgCache::VerIterator::FileList() const + {return VerFileIterator(*Owner,Owner->VerFileP + S->FileList);} +inline pkgCache::DescFileIterator pkgCache::DescIterator::FileList() const + {return DescFileIterator(*Owner,Owner->DescFileP + S->FileList);} + /*}}}*/ +#endif diff --git a/apt-pkg/cacheset.cc b/apt-pkg/cacheset.cc new file mode 100644 index 0000000..e52f762 --- /dev/null +++ b/apt-pkg/cacheset.cc @@ -0,0 +1,939 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Simple wrapper around a std::set to provide a similar interface to + a set of cache structures as to the complete set of all structures + in the pkgCache. Currently only Package is supported. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/cachefile.h> +#include <apt-pkg/cachefilter.h> +#include <apt-pkg/cacheset.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/depcache.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgrecords.h> +#include <apt-pkg/policy.h> +#include <apt-pkg/versionmatch.h> + +#include <list> +#include <string> +#include <vector> +#include <regex.h> +#include <stddef.h> +#include <stdio.h> +#include <string.h> + +#include <apti18n.h> + /*}}}*/ +namespace APT { +// PackageFrom - selecting the appropriate method for package selection /*{{{*/ +bool CacheSetHelper::PackageFrom(enum PkgSelector const select, PackageContainerInterface * const pci, + pkgCacheFile &Cache, std::string const &pattern) { + switch (select) { + case UNKNOWN: return false; + case REGEX: return PackageFromRegEx(pci, Cache, pattern); + case TASK: return PackageFromTask(pci, Cache, pattern); + case FNMATCH: return PackageFromFnmatch(pci, Cache, pattern); + case PACKAGENAME: return PackageFromPackageName(pci, Cache, pattern); + case STRING: return PackageFromString(pci, Cache, pattern); + case PATTERN: return PackageFromPattern(pci, Cache, pattern); + } + return false; +} + /*}}}*/ +// PackageFromTask - Return all packages in the cache from a specific task /*{{{*/ +bool CacheSetHelper::PackageFromTask(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern) { + size_t const archfound = pattern.find_last_of(':'); + std::string arch = "native"; + if (archfound != std::string::npos) { + arch = pattern.substr(archfound+1); + pattern.erase(archfound); + } + + if (pattern[pattern.length() -1] != '^') + return false; + pattern.erase(pattern.length()-1); + + if (unlikely(Cache.GetPkgCache() == 0 || Cache.GetDepCache() == 0)) + return false; + + bool const wasEmpty = pci->empty(); + if (wasEmpty == true) + pci->setConstructor(CacheSetHelper::TASK); + + // get the records + pkgRecords Recs(Cache); + + // build regexp for the task + regex_t Pattern; + char S[300]; + snprintf(S, sizeof(S), "^Task:.*[, ]%s([, ]|$)", pattern.c_str()); + if(regcomp(&Pattern,S, REG_EXTENDED | REG_NOSUB | REG_NEWLINE) != 0) { + _error->Error("Failed to compile task regexp"); + return false; + } + + bool found = false; + for (pkgCache::GrpIterator Grp = Cache->GrpBegin(); Grp.end() == false; ++Grp) { + pkgCache::PkgIterator Pkg = Grp.FindPkg(arch); + if (Pkg.end() == true) + continue; + pkgCache::VerIterator ver = Cache[Pkg].CandidateVerIter(Cache); + if(ver.end() == true) + continue; + + pkgRecords::Parser &parser = Recs.Lookup(ver.FileList()); + const char *start, *end; + parser.GetRec(start,end); + unsigned int const length = end - start; + if (unlikely(length == 0)) + continue; + char buf[length]; + strncpy(buf, start, length); + buf[length-1] = '\0'; + if (regexec(&Pattern, buf, 0, 0, 0) != 0) + continue; + + pci->insert(Pkg); + showPackageSelection(Pkg, CacheSetHelper::TASK, pattern); + found = true; + } + regfree(&Pattern); + + if (found == false) { + canNotFindPackage(CacheSetHelper::TASK, pci, Cache, pattern); + pci->setConstructor(CacheSetHelper::UNKNOWN); + return false; + } + + if (wasEmpty == false && pci->getConstructor() != CacheSetHelper::UNKNOWN) + pci->setConstructor(CacheSetHelper::UNKNOWN); + + return true; +} + /*}}}*/ +// PackageFromRegEx - Return all packages in the cache matching a pattern /*{{{*/ +bool CacheSetHelper::PackageFromRegEx(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern) { + static const char * const isregex = ".?+*|[^$"; + + if (_config->FindB("APT::Cmd::Pattern-Only", false)) + { + // Only allow explicit regexp pattern. + if (pattern.size() == 0 || (pattern[0] != '^' && pattern[pattern.size() - 1] != '$')) + return false; + } else { + if (pattern.find_first_of(isregex) == std::string::npos) + return false; + } + + bool const wasEmpty = pci->empty(); + if (wasEmpty == true) + pci->setConstructor(CacheSetHelper::REGEX); + + size_t archfound = pattern.find_last_of(':'); + std::string arch = "native"; + if (archfound != std::string::npos) { + arch = pattern.substr(archfound+1); + if (arch.find_first_of(isregex) == std::string::npos) + pattern.erase(archfound); + else + arch = "native"; + } + + if (unlikely(Cache.GetPkgCache() == 0)) + return false; + + APT::CacheFilter::PackageNameMatchesRegEx regexfilter(pattern); + + bool found = false; + for (pkgCache::GrpIterator Grp = Cache.GetPkgCache()->GrpBegin(); Grp.end() == false; ++Grp) { + if (regexfilter(Grp) == false) + continue; + pkgCache::PkgIterator Pkg = Grp.FindPkg(arch); + if (Pkg.end() == true) { + if (archfound == std::string::npos) + Pkg = Grp.FindPreferredPkg(true); + if (Pkg.end() == true) + continue; + } + + pci->insert(Pkg); + showPackageSelection(Pkg, CacheSetHelper::REGEX, pattern); + found = true; + } + + if (found == false) { + canNotFindPackage(CacheSetHelper::REGEX, pci, Cache, pattern); + pci->setConstructor(CacheSetHelper::UNKNOWN); + return false; + } + + if (wasEmpty == false && pci->getConstructor() != CacheSetHelper::UNKNOWN) + pci->setConstructor(CacheSetHelper::UNKNOWN); + + return true; +} + /*}}}*/ +// PackageFromFnmatch - Returns the package defined by this fnmatch /*{{{*/ +bool CacheSetHelper::PackageFromFnmatch(PackageContainerInterface * const pci, + pkgCacheFile &Cache, std::string pattern) +{ + static const char * const isfnmatch = ".?*[]!"; + // Whitelist approach: Anything not in here is not a valid pattern + static const char *const isfnmatch_strict = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-.:*"; + + if (_config->FindB("APT::Cmd::Pattern-Only", false) && pattern.find_first_not_of(isfnmatch_strict) != std::string::npos) + return false; + if (pattern.find_first_of(isfnmatch) == std::string::npos) + return false; + + bool const wasEmpty = pci->empty(); + if (wasEmpty == true) + pci->setConstructor(CacheSetHelper::FNMATCH); + + size_t archfound = pattern.find_last_of(':'); + std::string arch = "native"; + if (archfound != std::string::npos) { + arch = pattern.substr(archfound+1); + if (arch.find_first_of(isfnmatch) == std::string::npos) + pattern.erase(archfound); + else + arch = "native"; + } + + if (unlikely(Cache.GetPkgCache() == 0)) + return false; + + APT::CacheFilter::PackageNameMatchesFnmatch filter(pattern); + + bool found = false; + for (pkgCache::GrpIterator Grp = Cache.GetPkgCache()->GrpBegin(); Grp.end() == false; ++Grp) { + if (filter(Grp) == false) + continue; + pkgCache::PkgIterator Pkg = Grp.FindPkg(arch); + if (Pkg.end() == true) { + if (archfound == std::string::npos) + Pkg = Grp.FindPreferredPkg(true); + if (Pkg.end() == true) + continue; + } + + pci->insert(Pkg); + showPackageSelection(Pkg, CacheSetHelper::FNMATCH, pattern); + found = true; + } + + if (found == false) { + canNotFindPackage(CacheSetHelper::FNMATCH, pci, Cache, pattern); + pci->setConstructor(CacheSetHelper::UNKNOWN); + return false; + } + + if (wasEmpty == false && pci->getConstructor() != CacheSetHelper::UNKNOWN) + pci->setConstructor(CacheSetHelper::UNKNOWN); + + return true; +} + /*}}}*/ +// PackageFromPackageName - Returns the package defined by this string /*{{{*/ +bool CacheSetHelper::PackageFromPackageName(PackageContainerInterface * const pci, pkgCacheFile &Cache, + std::string pkg) { + if (unlikely(Cache.GetPkgCache() == 0)) + return false; + + std::string const pkgstring = pkg; + size_t const archfound = pkg.find_last_of(':'); + std::string arch; + if (archfound != std::string::npos) { + arch = pkg.substr(archfound+1); + pkg.erase(archfound); + if (arch == "all" || arch == "native") + arch = _config->Find("APT::Architecture"); + } + + pkgCache::GrpIterator Grp = Cache.GetPkgCache()->FindGrp(pkg); + if (Grp.end() == false) { + if (arch.empty() == true) { + pkgCache::PkgIterator Pkg = Grp.FindPreferredPkg(); + if (Pkg.end() == false) + { + pci->insert(Pkg); + return true; + } + } else { + bool found = false; + // for 'linux-any' return the first package matching, for 'linux-*' return all matches + bool const isGlobal = arch.find('*') != std::string::npos; + APT::CacheFilter::PackageArchitectureMatchesSpecification pams(arch); + for (pkgCache::PkgIterator Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg)) { + if (pams(Pkg) == false) + continue; + pci->insert(Pkg); + found = true; + if (isGlobal == false) + break; + } + if (found == true) + return true; + } + } + + pkgCache::PkgIterator Pkg = canNotFindPkgName(Cache, pkgstring); + if (Pkg.end() == true) + return false; + + pci->insert(Pkg); + return true; +} + /*}}}*/ +// PackageFromPattern - Return all packages matching a specific pattern /*{{{*/ +bool CacheSetHelper::PackageFromPattern(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string const &pattern) +{ + if (pattern.size() < 1 || (pattern[0] != '?' && pattern[0] != '~')) + return false; + + auto compiledPattern = APT::CacheFilter::ParsePattern(pattern, &Cache); + if (!compiledPattern) + return false; + + for (pkgCache::PkgIterator Pkg = Cache->PkgBegin(); Pkg.end() == false; ++Pkg) + { + if ((*compiledPattern)(Pkg) == false) + continue; + + pci->insert(Pkg); + } + return true; +} + /*}}}*/ +// PackageFromString - Return all packages matching a specific string /*{{{*/ +bool CacheSetHelper::PackageFromString(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &str) { + bool found = true; + _error->PushToStack(); + + if (PackageFrom(CacheSetHelper::PATTERN, pci, Cache, str) == false && + PackageFrom(CacheSetHelper::PACKAGENAME, pci, Cache, str) == false && + PackageFrom(CacheSetHelper::TASK, pci, Cache, str) == false && + // FIXME: hm, hm, regexp/fnmatch incompatible? + PackageFrom(CacheSetHelper::FNMATCH, pci, Cache, str) == false && + PackageFrom(CacheSetHelper::REGEX, pci, Cache, str) == false) + { + canNotFindPackage(CacheSetHelper::PACKAGENAME, pci, Cache, str); + found = false; + } + + if (found == true) + _error->RevertToStack(); + else + _error->MergeWithStack(); + return found; +} + /*}}}*/ +// PackageFromCommandLine - Return all packages specified on commandline /*{{{*/ +bool CacheSetHelper::PackageFromCommandLine(PackageContainerInterface * const pci, pkgCacheFile &Cache, const char **cmdline) { + bool found = false; + for (const char **I = cmdline; *I != 0; ++I) + found |= PackageFrom(CacheSetHelper::STRING, pci, Cache, *I); + return found; +} + /*}}}*/ +// FromModifierCommandLine - helper doing the work for PKG:GroupedFromCommandLine /*{{{*/ +bool CacheSetHelper::PackageFromModifierCommandLine(unsigned short &modID, PackageContainerInterface * const pci, + pkgCacheFile &Cache, const char * cmdline, + std::list<PkgModifier> const &mods) { + std::string str = cmdline; + unsigned short fallback = modID; + bool modifierPresent = false; + for (std::list<PkgModifier>::const_iterator mod = mods.begin(); + mod != mods.end(); ++mod) { + size_t const alength = strlen(mod->Alias); + switch(mod->Pos) { + case PkgModifier::POSTFIX: + if (str.compare(str.length() - alength, alength, + mod->Alias, 0, alength) != 0) + continue; + str.erase(str.length() - alength); + modID = mod->ID; + break; + case PkgModifier::PREFIX: + continue; + case PkgModifier::NONE: + continue; + } + modifierPresent = true; + break; + } + if (modifierPresent == true) { + bool const errors = showErrors(false); + bool const found = PackageFrom(PACKAGENAME, pci, Cache, cmdline); + showErrors(errors); + if (found == true) { + modID = fallback; + return true; + } + } + return PackageFrom(CacheSetHelper::PACKAGENAME, pci, Cache, str); +} + /*}}}*/ +// FromModifierCommandLine - helper doing the work for VER:GroupedFromCommandLine /*{{{*/ +bool VersionContainerInterface::FromModifierCommandLine(unsigned short &modID, + VersionContainerInterface * const vci, + pkgCacheFile &Cache, const char * cmdline, + std::list<Modifier> const &mods, + CacheSetHelper &helper) { + CacheSetHelper::VerSelector select = CacheSetHelper::NEWEST; + std::string str = cmdline; + if (unlikely(str.empty() == true)) + return false; + bool modifierPresent = false; + unsigned short fallback = modID; + for (std::list<Modifier>::const_iterator mod = mods.begin(); + mod != mods.end(); ++mod) { + if (modID == fallback && mod->ID == fallback) + select = mod->SelectVersion; + size_t const alength = strlen(mod->Alias); + switch(mod->Pos) { + case Modifier::POSTFIX: + if (str.length() <= alength || + str.compare(str.length() - alength, alength, mod->Alias, 0, alength) != 0) + continue; + str.erase(str.length() - alength); + modID = mod->ID; + select = mod->SelectVersion; + break; + case Modifier::PREFIX: + continue; + case Modifier::NONE: + continue; + } + modifierPresent = true; + break; + } + if (modifierPresent == true) { + bool const errors = helper.showErrors(false); + bool const found = VersionContainerInterface::FromString(vci, Cache, cmdline, select, helper, true); + helper.showErrors(errors); + if (found == true) { + modID = fallback; + return true; + } + } + return FromString(vci, Cache, str, select, helper); +} + /*}}}*/ +// FromCommandLine - Return all versions specified on commandline /*{{{*/ +bool VersionContainerInterface::FromCommandLine(VersionContainerInterface * const vci, + pkgCacheFile &Cache, const char **cmdline, + CacheSetHelper::VerSelector const fallback, + CacheSetHelper &helper) { + bool found = false; + for (const char **I = cmdline; *I != 0; ++I) + found |= VersionContainerInterface::FromString(vci, Cache, *I, fallback, helper); + return found; +} + /*}}}*/ +// FromString - Returns all versions spedcified by a string /*{{{*/ +bool VersionContainerInterface::FromString(VersionContainerInterface * const vci, + pkgCacheFile &Cache, std::string pkg, + CacheSetHelper::VerSelector const fallback, + CacheSetHelper &helper, + bool const onlyFromName) { + std::string ver; + bool verIsRel = false; + size_t const vertag = pkg.find_last_of("/="); + if (vertag != std::string::npos) { + ver = pkg.substr(vertag+1); + verIsRel = (pkg[vertag] == '/'); + pkg.erase(vertag); + } + + PackageSet pkgset; + if (onlyFromName == false) + helper.PackageFrom(CacheSetHelper::STRING, &pkgset, Cache, pkg); + else { + helper.PackageFrom(CacheSetHelper::PACKAGENAME, &pkgset, Cache, pkg); + } + + bool errors = true; + if (pkgset.getConstructor() != CacheSetHelper::UNKNOWN) + errors = helper.showErrors(false); + + bool found = false; + for (PackageSet::const_iterator P = pkgset.begin(); + P != pkgset.end(); ++P) { + if (vertag == std::string::npos) { + found |= VersionContainerInterface::FromPackage(vci, Cache, P, fallback, helper); + continue; + } + pkgCache::VerIterator V; + if (ver == "installed") + V = getInstalledVer(Cache, P, helper); + else if (ver == "candidate") + V = getCandidateVer(Cache, P, helper); + else if (ver == "newest") { + if (P->VersionList != 0) + V = P.VersionList(); + else + V = helper.canNotGetVersion(CacheSetHelper::NEWEST, Cache, P); + } else { + pkgVersionMatch Match(ver, (verIsRel == true ? pkgVersionMatch::Release : + pkgVersionMatch::Version)); + V = Match.Find(P); + helper.setLastVersionMatcher(ver); + if (V.end()) { + if (verIsRel == true) + V = helper.canNotGetVersion(CacheSetHelper::RELEASE, Cache, P); + else + V = helper.canNotGetVersion(CacheSetHelper::VERSIONNUMBER, Cache, P); + } + } + if (V.end() == true) + continue; + if (verIsRel == true) + helper.showVersionSelection(P, V, CacheSetHelper::RELEASE, ver); + else + helper.showVersionSelection(P, V, CacheSetHelper::VERSIONNUMBER, ver); + vci->insert(V); + found = true; + } + if (pkgset.getConstructor() != CacheSetHelper::UNKNOWN) + helper.showErrors(errors); + return found; +} + /*}}}*/ +// FromPackage - versions from package based on fallback /*{{{*/ +bool VersionContainerInterface::FromPackage(VersionContainerInterface * const vci, + pkgCacheFile &Cache, + pkgCache::PkgIterator const &P, + CacheSetHelper::VerSelector const fallback, + CacheSetHelper &helper) { + pkgCache::VerIterator V; + bool showErrors; + bool found = false; + switch(fallback) { + case CacheSetHelper::ALL: + if (P->VersionList != 0) + for (V = P.VersionList(); V.end() != true; ++V) + found |= vci->insert(V); + else + helper.canNotFindVersion(CacheSetHelper::ALL, vci, Cache, P); + break; + case CacheSetHelper::CANDANDINST: + found |= vci->insert(getInstalledVer(Cache, P, helper)); + found |= vci->insert(getCandidateVer(Cache, P, helper)); + break; + case CacheSetHelper::CANDIDATE: + found |= vci->insert(getCandidateVer(Cache, P, helper)); + break; + case CacheSetHelper::INSTALLED: + found |= vci->insert(getInstalledVer(Cache, P, helper)); + break; + case CacheSetHelper::CANDINST: + showErrors = helper.showErrors(false); + V = getCandidateVer(Cache, P, helper); + if (V.end() == true) + V = getInstalledVer(Cache, P, helper); + helper.showErrors(showErrors); + if (V.end() == false) + found |= vci->insert(V); + else + helper.canNotFindVersion(CacheSetHelper::CANDINST, vci, Cache, P); + break; + case CacheSetHelper::INSTCAND: + showErrors = helper.showErrors(false); + V = getInstalledVer(Cache, P, helper); + if (V.end() == true) + V = getCandidateVer(Cache, P, helper); + helper.showErrors(showErrors); + if (V.end() == false) + found |= vci->insert(V); + else + helper.canNotFindVersion(CacheSetHelper::INSTCAND, vci, Cache, P); + break; + case CacheSetHelper::NEWEST: + if (P->VersionList != 0) + found |= vci->insert(P.VersionList()); + else + helper.canNotFindVersion(CacheSetHelper::NEWEST, vci, Cache, P); + break; + case CacheSetHelper::RELEASE: + { + pkgVersionMatch Match(helper.getLastVersionMatcher(), pkgVersionMatch::Release); + V = Match.Find(P); + if (not V.end()) + found |= vci->insert(V); + else + helper.canNotFindVersion(CacheSetHelper::RELEASE, vci, Cache, P); + } + break; + case CacheSetHelper::VERSIONNUMBER: + { + pkgVersionMatch Match(helper.getLastVersionMatcher(), pkgVersionMatch::Version); + V = Match.Find(P); + if (not V.end()) + found |= vci->insert(V); + else + helper.canNotFindVersion(CacheSetHelper::VERSIONNUMBER, vci, Cache, P); + } + break; + } + return found; +} + /*}}}*/ +// FromDependency - versions satisfying a given dependency /*{{{*/ +bool VersionContainerInterface::FromDependency(VersionContainerInterface * const vci, + pkgCacheFile &Cache, + pkgCache::DepIterator const &D, + CacheSetHelper::VerSelector const selector, + CacheSetHelper &helper) +{ + bool found = false; + auto const insertVersion = [&](pkgCache::PkgIterator const &TP, pkgCache::VerIterator const &TV) { + if (not TV.end() && not D.IsIgnorable(TP) && D.IsSatisfied(TV)) + { + vci->insert(TV); + found = true; + } + }; + pkgCache::PkgIterator const T = D.TargetPkg(); + auto const insertAllTargetVersions = [&](auto const &getTargetVersion) { + insertVersion(T, getTargetVersion(T)); + for (auto Prv = T.ProvidesList(); not Prv.end(); ++Prv) + { + if (D.IsIgnorable(Prv)) + continue; + auto const OP = Prv.OwnerPkg(); + auto const TV = getTargetVersion(OP); + if (Prv.OwnerVer() == TV && D.IsSatisfied(Prv)) + { + vci->insert(TV); + found = true; + } + } + return found; + }; + switch(selector) { + case CacheSetHelper::ALL: + for (auto Ver = T.VersionList(); not Ver.end(); ++Ver) + { + insertVersion(T, Ver); + for (pkgCache::PrvIterator Prv = T.ProvidesList(); not Prv.end(); ++Prv) + if (not D.IsIgnorable(Prv)) + { + vci->insert(Prv.OwnerVer()); + found = true; + } + } + return found; + case CacheSetHelper::CANDANDINST: + found = FromDependency(vci, Cache, D, CacheSetHelper::CANDIDATE, helper); + found &= FromDependency(vci, Cache, D, CacheSetHelper::INSTALLED, helper); + return found; + case CacheSetHelper::CANDIDATE: + // skip looking if we have already cached that we will find nothing + if (((Cache[D] & pkgDepCache::DepCVer) == 0) != D.IsNegative()) + return found; + return insertAllTargetVersions([&](pkgCache::PkgIterator const &OP) { return Cache[OP].CandidateVerIter(Cache); }); + case CacheSetHelper::INSTALLED: + return insertAllTargetVersions([&](pkgCache::PkgIterator const &OP) { return OP.CurrentVer(); }); + case CacheSetHelper::CANDINST: + return FromDependency(vci, Cache, D, CacheSetHelper::CANDIDATE, helper) || + FromDependency(vci, Cache, D, CacheSetHelper::INSTALLED, helper); + case CacheSetHelper::INSTCAND: + return FromDependency(vci, Cache, D, CacheSetHelper::INSTALLED, helper) || + FromDependency(vci, Cache, D, CacheSetHelper::CANDIDATE, helper); + case CacheSetHelper::NEWEST: + return insertAllTargetVersions([&](pkgCache::PkgIterator const &OP) { return OP.VersionList(); }); + case CacheSetHelper::RELEASE: + case CacheSetHelper::VERSIONNUMBER: + // both make no sense here, so always false + return false; + } + return found; +} + /*}}}*/ +// getCandidateVer - Returns the candidate version of the given package /*{{{*/ +pkgCache::VerIterator VersionContainerInterface::getCandidateVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg, CacheSetHelper &helper) { + pkgCache::VerIterator Cand; + if (Cache.IsDepCacheBuilt() == true) { + Cand = Cache[Pkg].CandidateVerIter(Cache); + } else if (unlikely(Cache.GetPolicy() == nullptr)) { + return pkgCache::VerIterator(Cache); + } else { + Cand = Cache.GetPolicy()->GetCandidateVer(Pkg); + } + if (Cand.end() == true) + return helper.canNotGetVersion(CacheSetHelper::CANDIDATE, Cache, Pkg); + return Cand; +} + /*}}}*/ +// getInstalledVer - Returns the installed version of the given package /*{{{*/ +pkgCache::VerIterator VersionContainerInterface::getInstalledVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg, CacheSetHelper &helper) { + if (Pkg->CurrentVer == 0) + return helper.canNotGetVersion(CacheSetHelper::INSTALLED, Cache, Pkg); + return Pkg.CurrentVer(); +} + /*}}}*/ + +// canNotFindPackage - with the given selector and pattern /*{{{*/ +void CacheSetHelper::canNotFindPackage(enum PkgSelector const select, + PackageContainerInterface * const pci, pkgCacheFile &Cache, + std::string const &pattern) { + switch (select) { + case REGEX: canNotFindRegEx(pci, Cache, pattern); break; + case TASK: canNotFindTask(pci, Cache, pattern); break; + case FNMATCH: canNotFindFnmatch(pci, Cache, pattern); break; + case PACKAGENAME: canNotFindPackage(pci, Cache, pattern); break; + case STRING: canNotFindPackage(pci, Cache, pattern); break; + case PATTERN: canNotFindPackage(pci, Cache, pattern); break; + case UNKNOWN: break; + } +} +// canNotFindTask - handle the case no package is found for a task /*{{{*/ +void CacheSetHelper::canNotFindTask(PackageContainerInterface * const /*pci*/, pkgCacheFile &/*Cache*/, std::string pattern) { + if (ShowError == true) + _error->Insert(ErrorType, _("Couldn't find task '%s'"), pattern.c_str()); +} + /*}}}*/ +// canNotFindRegEx - handle the case no package is found by a regex /*{{{*/ +void CacheSetHelper::canNotFindRegEx(PackageContainerInterface * const /*pci*/, pkgCacheFile &/*Cache*/, std::string pattern) { + if (ShowError == true) + _error->Insert(ErrorType, _("Couldn't find any package by regex '%s'"), pattern.c_str()); +} + /*}}}*/ +// canNotFindFnmatch - handle the case no package is found by a fnmatch /*{{{*/ + void CacheSetHelper::canNotFindFnmatch(PackageContainerInterface * const /*pci*/, pkgCacheFile &/*Cache*/, std::string pattern) { + if (ShowError == true) + _error->Insert(ErrorType, _("Couldn't find any package by glob '%s'"), pattern.c_str()); +} + /*}}}*/ +// canNotFindPackage - handle the case no package is found from a string/*{{{*/ +void CacheSetHelper::canNotFindPackage(PackageContainerInterface * const /*pci*/, pkgCacheFile &/*Cache*/, std::string const &/*str*/) { +} + /*}}}*/ + /*}}}*/ +// canNotFindPkgName - handle the case no package has this name /*{{{*/ +pkgCache::PkgIterator CacheSetHelper::canNotFindPkgName(pkgCacheFile &Cache, + std::string const &str) { + if (ShowError == true) + _error->Insert(ErrorType, _("Unable to locate package %s"), str.c_str()); + return pkgCache::PkgIterator(Cache, 0); +} + /*}}}*/ +// canNotFindVersion - for package by selector /*{{{*/ +void CacheSetHelper::canNotFindVersion(enum VerSelector const select, VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) +{ + switch (select) { + case ALL: canNotFindAllVer(vci, Cache, Pkg); break; + case INSTCAND: canNotFindInstCandVer(vci, Cache, Pkg); break; + case CANDINST: canNotFindCandInstVer(vci, Cache, Pkg); break; + case NEWEST: canNotFindNewestVer(Cache, Pkg); break; + case CANDIDATE: canNotFindCandidateVer(Cache, Pkg); break; + case INSTALLED: canNotFindInstalledVer(Cache, Pkg); break; + case CANDANDINST: canNotGetCandInstVer(Cache, Pkg); break; + case RELEASE: canNotGetVerFromRelease(Cache, Pkg, getLastVersionMatcher()); break; + case VERSIONNUMBER: canNotGetVerFromVersionNumber(Cache, Pkg, getLastVersionMatcher()); break; + } +} +// canNotFindAllVer /*{{{*/ +void CacheSetHelper::canNotFindAllVer(VersionContainerInterface * const /*vci*/, pkgCacheFile &/*Cache*/, + pkgCache::PkgIterator const &Pkg) { + if (ShowError == true) + _error->Insert(ErrorType, _("Can't select versions from package '%s' as it is purely virtual"), Pkg.FullName(true).c_str()); +} + /*}}}*/ +// canNotFindInstCandVer /*{{{*/ +void CacheSetHelper::canNotFindInstCandVer(VersionContainerInterface * const /*vci*/, pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg) { + canNotGetInstCandVer(Cache, Pkg); +} + /*}}}*/ +// canNotFindInstCandVer /*{{{*/ +void CacheSetHelper::canNotFindCandInstVer(VersionContainerInterface * const /*vci*/, pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg) { + canNotGetCandInstVer(Cache, Pkg); +} + /*}}}*/ + /*}}}*/ +// canNotGetVersion - for package by selector /*{{{*/ +pkgCache::VerIterator CacheSetHelper::canNotGetVersion(enum VerSelector const select, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) { + switch (select) { + case NEWEST: return canNotFindNewestVer(Cache, Pkg); + case CANDIDATE: return canNotFindCandidateVer(Cache, Pkg); + case INSTALLED: return canNotFindInstalledVer(Cache, Pkg); + case CANDINST: return canNotGetCandInstVer(Cache, Pkg); + case INSTCAND: return canNotGetInstCandVer(Cache, Pkg); + case RELEASE: return canNotGetVerFromRelease(Cache, Pkg, getLastVersionMatcher()); + case VERSIONNUMBER: return canNotGetVerFromVersionNumber(Cache, Pkg, getLastVersionMatcher()); + case ALL: + case CANDANDINST: + // invalid in this branch + return pkgCache::VerIterator(Cache, 0); + } + return pkgCache::VerIterator(Cache, 0); +} +// canNotFindNewestVer /*{{{*/ +pkgCache::VerIterator CacheSetHelper::canNotFindNewestVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg) { + if (ShowError == true) + _error->Insert(ErrorType, _("Can't select newest version from package '%s' as it is purely virtual"), Pkg.FullName(true).c_str()); + return pkgCache::VerIterator(Cache, 0); +} + /*}}}*/ +// canNotFindCandidateVer /*{{{*/ +pkgCache::VerIterator CacheSetHelper::canNotFindCandidateVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg) { + if (ShowError == true) + _error->Insert(ErrorType, _("Can't select candidate version from package %s as it has no candidate"), Pkg.FullName(true).c_str()); + return pkgCache::VerIterator(Cache, 0); +} + /*}}}*/ +// canNotFindInstalledVer /*{{{*/ +pkgCache::VerIterator CacheSetHelper::canNotFindInstalledVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg) { + if (ShowError == true) + _error->Insert(ErrorType, _("Can't select installed version from package %s as it is not installed"), Pkg.FullName(true).c_str()); + return pkgCache::VerIterator(Cache, 0); +} + /*}}}*/ +// canNotFindInstCandVer /*{{{*/ +pkgCache::VerIterator CacheSetHelper::canNotGetInstCandVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg) { + if (ShowError == true) + _error->Insert(ErrorType, _("Can't select installed nor candidate version from package '%s' as it has neither of them"), Pkg.FullName(true).c_str()); + return pkgCache::VerIterator(Cache, 0); +} + /*}}}*/ +// canNotFindInstCandVer /*{{{*/ +pkgCache::VerIterator CacheSetHelper::canNotGetCandInstVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg) { + if (ShowError == true) + _error->Insert(ErrorType, _("Can't select installed nor candidate version from package '%s' as it has neither of them"), Pkg.FullName(true).c_str()); + return pkgCache::VerIterator(Cache, 0); +} + /*}}}*/ +// canNotFindMatchingVer /*{{{*/ +pkgCache::VerIterator CacheSetHelper::canNotGetVerFromRelease(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg, std::string const &release) { + if (ShowError == true) + _error->Insert(ErrorType, _("Release '%s' for '%s' was not found"), release.c_str(), Pkg.FullName(true).c_str()); + return pkgCache::VerIterator(Cache, 0); +} +pkgCache::VerIterator CacheSetHelper::canNotGetVerFromVersionNumber(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg, std::string const &verstr) { + if (ShowError == true) + _error->Insert(ErrorType, _("Version '%s' for '%s' was not found"), verstr.c_str(), Pkg.FullName(true).c_str()); + return pkgCache::VerIterator(Cache, 0); +} + /*}}}*/ + /*}}}*/ +// showPackageSelection - by selector and given pattern /*{{{*/ +void CacheSetHelper::showPackageSelection(pkgCache::PkgIterator const &pkg, enum PkgSelector const select, + std::string const &pattern) { + switch (select) { + case REGEX: showRegExSelection(pkg, pattern); break; + case TASK: showTaskSelection(pkg, pattern); break; + case FNMATCH: showFnmatchSelection(pkg, pattern); break; + case PATTERN: showPatternSelection(pkg, pattern); break; + case PACKAGENAME: /* no surprises here */ break; + case STRING: /* handled by the special cases */ break; + case UNKNOWN: break; + } +} +// showTaskSelection /*{{{*/ +void CacheSetHelper::showTaskSelection(pkgCache::PkgIterator const &/*pkg*/, + std::string const &/*pattern*/) { +} + /*}}}*/ +// showRegExSelection /*{{{*/ +void CacheSetHelper::showRegExSelection(pkgCache::PkgIterator const &/*pkg*/, + std::string const &/*pattern*/) { +} + /*}}}*/ +// showFnmatchSelection /*{{{*/ +void CacheSetHelper::showFnmatchSelection(pkgCache::PkgIterator const &/*pkg*/, + std::string const &/*pattern*/) { +} + /*}}}*/ +// showPatternSelection /*{{{*/ +void CacheSetHelper::showPatternSelection(pkgCache::PkgIterator const & /*pkg*/, + std::string const & /*pattern*/) +{ +} + /*}}}*/ + /*}}}*/ +// showVersionSelection /*{{{*/ +void CacheSetHelper::showVersionSelection(pkgCache::PkgIterator const &Pkg, + pkgCache::VerIterator const &Ver, enum VerSelector const select, std::string const &pattern) { + switch (select) { + case RELEASE: + showSelectedVersion(Pkg, Ver, pattern, true); + break; + case VERSIONNUMBER: + showSelectedVersion(Pkg, Ver, pattern, false); + break; + case NEWEST: + case CANDIDATE: + case INSTALLED: + case CANDINST: + case INSTCAND: + case ALL: + case CANDANDINST: + // not really surprises, but in fact: just not implemented + break; + } +} +void CacheSetHelper::showSelectedVersion(pkgCache::PkgIterator const &/*Pkg*/, + pkgCache::VerIterator const /*Ver*/, + std::string const &/*ver*/, + bool const /*verIsRel*/) { +} + /*}}}*/ + +class CacheSetHelper::Private { +public: + std::string version_or_release; +}; +std::string CacheSetHelper::getLastVersionMatcher() const { return d->version_or_release; } +void CacheSetHelper::setLastVersionMatcher(std::string const &matcher) { d->version_or_release = matcher; } + +CacheSetHelper::CacheSetHelper(bool const ShowError, GlobalError::MsgType ErrorType) : + ShowError(ShowError), ErrorType(ErrorType), d(new Private{}) {} +CacheSetHelper::~CacheSetHelper() { delete d; } + +PackageContainerInterface::PackageContainerInterface() : ConstructedBy(CacheSetHelper::UNKNOWN), d(NULL) {} +PackageContainerInterface::PackageContainerInterface(PackageContainerInterface const &by) : PackageContainerInterface() { *this = by; } +PackageContainerInterface::PackageContainerInterface(CacheSetHelper::PkgSelector const by) : ConstructedBy(by), d(NULL) {} +PackageContainerInterface& PackageContainerInterface::operator=(PackageContainerInterface const &other) { + if (this != &other) + this->ConstructedBy = other.ConstructedBy; + return *this; +} +PackageContainerInterface::~PackageContainerInterface() {} + +PackageUniverse::PackageUniverse(pkgCache * const Owner) : _cont(Owner), d(NULL) {} +PackageUniverse::PackageUniverse(pkgCacheFile * const Owner) : _cont(Owner->GetPkgCache()), d(NULL) {} +PackageUniverse::~PackageUniverse() {} + +VersionContainerInterface::VersionContainerInterface() : d(NULL) {} +VersionContainerInterface::VersionContainerInterface(VersionContainerInterface const &other) : VersionContainerInterface() { + *this = other; +}; +VersionContainerInterface& VersionContainerInterface::operator=(VersionContainerInterface const &) { + return *this; +} + +VersionContainerInterface::~VersionContainerInterface() {} +} diff --git a/apt-pkg/cacheset.h b/apt-pkg/cacheset.h new file mode 100644 index 0000000..4ea78bc --- /dev/null +++ b/apt-pkg/cacheset.h @@ -0,0 +1,1079 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/** \file cacheset.h + Wrappers around std::set to have set::iterators which behave + similar to the Iterators of the cache structures. + + Provides also a few helper methods which work with these sets */ + /*}}}*/ +#ifndef APT_CACHESET_H +#define APT_CACHESET_H +// Include Files /*{{{*/ +#include <fstream> +#include <map> +#include <set> +#if __cplusplus >= 201103L +#include <forward_list> +#include <initializer_list> +#include <unordered_set> +#endif +#include <algorithm> +#include <deque> +#include <iterator> +#include <list> +#include <string> +#include <vector> + +#include <stddef.h> + +#include <apt-pkg/error.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> + + /*}}}*/ + +class pkgCacheFile; + +namespace APT { +class PackageContainerInterface; +class VersionContainerInterface; + +class APT_PUBLIC CacheSetHelper { /*{{{*/ +/** \class APT::CacheSetHelper + Simple base class with a lot of virtual methods which can be overridden + to alter the behavior or the output of the CacheSets. + + This helper is passed around by the static methods in the CacheSets and + used every time they hit an error condition or something could be + printed out. +*/ +public: /*{{{*/ + CacheSetHelper(bool const ShowError = true, + GlobalError::MsgType ErrorType = GlobalError::ERROR); + virtual ~CacheSetHelper(); + + enum PkgSelector { UNKNOWN, REGEX, TASK, FNMATCH, PACKAGENAME, STRING, PATTERN }; + + virtual bool PackageFrom(enum PkgSelector const select, PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &pattern); + + virtual bool PackageFromCommandLine(PackageContainerInterface * const pci, pkgCacheFile &Cache, const char **cmdline); + + struct PkgModifier { + enum Position { NONE, PREFIX, POSTFIX }; + unsigned short ID; + const char * const Alias; + Position Pos; + PkgModifier (unsigned short const &id, const char * const alias, Position const &pos) : ID(id), Alias(alias), Pos(pos) {} + }; + virtual bool PackageFromModifierCommandLine(unsigned short &modID, PackageContainerInterface * const pci, + pkgCacheFile &Cache, const char * cmdline, + std::list<PkgModifier> const &mods); + + + /** \brief be notified about the package being selected via pattern + * + * Main use is probably to show a message to the user what happened + * + * \param pkg is the package which was selected + * \param select is the selection method which choose the package + * \param pattern is the string used by the selection method to pick the package + */ + virtual void showPackageSelection(pkgCache::PkgIterator const &pkg, PkgSelector const select, std::string const &pattern); + + /** \brief be notified if a package can't be found via pattern + * + * Can be used to show a message as well as to try something else to make it match + * + * \param select is the method tried for selection + * \param pci is the container the package should be inserted in + * \param Cache is the package universe available + * \param pattern is the string not matching anything + */ + virtual void canNotFindPackage(enum PkgSelector const select, PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &pattern); + + /** \brief specifies which version(s) we want to refer to */ + enum VerSelector { + /** by release string */ + RELEASE, + /** by version number string */ + VERSIONNUMBER, + /** All versions */ + ALL, + /** Candidate and installed version */ + CANDANDINST, + /** Candidate version */ + CANDIDATE, + /** Installed version */ + INSTALLED, + /** Candidate or if non installed version */ + CANDINST, + /** Installed or if non candidate version */ + INSTCAND, + /** Newest version */ + NEWEST + }; + + /** \brief be notified about the version being selected via pattern + * + * Main use is probably to show a message to the user what happened + * Note that at the moment this method is only called for RELEASE + * and VERSION selections, not for the others. + * + * \param Pkg is the package which was selected for + * \param Ver is the version selected + * \param select is the selection method which choose the version + * \param pattern is the string used by the selection method to pick the version + */ + virtual void showVersionSelection(pkgCache::PkgIterator const &Pkg, pkgCache::VerIterator const &Ver, + enum VerSelector const select, std::string const &pattern); + + /** \brief be notified if a version can't be found for a package + * + * Main use is probably to show a message to the user what happened + * + * \param select is the method tried for selection + * \param vci is the container the version should be inserted in + * \param Cache is the package universe available + * \param Pkg is the package we wanted a version from + */ + virtual void canNotFindVersion(enum VerSelector const select, VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); + + // the difference between canNotFind and canNotGet is that the later is more low-level + // and called from other places: In this case looking into the code is the only real answer… + virtual pkgCache::VerIterator canNotGetVersion(enum VerSelector const select, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); + + virtual pkgCache::PkgIterator canNotFindPkgName(pkgCacheFile &Cache, std::string const &str); + + bool showErrors() const { return ShowError; } + bool showErrors(bool const newValue) { if (ShowError == newValue) return ShowError; else return ((ShowError = newValue) == false); } + GlobalError::MsgType errorType() const { return ErrorType; } + GlobalError::MsgType errorType(GlobalError::MsgType const &newValue) + { + if (ErrorType == newValue) return ErrorType; + else { + GlobalError::MsgType const &oldValue = ErrorType; + ErrorType = newValue; + return oldValue; + } + } + + std::string getLastVersionMatcher() const; + void setLastVersionMatcher(std::string const &matcher); + /*}}}*/ +protected: + bool ShowError; + GlobalError::MsgType ErrorType; + + pkgCache::VerIterator canNotGetInstCandVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg); + pkgCache::VerIterator canNotGetCandInstVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg); + + pkgCache::VerIterator canNotGetVerFromRelease(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg, std::string const &release); + pkgCache::VerIterator canNotGetVerFromVersionNumber(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg, std::string const &verstr); + + bool PackageFromTask(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern); + bool PackageFromRegEx(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern); + bool PackageFromFnmatch(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern); + bool PackageFromPackageName(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern); + bool PackageFromString(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &pattern); + bool PackageFromPattern(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &pattern); +private: + void showTaskSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); + void showRegExSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); + void showFnmatchSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); + void showPatternSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); + void canNotFindTask(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string pattern); + void canNotFindRegEx(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string pattern); + void canNotFindFnmatch(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string pattern); + void canNotFindPackage(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string const &str); + void showSelectedVersion(pkgCache::PkgIterator const &Pkg, pkgCache::VerIterator const Ver, + std::string const &ver, bool const verIsRel); + void canNotFindAllVer(VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); + void canNotFindInstCandVer(VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); + void canNotFindCandInstVer(VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); + pkgCache::VerIterator canNotFindNewestVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); + pkgCache::VerIterator canNotFindCandidateVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); + pkgCache::VerIterator canNotFindInstalledVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); + + class Private; + Private * const d; +}; /*}}}*/ +// Iterator templates for our Containers /*{{{*/ +template<typename Interface, typename Master, typename iterator_type, typename container_iterator, typename container_value> class Container_iterator_base : + public Interface::template iterator_base<iterator_type> +{ +protected: + container_iterator _iter; +public: + using iterator_category = typename std::iterator_traits<container_iterator>::iterator_category; + using value_type = container_value; + using difference_type = std::ptrdiff_t; + using pointer = container_value*; + using reference = container_value&; + explicit Container_iterator_base(container_iterator const &i) : _iter(i) {} + inline container_value operator*(void) const { return static_cast<iterator_type const*>(this)->getType(); }; + operator container_iterator(void) const { return _iter; } + inline iterator_type& operator++() { ++_iter; return static_cast<iterator_type&>(*this); } + inline iterator_type operator++(int) { iterator_type tmp(*this); operator++(); return tmp; } + inline iterator_type operator+(typename container_iterator::difference_type const &n) { return iterator_type(_iter + n); } + inline iterator_type operator+=(typename container_iterator::difference_type const &n) { _iter += n; return static_cast<iterator_type&>(*this); } + inline iterator_type& operator--() { --_iter;; return static_cast<iterator_type&>(*this); } + inline iterator_type operator--(int) { iterator_type tmp(*this); operator--(); return tmp; } + inline iterator_type operator-(typename container_iterator::difference_type const &n) { return iterator_type(_iter - n); } + inline typename container_iterator::difference_type operator-(iterator_type const &b) { return (_iter - b._iter); } + inline iterator_type operator-=(typename container_iterator::difference_type const &n) { _iter -= n; return static_cast<iterator_type&>(*this); } + inline bool operator!=(iterator_type const &i) const { return _iter != i._iter; } + inline bool operator==(iterator_type const &i) const { return _iter == i._iter; } + inline bool operator<(iterator_type const &i) const { return _iter < i._iter; } + inline bool operator>(iterator_type const &i) const { return _iter > i._iter; } + inline bool operator<=(iterator_type const &i) const { return _iter <= i._iter; } + inline bool operator>=(iterator_type const &i) const { return _iter >= i._iter; } + inline typename container_iterator::reference operator[](typename container_iterator::difference_type const &n) const { return _iter[n]; } + + friend std::ostream& operator<<(std::ostream& out, iterator_type i) { return operator<<(out, *i); } + friend Master; +}; +template<class Interface, class Container, class Master> class Container_const_iterator : + public Container_iterator_base<Interface, Master, Container_const_iterator<Interface, Container, Master>, typename Container::const_iterator, typename Container::value_type> +{ + typedef Container_const_iterator<Interface, Container, Master> iterator_type; + typedef typename Container::const_iterator container_iterator; +public: + explicit Container_const_iterator(container_iterator i) : + Container_iterator_base<Interface, Master, iterator_type, container_iterator, typename Container::value_type>(i) {} + + inline typename Container::value_type getType(void) const { return *this->_iter; } +}; +template<class Interface, class Container, class Master> class Container_iterator : + public Container_iterator_base<Interface, Master, Container_iterator<Interface, Container, Master>, typename Container::iterator, typename Container::value_type> +{ + typedef Container_iterator<Interface, Container, Master> iterator_type; + typedef typename Container::iterator container_iterator; +public: + explicit Container_iterator(container_iterator const &i) : + Container_iterator_base<Interface, Master, iterator_type, container_iterator, typename Container::value_type>(i) {} + + operator typename Master::const_iterator() { return typename Master::const_iterator(this->_iter); } + inline typename Container::iterator::reference operator*(void) const { return *this->_iter; } + + inline typename Container::value_type getType(void) const { return *this->_iter; } +}; +template<class Interface, class Container, class Master> class Container_const_reverse_iterator : + public Container_iterator_base<Interface, Master, Container_const_reverse_iterator<Interface, Container, Master>, typename Container::const_reverse_iterator, typename Container::value_type> +{ + typedef Container_const_reverse_iterator<Interface, Container, Master> iterator_type; + typedef typename Container::const_reverse_iterator container_iterator; +public: + explicit Container_const_reverse_iterator(container_iterator i) : + Container_iterator_base<Interface, Master, iterator_type, container_iterator, typename Container::value_type>(i) {} + + inline typename Container::value_type getType(void) const { return *this->_iter; } +}; +template<class Interface, class Container, class Master> class Container_reverse_iterator : + public Container_iterator_base<Interface, Master, Container_reverse_iterator<Interface, Container, Master>, typename Container::reverse_iterator, typename Container::value_type> +{ + typedef Container_reverse_iterator<Interface, Container, Master> iterator_type; + typedef typename Container::reverse_iterator container_iterator; +public: + explicit Container_reverse_iterator(container_iterator i) : + Container_iterator_base<Interface, Master, iterator_type, container_iterator, typename Container::value_type>(i) {} + + operator typename Master::const_iterator() { return typename Master::const_iterator(this->_iter); } + inline iterator_type& operator=(iterator_type const &i) { this->_iter = i._iter; return static_cast<iterator_type&>(*this); } + inline iterator_type& operator=(container_iterator const &i) { this->_iter = i; return static_cast<iterator_type&>(*this); } + inline typename Container::reverse_iterator::reference operator*(void) const { return *this->_iter; } + + inline typename Container::value_type getType(void) const { return *this->_iter; } +}; + /*}}}*/ +class APT_PUBLIC PackageContainerInterface { /*{{{*/ +/** \class PackageContainerInterface + + * Interface ensuring that all operations can be executed on the yet to + * define concrete PackageContainer - access to all methods is possible, + * but in general the wrappers provided by the PackageContainer template + * are nicer to use. + + * This class mostly protects use from the need to write all implementation + * of the methods working on containers in the template */ +public: + template<class Itr> class iterator_base { /*{{{*/ + pkgCache::PkgIterator getType() const { return static_cast<Itr const*>(this)->getType(); }; + public: + operator pkgCache::PkgIterator(void) const { return getType(); } + + inline const char *Name() const {return getType().Name(); } + inline std::string FullName(bool const Pretty) const { return getType().FullName(Pretty); } + inline std::string FullName() const { return getType().FullName(); } + inline bool Purge() const {return getType().Purge(); } + inline const char *Arch() const {return getType().Arch(); } + inline pkgCache::GrpIterator Group() const { return getType().Group(); } + inline pkgCache::VerIterator VersionList() const { return getType().VersionList(); } + inline pkgCache::VerIterator CurrentVer() const { return getType().CurrentVer(); } + inline pkgCache::DepIterator RevDependsList() const { return getType().RevDependsList(); } + inline pkgCache::PrvIterator ProvidesList() const { return getType().ProvidesList(); } + inline pkgCache::PkgIterator::OkState State() const { return getType().State(); } + inline const char *CurVersion() const { return getType().CurVersion(); } + inline pkgCache *Cache() const { return getType().Cache(); } + inline unsigned long Index() const {return getType().Index();} + // we have only valid iterators here + inline bool end() const { return false; } + + inline pkgCache::Package const * operator->() const {return &*getType();} + }; + /*}}}*/ + + virtual bool insert(pkgCache::PkgIterator const &P) = 0; + virtual bool empty() const = 0; + virtual void clear() = 0; + virtual size_t size() const = 0; + + void setConstructor(CacheSetHelper::PkgSelector const by) { ConstructedBy = by; } + CacheSetHelper::PkgSelector getConstructor() const { return ConstructedBy; } + PackageContainerInterface(); + explicit PackageContainerInterface(CacheSetHelper::PkgSelector const by); + PackageContainerInterface(PackageContainerInterface const &by); + PackageContainerInterface& operator=(PackageContainerInterface const &other); + virtual ~PackageContainerInterface(); + +private: + CacheSetHelper::PkgSelector ConstructedBy; + void * const d; +}; + /*}}}*/ +template<class Container> class APT_PUBLIC PackageContainer : public PackageContainerInterface {/*{{{*/ +/** \class APT::PackageContainer + + Simple wrapper around a container class like std::set to provide a similar + interface to a set of packages as to the complete set of all packages in the + pkgCache. */ + Container _cont; +public: /*{{{*/ + /** \brief smell like a pkgCache::PkgIterator */ + typedef Container_const_iterator<PackageContainerInterface, Container, PackageContainer> const_iterator; + typedef Container_iterator<PackageContainerInterface, Container, PackageContainer> iterator; + typedef Container_const_reverse_iterator<PackageContainerInterface, Container, PackageContainer> const_reverse_iterator; + typedef Container_reverse_iterator<PackageContainerInterface, Container, PackageContainer> reverse_iterator; + typedef typename Container::value_type value_type; + typedef typename Container::pointer pointer; + typedef typename Container::const_pointer const_pointer; + typedef typename Container::reference reference; + typedef typename Container::const_reference const_reference; + typedef typename Container::difference_type difference_type; + typedef typename Container::size_type size_type; + typedef typename Container::allocator_type allocator_type; + + bool insert(pkgCache::PkgIterator const &P) APT_OVERRIDE { if (P.end() == true) return false; _cont.insert(P); return true; } + template<class Cont> void insert(PackageContainer<Cont> const &pkgcont) { _cont.insert((typename Cont::const_iterator)pkgcont.begin(), (typename Cont::const_iterator)pkgcont.end()); } + void insert(const_iterator begin, const_iterator end) { _cont.insert(begin, end); } + + bool empty() const APT_OVERRIDE { return _cont.empty(); } + void clear() APT_OVERRIDE { return _cont.clear(); } + size_t size() const APT_OVERRIDE { return _cont.size(); } +#if __GNUC__ >= 5 || (__GNUC_MINOR__ >= 9 && __GNUC__ >= 4) + iterator erase( const_iterator pos ) { return iterator(_cont.erase(pos._iter)); } + iterator erase( const_iterator first, const_iterator last ) { return iterator(_cont.erase(first._iter, last._iter)); } +#else + iterator erase( iterator pos ) { return iterator(_cont.erase(pos._iter)); } + iterator erase( iterator first, iterator last ) { return iterator(_cont.erase(first._iter, last._iter)); } +#endif + const_iterator begin() const { return const_iterator(_cont.begin()); } + const_iterator end() const { return const_iterator(_cont.end()); } + const_reverse_iterator rbegin() const { return const_reverse_iterator(_cont.rbegin()); } + const_reverse_iterator rend() const { return const_reverse_iterator(_cont.rend()); } +#if __cplusplus >= 201103L + const_iterator cbegin() const { return const_iterator(_cont.cbegin()); } + const_iterator cend() const { return const_iterator(_cont.cend()); } + const_reverse_iterator crbegin() const { return const_reverse_iterator(_cont.crbegin()); } + const_reverse_iterator crend() const { return const_reverse_iterator(_cont.crend()); } +#endif + iterator begin() { return iterator(_cont.begin()); } + iterator end() { return iterator(_cont.end()); } + reverse_iterator rbegin() { return reverse_iterator(_cont.rbegin()); } + reverse_iterator rend() { return reverse_iterator(_cont.rend()); } + const_iterator find(pkgCache::PkgIterator const &P) const { return const_iterator(_cont.find(P)); } + + PackageContainer() : PackageContainerInterface(CacheSetHelper::UNKNOWN) {} + explicit PackageContainer(CacheSetHelper::PkgSelector const &by) : PackageContainerInterface(by) {} + template<typename Itr> PackageContainer(Itr first, Itr last) : PackageContainerInterface(CacheSetHelper::UNKNOWN), _cont(first, last) {} +#if __cplusplus >= 201103L + PackageContainer(std::initializer_list<value_type> list) : PackageContainerInterface(CacheSetHelper::UNKNOWN), _cont(list) {} + void push_back(value_type&& P) { _cont.emplace_back(std::move(P)); } + template<typename... Args> void emplace_back(Args&&... args) { _cont.emplace_back(std::forward<Args>(args)...); } +#endif + void push_back(const value_type& P) { _cont.push_back(P); } + + /** \brief sort all included versions with given comparer + + Some containers are sorted by default, some are not and can't be, + but a few like std::vector can be sorted if need be, so this can be + specialized in later on. The default is that this will fail though. + Specifically, already sorted containers like std::set will return + false as well as there is no easy way to check that the given comparer + would sort in the same way the set is currently sorted + + \return \b true if the set was sorted, \b false if not. */ + template<class Compare> bool sort(Compare /*Comp*/) { return false; } + + /** \brief returns all packages in the cache who belong to the given task + + A simple helper responsible for search for all members of a task + in the cache. Optional it prints a notice about the + packages chosen cause of the given task. + \param Cache the packages are in + \param pattern name of the task + \param helper responsible for error and message handling */ + static PackageContainer FromTask(pkgCacheFile &Cache, std::string const &pattern, CacheSetHelper &helper) { + PackageContainer cont(CacheSetHelper::TASK); + helper.PackageFrom(CacheSetHelper::TASK, &cont, Cache, pattern); + return cont; + } + static PackageContainer FromTask(pkgCacheFile &Cache, std::string const &pattern) { + CacheSetHelper helper; + return FromTask(Cache, pattern, helper); + } + + /** \brief returns all packages in the cache whose name matches a given pattern + + A simple helper responsible for executing a regular expression on all + package names in the cache. Optional it prints a notice about the + packages chosen cause of the given package. + \param Cache the packages are in + \param pattern regular expression for package names + \param helper responsible for error and message handling */ + static PackageContainer FromRegEx(pkgCacheFile &Cache, std::string const &pattern, CacheSetHelper &helper) { + PackageContainer cont(CacheSetHelper::REGEX); + helper.PackageFrom(CacheSetHelper::REGEX, &cont, Cache, pattern); + return cont; + } + + static PackageContainer FromRegEx(pkgCacheFile &Cache, std::string const &pattern) { + CacheSetHelper helper; + return FromRegEx(Cache, pattern, helper); + } + + static PackageContainer FromFnmatch(pkgCacheFile &Cache, std::string const &pattern, CacheSetHelper &helper) { + PackageContainer cont(CacheSetHelper::FNMATCH); + helper.PackageFrom(CacheSetHelper::FNMATCH, &cont, Cache, pattern); + return cont; + } + static PackageContainer FromFnMatch(pkgCacheFile &Cache, std::string const &pattern) { + CacheSetHelper helper; + return FromFnmatch(Cache, pattern, helper); + } + + /** \brief returns all packages specified by a string + + \param Cache the packages are in + \param pattern String the package name(s) should be extracted from + \param helper responsible for error and message handling */ + static PackageContainer FromString(pkgCacheFile &Cache, std::string const &pattern, CacheSetHelper &helper) { + PackageContainer cont; + helper.PackageFrom(CacheSetHelper::PACKAGENAME, &cont, Cache, pattern); + return cont; + } + static PackageContainer FromString(pkgCacheFile &Cache, std::string const &pattern) { + CacheSetHelper helper; + return FromString(Cache, pattern, helper); + } + + /** \brief returns all packages specified on the commandline + + Get all package names from the commandline and executes regex's if needed. + No special package command is supported, just plain names. + \param Cache the packages are in + \param cmdline Command line the package names should be extracted from + \param helper responsible for error and message handling */ + static PackageContainer FromCommandLine(pkgCacheFile &Cache, const char **cmdline, CacheSetHelper &helper) { + PackageContainer cont; + helper.PackageFromCommandLine(&cont, Cache, cmdline); + return cont; + } + static PackageContainer FromCommandLine(pkgCacheFile &Cache, const char **cmdline) { + CacheSetHelper helper; + return FromCommandLine(Cache, cmdline, helper); + } + + /** \brief group packages by a action modifiers + + At some point it is needed to get from the same commandline + different package sets grouped by a modifier. Take + apt-get install apt awesome- + as an example. + \param Cache the packages are in + \param cmdline Command line the package names should be extracted from + \param mods list of modifiers the method should accept + \param fallback the default modifier group for a package + \param helper responsible for error and message handling */ + static std::map<unsigned short, PackageContainer> GroupedFromCommandLine( + pkgCacheFile &Cache, + const char **cmdline, + std::list<CacheSetHelper::PkgModifier> const &mods, + unsigned short const &fallback, + CacheSetHelper &helper) { + std::map<unsigned short, PackageContainer> pkgsets; + for (const char **I = cmdline; *I != 0; ++I) { + unsigned short modID = fallback; + PackageContainer pkgset; + helper.PackageFromModifierCommandLine(modID, &pkgset, Cache, *I, mods); + pkgsets[modID].insert(pkgset); + } + return pkgsets; + } + static std::map<unsigned short, PackageContainer> GroupedFromCommandLine( + pkgCacheFile &Cache, + const char **cmdline, + std::list<CacheSetHelper::PkgModifier> const &mods, + unsigned short const &fallback) { + CacheSetHelper helper; + return GroupedFromCommandLine(Cache, cmdline, + mods, fallback, helper); + } + /*}}}*/ +}; /*}}}*/ +// various specialisations for PackageContainer /*{{{*/ +template<> template<class Cont> void PackageContainer<std::list<pkgCache::PkgIterator> >::insert(PackageContainer<Cont> const &pkgcont) { + for (typename PackageContainer<Cont>::const_iterator p = pkgcont.begin(); p != pkgcont.end(); ++p) + _cont.push_back(*p); +} +#if __cplusplus >= 201103L +template<> template<class Cont> void PackageContainer<std::forward_list<pkgCache::PkgIterator> >::insert(PackageContainer<Cont> const &pkgcont) { + for (typename PackageContainer<Cont>::const_iterator p = pkgcont.begin(); p != pkgcont.end(); ++p) + _cont.push_front(*p); +} +#endif +template<> template<class Cont> void PackageContainer<std::deque<pkgCache::PkgIterator> >::insert(PackageContainer<Cont> const &pkgcont) { + for (typename PackageContainer<Cont>::const_iterator p = pkgcont.begin(); p != pkgcont.end(); ++p) + _cont.push_back(*p); +} +template<> template<class Cont> void PackageContainer<std::vector<pkgCache::PkgIterator> >::insert(PackageContainer<Cont> const &pkgcont) { + for (typename PackageContainer<Cont>::const_iterator p = pkgcont.begin(); p != pkgcont.end(); ++p) + _cont.push_back(*p); +} +// these are 'inline' as otherwise the linker has problems with seeing these untemplated +// specializations again and again - but we need to see them, so that library users can use them +template<> inline bool PackageContainer<std::list<pkgCache::PkgIterator> >::insert(pkgCache::PkgIterator const &P) { + if (P.end() == true) + return false; + _cont.push_back(P); + return true; +} +#if __cplusplus >= 201103L +template<> inline bool PackageContainer<std::forward_list<pkgCache::PkgIterator> >::insert(pkgCache::PkgIterator const &P) { + if (P.end() == true) + return false; + _cont.push_front(P); + return true; +} +#endif +template<> inline bool PackageContainer<std::deque<pkgCache::PkgIterator> >::insert(pkgCache::PkgIterator const &P) { + if (P.end() == true) + return false; + _cont.push_back(P); + return true; +} +template<> inline bool PackageContainer<std::vector<pkgCache::PkgIterator> >::insert(pkgCache::PkgIterator const &P) { + if (P.end() == true) + return false; + _cont.push_back(P); + return true; +} +template<> inline void PackageContainer<std::list<pkgCache::PkgIterator> >::insert(const_iterator begin, const_iterator end) { + for (const_iterator p = begin; p != end; ++p) + _cont.push_back(*p); +} +#if __cplusplus >= 201103L +template<> inline void PackageContainer<std::forward_list<pkgCache::PkgIterator> >::insert(const_iterator begin, const_iterator end) { + for (const_iterator p = begin; p != end; ++p) + _cont.push_front(*p); +} +#endif +template<> inline void PackageContainer<std::deque<pkgCache::PkgIterator> >::insert(const_iterator begin, const_iterator end) { + for (const_iterator p = begin; p != end; ++p) + _cont.push_back(*p); +} +template<> inline void PackageContainer<std::vector<pkgCache::PkgIterator> >::insert(const_iterator begin, const_iterator end) { + for (const_iterator p = begin; p != end; ++p) + _cont.push_back(*p); +} +#if APT_GCC_VERSION < 0x409 +template<> inline PackageContainer<std::set<pkgCache::PkgIterator> >::iterator PackageContainer<std::set<pkgCache::PkgIterator> >::erase(iterator i) { + _cont.erase(i._iter); + return end(); +} +template<> inline PackageContainer<std::set<pkgCache::PkgIterator> >::iterator PackageContainer<std::set<pkgCache::PkgIterator> >::erase(iterator first, iterator last) { + _cont.erase(first, last); + return end(); +} +#endif +template<> template<class Compare> inline bool PackageContainer<std::vector<pkgCache::PkgIterator> >::sort(Compare Comp) { + std::sort(_cont.begin(), _cont.end(), Comp); + return true; +} +template<> template<class Compare> inline bool PackageContainer<std::list<pkgCache::PkgIterator> >::sort(Compare Comp) { + _cont.sort(Comp); + return true; +} +#if __cplusplus >= 201103L +template<> template<class Compare> inline bool PackageContainer<std::forward_list<pkgCache::PkgIterator> >::sort(Compare Comp) { + _cont.sort(Comp); + return true; +} +#endif +template<> template<class Compare> inline bool PackageContainer<std::deque<pkgCache::PkgIterator> >::sort(Compare Comp) { + std::sort(_cont.begin(), _cont.end(), Comp); + return true; +} + /*}}}*/ + +// class PackageUniverse - pkgCache as PackageContainerInterface /*{{{*/ +/** \class PackageUniverse + + Wraps around our usual pkgCache, so that it can be stuffed into methods + expecting a PackageContainer. + + The wrapping is read-only in practice modeled by making erase and co + private methods. */ +class APT_PUBLIC PackageUniverse : public PackageContainerInterface { + pkgCache * const _cont; + void * const d; +public: + class const_iterator : public APT::Container_iterator_base<APT::PackageContainerInterface, PackageUniverse, PackageUniverse::const_iterator, pkgCache::PkgIterator, pkgCache::PkgIterator> + { + public: + explicit const_iterator(pkgCache::PkgIterator i): + Container_iterator_base<APT::PackageContainerInterface, PackageUniverse, PackageUniverse::const_iterator, pkgCache::PkgIterator, pkgCache::PkgIterator>(i) {} + + inline pkgCache::PkgIterator getType(void) const { return _iter; } + }; + typedef const_iterator iterator; + typedef pkgCache::PkgIterator value_type; + typedef typename pkgCache::PkgIterator* pointer; + typedef typename pkgCache::PkgIterator const* const_pointer; + typedef const pkgCache::PkgIterator& const_reference; + typedef const_reference reference; + typedef const_iterator::difference_type difference_type; + typedef std::make_unsigned<const_iterator::difference_type>::type size_type; + + + bool empty() const APT_OVERRIDE { return false; } + size_t size() const APT_OVERRIDE { return _cont->Head().PackageCount; } + + const_iterator begin() const { return const_iterator(_cont->PkgBegin()); } + const_iterator end() const { return const_iterator(_cont->PkgEnd()); } + const_iterator cbegin() const { return const_iterator(_cont->PkgBegin()); } + const_iterator cend() const { return const_iterator(_cont->PkgEnd()); } + iterator begin() { return iterator(_cont->PkgBegin()); } + iterator end() { return iterator(_cont->PkgEnd()); } + + pkgCache * data() const { return _cont; } + + explicit PackageUniverse(pkgCache * const Owner); + explicit PackageUniverse(pkgCacheFile * const Owner); + virtual ~PackageUniverse(); + +private: + APT_HIDDEN bool insert(pkgCache::PkgIterator const &) APT_OVERRIDE { return true; } + template<class Cont> APT_HIDDEN void insert(PackageContainer<Cont> const &) { } + APT_HIDDEN void insert(const_iterator, const_iterator) { } + + APT_HIDDEN void clear() APT_OVERRIDE { } + APT_HIDDEN iterator erase( const_iterator pos ); + APT_HIDDEN iterator erase( const_iterator first, const_iterator last ); +}; + /*}}}*/ +typedef PackageContainer<std::set<pkgCache::PkgIterator> > PackageSet; +#if __cplusplus >= 201103L +typedef PackageContainer<std::unordered_set<pkgCache::PkgIterator> > PackageUnorderedSet; +typedef PackageContainer<std::forward_list<pkgCache::PkgIterator> > PackageForwardList; +#endif +typedef PackageContainer<std::list<pkgCache::PkgIterator> > PackageList; +typedef PackageContainer<std::deque<pkgCache::PkgIterator> > PackageDeque; +typedef PackageContainer<std::vector<pkgCache::PkgIterator> > PackageVector; + +class APT_PUBLIC VersionContainerInterface { /*{{{*/ +/** \class APT::VersionContainerInterface + + Same as APT::PackageContainerInterface, just for Versions */ +public: + /** \brief smell like a pkgCache::VerIterator */ + template<class Itr> class iterator_base { /*{{{*/ + pkgCache::VerIterator getType() const { return static_cast<Itr const*>(this)->getType(); }; + public: + operator pkgCache::VerIterator(void) { return getType(); } + + inline pkgCache *Cache() const { return getType().Cache(); } + inline unsigned long Index() const {return getType().Index();} + inline int CompareVer(const pkgCache::VerIterator &B) const { return getType().CompareVer(B); } + inline const char *VerStr() const { return getType().VerStr(); } + inline const char *Section() const { return getType().Section(); } + inline const char *Arch() const { return getType().Arch(); } + inline pkgCache::PkgIterator ParentPkg() const { return getType().ParentPkg(); } + inline pkgCache::DescIterator DescriptionList() const { return getType().DescriptionList(); } + inline pkgCache::DescIterator TranslatedDescription() const { return getType().TranslatedDescription(); } + inline pkgCache::DepIterator DependsList() const { return getType().DependsList(); } + inline pkgCache::PrvIterator ProvidesList() const { return getType().ProvidesList(); } + inline pkgCache::VerFileIterator FileList() const { return getType().FileList(); } + inline bool Downloadable() const { return getType().Downloadable(); } + inline const char *PriorityType() const { return getType().PriorityType(); } + inline std::string RelStr() const { return getType().RelStr(); } + inline bool Automatic() const { return getType().Automatic(); } + inline pkgCache::VerFileIterator NewestFile() const { return getType().NewestFile(); } + // we have only valid iterators here + inline bool end() const { return false; } + + inline pkgCache::Version const * operator->() const { return &*getType(); } + }; + /*}}}*/ + + virtual bool insert(pkgCache::VerIterator const &V) = 0; + virtual bool empty() const = 0; + virtual void clear() = 0; + virtual size_t size() const = 0; + + struct Modifier { + unsigned short const ID; + const char * const Alias; + enum Position { NONE, PREFIX, POSTFIX } const Pos; + enum CacheSetHelper::VerSelector const SelectVersion; + Modifier (unsigned short const &id, const char * const alias, Position const &pos, + enum CacheSetHelper::VerSelector const select) : ID(id), Alias(alias), Pos(pos), + SelectVersion(select) {} + }; + + static bool FromCommandLine(VersionContainerInterface * const vci, pkgCacheFile &Cache, + const char **cmdline, CacheSetHelper::VerSelector const fallback, + CacheSetHelper &helper); + + static bool FromString(VersionContainerInterface * const vci, pkgCacheFile &Cache, + std::string pkg, CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper, + bool const onlyFromName = false); + + static bool FromPattern(VersionContainerInterface *const vci, pkgCacheFile &Cache, + std::string pkg, CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper); + + static bool FromPackage(VersionContainerInterface * const vci, pkgCacheFile &Cache, + pkgCache::PkgIterator const &P, CacheSetHelper::VerSelector const fallback, + CacheSetHelper &helper); + + static bool FromModifierCommandLine(unsigned short &modID, + VersionContainerInterface * const vci, + pkgCacheFile &Cache, const char * cmdline, + std::list<Modifier> const &mods, + CacheSetHelper &helper); + + + static bool FromDependency(VersionContainerInterface * const vci, + pkgCacheFile &Cache, + pkgCache::DepIterator const &D, + CacheSetHelper::VerSelector const selector, + CacheSetHelper &helper); + + VersionContainerInterface(); + VersionContainerInterface(VersionContainerInterface const &other); + VersionContainerInterface& operator=(VersionContainerInterface const &other); + virtual ~VersionContainerInterface(); +private: + void * const d; + +protected: /*{{{*/ + + /** \brief returns the candidate version of the package + + \param Cache to be used to query for information + \param Pkg we want the candidate version from this package + \param helper used in this container instance */ + static pkgCache::VerIterator getCandidateVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg, CacheSetHelper &helper); + + /** \brief returns the installed version of the package + + \param Cache to be used to query for information + \param Pkg we want the installed version from this package + \param helper used in this container instance */ + static pkgCache::VerIterator getInstalledVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg, CacheSetHelper &helper); + /*}}}*/ +}; + /*}}}*/ +template<class Container> class APT_PUBLIC VersionContainer : public VersionContainerInterface {/*{{{*/ +/** \class APT::VersionContainer + + Simple wrapper around a container class like std::set to provide a similar + interface to a set of versions as to the complete set of all versions in the + pkgCache. */ + Container _cont; +public: /*{{{*/ + + typedef Container_const_iterator<VersionContainerInterface, Container, VersionContainer> const_iterator; + typedef Container_iterator<VersionContainerInterface, Container, VersionContainer> iterator; + typedef Container_const_reverse_iterator<VersionContainerInterface, Container, VersionContainer> const_reverse_iterator; + typedef Container_reverse_iterator<VersionContainerInterface, Container, VersionContainer> reverse_iterator; + typedef typename Container::value_type value_type; + typedef typename Container::pointer pointer; + typedef typename Container::const_pointer const_pointer; + typedef typename Container::reference reference; + typedef typename Container::const_reference const_reference; + typedef typename Container::difference_type difference_type; + typedef typename Container::size_type size_type; + typedef typename Container::allocator_type allocator_type; + + bool insert(pkgCache::VerIterator const &V) APT_OVERRIDE { if (V.end() == true) return false; _cont.insert(V); return true; } + template<class Cont> void insert(VersionContainer<Cont> const &vercont) { _cont.insert((typename Cont::const_iterator)vercont.begin(), (typename Cont::const_iterator)vercont.end()); } + void insert(const_iterator begin, const_iterator end) { _cont.insert(begin, end); } + bool empty() const APT_OVERRIDE { return _cont.empty(); } + void clear() APT_OVERRIDE { return _cont.clear(); } + size_t size() const APT_OVERRIDE { return _cont.size(); } +#if APT_GCC_VERSION >= 0x409 + iterator erase( const_iterator pos ) { return iterator(_cont.erase(pos._iter)); } + iterator erase( const_iterator first, const_iterator last ) { return iterator(_cont.erase(first._iter, last._iter)); } +#else + iterator erase( iterator pos ) { return iterator(_cont.erase(pos._iter)); } + iterator erase( iterator first, iterator last ) { return iterator(_cont.erase(first._iter, last._iter)); } +#endif + const_iterator begin() const { return const_iterator(_cont.begin()); } + const_iterator end() const { return const_iterator(_cont.end()); } + const_reverse_iterator rbegin() const { return const_reverse_iterator(_cont.rbegin()); } + const_reverse_iterator rend() const { return const_reverse_iterator(_cont.rend()); } +#if __cplusplus >= 201103L + const_iterator cbegin() const { return const_iterator(_cont.cbegin()); } + const_iterator cend() const { return const_iterator(_cont.cend()); } + const_reverse_iterator crbegin() const { return const_reverse_iterator(_cont.crbegin()); } + const_reverse_iterator crend() const { return const_reverse_iterator(_cont.crend()); } +#endif + iterator begin() { return iterator(_cont.begin()); } + iterator end() { return iterator(_cont.end()); } + reverse_iterator rbegin() { return reverse_iterator(_cont.rbegin()); } + reverse_iterator rend() { return reverse_iterator(_cont.rend()); } + const_iterator find(pkgCache::VerIterator const &V) const { return const_iterator(_cont.find(V)); } + + VersionContainer() : VersionContainerInterface() {} + template<typename Itr> VersionContainer(Itr first, Itr last) : VersionContainerInterface(), _cont(first, last) {} +#if __cplusplus >= 201103L + VersionContainer(std::initializer_list<value_type> list) : VersionContainerInterface(), _cont(list) {} + void push_back(value_type&& P) { _cont.emplace_back(std::move(P)); } + template<typename... Args> void emplace_back(Args&&... args) { _cont.emplace_back(std::forward<Args>(args)...); } +#endif + void push_back(const value_type& P) { _cont.push_back(P); } + + /** \brief sort all included versions with given comparer + + Some containers are sorted by default, some are not and can't be, + but a few like std::vector can be sorted if need be, so this can be + specialized in later on. The default is that this will fail though. + Specifically, already sorted containers like std::set will return + false as well as there is no easy way to check that the given comparer + would sort in the same way the set is currently sorted + + \return \b true if the set was sorted, \b false if not. */ + template<class Compare> bool sort(Compare /*Comp*/) { return false; } + + /** \brief returns all versions specified on the commandline + + Get all versions from the commandline, uses given default version if + non specifically requested and executes regex's if needed on names. + \param Cache the packages and versions are in + \param cmdline Command line the versions should be extracted from + \param fallback version specification + \param helper responsible for error and message handling */ + static VersionContainer FromCommandLine(pkgCacheFile &Cache, const char **cmdline, + CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper) { + VersionContainer vercon; + VersionContainerInterface::FromCommandLine(&vercon, Cache, cmdline, fallback, helper); + return vercon; + } + static VersionContainer FromCommandLine(pkgCacheFile &Cache, const char **cmdline, + CacheSetHelper::VerSelector const fallback) { + CacheSetHelper helper; + return FromCommandLine(Cache, cmdline, fallback, helper); + } + static VersionContainer FromCommandLine(pkgCacheFile &Cache, const char **cmdline) { + return FromCommandLine(Cache, cmdline, CacheSetHelper::CANDINST); + } + static VersionContainer FromString(pkgCacheFile &Cache, std::string const &pkg, + CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper, + bool const /*onlyFromName = false*/) { + VersionContainer vercon; + VersionContainerInterface::FromString(&vercon, Cache, pkg, fallback, helper); + return vercon; + } + static VersionContainer FromString(pkgCacheFile &Cache, std::string pkg, + CacheSetHelper::VerSelector const fallback) { + CacheSetHelper helper; + return FromString(Cache, pkg, fallback, helper); + } + static VersionContainer FromString(pkgCacheFile &Cache, std::string pkg) { + return FromString(Cache, pkg, CacheSetHelper::CANDINST); + } + + + /** \brief returns all versions specified for the package + + \param Cache the package and versions are in + \param P the package in question + \param fallback the version(s) you want to get + \param helper the helper used for display and error handling */ + static VersionContainer FromPackage(pkgCacheFile &Cache, pkgCache::PkgIterator const &P, + CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper) { + VersionContainer vercon; + VersionContainerInterface::FromPackage(&vercon, Cache, P, fallback, helper); + return vercon; + } + static VersionContainer FromPackage(pkgCacheFile &Cache, pkgCache::PkgIterator const &P, + CacheSetHelper::VerSelector const fallback) { + CacheSetHelper helper; + return FromPackage(Cache, P, fallback, helper); + } + static VersionContainer FromPackage(pkgCacheFile &Cache, pkgCache::PkgIterator const &P) { + return FromPackage(Cache, P, CacheSetHelper::CANDIDATE); + } + + static std::map<unsigned short, VersionContainer> GroupedFromCommandLine( + pkgCacheFile &Cache, + const char **cmdline, + std::list<Modifier> const &mods, + unsigned short const fallback, + CacheSetHelper &helper) { + std::map<unsigned short, VersionContainer> versets; + for (const char **I = cmdline; *I != 0; ++I) { + unsigned short modID = fallback; + VersionContainer verset; + VersionContainerInterface::FromModifierCommandLine(modID, &verset, Cache, *I, mods, helper); + versets[modID].insert(verset); + } + return versets; + + } + static std::map<unsigned short, VersionContainer> GroupedFromCommandLine( + pkgCacheFile &Cache, const char **cmdline, + std::list<Modifier> const &mods, + unsigned short const fallback) { + CacheSetHelper helper; + return GroupedFromCommandLine(Cache, cmdline, + mods, fallback, helper); + } + + static VersionContainer FromDependency(pkgCacheFile &Cache, pkgCache::DepIterator const &D, + CacheSetHelper::VerSelector const selector, CacheSetHelper &helper) { + VersionContainer vercon; + VersionContainerInterface::FromDependency(&vercon, Cache, D, selector, helper); + return vercon; + } + static VersionContainer FromDependency(pkgCacheFile &Cache, pkgCache::DepIterator const &D, + CacheSetHelper::VerSelector const selector) { + CacheSetHelper helper; + return FromDependency(Cache, D, selector, helper); + } + static VersionContainer FromDependency(pkgCacheFile &Cache, pkgCache::DepIterator const &D) { + return FromDependency(Cache, D, CacheSetHelper::CANDIDATE); + } + /*}}}*/ +}; /*}}}*/ +// various specialisations for VersionContainer /*{{{*/ +template<> template<class Cont> void VersionContainer<std::list<pkgCache::VerIterator> >::insert(VersionContainer<Cont> const &vercont) { + for (typename VersionContainer<Cont>::const_iterator v = vercont.begin(); v != vercont.end(); ++v) + _cont.push_back(*v); +} +#if __cplusplus >= 201103L +template<> template<class Cont> void VersionContainer<std::forward_list<pkgCache::VerIterator> >::insert(VersionContainer<Cont> const &vercont) { + for (typename VersionContainer<Cont>::const_iterator v = vercont.begin(); v != vercont.end(); ++v) + _cont.push_front(*v); +} +#endif +template<> template<class Cont> void VersionContainer<std::deque<pkgCache::VerIterator> >::insert(VersionContainer<Cont> const &vercont) { + for (typename VersionContainer<Cont>::const_iterator v = vercont.begin(); v != vercont.end(); ++v) + _cont.push_back(*v); +} +template<> template<class Cont> void VersionContainer<std::vector<pkgCache::VerIterator> >::insert(VersionContainer<Cont> const &vercont) { + for (typename VersionContainer<Cont>::const_iterator v = vercont.begin(); v != vercont.end(); ++v) + _cont.push_back(*v); +} +// these are 'inline' as otherwise the linker has problems with seeing these untemplated +// specializations again and again - but we need to see them, so that library users can use them +template<> inline bool VersionContainer<std::list<pkgCache::VerIterator> >::insert(pkgCache::VerIterator const &V) { + if (V.end() == true) + return false; + _cont.push_back(V); + return true; +} +#if __cplusplus >= 201103L +template<> inline bool VersionContainer<std::forward_list<pkgCache::VerIterator> >::insert(pkgCache::VerIterator const &V) { + if (V.end() == true) + return false; + _cont.push_front(V); + return true; +} +#endif +template<> inline bool VersionContainer<std::deque<pkgCache::VerIterator> >::insert(pkgCache::VerIterator const &V) { + if (V.end() == true) + return false; + _cont.push_back(V); + return true; +} +template<> inline bool VersionContainer<std::vector<pkgCache::VerIterator> >::insert(pkgCache::VerIterator const &V) { + if (V.end() == true) + return false; + _cont.push_back(V); + return true; +} +template<> inline void VersionContainer<std::list<pkgCache::VerIterator> >::insert(const_iterator begin, const_iterator end) { + for (const_iterator v = begin; v != end; ++v) + _cont.push_back(*v); +} +#if __cplusplus >= 201103L +template<> inline void VersionContainer<std::forward_list<pkgCache::VerIterator> >::insert(const_iterator begin, const_iterator end) { + for (const_iterator v = begin; v != end; ++v) + _cont.push_front(*v); +} +#endif +template<> inline void VersionContainer<std::deque<pkgCache::VerIterator> >::insert(const_iterator begin, const_iterator end) { + for (const_iterator v = begin; v != end; ++v) + _cont.push_back(*v); +} +template<> inline void VersionContainer<std::vector<pkgCache::VerIterator> >::insert(const_iterator begin, const_iterator end) { + for (const_iterator v = begin; v != end; ++v) + _cont.push_back(*v); +} +#if APT_GCC_VERSION < 0x409 +template<> inline VersionContainer<std::set<pkgCache::VerIterator> >::iterator VersionContainer<std::set<pkgCache::VerIterator> >::erase(iterator i) { + _cont.erase(i._iter); + return end(); +} +template<> inline VersionContainer<std::set<pkgCache::VerIterator> >::iterator VersionContainer<std::set<pkgCache::VerIterator> >::erase(iterator first, iterator last) { + _cont.erase(first, last); + return end(); +} +#endif +template<> template<class Compare> inline bool VersionContainer<std::vector<pkgCache::VerIterator> >::sort(Compare Comp) { + std::sort(_cont.begin(), _cont.end(), Comp); + return true; +} +template<> template<class Compare> inline bool VersionContainer<std::list<pkgCache::VerIterator> >::sort(Compare Comp) { + _cont.sort(Comp); + return true; +} +#if __cplusplus >= 201103L +template<> template<class Compare> inline bool VersionContainer<std::forward_list<pkgCache::VerIterator> >::sort(Compare Comp) { + _cont.sort(Comp); + return true; +} +#endif +template<> template<class Compare> inline bool VersionContainer<std::deque<pkgCache::VerIterator> >::sort(Compare Comp) { + std::sort(_cont.begin(), _cont.end(), Comp); + return true; +} + /*}}}*/ + +typedef VersionContainer<std::set<pkgCache::VerIterator> > VersionSet; +#if __cplusplus >= 201103L +typedef VersionContainer<std::unordered_set<pkgCache::VerIterator> > VersionUnorderedSet; +typedef VersionContainer<std::forward_list<pkgCache::VerIterator> > VersionForwardList; +#endif +typedef VersionContainer<std::list<pkgCache::VerIterator> > VersionList; +typedef VersionContainer<std::deque<pkgCache::VerIterator> > VersionDeque; +typedef VersionContainer<std::vector<pkgCache::VerIterator> > VersionVector; +} +#endif diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc new file mode 100644 index 0000000..ea10d10 --- /dev/null +++ b/apt-pkg/cdrom.cc @@ -0,0 +1,989 @@ +/* + */ +#include <config.h> + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/cdrom.h> +#include <apt-pkg/cdromutl.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/indexcopy.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <fstream> +#include <iostream> +#include <iostream> +#include <sstream> +#include <string> +#include <vector> +#include <dirent.h> +#include <dlfcn.h> +#include <stdio.h> +#include <string.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <apti18n.h> + +#ifdef HAVE_UDEV +#include <libudev.h> +#endif + +using namespace std; + +// FindPackages - Find the package files on the CDROM /*{{{*/ +// --------------------------------------------------------------------- +/* We look over the cdrom for package files. This is a recursive + search that short circuits when it his a package file in the dir. + This speeds it up greatly as the majority of the size is in the + binary-* sub dirs. */ +bool pkgCdrom::FindPackages(string CD, + vector<string> &List, + vector<string> &SList, + vector<string> &SigList, + vector<string> &TransList, + string &InfoDir, pkgCdromStatus *log, + unsigned int Depth) +{ + static ino_t Inodes[9]; + DIR *D; + + // if we have a look we "pulse" now + if(log) + log->Update(); + + if (Depth >= 7) + return true; + + if (CD[CD.length()-1] != '/') + CD += '/'; + + if (chdir(CD.c_str()) != 0) + return _error->Errno("chdir","Unable to change to %s",CD.c_str()); + + // Look for a .disk subdirectory + if (InfoDir.empty() == true) + { + if (DirectoryExists(".disk") == true) + InfoDir = InfoDir + CD + ".disk/"; + } + + // Don't look into directories that have been marked to ignore. + if (RealFileExists(".aptignr") == true) + return true; + + /* Check _first_ for a signature file as apt-cdrom assumes that all files + under a Packages/Source file are in control of that file and stops + the scanning + */ + if (RealFileExists("Release.gpg") == true || RealFileExists("InRelease") == true) + { + SigList.push_back(CD); + } + + /* Aha! We found some package files. We assume that everything under + this dir is controlled by those package files so we don't look down + anymore */ + std::vector<APT::Configuration::Compressor> const compressor = APT::Configuration::getCompressors(); + for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) + { + if (RealFileExists(std::string("Packages").append(c->Extension).c_str()) == false) + continue; + + if (_config->FindB("Debug::aptcdrom",false) == true) + std::clog << "Found Packages in " << CD << std::endl; + List.push_back(CD); + + // Continue down if thorough is given + if (_config->FindB("APT::CDROM::Thorough",false) == false) + return true; + break; + } + for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) + { + if (RealFileExists(std::string("Sources").append(c->Extension).c_str()) == false) + continue; + + if (_config->FindB("Debug::aptcdrom",false) == true) + std::clog << "Found Sources in " << CD << std::endl; + SList.push_back(CD); + + // Continue down if thorough is given + if (_config->FindB("APT::CDROM::Thorough",false) == false) + return true; + break; + } + + // see if we find translation indices + if (DirectoryExists("i18n") == true) + { + D = opendir("i18n"); + for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + { + if(strncmp(Dir->d_name, "Translation-", strlen("Translation-")) != 0) + continue; + string file = Dir->d_name; + for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) + { + string fileext = flExtension(file); + if (file == fileext) + fileext.clear(); + else if (fileext.empty() == false) + fileext = "." + fileext; + + if (c->Extension == fileext) + { + if (_config->FindB("Debug::aptcdrom",false) == true) + std::clog << "Found translation " << Dir->d_name << " in " << CD << "i18n/" << std::endl; + file.erase(file.size() - fileext.size()); + TransList.push_back(CD + "i18n/" + file); + break; + } + } + } + closedir(D); + } + + D = opendir("."); + if (D == 0) + return _error->Errno("opendir","Unable to read %s",CD.c_str()); + + // Run over the directory + for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + { + // Skip some files.. + if (strcmp(Dir->d_name,".") == 0 || + strcmp(Dir->d_name,"..") == 0 || + strcmp(Dir->d_name,".disk") == 0 || + strcmp(Dir->d_name,"debian-installer") == 0) + continue; + + // See if the name is a sub directory + struct stat Buf; + if (stat(Dir->d_name,&Buf) != 0) + continue; + + if (S_ISDIR(Buf.st_mode) == 0) + continue; + + unsigned int I; + for (I = 0; I != Depth; I++) + if (Inodes[I] == Buf.st_ino) + break; + if (I != Depth) + continue; + + // Store the inodes weve seen + Inodes[Depth] = Buf.st_ino; + + // Descend + if (FindPackages(CD + Dir->d_name,List,SList,SigList,TransList,InfoDir,log,Depth+1) == false) + break; + + if (chdir(CD.c_str()) != 0) + { + _error->Errno("chdir","Unable to change to %s", CD.c_str()); + closedir(D); + return false; + } + }; + + closedir(D); + + return !_error->PendingError(); +} + /*}}}*/ +// Score - We compute a 'score' for a path /*{{{*/ +// --------------------------------------------------------------------- +/* Paths are scored based on how close they come to what I consider + normal. That is ones that have 'dist' 'stable' 'testing' will score + higher than ones without. */ +int pkgCdrom::Score(string Path) +{ + int Res = 0; + if (Path.find("stable/") != string::npos) + Res += 29; + if (Path.find("/binary-") != string::npos) + Res += 20; + if (Path.find("testing/") != string::npos) + Res += 28; + if (Path.find("unstable/") != string::npos) + Res += 27; + if (Path.find("/dists/") != string::npos) + Res += 40; + if (Path.find("/main/") != string::npos) + Res += 20; + if (Path.find("/contrib/") != string::npos) + Res += 20; + if (Path.find("/non-free/") != string::npos) + Res += 20; + if (Path.find("/non-free-firmware/") != string::npos) + Res += 20; + if (Path.find("/non-US/") != string::npos) + Res += 20; + if (Path.find("/source/") != string::npos) + Res += 10; + if (Path.find("/debian/") != string::npos) + Res -= 10; + + // check for symlinks in the patch leading to the actual file + // a symlink gets a big penalty + struct stat Buf; + string statPath = flNotFile(Path); + string cdromPath = _config->FindDir("Acquire::cdrom::mount"); + while(statPath != cdromPath && statPath != "./") { + statPath.resize(statPath.size()-1); // remove the trailing '/' + if (lstat(statPath.c_str(),&Buf) == 0) { + if(S_ISLNK(Buf.st_mode)) { + Res -= 60; + break; + } + } + statPath = flNotFile(statPath); // descent + } + + return Res; +} + /*}}}*/ +// DropBinaryArch - Dump dirs with a string like /binary-<foo>/ /*{{{*/ +// --------------------------------------------------------------------- +/* Here we drop everything that is not this machines arch */ +bool pkgCdrom::DropBinaryArch(vector<string> &List) +{ + + for (unsigned int I = 0; I < List.size(); I++) + { + const char *Str = List[I].c_str(); + const char *Start, *End; + if ((Start = strstr(Str,"/binary-")) == 0) + continue; + + // Between Start and End is the architecture + Start += 8; + if ((End = strstr(Start,"/")) != 0 && Start != End && + APT::Configuration::checkArchitecture(string(Start, End)) == true) + continue; // okay, architecture is accepted + + // not accepted -> Erase it + List.erase(List.begin() + I); + --I; // the next entry is at the same index after the erase + } + + return true; +} + /*}}}*/ +// DropTranslation - Dump unwanted Translation-<lang> files /*{{{*/ +// --------------------------------------------------------------------- +/* Here we drop everything that is not configured in Acquire::Languages */ +bool pkgCdrom::DropTranslation(vector<string> &List) +{ + for (unsigned int I = 0; I < List.size(); I++) + { + const char *Start; + if ((Start = strstr(List[I].c_str(), "/Translation-")) == NULL) + continue; + Start += strlen("/Translation-"); + + if (APT::Configuration::checkLanguage(Start, true) == true) + continue; + + // not accepted -> Erase it + List.erase(List.begin() + I); + --I; // the next entry is at the same index after the erase + } + + return true; +} + /*}}}*/ +// DropRepeats - Drop repeated files resulting from symlinks /*{{{*/ +// --------------------------------------------------------------------- +/* Here we go and stat every file that we found and strip dup inodes. */ +bool pkgCdrom::DropRepeats(vector<string> &List,const char *Name) +{ + bool couldFindAllFiles = true; + // Get a list of all the inodes + ino_t *Inodes = new ino_t[List.size()]; + for (unsigned int I = 0; I != List.size(); ++I) + { + struct stat Buf; + bool found = false; + + std::vector<APT::Configuration::Compressor> const compressor = APT::Configuration::getCompressors(); + for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) + { + std::string const filename = List[I] + Name + c->Extension; + if (stat(filename.c_str(), &Buf) != 0) + continue; + Inodes[I] = Buf.st_ino; + found = true; + break; + } + + if (found == false) + { + _error->Errno("stat","Failed to stat %s%s",List[I].c_str(), Name); + couldFindAllFiles = false; + Inodes[I] = 0; + } + } + + // Look for dups + for (unsigned int I = 0; I != List.size(); I++) + { + if (Inodes[I] == 0) + continue; + for (unsigned int J = I+1; J < List.size(); J++) + { + // No match + if (Inodes[J] == 0 || Inodes[J] != Inodes[I]) + continue; + + // We score the two paths.. and erase one + int ScoreA = Score(List[I]); + int ScoreB = Score(List[J]); + if (ScoreA < ScoreB) + { + List[I] = string(); + break; + } + + List[J] = string(); + } + } + delete[] Inodes; + + // Wipe erased entries + for (unsigned int I = 0; I < List.size();) + { + if (List[I].empty() == false) + I++; + else + List.erase(List.begin()+I); + } + + return couldFindAllFiles; +} + /*}}}*/ +// ReduceSourceList - Takes the path list and reduces it /*{{{*/ +// --------------------------------------------------------------------- +/* This takes the list of source list expressed entries and collects + similar ones to form a single entry for each dist */ +void pkgCdrom::ReduceSourcelist(string /*CD*/,vector<string> &List) +{ + sort(List.begin(),List.end()); + + // Collect similar entries + for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) + { + // Find a space.. + string::size_type Space = (*I).find(' '); + if (Space == string::npos) + continue; + string::size_type SSpace = (*I).find(' ',Space + 1); + if (SSpace == string::npos) + continue; + + string Word1 = string(*I,Space,SSpace-Space); + string Prefix = string(*I,0,Space); + string Component = string(*I,SSpace); + for (vector<string>::iterator J = List.begin(); J != I; ++J) + { + // Find a space.. + string::size_type Space2 = (*J).find(' '); + if (Space2 == string::npos) + continue; + string::size_type SSpace2 = (*J).find(' ',Space2 + 1); + if (SSpace2 == string::npos) + continue; + + if (string(*J,0,Space2) != Prefix) + continue; + if (string(*J,Space2,SSpace2-Space2) != Word1) + continue; + + string Component2 = string(*J, SSpace2) + " "; + if (Component2.find(Component + " ") == std::string::npos) + *J += Component; + I->clear(); + } + } + + // Wipe erased entries + for (unsigned int I = 0; I < List.size();) + { + if (List[I].empty() == false) + I++; + else + List.erase(List.begin()+I); + } +} + /*}}}*/ +// WriteDatabase - Write the CDROM Database file /*{{{*/ +// --------------------------------------------------------------------- +/* We rewrite the configuration class associated with the cdrom database. */ +bool pkgCdrom::WriteDatabase(Configuration &Cnf) +{ + string DFile = _config->FindFile("Dir::State::cdroms"); + string NewFile = DFile + ".new"; + + RemoveFile("WriteDatabase", NewFile); + ofstream Out(NewFile.c_str()); + if (!Out) + return _error->Errno("ofstream::ofstream", + "Failed to open %s.new",DFile.c_str()); + + /* Write out all of the configuration directives by walking the + configuration tree */ + Cnf.Dump(Out, NULL, "%F \"%v\";\n", false); + + Out.close(); + + if (FileExists(DFile) == true) + rename(DFile.c_str(), (DFile + '~').c_str()); + if (rename(NewFile.c_str(),DFile.c_str()) != 0) + return _error->Errno("rename","Failed to rename %s.new to %s", + DFile.c_str(),DFile.c_str()); + + return true; +} + /*}}}*/ +// WriteSourceList - Write an updated sourcelist /*{{{*/ +// --------------------------------------------------------------------- +/* This reads the old source list and copies it into the new one. It + appends the new CDROM entries just after the first block of comments. + This places them first in the file. It also removes any old entries + that were the same. */ +bool pkgCdrom::WriteSourceList(string Name,vector<string> &List,bool Source) +{ + if (List.empty() == true) + return true; + + string File = _config->FindFile("Dir::Etc::sourcelist"); + + FileFd F(FileExists(File) ? File : "/dev/null", FileFd::ReadOnly); + if (not F.IsOpen() || F.Failed()) + return _error->Errno("WriteSourceList", "Opening %s failed", File.c_str()); + + string NewFile = File + ".new"; + RemoveFile("WriteSourceList", NewFile); + ofstream Out(NewFile.c_str()); + if (!Out) + return _error->Errno("ofstream::ofstream", + "Failed to open %s.new",File.c_str()); + + // Create a short uri without the path + string ShortURI = "cdrom:[" + Name + "]/"; + string ShortURI2 = "cdrom:" + Name + "/"; // For Compatibility + + string Type; + if (Source == true) + Type = "deb-src"; + else + Type = "deb"; + + int CurLine = 0; + bool First = true; + std::string Buffer; + while (F.ReadLine(Buffer)) + { + ++CurLine; + auto const Cleaned = APT::String::Strip(SubstVar(Buffer, "\t", " ")); + + // Comment or blank + if (Cleaned.empty() || Cleaned[0] == '#') + { + Out << Buffer << endl; + continue; + } + + if (First == true) + { + for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) + { + string::size_type Space = (*I).find(' '); + if (Space == string::npos) + return _error->Error("Internal error"); + Out << Type << " cdrom:[" << Name << "]/" << string(*I,0,Space) << + " " << string(*I,Space+1) << endl; + } + } + First = false; + + // Grok it + string cType; + string URI; + const char *C = Cleaned.c_str(); + if (ParseQuoteWord(C,cType) == false || + ParseQuoteWord(C,URI) == false) + { + Out << Buffer << endl; + continue; + } + + // Emit lines like this one + if (cType != Type || (string(URI,0,ShortURI.length()) != ShortURI && + string(URI,0,ShortURI.length()) != ShortURI2)) + { + Out << Buffer << endl; + continue; + } + } + + // Just in case the file was empty + if (First == true) + { + for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) + { + string::size_type Space = (*I).find(' '); + if (Space == string::npos) + return _error->Error("Internal error"); + + Out << "deb cdrom:[" << Name << "]/" << string(*I,0,Space) << + " " << string(*I,Space+1) << endl; + } + } + + Out.close(); + + rename(File.c_str(), (File + '~').c_str()); + if (rename(NewFile.c_str(),File.c_str()) != 0) + return _error->Errno("rename","Failed to rename %s.new to %s", + File.c_str(),File.c_str()); + + return true; +} + /*}}}*/ +bool pkgCdrom::UnmountCDROM(std::string const &CDROM, pkgCdromStatus * const log)/*{{{*/ +{ + if (_config->FindB("APT::CDROM::NoMount",false) == true) + return true; + if (log != NULL) + log->Update(_("Unmounting CD-ROM...\n"), STEP_LAST); + return UnmountCdrom(CDROM); +} + /*}}}*/ +bool pkgCdrom::MountAndIdentCDROM(Configuration &Database, std::string &CDROM, std::string &ident, pkgCdromStatus * const log, bool const interactive)/*{{{*/ +{ + // Startup + CDROM = _config->FindDir("Acquire::cdrom::mount"); + if (CDROM[0] == '.') + CDROM= SafeGetCWD() + '/' + CDROM; + + if (log != NULL) + { + string msg; + log->SetTotal(STEP_LAST); + strprintf(msg, _("Using CD-ROM mount point %s\n"), CDROM.c_str()); + log->Update(msg, STEP_PREPARE); + } + + // Unmount the CD and get the user to put in the one they want + if (_config->FindB("APT::CDROM::NoMount", false) == false) + { + if (interactive == true) + { + UnmountCDROM(CDROM, log); + + if(log != NULL) + { + log->Update(_("Waiting for disc...\n"), STEP_WAIT); + if(!log->ChangeCdrom()) { + // user aborted + return false; + } + } + } + + // Mount the new CDROM + if(log != NULL) + log->Update(_("Mounting CD-ROM...\n"), STEP_MOUNT); + + if (MountCdrom(CDROM) == false) + return _error->Error("Failed to mount the cdrom."); + } + + if (IsMounted(CDROM) == false) + return _error->Error("Failed to mount the cdrom."); + + // Hash the CD to get an ID + if (log != NULL) + log->Update(_("Identifying... "), STEP_IDENT); + + if (IdentCdrom(CDROM,ident) == false) + { + ident = ""; + if (log != NULL) + log->Update("\n"); + UnmountCDROM(CDROM, NULL); + return false; + } + + if (log != NULL) + { + string msg; + strprintf(msg, "[%s]\n", ident.c_str()); + log->Update(msg); + } + + // Read the database + string DFile = _config->FindFile("Dir::State::cdroms"); + if (FileExists(DFile) == true) + { + if (ReadConfigFile(Database,DFile) == false) + { + UnmountCDROM(CDROM, NULL); + return _error->Error("Unable to read the cdrom database %s", + DFile.c_str()); + } + } + return true; +} + /*}}}*/ +bool pkgCdrom::Ident(string &ident, pkgCdromStatus *log) /*{{{*/ +{ + Configuration Database; + std::string CDROM; + if (MountAndIdentCDROM(Database, CDROM, ident, log, false) == false) + return false; + + if (log != NULL) + { + string msg; + strprintf(msg, _("Stored label: %s\n"), + Database.Find("CD::"+ident).c_str()); + log->Update(msg); + } + + // Unmount and finish + UnmountCDROM(CDROM, log); + return true; +} + /*}}}*/ +bool pkgCdrom::Add(pkgCdromStatus *log) /*{{{*/ +{ + Configuration Database; + std::string ID, CDROM; + if (MountAndIdentCDROM(Database, CDROM, ID, log, true) == false) + return false; + + if(log != NULL) + log->Update(_("Scanning disc for index files...\n"),STEP_SCAN); + + // Get the CD structure + vector<string> List; + vector<string> SourceList; + vector<string> SigList; + vector<string> TransList; + string StartDir = SafeGetCWD(); + string InfoDir; + if (FindPackages(CDROM,List,SourceList, SigList,TransList,InfoDir,log) == false) + { + if (log != NULL) + log->Update("\n"); + UnmountCDROM(CDROM, NULL); + return false; + } + + if (chdir(StartDir.c_str()) != 0) + { + UnmountCDROM(CDROM, NULL); + return _error->Errno("chdir","Unable to change to %s", StartDir.c_str()); + } + + if (_config->FindB("Debug::aptcdrom",false) == true) + { + cout << "I found (binary):" << endl; + for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) + cout << *I << endl; + cout << "I found (source):" << endl; + for (vector<string>::iterator I = SourceList.begin(); I != SourceList.end(); ++I) + cout << *I << endl; + cout << "I found (Signatures):" << endl; + for (vector<string>::iterator I = SigList.begin(); I != SigList.end(); ++I) + cout << *I << endl; + } + + //log->Update(_("Cleaning package lists..."), STEP_CLEAN); + + // Fix up the list + DropBinaryArch(List); + DropRepeats(List,"Packages"); + DropRepeats(SourceList,"Sources"); + // FIXME: We ignore stat() errors here as we usually have only one of those in use + // This has little potential to drop 'valid' stat() errors as we know that one of these + // files need to exist, but it would be better if we would check it here + _error->PushToStack(); + DropRepeats(SigList,"Release.gpg"); + DropRepeats(SigList,"InRelease"); + _error->RevertToStack(); + DropRepeats(TransList,""); + if (_config->FindB("APT::CDROM::DropTranslation", true) == true) + DropTranslation(TransList); + if(log != NULL) { + string msg; + strprintf(msg, _("Found %zu package indexes, %zu source indexes, " + "%zu translation indexes and %zu signatures\n"), + List.size(), SourceList.size(), TransList.size(), + SigList.size()); + log->Update(msg, STEP_SCAN); + } + + if (List.empty() == true && SourceList.empty() == true) + { + UnmountCDROM(CDROM, NULL); + return _error->Error(_("Unable to locate any package files, perhaps this is not a Debian Disc or the wrong architecture?")); + } + + // Check if the CD is in the database + string Name; + if (Database.Exists("CD::" + ID) == false || + _config->FindB("APT::CDROM::Rename",false) == true) + { + // Try to use the CDs label if at all possible + if (InfoDir.empty() == false && + FileExists(InfoDir + "/info") == true) + { + ifstream F((InfoDir + "/info").c_str()); + if (F.good() == true) + getline(F,Name); + + if (Name.empty() == false) + { + // Escape special characters + string::iterator J = Name.begin(); + for (; J != Name.end(); ++J) + if (*J == '"' || *J == ']' || *J == '[') + *J = '_'; + + if(log != NULL) + { + string msg; + strprintf(msg, _("Found label '%s'\n"), Name.c_str()); + log->Update(msg); + } + Database.Set("CD::" + ID + "::Label",Name); + } + } + + if (_config->FindB("APT::CDROM::Rename",false) == true || + Name.empty() == true) + { + if(log == NULL) + { + UnmountCDROM(CDROM, NULL); + return _error->Error("No disc name found and no way to ask for it"); + } + + while(true) { + if(!log->AskCdromName(Name)) { + // user canceld + UnmountCDROM(CDROM, NULL); + return false; + } + cout << "Name: '" << Name << "'" << endl; + + if (Name.empty() == false && + Name.find('"') == string::npos && + Name.find('[') == string::npos && + Name.find(']') == string::npos) + break; + log->Update(_("That is not a valid name, try again.\n")); + } + } + } + else + Name = Database.Find("CD::" + ID); + + // Escape special characters + string::iterator J = Name.begin(); + for (; J != Name.end(); ++J) + if (*J == '"' || *J == ']' || *J == '[') + *J = '_'; + + Database.Set("CD::" + ID,Name); + if(log != NULL) + { + string msg; + strprintf(msg, _("This disc is called: \n'%s'\n"), Name.c_str()); + log->Update(msg); + log->Update(_("Copying package lists..."), STEP_COPY); + } + + // check for existence and possibly create state directory for copying + string const listDir = _config->FindDir("Dir::State::lists"); + string const partialListDir = listDir + "partial/"; + mode_t const mode = umask(S_IWGRP | S_IWOTH); + bool const creation_fail = (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::State"), partialListDir) == false && + CreateAPTDirectoryIfNeeded(listDir, partialListDir) == false); + umask(mode); + if (creation_fail == true) + { + UnmountCDROM(CDROM, NULL); + return _error->Errno("cdrom", _("List directory %s is missing."), (listDir + "partial").c_str()); + } + + // take care of the signatures and copy them if they are ok + // (we do this before PackageCopy as it modifies "List" and "SourceList") + SigVerify SignVerify; + SignVerify.CopyAndVerify(CDROM, Name, SigList, List, SourceList); + + // Copy the package files to the state directory + PackageCopy Copy; + SourceCopy SrcCopy; + TranslationsCopy TransCopy; + if (Copy.CopyPackages(CDROM,Name,List, log) == false || + SrcCopy.CopyPackages(CDROM,Name,SourceList, log) == false || + TransCopy.CopyTranslations(CDROM,Name,TransList, log) == false) + { + UnmountCDROM(CDROM, NULL); + return false; + } + + // reduce the List so that it takes less space in sources.list + ReduceSourcelist(CDROM,List); + ReduceSourcelist(CDROM,SourceList); + + // Write the database and sourcelist + if (_config->FindB("APT::cdrom::NoAct",false) == false) + { + if (WriteDatabase(Database) == false) + { + UnmountCDROM(CDROM, NULL); + return false; + } + + if(log != NULL) + log->Update(_("Writing new source list\n"), STEP_WRITE); + if (WriteSourceList(Name,List,false) == false || + WriteSourceList(Name,SourceList,true) == false) + { + UnmountCDROM(CDROM, NULL); + return false; + } + } + + // Print the sourcelist entries + if(log != NULL) + log->Update(_("Source list entries for this disc are:\n")); + + for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) + { + string::size_type Space = (*I).find(' '); + if (Space == string::npos) + { + UnmountCDROM(CDROM, NULL); + return _error->Error("Internal error"); + } + + if(log != NULL) + { + stringstream msg; + msg << "deb cdrom:[" << Name << "]/" << string(*I,0,Space) << + " " << string(*I,Space+1) << endl; + log->Update(msg.str()); + } + } + + for (vector<string>::iterator I = SourceList.begin(); I != SourceList.end(); ++I) + { + string::size_type Space = (*I).find(' '); + if (Space == string::npos) + { + UnmountCDROM(CDROM, NULL); + return _error->Error("Internal error"); + } + + if(log != NULL) { + stringstream msg; + msg << "deb-src cdrom:[" << Name << "]/" << string(*I,0,Space) << + " " << string(*I,Space+1) << endl; + log->Update(msg.str()); + } + } + + // Unmount and finish + UnmountCDROM(CDROM, log); + return true; +} + /*}}}*/ + +pkgUdevCdromDevices::pkgUdevCdromDevices() /*{{{*/ + : d(NULL) +{ +} + /*}}}*/ +// convenience interface, this will just call ScanForRemovable /*{{{*/ +vector<CdromDevice> pkgUdevCdromDevices::Scan() +{ + bool CdromOnly = _config->FindB("APT::cdrom::CdromOnly", true); + return ScanForRemovable(CdromOnly); +} + /*}}}*/ +vector<CdromDevice> pkgUdevCdromDevices::ScanForRemovable(bool CdromOnly)/*{{{*/ +{ + vector<CdromDevice> cdrom_devices; +#ifdef HAVE_UDEV + struct udev_enumerate *enumerate; + struct udev_list_entry *l, *devices; + struct udev *udev_ctx; + + udev_ctx = udev_new(); + enumerate = udev_enumerate_new (udev_ctx); + if (CdromOnly) + udev_enumerate_add_match_property(enumerate, "ID_CDROM", "1"); + else { + udev_enumerate_add_match_sysattr(enumerate, "removable", "1"); + } + + udev_enumerate_scan_devices (enumerate); + devices = udev_enumerate_get_list_entry (enumerate); + for (l = devices; l != NULL; l = udev_list_entry_get_next (l)) + { + CdromDevice cdrom; + struct udev_device *udevice; + udevice = udev_device_new_from_syspath (udev_enumerate_get_udev (enumerate), udev_list_entry_get_name (l)); + if (udevice == NULL) + continue; + const char* devnode = udev_device_get_devnode(udevice); + + // try fstab_dir first + string mountpath; + const char* mp = udev_device_get_property_value(udevice, "FSTAB_DIR"); + if (mp) + mountpath = string(mp); + else + mountpath = FindMountPointForDevice(devnode); + + // fill in the struct + cdrom.DeviceName = string(devnode); + if (mountpath != "") { + cdrom.MountPath = mountpath; + string s = mountpath; + cdrom.Mounted = IsMounted(s); + } else { + cdrom.Mounted = false; + cdrom.MountPath = ""; + } + cdrom_devices.push_back(cdrom); + } +#endif + return cdrom_devices; +} + /*}}}*/ + +pkgUdevCdromDevices::~pkgUdevCdromDevices() /*{{{*/ +{ +} + /*}}}*/ + +pkgCdromStatus::pkgCdromStatus() : d(NULL), totalSteps(0) {} +pkgCdromStatus::~pkgCdromStatus() {} + +pkgCdrom::pkgCdrom() : d(NULL) {} +pkgCdrom::~pkgCdrom() {} diff --git a/apt-pkg/cdrom.h b/apt-pkg/cdrom.h new file mode 100644 index 0000000..d3fc771 --- /dev/null +++ b/apt-pkg/cdrom.h @@ -0,0 +1,110 @@ +#ifndef PKGLIB_CDROM_H +#define PKGLIB_CDROM_H + +#include <apt-pkg/macros.h> + +#include <string> +#include <vector> + +#include <stddef.h> + + +class Configuration; +class OpProgress; + +class APT_PUBLIC pkgCdromStatus /*{{{*/ +{ + void * const d; + protected: + int totalSteps; + + public: + pkgCdromStatus(); + virtual ~pkgCdromStatus(); + + // total steps + virtual void SetTotal(int total) { totalSteps = total; }; + // update steps, will be called regularly as a "pulse" + virtual void Update(std::string text="", int current=0) = 0; + + // ask for cdrom insert + virtual bool ChangeCdrom() = 0; + // ask for cdrom name + virtual bool AskCdromName(std::string &Name) = 0; + // Progress indicator for the Index rewriter + virtual OpProgress* GetOpProgress() {return NULL; }; +}; + /*}}}*/ +class APT_PUBLIC pkgCdrom /*{{{*/ +{ + protected: + enum { + STEP_PREPARE = 1, + STEP_UNMOUNT, + STEP_WAIT, + STEP_MOUNT, + STEP_IDENT, + STEP_SCAN, + STEP_COPY, + STEP_WRITE, + STEP_UNMOUNT3, + STEP_LAST + }; + + + bool FindPackages(std::string CD, + std::vector<std::string> &List, + std::vector<std::string> &SList, + std::vector<std::string> &SigList, + std::vector<std::string> &TransList, + std::string &InfoDir, pkgCdromStatus *log, + unsigned int Depth = 0); + bool DropBinaryArch(std::vector<std::string> &List); + bool DropRepeats(std::vector<std::string> &List,const char *Name); + bool DropTranslation(std::vector<std::string> &List); + void ReduceSourcelist(std::string CD,std::vector<std::string> &List); + bool WriteDatabase(Configuration &Cnf); + bool WriteSourceList(std::string Name,std::vector<std::string> &List,bool Source); + int Score(std::string Path); + + public: + bool Ident(std::string &ident, pkgCdromStatus *log); + bool Add(pkgCdromStatus *log); + + pkgCdrom(); + virtual ~pkgCdrom(); + + private: + void * const d; + + APT_HIDDEN bool MountAndIdentCDROM(Configuration &Database, std::string &CDROM, + std::string &ident, pkgCdromStatus * const log, bool const interactive); + APT_HIDDEN bool UnmountCDROM(std::string const &CDROM, pkgCdromStatus * const log); +}; + /*}}}*/ + + +// class that uses libudev to find cdrom/removable devices dynamically +struct APT_PUBLIC CdromDevice /*{{{*/ +{ + std::string DeviceName; + bool Mounted; + std::string MountPath; +}; + /*}}}*/ +class APT_PUBLIC pkgUdevCdromDevices /*{{{*/ +{ + void * const d; + public: + pkgUdevCdromDevices(); + virtual ~pkgUdevCdromDevices(); + + // convenience interface, this will just call ScanForRemovable + // with "APT::cdrom::CdromOnly" + std::vector<CdromDevice> Scan(); + + std::vector<CdromDevice> ScanForRemovable(bool CdromOnly); +}; + /*}}}*/ + +#endif diff --git a/apt-pkg/clean.cc b/apt-pkg/clean.cc new file mode 100644 index 0000000..9dd56e6 --- /dev/null +++ b/apt-pkg/clean.cc @@ -0,0 +1,134 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Clean - Clean out downloaded directories + + ##################################################################### */ + /*}}}*/ +// Includes /*{{{*/ +#include <config.h> + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/clean.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/strutl.h> + +#include <string> +#include <dirent.h> +#include <fcntl.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ +// ArchiveCleaner::Go - Perform smart cleanup of the archive /*{{{*/ +// --------------------------------------------------------------------- +/* Scan the directory for files to erase, we check the version information + against our database to see if it is interesting */ +bool pkgArchiveCleaner::Go(std::string Dir,pkgCache &Cache) +{ + bool CleanInstalled = _config->FindB("APT::Clean-Installed",true); + + if(Dir == "/") + return _error->Error(_("Clean of %s is not supported"), Dir.c_str()); + + // non-existing directories are always clean + // we do not check for a directory explicitly to support symlinks + if (FileExists(Dir) == false) + return true; + + int const dirfd = open(Dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (dirfd == -1) + return _error->Errno("open",_("Unable to read %s"),Dir.c_str()); + DIR * const D = fdopendir(dirfd); + if (D == nullptr) + return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + + for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + { + // Skip some files.. + if (strcmp(Dir->d_name, "lock") == 0 || + strcmp(Dir->d_name, "partial") == 0 || + strcmp(Dir->d_name, "auxfiles") == 0 || + strcmp(Dir->d_name, "lost+found") == 0 || + strcmp(Dir->d_name, ".") == 0 || + strcmp(Dir->d_name, "..") == 0) + continue; + + struct stat St; + if (fstatat(dirfd, Dir->d_name,&St, 0) != 0) + { + _error->Errno("stat",_("Unable to stat %s."),Dir->d_name); + closedir(D); + return false; + } + + // Grab the package name + const char *I = Dir->d_name; + for (; *I != 0 && *I != '_';I++); + if (*I != '_') + continue; + std::string Pkg = DeQuoteString(std::string(Dir->d_name,I-Dir->d_name)); + + // Grab the version + const char *Start = I + 1; + for (I = Start; *I != 0 && *I != '_';I++); + if (*I != '_') + continue; + std::string Ver = DeQuoteString(std::string(Start,I-Start)); + + // Grab the arch + Start = I + 1; + for (I = Start; *I != 0 && *I != '.' ;I++); + if (*I != '.') + continue; + std::string const Arch = DeQuoteString(std::string(Start,I-Start)); + + // ignore packages of unconfigured architectures + if (APT::Configuration::checkArchitecture(Arch) == false) + continue; + + // Lookup the package + pkgCache::PkgIterator P = Cache.FindPkg(Pkg, Arch); + if (P.end() != true) + { + pkgCache::VerIterator V = P.VersionList(); + for (; V.end() == false; ++V) + { + // See if we can fetch this version at all + bool IsFetchable = false; + for (pkgCache::VerFileIterator J = V.FileList(); + J.end() == false; ++J) + { + if (CleanInstalled == true && + J.File().Flagged(pkgCache::Flag::NotSource)) + continue; + IsFetchable = true; + break; + } + + // See if this version matches the file + if (IsFetchable == true && Ver == V.VerStr()) + break; + } + + // We found a match, keep the file + if (V.end() == false) + continue; + } + + Erase(dirfd, Dir->d_name, Pkg, Ver, St); + } + closedir(D); + return true; +} + /*}}}*/ + +pkgArchiveCleaner::pkgArchiveCleaner() : d(NULL) {} +pkgArchiveCleaner::~pkgArchiveCleaner() {} diff --git a/apt-pkg/clean.h b/apt-pkg/clean.h new file mode 100644 index 0000000..21b90e5 --- /dev/null +++ b/apt-pkg/clean.h @@ -0,0 +1,38 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Clean - Clean out downloaded directories + + ##################################################################### */ + /*}}}*/ +#ifndef APTPKG_CLEAN_H +#define APTPKG_CLEAN_H + + +#include <string> + +#include <apt-pkg/macros.h> + +class pkgCache; + +class APT_PUBLIC pkgArchiveCleaner +{ + /** \brief dpointer placeholder (for later in case we need it) */ + void * const d; + + protected: + virtual void Erase(int const dirfd, char const * const File, + std::string const &Pkg,std::string const &Ver, + struct stat const &St) = 0; + + public: + + bool Go(std::string Dir,pkgCache &Cache); + + pkgArchiveCleaner(); + virtual ~pkgArchiveCleaner(); +}; + + +#endif diff --git a/apt-pkg/contrib/arfile.cc b/apt-pkg/contrib/arfile.cc new file mode 100644 index 0000000..6d4a1f1 --- /dev/null +++ b/apt-pkg/contrib/arfile.cc @@ -0,0 +1,179 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + AR File - Handle an 'AR' archive + + AR Archives have plain text headers at the start of each file + section. The headers are aligned on a 2 byte boundary. + + Information about the structure of AR files can be found in ar(5) + on a BSD system, or in the binutils source. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/arfile.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/strutl.h> + +#include <string> +#include <string.h> +#include <sys/types.h> + +#include <apti18n.h> + /*}}}*/ + +struct ARArchive::MemberHeader +{ + char Name[16]; + char MTime[12]; + char UID[6]; + char GID[6]; + char Mode[8]; + char Size[10]; + char Magic[2]; +}; + +// ARArchive::ARArchive - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +ARArchive::ARArchive(FileFd &File) : List(0), File(File) +{ + LoadHeaders(); +} + /*}}}*/ +// ARArchive::~ARArchive - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +ARArchive::~ARArchive() +{ + while (List != 0) + { + Member *Tmp = List; + List = List->Next; + delete Tmp; + } +} + /*}}}*/ +// ARArchive::LoadHeaders - Load the headers from each file /*{{{*/ +// --------------------------------------------------------------------- +/* AR files are structured with a 8 byte magic string followed by a 60 + byte plain text header then the file data, another header, data, etc */ +bool ARArchive::LoadHeaders() +{ + off_t Left = File.Size(); + + // Check the magic byte + char Magic[8]; + if (File.Read(Magic,sizeof(Magic)) == false) + return false; + if (memcmp(Magic,"!<arch>\012",sizeof(Magic)) != 0) + return _error->Error(_("Invalid archive signature")); + Left -= sizeof(Magic); + + // Read the member list + while (Left > 0) + { + MemberHeader Head; + if (File.Read(&Head,sizeof(Head)) == false) + return _error->Error(_("Error reading archive member header")); + Left -= sizeof(Head); + + // Convert all of the integer members + Member *Memb = new Member(); + if (StrToNum(Head.MTime,Memb->MTime,sizeof(Head.MTime)) == false || + StrToNum(Head.UID,Memb->UID,sizeof(Head.UID)) == false || + StrToNum(Head.GID,Memb->GID,sizeof(Head.GID)) == false || + StrToNum(Head.Mode,Memb->Mode,sizeof(Head.Mode),8) == false || + StrToNum(Head.Size,Memb->Size,sizeof(Head.Size)) == false) + { + delete Memb; + return _error->Error(_("Invalid archive member header")); + } + + if (Left < 0 || Memb->Size > static_cast<unsigned long long>(Left)) + { + delete Memb; + return _error->Error(_("Invalid archive member header")); + } + // Check for an extra long name string + if (memcmp(Head.Name,"#1/",3) == 0) + { + char S[300]; + unsigned long Len; + if (StrToNum(Head.Name+3,Len,sizeof(Head.Size)-3) == false || + Len >= sizeof(S)) + { + delete Memb; + return _error->Error(_("Invalid archive member header")); + } + + if (Len > Memb->Size) + { + delete Memb; + return _error->Error(_("Invalid archive member header")); + } + + if (File.Read(S,Len) == false) + { + delete Memb; + return false; + } + S[Len] = 0; + Memb->Name = S; + Memb->Size -= Len; + Left -= Len; + } + else + { + unsigned int I = sizeof(Head.Name) - 1; + for (; Head.Name[I] == ' ' || Head.Name[I] == '/'; I--) + { + if (I == 0) + { + delete Memb; + return _error->Error(_("Invalid archive member header")); + } + } + Memb->Name = std::string(Head.Name,I+1); + } + + // Account for the AR header alignment + off_t Skip = Memb->Size % 2; + + // Add it to the list + Memb->Next = List; + List = Memb; + Memb->Start = File.Tell(); + if (File.Skip(Memb->Size + Skip) == false) + return false; + if (Left < (off_t)(Memb->Size + Skip)) + return _error->Error(_("Archive is too short")); + Left -= Memb->Size + Skip; + } + if (Left != 0) + return _error->Error(_("Failed to read the archive headers")); + + return true; +} + /*}}}*/ +// ARArchive::FindMember - Find a name in the member list /*{{{*/ +// --------------------------------------------------------------------- +/* Find a member with the given name */ +const ARArchive::Member *ARArchive::FindMember(const char *Name) const +{ + const Member *Res = List; + while (Res != 0) + { + if (Res->Name == Name) + return Res; + Res = Res->Next; + } + + return 0; +} + /*}}}*/ diff --git a/apt-pkg/contrib/arfile.h b/apt-pkg/contrib/arfile.h new file mode 100644 index 0000000..9b13ed9 --- /dev/null +++ b/apt-pkg/contrib/arfile.h @@ -0,0 +1,66 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + AR File - Handle an 'AR' archive + + This is a reader for the usual 4.4 BSD AR format. It allows raw + stream access to a single member at a time. Basically all this class + provides is header parsing and verification. It is up to the client + to correctly make use of the stream start/stop points. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_ARFILE_H +#define PKGLIB_ARFILE_H + +#include <apt-pkg/macros.h> +#include <string> + +class FileFd; + +class APT_PUBLIC ARArchive +{ + struct MemberHeader; + public: + struct Member; + + protected: + + // Linked list of members + Member *List; + + bool LoadHeaders(); + + public: + + // The stream file + FileFd &File; + + // Locate a member by name + const Member *FindMember(const char *Name) const; + inline Member *Members() { return List; } + + APT_PUBLIC explicit ARArchive(FileFd &File); + APT_PUBLIC ~ARArchive(); +}; + +// A member of the archive +struct ARArchive::Member +{ + // Fields from the header + std::string Name; + unsigned long MTime; + unsigned long UID; + unsigned long GID; + unsigned long Mode; + unsigned long long Size; + + // Location of the data. + unsigned long long Start; + Member *Next; + + Member() : Start(0), Next(0) {}; +}; + +#endif diff --git a/apt-pkg/contrib/cdromutl.cc b/apt-pkg/contrib/cdromutl.cc new file mode 100644 index 0000000..f402017 --- /dev/null +++ b/apt-pkg/contrib/cdromutl.cc @@ -0,0 +1,293 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + CDROM Utilities - Some functions to manipulate CDROM mounts. + + These are here for the cdrom method and apt-cdrom. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/cdromutl.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/strutl.h> + +#include <iostream> +#include <string> +#include <vector> +#include <dirent.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ + +using std::string; + +// IsMounted - Returns true if the mount point is mounted /*{{{*/ +// --------------------------------------------------------------------- +/* This is a simple algorithm that should always work, we stat the mount point + and the '..' file in the mount point and see if they are on the same device. + By definition if they are the same then it is not mounted. This should + account for symlinked mount points as well. */ +bool IsMounted(string &Path) +{ + if (Path.empty() == true) + return false; + + // Need that trailing slash for directories + if (Path[Path.length() - 1] != '/') + Path += '/'; + + // if the path has a ".disk" directory we treat it as mounted + // this way even extracted copies of disks are recognized + if (DirectoryExists(Path + ".disk/") == true) + return true; + + /* First we check if the path is actually mounted, we do this by + stating the path and the previous directory (careful of links!) + and comparing their device fields. */ + struct stat Buf,Buf2; + if (stat(Path.c_str(),&Buf) != 0 || + stat((Path + "../").c_str(),&Buf2) != 0) + return _error->Errno("stat",_("Unable to stat the mount point %s"),Path.c_str()); + + if (Buf.st_dev == Buf2.st_dev) + return false; + return true; +} + /*}}}*/ +// UnmountCdrom - Unmount a cdrom /*{{{*/ +// --------------------------------------------------------------------- +/* Forking umount works much better than the umount syscall which can + leave /etc/mtab inconsistent. We drop all messages this produces. */ +bool UnmountCdrom(string Path) +{ + // do not generate errors, even if the mountpoint does not exist + // the mountpoint might be auto-created by the mount command + // and a non-existing mountpoint is surely not mounted + _error->PushToStack(); + bool const mounted = IsMounted(Path); + _error->RevertToStack(); + if (mounted == false) + return true; + + for (int i=0;i<3;i++) + { + + int Child = ExecFork(); + + // The child + if (Child == 0) + { + // Make all the fds /dev/null + int const null_fd = open("/dev/null",O_RDWR); + for (int I = 0; I != 3; ++I) + dup2(null_fd, I); + + if (_config->Exists("Acquire::cdrom::"+Path+"::UMount") == true) + { + if (system(_config->Find("Acquire::cdrom::"+Path+"::UMount").c_str()) != 0) + _exit(100); + _exit(0); + } + else + { + const char * const Args[] = { + "umount", + Path.c_str(), + nullptr + }; + execvp(Args[0], const_cast<char **>(Args)); + _exit(100); + } + } + + // if it can not be umounted, give it a bit more time + // this can happen when auto-mount magic or fs/cdrom prober attack + if (ExecWait(Child,"umount",true) == true) + return true; + sleep(1); + } + + return false; +} + /*}}}*/ +// MountCdrom - Mount a cdrom /*{{{*/ +// --------------------------------------------------------------------- +/* We fork mount and drop all messages */ +bool MountCdrom(string Path, string DeviceName) +{ + // do not generate errors, even if the mountpoint does not exist + // the mountpoint might be auto-created by the mount command + _error->PushToStack(); + bool const mounted = IsMounted(Path); + _error->RevertToStack(); + if (mounted == true) + return true; + + int Child = ExecFork(); + + // The child + if (Child == 0) + { + // Make all the fds /dev/null + int const null_fd = open("/dev/null",O_RDWR); + for (int I = 0; I != 3; ++I) + dup2(null_fd, I); + + if (_config->Exists("Acquire::cdrom::"+Path+"::Mount") == true) + { + if (system(_config->Find("Acquire::cdrom::"+Path+"::Mount").c_str()) != 0) + _exit(100); + _exit(0); + } + else + { + const char *Args[10]; + Args[0] = "mount"; + if (DeviceName == "") + { + Args[1] = Path.c_str(); + Args[2] = 0; + } else { + Args[1] = DeviceName.c_str(); + Args[2] = Path.c_str(); + Args[3] = 0; + } + execvp(Args[0],(char **)Args); + _exit(100); + } + } + + // Wait for mount + return ExecWait(Child,"mount",true); +} + /*}}}*/ +// IdentCdrom - Generate a unique string for this CD /*{{{*/ +// --------------------------------------------------------------------- +/* We convert everything we hash into a string, this prevents byte size/order + from effecting the outcome. */ +bool IdentCdrom(string CD,string &Res,unsigned int Version) +{ + Hashes Hash(Hashes::MD5SUM); + bool writable_media = false; + + int dirfd = open(CD.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (dirfd == -1) + return _error->Errno("open",_("Unable to read %s"),CD.c_str()); + + // if we are on a writable medium (like a usb-stick) that is just + // used like a cdrom don't use "." as it will constantly change, + // use .disk instead + if (faccessat(dirfd, ".", W_OK, 0) == 0) + { + int diskfd = openat(dirfd, "./.disk", O_RDONLY | O_DIRECTORY | O_CLOEXEC, 0); + if (diskfd != -1) + { + close(dirfd); + dirfd = diskfd; + writable_media = true; + CD = CD.append("/.disk"); + if (_config->FindB("Debug::aptcdrom",false) == true) + std::clog << "Found writable cdrom, using alternative path: " << CD + << std::endl; + } + } + + DIR * const D = fdopendir(dirfd); + if (D == nullptr) + return _error->Errno("opendir",_("Unable to read %s"),CD.c_str()); + + /* Run over the directory, we assume that the reader order will never + change as the media is read-only. In theory if the kernel did + some sort of wacked caching this might not be true.. */ + for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + { + // Skip some files.. + if (strcmp(Dir->d_name,".") == 0 || + strcmp(Dir->d_name,"..") == 0) + continue; + + std::string S; + if (Version <= 1) + S = std::to_string(Dir->d_ino); + else + { + struct stat Buf; + if (fstatat(dirfd, Dir->d_name, &Buf, 0) != 0) + continue; + S = std::to_string(Buf.st_mtime); + } + + Hash.Add(S.c_str()); + Hash.Add(Dir->d_name); + } + + // Some stats from the fsys + std::string S; + if (_config->FindB("Debug::identcdrom",false) == false) + { + struct statvfs Buf; + if (fstatvfs(dirfd, &Buf) != 0) + return _error->Errno("statfs",_("Failed to stat the cdrom")); + + // We use a kilobyte block size to avoid overflow + S = std::to_string(Buf.f_blocks * (Buf.f_bsize / 1024)); + if (writable_media == false) + S.append(" ").append(std::to_string(Buf.f_bfree * (Buf.f_bsize / 1024))); + Hash.Add(S.c_str(), S.length()); + strprintf(S, "-%u", Version); + } + else + strprintf(S, "-%u.debug", Version); + + closedir(D); + Res = Hash.GetHashString(Hashes::MD5SUM).HashValue().append(std::move(S)); + return true; +} + /*}}}*/ +// FindMountPointForDevice - Find mountpoint for the given device /*{{{*/ +string FindMountPointForDevice(const char *devnode) +{ + // this is the order that mount uses as well + std::vector<std::string> const mounts = _config->FindVector("Dir::state::MountPoints", "/etc/mtab,/proc/mount"); + + for (std::vector<std::string>::const_iterator m = mounts.begin(); m != mounts.end(); ++m) + if (FileExists(*m) == true) + { + char * line = NULL; + size_t line_len = 0; + FILE * f = fopen(m->c_str(), "r"); + while(getline(&line, &line_len, f) != -1) + { + char * out[] = { NULL, NULL, NULL }; + TokSplitString(' ', line, out, 3); + if (out[2] != NULL || out[1] == NULL || out[0] == NULL) + continue; + if (strcmp(out[0], devnode) != 0) + continue; + fclose(f); + // unescape the \0XXX chars in the path + string mount_point = out[1]; + free(line); + return DeEscapeString(mount_point); + } + fclose(f); + free(line); + } + + return string(); +} + /*}}}*/ diff --git a/apt-pkg/contrib/cdromutl.h b/apt-pkg/contrib/cdromutl.h new file mode 100644 index 0000000..1384cea --- /dev/null +++ b/apt-pkg/contrib/cdromutl.h @@ -0,0 +1,24 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + CDROM Utilities - Some functions to manipulate CDROM mounts. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_CDROMUTL_H +#define PKGLIB_CDROMUTL_H + +#include <apt-pkg/macros.h> + +#include <string> + + +// mount cdrom, DeviceName (e.g. /dev/sr0) is optional +APT_PUBLIC bool MountCdrom(std::string Path, std::string DeviceName=""); +APT_PUBLIC bool UnmountCdrom(std::string Path); +APT_PUBLIC bool IdentCdrom(std::string CD,std::string &Res,unsigned int Version = 2); +APT_PUBLIC bool IsMounted(std::string &Path); +APT_PUBLIC std::string FindMountPointForDevice(const char *device); + +#endif diff --git a/apt-pkg/contrib/cmndline.cc b/apt-pkg/contrib/cmndline.cc new file mode 100644 index 0000000..998af2f --- /dev/null +++ b/apt-pkg/contrib/cmndline.cc @@ -0,0 +1,445 @@ +// -*- mode: cpp; mode: fold -*- +// SPDX-License-Identifier: GPL-2.0+ +// Description /*{{{*/ +/* ###################################################################### + + Command Line Class - Sophisticated command line parser + + This file had this historic note, but now includes further changes + under the GPL-2.0+: + + This source is placed in the Public Domain, do with it what you will + It was originally written by Jason Gunthorpe <jgg@debian.org>. + + ##################################################################### */ + /*}}}*/ +// Include files /*{{{*/ +#include <config.h> + +#include <apt-pkg/cmndline.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/strutl.h> + +#include <string> +#include <stddef.h> +#include <stdlib.h> +#include <string.h> + +#include <apti18n.h> + /*}}}*/ +using namespace std; + +// CommandLine::CommandLine - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +CommandLine::CommandLine(Args *AList,Configuration *Conf) : ArgList(AList), + Conf(Conf), FileList(0) +{ +} +CommandLine::CommandLine() : ArgList(NULL), Conf(NULL), FileList(0) +{ +} + /*}}}*/ +// CommandLine::~CommandLine - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +CommandLine::~CommandLine() +{ + delete [] FileList; +} + /*}}}*/ +// CommandLine::GetCommand - return the first non-option word /*{{{*/ +char const * CommandLine::GetCommand(Dispatch const * const Map, + unsigned int const argc, char const * const * const argv) +{ + // if there is a -- on the line there must be the word we search for either + // before it (as -- marks the end of the options) or right after it (as we can't + // decide if the command is actually an option, given that in theory, you could + // have parameters named like commands) + for (size_t i = 1; i < argc; ++i) + { + if (strcmp(argv[i], "--") != 0) + continue; + // check if command is before -- + for (size_t k = 1; k < i; ++k) + for (size_t j = 0; Map[j].Match != NULL; ++j) + if (strcmp(argv[k], Map[j].Match) == 0) + return Map[j].Match; + // see if the next token after -- is the command + ++i; + if (i < argc) + for (size_t j = 0; Map[j].Match != NULL; ++j) + if (strcmp(argv[i], Map[j].Match) == 0) + return Map[j].Match; + // we found a --, but not a command + return NULL; + } + // no --, so search for the first word matching a command + // FIXME: How like is it that an option parameter will be also a valid Match ? + for (size_t i = 1; i < argc; ++i) + { + if (*(argv[i]) == '-') + continue; + for (size_t j = 0; Map[j].Match != NULL; ++j) + if (strcmp(argv[i], Map[j].Match) == 0) + return Map[j].Match; + } + return NULL; +} + /*}}}*/ +// CommandLine::Parse - Main action member /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool CommandLine::Parse(int argc,const char **argv) +{ + delete [] FileList; + FileList = new const char *[argc]; + const char **Files = FileList; + int I; + for (I = 1; I != argc; I++) + { + const char *Opt = argv[I]; + + // It is not an option + if (*Opt != '-') + { + *Files++ = Opt; + continue; + } + + Opt++; + + // Double dash signifies the end of option processing + if (*Opt == '-' && Opt[1] == 0) + { + I++; + break; + } + + // Single dash is a short option + if (*Opt != '-') + { + // Iterate over each letter + while (*Opt != 0) + { + // Search for the option + Args *A; + for (A = ArgList; A->end() == false && A->ShortOpt != *Opt; A++); + if (A->end() == true) + return _error->Error(_("Command line option '%c' [from %s] is not understood in combination with the other options."),*Opt,argv[I]); + + if (HandleOpt(I,argc,argv,Opt,A) == false) + return false; + if (*Opt != 0) + Opt++; + } + continue; + } + + Opt++; + + // Match up to a = against the list + Args *A; + const char *OptEnd = strchrnul(Opt, '='); + for (A = ArgList; A->end() == false && + (A->LongOpt == 0 || stringcasecmp(Opt,OptEnd,A->LongOpt) != 0); + ++A); + + // Failed, look for a word after the first - (no-foo) + bool PreceedMatch = false; + if (A->end() == true) + { + Opt = (const char*) memchr(Opt, '-', OptEnd - Opt); + if (Opt == NULL) + return _error->Error(_("Command line option %s is not understood in combination with the other options"),argv[I]); + Opt++; + + for (A = ArgList; A->end() == false && + (A->LongOpt == 0 || stringcasecmp(Opt,OptEnd,A->LongOpt) != 0); + ++A); + + // Failed again.. + if (A->end() == true && OptEnd - Opt != 1) + return _error->Error(_("Command line option %s is not understood in combination with the other options"),argv[I]); + + // The option could be a single letter option prefixed by a no-.. + if (A->end() == true) + { + for (A = ArgList; A->end() == false && A->ShortOpt != *Opt; A++); + + if (A->end() == true) + return _error->Error(_("Command line option %s is not understood in combination with the other options"),argv[I]); + } + + // The option is not boolean + if (A->IsBoolean() == false) + return _error->Error(_("Command line option %s is not boolean"),argv[I]); + PreceedMatch = true; + } + + // Deal with it. + OptEnd--; + if (HandleOpt(I,argc,argv,OptEnd,A,PreceedMatch) == false) + return false; + } + + // Copy any remaining file names over + for (; I != argc; I++) + *Files++ = argv[I]; + *Files = 0; + + SaveInConfig(argc, argv); + + return true; +} + /*}}}*/ +// CommandLine::HandleOpt - Handle a single option including all flags /*{{{*/ +// --------------------------------------------------------------------- +/* This is a helper function for parser, it looks at a given argument + and looks for specific patterns in the string, it gets tokanized + -ruffly- like -*[yes|true|enable]-(o|longopt)[=][ ][argument] */ +bool CommandLine::HandleOpt(int &I,int argc,const char *argv[], + const char *&Opt,Args *A,bool PreceedMatch) +{ + const char *Argument = 0; + bool CertainArg = false; + int IncI = 0; + + /* Determine the possible location of an option or 0 if their is + no option */ + if (Opt[1] == 0) + { + if (I + 1 < argc && argv[I+1][0] != '-') + Argument = argv[I+1]; + + IncI = 1; + } + else + { + if (Opt[1] == '=') + { + CertainArg = true; + Argument = Opt + 2; + } + else + Argument = Opt + 1; + } + + // Option is an argument set + if ((A->Flags & HasArg) == HasArg) + { + if (Argument == 0) + return _error->Error(_("Option %s requires an argument."),argv[I]); + Opt += strlen(Opt); + I += IncI; + + // Parse a configuration file + if ((A->Flags & ConfigFile) == ConfigFile) + return ReadConfigFile(*Conf,Argument); + + // Arbitrary item specification + if ((A->Flags & ArbItem) == ArbItem) + { + const char * const J = strchr(Argument, '='); + if (J == nullptr) + return _error->Error(_("Option %s: Configuration item specification must have an =<val>."),argv[I]); + + Conf->Set(string(Argument,J-Argument), J+1); + return true; + } + + const char *I = strchrnul(A->ConfName, ' '); + if (*I == ' ') + Conf->Set(string(A->ConfName,0,I-A->ConfName),string(I+1) + Argument); + else + Conf->Set(A->ConfName,string(I) + Argument); + + return true; + } + + // Option is an integer level + if ((A->Flags & IntLevel) == IntLevel) + { + // There might be an argument + if (Argument != 0) + { + char *EndPtr; + unsigned long Value = strtol(Argument,&EndPtr,10); + + // Conversion failed and the argument was specified with an =s + if (EndPtr == Argument && CertainArg == true) + return _error->Error(_("Option %s requires an integer argument, not '%s'"),argv[I],Argument); + + // Conversion was ok, set the value and return + if (EndPtr != 0 && EndPtr != Argument && *EndPtr == 0) + { + Conf->Set(A->ConfName,Value); + Opt += strlen(Opt); + I += IncI; + return true; + } + } + + // Increase the level + Conf->Set(A->ConfName,Conf->FindI(A->ConfName)+1); + return true; + } + + // Option is a boolean + int Sense = -1; // -1 is unspecified, 0 is yes 1 is no + + // Look for an argument. + while (1) + { + // Look at preceding text + char Buffer[300]; + if (Argument == 0) + { + if (PreceedMatch == false) + break; + + if (strlen(argv[I]) >= sizeof(Buffer)) + return _error->Error(_("Option '%s' is too long"),argv[I]); + + // Skip the leading dash + const char *J = argv[I]; + for (; *J == '-'; J++) + ; + + const char *JEnd = strchr(J, '-'); + if (JEnd != NULL) + { + strncpy(Buffer,J,JEnd - J); + Buffer[JEnd - J] = 0; + Argument = Buffer; + CertainArg = true; + } + else + break; + } + + // Check for boolean + Sense = StringToBool(Argument); + if (Sense >= 0) + { + // Eat the argument + if (Argument != Buffer) + { + Opt += strlen(Opt); + I += IncI; + } + break; + } + + if (CertainArg == true) + return _error->Error(_("Sense %s is not understood, try true or false."),Argument); + + Argument = 0; + } + + // Indeterminate sense depends on the flag + if (Sense == -1) + { + if ((A->Flags & InvBoolean) == InvBoolean) + Sense = 0; + else + Sense = 1; + } + + Conf->Set(A->ConfName,Sense); + return true; +} + /*}}}*/ +// CommandLine::FileSize - Count the number of filenames /*{{{*/ +// --------------------------------------------------------------------- +/* */ +unsigned int CommandLine::FileSize() const +{ + unsigned int Count = 0; + for (const char **I = FileList; I != 0 && *I != 0; I++) + Count++; + return Count; +} + /*}}}*/ +// CommandLine::DispatchArg - Do something with the first arg /*{{{*/ +bool CommandLine::DispatchArg(Dispatch const * const Map,bool NoMatch) +{ + int I; + for (I = 0; Map[I].Match != 0; I++) + { + if (strcmp(FileList[0],Map[I].Match) == 0) + { + bool Res = Map[I].Handler(*this); + if (Res == false && _error->PendingError() == false) + _error->Error("Handler silently failed"); + return Res; + } + } + + // No matching name + if (Map[I].Match == 0) + { + if (NoMatch == true) + _error->Error(_("Invalid operation %s"),FileList[0]); + } + + return false; +} + /*}}}*/ +// CommandLine::SaveInConfig - for output later in a logfile or so /*{{{*/ +// --------------------------------------------------------------------- +/* We save the commandline here to have it around later for e.g. logging. + It feels a bit like a hack here and isn't bulletproof, but it is better + than nothing after all. */ +void CommandLine::SaveInConfig(unsigned int const &argc, char const * const * const argv) +{ + char cmdline[100 + argc * 50]; + memset(cmdline, 0, sizeof(cmdline)); + unsigned int length = 0; + bool lastWasOption = false; + bool closeQuote = false; + for (unsigned int i = 0; i < argc && length < sizeof(cmdline); ++i, ++length) + { + for (unsigned int j = 0; argv[i][j] != '\0' && length < sizeof(cmdline)-2; ++j) + { + // we can't really sensibly deal with quoting so skip it + if (strchr("\"\'\r\n", argv[i][j]) != nullptr) + continue; + cmdline[length++] = argv[i][j]; + if (lastWasOption == true && argv[i][j] == '=') + { + // That is possibly an option: Quote it if it includes spaces, + // the benefit is that this will eliminate also most false positives + const char* c = strchr(&argv[i][j+1], ' '); + if (c == NULL) continue; + cmdline[length++] = '\''; + closeQuote = true; + } + } + if (closeQuote == true) + { + cmdline[length++] = '\''; + closeQuote = false; + } + // Problem: detects also --hello + if (cmdline[length-1] == 'o') + lastWasOption = true; + cmdline[length] = ' '; + } + cmdline[--length] = '\0'; + _config->Set("CommandLine::AsString", cmdline); +} + /*}}}*/ +CommandLine::Args CommandLine::MakeArgs(char ShortOpt, char const *LongOpt, char const *ConfName, unsigned long Flags)/*{{{*/ +{ + /* In theory, this should be a constructor for CommandLine::Args instead, + but this breaks compatibility as gcc thinks this is a c++11 initializer_list */ + CommandLine::Args arg; + arg.ShortOpt = ShortOpt; + arg.LongOpt = LongOpt; + arg.ConfName = ConfName; + arg.Flags = Flags; + return arg; +} + /*}}}*/ diff --git a/apt-pkg/contrib/cmndline.h b/apt-pkg/contrib/cmndline.h new file mode 100644 index 0000000..40d384f --- /dev/null +++ b/apt-pkg/contrib/cmndline.h @@ -0,0 +1,113 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Command Line Class - Sophisticated command line parser + + This class provides a unified command line parser/option handliner/ + configuration mechanism. It allows the caller to specify the option + set and map the option set into the configuration class or other + special functioning. + + Filenames are stripped from the option stream and put into their + own array. + + The argument descriptor array can be initialized as: + + CommandLine::Args Args[] = + {{'q',"quiet","apt::get::quiet",CommandLine::IntLevel}, + {0,0,0,0}}; + + The flags mean, + HasArg - Means the argument has a value + IntLevel - Means the argument is an integer level indication, the + following -qqqq (+3) -q5 (=5) -q=5 (=5) are valid + Boolean - Means it is true/false or yes/no. + -d (true) --no-d (false) --yes-d (true) + --long (true) --no-long (false) --yes-long (true) + -d=yes (true) -d=no (false) Words like enable, disable, + true false, yes no and on off are recognized in logical + places. + InvBoolean - Same as boolean but the case with no specified sense + (first case) is set to false. + ConfigFile - Means this flag should be interprited as the name of + a config file to read in at this point in option processing. + Implies HasArg. + ArbItem - Means the item is an arbitrary configuration string of + the form item=value, where item is passed directly + to the configuration class. + The default, if the flags are 0 is to use Boolean + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_CMNDLINE_H +#define PKGLIB_CMNDLINE_H + +#include <apt-pkg/macros.h> + + +class Configuration; + +class APT_PUBLIC CommandLine +{ + public: + struct Args; + struct Dispatch; + struct DispatchWithHelp; + + protected: + + Args *ArgList; + Configuration *Conf; + bool HandleOpt(int &I,int argc,const char *argv[], + const char *&Opt,Args *A,bool PreceedeMatch = false); + void static SaveInConfig(unsigned int const &argc, char const * const * const argv); + + public: + + enum AFlags + { + HasArg = (1 << 0), + IntLevel = (1 << 1), + Boolean = (1 << 2), + InvBoolean = (1 << 3), + ConfigFile = (1 << 4) | HasArg, + ArbItem = (1 << 5) | HasArg + }; + + const char **FileList; + + bool Parse(int argc,const char **argv); + void ShowHelp(); + unsigned int FileSize() const APT_PURE; + bool DispatchArg(Dispatch const * const List,bool NoMatch = true); + + static char const * GetCommand(Dispatch const * const Map, + unsigned int const argc, char const * const * const argv) APT_PURE; + + static CommandLine::Args MakeArgs(char ShortOpt, char const *LongOpt, + char const *ConfName, unsigned long Flags) APT_PURE; + + CommandLine(); + CommandLine(Args *AList,Configuration *Conf); + ~CommandLine(); +}; + +struct CommandLine::Args +{ + char ShortOpt; + const char *LongOpt; + const char *ConfName; + unsigned long Flags; + + inline bool end() {return ShortOpt == 0 && LongOpt == 0;}; + inline bool IsBoolean() {return Flags == 0 || (Flags & (Boolean|InvBoolean)) != 0;}; +}; + +struct CommandLine::Dispatch +{ + const char *Match; + bool (*Handler)(CommandLine &); +}; + +#endif diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc new file mode 100644 index 0000000..1134ed6 --- /dev/null +++ b/apt-pkg/contrib/configuration.cc @@ -0,0 +1,1194 @@ +// -*- mode: cpp; mode: fold -*- +// SPDX-License-Identifier: GPL-2.0+ +// Description /*{{{*/ +/* ###################################################################### + + Configuration Class + + This class provides a configuration file and command line parser + for a tree-oriented configuration environment. All runtime configuration + is stored in here. + + This file had this historic note, but now includes further changes + under the GPL-2.0+: + + This source is placed in the Public Domain, do with it what you will + It was originally written by Jason Gunthorpe <jgg@debian.org>. + + ##################################################################### */ + /*}}}*/ +// Include files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/string_view.h> + +#include <ctype.h> +#include <regex.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <algorithm> +#include <array> +#include <fstream> +#include <iterator> +#include <numeric> +#include <sstream> +#include <stack> +#include <string> +#include <unordered_map> +#include <vector> + +#include <apti18n.h> + +using namespace std; + /*}}}*/ + +Configuration *_config = new Configuration; + +/* TODO: This config verification shouldn't be using a static variable + but a Cnf-member – but that would need ABI breaks and stuff and for now + that really is an apt-dev-only tool, so it isn't that bad that it is + unusable and allaround a bit strange */ +enum class APT_HIDDEN ConfigType { UNDEFINED, INT, BOOL, STRING, STRING_OR_BOOL, STRING_OR_LIST, FILE, DIR, LIST, PROGRAM_PATH = FILE }; +APT_HIDDEN std::unordered_map<std::string, ConfigType> apt_known_config {}; +static std::string getConfigTypeString(ConfigType const type) /*{{{*/ +{ + switch (type) + { + case ConfigType::UNDEFINED: return "UNDEFINED"; + case ConfigType::INT: return "INT"; + case ConfigType::BOOL: return "BOOL"; + case ConfigType::STRING: return "STRING"; + case ConfigType::STRING_OR_BOOL: return "STRING_OR_BOOL"; + case ConfigType::FILE: return "FILE"; + case ConfigType::DIR: return "DIR"; + case ConfigType::LIST: return "LIST"; + case ConfigType::STRING_OR_LIST: return "STRING_OR_LIST"; + } + return "UNKNOWN"; +} + /*}}}*/ +static ConfigType getConfigType(std::string const &type) /*{{{*/ +{ + if (type == "<INT>") + return ConfigType::INT; + else if (type == "<BOOL>") + return ConfigType::BOOL; + else if (type == "<STRING>") + return ConfigType::STRING; + else if (type == "<STRING_OR_BOOL>") + return ConfigType::STRING_OR_BOOL; + else if (type == "<FILE>") + return ConfigType::FILE; + else if (type == "<DIR>") + return ConfigType::DIR; + else if (type == "<LIST>") + return ConfigType::LIST; + else if (type == "<STRING_OR_LIST>") + return ConfigType::STRING_OR_LIST; + else if (type == "<PROGRAM_PATH>") + return ConfigType::PROGRAM_PATH; + return ConfigType::UNDEFINED; +} + /*}}}*/ +// checkFindConfigOptionType - workhorse of option checking /*{{{*/ +static void checkFindConfigOptionTypeInternal(std::string name, ConfigType const type) +{ + std::transform(name.begin(), name.end(), name.begin(), ::tolower); + auto known = apt_known_config.find(name); + if (known == apt_known_config.cend()) + { + auto const rcolon = name.rfind(':'); + if (rcolon != std::string::npos) + { + known = apt_known_config.find(name.substr(0, rcolon) + ":*"); + if (known == apt_known_config.cend()) + { + auto const parts = StringSplit(name, "::"); + size_t psize = parts.size(); + if (psize > 1) + { + for (size_t max = psize; max != 1; --max) + { + std::ostringstream os; + std::copy(parts.begin(), parts.begin() + max, std::ostream_iterator<std::string>(os, "::")); + os << "**"; + known = apt_known_config.find(os.str()); + if (known != apt_known_config.cend() && known->second == ConfigType::UNDEFINED) + return; + } + for (size_t max = psize - 1; max != 1; --max) + { + std::ostringstream os; + std::copy(parts.begin(), parts.begin() + max - 1, std::ostream_iterator<std::string>(os, "::")); + os << "*::"; + std::copy(parts.begin() + max + 1, parts.end() - 1, std::ostream_iterator<std::string>(os, "::")); + os << *(parts.end() - 1); + known = apt_known_config.find(os.str()); + if (known != apt_known_config.cend()) + break; + } + } + } + } + } + if (known == apt_known_config.cend()) + _error->Warning("Using unknown config option »%s« of type %s", + name.c_str(), getConfigTypeString(type).c_str()); + else if (known->second != type) + { + if (known->second == ConfigType::DIR && type == ConfigType::FILE) + ; // implementation detail + else if (type == ConfigType::STRING && (known->second == ConfigType::FILE || known->second == ConfigType::DIR)) + ; // TODO: that might be an error or not, we will figure this out later + else if (known->second == ConfigType::STRING_OR_BOOL && (type == ConfigType::BOOL || type == ConfigType::STRING)) + ; + else if (known->second == ConfigType::STRING_OR_LIST && (type == ConfigType::LIST || type == ConfigType::STRING)) + ; + else + _error->Warning("Using config option »%s« of type %s as a type %s", + name.c_str(), getConfigTypeString(known->second).c_str(), getConfigTypeString(type).c_str()); + } +} +static void checkFindConfigOptionType(char const * const name, ConfigType const type) +{ + if (apt_known_config.empty()) + return; + checkFindConfigOptionTypeInternal(name, type); +} + /*}}}*/ +static bool LoadConfigurationIndex(std::string const &filename) /*{{{*/ +{ + apt_known_config.clear(); + if (filename.empty()) + return true; + Configuration Idx; + if (ReadConfigFile(Idx, filename) == false) + return false; + + Configuration::Item const * Top = Idx.Tree(nullptr); + if (unlikely(Top == nullptr)) + return false; + + do { + if (Top->Value.empty() == false) + { + std::string fulltag = Top->FullTag(); + std::transform(fulltag.begin(), fulltag.end(), fulltag.begin(), ::tolower); + apt_known_config.emplace(std::move(fulltag), getConfigType(Top->Value)); + } + + if (Top->Child != nullptr) + { + Top = Top->Child; + continue; + } + + while (Top != nullptr && Top->Next == nullptr) + Top = Top->Parent; + if (Top != nullptr) + Top = Top->Next; + } while (Top != nullptr); + + return true; +} + /*}}}*/ + +// Configuration::Configuration - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +Configuration::Configuration() : ToFree(true) +{ + Root = new Item; +} +Configuration::Configuration(const Item *Root) : Root((Item *)Root), ToFree(false) +{ +} + /*}}}*/ +// Configuration::~Configuration - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +Configuration::~Configuration() +{ + if (ToFree == false) + return; + + Item *Top = Root; + for (; Top != 0;) + { + if (Top->Child != 0) + { + Top = Top->Child; + continue; + } + + while (Top != 0 && Top->Next == 0) + { + Item *Parent = Top->Parent; + delete Top; + Top = Parent; + } + if (Top != 0) + { + Item *Next = Top->Next; + delete Top; + Top = Next; + } + } +} + /*}}}*/ +// Configuration::Lookup - Lookup a single item /*{{{*/ +// --------------------------------------------------------------------- +/* This will lookup a single item by name below another item. It is a + helper function for the main lookup function */ +Configuration::Item *Configuration::Lookup(Item *Head,const char *S, + unsigned long const &Len,bool const &Create) +{ + int Res = 1; + Item *I = Head->Child; + Item **Last = &Head->Child; + + // Empty strings match nothing. They are used for lists. + if (Len != 0) + { + for (; I != 0; Last = &I->Next, I = I->Next) + if (Len == I->Tag.length() && (Res = stringcasecmp(I->Tag,S,S + Len)) == 0) + break; + } + else + for (; I != 0; Last = &I->Next, I = I->Next); + + if (Res == 0) + return I; + if (Create == false) + return 0; + + I = new Item; + I->Tag.assign(S,Len); + I->Next = *Last; + I->Parent = Head; + *Last = I; + return I; +} + /*}}}*/ +// Configuration::Lookup - Lookup a fully scoped item /*{{{*/ +// --------------------------------------------------------------------- +/* This performs a fully scoped lookup of a given name, possibly creating + new items */ +Configuration::Item *Configuration::Lookup(const char *Name,bool const &Create) +{ + if (Name == 0) + return Root->Child; + + const char *Start = Name; + const char *End = Start + strlen(Name); + const char *TagEnd = Name; + Item *Itm = Root; + for (; End - TagEnd >= 2; TagEnd++) + { + if (TagEnd[0] == ':' && TagEnd[1] == ':') + { + Itm = Lookup(Itm,Start,TagEnd - Start,Create); + if (Itm == 0) + return 0; + TagEnd = Start = TagEnd + 2; + } + } + + // This must be a trailing ::, we create unique items in a list + if (End - Start == 0) + { + if (Create == false) + return 0; + } + + Itm = Lookup(Itm,Start,End - Start,Create); + return Itm; +} + /*}}}*/ +// Configuration::Find - Find a value /*{{{*/ +// --------------------------------------------------------------------- +/* */ +string Configuration::Find(const char *Name,const char *Default) const +{ + checkFindConfigOptionType(Name, ConfigType::STRING); + const Item *Itm = Lookup(Name); + if (Itm == 0 || Itm->Value.empty() == true) + { + if (Default == 0) + return ""; + else + return Default; + } + + return Itm->Value; +} + /*}}}*/ +// Configuration::FindFile - Find a Filename /*{{{*/ +// --------------------------------------------------------------------- +/* Directories are stored as the base dir in the Parent node and the + sub directory in sub nodes with the final node being the end filename + */ +string Configuration::FindFile(const char *Name,const char *Default) const +{ + checkFindConfigOptionType(Name, ConfigType::FILE); + const Item *RootItem = Lookup("RootDir"); + std::string result = (RootItem == 0) ? "" : RootItem->Value; + if(result.empty() == false && result[result.size() - 1] != '/') + result.push_back('/'); + + const Item *Itm = Lookup(Name); + if (Itm == 0 || Itm->Value.empty() == true) + { + if (Default != 0) + result.append(Default); + } + else + { + string val = Itm->Value; + while (Itm->Parent != 0) + { + if (Itm->Parent->Value.empty() == true) + { + Itm = Itm->Parent; + continue; + } + + // Absolute + if (val.length() >= 1 && val[0] == '/') + { + if (val.compare(0, 9, "/dev/null") == 0) + val.erase(9); + break; + } + + // ~/foo or ./foo + if (val.length() >= 2 && (val[0] == '~' || val[0] == '.') && val[1] == '/') + break; + + // ../foo + if (val.length() >= 3 && val[0] == '.' && val[1] == '.' && val[2] == '/') + break; + + if (Itm->Parent->Value.end()[-1] != '/') + val.insert(0, "/"); + + val.insert(0, Itm->Parent->Value); + Itm = Itm->Parent; + } + result.append(val); + } + return flNormalize(result); +} + /*}}}*/ +// Configuration::FindDir - Find a directory name /*{{{*/ +// --------------------------------------------------------------------- +/* This is like findfile execept the result is terminated in a / */ +string Configuration::FindDir(const char *Name,const char *Default) const +{ + checkFindConfigOptionType(Name, ConfigType::DIR); + string Res = FindFile(Name,Default); + if (Res.end()[-1] != '/') + { + size_t const found = Res.rfind("/dev/null"); + if (found != string::npos && found == Res.size() - 9) + return Res; // /dev/null returning + return Res + '/'; + } + return Res; +} + /*}}}*/ +// Configuration::FindVector - Find a vector of values /*{{{*/ +// --------------------------------------------------------------------- +/* Returns a vector of config values under the given item */ +vector<string> Configuration::FindVector(const char *Name, std::string const &Default, bool const Keys) const +{ + checkFindConfigOptionType(Name, ConfigType::LIST); + vector<string> Vec; + const Item *Top = Lookup(Name); + if (Top == NULL) + return VectorizeString(Default, ','); + + if (Top->Value.empty() == false) + return VectorizeString(Top->Value, ','); + + Item *I = Top->Child; + while(I != NULL) + { + Vec.push_back(Keys ? I->Tag : I->Value); + I = I->Next; + } + if (Vec.empty() == true) + return VectorizeString(Default, ','); + + return Vec; +} + /*}}}*/ +// Configuration::FindI - Find an integer value /*{{{*/ +// --------------------------------------------------------------------- +/* */ +int Configuration::FindI(const char *Name,int const &Default) const +{ + checkFindConfigOptionType(Name, ConfigType::INT); + const Item *Itm = Lookup(Name); + if (Itm == 0 || Itm->Value.empty() == true) + return Default; + + char *End; + int Res = strtol(Itm->Value.c_str(),&End,0); + if (End == Itm->Value.c_str()) + return Default; + + return Res; +} + /*}}}*/ +// Configuration::FindB - Find a boolean type /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool Configuration::FindB(const char *Name,bool const &Default) const +{ + checkFindConfigOptionType(Name, ConfigType::BOOL); + const Item *Itm = Lookup(Name); + if (Itm == 0 || Itm->Value.empty() == true) + return Default; + + return StringToBool(Itm->Value,Default); +} + /*}}}*/ +// Configuration::FindAny - Find an arbitrary type /*{{{*/ +// --------------------------------------------------------------------- +/* a key suffix of /f, /d, /b or /i calls Find{File,Dir,B,I} */ +string Configuration::FindAny(const char *Name,const char *Default) const +{ + string key = Name; + char type = 0; + + if (key.size() > 2 && key.end()[-2] == '/') + { + type = key.end()[-1]; + key.resize(key.size() - 2); + } + + switch (type) + { + // file + case 'f': + return FindFile(key.c_str(), Default); + + // directory + case 'd': + return FindDir(key.c_str(), Default); + + // bool + case 'b': + return FindB(key, Default) ? "true" : "false"; + + // int + case 'i': + { + char buf[16]; + snprintf(buf, sizeof(buf)-1, "%d", FindI(key, Default ? atoi(Default) : 0 )); + return buf; + } + } + + // fallback + return Find(Name, Default); +} + /*}}}*/ +// Configuration::CndSet - Conditional Set a value /*{{{*/ +// --------------------------------------------------------------------- +/* This will not overwrite */ +void Configuration::CndSet(const char *Name,const string &Value) +{ + Item *Itm = Lookup(Name,true); + if (Itm == 0) + return; + if (Itm->Value.empty() == true) + Itm->Value = Value; +} + /*}}}*/ +// Configuration::Set - Set an integer value /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void Configuration::CndSet(const char *Name,int const Value) +{ + Item *Itm = Lookup(Name,true); + if (Itm == 0 || Itm->Value.empty() == false) + return; + char S[300]; + snprintf(S,sizeof(S),"%i",Value); + Itm->Value = S; +} + /*}}}*/ +// Configuration::Set - Set a value /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void Configuration::Set(const char *Name,const string &Value) +{ + Item *Itm = Lookup(Name,true); + if (Itm == 0) + return; + Itm->Value = Value; +} + /*}}}*/ +// Configuration::Set - Set an integer value /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void Configuration::Set(const char *Name,int const &Value) +{ + Item *Itm = Lookup(Name,true); + if (Itm == 0) + return; + char S[300]; + snprintf(S,sizeof(S),"%i",Value); + Itm->Value = S; +} + /*}}}*/ +// Configuration::Clear - Clear an single value from a list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void Configuration::Clear(string const &Name, int const &Value) +{ + char S[300]; + snprintf(S,sizeof(S),"%i",Value); + Clear(Name, S); +} + /*}}}*/ +// Configuration::Clear - Clear an single value from a list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void Configuration::Clear(string const &Name, string const &Value) +{ + Item *Top = Lookup(Name.c_str(),false); + if (Top == 0 || Top->Child == 0) + return; + + Item *Tmp, *Prev, *I; + Prev = I = Top->Child; + + while(I != NULL) + { + if(I->Value == Value) + { + Tmp = I; + // was first element, point parent to new first element + if(Top->Child == Tmp) + Top->Child = I->Next; + I = I->Next; + Prev->Next = I; + delete Tmp; + } else { + Prev = I; + I = I->Next; + } + } + +} + /*}}}*/ +// Configuration::Clear - Clear everything /*{{{*/ +// --------------------------------------------------------------------- +void Configuration::Clear() +{ + const Configuration::Item *Top = Tree(0); + while( Top != 0 ) + { + Clear(Top->FullTag()); + Top = Top->Next; + } +} + /*}}}*/ +// Configuration::Clear - Clear an entire tree /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void Configuration::Clear(string const &Name) +{ + Item *Top = Lookup(Name.c_str(),false); + if (Top == 0) + return; + + Top->Value.clear(); + Item *Stop = Top; + Top = Top->Child; + Stop->Child = 0; + for (; Top != 0;) + { + if (Top->Child != 0) + { + Top = Top->Child; + continue; + } + + while (Top != 0 && Top->Next == 0) + { + Item *Tmp = Top; + Top = Top->Parent; + delete Tmp; + + if (Top == Stop) + return; + } + + Item *Tmp = Top; + if (Top != 0) + Top = Top->Next; + delete Tmp; + } +} + /*}}}*/ +void Configuration::MoveSubTree(char const * const OldRootName, char const * const NewRootName)/*{{{*/ +{ + // prevent NewRoot being a subtree of OldRoot + if (OldRootName == nullptr) + return; + if (NewRootName != nullptr) + { + if (strcmp(OldRootName, NewRootName) == 0) + return; + std::string const oldroot = std::string(OldRootName) + "::"; + if (strcasestr(NewRootName, oldroot.c_str()) != NULL) + return; + } + + Item * Top; + Item const * const OldRoot = Top = Lookup(OldRootName, false); + if (Top == nullptr) + return; + std::string NewRoot; + if (NewRootName != nullptr) + NewRoot.append(NewRootName).append("::"); + + Top->Value.clear(); + Item * const Stop = Top; + Top = Top->Child; + Stop->Child = 0; + for (; Top != 0;) + { + if (Top->Child != 0) + { + Top = Top->Child; + continue; + } + + while (Top != 0 && Top->Next == 0) + { + Set(NewRoot + Top->FullTag(OldRoot), Top->Value); + Item const * const Tmp = Top; + Top = Top->Parent; + delete Tmp; + + if (Top == Stop) + return; + } + + Set(NewRoot + Top->FullTag(OldRoot), Top->Value); + Item const * const Tmp = Top; + if (Top != 0) + Top = Top->Next; + delete Tmp; + } +} + /*}}}*/ +// Configuration::Exists - Returns true if the Name exists /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool Configuration::Exists(const char *Name) const +{ + const Item *Itm = Lookup(Name); + if (Itm == 0) + return false; + return true; +} + /*}}}*/ +// Configuration::ExistsAny - Returns true if the Name, possibly /*{{{*/ +// --------------------------------------------------------------------- +/* qualified by /[fdbi] exists */ +bool Configuration::ExistsAny(const char *Name) const +{ + string key = Name; + + if (key.size() > 2 && key.end()[-2] == '/') + { + if (key.find_first_of("fdbi",key.size()-1) < key.size()) + { + key.resize(key.size() - 2); + if (Exists(key.c_str())) + return true; + } + else + { + _error->Warning(_("Unrecognized type abbreviation: '%c'"), key.end()[-3]); + } + } + return Exists(Name); +} + /*}}}*/ +// Configuration::Dump - Dump the config /*{{{*/ +// --------------------------------------------------------------------- +/* Dump the entire configuration space */ +void Configuration::Dump(ostream& str) +{ + Dump(str, NULL, "%F \"%v\";\n", true); +} +void Configuration::Dump(ostream& str, char const * const root, + char const * const formatstr, bool const emptyValue) +{ + const Configuration::Item* Top = Tree(root); + if (Top == 0) + return; + const Configuration::Item* const Root = (root == NULL) ? NULL : Top; + std::vector<std::string> const format = VectorizeString(formatstr, '%'); + + /* Write out all of the configuration directives by walking the + configuration tree */ + do { + if (emptyValue == true || Top->Value.empty() == emptyValue) + { + std::vector<std::string>::const_iterator f = format.begin(); + str << *f; + for (++f; f != format.end(); ++f) + { + if (f->empty() == true) + { + ++f; + str << '%' << *f; + continue; + } + char const type = (*f)[0]; + if (type == 'f') + str << Top->FullTag(); + else if (type == 't') + str << Top->Tag; + else if (type == 'v') + str << Top->Value; + else if (type == 'F') + str << QuoteString(Top->FullTag(), "=\"\n"); + else if (type == 'T') + str << QuoteString(Top->Tag, "=\"\n"); + else if (type == 'V') + str << QuoteString(Top->Value, "=\"\n"); + else if (type == 'n') + str << "\n"; + else if (type == 'N') + str << "\t"; + else + str << '%' << type; + str << f->c_str() + 1; + } + } + + if (Top->Child != 0) + { + Top = Top->Child; + continue; + } + + while (Top != 0 && Top->Next == 0) + Top = Top->Parent; + if (Top != 0) + Top = Top->Next; + + if (Root != NULL) + { + const Configuration::Item* I = Top; + while(I != 0) + { + if (I == Root) + break; + else + I = I->Parent; + } + if (I == 0) + break; + } + } while (Top != 0); +} + /*}}}*/ + +// Configuration::Item::FullTag - Return the fully scoped tag /*{{{*/ +// --------------------------------------------------------------------- +/* Stop sets an optional max recursion depth if this item is being viewed as + part of a sub tree. */ +string Configuration::Item::FullTag(const Item *Stop) const +{ + if (Parent == 0 || Parent->Parent == 0 || Parent == Stop) + return Tag; + return Parent->FullTag(Stop) + "::" + Tag; +} + /*}}}*/ + +// ReadConfigFile - Read a configuration file /*{{{*/ +// --------------------------------------------------------------------- +/* The configuration format is very much like the named.conf format + used in bind8, in fact this routine can parse most named.conf files. + Sectional config files are like bind's named.conf where there are + sections like 'zone "foo.org" { .. };' This causes each section to be + added in with a tag like "zone::foo.org" instead of being split + tag/value. AsSectional enables Sectional parsing.*/ +static void leaveCurrentScope(std::stack<std::string> &Stack, std::string &ParentTag) +{ + if (Stack.empty()) + ParentTag.clear(); + else + { + ParentTag = Stack.top(); + Stack.pop(); + } +} +bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectional, + unsigned const &Depth) +{ + // Open the stream for reading + FileFd F; + if (OpenConfigurationFileFd(FName, F) == false) + return false; + + string LineBuffer; + std::stack<std::string> Stack; + + // Parser state + string ParentTag; + + int CurLine = 0; + bool InComment = false; + while (F.Eof() == false) + { + // The raw input line. + std::string Input; + if (F.ReadLine(Input) == false) + Input.clear(); + // The input line with comments stripped. + std::string Fragment; + + // Expand tabs in the input line and remove leading and trailing whitespace. + Input = APT::String::Strip(SubstVar(Input, "\t", " ")); + CurLine++; + + // Now strip comments; if the whole line is contained in a + // comment, skip this line. + APT::StringView Line{Input.data(), Input.size()}; + + // continued Multi line comment + if (InComment) + { + size_t end = Line.find("*/"); + if (end != APT::StringView::npos) + { + Line.remove_prefix(end + 2); + InComment = false; + } + else + continue; + } + + // Discard single line comments + { + size_t start = 0; + while ((start = Line.find("//", start)) != APT::StringView::npos) + { + if (std::count(Line.begin(), Line.begin() + start, '"') % 2 != 0) + { + ++start; + continue; + } + Line.remove_suffix(Line.length() - start); + break; + } + using APT::operator""_sv; + constexpr std::array<APT::StringView, 3> magicComments { "clear"_sv, "include"_sv, "x-apt-configure-index"_sv }; + start = 0; + while ((start = Line.find('#', start)) != APT::StringView::npos) + { + if (std::count(Line.begin(), Line.begin() + start, '"') % 2 != 0 || + std::any_of(magicComments.begin(), magicComments.end(), [&](auto const m) { return Line.compare(start+1, m.length(), m) == 0; })) + { + ++start; + continue; + } + Line.remove_suffix(Line.length() - start); + break; + } + } + + // Look for multi line comments and build up the + // fragment. + Fragment.reserve(Line.length()); + { + size_t start = 0; + while ((start = Line.find("/*", start)) != APT::StringView::npos) + { + if (std::count(Line.begin(), Line.begin() + start, '"') % 2 != 0) + { + start += 2; + continue; + } + Fragment.append(Line.data(), start); + auto const end = Line.find("*/", start + 2); + if (end == APT::StringView::npos) + { + Line.clear(); + InComment = true; + break; + } + else + Line.remove_prefix(end + 2); + start = 0; + } + if (not Line.empty()) + Fragment.append(Line.data(), Line.length()); + } + + // Skip blank lines. + if (Fragment.empty()) + continue; + + // The line has actual content; interpret what it means. + bool InQuote = false; + auto Start = Fragment.cbegin(); + auto End = Fragment.cend(); + for (std::string::const_iterator I = Start; + I != End; ++I) + { + if (*I == '"') + InQuote = !InQuote; + + if (InQuote == false && (*I == '{' || *I == ';' || *I == '}')) + { + // Put the last fragment into the buffer + std::string::const_iterator NonWhitespaceStart = Start; + std::string::const_iterator NonWhitespaceStop = I; + for (; NonWhitespaceStart != I && isspace(*NonWhitespaceStart) != 0; ++NonWhitespaceStart) + ; + for (; NonWhitespaceStop != NonWhitespaceStart && isspace(NonWhitespaceStop[-1]) != 0; --NonWhitespaceStop) + ; + if (LineBuffer.empty() == false && NonWhitespaceStop - NonWhitespaceStart != 0) + LineBuffer += ' '; + LineBuffer += string(NonWhitespaceStart, NonWhitespaceStop); + + // Drop this from the input string, saving the character + // that terminated the construct we just closed. (i.e., a + // brace or a semicolon) + char TermChar = *I; + Start = I + 1; + + // Syntax Error + if (TermChar == '{' && LineBuffer.empty() == true) + return _error->Error(_("Syntax error %s:%u: Block starts with no name."),FName.c_str(),CurLine); + + // No string on this line + if (LineBuffer.empty() == true) + { + if (TermChar == '}') + leaveCurrentScope(Stack, ParentTag); + continue; + } + + // Parse off the tag + string Tag; + const char *Pos = LineBuffer.c_str(); + if (ParseQuoteWord(Pos,Tag) == false) + return _error->Error(_("Syntax error %s:%u: Malformed tag"),FName.c_str(),CurLine); + + // Parse off the word + string Word; + bool NoWord = false; + if (ParseCWord(Pos,Word) == false && + ParseQuoteWord(Pos,Word) == false) + { + if (TermChar != '{') + { + Word = Tag; + Tag = ""; + } + else + NoWord = true; + } + if (strlen(Pos) != 0) + return _error->Error(_("Syntax error %s:%u: Extra junk after value"),FName.c_str(),CurLine); + + // Go down a level + if (TermChar == '{') + { + Stack.push(ParentTag); + + /* Make sectional tags incorporate the section into the + tag string */ + if (AsSectional == true && Word.empty() == false) + { + Tag.append("::").append(Word); + Word.clear(); + } + + if (ParentTag.empty() == true) + ParentTag = Tag; + else + ParentTag.append("::").append(Tag); + Tag.clear(); + } + + // Generate the item name + string Item; + if (ParentTag.empty() == true) + Item = Tag; + else + { + if (TermChar != '{' || Tag.empty() == false) + Item = ParentTag + "::" + Tag; + else + Item = ParentTag; + } + + // Specials + if (Tag.length() >= 1 && Tag[0] == '#') + { + if (ParentTag.empty() == false) + return _error->Error(_("Syntax error %s:%u: Directives can only be done at the top level"),FName.c_str(),CurLine); + Tag.erase(Tag.begin()); + if (Tag == "clear") + Conf.Clear(Word); + else if (Tag == "include") + { + if (Depth > 10) + return _error->Error(_("Syntax error %s:%u: Too many nested includes"),FName.c_str(),CurLine); + if (Word.length() > 2 && Word.end()[-1] == '/') + { + if (ReadConfigDir(Conf,Word,AsSectional,Depth+1) == false) + return _error->Error(_("Syntax error %s:%u: Included from here"),FName.c_str(),CurLine); + } + else + { + if (ReadConfigFile(Conf,Word,AsSectional,Depth+1) == false) + return _error->Error(_("Syntax error %s:%u: Included from here"),FName.c_str(),CurLine); + } + } + else if (Tag == "x-apt-configure-index") + { + if (LoadConfigurationIndex(Word) == false) + return _error->Warning("Loading the configure index %s in file %s:%u failed!", Word.c_str(), FName.c_str(), CurLine); + } + else + return _error->Error(_("Syntax error %s:%u: Unsupported directive '%s'"),FName.c_str(),CurLine,Tag.c_str()); + } + else if (Tag.empty() == true && NoWord == false && Word == "#clear") + return _error->Error(_("Syntax error %s:%u: clear directive requires an option tree as argument"),FName.c_str(),CurLine); + else + { + // Set the item in the configuration class + if (NoWord == false) + Conf.Set(Item,Word); + } + + // Empty the buffer + LineBuffer.clear(); + + // Move up a tag, but only if there is no bit to parse + if (TermChar == '}') + leaveCurrentScope(Stack, ParentTag); + } + } + + // Store the remaining text, if any, in the current line buffer. + + // NB: could change this to use string-based operations; I'm + // using strstrip now to ensure backwards compatibility. + // -- dburrows 2008-04-01 + { + char *Buffer = new char[End - Start + 1]; + try + { + std::copy(Start, End, Buffer); + Buffer[End - Start] = '\0'; + + const char *Stripd = _strstrip(Buffer); + if (*Stripd != 0 && LineBuffer.empty() == false) + LineBuffer += " "; + LineBuffer += Stripd; + } + catch(...) + { + delete[] Buffer; + throw; + } + delete[] Buffer; + } + } + + if (LineBuffer.empty() == false) + return _error->Error(_("Syntax error %s:%u: Extra junk at end of file"),FName.c_str(),CurLine); + return true; +} + /*}}}*/ +// ReadConfigDir - Read a directory of config files /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool ReadConfigDir(Configuration &Conf,const string &Dir, + bool const &AsSectional, unsigned const &Depth) +{ + _error->PushToStack(); + auto const files = GetListOfFilesInDir(Dir, "conf", true, true); + auto const successfulList = not _error->PendingError(); + _error->MergeWithStack(); + return std::accumulate(files.cbegin(), files.cend(), true, [&](bool good, auto const &file) { + return ReadConfigFile(Conf, file, AsSectional, Depth) && good; + }) && successfulList; +} + /*}}}*/ +// MatchAgainstConfig Constructor /*{{{*/ +Configuration::MatchAgainstConfig::MatchAgainstConfig(char const * Config) +{ + std::vector<std::string> const strings = _config->FindVector(Config); + for (std::vector<std::string>::const_iterator s = strings.begin(); + s != strings.end(); ++s) + { + regex_t *p = new regex_t; + if (regcomp(p, s->c_str(), REG_EXTENDED | REG_ICASE | REG_NOSUB) == 0) + patterns.push_back(p); + else + { + regfree(p); + delete p; + _error->Warning("Invalid regular expression '%s' in configuration " + "option '%s' will be ignored.", + s->c_str(), Config); + continue; + } + } + if (strings.empty() == true) + patterns.push_back(NULL); +} + /*}}}*/ +// MatchAgainstConfig Destructor /*{{{*/ +Configuration::MatchAgainstConfig::~MatchAgainstConfig() +{ + clearPatterns(); +} +void Configuration::MatchAgainstConfig::clearPatterns() +{ + for(std::vector<regex_t *>::const_iterator p = patterns.begin(); + p != patterns.end(); ++p) + { + if (*p == NULL) continue; + regfree(*p); + delete *p; + } + patterns.clear(); +} + /*}}}*/ +// MatchAgainstConfig::Match - returns true if a pattern matches /*{{{*/ +bool Configuration::MatchAgainstConfig::Match(char const * str) const +{ + for(std::vector<regex_t *>::const_iterator p = patterns.begin(); + p != patterns.end(); ++p) + if (*p != NULL && regexec(*p, str, 0, 0, 0) == 0) + return true; + + return false; +} + /*}}}*/ diff --git a/apt-pkg/contrib/configuration.h b/apt-pkg/contrib/configuration.h new file mode 100644 index 0000000..6ebf28d --- /dev/null +++ b/apt-pkg/contrib/configuration.h @@ -0,0 +1,152 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Configuration Class + + This class provides a configuration file and command line parser + for a tree-oriented configuration environment. All runtime configuration + is stored in here. + + Each configuration name is given as a fully scoped string such as + Foo::Bar + And has associated with it a text string. The Configuration class only + provides storage and lookup for this tree, other classes provide + configuration file formats (and parsers/emitters if needed). + + Most things can get by quite happily with, + cout << _config->Find("Foo::Bar") << endl; + + A special extension, support for ordered lists is provided by using the + special syntax, "block::list::" the trailing :: designates the + item as a list. To access the list you must use the tree function on + "block::list". + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_CONFIGURATION_H +#define PKGLIB_CONFIGURATION_H + +#include <regex.h> + +#include <iostream> +#include <string> +#include <vector> + +#include <apt-pkg/macros.h> + + +class APT_PUBLIC Configuration +{ + public: + + struct Item + { + std::string Value; + std::string Tag; + Item *Parent; + Item *Child; + Item *Next; + + std::string FullTag(const Item *Stop = 0) const; + + Item() : Parent(0), Child(0), Next(0) {}; + }; + + private: + + Item *Root; + bool ToFree; + + Item *Lookup(Item *Head,const char *S,unsigned long const &Len,bool const &Create); + Item *Lookup(const char *Name,const bool &Create); + inline const Item *Lookup(const char *Name) const + { + return const_cast<Configuration *>(this)->Lookup(Name,false); + } + + public: + + std::string Find(const char *Name,const char *Default = 0) const; + std::string Find(std::string const &Name,const char *Default = 0) const {return Find(Name.c_str(),Default);}; + std::string Find(std::string const &Name, std::string const &Default) const {return Find(Name.c_str(),Default.c_str());}; + std::string FindFile(const char *Name,const char *Default = 0) const; + std::string FindDir(const char *Name,const char *Default = 0) const; + /** return a list of child options + * + * Options like Acquire::Languages are handled as lists which + * can be overridden and have a default. For the later two a comma + * separated list of values is supported. + * + * \param Name of the parent node + * \param Default list of values separated by commas */ + std::vector<std::string> FindVector(const char *Name, std::string const &Default = "", bool const Keys = false) const; + std::vector<std::string> FindVector(std::string const &Name, std::string const &Default = "", bool const Keys = false) const { return FindVector(Name.c_str(), Default, Keys); }; + + int FindI(const char *Name,int const &Default = 0) const; + int FindI(std::string const &Name,int const &Default = 0) const {return FindI(Name.c_str(),Default);}; + bool FindB(const char *Name,bool const &Default = false) const; + bool FindB(std::string const &Name,bool const &Default = false) const {return FindB(Name.c_str(),Default);}; + std::string FindAny(const char *Name,const char *Default = 0) const; + + inline void Set(const std::string &Name,const std::string &Value) {Set(Name.c_str(),Value);}; + void CndSet(const char *Name,const std::string &Value); + void CndSet(const char *Name,const int Value); + void Set(const char *Name,const std::string &Value); + void Set(const char *Name,const int &Value); + + inline bool Exists(const std::string &Name) const {return Exists(Name.c_str());}; + bool Exists(const char *Name) const; + bool ExistsAny(const char *Name) const; + + void MoveSubTree(char const * const OldRoot, char const * const NewRoot); + + // clear a whole tree + void Clear(const std::string &Name); + void Clear(); + + // remove a certain value from a list (e.g. the list of "APT::Keep-Fds") + void Clear(std::string const &List, std::string const &Value); + void Clear(std::string const &List, int const &Value); + + inline const Item *Tree(const char *Name) const {return Lookup(Name);}; + + inline void Dump() { Dump(std::clog); }; + void Dump(std::ostream& str); + void Dump(std::ostream& str, char const * const root, + char const * const format, bool const emptyValue); + + explicit Configuration(const Item *Root); + Configuration(); + ~Configuration(); + + /** \brief match a string against a configurable list of patterns */ + class MatchAgainstConfig + { + std::vector<regex_t *> patterns; + APT_HIDDEN void clearPatterns(); + + public: + explicit MatchAgainstConfig(char const * Config); + virtual ~MatchAgainstConfig(); + + /** \brief Returns \b true for a string matching one of the patterns */ + bool Match(char const * str) const; + bool Match(std::string const &str) const { return Match(str.c_str()); }; + + /** \brief returns if the matcher setup was successful */ + bool wasConstructedSuccessfully() const { return patterns.empty() == false; } + }; +}; + +APT_PUBLIC extern Configuration *_config; + +APT_PUBLIC bool ReadConfigFile(Configuration &Conf,const std::string &FName, + bool const &AsSectional = false, + unsigned const &Depth = 0); + +APT_PUBLIC bool ReadConfigDir(Configuration &Conf,const std::string &Dir, + bool const &AsSectional = false, + unsigned const &Depth = 0); + +#endif diff --git a/apt-pkg/contrib/error.cc b/apt-pkg/contrib/error.cc new file mode 100644 index 0000000..dd28af1 --- /dev/null +++ b/apt-pkg/contrib/error.cc @@ -0,0 +1,336 @@ +// -*- mode: cpp; mode: fold -*- +// SPDX-License-Identifier: GPL-2.0+ +// Description /*{{{*/ +/* ###################################################################### + + Global Error Class - Global error mechanism + + We use a simple STL vector to store each error record. A PendingFlag + is kept which indicates when the vector contains a Sever error. + + This file had this historic note, but now includes further changes + under the GPL-2.0+: + + This source is placed in the Public Domain, do with it what you will + It was originally written by Jason Gunthorpe. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> + +#include <algorithm> +#include <cstring> +#include <iostream> +#include <list> +#include <string> +#include <errno.h> +#include <stdarg.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + + /*}}}*/ + +// Global Error Object /*{{{*/ +GlobalError *_GetErrorObj() +{ + static thread_local GlobalError Obj; + return &Obj; +} + /*}}}*/ +// GlobalError::GlobalError - Constructor /*{{{*/ +GlobalError::GlobalError() : PendingFlag(false) {} + /*}}}*/ +// GlobalError::FatalE, Errno, WarningE, NoticeE and DebugE - Add to the list/*{{{*/ +#define GEMessage(NAME, TYPE) \ +bool GlobalError::NAME (const char *Function, const char *Description,...) { \ + va_list args; \ + size_t msgSize = 400; \ + int const errsv = errno; \ + bool retry; \ + do { \ + va_start(args,Description); \ + retry = InsertErrno(TYPE, Function, Description, args, errsv, msgSize); \ + va_end(args); \ + } while (retry); \ + return false; \ +} +GEMessage(FatalE, FATAL) +GEMessage(Errno, ERROR) +GEMessage(WarningE, WARNING) +GEMessage(NoticeE, NOTICE) +GEMessage(DebugE, DEBUG) +#undef GEMessage + /*}}}*/ +// GlobalError::InsertErrno - Get part of the errortype string from errno/*{{{*/ +bool GlobalError::InsertErrno(MsgType const &type, const char *Function, + const char *Description,...) { + va_list args; + size_t msgSize = 400; + int const errsv = errno; + bool retry; + do { + va_start(args,Description); + retry = InsertErrno(type, Function, Description, args, errsv, msgSize); + va_end(args); + } while (retry); + return false; +} + /*}}}*/ +// GlobalError::InsertErrno - formats an error message with the errno /*{{{*/ +bool GlobalError::InsertErrno(MsgType type, const char* Function, + const char* Description, va_list &args, + int const errsv, size_t &msgSize) { + char* S = (char*) malloc(msgSize); + int const n = snprintf(S, msgSize, "%s - %s (%i: %s)", Description, + Function, errsv, strerror(errsv)); + if (n > -1 && ((unsigned int) n) < msgSize); + else { + if (n > -1) + msgSize = n + 1; + else + msgSize *= 2; + free(S); + return true; + } + + bool const geins = Insert(type, S, args, msgSize); + free(S); + return geins; +} + /*}}}*/ +// GlobalError::Fatal, Error, Warning, Notice and Debug - Add to the list/*{{{*/ +#define GEMessage(NAME, TYPE) \ +bool GlobalError::NAME (const char *Description,...) { \ + va_list args; \ + size_t msgSize = 400; \ + bool retry; \ + do { \ + va_start(args,Description); \ + retry = Insert(TYPE, Description, args, msgSize); \ + va_end(args); \ + } while (retry); \ + return false; \ +} +GEMessage(Fatal, FATAL) +GEMessage(Error, ERROR) +GEMessage(Warning, WARNING) +GEMessage(Notice, NOTICE) +GEMessage(Debug, DEBUG) +#undef GEMessage + /*}}}*/ +// GlobalError::Insert - Add a errotype message to the list /*{{{*/ +bool GlobalError::Insert(MsgType const &type, const char *Description,...) +{ + va_list args; + size_t msgSize = 400; + bool retry; + do { + va_start(args,Description); + retry = Insert(type, Description, args, msgSize); + va_end(args); + } while (retry); + return false; +} + /*}}}*/ +// GlobalError::Insert - Insert a new item at the end /*{{{*/ +bool GlobalError::Insert(MsgType type, const char* Description, + va_list &args, size_t &msgSize) { + char* S = (char*) malloc(msgSize); + int const n = vsnprintf(S, msgSize, Description, args); + if (n > -1 && ((unsigned int) n) < msgSize); + else { + if (n > -1) + msgSize = n + 1; + else + msgSize *= 2; + free(S); + return true; + } + + Item const m(S, type); + Messages.push_back(m); + + if (type == ERROR || type == FATAL) + PendingFlag = true; + + if (type == FATAL || type == DEBUG) + std::clog << m << std::endl; + + free(S); + return false; +} + /*}}}*/ +// GlobalError::PopMessage - Pulls a single message out /*{{{*/ +bool GlobalError::PopMessage(std::string &Text) { + if (Messages.empty() == true) + return false; + + Item const msg = Messages.front(); + Messages.pop_front(); + + bool const Ret = (msg.Type == ERROR || msg.Type == FATAL); + Text = msg.Text; + if (PendingFlag == false || Ret == false) + return Ret; + + // check if another error message is pending + for (std::list<Item>::const_iterator m = Messages.begin(); + m != Messages.end(); ++m) + if (m->Type == ERROR || m->Type == FATAL) + return Ret; + + PendingFlag = false; + return Ret; +} + /*}}}*/ +// GlobalError::DumpErrors - Dump all of the errors/warns to cerr /*{{{*/ +void GlobalError::DumpErrors(std::ostream &out, MsgType const &threshold, + bool const &mergeStack) { + if (mergeStack == true) + for (std::list<MsgStack>::const_reverse_iterator s = Stacks.rbegin(); + s != Stacks.rend(); ++s) + std::copy(s->Messages.begin(), s->Messages.end(), std::front_inserter(Messages)); + + std::for_each(Messages.begin(), Messages.end(), [&threshold, &out](Item const &m) { + if (m.Type >= threshold) + out << m << std::endl; + }); + + Discard(); +} + /*}}}*/ +// GlobalError::Discard - Discard /*{{{*/ +void GlobalError::Discard() { + Messages.clear(); + PendingFlag = false; +} + /*}}}*/ +// GlobalError::empty - does our error list include anything? /*{{{*/ +bool GlobalError::empty(MsgType const &threshold) const { + if (PendingFlag == true) + return false; + + if (Messages.empty() == true) + return true; + + return std::find_if(Messages.begin(), Messages.end(), [&threshold](Item const &m) { + return m.Type >= threshold; + }) == Messages.end(); +} + /*}}}*/ +// GlobalError::PushToStack /*{{{*/ +void GlobalError::PushToStack() { + Stacks.emplace_back(Messages, PendingFlag); + Discard(); +} + /*}}}*/ +// GlobalError::RevertToStack /*{{{*/ +void GlobalError::RevertToStack() { + Discard(); + MsgStack pack = Stacks.back(); + Messages = pack.Messages; + PendingFlag = pack.PendingFlag; + Stacks.pop_back(); +} + /*}}}*/ +// GlobalError::MergeWithStack /*{{{*/ +void GlobalError::MergeWithStack() { + MsgStack pack = Stacks.back(); + Messages.splice(Messages.begin(), pack.Messages); + PendingFlag = PendingFlag || pack.PendingFlag; + Stacks.pop_back(); +} + /*}}}*/ + +// GlobalError::Item::operator<< /*{{{*/ +APT_HIDDEN std::ostream &operator<<(std::ostream &out, GlobalError::Item i) +{ + static constexpr auto COLOR_RESET = "\033[0m"; + static constexpr auto COLOR_NOTICE = "\033[33m"; // normal yellow + static constexpr auto COLOR_WARN = "\033[1;33m"; // bold yellow + static constexpr auto COLOR_ERROR = "\033[1;31m"; // bold red + + bool use_color = _config->FindB("APT::Color", false); + + if (use_color) + { + switch (i.Type) + { + case GlobalError::FATAL: + case GlobalError::ERROR: + out << COLOR_ERROR; + break; + case GlobalError::WARNING: + out << COLOR_WARN; + break; + case GlobalError::NOTICE: + out << COLOR_NOTICE; + break; + default: + break; + } + } + + switch (i.Type) + { + case GlobalError::FATAL: + case GlobalError::ERROR: + out << 'E'; + break; + case GlobalError::WARNING: + out << 'W'; + break; + case GlobalError::NOTICE: + out << 'N'; + break; + case GlobalError::DEBUG: + out << 'D'; + break; + } + out << ": "; + + if (use_color) + { + switch (i.Type) + { + case GlobalError::FATAL: + case GlobalError::ERROR: + case GlobalError::WARNING: + case GlobalError::NOTICE: + out << COLOR_RESET; + break; + default: + break; + } + } + + std::string::size_type line_start = 0; + std::string::size_type line_end; + while ((line_end = i.Text.find_first_of("\n\r", line_start)) != std::string::npos) + { + if (line_start != 0) + out << std::endl + << " "; + out << i.Text.substr(line_start, line_end - line_start); + line_start = i.Text.find_first_not_of("\n\r", line_end + 1); + if (line_start == std::string::npos) + break; + } + if (line_start == 0) + out << i.Text; + else if (line_start != std::string::npos) + out << std::endl + << " " << i.Text.substr(line_start); + + if (use_color) + out << COLOR_RESET; + + return out; +} + /*}}}*/ diff --git a/apt-pkg/contrib/error.h b/apt-pkg/contrib/error.h new file mode 100644 index 0000000..dd50b9c --- /dev/null +++ b/apt-pkg/contrib/error.h @@ -0,0 +1,349 @@ +// -*- mode: cpp; mode: fold -*- +// SPDX-License-Identifier: GPL-2.0+ +// Description /*{{{*/ +/* ###################################################################### + + Global Error Class - Global error mechanism + + This class has a single global instance. When a function needs to + generate an error condition, such as a read error, it calls a member + in this class to add the error to a stack of errors. + + By using a stack the problem with a scheme like errno is removed and + it allows a very detailed account of what went wrong to be transmitted + to the UI for display. (Errno has problems because each function sets + errno to 0 if it didn't have an error thus eraseing erno in the process + of cleanup) + + Several predefined error generators are provided to handle common + things like errno. The general idea is that all methods return a bool. + If the bool is true then things are OK, if it is false then things + should start being undone and the stack should unwind under program + control. + + A Warning should not force the return of false. Things did not fail, but + they might have had unexpected problems. Errors are stored in a FIFO + so Pop will return the first item.. + + I have some thoughts about extending this into a more general UI<-> + Engine interface, ie allowing the Engine to say 'The disk is full' in + a dialog that says 'Panic' and 'Retry'.. The error generator functions + like errno, Warning and Error return false always so this is normal: + if (open(..)) + return _error->Errno(..); + + This file had this historic note, but now includes further changes + under the GPL-2.0+: + + This source is placed in the Public Domain, do with it what you will + It was originally written by Jason Gunthorpe. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_ERROR_H +#define PKGLIB_ERROR_H + +#include <apt-pkg/macros.h> + +#include <iostream> +#include <list> +#include <string> + +#include <stdarg.h> +#include <stddef.h> + +class APT_PUBLIC GlobalError /*{{{*/ +{ +public: /*{{{*/ + /** \brief a message can have one of following severity */ + enum MsgType { + /** \brief Message will be printed instantly as it is likely that + this error will lead to a complete crash */ + FATAL = 40, + /** \brief An error does hinder the correct execution and should be corrected */ + ERROR = 30, + /** \brief indicates problem that can lead to errors later on */ + WARNING = 20, + /** \brief deprecation warnings, old fallback behavior, … */ + NOTICE = 10, + /** \brief for developers only in areas it is hard to print something directly */ + DEBUG = 0 + }; + + /** \brief add a fatal error message with errno to the list + * + * \param Function name of the function generating the error + * \param Description format string for the error message + * + * \return \b false + */ + bool FatalE(const char *Function,const char *Description,...) APT_PRINTF(3) APT_COLD; + + /** \brief add an Error message with errno to the list + * + * \param Function name of the function generating the error + * \param Description format string for the error message + * + * \return \b false + */ + bool Errno(const char *Function,const char *Description,...) APT_PRINTF(3) APT_COLD; + + /** \brief add a warning message with errno to the list + * + * A warning should be considered less severe than an error and + * may be ignored by the client. + * + * \param Function Name of the function generates the warning. + * \param Description Format string for the warning message. + * + * \return \b false + */ + bool WarningE(const char *Function,const char *Description,...) APT_PRINTF(3) APT_COLD; + + /** \brief add a notice message with errno to the list + * + * \param Function name of the function generating the error + * \param Description format string for the error message + * + * \return \b false + */ + bool NoticeE(const char *Function,const char *Description,...) APT_PRINTF(3) APT_COLD; + + /** \brief add a debug message with errno to the list + * + * \param Function name of the function generating the error + * \param Description format string for the error message + * + * \return \b false + */ + bool DebugE(const char *Function,const char *Description,...) APT_PRINTF(3) APT_COLD; + + /** \brief adds an errno message with the given type + * + * \param type of the error message + * \param Function which failed + * \param Description of the error + */ + bool InsertErrno(MsgType const &type, const char* Function, + const char* Description,...) APT_PRINTF(4) APT_COLD; + + /** \brief adds an errno message with the given type + * + * args needs to be initialized with va_start and terminated + * with va_end by the caller. msgSize is also an out-parameter + * in case the msgSize was not enough to store the complete message. + * + * \param type of the error message + * \param Function which failed + * \param Description is the format string for args + * \param args list from a printf-like function + * \param errsv is the errno the error is for + * \param msgSize is the size of the char[] used to store message + * \return true if the message was added, false if not - the caller + * should call this method again in that case + */ + bool InsertErrno(MsgType type, const char* Function, + const char* Description, va_list &args, + int const errsv, size_t &msgSize) APT_COLD; + + /** \brief add an fatal error message to the list + * + * Most of the stuff we consider as "error" is also "fatal" for + * the user as the application will not have the expected result, + * but a fatal message here means that it gets printed directly + * to stderr in addition to adding it to the list as the error + * leads sometimes to crashes and a maybe duplicated message + * is better than "Segfault" as the only displayed text + * + * \param Description Format string for the fatal error message. + * + * \return \b false + */ + bool Fatal(const char *Description,...) APT_PRINTF(2) APT_COLD; + + /** \brief add an Error message to the list + * + * \param Description Format string for the error message. + * + * \return \b false + */ + bool Error(const char *Description,...) APT_PRINTF(2) APT_COLD; + + /** \brief add a warning message to the list + * + * A warning should be considered less severe than an error and + * may be ignored by the client. + * + * \param Description Format string for the message + * + * \return \b false + */ + bool Warning(const char *Description,...) APT_PRINTF(2) APT_COLD; + + /** \brief add a notice message to the list + * + * A notice should be considered less severe than an error or a + * warning and can be ignored by the client without further problems + * for some times, but he should consider fixing the problem. + * This error type can be used for e.g. deprecation warnings of options. + * + * \param Description Format string for the message + * + * \return \b false + */ + bool Notice(const char *Description,...) APT_PRINTF(2) APT_COLD; + + /** \brief add a debug message to the list + * + * \param Description Format string for the message + * + * \return \b false + */ + bool Debug(const char *Description,...) APT_PRINTF(2) APT_COLD; + + /** \brief adds an error message with the given type + * + * \param type of the error message + * \param Description of the error + */ + bool Insert(MsgType const &type, const char* Description,...) APT_PRINTF(3) APT_COLD; + + /** \brief adds an error message with the given type + * + * args needs to be initialized with va_start and terminated + * with va_end by the caller. msgSize is also an out-parameter + * in case the msgSize was not enough to store the complete message. + * + * \param type of the error message + * \param Description is the format string for args + * \param args list from a printf-like function + * \param msgSize is the size of the char[] used to store message + * \return true if the message was added, false if not - the caller + * should call this method again in that case + */ + bool Insert(MsgType type, const char* Description, + va_list &args, size_t &msgSize) APT_COLD; + + /** \brief is an error in the list? + * + * \return \b true if an error is included in the list, \b false otherwise + */ + inline bool PendingError() const APT_PURE {return PendingFlag;}; + + /** \brief is the list empty? + * + * Can be used to check if the current stack level doesn't include + * anything equal or more severe than a given threshold, defaulting + * to warning level for historic reasons. + * + * \param threshold minimum level considered + * + * \return \b true if the list is empty, \b false otherwise + */ + bool empty(MsgType const &threshold = WARNING) const APT_PURE; + + /** \brief returns and removes the first (or last) message in the list + * + * \param[out] Text message of the first/last item + * + * \return \b true if the message was an error, \b false otherwise + */ + bool PopMessage(std::string &Text); + + /** \brief clears the list of messages */ + void Discard(); + + /** \brief outputs the list of messages to the given stream + * + * Note that all messages are discarded, even undisplayed ones. + * + * \param[out] out output stream to write the messages in + * \param threshold minimum level considered + * \param mergeStack if true recursively dumps the entire stack + */ + void DumpErrors(std::ostream &out, MsgType const &threshold = WARNING, + bool const &mergeStack = true); + + /** \brief dumps the list of messages to std::cerr + * + * Note that all messages are discarded, also the notices + * displayed or not. + * + * \param threshold minimum level printed + */ + void inline DumpErrors(MsgType const &threshold) { + DumpErrors(std::cerr, threshold); + } + + // mvo: we do this instead of using a default parameter in the + // previous declaration to avoid a (subtle) API break for + // e.g. sigc++ and mem_fun0 + /** \brief dumps the messages of type WARNING or higher to std::cerr + * + * Note that all messages are discarded, displayed or not. + * + */ + void inline DumpErrors() { + DumpErrors(WARNING); + } + + /** \brief put the current Messages into the stack + * + * All "old" messages will be pushed into a stack to + * them later back, but for now the Message query will be + * empty and performs as no messages were present before. + * + * The stack can be as deep as you want - all stack operations + * will only operate on the last element in the stack. + */ + void PushToStack(); + + /** \brief throw away all current messages */ + void RevertToStack(); + + /** \brief merge current and stack together */ + void MergeWithStack(); + + /** \brief return the deep of the stack */ + size_t StackCount() const APT_PURE { + return Stacks.size(); + } + + GlobalError(); + /*}}}*/ +private: /*{{{*/ + struct Item { + std::string Text; + MsgType Type; + + Item(char const *Text, MsgType const &Type) : + Text(Text), Type(Type) {}; + + APT_HIDDEN friend std::ostream &operator<<(std::ostream &out, Item i); + }; + + APT_HIDDEN friend std::ostream &operator<<(std::ostream &out, Item i); + + std::list<Item> Messages; + bool PendingFlag; + + struct MsgStack { + std::list<Item> Messages; + bool const PendingFlag; + + MsgStack(std::list<Item> const &Messages, bool const &Pending) : + Messages(Messages), PendingFlag(Pending) {}; + }; + + std::list<MsgStack> Stacks; + /*}}}*/ +}; + /*}}}*/ + +// The 'extra-ansi' syntax is used to help with collisions. +APT_PUBLIC GlobalError *_GetErrorObj(); +static struct { + inline GlobalError* operator ->() { return _GetErrorObj(); } +} _error APT_UNUSED; + +#endif diff --git a/apt-pkg/contrib/extracttar.cc b/apt-pkg/contrib/extracttar.cc new file mode 100644 index 0000000..dc24375 --- /dev/null +++ b/apt-pkg/contrib/extracttar.cc @@ -0,0 +1,325 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Extract a Tar - Tar Extractor + + Some performance measurements showed that zlib performed quite poorly + in comparison to a forked gzip process. This tar extractor makes use + of the fact that dup'd file descriptors have the same seek pointer + and that gzip will not read past the end of a compressed stream, + even if there is more data. We use the dup property to track extraction + progress and the gzip feature to just feed gzip a fd in the middle + of an AR file. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/dirstream.h> +#include <apt-pkg/error.h> +#include <apt-pkg/extracttar.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <iostream> +#include <string> +#include <fcntl.h> +#include <signal.h> +#include <string.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +// The on disk header for a tar file. +struct ExtractTar::TarHeader +{ + char Name[100]; + char Mode[8]; + char UserID[8]; + char GroupID[8]; + char Size[12]; + char MTime[12]; + char Checksum[8]; + char LinkFlag; + char LinkName[100]; + char MagicNumber[8]; + char UserName[32]; + char GroupName[32]; + char Major[8]; + char Minor[8]; +}; + +// We need to read long names (names and link targets) into memory, so let's +// have a limit (shamelessly stolen from libarchive) to avoid people OOMing +// us with large streams. +static const unsigned long long APT_LONGNAME_LIMIT = 1048576llu; + +// A file size limit that we allow extracting. Currently, that's 128 GB. +// We also should leave some wiggle room for code adding files to it, and +// possibly conversion for signed, so this should not be larger than like 2**62. +static const unsigned long long APT_FILESIZE_LIMIT = 1llu << 37; + +// ExtractTar::ExtractTar - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +ExtractTar::ExtractTar(FileFd &Fd,unsigned long long Max,string DecompressionProgram) + : File(Fd), MaxInSize(Max), DecompressProg(DecompressionProgram) +{ + GZPid = -1; + Eof = false; +} + /*}}}*/ +// ExtractTar::ExtractTar - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +ExtractTar::~ExtractTar() +{ + // Error close + Done(); +} + /*}}}*/ +// ExtractTar::Done - Reap the gzip sub process /*{{{*/ +bool ExtractTar::Done() +{ + return InFd.Close(); +} + /*}}}*/ +// ExtractTar::StartGzip - Startup gzip /*{{{*/ +// --------------------------------------------------------------------- +/* This creates a gzip sub process that has its input as the file itself. + If this tar file is embedded into something like an ar file then + gzip will efficiently ignore the extra bits. */ +bool ExtractTar::StartGzip() +{ + if (DecompressProg.empty()) + { + InFd.OpenDescriptor(File.Fd(), FileFd::ReadOnly, FileFd::None, false); + return true; + } + + std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors(); + std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin(); + for (; compressor != compressors.end(); ++compressor) { + if (compressor->Name == DecompressProg) { + return InFd.OpenDescriptor(File.Fd(), FileFd::ReadOnly, *compressor, false); + } + } + + return _error->Error(_("Cannot find a configured compressor for '%s'"), + DecompressProg.c_str()); + +} + /*}}}*/ +// ExtractTar::Go - Perform extraction /*{{{*/ +// --------------------------------------------------------------------- +/* This reads each 512 byte block from the archive and extracts the header + information into the Item structure. Then it resolves the UID/GID and + invokes the correct processing function. */ +bool ExtractTar::Go(pkgDirStream &Stream) +{ + if (StartGzip() == false) + return false; + + // Loop over all blocks + string LastLongLink, ItemLink; + string LastLongName, ItemName; + while (1) + { + bool BadRecord = false; + unsigned char Block[512]; + if (InFd.Read(Block,sizeof(Block),true) == false) + return false; + + if (InFd.Eof() == true) + break; + + // Get the checksum + TarHeader *Tar = (TarHeader *)Block; + unsigned long CheckSum; + if (StrToNum(Tar->Checksum,CheckSum,sizeof(Tar->Checksum),8) == false) + return _error->Error(_("Corrupted archive")); + + /* Compute the checksum field. The actual checksum is blanked out + with spaces so it is not included in the computation */ + unsigned long NewSum = 0; + memset(Tar->Checksum,' ',sizeof(Tar->Checksum)); + for (int I = 0; I != sizeof(Block); I++) + NewSum += Block[I]; + + /* Check for a block of nulls - in this case we kill gzip, GNU tar + does this.. */ + if (NewSum == ' '*sizeof(Tar->Checksum)) + return Done(); + + if (NewSum != CheckSum) + return _error->Error(_("Tar checksum failed, archive corrupted")); + + // Decode all of the fields + pkgDirStream::Item Itm; + if (StrToNum(Tar->Mode,Itm.Mode,sizeof(Tar->Mode),8) == false || + (Base256ToNum(Tar->UserID,Itm.UID,8) == false && + StrToNum(Tar->UserID,Itm.UID,sizeof(Tar->UserID),8) == false) || + (Base256ToNum(Tar->GroupID,Itm.GID,8) == false && + StrToNum(Tar->GroupID,Itm.GID,sizeof(Tar->GroupID),8) == false) || + (Base256ToNum(Tar->Size,Itm.Size,12) == false && + StrToNum(Tar->Size,Itm.Size,sizeof(Tar->Size),8) == false) || + (Base256ToNum(Tar->MTime,Itm.MTime,12) == false && + StrToNum(Tar->MTime,Itm.MTime,sizeof(Tar->MTime),8) == false) || + StrToNum(Tar->Major,Itm.Major,sizeof(Tar->Major),8) == false || + StrToNum(Tar->Minor,Itm.Minor,sizeof(Tar->Minor),8) == false) + return _error->Error(_("Corrupted archive")); + + // Security check. Prevents overflows below the code when rounding up in skip/copy code, + // and provides modest protection against decompression bombs. + if (Itm.Size > APT_FILESIZE_LIMIT) + return _error->Error("Tar member too large: %llu > %llu bytes", Itm.Size, APT_FILESIZE_LIMIT); + + // Grab the filename and link target: use last long name if one was + // set, otherwise use the header value as-is, but remember that it may + // fill the entire 100-byte block and needs to be zero-terminated. + // See Debian Bug #689582. + if (LastLongName.empty() == false) + Itm.Name = (char *)LastLongName.c_str(); + else + Itm.Name = (char *)ItemName.assign(Tar->Name, sizeof(Tar->Name)).c_str(); + if (Itm.Name[0] == '.' && Itm.Name[1] == '/' && Itm.Name[2] != 0) + Itm.Name += 2; + + if (LastLongLink.empty() == false) + Itm.LinkTarget = (char *)LastLongLink.c_str(); + else + Itm.LinkTarget = (char *)ItemLink.assign(Tar->LinkName, sizeof(Tar->LinkName)).c_str(); + + // Convert the type over + switch (Tar->LinkFlag) + { + case NormalFile0: + case NormalFile: + Itm.Type = pkgDirStream::Item::File; + break; + + case HardLink: + Itm.Type = pkgDirStream::Item::HardLink; + break; + + case SymbolicLink: + Itm.Type = pkgDirStream::Item::SymbolicLink; + break; + + case CharacterDevice: + Itm.Type = pkgDirStream::Item::CharDevice; + break; + + case BlockDevice: + Itm.Type = pkgDirStream::Item::BlockDevice; + break; + + case Directory: + Itm.Type = pkgDirStream::Item::Directory; + break; + + case FIFO: + Itm.Type = pkgDirStream::Item::FIFO; + break; + + case GNU_LongLink: + { + unsigned long long Length = Itm.Size; + unsigned char Block[512]; + if (Length > APT_LONGNAME_LIMIT) + return _error->Error("Long name to large: %llu bytes > %llu bytes", Length, APT_LONGNAME_LIMIT); + while (Length > 0) + { + if (InFd.Read(Block,sizeof(Block),true) == false) + return false; + if (Length <= sizeof(Block)) + { + LastLongLink.append(Block,Block+sizeof(Block)); + break; + } + LastLongLink.append(Block,Block+sizeof(Block)); + Length -= sizeof(Block); + } + continue; + } + + case GNU_LongName: + { + unsigned long long Length = Itm.Size; + unsigned char Block[512]; + if (Length > APT_LONGNAME_LIMIT) + return _error->Error("Long name to large: %llu bytes > %llu bytes", Length, APT_LONGNAME_LIMIT); + while (Length > 0) + { + if (InFd.Read(Block,sizeof(Block),true) == false) + return false; + if (Length < sizeof(Block)) + { + LastLongName.append(Block,Block+sizeof(Block)); + break; + } + LastLongName.append(Block,Block+sizeof(Block)); + Length -= sizeof(Block); + } + continue; + } + + default: + BadRecord = true; + _error->Warning(_("Unknown TAR header type %u"), (unsigned)Tar->LinkFlag); + break; + } + + int Fd = -1; + if (not BadRecord && not Stream.DoItem(Itm, Fd)) + return false; + + if (Fd == -1 || Fd < -2 || BadRecord) + { + if (Itm.Size > 0 && not InFd.Skip(((Itm.Size + (sizeof(Block) - 1)) / sizeof(Block)) * sizeof(Block))) + return false; + } + else if (Itm.Size != 0) + { + // Copy the file over the FD + auto Size = Itm.Size; + unsigned char Junk[32*1024]; + do + { + auto const Read = std::min<unsigned long long>(Size, sizeof(Junk)); + if (not InFd.Read(Junk, ((Read + (sizeof(Block) - 1)) / sizeof(Block)) * sizeof(Block))) + return false; + + if (Fd > 0) + { + if (not FileFd::Write(Fd, Junk, Read)) + return Stream.Fail(Itm, Fd); + } + // An Fd of -2 means to send to a special processing function + else if (Fd == -2) + { + if (not Stream.Process(Itm, Junk, Read, Itm.Size - Size)) + return Stream.Fail(Itm, Fd); + } + + Size -= Read; + } while (Size != 0); + } + + // And finish up + if (not BadRecord && not Stream.FinishedFile(Itm, Fd)) + return false; + LastLongName.erase(); + LastLongLink.erase(); + } + + return Done(); +} + /*}}}*/ diff --git a/apt-pkg/contrib/extracttar.h b/apt-pkg/contrib/extracttar.h new file mode 100644 index 0000000..a3c862a --- /dev/null +++ b/apt-pkg/contrib/extracttar.h @@ -0,0 +1,55 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Extract a Tar - Tar Extractor + + The tar extractor takes an ordinary gzip compressed tar stream from + the given file and explodes it, passing the individual items to the + given Directory Stream for processing. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_EXTRACTTAR_H +#define PKGLIB_EXTRACTTAR_H + +#include <apt-pkg/fileutl.h> +#include <apt-pkg/macros.h> + +#include <string> + + +class pkgDirStream; + +class APT_PUBLIC ExtractTar +{ + protected: + + struct TarHeader; + + // The varios types items can be + enum ItemType {NormalFile0 = '\0',NormalFile = '0',HardLink = '1', + SymbolicLink = '2',CharacterDevice = '3', + BlockDevice = '4',Directory = '5',FIFO = '6', + GNU_LongLink = 'K',GNU_LongName = 'L'}; + + FileFd &File; + unsigned long long MaxInSize; + int GZPid; + FileFd InFd; + bool Eof; + std::string DecompressProg; + + // Fork and reap gzip + bool StartGzip(); + bool Done(); + + public: + + bool Go(pkgDirStream &Stream); + + ExtractTar(FileFd &Fd,unsigned long long Max,std::string DecompressionProgram); + virtual ~ExtractTar(); +}; + +#endif diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc new file mode 100644 index 0000000..fad51de --- /dev/null +++ b/apt-pkg/contrib/fileutl.cc @@ -0,0 +1,3483 @@ +// -*- mode: cpp; mode: fold -*- +// SPDX-License-Identifier: GPL-2.0+ +// Description /*{{{*/ +/* ###################################################################### + + File Utilities + + CopyFile - Buffered copy of a single file + GetLock - dpkg compatible lock file manipulation (fcntl) + + This file had this historic note, but now includes further changes + under the GPL-2.0+: + + Most of this source is placed in the Public Domain, do with it what + you will + It was originally written by Jason Gunthorpe <jgg@debian.org>. + FileFd gzip support added by Martin Pitt <martin.pitt@canonical.com> + + The exception is RunScripts() it is under the GPLv2 + + We believe that this reference to GPLv2 was not meant to exclude later + versions as that would have changed the overall project license from GPL-2+ + to GPL-2. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgsystem.h> +#include <apt-pkg/strutl.h> + +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <iostream> +#include <string> +#include <vector> +#include <ctype.h> +#include <dirent.h> +#include <errno.h> +#include <fcntl.h> +#include <glob.h> +#include <grp.h> +#include <pwd.h> +#include <signal.h> +#include <stdarg.h> +#include <stddef.h> +#include <stdio.h> +#include <sys/select.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/wait.h> +#include <time.h> +#include <unistd.h> + +#include <algorithm> +#include <memory> +#include <set> + +#ifdef HAVE_ZLIB +#include <zlib.h> +#endif +#ifdef HAVE_BZ2 +#include <bzlib.h> +#endif +#ifdef HAVE_LZMA +#include <lzma.h> +#endif +#ifdef HAVE_LZ4 +#include <lz4frame.h> +#endif +#ifdef HAVE_ZSTD +#include <zstd.h> +#endif +#ifdef HAVE_SYSTEMD +#include <systemd/sd-bus.h> +#endif +#include <endian.h> +#include <stdint.h> + +#if __gnu_linux__ +#include <sys/prctl.h> +#endif + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +// RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool RunScripts(const char *Cnf) +{ + Configuration::Item const *Opts = _config->Tree(Cnf); + if (Opts == 0 || Opts->Child == 0) + return true; + Opts = Opts->Child; + + // Fork for running the system calls + pid_t Child = ExecFork(); + + // This is the child + if (Child == 0) + { + if (_system != nullptr && _system->IsLocked() == true && (stringcasecmp(Cnf, "dpkg::post-invoke") == 0 || stringcasecmp(Cnf, "dpkg::pre-invoke") == 0)) + setenv("DPKG_FRONTEND_LOCKED", "true", 1); + if (_config->FindDir("DPkg::Chroot-Directory","/") != "/") + { + std::cerr << "Chrooting into " + << _config->FindDir("DPkg::Chroot-Directory") + << std::endl; + if (chroot(_config->FindDir("DPkg::Chroot-Directory","/").c_str()) != 0) + _exit(100); + } + + if (chdir("/tmp/") != 0) + _exit(100); + + unsigned int Count = 1; + for (; Opts != 0; Opts = Opts->Next, Count++) + { + if (Opts->Value.empty() == true) + continue; + + if(_config->FindB("Debug::RunScripts", false) == true) + std::clog << "Running external script: '" + << Opts->Value << "'" << std::endl; + + if (system(Opts->Value.c_str()) != 0) + _exit(100+Count); + } + _exit(0); + } + + // Wait for the child + int Status = 0; + while (waitpid(Child,&Status,0) != Child) + { + if (errno == EINTR) + continue; + return _error->Errno("waitpid","Couldn't wait for subprocess"); + } + + // Check for an error code. + if (WIFEXITED(Status) == 0 || WEXITSTATUS(Status) != 0) + { + unsigned int Count = WEXITSTATUS(Status); + if (Count > 100) + { + Count -= 100; + for (; Opts != 0 && Count != 1; Opts = Opts->Next, Count--); + _error->Error("Problem executing scripts %s '%s'",Cnf,Opts->Value.c_str()); + } + + return _error->Error("Sub-process returned an error code"); + } + + return true; +} + /*}}}*/ + +// CopyFile - Buffered copy of a file /*{{{*/ +// --------------------------------------------------------------------- +/* The caller is expected to set things so that failure causes erasure */ +bool CopyFile(FileFd &From,FileFd &To) +{ + if (From.IsOpen() == false || To.IsOpen() == false || + From.Failed() == true || To.Failed() == true) + return false; + + // Buffered copy between fds + constexpr size_t BufSize = APT_BUFFER_SIZE; + std::unique_ptr<unsigned char[]> Buf(new unsigned char[BufSize]); + unsigned long long ToRead = 0; + do { + if (From.Read(Buf.get(),BufSize, &ToRead) == false || + To.Write(Buf.get(),ToRead) == false) + return false; + } while (ToRead != 0); + + return true; +} + /*}}}*/ +bool RemoveFileAt(char const * const Function, int const dirfd, std::string const &FileName)/*{{{*/ +{ + if (FileName == "/dev/null") + return true; + errno = 0; + if (unlinkat(dirfd, FileName.c_str(), 0) != 0) + { + if (errno == ENOENT) + return true; + + return _error->WarningE(Function,_("Problem unlinking the file %s"), FileName.c_str()); + } + return true; +} + /*}}}*/ +bool RemoveFile(char const * const Function, std::string const &FileName)/*{{{*/ +{ + if (FileName == "/dev/null") + return true; + errno = 0; + if (unlink(FileName.c_str()) != 0) + { + if (errno == ENOENT) + return true; + + return _error->WarningE(Function,_("Problem unlinking the file %s"), FileName.c_str()); + } + return true; +} + /*}}}*/ +// GetLock - Gets a lock file /*{{{*/ +// --------------------------------------------------------------------- +/* This will create an empty file of the given name and lock it. Once this + is done all other calls to GetLock in any other process will fail with + -1. The return result is the fd of the file, the call should call + close at some time. */ + +static std::string GetProcessName(int pid) +{ + struct HideError + { + int err; + HideError() : err(errno) { _error->PushToStack(); } + ~HideError() + { + errno = err; + _error->RevertToStack(); + } + } hideError; + std::string path; + strprintf(path, "/proc/%d/status", pid); + FileFd status(path, FileFd::ReadOnly); + std::string line; + while (status.ReadLine(line)) + { + if (line.substr(0, 5) == "Name:") + return line.substr(6); + } + return ""; +} +int GetLock(string File,bool Errors) +{ + // GetLock() is used in aptitude on directories with public-write access + // Use O_NOFOLLOW here to prevent symlink traversal attacks + int FD = open(File.c_str(),O_RDWR | O_CREAT | O_NOFOLLOW,0640); + if (FD < 0) + { + // Read only .. can't have locking problems there. + if (errno == EROFS) + { + _error->Warning(_("Not using locking for read only lock file %s"),File.c_str()); + return dup(0); // Need something for the caller to close + } + + if (Errors == true) + _error->Errno("open",_("Could not open lock file %s"),File.c_str()); + + // Feh.. We do this to distinguish the lock vs open case.. + errno = EPERM; + return -1; + } + SetCloseExec(FD,true); + + // Acquire a write lock + struct flock fl; + fl.l_type = F_WRLCK; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + if (fcntl(FD,F_SETLK,&fl) == -1) + { + // always close to not leak resources + int Tmp = errno; + + if ((errno == EACCES || errno == EAGAIN)) + { + fl.l_type = F_WRLCK; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + fl.l_pid = -1; + fcntl(FD, F_GETLK, &fl); + } + else + { + fl.l_pid = -1; + } + close(FD); + errno = Tmp; + + if (errno == ENOLCK) + { + _error->Warning(_("Not using locking for nfs mounted lock file %s"),File.c_str()); + return dup(0); // Need something for the caller to close + } + + if (Errors == true) + { + // We only do the lookup in the if ((errno == EACCES || errno == EAGAIN)) + // case, so we do not need to show the errno strerrr here... + if (fl.l_pid != -1) + { + auto name = GetProcessName(fl.l_pid); + if (name.empty()) + _error->Error(_("Could not get lock %s. It is held by process %d"), File.c_str(), fl.l_pid); + else + _error->Error(_("Could not get lock %s. It is held by process %d (%s)"), File.c_str(), fl.l_pid, name.c_str()); + } + else + _error->Errno("open", _("Could not get lock %s"), File.c_str()); + + _error->Notice(_("Be aware that removing the lock file is not a solution and may break your system.")); + } + + return -1; + } + + return FD; +} + /*}}}*/ +// FileExists - Check if a file exists /*{{{*/ +// --------------------------------------------------------------------- +/* Beware: Directories are also files! */ +bool FileExists(string File) +{ + struct stat Buf; + if (stat(File.c_str(),&Buf) != 0) + return false; + return true; +} + /*}}}*/ +// RealFileExists - Check if a file exists and if it is really a file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool RealFileExists(string File) +{ + struct stat Buf; + if (stat(File.c_str(),&Buf) != 0) + return false; + return ((Buf.st_mode & S_IFREG) != 0); +} + /*}}}*/ +// DirectoryExists - Check if a directory exists and is really one /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool DirectoryExists(string const &Path) +{ + struct stat Buf; + if (stat(Path.c_str(),&Buf) != 0) + return false; + return ((Buf.st_mode & S_IFDIR) != 0); +} + /*}}}*/ +// CreateDirectory - poor man's mkdir -p guarded by a parent directory /*{{{*/ +// --------------------------------------------------------------------- +/* This method will create all directories needed for path in good old + mkdir -p style but refuses to do this if Parent is not a prefix of + this Path. Example: /var/cache/ and /var/cache/apt/archives are given, + so it will create apt/archives if /var/cache exists - on the other + hand if the parent is /var/lib the creation will fail as this path + is not a parent of the path to be generated. */ +bool CreateDirectory(string const &Parent, string const &Path) +{ + if (Parent.empty() == true || Path.empty() == true) + return false; + + if (DirectoryExists(Path) == true) + return true; + + if (DirectoryExists(Parent) == false) + return false; + + // we are not going to create directories "into the blue" + if (Path.compare(0, Parent.length(), Parent) != 0) + return false; + + vector<string> const dirs = VectorizeString(Path.substr(Parent.size()), '/'); + string progress = Parent; + for (vector<string>::const_iterator d = dirs.begin(); d != dirs.end(); ++d) + { + if (d->empty() == true) + continue; + + progress.append("/").append(*d); + if (DirectoryExists(progress) == true) + continue; + + if (mkdir(progress.c_str(), 0755) != 0) + return false; + } + return true; +} + /*}}}*/ +// CreateAPTDirectoryIfNeeded - ensure that the given directory exists /*{{{*/ +// --------------------------------------------------------------------- +/* a small wrapper around CreateDirectory to check if it exists and to + remove the trailing "/apt/" from the parent directory if needed */ +bool CreateAPTDirectoryIfNeeded(string const &Parent, string const &Path) +{ + if (DirectoryExists(Path) == true) + return true; + + size_t const len = Parent.size(); + if (len > 5 && Parent.find("/apt/", len - 6, 5) == len - 5) + { + if (CreateDirectory(Parent.substr(0,len-5), Path) == true) + return true; + } + else if (CreateDirectory(Parent, Path) == true) + return true; + + return false; +} + /*}}}*/ +// GetListOfFilesInDir - returns a vector of files in the given dir /*{{{*/ +// --------------------------------------------------------------------- +/* If an extension is given only files with this extension are included + in the returned vector, otherwise every "normal" file is included. */ +std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext, + bool const &SortList, bool const &AllowNoExt) +{ + std::vector<string> ext; + ext.reserve(2); + if (Ext.empty() == false) + ext.push_back(Ext); + if (AllowNoExt == true && ext.empty() == false) + ext.push_back(""); + return GetListOfFilesInDir(Dir, ext, SortList); +} +std::vector<string> GetListOfFilesInDir(string const &Dir, std::vector<string> const &Ext, + bool const &SortList) +{ + // Attention debuggers: need to be set with the environment config file! + bool const Debug = _config->FindB("Debug::GetListOfFilesInDir", false); + if (Debug == true) + { + std::clog << "Accept in " << Dir << " only files with the following " << Ext.size() << " extensions:" << std::endl; + if (Ext.empty() == true) + std::clog << "\tNO extension" << std::endl; + else + for (std::vector<string>::const_iterator e = Ext.begin(); + e != Ext.end(); ++e) + std::clog << '\t' << (e->empty() == true ? "NO" : *e) << " extension" << std::endl; + } + + std::vector<string> List; + + if (DirectoryExists(Dir) == false) + { + _error->Error(_("List of files can't be created as '%s' is not a directory"), Dir.c_str()); + return List; + } + + Configuration::MatchAgainstConfig SilentIgnore("Dir::Ignore-Files-Silently"); + DIR *D = opendir(Dir.c_str()); + if (D == 0) + { + if (errno == EACCES) + _error->WarningE("opendir", _("Unable to read %s"), Dir.c_str()); + else + _error->Errno("opendir", _("Unable to read %s"), Dir.c_str()); + return List; + } + + for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D)) + { + // skip "hidden" files + if (Ent->d_name[0] == '.') + continue; + + // Make sure it is a file and not something else + string const File = flCombine(Dir,Ent->d_name); +#ifdef _DIRENT_HAVE_D_TYPE + if (Ent->d_type != DT_REG) +#endif + { + if (RealFileExists(File) == false) + { + // do not show ignoration warnings for directories + if ( +#ifdef _DIRENT_HAVE_D_TYPE + Ent->d_type == DT_DIR || +#endif + DirectoryExists(File) == true) + continue; + if (SilentIgnore.Match(Ent->d_name) == false) + _error->Notice(_("Ignoring '%s' in directory '%s' as it is not a regular file"), Ent->d_name, Dir.c_str()); + continue; + } + } + + // check for accepted extension: + // no extension given -> periods are bad as hell! + // extensions given -> "" extension allows no extension + if (Ext.empty() == false) + { + string d_ext = flExtension(Ent->d_name); + if (d_ext == Ent->d_name) // no extension + { + if (std::find(Ext.begin(), Ext.end(), "") == Ext.end()) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → no extension" << std::endl; + if (SilentIgnore.Match(Ent->d_name) == false) + _error->Notice(_("Ignoring file '%s' in directory '%s' as it has no filename extension"), Ent->d_name, Dir.c_str()); + continue; + } + } + else if (std::find(Ext.begin(), Ext.end(), d_ext) == Ext.end()) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → bad extension »" << flExtension(Ent->d_name) << "«" << std::endl; + if (SilentIgnore.Match(Ent->d_name) == false) + _error->Notice(_("Ignoring file '%s' in directory '%s' as it has an invalid filename extension"), Ent->d_name, Dir.c_str()); + continue; + } + } + + // Skip bad filenames ala run-parts + const char *C = Ent->d_name; + for (; *C != 0; ++C) + if (isalpha(*C) == 0 && isdigit(*C) == 0 + && *C != '_' && *C != '-' && *C != ':') { + // no required extension -> dot is a bad character + if (*C == '.' && Ext.empty() == false) + continue; + break; + } + + // we don't reach the end of the name -> bad character included + if (*C != 0) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → bad character »" + << *C << "« in filename (period allowed: " << (Ext.empty() ? "no" : "yes") << ")" << std::endl; + continue; + } + + // skip filenames which end with a period. These are never valid + if (*(C - 1) == '.') + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → Period as last character" << std::endl; + continue; + } + + if (Debug == true) + std::clog << "Accept file: " << Ent->d_name << " in " << Dir << std::endl; + List.push_back(File); + } + closedir(D); + + if (SortList == true) + std::sort(List.begin(),List.end()); + return List; +} +std::vector<string> GetListOfFilesInDir(string const &Dir, bool SortList) +{ + bool const Debug = _config->FindB("Debug::GetListOfFilesInDir", false); + if (Debug == true) + std::clog << "Accept in " << Dir << " all regular files" << std::endl; + + std::vector<string> List; + + if (DirectoryExists(Dir) == false) + { + _error->Error(_("List of files can't be created as '%s' is not a directory"), Dir.c_str()); + return List; + } + + DIR *D = opendir(Dir.c_str()); + if (D == 0) + { + _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + return List; + } + + for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D)) + { + // skip "hidden" files + if (Ent->d_name[0] == '.') + continue; + + // Make sure it is a file and not something else + string const File = flCombine(Dir,Ent->d_name); +#ifdef _DIRENT_HAVE_D_TYPE + if (Ent->d_type != DT_REG) +#endif + { + if (RealFileExists(File) == false) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → it is not a real file" << std::endl; + continue; + } + } + + // Skip bad filenames ala run-parts + const char *C = Ent->d_name; + for (; *C != 0; ++C) + if (isalpha(*C) == 0 && isdigit(*C) == 0 + && *C != '_' && *C != '-' && *C != '.') + break; + + // we don't reach the end of the name -> bad character included + if (*C != 0) + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → bad character »" << *C << "« in filename" << std::endl; + continue; + } + + // skip filenames which end with a period. These are never valid + if (*(C - 1) == '.') + { + if (Debug == true) + std::clog << "Bad file: " << Ent->d_name << " → Period as last character" << std::endl; + continue; + } + + if (Debug == true) + std::clog << "Accept file: " << Ent->d_name << " in " << Dir << std::endl; + List.push_back(File); + } + closedir(D); + + if (SortList == true) + std::sort(List.begin(),List.end()); + return List; +} + /*}}}*/ +// SafeGetCWD - This is a safer getcwd that returns a dynamic string /*{{{*/ +// --------------------------------------------------------------------- +/* We return / on failure. */ +string SafeGetCWD() +{ + // Stash the current dir. + char S[300]; + S[0] = 0; + if (getcwd(S,sizeof(S)-2) == 0) + return "/"; + unsigned int Len = strlen(S); + S[Len] = '/'; + S[Len+1] = 0; + return S; +} + /*}}}*/ +// GetModificationTime - Get the mtime of the given file or -1 on error /*{{{*/ +// --------------------------------------------------------------------- +/* We return / on failure. */ +time_t GetModificationTime(string const &Path) +{ + struct stat St; + if (stat(Path.c_str(), &St) < 0) + return -1; + return St.st_mtime; +} + /*}}}*/ +// flNotDir - Strip the directory from the filename /*{{{*/ +// --------------------------------------------------------------------- +/* */ +string flNotDir(string File) +{ + string::size_type Res = File.rfind('/'); + if (Res == string::npos) + return File; + Res++; + return string(File,Res,Res - File.length()); +} + /*}}}*/ +// flNotFile - Strip the file from the directory name /*{{{*/ +// --------------------------------------------------------------------- +/* Result ends in a / */ +string flNotFile(string File) +{ + string::size_type Res = File.rfind('/'); + if (Res == string::npos) + return "./"; + Res++; + return string(File,0,Res); +} + /*}}}*/ +// flExtension - Return the extension for the file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +string flExtension(string File) +{ + string::size_type Res = File.rfind('.'); + if (Res == string::npos) + return File; + Res++; + return string(File,Res); +} + /*}}}*/ +// flNoLink - If file is a symlink then deref it /*{{{*/ +// --------------------------------------------------------------------- +/* If the name is not a link then the returned path is the input. */ +string flNoLink(string File) +{ + struct stat St; + if (lstat(File.c_str(),&St) != 0 || S_ISLNK(St.st_mode) == 0) + return File; + if (stat(File.c_str(),&St) != 0) + return File; + + /* Loop resolving the link. There is no need to limit the number of + loops because the stat call above ensures that the symlink is not + circular */ + char Buffer[1024]; + string NFile = File; + while (1) + { + // Read the link + ssize_t Res; + if ((Res = readlink(NFile.c_str(),Buffer,sizeof(Buffer))) <= 0 || + (size_t)Res >= sizeof(Buffer)) + return File; + + // Append or replace the previous path + Buffer[Res] = 0; + if (Buffer[0] == '/') + NFile = Buffer; + else + NFile = flNotFile(NFile) + Buffer; + + // See if we are done + if (lstat(NFile.c_str(),&St) != 0) + return File; + if (S_ISLNK(St.st_mode) == 0) + return NFile; + } +} + /*}}}*/ +// flCombine - Combine a file and a directory /*{{{*/ +// --------------------------------------------------------------------- +/* If the file is an absolute path then it is just returned, otherwise + the directory is pre-pended to it. */ +string flCombine(string Dir,string File) +{ + if (File.empty() == true) + return string(); + + if (File[0] == '/' || Dir.empty() == true) + return File; + if (File.length() >= 2 && File[0] == '.' && File[1] == '/') + return File; + if (Dir[Dir.length()-1] == '/') + return Dir + File; + return Dir + '/' + File; +} + /*}}}*/ +// flAbsPath - Return the absolute path of the filename /*{{{*/ +// --------------------------------------------------------------------- +/* */ +string flAbsPath(string File) +{ + char *p = realpath(File.c_str(), NULL); + if (p == NULL) + { + _error->Errno("realpath", "flAbsPath on %s failed", File.c_str()); + return ""; + } + std::string AbsPath(p); + free(p); + return AbsPath; +} + /*}}}*/ +std::string flNormalize(std::string file) /*{{{*/ +{ + if (file.empty()) + return file; + // do some normalisation by removing // and /./ from the path + size_t found = string::npos; + while ((found = file.find("/./")) != string::npos) + file.replace(found, 3, "/"); + while ((found = file.find("//")) != string::npos) + file.replace(found, 2, "/"); + + if (APT::String::Startswith(file, "/dev/null")) + { + file.erase(strlen("/dev/null")); + return file; + } + return file; +} + /*}}}*/ +// SetCloseExec - Set the close on exec flag /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void SetCloseExec(int Fd,bool Close) +{ + if (fcntl(Fd,F_SETFD,(Close == false)?0:FD_CLOEXEC) != 0) + { + cerr << "FATAL -> Could not set close on exec " << strerror(errno) << endl; + exit(100); + } +} + /*}}}*/ +// SetNonBlock - Set the nonblocking flag /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void SetNonBlock(int Fd,bool Block) +{ + int Flags = fcntl(Fd,F_GETFL) & (~O_NONBLOCK); + if (fcntl(Fd,F_SETFL,Flags | ((Block == false)?0:O_NONBLOCK)) != 0) + { + cerr << "FATAL -> Could not set non-blocking flag " << strerror(errno) << endl; + exit(100); + } +} + /*}}}*/ +// WaitFd - Wait for a FD to become readable /*{{{*/ +// --------------------------------------------------------------------- +/* This waits for a FD to become readable using select. It is useful for + applications making use of non-blocking sockets. The timeout is + in seconds. */ +bool WaitFd(int Fd,bool write,unsigned long timeout) +{ + fd_set Set; + struct timeval tv; + FD_ZERO(&Set); + FD_SET(Fd,&Set); + tv.tv_sec = timeout; + tv.tv_usec = 0; + if (write == true) + { + int Res; + do + { + Res = select(Fd+1,0,&Set,0,(timeout != 0?&tv:0)); + } + while (Res < 0 && errno == EINTR); + + if (Res <= 0) + return false; + } + else + { + int Res; + do + { + Res = select(Fd+1,&Set,0,0,(timeout != 0?&tv:0)); + } + while (Res < 0 && errno == EINTR); + + if (Res <= 0) + return false; + } + + return true; +} + /*}}}*/ +// MergeKeepFdsFromConfiguration - Merge APT::Keep-Fds configuration /*{{{*/ +// --------------------------------------------------------------------- +/* This is used to merge the APT::Keep-Fds with the provided KeepFDs + * set. + */ +void MergeKeepFdsFromConfiguration(std::set<int> &KeepFDs) +{ + Configuration::Item const *Opts = _config->Tree("APT::Keep-Fds"); + if (Opts != 0 && Opts->Child != 0) + { + Opts = Opts->Child; + for (; Opts != 0; Opts = Opts->Next) + { + if (Opts->Value.empty() == true) + continue; + int fd = atoi(Opts->Value.c_str()); + KeepFDs.insert(fd); + } + } +} + /*}}}*/ +// ExecFork - Magical fork that sanitizes the context before execing /*{{{*/ +// --------------------------------------------------------------------- +/* This is used if you want to cleanse the environment for the forked + child, it fixes up the important signals and nukes all of the fds, + otherwise acts like normal fork. */ +pid_t ExecFork() +{ + set<int> KeepFDs; + // we need to merge the Keep-Fds as external tools like + // debconf-apt-progress use it + MergeKeepFdsFromConfiguration(KeepFDs); + return ExecFork(KeepFDs); +} + +pid_t ExecFork(std::set<int> KeepFDs) +{ + // Fork off the process + pid_t Process = fork(); + if (Process < 0) + { + cerr << "FATAL -> Failed to fork." << endl; + exit(100); + } + + // Spawn the subprocess + if (Process == 0) + { + // Setup the signals + signal(SIGPIPE,SIG_DFL); + signal(SIGQUIT,SIG_DFL); + signal(SIGINT,SIG_DFL); + signal(SIGWINCH,SIG_DFL); + signal(SIGCONT,SIG_DFL); + signal(SIGTSTP,SIG_DFL); + + DIR *dir = opendir("/proc/self/fd"); + if (dir != NULL) + { + struct dirent *ent; + while ((ent = readdir(dir))) + { + int fd = atoi(ent->d_name); + // If fd > 0, it was a fd number and not . or .. + if (fd >= 3 && KeepFDs.find(fd) == KeepFDs.end()) + fcntl(fd,F_SETFD,FD_CLOEXEC); + } + closedir(dir); + } else { + long ScOpenMax = sysconf(_SC_OPEN_MAX); + // Close all of our FDs - just in case + for (int K = 3; K != ScOpenMax; K++) + { + if(KeepFDs.find(K) == KeepFDs.end()) + fcntl(K,F_SETFD,FD_CLOEXEC); + } + } + } + + return Process; +} + /*}}}*/ +// ExecWait - Fancy waitpid /*{{{*/ +// --------------------------------------------------------------------- +/* Waits for the given sub process. If Reap is set then no errors are + generated. Otherwise a failed subprocess will generate a proper descriptive + message */ +bool ExecWait(pid_t Pid,const char *Name,bool Reap) +{ + if (Pid <= 1) + return true; + + // Wait and collect the error code + int Status; + while (waitpid(Pid,&Status,0) != Pid) + { + if (errno == EINTR) + continue; + + if (Reap == true) + return false; + + return _error->Error(_("Waited for %s but it wasn't there"),Name); + } + + + // Check for an error code. + if (WIFEXITED(Status) == 0 || WEXITSTATUS(Status) != 0) + { + if (Reap == true) + return false; + if (WIFSIGNALED(Status) != 0) + { + if( WTERMSIG(Status) == SIGSEGV) + return _error->Error(_("Sub-process %s received a segmentation fault."),Name); + else + return _error->Error(_("Sub-process %s received signal %u."),Name, WTERMSIG(Status)); + } + + if (WIFEXITED(Status) != 0) + return _error->Error(_("Sub-process %s returned an error code (%u)"),Name,WEXITSTATUS(Status)); + + return _error->Error(_("Sub-process %s exited unexpectedly"),Name); + } + + return true; +} + /*}}}*/ +// StartsWithGPGClearTextSignature - Check if a file is Pgp/GPG clearsigned /*{{{*/ +bool StartsWithGPGClearTextSignature(string const &FileName) +{ + FILE* gpg = fopen(FileName.c_str(), "r"); + if (gpg == nullptr) + return false; + + char * lineptr = nullptr; + size_t n = 0; + errno = 0; + ssize_t const result = getline(&lineptr, &n, gpg); + if (errno != 0) + { + _error->Errno("getline", "Could not read from %s", FileName.c_str()); + fclose(gpg); + free(lineptr); + return false; + } + fclose(gpg); + + _strrstrip(lineptr); + static const char* SIGMSG = "-----BEGIN PGP SIGNED MESSAGE-----"; + if (result == -1 || strcmp(lineptr, SIGMSG) != 0) + { + free(lineptr); + return false; + } + free(lineptr); + return true; +} + /*}}}*/ +// ChangeOwnerAndPermissionOfFile - set file attributes to requested values /*{{{*/ +bool ChangeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode) +{ + if (strcmp(file, "/dev/null") == 0) + return true; + bool Res = true; + if (getuid() == 0 && strlen(user) != 0 && strlen(group) != 0) // if we aren't root, we can't chown, so don't try it + { + // ensure the file is owned by root and has good permissions + struct passwd const * const pw = getpwnam(user); + struct group const * const gr = getgrnam(group); + if (pw != NULL && gr != NULL && lchown(file, pw->pw_uid, gr->gr_gid) != 0) + Res &= _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file); + } + struct stat Buf; + if (lstat(file, &Buf) != 0 || S_ISLNK(Buf.st_mode)) + return Res; + if (chmod(file, mode) != 0) + Res &= _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file); + return Res; +} + /*}}}*/ + +struct APT_HIDDEN simple_buffer { /*{{{*/ + size_t buffersize_max = 0; + unsigned long long bufferstart = 0; + unsigned long long bufferend = 0; + char *buffer = nullptr; + + simple_buffer() { + reset(4096); + } + ~simple_buffer() { + delete[] buffer; + } + + const char *get() const { return buffer + bufferstart; } + char *get() { return buffer + bufferstart; } + const char *getend() const { return buffer + bufferend; } + char *getend() { return buffer + bufferend; } + bool empty() const { return bufferend <= bufferstart; } + bool full() const { return bufferend == buffersize_max; } + unsigned long long free() const { return buffersize_max - bufferend; } + unsigned long long size() const { return bufferend-bufferstart; } + void reset(size_t size) + { + if (size > buffersize_max) { + delete[] buffer; + buffersize_max = size; + buffer = new char[size]; + } + reset(); + } + void reset() { bufferend = bufferstart = 0; } + ssize_t read(void *to, unsigned long long requested_size) APT_MUSTCHECK + { + if (size() < requested_size) + requested_size = size(); + memcpy(to, buffer + bufferstart, requested_size); + bufferstart += requested_size; + if (bufferstart == bufferend) + bufferstart = bufferend = 0; + return requested_size; + } + ssize_t write(const void *from, unsigned long long requested_size) APT_MUSTCHECK + { + if (free() < requested_size) + requested_size = free(); + memcpy(getend(), from, requested_size); + bufferend += requested_size; + if (bufferstart == bufferend) + bufferstart = bufferend = 0; + return requested_size; + } +}; + /*}}}*/ + +class APT_HIDDEN FileFdPrivate { /*{{{*/ + friend class BufferedWriteFileFdPrivate; +protected: + FileFd * const filefd; + simple_buffer buffer; + int compressed_fd; + pid_t compressor_pid; + bool is_pipe; + APT::Configuration::Compressor compressor; + unsigned int openmode; + unsigned long long seekpos; +public: + + explicit FileFdPrivate(FileFd * const pfilefd) : filefd(pfilefd), + compressed_fd(-1), compressor_pid(-1), is_pipe(false), + openmode(0), seekpos(0) {}; + virtual APT::Configuration::Compressor get_compressor() const + { + return compressor; + } + virtual void set_compressor(APT::Configuration::Compressor const &compressor) + { + this->compressor = compressor; + } + virtual unsigned int get_openmode() const + { + return openmode; + } + virtual void set_openmode(unsigned int openmode) + { + this->openmode = openmode; + } + virtual bool get_is_pipe() const + { + return is_pipe; + } + virtual void set_is_pipe(bool is_pipe) + { + this->is_pipe = is_pipe; + } + virtual unsigned long long get_seekpos() const + { + return seekpos; + } + virtual void set_seekpos(unsigned long long seekpos) + { + this->seekpos = seekpos; + } + + virtual bool InternalOpen(int const iFd, unsigned int const Mode) = 0; + ssize_t InternalRead(void * To, unsigned long long Size) + { + // Drain the buffer if needed. + if (buffer.empty() == false) + { + return buffer.read(To, Size); + } + return InternalUnbufferedRead(To, Size); + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) = 0; + virtual bool InternalReadError() { return filefd->FileFdErrno("read",_("Read error")); } + virtual char * InternalReadLine(char * To, unsigned long long Size) + { + if (unlikely(Size == 0)) + return nullptr; + // Read one byte less than buffer size to have space for trailing 0. + --Size; + + char * const InitialTo = To; + + while (Size > 0) { + if (buffer.empty() == true) + { + buffer.reset(); + unsigned long long actualread = 0; + if (filefd->Read(buffer.getend(), buffer.free(), &actualread) == false) + return nullptr; + buffer.bufferend = actualread; + if (buffer.size() == 0) + { + if (To == InitialTo) + return nullptr; + break; + } + filefd->Flags &= ~FileFd::HitEof; + } + + unsigned long long const OutputSize = std::min(Size, buffer.size()); + char const * const newline = static_cast<char const *>(memchr(buffer.get(), '\n', OutputSize)); + // Read until end of line or up to Size bytes from the buffer. + unsigned long long actualread = buffer.read(To, + (newline != nullptr) + ? (newline - buffer.get()) + 1 + : OutputSize); + To += actualread; + Size -= actualread; + if (newline != nullptr) + break; + } + *To = '\0'; + return InitialTo; + } + virtual bool InternalFlush() + { + return true; + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) = 0; + virtual bool InternalWriteError() { return filefd->FileFdErrno("write",_("Write error")); } + virtual bool InternalSeek(unsigned long long const To) + { + // Our poor man seeking is costly, so try to avoid it + unsigned long long const iseekpos = filefd->Tell(); + if (iseekpos == To) + return true; + else if (iseekpos < To) + return filefd->Skip(To - iseekpos); + + if ((openmode & FileFd::ReadOnly) != FileFd::ReadOnly) + return filefd->FileFdError("Reopen is only implemented for read-only files!"); + InternalClose(filefd->FileName); + if (filefd->iFd != -1) + close(filefd->iFd); + filefd->iFd = -1; + if (filefd->TemporaryFileName.empty() == false) + filefd->iFd = open(filefd->TemporaryFileName.c_str(), O_RDONLY); + else if (filefd->FileName.empty() == false) + filefd->iFd = open(filefd->FileName.c_str(), O_RDONLY); + else + { + if (compressed_fd > 0) + if (lseek(compressed_fd, 0, SEEK_SET) != 0) + filefd->iFd = compressed_fd; + if (filefd->iFd < 0) + return filefd->FileFdError("Reopen is not implemented for pipes opened with FileFd::OpenDescriptor()!"); + } + + if (filefd->OpenInternDescriptor(openmode, compressor) == false) + return filefd->FileFdError("Seek on file %s because it couldn't be reopened", filefd->FileName.c_str()); + + buffer.reset(); + set_seekpos(0); + if (To != 0) + return filefd->Skip(To); + + seekpos = To; + return true; + } + virtual bool InternalSkip(unsigned long long Over) + { + unsigned long long constexpr buffersize = 1024; + char buffer[buffersize]; + while (Over != 0) + { + unsigned long long toread = std::min(buffersize, Over); + if (filefd->Read(buffer, toread) == false) + return filefd->FileFdError("Unable to seek ahead %llu",Over); + Over -= toread; + } + return true; + } + virtual bool InternalTruncate(unsigned long long const) + { + return filefd->FileFdError("Truncating compressed files is not implemented (%s)", filefd->FileName.c_str()); + } + virtual unsigned long long InternalTell() + { + // In theory, we could just return seekpos here always instead of + // seeking around, but not all users of FileFd use always Seek() and co + // so d->seekpos isn't always true and we can just use it as a hint if + // we have nothing else, but not always as an authority… + return seekpos - buffer.size(); + } + virtual unsigned long long InternalSize() + { + unsigned long long size = 0; + unsigned long long const oldSeek = filefd->Tell(); + unsigned long long constexpr ignoresize = 1024; + char ignore[ignoresize]; + unsigned long long read = 0; + do { + if (filefd->Read(ignore, ignoresize, &read) == false) + { + filefd->Seek(oldSeek); + return 0; + } + } while(read != 0); + size = filefd->Tell(); + filefd->Seek(oldSeek); + return size; + } + virtual bool InternalClose(std::string const &FileName) = 0; + virtual bool InternalStream() const { return false; } + virtual bool InternalAlwaysAutoClose() const { return true; } + + virtual ~FileFdPrivate() {} +}; + /*}}}*/ +class APT_HIDDEN BufferedWriteFileFdPrivate : public FileFdPrivate { /*{{{*/ +protected: + FileFdPrivate *wrapped; + simple_buffer writebuffer; + +public: + + explicit BufferedWriteFileFdPrivate(FileFdPrivate *Priv) : + FileFdPrivate(Priv->filefd), wrapped(Priv) {}; + + virtual APT::Configuration::Compressor get_compressor() const APT_OVERRIDE + { + return wrapped->get_compressor(); + } + virtual void set_compressor(APT::Configuration::Compressor const &compressor) APT_OVERRIDE + { + return wrapped->set_compressor(compressor); + } + virtual unsigned int get_openmode() const APT_OVERRIDE + { + return wrapped->get_openmode(); + } + virtual void set_openmode(unsigned int openmode) APT_OVERRIDE + { + return wrapped->set_openmode(openmode); + } + virtual bool get_is_pipe() const APT_OVERRIDE + { + return wrapped->get_is_pipe(); + } + virtual void set_is_pipe(bool is_pipe) APT_OVERRIDE + { + FileFdPrivate::set_is_pipe(is_pipe); + wrapped->set_is_pipe(is_pipe); + } + virtual unsigned long long get_seekpos() const APT_OVERRIDE + { + return wrapped->get_seekpos(); + } + virtual void set_seekpos(unsigned long long seekpos) APT_OVERRIDE + { + return wrapped->set_seekpos(seekpos); + } + virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE + { + if (InternalFlush() == false) + return false; + return wrapped->InternalOpen(iFd, Mode); + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE + { + if (InternalFlush() == false) + return -1; + return wrapped->InternalUnbufferedRead(To, Size); + + } + virtual bool InternalReadError() APT_OVERRIDE + { + return wrapped->InternalReadError(); + } + virtual char * InternalReadLine(char * To, unsigned long long Size) APT_OVERRIDE + { + if (InternalFlush() == false) + return nullptr; + return wrapped->InternalReadLine(To, Size); + } + virtual bool InternalFlush() APT_OVERRIDE + { + while (writebuffer.empty() == false) { + auto written = wrapped->InternalWrite(writebuffer.get(), + writebuffer.size()); + // Ignore interrupted syscalls + if (written < 0 && errno == EINTR) + continue; + if (written < 0) + return wrapped->InternalWriteError(); + + writebuffer.bufferstart += written; + } + writebuffer.reset(); + return wrapped->InternalFlush(); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE + { + // Optimisation: If the buffer is empty and we have more to write than + // would fit in the buffer (or equal number of bytes), write directly. + if (writebuffer.empty() == true && Size >= writebuffer.free()) + return wrapped->InternalWrite(From, Size); + + // Write as much into the buffer as possible and then flush if needed + auto written = writebuffer.write(From, Size); + + if (writebuffer.full() && InternalFlush() == false) + return -1; + + return written; + } + virtual bool InternalWriteError() APT_OVERRIDE + { + return wrapped->InternalWriteError(); + } + virtual bool InternalSeek(unsigned long long const To) APT_OVERRIDE + { + if (InternalFlush() == false) + return false; + return wrapped->InternalSeek(To); + } + virtual bool InternalSkip(unsigned long long Over) APT_OVERRIDE + { + if (InternalFlush() == false) + return false; + return wrapped->InternalSkip(Over); + } + virtual bool InternalTruncate(unsigned long long const Size) APT_OVERRIDE + { + if (InternalFlush() == false) + return false; + return wrapped->InternalTruncate(Size); + } + virtual unsigned long long InternalTell() APT_OVERRIDE + { + if (InternalFlush() == false) + return -1; + return wrapped->InternalTell(); + } + virtual unsigned long long InternalSize() APT_OVERRIDE + { + if (InternalFlush() == false) + return -1; + return wrapped->InternalSize(); + } + virtual bool InternalClose(std::string const &FileName) APT_OVERRIDE + { + return wrapped->InternalClose(FileName); + } + virtual bool InternalAlwaysAutoClose() const APT_OVERRIDE + { + return wrapped->InternalAlwaysAutoClose(); + } + virtual ~BufferedWriteFileFdPrivate() + { + delete wrapped; + } +}; + /*}}}*/ +class APT_HIDDEN GzipFileFdPrivate: public FileFdPrivate { /*{{{*/ +#ifdef HAVE_ZLIB +public: + gzFile gz; + virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE + { + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + gz = gzdopen(iFd, "r+"); + else if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) + gz = gzdopen(iFd, "w"); + else + gz = gzdopen(iFd, "r"); + filefd->Flags |= FileFd::Compressed; + return gz != nullptr; + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE + { + return gzread(gz, To, Size); + } + virtual bool InternalReadError() APT_OVERRIDE + { + int err; + char const * const errmsg = gzerror(gz, &err); + if (err != Z_ERRNO) + return filefd->FileFdError("gzread: %s (%d: %s)", _("Read error"), err, errmsg); + return FileFdPrivate::InternalReadError(); + } + virtual char * InternalReadLine(char * To, unsigned long long Size) APT_OVERRIDE + { + return gzgets(gz, To, Size); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE + { + return gzwrite(gz,From,Size); + } + virtual bool InternalWriteError() APT_OVERRIDE + { + int err; + char const * const errmsg = gzerror(gz, &err); + if (err != Z_ERRNO) + return filefd->FileFdError("gzwrite: %s (%d: %s)", _("Write error"), err, errmsg); + return FileFdPrivate::InternalWriteError(); + } + virtual bool InternalSeek(unsigned long long const To) APT_OVERRIDE + { + off_t const res = gzseek(gz, To, SEEK_SET); + if (res != (off_t)To) + return filefd->FileFdError("Unable to seek to %llu", To); + seekpos = To; + buffer.reset(); + return true; + } + virtual bool InternalSkip(unsigned long long Over) APT_OVERRIDE + { + if (Over >= buffer.size()) + { + Over -= buffer.size(); + buffer.reset(); + } + else + { + buffer.bufferstart += Over; + return true; + } + if (Over == 0) + return true; + off_t const res = gzseek(gz, Over, SEEK_CUR); + if (res < 0) + return filefd->FileFdError("Unable to seek ahead %llu",Over); + seekpos = res; + return true; + } + virtual unsigned long long InternalTell() APT_OVERRIDE + { + return gztell(gz) - buffer.size(); + } + virtual unsigned long long InternalSize() APT_OVERRIDE + { + unsigned long long filesize = FileFdPrivate::InternalSize(); + // only check gzsize if we are actually a gzip file, just checking for + // "gz" is not sufficient as uncompressed files could be opened with + // gzopen in "direct" mode as well + if (filesize == 0 || gzdirect(gz)) + return filesize; + + off_t const oldPos = lseek(filefd->iFd, 0, SEEK_CUR); + /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do + * this ourselves; the original (uncompressed) file size is the last 32 + * bits of the file */ + // FIXME: Size for gz-files is limited by 32bit… no largefile support + if (lseek(filefd->iFd, -4, SEEK_END) < 0) + { + filefd->FileFdErrno("lseek","Unable to seek to end of gzipped file"); + return 0; + } + uint32_t size = 0; + if (read(filefd->iFd, &size, 4) != 4) + { + filefd->FileFdErrno("read","Unable to read original size of gzipped file"); + return 0; + } + size = le32toh(size); + + if (lseek(filefd->iFd, oldPos, SEEK_SET) < 0) + { + filefd->FileFdErrno("lseek","Unable to seek in gzipped file"); + return 0; + } + return size; + } + virtual bool InternalClose(std::string const &FileName) APT_OVERRIDE + { + if (gz == nullptr) + return true; + int const e = gzclose(gz); + gz = nullptr; + // gzdclose() on empty files always fails with "buffer error" here, ignore that + if (e != 0 && e != Z_BUF_ERROR) + return _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str()); + return true; + } + + explicit GzipFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), gz(nullptr) {} + virtual ~GzipFileFdPrivate() { InternalClose(""); } +#endif +}; + /*}}}*/ +class APT_HIDDEN Bz2FileFdPrivate: public FileFdPrivate { /*{{{*/ +#ifdef HAVE_BZ2 + BZFILE* bz2; +public: + virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE + { + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + bz2 = BZ2_bzdopen(iFd, "r+"); + else if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) + bz2 = BZ2_bzdopen(iFd, "w"); + else + bz2 = BZ2_bzdopen(iFd, "r"); + filefd->Flags |= FileFd::Compressed; + return bz2 != nullptr; + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE + { + return BZ2_bzread(bz2, To, Size); + } + virtual bool InternalReadError() APT_OVERRIDE + { + int err; + char const * const errmsg = BZ2_bzerror(bz2, &err); + if (err != BZ_IO_ERROR) + return filefd->FileFdError("BZ2_bzread: %s %s (%d: %s)", filefd->FileName.c_str(), _("Read error"), err, errmsg); + return FileFdPrivate::InternalReadError(); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE + { + return BZ2_bzwrite(bz2, (void*)From, Size); + } + virtual bool InternalWriteError() APT_OVERRIDE + { + int err; + char const * const errmsg = BZ2_bzerror(bz2, &err); + if (err != BZ_IO_ERROR) + return filefd->FileFdError("BZ2_bzwrite: %s %s (%d: %s)", filefd->FileName.c_str(), _("Write error"), err, errmsg); + return FileFdPrivate::InternalWriteError(); + } + virtual bool InternalStream() const APT_OVERRIDE { return true; } + virtual bool InternalClose(std::string const &) APT_OVERRIDE + { + if (bz2 == nullptr) + return true; + BZ2_bzclose(bz2); + bz2 = nullptr; + return true; + } + + explicit Bz2FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), bz2(nullptr) {} + virtual ~Bz2FileFdPrivate() { InternalClose(""); } +#endif +}; + /*}}}*/ +class APT_HIDDEN Lz4FileFdPrivate: public FileFdPrivate { /*{{{*/ + static constexpr unsigned long long LZ4_HEADER_SIZE = 19; + static constexpr unsigned long long LZ4_FOOTER_SIZE = 4; +#ifdef HAVE_LZ4 + LZ4F_decompressionContext_t dctx; + LZ4F_compressionContext_t cctx; + LZ4F_errorCode_t res; + FileFd backend; + simple_buffer lz4_buffer; + // Count of bytes that the decompressor expects to read next, or buffer size. + size_t next_to_load = APT_BUFFER_SIZE; +public: + virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE + { + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + return _error->Error("lz4 only supports write or read mode"); + + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) { + res = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); + lz4_buffer.reset(LZ4F_compressBound(APT_BUFFER_SIZE, nullptr) + + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE); + } else { + res = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + lz4_buffer.reset(APT_BUFFER_SIZE); + } + + filefd->Flags |= FileFd::Compressed; + + if (LZ4F_isError(res)) + return false; + + unsigned int flags = (Mode & (FileFd::WriteOnly|FileFd::ReadOnly)); + if (backend.OpenDescriptor(iFd, flags, FileFd::None, true) == false) + return false; + + // Write the file header + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) + { + res = LZ4F_compressBegin(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr); + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return false; + } + + return true; + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE + { + /* Keep reading as long as the compressor still wants to read */ + while (next_to_load) { + // Fill compressed buffer; + if (lz4_buffer.empty()) { + unsigned long long read; + /* Reset - if LZ4 decompressor wants to read more, allocate more */ + lz4_buffer.reset(next_to_load); + if (backend.Read(lz4_buffer.getend(), lz4_buffer.free(), &read) == false) + return -1; + lz4_buffer.bufferend += read; + + /* Expected EOF */ + if (read == 0) { + res = -1; + return filefd->FileFdError("LZ4F: %s %s", + filefd->FileName.c_str(), + _("Unexpected end of file")), -1; + } + } + // Drain compressed buffer as far as possible. + size_t in = lz4_buffer.size(); + size_t out = Size; + + res = LZ4F_decompress(dctx, To, &out, lz4_buffer.get(), &in, nullptr); + if (LZ4F_isError(res)) + return -1; + + next_to_load = res; + lz4_buffer.bufferstart += in; + + if (out != 0) + return out; + } + + return 0; + } + virtual bool InternalReadError() APT_OVERRIDE + { + char const * const errmsg = LZ4F_getErrorName(res); + + return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Read error"), res, errmsg); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE + { + unsigned long long const towrite = std::min(APT_BUFFER_SIZE, Size); + + res = LZ4F_compressUpdate(cctx, + lz4_buffer.buffer, lz4_buffer.buffersize_max, + From, towrite, nullptr); + + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return -1; + + return towrite; + } + virtual bool InternalWriteError() APT_OVERRIDE + { + char const * const errmsg = LZ4F_getErrorName(res); + + return filefd->FileFdError("LZ4F: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Write error"), res, errmsg); + } + virtual bool InternalStream() const APT_OVERRIDE { return true; } + + virtual bool InternalFlush() APT_OVERRIDE + { + return backend.Flush(); + } + + virtual bool InternalClose(std::string const &) APT_OVERRIDE + { + /* Reset variables */ + res = 0; + next_to_load = APT_BUFFER_SIZE; + + if (cctx != nullptr) + { + if (filefd->Failed() == false) + { + res = LZ4F_compressEnd(cctx, lz4_buffer.buffer, lz4_buffer.buffersize_max, nullptr); + if (LZ4F_isError(res) || backend.Write(lz4_buffer.buffer, res) == false) + return false; + if (!backend.Flush()) + return false; + } + if (!backend.Close()) + return false; + + res = LZ4F_freeCompressionContext(cctx); + cctx = nullptr; + } + + if (dctx != nullptr) + { + res = LZ4F_freeDecompressionContext(dctx); + dctx = nullptr; + } + if (backend.IsOpen()) + { + backend.Close(); + filefd->iFd = -1; + } + + return LZ4F_isError(res) == false; + } + + explicit Lz4FileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), dctx(nullptr), cctx(nullptr) {} + virtual ~Lz4FileFdPrivate() { + InternalClose(""); + } +#endif +}; + /*}}}*/ +class APT_HIDDEN ZstdFileFdPrivate : public FileFdPrivate /*{{{*/ +{ +#ifdef HAVE_ZSTD + ZSTD_DStream *dctx; + ZSTD_CStream *cctx; + size_t res = 0; + FileFd backend; + simple_buffer zstd_buffer; + // Count of bytes that the decompressor expects to read next, or buffer size. + size_t next_to_load = APT_BUFFER_SIZE; + + public: + virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE + { + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + return _error->Error("zstd only supports write or read mode"); + + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) + { + cctx = ZSTD_createCStream(); + res = ZSTD_initCStream(cctx, findLevel(compressor.CompressArgs)); + zstd_buffer.reset(APT_BUFFER_SIZE); + } + else + { + dctx = ZSTD_createDStream(); + res = ZSTD_initDStream(dctx); + zstd_buffer.reset(APT_BUFFER_SIZE); + } + + filefd->Flags |= FileFd::Compressed; + + if (ZSTD_isError(res)) + return false; + + unsigned int flags = (Mode & (FileFd::WriteOnly | FileFd::ReadOnly)); + if (backend.OpenDescriptor(iFd, flags, FileFd::None, true) == false) + return false; + + return true; + } + virtual ssize_t InternalUnbufferedRead(void *const To, unsigned long long const Size) APT_OVERRIDE + { + /* Keep reading as long as the compressor still wants to read */ + while (true) + { + // Fill compressed buffer; + if (zstd_buffer.empty()) + { + unsigned long long read; + /* Reset - if LZ4 decompressor wants to read more, allocate more */ + zstd_buffer.reset(next_to_load); + if (backend.Read(zstd_buffer.getend(), zstd_buffer.free(), &read) == false) + return -1; + zstd_buffer.bufferend += read; + + if (read == 0) + { + /* Expected EOF */ + if (next_to_load == 0) + return 0; + + res = -1; + return filefd->FileFdError("ZSTD: %s %s", + filefd->FileName.c_str(), + _("Unexpected end of file")), + -1; + } + } + // Drain compressed buffer as far as possible. + ZSTD_inBuffer in = { + .src = zstd_buffer.get(), + .size = zstd_buffer.size(), + .pos = 0, + }; + ZSTD_outBuffer out = { + .dst = To, + .size = Size, + .pos = 0, + }; + + next_to_load = res = ZSTD_decompressStream(dctx, &out, &in); + + if (res == 0) + { + res = ZSTD_initDStream(dctx); + } + + if (ZSTD_isError(res)) + return -1; + + zstd_buffer.bufferstart += in.pos; + + if (out.pos != 0) + return out.pos; + } + + return 0; + } + virtual bool InternalReadError() APT_OVERRIDE + { + char const *const errmsg = ZSTD_getErrorName(res); + + return filefd->FileFdError("ZSTD: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Read error"), res, errmsg); + } + virtual ssize_t InternalWrite(void const *const From, unsigned long long const Size) APT_OVERRIDE + { + // Drain compressed buffer as far as possible. + ZSTD_outBuffer out = { + .dst = zstd_buffer.buffer, + .size = zstd_buffer.buffersize_max, + .pos = 0, + }; + ZSTD_inBuffer in = { + .src = From, + .size = Size, + .pos = 0, + }; + + res = ZSTD_compressStream(cctx, &out, &in); + + if (ZSTD_isError(res) || backend.Write(zstd_buffer.buffer, out.pos) == false) + return -1; + + return in.pos; + } + + virtual bool InternalWriteError() APT_OVERRIDE + { + char const *const errmsg = ZSTD_getErrorName(res); + + return filefd->FileFdError("ZSTD: %s %s (%zu: %s)", filefd->FileName.c_str(), _("Write error"), res, errmsg); + } + virtual bool InternalStream() const APT_OVERRIDE { return true; } + + virtual bool InternalFlush() APT_OVERRIDE + { + return backend.Flush(); + } + + virtual bool InternalClose(std::string const &) APT_OVERRIDE + { + /* Reset variables */ + res = 0; + next_to_load = APT_BUFFER_SIZE; + + if (cctx != nullptr) + { + if (filefd->Failed() == false) + { + do + { + ZSTD_outBuffer out = { + .dst = zstd_buffer.buffer, + .size = zstd_buffer.buffersize_max, + .pos = 0, + }; + res = ZSTD_endStream(cctx, &out); + if (ZSTD_isError(res) || backend.Write(zstd_buffer.buffer, out.pos) == false) + return false; + } while (res > 0); + + if (!backend.Flush()) + return false; + } + if (!backend.Close()) + return false; + + res = ZSTD_freeCStream(cctx); + cctx = nullptr; + } + + if (dctx != nullptr) + { + res = ZSTD_freeDStream(dctx); + dctx = nullptr; + } + if (backend.IsOpen()) + { + backend.Close(); + filefd->iFd = -1; + } + + return ZSTD_isError(res) == false; + } + + static uint32_t findLevel(std::vector<std::string> const &Args) + { + for (auto a = Args.rbegin(); a != Args.rend(); ++a) + { + if (a->size() >= 2 && (*a)[0] == '-' && (*a)[1] != '-') + { + auto const level = a->substr(1); + auto const notANumber = level.find_first_not_of("0123456789"); + if (notANumber != std::string::npos) + continue; + + return (uint32_t)stoi(level); + } + } + return 19; + } + + explicit ZstdFileFdPrivate(FileFd *const filefd) : FileFdPrivate(filefd), dctx(nullptr), cctx(nullptr) {} + virtual ~ZstdFileFdPrivate() + { + InternalClose(""); + } +#endif +}; + /*}}}*/ +class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/ +#ifdef HAVE_LZMA + struct LZMAFILE { + FILE* file; + FileFd * const filefd; + uint8_t buffer[4096]; + lzma_stream stream; + lzma_ret err; + bool eof; + bool compressing; + + explicit LZMAFILE(FileFd * const fd) : file(nullptr), filefd(fd), eof(false), compressing(false) { buffer[0] = '\0'; } + ~LZMAFILE() + { + if (compressing == true && filefd->Failed() == false) + { + size_t constexpr buffersize = sizeof(buffer)/sizeof(buffer[0]); + while(true) + { + stream.avail_out = buffersize; + stream.next_out = buffer; + err = lzma_code(&stream, LZMA_FINISH); + if (err != LZMA_OK && err != LZMA_STREAM_END) + { + _error->Error("~LZMAFILE: Compress finalisation failed"); + break; + } + size_t const n = buffersize - stream.avail_out; + if (n && fwrite(buffer, 1, n, file) != n) + { + _error->Errno("~LZMAFILE",_("Write error")); + break; + } + if (err == LZMA_STREAM_END) + break; + } + } + lzma_end(&stream); + fclose(file); + } + }; + LZMAFILE* lzma; + static uint32_t findXZlevel(std::vector<std::string> const &Args) + { + for (auto a = Args.rbegin(); a != Args.rend(); ++a) + if (a->empty() == false && (*a)[0] == '-' && (*a)[1] != '-') + { + auto const number = a->find_last_of("0123456789"); + if (number == std::string::npos) + continue; + auto const extreme = a->find("e", number); + uint32_t level = (extreme != std::string::npos) ? LZMA_PRESET_EXTREME : 0; + switch ((*a)[number]) + { + case '0': return level | 0; + case '1': return level | 1; + case '2': return level | 2; + case '3': return level | 3; + case '4': return level | 4; + case '5': return level | 5; + case '6': return level | 6; + case '7': return level | 7; + case '8': return level | 8; + case '9': return level | 9; + } + } + return 6; + } +public: + virtual bool InternalOpen(int const iFd, unsigned int const Mode) APT_OVERRIDE + { + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + return filefd->FileFdError("ReadWrite mode is not supported for lzma/xz files %s", filefd->FileName.c_str()); + + if (lzma == nullptr) + lzma = new LzmaFileFdPrivate::LZMAFILE(filefd); + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) + lzma->file = fdopen(iFd, "w"); + else + lzma->file = fdopen(iFd, "r"); + filefd->Flags |= FileFd::Compressed; + if (lzma->file == nullptr) + return false; + + lzma_stream tmp_stream = LZMA_STREAM_INIT; + lzma->stream = tmp_stream; + + if ((Mode & FileFd::WriteOnly) == FileFd::WriteOnly) + { + uint32_t const xzlevel = findXZlevel(compressor.CompressArgs); + if (compressor.Name == "xz") + { + if (lzma_easy_encoder(&lzma->stream, xzlevel, LZMA_CHECK_CRC64) != LZMA_OK) + return false; + } + else + { + lzma_options_lzma options; + lzma_lzma_preset(&options, xzlevel); + if (lzma_alone_encoder(&lzma->stream, &options) != LZMA_OK) + return false; + } + lzma->compressing = true; + } + else + { + uint64_t constexpr memlimit = 1024 * 1024 * 500; + if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK) + return false; + lzma->compressing = false; + } + return true; + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE + { + ssize_t Res; + if (lzma->eof == true) + return 0; + + lzma->stream.next_out = (uint8_t *) To; + lzma->stream.avail_out = Size; + if (lzma->stream.avail_in == 0) + { + lzma->stream.next_in = lzma->buffer; + lzma->stream.avail_in = fread(lzma->buffer, 1, sizeof(lzma->buffer)/sizeof(lzma->buffer[0]), lzma->file); + } + lzma->err = lzma_code(&lzma->stream, LZMA_RUN); + if (lzma->err == LZMA_STREAM_END) + { + lzma->eof = true; + Res = Size - lzma->stream.avail_out; + } + else if (lzma->err != LZMA_OK) + { + Res = -1; + errno = 0; + } + else + { + Res = Size - lzma->stream.avail_out; + if (Res == 0) + { + // lzma run was okay, but produced no output… + Res = -1; + errno = EINTR; + } + } + return Res; + } + virtual bool InternalReadError() APT_OVERRIDE + { + return filefd->FileFdError("lzma_read: %s (%d)", _("Read error"), lzma->err); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE + { + ssize_t Res; + lzma->stream.next_in = (uint8_t *)From; + lzma->stream.avail_in = Size; + lzma->stream.next_out = lzma->buffer; + lzma->stream.avail_out = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]); + lzma->err = lzma_code(&lzma->stream, LZMA_RUN); + if (lzma->err != LZMA_OK) + return -1; + size_t const n = sizeof(lzma->buffer)/sizeof(lzma->buffer[0]) - lzma->stream.avail_out; + size_t const m = (n == 0) ? 0 : fwrite(lzma->buffer, 1, n, lzma->file); + if (m != n) + { + Res = -1; + errno = 0; + } + else + { + Res = Size - lzma->stream.avail_in; + if (Res == 0) + { + // lzma run was okay, but produced no output… + Res = -1; + errno = EINTR; + } + } + return Res; + } + virtual bool InternalWriteError() APT_OVERRIDE + { + return filefd->FileFdError("lzma_write: %s (%d)", _("Write error"), lzma->err); + } + virtual bool InternalStream() const APT_OVERRIDE { return true; } + virtual bool InternalClose(std::string const &) APT_OVERRIDE + { + delete lzma; + lzma = nullptr; + return true; + } + + explicit LzmaFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd), lzma(nullptr) {} + virtual ~LzmaFileFdPrivate() { InternalClose(""); } +#endif +}; + /*}}}*/ +class APT_HIDDEN PipedFileFdPrivate: public FileFdPrivate /*{{{*/ +/* if we don't have a specific class dealing with library calls, we (un)compress + by executing a specified binary and pipe in/out what we need */ +{ +public: + virtual bool InternalOpen(int const, unsigned int const Mode) APT_OVERRIDE + { + // collect zombies here in case we reopen + if (compressor_pid > 0) + ExecWait(compressor_pid, "FileFdCompressor", true); + + if ((Mode & FileFd::ReadWrite) == FileFd::ReadWrite) + return filefd->FileFdError("ReadWrite mode is not supported for file %s", filefd->FileName.c_str()); + if (compressor.Binary == "false") + return filefd->FileFdError("libapt has inbuilt support for the %s compression," + " but was forced to ignore it in favor of an external binary – which isn't installed.", compressor.Name.c_str()); + + bool const Comp = (Mode & FileFd::WriteOnly) == FileFd::WriteOnly; + if (Comp == false && filefd->iFd != -1) + { + // Handle 'decompression' of empty files + struct stat Buf; + if (fstat(filefd->iFd, &Buf) != 0) + return filefd->FileFdErrno("fstat", "Could not stat fd %d for file %s", filefd->iFd, filefd->FileName.c_str()); + if (Buf.st_size == 0 && S_ISFIFO(Buf.st_mode) == false) + return true; + + // We don't need the file open - instead let the compressor open it + // as he properly knows better how to efficiently read from 'his' file + if (filefd->FileName.empty() == false) + { + close(filefd->iFd); + filefd->iFd = -1; + } + } + + // Create a data pipe + int Pipe[2] = {-1,-1}; + if (pipe(Pipe) != 0) + return filefd->FileFdErrno("pipe",_("Failed to create subprocess IPC")); + for (int J = 0; J != 2; J++) + SetCloseExec(Pipe[J],true); + + compressed_fd = filefd->iFd; + set_is_pipe(true); + + if (Comp == true) + filefd->iFd = Pipe[1]; + else + filefd->iFd = Pipe[0]; + + // The child.. + compressor_pid = ExecFork(); + if (compressor_pid == 0) + { + if (Comp == true) + { + dup2(compressed_fd,STDOUT_FILENO); + dup2(Pipe[0],STDIN_FILENO); + } + else + { + if (compressed_fd != -1) + dup2(compressed_fd,STDIN_FILENO); + dup2(Pipe[1],STDOUT_FILENO); + } + int const nullfd = open("/dev/null", O_WRONLY); + if (nullfd != -1) + { + dup2(nullfd,STDERR_FILENO); + close(nullfd); + } + + SetCloseExec(STDOUT_FILENO,false); + SetCloseExec(STDIN_FILENO,false); + + std::vector<char const*> Args; + Args.push_back(compressor.Binary.c_str()); + std::vector<std::string> const * const addArgs = + (Comp == true) ? &(compressor.CompressArgs) : &(compressor.UncompressArgs); + for (std::vector<std::string>::const_iterator a = addArgs->begin(); + a != addArgs->end(); ++a) + Args.push_back(a->c_str()); + if (Comp == false && filefd->FileName.empty() == false) + { + // commands not needing arguments, do not need to be told about using standard output + // in reality, only testcases with tools like cat, rev, rot13, … are able to trigger this + if (compressor.CompressArgs.empty() == false && compressor.UncompressArgs.empty() == false) + Args.push_back("--stdout"); + if (filefd->TemporaryFileName.empty() == false) + Args.push_back(filefd->TemporaryFileName.c_str()); + else + Args.push_back(filefd->FileName.c_str()); + } + Args.push_back(NULL); + + execvp(Args[0],(char **)&Args[0]); + cerr << _("Failed to exec compressor ") << Args[0] << endl; + _exit(100); + } + if (Comp == true) + close(Pipe[0]); + else + close(Pipe[1]); + + return true; + } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE + { + return read(filefd->iFd, To, Size); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE + { + return write(filefd->iFd, From, Size); + } + virtual bool InternalClose(std::string const &) APT_OVERRIDE + { + bool Ret = true; + if (filefd->iFd != -1) + { + close(filefd->iFd); + filefd->iFd = -1; + } + if (compressor_pid > 0) + Ret &= ExecWait(compressor_pid, "FileFdCompressor", true); + compressor_pid = -1; + return Ret; + } + explicit PipedFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {} + virtual ~PipedFileFdPrivate() { InternalClose(""); } +}; + /*}}}*/ +class APT_HIDDEN DirectFileFdPrivate: public FileFdPrivate /*{{{*/ +{ +public: + virtual bool InternalOpen(int const, unsigned int const) APT_OVERRIDE { return true; } + virtual ssize_t InternalUnbufferedRead(void * const To, unsigned long long const Size) APT_OVERRIDE + { + return read(filefd->iFd, To, Size); + } + virtual ssize_t InternalWrite(void const * const From, unsigned long long const Size) APT_OVERRIDE + { + // files opened read+write are strange and only really "supported" for direct files + if (buffer.size() != 0) + { + lseek(filefd->iFd, -buffer.size(), SEEK_CUR); + buffer.reset(); + } + return write(filefd->iFd, From, Size); + } + virtual bool InternalSeek(unsigned long long const To) APT_OVERRIDE + { + off_t const res = lseek(filefd->iFd, To, SEEK_SET); + if (res != (off_t)To) + return filefd->FileFdError("Unable to seek to %llu", To); + seekpos = To; + buffer.reset(); + return true; + } + virtual bool InternalSkip(unsigned long long Over) APT_OVERRIDE + { + if (Over >= buffer.size()) + { + Over -= buffer.size(); + buffer.reset(); + } + else + { + buffer.bufferstart += Over; + return true; + } + if (Over == 0) + return true; + off_t const res = lseek(filefd->iFd, Over, SEEK_CUR); + if (res < 0) + return filefd->FileFdError("Unable to seek ahead %llu",Over); + seekpos = res; + return true; + } + virtual bool InternalTruncate(unsigned long long const To) APT_OVERRIDE + { + if (buffer.size() != 0) + { + unsigned long long const seekpos = lseek(filefd->iFd, 0, SEEK_CUR); + if ((seekpos - buffer.size()) >= To) + buffer.reset(); + else if (seekpos >= To) + buffer.bufferend = (To - seekpos) + buffer.bufferstart; + else + buffer.reset(); + } + if (ftruncate(filefd->iFd, To) != 0) + return filefd->FileFdError("Unable to truncate to %llu",To); + return true; + } + virtual unsigned long long InternalTell() APT_OVERRIDE + { + return lseek(filefd->iFd,0,SEEK_CUR) - buffer.size(); + } + virtual unsigned long long InternalSize() APT_OVERRIDE + { + return filefd->FileSize(); + } + virtual bool InternalClose(std::string const &) APT_OVERRIDE { return true; } + virtual bool InternalAlwaysAutoClose() const APT_OVERRIDE { return false; } + + explicit DirectFileFdPrivate(FileFd * const filefd) : FileFdPrivate(filefd) {} + virtual ~DirectFileFdPrivate() { InternalClose(""); } +}; + /*}}}*/ +// FileFd Constructors /*{{{*/ +FileFd::FileFd(std::string FileName,unsigned int const Mode,unsigned long AccessMode) : iFd(-1), Flags(0), d(NULL) +{ + Open(FileName,Mode, None, AccessMode); +} +FileFd::FileFd(std::string FileName,unsigned int const Mode, CompressMode Compress, unsigned long AccessMode) : iFd(-1), Flags(0), d(NULL) +{ + Open(FileName,Mode, Compress, AccessMode); +} +FileFd::FileFd() : iFd(-1), Flags(AutoClose), d(NULL) {} +FileFd::FileFd(int const Fd, unsigned int const Mode, CompressMode Compress) : iFd(-1), Flags(0), d(NULL) +{ + OpenDescriptor(Fd, Mode, Compress); +} +FileFd::FileFd(int const Fd, bool const AutoClose) : iFd(-1), Flags(0), d(NULL) +{ + OpenDescriptor(Fd, ReadWrite, None, AutoClose); +} + /*}}}*/ +// FileFd::Open - Open a file /*{{{*/ +// --------------------------------------------------------------------- +/* The most commonly used open mode combinations are given with Mode */ +bool FileFd::Open(string FileName,unsigned int const Mode,CompressMode Compress, unsigned long const AccessMode) +{ + if (Mode == ReadOnlyGzip) + return Open(FileName, ReadOnly, Gzip, AccessMode); + + if (Compress == Auto && (Mode & WriteOnly) == WriteOnly) + return FileFdError("Autodetection on %s only works in ReadOnly openmode!", FileName.c_str()); + + std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors(); + std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin(); + if (Compress == Auto) + { + for (; compressor != compressors.end(); ++compressor) + { + std::string file = FileName + compressor->Extension; + if (FileExists(file) == false) + continue; + FileName = file; + break; + } + } + else if (Compress == Extension) + { + std::string::size_type const found = FileName.find_last_of('.'); + std::string ext; + if (found != std::string::npos) + { + ext = FileName.substr(found); + if (ext == ".new" || ext == ".bak") + { + std::string::size_type const found2 = FileName.find_last_of('.', found - 1); + if (found2 != std::string::npos) + ext = FileName.substr(found2, found - found2); + else + ext.clear(); + } + } + for (; compressor != compressors.end(); ++compressor) + if (ext == compressor->Extension) + break; + // no matching extension - assume uncompressed (imagine files like 'example.org_Packages') + if (compressor == compressors.end()) + for (compressor = compressors.begin(); compressor != compressors.end(); ++compressor) + if (compressor->Name == ".") + break; + } + else + { + std::string name; + switch (Compress) + { + case None: name = "."; break; + case Gzip: name = "gzip"; break; + case Bzip2: name = "bzip2"; break; + case Lzma: name = "lzma"; break; + case Xz: name = "xz"; break; + case Lz4: name = "lz4"; break; + case Zstd: name = "zstd"; break; + case Auto: + case Extension: + // Unreachable + return FileFdError("Opening File %s in None, Auto or Extension should be already handled?!?", FileName.c_str()); + } + for (; compressor != compressors.end(); ++compressor) + if (compressor->Name == name) + break; + if (compressor == compressors.end()) + return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str()); + } + + if (compressor == compressors.end()) + return FileFdError("Can't find a match for specified compressor mode for file %s", FileName.c_str()); + return Open(FileName, Mode, *compressor, AccessMode); +} +bool FileFd::Open(string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor, unsigned long const AccessMode) +{ + Close(); + Flags = AutoClose; + + if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0) + return FileFdError("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str()); + if ((Mode & ReadWrite) == 0) + return FileFdError("No openmode provided in FileFd::Open for %s", FileName.c_str()); + + unsigned int OpenMode = Mode; + if (FileName == "/dev/null") + OpenMode = OpenMode & ~(Atomic | Exclusive | Create | Empty); + + if ((OpenMode & Atomic) == Atomic) + { + Flags |= Replace; + } + else if ((OpenMode & (Exclusive | Create)) == (Exclusive | Create)) + { + // for atomic, this will be done by rename in Close() + RemoveFile("FileFd::Open", FileName); + } + if ((OpenMode & Empty) == Empty) + { + struct stat Buf; + if (lstat(FileName.c_str(),&Buf) == 0 && S_ISLNK(Buf.st_mode)) + RemoveFile("FileFd::Open", FileName); + } + + int fileflags = 0; + #define if_FLAGGED_SET(FLAG, MODE) if ((OpenMode & FLAG) == FLAG) fileflags |= MODE + if_FLAGGED_SET(ReadWrite, O_RDWR); + else if_FLAGGED_SET(ReadOnly, O_RDONLY); + else if_FLAGGED_SET(WriteOnly, O_WRONLY); + + if_FLAGGED_SET(Create, O_CREAT); + if_FLAGGED_SET(Empty, O_TRUNC); + if_FLAGGED_SET(Exclusive, O_EXCL); + #undef if_FLAGGED_SET + + if ((OpenMode & Atomic) == Atomic) + { + char *name = strdup((FileName + ".XXXXXX").c_str()); + + if((iFd = mkstemp(name)) == -1) + { + free(name); + return FileFdErrno("mkstemp", "Could not create temporary file for %s", FileName.c_str()); + } + + TemporaryFileName = string(name); + free(name); + + // umask() will always set the umask and return the previous value, so + // we first set the umask and then reset it to the old value + mode_t const CurrentUmask = umask(0); + umask(CurrentUmask); + // calculate the actual file permissions (just like open/creat) + mode_t const FilePermissions = (AccessMode & ~CurrentUmask); + + if(fchmod(iFd, FilePermissions) == -1) + return FileFdErrno("fchmod", "Could not change permissions for temporary file %s", TemporaryFileName.c_str()); + } + else + iFd = open(FileName.c_str(), fileflags, AccessMode); + + this->FileName = FileName; + if (iFd == -1 || OpenInternDescriptor(OpenMode, compressor) == false) + { + if (iFd != -1) + { + close (iFd); + iFd = -1; + } + return FileFdErrno("open",_("Could not open file %s"), FileName.c_str()); + } + + SetCloseExec(iFd,true); + return true; +} + /*}}}*/ +// FileFd::OpenDescriptor - Open a filedescriptor /*{{{*/ +bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose) +{ + std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors(); + std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin(); + std::string name; + + // compat with the old API + if (Mode == ReadOnlyGzip && Compress == None) + Compress = Gzip; + + switch (Compress) + { + case None: name = "."; break; + case Gzip: name = "gzip"; break; + case Bzip2: name = "bzip2"; break; + case Lzma: name = "lzma"; break; + case Xz: name = "xz"; break; + case Lz4: name = "lz4"; break; + case Zstd: name = "zstd"; break; + case Auto: + case Extension: + if (AutoClose == true && Fd != -1) + close(Fd); + return FileFdError("Opening Fd %d in Auto or Extension compression mode is not supported", Fd); + } + for (; compressor != compressors.end(); ++compressor) + if (compressor->Name == name) + break; + if (compressor == compressors.end()) + { + if (AutoClose == true && Fd != -1) + close(Fd); + return FileFdError("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str()); + } + return OpenDescriptor(Fd, Mode, *compressor, AutoClose); +} +bool FileFd::OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration::Compressor const &compressor, bool AutoClose) +{ + Close(); + Flags = (AutoClose) ? FileFd::AutoClose : 0; + iFd = Fd; + this->FileName = ""; + if (OpenInternDescriptor(Mode, compressor) == false) + { + if (iFd != -1 && ( + (Flags & Compressed) == Compressed || + AutoClose == true)) + { + close (iFd); + iFd = -1; + } + return FileFdError(_("Could not open file descriptor %d"), Fd); + } + return true; +} +bool FileFd::OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor) +{ + if (iFd == -1) + return false; + + if (d != nullptr) + d->InternalClose(FileName); + + if (d == nullptr) + { + if (false) + /* dummy so that the rest can be 'else if's */; +#define APT_COMPRESS_INIT(NAME, CONSTRUCTOR) \ + else if (compressor.Name == NAME) \ + d = new CONSTRUCTOR(this) +#ifdef HAVE_ZLIB + APT_COMPRESS_INIT("gzip", GzipFileFdPrivate); +#endif +#ifdef HAVE_BZ2 + APT_COMPRESS_INIT("bzip2", Bz2FileFdPrivate); +#endif +#ifdef HAVE_LZMA + APT_COMPRESS_INIT("xz", LzmaFileFdPrivate); + APT_COMPRESS_INIT("lzma", LzmaFileFdPrivate); +#endif +#ifdef HAVE_LZ4 + APT_COMPRESS_INIT("lz4", Lz4FileFdPrivate); +#endif +#ifdef HAVE_ZSTD + APT_COMPRESS_INIT("zstd", ZstdFileFdPrivate); +#endif +#undef APT_COMPRESS_INIT + else if (compressor.Name == "." || compressor.Binary.empty() == true) + d = new DirectFileFdPrivate(this); + else + d = new PipedFileFdPrivate(this); + + if (Mode & BufferedWrite) + d = new BufferedWriteFileFdPrivate(d); + + d->set_openmode(Mode); + d->set_compressor(compressor); + if ((Flags & AutoClose) != AutoClose && d->InternalAlwaysAutoClose()) + { + // Need to duplicate fd here or gz/bz2 close for cleanup will close the fd as well + int const internFd = dup(iFd); + if (internFd == -1) + return FileFdErrno("OpenInternDescriptor", _("Could not open file descriptor %d"), iFd); + iFd = internFd; + } + } + return d->InternalOpen(iFd, Mode); +} + /*}}}*/ +// FileFd::~File - Closes the file /*{{{*/ +// --------------------------------------------------------------------- +/* If the proper modes are selected then we close the Fd and possibly + unlink the file on error. */ +FileFd::~FileFd() +{ + Close(); + if (d != NULL) + d->InternalClose(FileName); + delete d; + d = NULL; +} + /*}}}*/ +// FileFd::Read - Read a bit of the file /*{{{*/ +// --------------------------------------------------------------------- +/* We are careful to handle interruption by a signal while reading + gracefully. */ +bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) +{ + if (d == nullptr || Failed()) + return false; + ssize_t Res = 1; + errno = 0; + if (Actual != 0) + *Actual = 0; + *((char *)To) = '\0'; + while (Res > 0 && Size > 0) + { + Res = d->InternalRead(To, Size); + + if (Res < 0) + { + if (errno == EINTR) + { + // trick the while-loop into running again + Res = 1; + errno = 0; + continue; + } + return d->InternalReadError(); + } + + To = (char *)To + Res; + Size -= Res; + if (d != NULL) + d->set_seekpos(d->get_seekpos() + Res); + if (Actual != 0) + *Actual += Res; + } + + if (Size == 0) + return true; + + // Eof handling + if (Actual != 0) + { + Flags |= HitEof; + return true; + } + + return FileFdError(_("read, still have %llu to read but none left"), Size); +} +bool FileFd::Read(int const Fd, void *To, unsigned long long Size, unsigned long long * const Actual) +{ + ssize_t Res = 1; + errno = 0; + if (Actual != nullptr) + *Actual = 0; + *static_cast<char *>(To) = '\0'; + while (Res > 0 && Size > 0) + { + Res = read(Fd, To, Size); + if (Res < 0) + { + if (errno == EINTR) + { + Res = 1; + errno = 0; + continue; + } + return _error->Errno("read", _("Read error")); + } + To = static_cast<char *>(To) + Res; + Size -= Res; + if (Actual != 0) + *Actual += Res; + } + if (Size == 0) + return true; + if (Actual != nullptr) + return true; + return _error->Error(_("read, still have %llu to read but none left"), Size); +} + /*}}}*/ +// FileFd::ReadLine - Read a complete line from the file /*{{{*/ +char* FileFd::ReadLine(char *To, unsigned long long const Size) +{ + *To = '\0'; + if (d == nullptr || Failed()) + return nullptr; + return d->InternalReadLine(To, Size); +} +bool FileFd::ReadLine(std::string &To) +{ + To.clear(); + if (d == nullptr || Failed()) + return false; + constexpr size_t buflen = 4096; + char buffer[buflen]; + size_t len; + do + { + if (d->InternalReadLine(buffer, buflen) == nullptr) + return false; + len = strlen(buffer); + To.append(buffer, len); + } while (len == buflen - 1 && buffer[len - 2] != '\n'); + // remove the newline at the end + auto const i = To.find_last_not_of("\r\n"); + if (i == std::string::npos) + To.clear(); + else + To.erase(i + 1); + return true; +} + /*}}}*/ +// FileFd::Flush - Flush the file /*{{{*/ +bool FileFd::Flush() +{ + if (Failed()) + return false; + if (d == nullptr) + return true; + + return d->InternalFlush(); +} + /*}}}*/ +// FileFd::Write - Write to the file /*{{{*/ +bool FileFd::Write(const void *From,unsigned long long Size) +{ + if (d == nullptr || Failed()) + return false; + ssize_t Res = 1; + errno = 0; + while (Res > 0 && Size > 0) + { + Res = d->InternalWrite(From, Size); + + if (Res < 0) + { + if (errno == EINTR) + { + // trick the while-loop into running again + Res = 1; + errno = 0; + continue; + } + return d->InternalWriteError(); + } + + From = (char const *)From + Res; + Size -= Res; + if (d != NULL) + d->set_seekpos(d->get_seekpos() + Res); + } + + if (Size == 0) + return true; + + return FileFdError(_("write, still have %llu to write but couldn't"), Size); +} +bool FileFd::Write(int Fd, const void *From, unsigned long long Size) +{ + ssize_t Res = 1; + errno = 0; + while (Res > 0 && Size > 0) + { + Res = write(Fd,From,Size); + if (Res < 0 && errno == EINTR) + continue; + if (Res < 0) + return _error->Errno("write",_("Write error")); + + From = (char const *)From + Res; + Size -= Res; + } + + if (Size == 0) + return true; + + return _error->Error(_("write, still have %llu to write but couldn't"), Size); +} + /*}}}*/ +// FileFd::Seek - Seek in the file /*{{{*/ +bool FileFd::Seek(unsigned long long To) +{ + if (d == nullptr || Failed()) + return false; + Flags &= ~HitEof; + return d->InternalSeek(To); +} + /*}}}*/ +// FileFd::Skip - Skip over data in the file /*{{{*/ +bool FileFd::Skip(unsigned long long Over) +{ + if (d == nullptr || Failed()) + return false; + return d->InternalSkip(Over); +} + /*}}}*/ +// FileFd::Truncate - Truncate the file /*{{{*/ +bool FileFd::Truncate(unsigned long long To) +{ + if (d == nullptr || Failed()) + return false; + // truncating /dev/null is always successful - as we get an error otherwise + if (To == 0 && FileName == "/dev/null") + return true; + return d->InternalTruncate(To); +} + /*}}}*/ +// FileFd::Tell - Current seek position /*{{{*/ +// --------------------------------------------------------------------- +/* */ +unsigned long long FileFd::Tell() +{ + if (d == nullptr || Failed()) + return false; + off_t const Res = d->InternalTell(); + if (Res == (off_t)-1) + FileFdErrno("lseek","Failed to determine the current file position"); + d->set_seekpos(Res); + return Res; +} + /*}}}*/ +static bool StatFileFd(char const * const msg, int const iFd, std::string const &FileName, struct stat &Buf, FileFdPrivate * const d) /*{{{*/ +{ + bool ispipe = (d != NULL && d->get_is_pipe() == true); + if (ispipe == false) + { + if (fstat(iFd,&Buf) != 0) + // higher-level code will generate more meaningful messages, + // even translated this would be meaningless for users + return _error->Errno("fstat", "Unable to determine %s for fd %i", msg, iFd); + if (FileName.empty() == false) + ispipe = S_ISFIFO(Buf.st_mode); + } + + // for compressor pipes st_size is undefined and at 'best' zero + if (ispipe == true) + { + // we set it here, too, as we get the info here for free + // in theory the Open-methods should take care of it already + if (d != NULL) + d->set_is_pipe(true); + if (stat(FileName.c_str(), &Buf) != 0) + return _error->Errno("fstat", "Unable to determine %s for file %s", msg, FileName.c_str()); + } + return true; +} + /*}}}*/ +// FileFd::FileSize - Return the size of the file /*{{{*/ +unsigned long long FileFd::FileSize() +{ + struct stat Buf; + if (StatFileFd("file size", iFd, FileName, Buf, d) == false) + { + Flags |= Fail; + return 0; + } + return Buf.st_size; +} + /*}}}*/ +// FileFd::ModificationTime - Return the time of last touch /*{{{*/ +time_t FileFd::ModificationTime() +{ + struct stat Buf; + if (StatFileFd("modification time", iFd, FileName, Buf, d) == false) + { + Flags |= Fail; + return 0; + } + return Buf.st_mtime; +} + /*}}}*/ +// FileFd::Size - Return the size of the content in the file /*{{{*/ +unsigned long long FileFd::Size() +{ + if (d == nullptr) + return 0; + return d->InternalSize(); +} + /*}}}*/ +// FileFd::Close - Close the file if the close flag is set /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool FileFd::Close() +{ + if (Failed() == false && Flush() == false) + return false; + if (iFd == -1) + return true; + + bool Res = true; + if ((Flags & AutoClose) == AutoClose) + { + if ((Flags & Compressed) != Compressed && iFd > 0 && close(iFd) != 0) + Res &= _error->Errno("close",_("Problem closing the file %s"), FileName.c_str()); + } + + if (d != NULL) + { + Res &= d->InternalClose(FileName); + delete d; + d = NULL; + } + + if ((Flags & Replace) == Replace) { + if (Failed() == false && rename(TemporaryFileName.c_str(), FileName.c_str()) != 0) + Res &= _error->Errno("rename",_("Problem renaming the file %s to %s"), TemporaryFileName.c_str(), FileName.c_str()); + + FileName = TemporaryFileName; // for the unlink() below. + TemporaryFileName.clear(); + } + + iFd = -1; + + if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail && + FileName.empty() == false) + Res &= RemoveFile("FileFd::Close", FileName); + + if (Res == false) + Flags |= Fail; + return Res; +} + /*}}}*/ +// FileFd::Sync - Sync the file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool FileFd::Sync() +{ + if (fsync(iFd) != 0) + return FileFdErrno("sync",_("Problem syncing the file")); + return true; +} + /*}}}*/ +// FileFd::FileFdErrno - set Fail and call _error->Errno *{{{*/ +bool FileFd::FileFdErrno(const char *Function, const char *Description,...) +{ + Flags |= Fail; + va_list args; + size_t msgSize = 400; + int const errsv = errno; + bool retry; + do { + va_start(args,Description); + retry = _error->InsertErrno(GlobalError::ERROR, Function, Description, args, errsv, msgSize); + va_end(args); + } while (retry); + return false; +} + /*}}}*/ +// FileFd::FileFdError - set Fail and call _error->Error *{{{*/ +bool FileFd::FileFdError(const char *Description,...) { + Flags |= Fail; + va_list args; + size_t msgSize = 400; + bool retry; + do { + va_start(args,Description); + retry = _error->Insert(GlobalError::ERROR, Description, args, msgSize); + va_end(args); + } while (retry); + return false; +} + /*}}}*/ +// Glob - wrapper around "glob()" /*{{{*/ +std::vector<std::string> Glob(std::string const &pattern, int flags) +{ + std::vector<std::string> result; + glob_t globbuf; + int glob_res; + unsigned int i; + + glob_res = glob(pattern.c_str(), flags, NULL, &globbuf); + + if (glob_res != 0) + { + if(glob_res != GLOB_NOMATCH) { + _error->Errno("glob", "Problem with glob"); + return result; + } + } + + // append results + for(i=0;i<globbuf.gl_pathc;i++) + result.push_back(string(globbuf.gl_pathv[i])); + + globfree(&globbuf); + return result; +} + /*}}}*/ +static std::string APT_NONNULL(1) GetTempDirEnv(char const * const env) /*{{{*/ +{ + const char *tmpdir = getenv(env); + +#ifdef P_tmpdir + if (!tmpdir) + tmpdir = P_tmpdir; +#endif + + struct stat st; + if (!tmpdir || strlen(tmpdir) == 0 || // tmpdir is set + stat(tmpdir, &st) != 0 || (st.st_mode & S_IFDIR) == 0) // exists and is directory + tmpdir = "/tmp"; + else if (geteuid() != 0 && // root can do everything anyway + faccessat(AT_FDCWD, tmpdir, R_OK | W_OK | X_OK, AT_EACCESS) != 0) // current user has rwx access to directory + tmpdir = "/tmp"; + + return string(tmpdir); +} + /*}}}*/ +std::string GetTempDir() /*{{{*/ +{ + return GetTempDirEnv("TMPDIR"); +} +std::string GetTempDir(std::string const &User) +{ + // no need/possibility to drop privs + if(getuid() != 0 || User.empty() || User == "root") + return GetTempDir(); + + struct passwd const * const pw = getpwnam(User.c_str()); + if (pw == NULL) + return GetTempDir(); + + gid_t const old_euid = geteuid(); + gid_t const old_egid = getegid(); + if (setegid(pw->pw_gid) != 0) + _error->Errno("setegid", "setegid %u failed", pw->pw_gid); + if (seteuid(pw->pw_uid) != 0) + _error->Errno("seteuid", "seteuid %u failed", pw->pw_uid); + + std::string const tmp = GetTempDir(); + + if (seteuid(old_euid) != 0) + _error->Errno("seteuid", "seteuid %u failed", old_euid); + if (setegid(old_egid) != 0) + _error->Errno("setegid", "setegid %u failed", old_egid); + + return tmp; +} + /*}}}*/ +FileFd* GetTempFile(std::string const &Prefix, bool ImmediateUnlink, FileFd * const TmpFd) /*{{{*/ +{ + return GetTempFile(Prefix, ImmediateUnlink, TmpFd, false); +} +FileFd* GetTempFile(std::string const &Prefix, bool ImmediateUnlink, FileFd * const TmpFd, bool Buffered) +{ + std::string fn; + std::string const tempdir = GetTempDir(); + int fd = -1; +#ifdef O_TMPFILE + if (ImmediateUnlink) + fd = open(tempdir.c_str(), O_RDWR|O_TMPFILE|O_EXCL|O_CLOEXEC, 0600); + if (fd < 0) +#endif + { + auto const suffix = Prefix.find(".XXXXXX."); + std::vector<char> buffer(tempdir.length() + 1 + Prefix.length() + (suffix == std::string::npos ? 7 : 0) + 1, '\0'); + if (suffix != std::string::npos) + { + if (snprintf(buffer.data(), buffer.size(), "%s/%s", tempdir.c_str(), Prefix.c_str()) > 0) + { + ssize_t const suffixlen = (buffer.size() - 1) - (tempdir.length() + 1 + suffix + 7); + if (likely(suffixlen > 0)) + fd = mkstemps(buffer.data(), suffixlen); + } + } + else + { + if (snprintf(buffer.data(), buffer.size(), "%s/%s.XXXXXX", tempdir.c_str(), Prefix.c_str()) > 0) + fd = mkstemp(buffer.data()); + } + fn.assign(buffer.data(), buffer.size() - 1); + if (ImmediateUnlink && fd != -1) + unlink(fn.c_str()); + } + if (fd < 0) + { + _error->Errno("GetTempFile",_("Unable to mkstemp %s"), fn.c_str()); + return nullptr; + } + FileFd * const Fd = TmpFd == nullptr ? new FileFd() : TmpFd; + if (not Fd->OpenDescriptor(fd, FileFd::ReadWrite | (Buffered ? FileFd::BufferedWrite : 0), FileFd::None, true)) + { + _error->Errno("GetTempFile",_("Unable to write to %s"),fn.c_str()); + if (TmpFd == nullptr) + delete Fd; + return nullptr; + } + if (not ImmediateUnlink) + Fd->SetFileName(fn); + return Fd; +} + /*}}}*/ +bool Rename(std::string From, std::string To) /*{{{*/ +{ + if (rename(From.c_str(),To.c_str()) != 0) + { + _error->Error(_("rename failed, %s (%s -> %s)."),strerror(errno), + From.c_str(),To.c_str()); + return false; + } + return true; +} + /*}}}*/ +bool Popen(const char *Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode, bool CaptureStderr, bool Sandbox) /*{{{*/ +{ + int fd; + if (Mode != FileFd::ReadOnly && Mode != FileFd::WriteOnly) + return _error->Error("Popen supports ReadOnly (x)or WriteOnly mode only"); + + int Pipe[2] = {-1, -1}; + if(pipe(Pipe) != 0) + return _error->Errno("pipe", _("Failed to create subprocess IPC")); + + std::set<int> keep_fds; + keep_fds.insert(Pipe[0]); + keep_fds.insert(Pipe[1]); + Child = ExecFork(keep_fds); + if(Child < 0) + return _error->Errno("fork", "Failed to fork"); + if(Child == 0) + { + if (Sandbox && (getuid() == 0 || geteuid() == 0) && !DropPrivileges()) + { + _error->DumpErrors(); + _exit(1); + } + if(Mode == FileFd::ReadOnly) + { + close(Pipe[0]); + fd = Pipe[1]; + } + else if(Mode == FileFd::WriteOnly) + { + close(Pipe[1]); + fd = Pipe[0]; + } + + if(Mode == FileFd::ReadOnly) + { + dup2(fd, 1); + if (CaptureStderr == true) + dup2(fd, 2); + } else if(Mode == FileFd::WriteOnly) + dup2(fd, 0); + + execv(Args[0], (char**)Args); + _exit(100); + } + if(Mode == FileFd::ReadOnly) + { + close(Pipe[1]); + fd = Pipe[0]; + } + else if(Mode == FileFd::WriteOnly) + { + close(Pipe[0]); + fd = Pipe[1]; + } + else + return _error->Error("Popen supports ReadOnly (x)or WriteOnly mode only"); + Fd.OpenDescriptor(fd, Mode, FileFd::None, true); + + return true; +} + /*}}}*/ +bool DropPrivileges() /*{{{*/ +{ + if(_config->FindB("Debug::NoDropPrivs", false) == true) + return true; + +#if __gnu_linux__ +#if defined(PR_SET_NO_NEW_PRIVS) && ( PR_SET_NO_NEW_PRIVS != 38 ) +#error "PR_SET_NO_NEW_PRIVS is defined, but with a different value than expected!" +#endif + // see prctl(2), needs linux3.5 at runtime - magic constant to avoid it at buildtime + int ret = prctl(38, 1, 0, 0, 0); + // ignore EINVAL - kernel is too old to understand the option + if(ret < 0 && errno != EINVAL) + _error->Warning("PR_SET_NO_NEW_PRIVS failed with %i", ret); +#endif + + // empty setting disables privilege dropping - this also ensures + // backward compatibility, see bug #764506 + const std::string toUser = _config->Find("APT::Sandbox::User"); + if (toUser.empty() || toUser == "root") + return true; + + // a lot can go wrong trying to drop privileges completely, + // so ideally we would like to verify that we have done it – + // but the verify asks for too much in case of fakeroot (and alike) + // [Specific checks can be overridden with dedicated options] + bool const VerifySandboxing = _config->FindB("APT::Sandbox::Verify", false); + + // uid will be 0 in the end, but gid might be different anyway + uid_t const old_uid = getuid(); + gid_t const old_gid = getgid(); + + if (old_uid != 0) + return true; + + struct passwd *pw = getpwnam(toUser.c_str()); + if (pw == NULL) + return _error->Error("No user %s, can not drop rights", toUser.c_str()); + + // Do not change the order here, it might break things + // Get rid of all our supplementary groups first + if (setgroups(1, &pw->pw_gid)) + return _error->Errno("setgroups", "Failed to setgroups"); + + // Now change the group ids to the new user +#ifdef HAVE_SETRESGID + if (setresgid(pw->pw_gid, pw->pw_gid, pw->pw_gid) != 0) + return _error->Errno("setresgid", "Failed to set new group ids"); +#else + if (setegid(pw->pw_gid) != 0) + return _error->Errno("setegid", "Failed to setegid"); + + if (setgid(pw->pw_gid) != 0) + return _error->Errno("setgid", "Failed to setgid"); +#endif + + // Change the user ids to the new user +#ifdef HAVE_SETRESUID + if (setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid) != 0) + return _error->Errno("setresuid", "Failed to set new user ids"); +#else + if (setuid(pw->pw_uid) != 0) + return _error->Errno("setuid", "Failed to setuid"); + if (seteuid(pw->pw_uid) != 0) + return _error->Errno("seteuid", "Failed to seteuid"); +#endif + + // disabled by default as fakeroot doesn't implement getgroups currently (#806521) + if (VerifySandboxing == true || _config->FindB("APT::Sandbox::Verify::Groups", false) == true) + { + // Verify that the user isn't still in any supplementary groups + long const ngroups_max = sysconf(_SC_NGROUPS_MAX); + std::unique_ptr<gid_t[]> gidlist(new gid_t[ngroups_max]); + if (unlikely(gidlist == NULL)) + return _error->Error("Allocation of a list of size %lu for getgroups failed", ngroups_max); + ssize_t gidlist_nr; + if ((gidlist_nr = getgroups(ngroups_max, gidlist.get())) < 0) + return _error->Errno("getgroups", "Could not get new groups (%lu)", ngroups_max); + for (ssize_t i = 0; i < gidlist_nr; ++i) + if (gidlist[i] != pw->pw_gid) + return _error->Error("Could not switch group, user %s is still in group %d", toUser.c_str(), gidlist[i]); + } + + // enabled by default as all fakeroot-lookalikes should fake that accordingly + if (VerifySandboxing == true || _config->FindB("APT::Sandbox::Verify::IDs", true) == true) + { + // Verify that gid, egid, uid, and euid changed + if (getgid() != pw->pw_gid) + return _error->Error("Could not switch group"); + if (getegid() != pw->pw_gid) + return _error->Error("Could not switch effective group"); + if (getuid() != pw->pw_uid) + return _error->Error("Could not switch user"); + if (geteuid() != pw->pw_uid) + return _error->Error("Could not switch effective user"); + +#ifdef HAVE_GETRESUID + // verify that the saved set-user-id was changed as well + uid_t ruid = 0; + uid_t euid = 0; + uid_t suid = 0; + if (getresuid(&ruid, &euid, &suid)) + return _error->Errno("getresuid", "Could not get saved set-user-ID"); + if (suid != pw->pw_uid) + return _error->Error("Could not switch saved set-user-ID"); +#endif + +#ifdef HAVE_GETRESGID + // verify that the saved set-group-id was changed as well + gid_t rgid = 0; + gid_t egid = 0; + gid_t sgid = 0; + if (getresgid(&rgid, &egid, &sgid)) + return _error->Errno("getresuid", "Could not get saved set-group-ID"); + if (sgid != pw->pw_gid) + return _error->Error("Could not switch saved set-group-ID"); +#endif + } + + // disabled as fakeroot doesn't forbid (by design) (re)gaining root from unprivileged + if (VerifySandboxing == true || _config->FindB("APT::Sandbox::Verify::Regain", false) == true) + { + // Check that uid and gid changes do not work anymore + if (pw->pw_gid != old_gid && (setgid(old_gid) != -1 || setegid(old_gid) != -1)) + return _error->Error("Could restore a gid to root, privilege dropping did not work"); + + if (pw->pw_uid != old_uid && (setuid(old_uid) != -1 || seteuid(old_uid) != -1)) + return _error->Error("Could restore a uid to root, privilege dropping did not work"); + } + + if (_config->FindB("APT::Sandbox::ResetEnvironment", true)) + { + setenv("HOME", pw->pw_dir, 1); + setenv("USER", pw->pw_name, 1); + setenv("USERNAME", pw->pw_name, 1); + setenv("LOGNAME", pw->pw_name, 1); + auto const shell = flNotDir(pw->pw_shell); + if (shell == "false" || shell == "nologin") + setenv("SHELL", "/bin/sh", 1); + else + setenv("SHELL", pw->pw_shell, 1); + auto const apt_setenv_tmp = [](char const * const env) { + auto const tmpdir = getenv(env); + if (tmpdir != nullptr) + { + auto const ourtmpdir = GetTempDirEnv(env); + if (ourtmpdir != tmpdir) + setenv(env, ourtmpdir.c_str(), 1); + } + }; + apt_setenv_tmp("TMPDIR"); + apt_setenv_tmp("TEMPDIR"); + apt_setenv_tmp("TMP"); + apt_setenv_tmp("TEMP"); + } + + return true; +} + /*}}}*/ +bool OpenConfigurationFileFd(std::string const &File, FileFd &Fd) /*{{{*/ +{ + int const fd = open(File.c_str(), O_RDONLY | O_CLOEXEC | O_NOCTTY); + if (fd == -1) + return _error->WarningE("open", _("Unable to read %s"), File.c_str()); + APT::Configuration::Compressor none(".", "", "", nullptr, nullptr, 0); + if (Fd.OpenDescriptor(fd, FileFd::ReadOnly, none, true) == false) + return false; + Fd.SetFileName(File); + return true; +} + /*}}}*/ +int Inhibit(const char *what, const char *who, const char *why, const char *mode) /*{{{*/ +{ +#ifdef HAVE_SYSTEMD + sd_bus_error error = SD_BUS_ERROR_NULL; + sd_bus_message *m = NULL; + sd_bus *bus = NULL; + int fd; + int r; + + r = sd_bus_open_system(&bus); + if (r < 0) + goto out; + + r = sd_bus_call_method(bus, + "org.freedesktop.login1", + "/org/freedesktop/login1", + "org.freedesktop.login1.Manager", + "Inhibit", + &error, + &m, + "ssss", + what, + who, + why, + mode); + if (r < 0) + goto out; + + r = sd_bus_message_read(m, "h", &fd); + if (r < 0) + goto out; + + // We received a file descriptor, return it - systemd will close the read fd + // on free, so let's duplicate it here. + r = dup(fd); +out: + sd_bus_error_free(&error); + sd_bus_message_unref(m); + sd_bus_unref(bus); + return r; +#else + return -ENOTSUP; +#endif +} + /*}}}*/ diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h new file mode 100644 index 0000000..970b118 --- /dev/null +++ b/apt-pkg/contrib/fileutl.h @@ -0,0 +1,287 @@ +// -*- mode: cpp; mode: fold -*- +// SPDX-License-Identifier: GPL-2.0+ +// Description /*{{{*/ +/* ###################################################################### + + File Utilities + + CopyFile - Buffered copy of a single file + GetLock - dpkg compatible lock file manipulation (fcntl) + FileExists - Returns true if the file exists + SafeGetCWD - Returns the CWD in a string with overrun protection + + The file class is a handy abstraction for various functions+classes + that need to accept filenames. + + This file had this historic note, but now includes further changes + under the GPL-2.0+: + + This source is placed in the Public Domain, do with it what you will + It was originally written by Jason Gunthorpe. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_FILEUTL_H +#define PKGLIB_FILEUTL_H + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/macros.h> + +#include <set> +#include <string> +#include <vector> +#include <sys/stat.h> +#include <time.h> + +/* Define this for python-apt */ +#define APT_HAS_GZIP 1 + +class FileFdPrivate; +class APT_PUBLIC FileFd +{ + friend class FileFdPrivate; + friend class GzipFileFdPrivate; + friend class Bz2FileFdPrivate; + friend class LzmaFileFdPrivate; + friend class Lz4FileFdPrivate; + friend class ZstdFileFdPrivate; + friend class DirectFileFdPrivate; + friend class PipedFileFdPrivate; + protected: + int iFd; + + enum LocalFlags {AutoClose = (1<<0),Fail = (1<<1),DelOnFail = (1<<2), + HitEof = (1<<3), Replace = (1<<4), Compressed = (1<<5) }; + unsigned long Flags; + std::string FileName; + std::string TemporaryFileName; + + public: + enum OpenMode { + ReadOnly = (1 << 0), + WriteOnly = (1 << 1), + ReadWrite = ReadOnly | WriteOnly, + + Create = (1 << 2), + Exclusive = (1 << 3), + Atomic = Exclusive | (1 << 4), + Empty = (1 << 5), + BufferedWrite = (1 << 6), + + WriteEmpty = ReadWrite | Create | Empty, + WriteExists = ReadWrite, + WriteAny = ReadWrite | Create, + WriteTemp = ReadWrite | Create | Exclusive, + ReadOnlyGzip, + WriteAtomic = ReadWrite | Create | Atomic + }; + enum CompressMode + { + Auto = 'A', + None = 'N', + Extension = 'E', + Gzip = 'G', + Bzip2 = 'B', + Lzma = 'L', + Xz = 'X', + Lz4 = '4', + Zstd = 'Z' + }; + + inline bool Read(void *To,unsigned long long Size,bool AllowEof) + { + unsigned long long Jnk; + if (AllowEof) + return Read(To,Size,&Jnk); + return Read(To,Size); + } + bool Read(void *To,unsigned long long Size,unsigned long long *Actual = 0); + bool static Read(int const Fd, void *To, unsigned long long Size, unsigned long long * const Actual = 0); + /** read a complete line or until buffer is full + * + * The buffer will always be \\0 terminated, so at most Size-1 characters are read. + * If the buffer holds a complete line the last character (before \\0) will be + * the newline character \\n otherwise the line was longer than the buffer. + * + * @param To buffer which will hold the line + * @param Size of the buffer to fill + * @param \b nullptr is returned in error cases, otherwise + * the parameter \b To now filled with the line. + */ + char* ReadLine(char *To, unsigned long long const Size); + /** read a complete line from the file + * + * Similar to std::getline() the string does \b not include + * the newline, but just the content of the line as the newline + * is not needed to distinguish cases as for the other #ReadLine method. + * + * @param To string which will hold the line + * @return \b true if successful, otherwise \b false + */ + bool ReadLine(std::string &To); + bool Flush(); + bool Write(const void *From,unsigned long long Size); + bool static Write(int Fd, const void *From, unsigned long long Size); + bool Seek(unsigned long long To); + bool Skip(unsigned long long To); + bool Truncate(unsigned long long To); + unsigned long long Tell(); + // the size of the file content (compressed files will be uncompressed first) + unsigned long long Size(); + // the size of the file itself + unsigned long long FileSize(); + time_t ModificationTime(); + + bool Open(std::string FileName,unsigned int const Mode,CompressMode Compress,unsigned long const AccessMode = 0666); + bool Open(std::string FileName,unsigned int const Mode,APT::Configuration::Compressor const &compressor,unsigned long const AccessMode = 0666); + inline bool Open(std::string const &FileName,unsigned int const Mode, unsigned long const AccessMode = 0666) { + return Open(FileName, Mode, None, AccessMode); + }; + bool OpenDescriptor(int Fd, unsigned int const Mode, CompressMode Compress, bool AutoClose=false); + bool OpenDescriptor(int Fd, unsigned int const Mode, APT::Configuration::Compressor const &compressor, bool AutoClose=false); + inline bool OpenDescriptor(int Fd, unsigned int const Mode, bool AutoClose=false) { + return OpenDescriptor(Fd, Mode, None, AutoClose); + }; + bool Close(); + bool Sync(); + + // Simple manipulators + inline int Fd() {return iFd;}; + inline void Fd(int fd) { OpenDescriptor(fd, ReadWrite);}; + + inline bool IsOpen() {return iFd >= 0;}; + inline bool Failed() {return (Flags & Fail) == Fail;}; + inline void EraseOnFailure() {Flags |= DelOnFail;}; + inline void OpFail() {Flags |= Fail;}; + inline bool Eof() {return (Flags & HitEof) == HitEof;}; + inline bool IsCompressed() {return (Flags & Compressed) == Compressed;}; + inline std::string &Name() {return FileName;}; + inline void SetFileName(std::string const &name) { FileName = name; }; + + FileFd(std::string FileName,unsigned int const Mode,unsigned long AccessMode = 0666); + FileFd(std::string FileName,unsigned int const Mode, CompressMode Compress, unsigned long AccessMode = 0666); + FileFd(); + FileFd(int const Fd, unsigned int const Mode = ReadWrite, CompressMode Compress = None); + FileFd(int const Fd, bool const AutoClose); + virtual ~FileFd(); + + private: + FileFdPrivate * d; + APT_HIDDEN FileFd(const FileFd &); + APT_HIDDEN FileFd & operator=(const FileFd &); + APT_HIDDEN bool OpenInternDescriptor(unsigned int const Mode, APT::Configuration::Compressor const &compressor); + + // private helpers to set Fail flag and call _error->Error + APT_HIDDEN bool FileFdErrno(const char* Function, const char* Description,...) APT_PRINTF(3) APT_COLD; + APT_HIDDEN bool FileFdError(const char* Description,...) APT_PRINTF(2) APT_COLD; +}; + +APT_PUBLIC bool RunScripts(const char *Cnf); +APT_PUBLIC bool CopyFile(FileFd &From,FileFd &To); +APT_PUBLIC bool RemoveFile(char const * const Function, std::string const &FileName); +APT_PUBLIC bool RemoveFileAt(char const * const Function, int const dirfd, std::string const &FileName); +APT_PUBLIC int GetLock(std::string File,bool Errors = true); +APT_PUBLIC bool FileExists(std::string File); +APT_PUBLIC bool RealFileExists(std::string File); +APT_PUBLIC bool DirectoryExists(std::string const &Path); +APT_PUBLIC bool CreateDirectory(std::string const &Parent, std::string const &Path); +APT_PUBLIC time_t GetModificationTime(std::string const &Path); +APT_PUBLIC bool Rename(std::string From, std::string To); + +APT_PUBLIC std::string GetTempDir(); +APT_PUBLIC std::string GetTempDir(std::string const &User); +APT_PUBLIC FileFd* GetTempFile(std::string const &Prefix = "", + bool ImmediateUnlink = true, + FileFd * const TmpFd = NULL); + +// FIXME: GetTempFile should always return a buffered file +APT_HIDDEN FileFd* GetTempFile(std::string const &Prefix, + bool ImmediateUnlink , + FileFd * const TmpFd, + bool Buffered); + +/** \brief Ensure the existence of the given Path + * + * \param Parent directory of the Path directory - a trailing + * /apt/ will be removed before CreateDirectory call. + * \param Path which should exist after (successful) call + */ +APT_PUBLIC bool CreateAPTDirectoryIfNeeded(std::string const &Parent, std::string const &Path); + +APT_PUBLIC std::vector<std::string> GetListOfFilesInDir(std::string const &Dir, std::string const &Ext, + bool const &SortList, bool const &AllowNoExt=false); +APT_PUBLIC std::vector<std::string> GetListOfFilesInDir(std::string const &Dir, std::vector<std::string> const &Ext, + bool const &SortList); +APT_PUBLIC std::vector<std::string> GetListOfFilesInDir(std::string const &Dir, bool SortList); +APT_PUBLIC std::string SafeGetCWD(); +APT_PUBLIC void SetCloseExec(int Fd,bool Close); +APT_PUBLIC void SetNonBlock(int Fd,bool Block); +APT_PUBLIC bool WaitFd(int Fd,bool write = false,unsigned long timeout = 0); +APT_PUBLIC pid_t ExecFork(); +APT_PUBLIC pid_t ExecFork(std::set<int> keep_fds); +APT_PUBLIC void MergeKeepFdsFromConfiguration(std::set<int> &keep_fds); +APT_PUBLIC bool ExecWait(pid_t Pid,const char *Name,bool Reap = false); + +// check if the given file starts with a PGP cleartext signature +APT_PUBLIC bool StartsWithGPGClearTextSignature(std::string const &FileName); + +/** change file attributes to requested known good values + * + * The method skips the user:group setting if not root. + * + * @param requester is printed as functionname in error cases + * @param file is the file to be modified + * @param user is the (new) owner of the file, e.g. _apt + * @param group is the (new) group owning the file, e.g. root + * @param mode is the access mode of the file, e.g. 0644 + */ +APT_PUBLIC bool ChangeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode); + +/** + * \brief Drop privileges + * + * Drop the privileges to the user _apt (or the one specified in + * APT::Sandbox::User). This does not set the supplementary group + * ids up correctly, it only uses the default group. Also prevent + * the process from gaining any new privileges afterwards, at least + * on Linux. + * + * \return true on success, false on failure with _error set + */ +APT_PUBLIC bool DropPrivileges(); + +// File string manipulators +APT_PUBLIC std::string flNotDir(std::string File); +APT_PUBLIC std::string flNotFile(std::string File); +APT_PUBLIC std::string flNoLink(std::string File); +APT_PUBLIC std::string flExtension(std::string File); +APT_PUBLIC std::string flCombine(std::string Dir,std::string File); + +/** \brief Takes a file path and returns the absolute path + */ +APT_PUBLIC std::string flAbsPath(std::string File); +/** \brief removes superfluous /./ and // from path */ +APT_HIDDEN std::string flNormalize(std::string file); + +// simple c++ glob +APT_PUBLIC std::vector<std::string> Glob(std::string const &pattern, int flags=0); + +/** \brief Popen() implementation that execv() instead of using a shell + * + * \param Args the execv style command to run + * \param FileFd is a reference to the FileFd to use for input or output + * \param Child a reference to the integer that stores the child pid + * Note that you must call ExecWait() or similar to cleanup + * \param Mode is either FileFd::ReadOnly or FileFd::WriteOnly + * \param CaptureStderr True if we should capture stderr in addition to stdout. + * (default: True). + * \param Sandbox True if this should run sandboxed + * \return true on success, false on failure with _error set + */ +APT_PUBLIC bool Popen(const char *Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode, bool CaptureStderr = true, bool Sandbox = false); + +APT_HIDDEN bool OpenConfigurationFileFd(std::string const &File, FileFd &Fd); + +APT_HIDDEN int Inhibit(const char *what, const char *who, const char *why, const char *mode); + +#endif diff --git a/apt-pkg/contrib/gpgv.cc b/apt-pkg/contrib/gpgv.cc new file mode 100644 index 0000000..3368ece --- /dev/null +++ b/apt-pkg/contrib/gpgv.cc @@ -0,0 +1,568 @@ +// -*- mode: cpp; mode: fold -*- +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/gpgv.h> +#include <apt-pkg/strutl.h> + +#include <errno.h> +#include <fcntl.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/wait.h> +#include <unistd.h> + +#include <algorithm> +#include <fstream> +#include <iostream> +#include <memory> +#include <sstream> +#include <string> +#include <vector> + +#include <apti18n.h> + /*}}}*/ + +// syntactic sugar to wrap a raw pointer with a custom deleter in a std::unique_ptr +static std::unique_ptr<char, decltype(&free)> make_unique_char(void *const str = nullptr) +{ + return {static_cast<char *>(str), &free}; +} +static std::unique_ptr<FILE, decltype(&fclose)> make_unique_FILE(std::string const &filename, char const *const mode) +{ + return {fopen(filename.c_str(), mode), &fclose}; +} + +class LineBuffer /*{{{*/ +{ + char *buffer = nullptr; + size_t buffer_size = 0; + int line_length = 0; + // a "normal" find_last_not_of returns npos if not found + int find_last_not_of_length(APT::StringView const bad) const + { + for (int result = line_length - 1; result >= 0; --result) + if (bad.find(buffer[result]) == APT::StringView::npos) + return result + 1; + return 0; + } + + public: + bool empty() const noexcept { return view().empty(); } + APT::StringView view() const noexcept { return {buffer, static_cast<size_t>(line_length)}; } + bool starts_with(APT::StringView const start) const { return view().substr(0, start.size()) == start; } + + bool writeTo(FileFd *const to, size_t offset = 0) const + { + if (to == nullptr) + return true; + return to->Write(buffer + offset, line_length - offset); + } + bool writeLineTo(FileFd *const to) const + { + if (to == nullptr) + return true; + buffer[line_length] = '\n'; + bool const result = to->Write(buffer, line_length + 1); + buffer[line_length] = '\0'; + return result; + } + bool writeNewLineIf(FileFd *const to, bool const condition) const + { + if (not condition || to == nullptr) + return true; + return to->Write("\n", 1); + } + + bool readFrom(FILE *stream, std::string const &InFile, bool acceptEoF = false) + { + errno = 0; + line_length = getline(&buffer, &buffer_size, stream); + if (errno != 0) + return _error->Errno("getline", "Could not read from %s", InFile.c_str()); + if (line_length == -1) + { + if (acceptEoF) + return false; + return _error->Error("Splitting of clearsigned file %s failed as it doesn't contain all expected parts", InFile.c_str()); + } + // a) remove newline characters, so we can work consistently with lines + line_length = find_last_not_of_length("\n\r"); + // b) remove trailing whitespaces as defined by rfc4880 §7.1 + line_length = find_last_not_of_length(" \t"); + buffer[line_length] = '\0'; + return true; + } + + ~LineBuffer() { free(buffer); } +}; +static bool operator==(LineBuffer const &buf, APT::StringView const exp) noexcept +{ + return buf.view() == exp; +} +static bool operator!=(LineBuffer const &buf, APT::StringView const exp) noexcept +{ + return buf.view() != exp; +} + /*}}}*/ +// ExecGPGV - returns the command needed for verify /*{{{*/ +// --------------------------------------------------------------------- +/* Generating the commandline for calling gpg is somehow complicated as + we need to add multiple keyrings and user supplied options. + Also, as gpg has no options to enforce a certain reduced style of + clear-signed files (=the complete content of the file is signed and + the content isn't encoded) we do a divide and conquer approach here + and split up the clear-signed file in message and signature for gpg. + And as a cherry on the cake, we use our apt-key wrapper to do part + of the lifting in regards to merging keyrings. Fun for the whole family. +*/ +static void APT_PRINTF(4) apt_error(std::ostream &outterm, int const statusfd, int fd[2], const char *format, ...) +{ + std::ostringstream outstr; + std::ostream &out = (statusfd == -1) ? outterm : outstr; + va_list args; + ssize_t size = 400; + while (true) { + bool ret; + va_start(args,format); + ret = iovprintf(out, format, args, size); + va_end(args); + if (ret) + break; + } + if (statusfd != -1) + { + auto const errtag = "[APTKEY:] ERROR "; + outstr << '\n'; + auto const errtext = outstr.str(); + if (not FileFd::Write(fd[1], errtag, strlen(errtag)) || + not FileFd::Write(fd[1], errtext.data(), errtext.size())) + outterm << errtext << std::flush; + } +} +void ExecGPGV(std::string const &File, std::string const &FileGPG, + int const &statusfd, int fd[2], std::string const &key) +{ + #define EINTERNAL 111 + std::string const aptkey = _config->Find("Dir::Bin::apt-key", CMAKE_INSTALL_FULL_BINDIR "/apt-key"); + + bool const Debug = _config->FindB("Debug::Acquire::gpgv", false); + struct exiter { + std::vector<const char *> files; + void operator ()(int code) APT_NORETURN { + std::for_each(files.begin(), files.end(), unlink); + exit(code); + } + } local_exit; + + + std::vector<const char *> Args; + Args.reserve(10); + + Args.push_back(aptkey.c_str()); + Args.push_back("--quiet"); + Args.push_back("--readonly"); + auto const keysFileFpr = VectorizeString(key, ','); + for (auto const &k: keysFileFpr) + { + if (unlikely(k.empty())) + continue; + if (k[0] == '/') + { + Args.push_back("--keyring"); + Args.push_back(k.c_str()); + } + else + { + Args.push_back("--keyid"); + Args.push_back(k.c_str()); + } + } + Args.push_back("verify"); + + char statusfdstr[10]; + if (statusfd != -1) + { + Args.push_back("--status-fd"); + snprintf(statusfdstr, sizeof(statusfdstr), "%i", statusfd); + Args.push_back(statusfdstr); + } + + Configuration::Item const *Opts; + Opts = _config->Tree("Acquire::gpgv::Options"); + if (Opts != 0) + { + Opts = Opts->Child; + for (; Opts != 0; Opts = Opts->Next) + { + if (Opts->Value.empty()) + continue; + Args.push_back(Opts->Value.c_str()); + } + } + + enum { DETACHED, CLEARSIGNED } releaseSignature = (FileGPG != File) ? DETACHED : CLEARSIGNED; + auto sig = make_unique_char(); + auto data = make_unique_char(); + auto conf = make_unique_char(); + + // Dump the configuration so apt-key picks up the correct Dir values + { + { + std::string tmpfile; + strprintf(tmpfile, "%s/apt.conf.XXXXXX", GetTempDir().c_str()); + conf.reset(strdup(tmpfile.c_str())); + } + if (conf == nullptr) { + apt_error(std::cerr, statusfd, fd, "Couldn't create tempfile names for passing config to apt-key"); + local_exit(EINTERNAL); + } + int confFd = mkstemp(conf.get()); + if (confFd == -1) { + apt_error(std::cerr, statusfd, fd, "Couldn't create temporary file %s for passing config to apt-key", conf.get()); + local_exit(EINTERNAL); + } + local_exit.files.push_back(conf.get()); + + std::ofstream confStream(conf.get()); + close(confFd); + _config->Dump(confStream); + confStream.close(); + setenv("APT_CONFIG", conf.get(), 1); + } + + // Tell apt-key not to emit warnings + setenv("APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE", "1", 1); + + if (releaseSignature == DETACHED) + { + auto detached = make_unique_FILE(FileGPG, "r"); + if (detached.get() == nullptr) + { + apt_error(std::cerr, statusfd, fd, "Detached signature file '%s' could not be opened", FileGPG.c_str()); + local_exit(EINTERNAL); + } + LineBuffer buf; + bool open_signature = false; + bool found_badcontent = false; + size_t found_signatures = 0; + while (buf.readFrom(detached.get(), FileGPG, true)) + { + if (open_signature) + { + if (buf == "-----END PGP SIGNATURE-----") + open_signature = false; + else if (buf.starts_with("-")) + { + // the used Radix-64 is not using dash for any value, so a valid line can't + // start with one. Header keys could, but no existent one does and seems unlikely. + // Instead it smells a lot like a header the parser didn't recognize. + apt_error(std::cerr, statusfd, fd, "Detached signature file '%s' contains unexpected line starting with a dash", FileGPG.c_str()); + local_exit(112); + } + } + else //if (not open_signature) + { + if (buf == "-----BEGIN PGP SIGNATURE-----") + { + open_signature = true; + ++found_signatures; + if (found_badcontent) + break; + } + else + { + found_badcontent = true; + if (found_signatures != 0) + break; + } + } + } + if (found_signatures == 0 && statusfd != -1) + { + auto const errtag = "[GNUPG:] NODATA\n"; + FileFd::Write(fd[1], errtag, strlen(errtag)); + // guess if this is a binary signature, we never officially supported them, + // but silently accepted them via passing them unchecked to gpgv + if (found_badcontent) + { + rewind(detached.get()); + auto ptag = fgetc(detached.get()); + // §4.2 says that the first bit is always set and gpg seems to generate + // only old format which is indicated by the second bit not set + if (ptag != EOF && (ptag & 0x80) != 0 && (ptag & 0x40) == 0) + { + apt_error(std::cerr, statusfd, fd, "Detached signature file '%s' is in unsupported binary format", FileGPG.c_str()); + local_exit(112); + } + } + // This is not an attack attempt but a file even gpgv would complain about + // likely the result of a paywall which is covered by the gpgv method + local_exit(113); + } + else if (found_badcontent) + { + apt_error(std::cerr, statusfd, fd, "Detached signature file '%s' contains lines not belonging to a signature", FileGPG.c_str()); + local_exit(112); + } + if (open_signature) + { + apt_error(std::cerr, statusfd, fd, "Detached signature file '%s' contains unclosed signatures", FileGPG.c_str()); + local_exit(112); + } + + Args.push_back(FileGPG.c_str()); + Args.push_back(File.c_str()); + } + else // clear-signed file + { + FileFd signature; + if (GetTempFile("apt.sig", false, &signature) == nullptr) + local_exit(EINTERNAL); + sig.reset(strdup(signature.Name().c_str())); + local_exit.files.push_back(sig.get()); + FileFd message; + if (GetTempFile("apt.data", false, &message) == nullptr) + local_exit(EINTERNAL); + data.reset(strdup(message.Name().c_str())); + local_exit.files.push_back(data.get()); + + if (signature.Failed() || message.Failed() || + not SplitClearSignedFile(File, &message, nullptr, &signature)) + { + apt_error(std::cerr, statusfd, fd, "Splitting up %s into data and signature failed", File.c_str()); + local_exit(112); + } + Args.push_back(sig.get()); + Args.push_back(data.get()); + } + + Args.push_back(NULL); + + if (Debug) + { + std::clog << "Preparing to exec: "; + for (std::vector<const char *>::const_iterator a = Args.begin(); *a != NULL; ++a) + std::clog << " " << *a; + std::clog << std::endl; + } + + if (statusfd != -1) + { + int const nullfd = open("/dev/null", O_WRONLY); + close(fd[0]); + // Redirect output to /dev/null; we read from the status fd + if (statusfd != STDOUT_FILENO) + dup2(nullfd, STDOUT_FILENO); + if (statusfd != STDERR_FILENO) + dup2(nullfd, STDERR_FILENO); + // Redirect the pipe to the status fd (3) + dup2(fd[1], statusfd); + + putenv((char *)"LANG="); + putenv((char *)"LC_ALL="); + putenv((char *)"LC_MESSAGES="); + } + + + // We have created tempfiles we have to clean up + // and we do an additional check, so fork yet another time … + pid_t pid = ExecFork(); + if(pid < 0) { + apt_error(std::cerr, statusfd, fd, "Fork failed for %s to check %s", Args[0], File.c_str()); + local_exit(EINTERNAL); + } + if(pid == 0) + { + if (statusfd != -1) + dup2(fd[1], statusfd); + execvp(Args[0], (char **) &Args[0]); + apt_error(std::cerr, statusfd, fd, "Couldn't execute %s to check %s", Args[0], File.c_str()); + local_exit(EINTERNAL); + } + + // Wait and collect the error code - taken from WaitPid as we need the exact Status + int Status; + while (waitpid(pid,&Status,0) != pid) + { + if (errno == EINTR) + continue; + apt_error(std::cerr, statusfd, fd, _("Waited for %s but it wasn't there"), "apt-key"); + local_exit(EINTERNAL); + } + + // check if it exit'ed normally … + if (not WIFEXITED(Status)) + { + apt_error(std::cerr, statusfd, fd, _("Sub-process %s exited unexpectedly"), "apt-key"); + local_exit(EINTERNAL); + } + + // … and with a good exit code + if (WEXITSTATUS(Status) != 0) + { + // we forward the statuscode, so don't generate a message on the fd in this case + apt_error(std::cerr, -1, fd, _("Sub-process %s returned an error code (%u)"), "apt-key", WEXITSTATUS(Status)); + local_exit(WEXITSTATUS(Status)); + } + + // everything fine + local_exit(0); +} + /*}}}*/ +// SplitClearSignedFile - split message into data/signature /*{{{*/ +bool SplitClearSignedFile(std::string const &InFile, FileFd * const ContentFile, + std::vector<std::string> * const ContentHeader, FileFd * const SignatureFile) +{ + auto in = make_unique_FILE(InFile, "r"); + if (in.get() == nullptr) + return _error->Errno("fopen", "can not open %s", InFile.c_str()); + + struct ScopedErrors + { + ScopedErrors() { _error->PushToStack(); } + ~ScopedErrors() { _error->MergeWithStack(); } + } scoped; + LineBuffer buf; + + // start of the message + if (not buf.readFrom(in.get(), InFile)) + return false; // empty or read error + if (buf != "-----BEGIN PGP SIGNED MESSAGE-----") + { + // this might be an unsigned file we don't want to report errors for, + // but still finish unsuccessful none the less. + while (buf.readFrom(in.get(), InFile, true)) + if (buf == "-----BEGIN PGP SIGNED MESSAGE-----") + return _error->Error("Clearsigned file '%s' does not start with a signed message block.", InFile.c_str()); + + return false; + } + + // save "Hash" Armor Headers + while (true) + { + if (not buf.readFrom(in.get(), InFile)) + return false; + if (buf.empty()) + break; // empty line ends the Armor Headers + if (buf.starts_with("-")) + // § 6.2 says unknown keys should be reported to the user. We don't go that far, + // but we assume that there will never be a header key starting with a dash + return _error->Error("Clearsigned file '%s' contains unexpected line starting with a dash (%s)", InFile.c_str(), "armor"); + if (ContentHeader != nullptr && buf.starts_with("Hash: ")) + ContentHeader->push_back(buf.view().to_string()); + } + + // the message itself + bool first_line = true; + while (true) + { + if (not buf.readFrom(in.get(), InFile)) + return false; + + if (buf.starts_with("-")) + { + if (buf == "-----BEGIN PGP SIGNATURE-----") + { + if (not buf.writeLineTo(SignatureFile)) + return false; + break; + } + else if (buf.starts_with("- ")) + { + // we don't have any fields which need to be dash-escaped, + // but implementations are free to escape all lines … + if (not buf.writeNewLineIf(ContentFile, not first_line) || not buf.writeTo(ContentFile, 2)) + return false; + } + else + // § 7.1 says a client should warn, but we don't really work with files which + // should contain lines starting with a dash, so it is a lot more likely that + // this is an attempt to trick our parser vs. gpgv parser into ignoring a header + return _error->Error("Clearsigned file '%s' contains unexpected line starting with a dash (%s)", InFile.c_str(), "msg"); + } + else if (not buf.writeNewLineIf(ContentFile, not first_line) || not buf.writeTo(ContentFile)) + return false; + first_line = false; + } + + // collect all signatures + bool open_signature = true; + while (true) + { + if (not buf.readFrom(in.get(), InFile, true)) + break; + + if (open_signature) + { + if (buf == "-----END PGP SIGNATURE-----") + open_signature = false; + else if (buf.starts_with("-")) + // the used Radix-64 is not using dash for any value, so a valid line can't + // start with one. Header keys could, but no existent one does and seems unlikely. + // Instead it smells a lot like a header the parser didn't recognize. + return _error->Error("Clearsigned file '%s' contains unexpected line starting with a dash (%s)", InFile.c_str(), "sig"); + } + else //if (not open_signature) + { + if (buf == "-----BEGIN PGP SIGNATURE-----") + open_signature = true; + else + return _error->Error("Clearsigned file '%s' contains unsigned lines.", InFile.c_str()); + } + + if (not buf.writeLineTo(SignatureFile)) + return false; + } + if (open_signature) + return _error->Error("Signature in file %s wasn't closed", InFile.c_str()); + + // Flush the files + if (SignatureFile != nullptr) + SignatureFile->Flush(); + if (ContentFile != nullptr) + ContentFile->Flush(); + + // Catch-all for "unhandled" read/sync errors + if (_error->PendingError()) + return false; + return true; +} + /*}}}*/ +bool OpenMaybeClearSignedFile(std::string const &ClearSignedFileName, FileFd &MessageFile) /*{{{*/ +{ + // Buffered file + if (GetTempFile("clearsigned.message", true, &MessageFile, true) == nullptr) + return false; + if (MessageFile.Failed()) + return _error->Error("Couldn't open temporary file to work with %s", ClearSignedFileName.c_str()); + + _error->PushToStack(); + bool const splitDone = SplitClearSignedFile(ClearSignedFileName, &MessageFile, NULL, NULL); + bool const errorDone = _error->PendingError(); + _error->MergeWithStack(); + if (not splitDone) + { + MessageFile.Close(); + + if (errorDone) + return false; + + // we deal with an unsigned file + MessageFile.Open(ClearSignedFileName, FileFd::ReadOnly); + } + else // clear-signed + { + if (not MessageFile.Seek(0)) + return _error->Errno("lseek", "Unable to seek back in message for file %s", ClearSignedFileName.c_str()); + } + + return not MessageFile.Failed(); +} + /*}}}*/ diff --git a/apt-pkg/contrib/gpgv.h b/apt-pkg/contrib/gpgv.h new file mode 100644 index 0000000..1cabed4 --- /dev/null +++ b/apt-pkg/contrib/gpgv.h @@ -0,0 +1,89 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Helpers to deal with gpgv better and more easily + + ##################################################################### */ + /*}}}*/ +#ifndef CONTRIB_GPGV_H +#define CONTRIB_GPGV_H + +#include <apt-pkg/macros.h> + +#include <string> +#include <vector> + + +class FileFd; + +/** \brief generates and run the command to verify a file with gpgv + * + * If File and FileSig specify the same file it is assumed that we + * deal with a clear-signed message. Note that the method will accept + * and validate files which include additional (unsigned) messages + * without complaining. Do NOT open files accepted by this method + * for reading. Use #OpenMaybeClearSignedFile to access the message + * instead to ensure you are only reading signed data. + * + * The method does not return, but has some notable exit-codes: + * 111 signals an internal error like the inability to execute gpgv, + * 112 indicates a clear-signed file which doesn't include a message, + * which can happen if APT is run while on a network requiring + * authentication before usage (e.g. in hotels) + * All other exit-codes are passed-through from gpgv. + * + * @param File is the message (unsigned or clear-signed) + * @param FileSig is the signature (detached or clear-signed) + * @param statusfd is the fd given to gpgv as --status-fd + * @param fd is used as a pipe for the standard output of gpgv + * @param key is the specific one to be used instead of using all + */ +APT_PUBLIC void ExecGPGV(std::string const &File, std::string const &FileSig, + int const &statusfd, int fd[2], std::string const &Key = "") APT_NORETURN; +inline APT_NORETURN void ExecGPGV(std::string const &File, std::string const &FileSig, + int const &statusfd = -1) { + int fd[2]; + ExecGPGV(File, FileSig, statusfd, fd); +} + +/** \brief Split an inline signature into message and signature + * + * Takes a clear-signed message and puts the first signed message + * in the content file and all signatures following it into the + * second. Unsigned messages, additional messages as well as + * whitespaces are discarded. The resulting files are suitable to + * be checked with gpgv. + * + * If a FileFd pointers is NULL it will not be used and the content + * which would have been written to it is silently discarded. + * + * The content of the split files is undefined if the splitting was + * unsuccessful. + * + * Note that trying to split an unsigned file will fail, but + * not generate an error message. + * + * @param InFile is the clear-signed file + * @param ContentFile is the FileFd the message will be written to + * @param ContentHeader is a list of all required Amored Headers for the message + * @param SignatureFile is the FileFd all signatures will be written to + * @return true if the splitting was successful, false otherwise + */ +APT_PUBLIC bool SplitClearSignedFile(std::string const &InFile, FileFd * const ContentFile, + std::vector<std::string> * const ContentHeader, FileFd * const SignatureFile); + +/** \brief open a file which might be clear-signed + * + * This method tries to extract the (signed) message of a file. + * If the file isn't signed it will just open the given filename. + * Otherwise the message is extracted to a temporary file which + * will be opened instead. + * + * @param ClearSignedFileName is the name of the file to open + * @param[out] MessageFile is the FileFd in which the file will be opened + * @return true if opening was successful, otherwise false + */ +APT_PUBLIC bool OpenMaybeClearSignedFile(std::string const &ClearSignedFileName, FileFd &MessageFile); + +#endif diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc new file mode 100644 index 0000000..313b1d3 --- /dev/null +++ b/apt-pkg/contrib/hashes.cc @@ -0,0 +1,464 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Hashes - Simple wrapper around the hash functions + + This is just used to make building the methods simpler, this is the + only interface required.. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile-keys.h> +#include <apt-pkg/tagfile.h> + +#include <algorithm> +#include <iostream> +#include <string> +#include <assert.h> +#include <stddef.h> +#include <stdlib.h> +#include <unistd.h> + +#include <gcrypt.h> + /*}}}*/ + +static const constexpr struct HashAlgo +{ + const char *name; + int gcryAlgo; + Hashes::SupportedHashes ourAlgo; +} Algorithms[] = { + {"MD5Sum", GCRY_MD_MD5, Hashes::MD5SUM}, + {"SHA1", GCRY_MD_SHA1, Hashes::SHA1SUM}, + {"SHA256", GCRY_MD_SHA256, Hashes::SHA256SUM}, + {"SHA512", GCRY_MD_SHA512, Hashes::SHA512SUM}, +}; + +const char * HashString::_SupportedHashes[] = +{ + "SHA512", "SHA256", "SHA1", "MD5Sum", "Checksum-FileSize", NULL +}; +std::vector<HashString::HashSupportInfo> HashString::SupportedHashesInfo() +{ + return {{ + { "SHA512", pkgTagSection::Key::SHA512,"Checksums-Sha512", pkgTagSection::Key::Checksums_Sha512}, + { "SHA256", pkgTagSection::Key::SHA256, "Checksums-Sha256", pkgTagSection::Key::Checksums_Sha256}, + { "SHA1", pkgTagSection::Key::SHA1, "Checksums-Sha1", pkgTagSection::Key::Checksums_Sha1 }, + { "MD5Sum", pkgTagSection::Key::MD5sum, "Files", pkgTagSection::Key::Files }, + }}; +} + +HashString::HashString() +{ +} + +HashString::HashString(std::string Type, std::string Hash) : Type(Type), Hash(Hash) +{ +} + +HashString::HashString(std::string StringedHash) /*{{{*/ +{ + if (StringedHash.find(":") == std::string::npos) + { + // legacy: md5sum without "MD5Sum:" prefix + if (StringedHash.size() == 32) + { + Type = "MD5Sum"; + Hash = StringedHash; + } + if(_config->FindB("Debug::Hashes",false) == true) + std::clog << "HashString(string): invalid StringedHash " << StringedHash << std::endl; + return; + } + std::string::size_type pos = StringedHash.find(":"); + Type = StringedHash.substr(0,pos); + Hash = StringedHash.substr(pos+1, StringedHash.size() - pos); + + if(_config->FindB("Debug::Hashes",false) == true) + std::clog << "HashString(string): " << Type << " : " << Hash << std::endl; +} + /*}}}*/ +bool HashString::VerifyFile(std::string filename) const /*{{{*/ +{ + std::string fileHash = GetHashForFile(filename); + + if(_config->FindB("Debug::Hashes",false) == true) + std::clog << "HashString::VerifyFile: got: " << fileHash << " expected: " << toStr() << std::endl; + + return (fileHash == Hash); +} + /*}}}*/ +bool HashString::FromFile(std::string filename) /*{{{*/ +{ + // pick the strongest hash + if (Type == "") + Type = _SupportedHashes[0]; + + Hash = GetHashForFile(filename); + return true; +} + /*}}}*/ +std::string HashString::GetHashForFile(std::string filename) const /*{{{*/ +{ + std::string fileHash; + + FileFd Fd(filename, FileFd::ReadOnly); + if(strcasecmp(Type.c_str(), "MD5Sum") == 0) + { + Hashes MD5(Hashes::MD5SUM); + MD5.AddFD(Fd); + fileHash = MD5.GetHashString(Hashes::MD5SUM).Hash; + } + else if (strcasecmp(Type.c_str(), "SHA1") == 0) + { + Hashes SHA1(Hashes::SHA1SUM); + SHA1.AddFD(Fd); + fileHash = SHA1.GetHashString(Hashes::SHA1SUM).Hash; + } + else if (strcasecmp(Type.c_str(), "SHA256") == 0) + { + Hashes SHA256(Hashes::SHA256SUM); + SHA256.AddFD(Fd); + fileHash = SHA256.GetHashString(Hashes::SHA256SUM).Hash; + } + else if (strcasecmp(Type.c_str(), "SHA512") == 0) + { + Hashes SHA512(Hashes::SHA512SUM); + SHA512.AddFD(Fd); + fileHash = SHA512.GetHashString(Hashes::SHA512SUM).Hash; + } + else if (strcasecmp(Type.c_str(), "Checksum-FileSize") == 0) + strprintf(fileHash, "%llu", Fd.FileSize()); + Fd.Close(); + + return fileHash; +} + /*}}}*/ +const char** HashString::SupportedHashes() /*{{{*/ +{ + return _SupportedHashes; +} + /*}}}*/ +APT_PURE bool HashString::empty() const /*{{{*/ +{ + return (Type.empty() || Hash.empty()); +} + /*}}}*/ + +APT_PURE static bool IsConfigured(const char *name, const char *what) +{ + std::string option; + strprintf(option, "APT::Hashes::%s::%s", name, what); + return _config->FindB(option, false); +} + +APT_PURE bool HashString::usable() const /*{{{*/ +{ + return ( + (Type != "Checksum-FileSize") && + (Type != "MD5Sum") && + (Type != "SHA1") && + !IsConfigured(Type.c_str(), "Untrusted") + ); +} + /*}}}*/ +std::string HashString::toStr() const /*{{{*/ +{ + return Type + ":" + Hash; +} + /*}}}*/ +APT_PURE bool HashString::operator==(HashString const &other) const /*{{{*/ +{ + return (strcasecmp(Type.c_str(), other.Type.c_str()) == 0 && Hash == other.Hash); +} +APT_PURE bool HashString::operator!=(HashString const &other) const +{ + return !(*this == other); +} + /*}}}*/ + +bool HashStringList::usable() const /*{{{*/ +{ + if (empty() == true) + return false; + std::string const forcedType = _config->Find("Acquire::ForceHash", ""); + if (forcedType.empty() == true) + { + // See if there is at least one usable hash + return std::any_of(list.begin(), list.end(), [](auto const &hs) { return hs.usable(); }); + } + return find(forcedType) != NULL; +} + /*}}}*/ +HashString const * HashStringList::find(char const * const type) const /*{{{*/ +{ + if (type == NULL || type[0] == '\0') + { + std::string const forcedType = _config->Find("Acquire::ForceHash", ""); + if (forcedType.empty() == false) + return find(forcedType.c_str()); + for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t) + for (std::vector<HashString>::const_iterator hs = list.begin(); hs != list.end(); ++hs) + if (strcasecmp(hs->HashType().c_str(), *t) == 0) + return &*hs; + return NULL; + } + for (std::vector<HashString>::const_iterator hs = list.begin(); hs != list.end(); ++hs) + if (strcasecmp(hs->HashType().c_str(), type) == 0) + return &*hs; + return NULL; +} + /*}}}*/ +unsigned long long HashStringList::FileSize() const /*{{{*/ +{ + HashString const * const hsf = find("Checksum-FileSize"); + if (hsf == NULL) + return 0; + std::string const hv = hsf->HashValue(); + return strtoull(hv.c_str(), NULL, 10); +} + /*}}}*/ +bool HashStringList::FileSize(unsigned long long const Size) /*{{{*/ +{ + return push_back(HashString("Checksum-FileSize", std::to_string(Size))); +} + /*}}}*/ +bool HashStringList::supported(char const * const type) /*{{{*/ +{ + for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t) + if (strcasecmp(*t, type) == 0) + return true; + return false; +} + /*}}}*/ +bool HashStringList::push_back(const HashString &hashString) /*{{{*/ +{ + if (hashString.HashType().empty() == true || + hashString.HashValue().empty() == true || + supported(hashString.HashType().c_str()) == false) + return false; + + // ensure that each type is added only once + HashString const * const hs = find(hashString.HashType().c_str()); + if (hs != NULL) + return *hs == hashString; + + list.push_back(hashString); + return true; +} + /*}}}*/ +bool HashStringList::VerifyFile(std::string filename) const /*{{{*/ +{ + if (usable() == false) + return false; + + Hashes hashes(*this); + FileFd file(filename, FileFd::ReadOnly); + HashString const * const hsf = find("Checksum-FileSize"); + if (hsf != NULL) + { + std::string fileSize; + strprintf(fileSize, "%llu", file.FileSize()); + if (hsf->HashValue() != fileSize) + return false; + } + hashes.AddFD(file); + HashStringList const hsl = hashes.GetHashStringList(); + return hsl == *this; +} + /*}}}*/ +bool HashStringList::operator==(HashStringList const &other) const /*{{{*/ +{ + std::string const forcedType = _config->Find("Acquire::ForceHash", ""); + if (forcedType.empty() == false) + { + HashString const * const hs = find(forcedType); + HashString const * const ohs = other.find(forcedType); + if (hs == NULL || ohs == NULL) + return false; + return *hs == *ohs; + } + short matches = 0; + for (const_iterator hs = begin(); hs != end(); ++hs) + { + HashString const * const ohs = other.find(hs->HashType()); + if (ohs == NULL) + continue; + if (*hs != *ohs) + return false; + ++matches; + } + if (matches == 0) + return false; + return true; +} +bool HashStringList::operator!=(HashStringList const &other) const +{ + return !(*this == other); +} + /*}}}*/ + +// PrivateHashes /*{{{*/ +class PrivateHashes { +public: + unsigned long long FileSize; + gcry_md_hd_t hd; + + void maybeInit() + { + + // Yikes, we got to initialize libgcrypt, or we get warnings. But we + // abstract away libgcrypt in Hashes from our users - they are not + // supposed to know what the hashing backend is, so we can't force + // them to init themselves as libgcrypt folks want us to. So this + // only leaves us with this option... + if (!gcry_control(GCRYCTL_INITIALIZATION_FINISHED_P)) + { + if (!gcry_check_version(nullptr)) + { + fprintf(stderr, "libgcrypt is too old (need %s, have %s)\n", + "nullptr", gcry_check_version(NULL)); + exit(2); + } + + gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0); + } + } + + explicit PrivateHashes(unsigned int const CalcHashes) : FileSize(0) + { + maybeInit(); + gcry_md_open(&hd, 0, 0); + for (auto & Algo : Algorithms) + { + if ((CalcHashes & Algo.ourAlgo) == Algo.ourAlgo) + gcry_md_enable(hd, Algo.gcryAlgo); + } + } + + explicit PrivateHashes(HashStringList const &Hashes) : FileSize(0) { + maybeInit(); + gcry_md_open(&hd, 0, 0); + for (auto & Algo : Algorithms) + { + if (not Hashes.usable() || Hashes.find(Algo.name) != NULL) + gcry_md_enable(hd, Algo.gcryAlgo); + } + } + ~PrivateHashes() + { + gcry_md_close(hd); + } +}; + /*}}}*/ +// Hashes::Add* - Add the contents of data or FD /*{{{*/ +bool Hashes::Add(const unsigned char * const Data, unsigned long long const Size) +{ + if (Size != 0) + { + gcry_md_write(d->hd, Data, Size); + d->FileSize += Size; + } + return true; +} +bool Hashes::AddFD(int const Fd,unsigned long long Size) +{ + unsigned char Buf[APT_BUFFER_SIZE]; + bool const ToEOF = (Size == UntilEOF); + while (Size != 0 || ToEOF) + { + decltype(Size) n = sizeof(Buf); + if (!ToEOF) n = std::min(Size, n); + ssize_t const Res = read(Fd,Buf,n); + if (Res < 0 || (!ToEOF && Res != (ssize_t) n)) // error, or short read + return false; + if (ToEOF && Res == 0) // EOF + break; + Size -= Res; + if (Add(Buf, Res) == false) + return false; + } + return true; +} +bool Hashes::AddFD(FileFd &Fd,unsigned long long Size) +{ + unsigned char Buf[APT_BUFFER_SIZE]; + bool const ToEOF = (Size == 0); + while (Size != 0 || ToEOF) + { + decltype(Size) n = sizeof(Buf); + if (!ToEOF) n = std::min(Size, n); + decltype(Size) a = 0; + if (Fd.Read(Buf, n, &a) == false) // error + return false; + if (ToEOF == false) + { + if (a != n) // short read + return false; + } + else if (a == 0) // EOF + break; + Size -= a; + if (Add(Buf, a) == false) + return false; + } + return true; +} + /*}}}*/ + +static APT_PURE std::string HexDigest(gcry_md_hd_t hd, int algo) +{ + char Conv[16] = + {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', + 'c', 'd', 'e', 'f'}; + + auto Size = gcry_md_get_algo_dlen(algo); + assert(Size <= 512/8); + char Result[((Size)*2) + 1]; + Result[(Size)*2] = 0; + + auto Sum = gcry_md_read(hd, algo); + + // Convert each char into two letters + size_t J = 0; + size_t I = 0; + for (; I != (Size)*2; J++, I += 2) + { + Result[I] = Conv[Sum[J] >> 4]; + Result[I + 1] = Conv[Sum[J] & 0xF]; + } + return std::string(Result); +}; + +HashStringList Hashes::GetHashStringList() +{ + HashStringList hashes; + for (auto & Algo : Algorithms) + if (gcry_md_is_enabled(d->hd, Algo.gcryAlgo)) + hashes.push_back(HashString(Algo.name, HexDigest(d->hd, Algo.gcryAlgo))); + hashes.FileSize(d->FileSize); + + return hashes; +} + +HashString Hashes::GetHashString(SupportedHashes hash) +{ + for (auto & Algo : Algorithms) + if (hash == Algo.ourAlgo) + return HashString(Algo.name, HexDigest(d->hd, Algo.gcryAlgo)); + + abort(); +} +Hashes::Hashes() : d(new PrivateHashes(~0)) { } +Hashes::Hashes(unsigned int const Hashes) : d(new PrivateHashes(Hashes)) {} +Hashes::Hashes(HashStringList const &Hashes) : d(new PrivateHashes(Hashes)) {} +Hashes::~Hashes() { delete d; } diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h new file mode 100644 index 0000000..e259b4e --- /dev/null +++ b/apt-pkg/contrib/hashes.h @@ -0,0 +1,222 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Hashes - Simple wrapper around the hash functions + + This is just used to make building the methods simpler, this is the + only interface required.. + + ##################################################################### */ + /*}}}*/ +#ifndef APTPKG_HASHES_H +#define APTPKG_HASHES_H + +#include <apt-pkg/macros.h> + +#ifdef APT_COMPILING_APT +#include <apt-pkg/string_view.h> +#include <apt-pkg/tagfile-keys.h> +#endif + +#include <cstring> +#include <string> +#include <vector> + + + +class FileFd; + +// helper class that contains hash function name +// and hash +class APT_PUBLIC HashString +{ + protected: + std::string Type; + std::string Hash; + static const char * _SupportedHashes[10]; + + // internal helper + std::string GetHashForFile(std::string filename) const; + + public: + HashString(std::string Type, std::string Hash); + explicit HashString(std::string StringedHashString); // init from str as "type:hash" + HashString(); + + // get hash type used + std::string HashType() const { return Type; }; + std::string HashValue() const { return Hash; }; + + // verify the given filename against the currently loaded hash + bool VerifyFile(std::string filename) const; + + // generate a hash string from the given filename + bool FromFile(std::string filename); + + + // helper + std::string toStr() const; // convert to str as "type:hash" + bool empty() const; + bool usable() const; + bool operator==(HashString const &other) const; + bool operator!=(HashString const &other) const; + + // return the list of hashes we support + static APT_PURE const char** SupportedHashes(); +#ifdef APT_COMPILING_APT + struct APT_HIDDEN HashSupportInfo { + APT::StringView name; + pkgTagSection::Key namekey; + APT::StringView chksumsname; + pkgTagSection::Key chksumskey; + }; + APT_HIDDEN static std::vector<HashSupportInfo> SupportedHashesInfo(); +#endif +}; + +class APT_PUBLIC HashStringList +{ + public: + /** find best hash if no specific one is requested + * + * @param type of the checksum to return, can be \b NULL + * @return If type is \b NULL (or the empty string) it will + * return the 'best' hash; otherwise the hash which was + * specifically requested. If no hash is found \b NULL will be returned. + */ + HashString const * find(char const * const type) const; + HashString const * find(std::string const &type) const { return find(type.c_str()); } + + /** finds the filesize hash and returns it as number + * + * @return beware: if the size isn't known we return \b 0 here, + * just like we would do for an empty file. If that is a problem + * for you have to get the size manually out of the list. + */ + unsigned long long FileSize() const; + + /** sets the filesize hash + * + * @param Size of the file + * @return @see #push_back + */ + bool FileSize(unsigned long long const Size); + + /** check if the given hash type is supported + * + * @param type to check + * @return true if supported, otherwise false + */ + static APT_PURE bool supported(char const * const type); + /** add the given #HashString to the list + * + * @param hashString to add + * @return true if the hash is added because it is supported and + * not already a different hash of the same type included, otherwise false + */ + bool push_back(const HashString &hashString); + /** @return size of the list of HashStrings */ + size_t size() const { return list.size(); } + + /** verify file against all hashes in the list + * + * @param filename to verify + * @return true if the file matches the hashsum, otherwise false + */ + bool VerifyFile(std::string filename) const; + + /** is the list empty ? + * + * @return \b true if the list is empty, otherwise \b false + */ + bool empty() const { return list.empty(); } + + /** has the list at least one good entry + * + * similar to #empty, but handles forced hashes. + * + * @return if no hash is forced, same result as #empty, + * if one is forced \b true if this has is available, \b false otherwise + */ + bool usable() const; + + typedef std::vector<HashString>::const_iterator const_iterator; + + /** iterator to the first element */ + const_iterator begin() const { return list.begin(); } + + /** iterator to the end element */ + const_iterator end() const { return list.end(); } + + /** start fresh with a clear list */ + void clear() { list.clear(); } + + /** compare two HashStringList for similarity. + * + * Two lists are similar if at least one hashtype is in both lists + * and the hashsum matches. All hashes are checked by default, + * if one doesn't match false is returned regardless of how many + * matched before. If a hash is forced, only this hash is compared, + * all others are ignored. + */ + bool operator==(HashStringList const &other) const; + bool operator!=(HashStringList const &other) const; + + HashStringList() {} + + // simplifying API-compatibility constructors + explicit HashStringList(std::string const &hash) { + if (hash.empty() == false) + list.push_back(HashString(hash)); + } + explicit HashStringList(char const * const hash) { + if (hash != NULL && hash[0] != '\0') + list.push_back(HashString(hash)); + } + + private: + std::vector<HashString> list; +}; + +class PrivateHashes; +class APT_PUBLIC Hashes +{ + PrivateHashes * const d; + public: + static const int UntilEOF = 0; + + bool Add(const unsigned char * const Data, unsigned long long const Size) APT_NONNULL(2); + inline bool Add(const char * const Data) APT_NONNULL(2) + {return Add(reinterpret_cast<unsigned char const *>(Data),strlen(Data));}; + inline bool Add(const char *const Data, unsigned long long const Size) APT_NONNULL(2) + { + return Add(reinterpret_cast<unsigned char const *>(Data), Size); + }; + inline bool Add(const unsigned char * const Beg,const unsigned char * const End) APT_NONNULL(2,3) + {return Add(Beg,End-Beg);}; + + enum SupportedHashes { MD5SUM = (1 << 0), SHA1SUM = (1 << 1), SHA256SUM = (1 << 2), + SHA512SUM = (1 << 3) }; + bool AddFD(int const Fd,unsigned long long Size = 0); + bool AddFD(FileFd &Fd,unsigned long long Size = 0); + + HashStringList GetHashStringList(); + + /** Get a specific hash. It is an error to use a hash that was not hashes */ + HashString GetHashString(SupportedHashes hash); + + /** create a Hashes object to calculate all supported hashes + * + * If ALL is too much, you can limit which Hashes are calculated + * with the following other constructors which mention explicitly + * which hashes to generate. */ + Hashes(); + /** @param Hashes bitflag composed of #SupportedHashes */ + explicit Hashes(unsigned int const Hashes); + /** @param Hashes is a list of hashes */ + explicit Hashes(HashStringList const &Hashes); + virtual ~Hashes(); +}; + +#endif diff --git a/apt-pkg/contrib/header-is-private.h b/apt-pkg/contrib/header-is-private.h new file mode 100644 index 0000000..201a40c --- /dev/null +++ b/apt-pkg/contrib/header-is-private.h @@ -0,0 +1,3 @@ +#ifndef APT_COMPILING_APT +#error Internal header without ABI stability. Should only be used by apt itself! +#endif diff --git a/apt-pkg/contrib/macros.h b/apt-pkg/contrib/macros.h new file mode 100644 index 0000000..c08cd24 --- /dev/null +++ b/apt-pkg/contrib/macros.h @@ -0,0 +1,131 @@ +// -*- mode: cpp; mode: fold -*- +// SPDX-License-Identifier: GPL-2.0+ +// Description /*{{{*/ +/* ###################################################################### + + Macros Header - Various useful macro definitions + + This file had this historic note, but now includes further changes + under the GPL-2.0+: + + This source is placed in the Public Domain, do with it what you will + It was originally written by Brian C. White. + + ##################################################################### */ + /*}}}*/ +// Private header +#ifndef MACROS_H +#define MACROS_H + +/* Useful count macro, use on an array of things and it will return the + number of items in the array */ +#define APT_ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) + +// Flag Macros +#define FLAG(f) (1L << (f)) +#define SETFLAG(v,f) ((v) |= FLAG(f)) +#define CLRFLAG(v,f) ((v) &=~FLAG(f)) +#define CHKFLAG(v,f) ((v) & FLAG(f) ? true : false) + +#ifdef __GNUC__ +#define APT_GCC_VERSION (__GNUC__ << 8 | __GNUC_MINOR__) +#else +#define APT_GCC_VERSION 0 +#endif + +#ifdef APT_COMPILING_APT +/* likely() and unlikely() can be used to mark boolean expressions + as (not) likely true which will help the compiler to optimise */ +#if APT_GCC_VERSION >= 0x0300 + #define likely(x) __builtin_expect (!!(x), 1) + #define unlikely(x) __builtin_expect (!!(x), 0) +#else + #define likely(x) (x) + #define unlikely(x) (x) +#endif +#endif + +#if APT_GCC_VERSION >= 0x0300 + #define APT_DEPRECATED __attribute__ ((deprecated)) + #define APT_DEPRECATED_MSG(X) __attribute__ ((deprecated(X))) + // __attribute__((const)) is too dangerous for us, we end up using it wrongly + #define APT_PURE __attribute__((pure)) + #define APT_NORETURN __attribute__((noreturn)) + #define APT_PRINTF(n) __attribute__((format(printf, n, n + 1))) + #define APT_WEAK __attribute__((weak)); + #define APT_UNUSED __attribute__((unused)) +#else + #define APT_DEPRECATED + #define APT_DEPRECATED_MSG + #define APT_PURE + #define APT_NORETURN + #define APT_PRINTF(n) + #define APT_WEAK + #define APT_UNUSED +#endif + +#if APT_GCC_VERSION > 0x0302 + #define APT_NONNULL(...) __attribute__((nonnull(__VA_ARGS__))) + #define APT_MUSTCHECK __attribute__((warn_unused_result)) +#else + #define APT_NONNULL(...) + #define APT_MUSTCHECK +#endif + +#if APT_GCC_VERSION >= 0x0400 + #define APT_SENTINEL __attribute__((sentinel)) + #define APT_PUBLIC __attribute__ ((visibility ("default"))) + #define APT_HIDDEN __attribute__ ((visibility ("hidden"))) +#else + #define APT_SENTINEL + #define APT_PUBLIC + #define APT_HIDDEN +#endif + +// cold functions are unlikely() to be called +#if APT_GCC_VERSION >= 0x0403 + #define APT_COLD __attribute__ ((__cold__)) + #define APT_HOT __attribute__ ((__hot__)) +#else + #define APT_COLD + #define APT_HOT +#endif + + +#if __GNUC__ >= 4 + #define APT_IGNORE_DEPRECATED_PUSH \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") + #define APT_IGNORE_DEPRECATED_POP \ + _Pragma("GCC diagnostic pop") + /* gcc has various problems with this shortcut, so prefer the long form */ + #define APT_IGNORE_DEPRECATED(XXX) \ + APT_IGNORE_DEPRECATED_PUSH \ + XXX \ + APT_IGNORE_DEPRECATED_POP +#else + #define APT_IGNORE_DEPRECATED_PUSH + #define APT_IGNORE_DEPRECATED_POP + #define APT_IGNORE_DEPRECATED(XXX) XXX +#endif + +#if __cplusplus >= 201103L + #define APT_OVERRIDE override +#else + #define APT_OVERRIDE /* no c++11 standard */ +#endif + +// These lines are extracted by the makefiles and the buildsystem +// Increasing MAJOR or MINOR results in the need of recompiling all +// reverse-dependencies of libapt-pkg against the new SONAME. +// Non-ABI-Breaks should only increase RELEASE number. +// See also buildlib/libversion.mak +#define APT_PKG_MAJOR 6 +#define APT_PKG_MINOR 0 +#define APT_PKG_RELEASE 0 +#define APT_PKG_ABI ((APT_PKG_MAJOR * 100) + APT_PKG_MINOR) + +/* Should be a multiple of the common page size (4096) */ +static constexpr unsigned long long APT_BUFFER_SIZE = 64 * 1024; + +#endif diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc new file mode 100644 index 0000000..0568e1c --- /dev/null +++ b/apt-pkg/contrib/mmap.cc @@ -0,0 +1,504 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + MMap Class - Provides 'real' mmap or a faked mmap using read(). + + MMap cover class. + + Some broken versions of glibc2 (libc6) have a broken definition + of mmap that accepts a char * -- all other systems (and libc5) use + void *. We can't safely do anything here that would be portable, so + libc6 generates warnings -- which should be errors, g++ isn't properly + strict. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#define _DEFAULT_SOURCE +#include <config.h> + +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/mmap.h> + +#include <cstdint> +#include <cstring> +#include <string> +#include <errno.h> +#include <stdlib.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ + +// MMap::MMap - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +MMap::MMap(FileFd &F,unsigned long Flags) : Flags(Flags), iSize(0), + Base(nullptr), SyncToFd(nullptr) +{ + if ((Flags & NoImmMap) != NoImmMap) + Map(F); +} + /*}}}*/ +// MMap::MMap - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +MMap::MMap(unsigned long Flags) : Flags(Flags), iSize(0), + Base(nullptr), SyncToFd(nullptr) +{ +} + /*}}}*/ +// MMap::~MMap - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +MMap::~MMap() +{ + Close(); +} + /*}}}*/ +// MMap::Map - Perform the mapping /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool MMap::Map(FileFd &Fd) +{ + iSize = Fd.Size(); + + // Set the permissions. + int Prot = PROT_READ; + int Map = MAP_SHARED; + if ((Flags & ReadOnly) != ReadOnly) + Prot |= PROT_WRITE; + if ((Flags & Public) != Public) + Map = MAP_PRIVATE; + + if (iSize == 0) + return _error->Error(_("Can't mmap an empty file")); + + // We can't mmap compressed fd's directly, so we need to read it completely + if (Fd.IsCompressed() == true) + { + if ((Flags & ReadOnly) != ReadOnly) + return _error->Error("Compressed file %s can only be mapped readonly", Fd.Name().c_str()); + Base = malloc(iSize); + if (unlikely(Base == nullptr)) + return _error->Errno("MMap-compressed-malloc", _("Couldn't make mmap of %llu bytes"), iSize); + SyncToFd = new FileFd(); + if (Fd.Seek(0L) == false || Fd.Read(Base, iSize) == false) + return _error->Error("Compressed file %s can't be read into mmap", Fd.Name().c_str()); + return true; + } + + // Map it. + Base = (Flags & Fallback) ? MAP_FAILED : mmap(0,iSize,Prot,Map,Fd.Fd(),0); + if (Base == MAP_FAILED) + { + if (errno == ENODEV || errno == EINVAL || (Flags & Fallback)) + { + // The filesystem doesn't support this particular kind of mmap. + // So we allocate a buffer and read the whole file into it. + if ((Flags & ReadOnly) == ReadOnly) + { + // for readonly, we don't need sync, so make it simple + Base = malloc(iSize); + if (unlikely(Base == nullptr)) + return _error->Errno("MMap-malloc", _("Couldn't make mmap of %llu bytes"), iSize); + SyncToFd = new FileFd(); + return Fd.Read(Base, iSize); + } + // FIXME: Writing to compressed fd's ? + int const dupped_fd = dup(Fd.Fd()); + if (dupped_fd == -1) + return _error->Errno("mmap", _("Couldn't duplicate file descriptor %i"), Fd.Fd()); + + Base = calloc(iSize, 1); + if (unlikely(Base == nullptr)) + return _error->Errno("MMap-calloc", _("Couldn't make mmap of %llu bytes"), iSize); + SyncToFd = new FileFd (dupped_fd); + if (!SyncToFd->Seek(0L) || !SyncToFd->Read(Base, iSize)) + return false; + } + else + return _error->Errno("MMap-mmap", _("Couldn't make mmap of %llu bytes"), iSize); + } + + return true; +} + /*}}}*/ +// MMap::Close - Close the map /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool MMap::Close(bool DoSync) +{ + if ((Flags & UnMapped) == UnMapped || validData() == false || iSize == 0) + return true; + + if (DoSync == true) + Sync(); + + if (SyncToFd != NULL) + { + free(Base); + delete SyncToFd; + SyncToFd = NULL; + } + else + { + if (munmap((char *)Base, iSize) != 0) + _error->WarningE("mmap", _("Unable to close mmap")); + } + + iSize = 0; + Base = 0; + return true; +} + /*}}}*/ +// MMap::Sync - Synchronize the map with the disk /*{{{*/ +// --------------------------------------------------------------------- +/* This is done in synchronous mode - the docs indicate that this will + not return till all IO is complete */ +bool MMap::Sync() +{ + if ((Flags & UnMapped) == UnMapped) + return true; + + if ((Flags & ReadOnly) != ReadOnly) + { + if (SyncToFd != NULL) + { + if (!SyncToFd->Seek(0) || !SyncToFd->Write(Base, iSize)) + return false; + } + else + { +#ifdef _POSIX_SYNCHRONIZED_IO + if (msync((char *)Base, iSize, MS_SYNC) < 0) + return _error->Errno("msync", _("Unable to synchronize mmap")); +#endif + } + } + return true; +} + /*}}}*/ +// MMap::Sync - Synchronize a section of the file to disk /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool MMap::Sync(unsigned long Start,unsigned long Stop) +{ + if ((Flags & UnMapped) == UnMapped) + return true; + + if ((Flags & ReadOnly) != ReadOnly) + { + if (SyncToFd != 0) + { + if (!SyncToFd->Seek(0) || + !SyncToFd->Write (((char *)Base)+Start, Stop-Start)) + return false; + } + else + { +#ifdef _POSIX_SYNCHRONIZED_IO + unsigned long long const PSize = sysconf(_SC_PAGESIZE); + if (msync((char *)Base+(Start/PSize)*PSize, Stop - Start, MS_SYNC) < 0) + return _error->Errno("msync", _("Unable to synchronize mmap")); +#endif + } + } + return true; +} + /*}}}*/ + +// DynamicMMap::DynamicMMap - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long const &Workspace, + unsigned long const &Grow, unsigned long const &Limit) : + MMap(F,Flags | NoImmMap), Fd(&F), WorkSpace(Workspace), + GrowFactor(Grow), Limit(Limit) +{ + // disable Moveable if we don't grow + if (Grow == 0) + this->Flags &= ~Moveable; + +#ifndef __linux__ + // kfreebsd doesn't have mremap, so we use the fallback + if ((this->Flags & Moveable) == Moveable) + this->Flags |= Fallback; +#endif + + unsigned long long EndOfFile = Fd->Size(); + if (EndOfFile > WorkSpace) + WorkSpace = EndOfFile; + else if(WorkSpace > 0) + { + Fd->Seek(WorkSpace - 1); + char C = 0; + Fd->Write(&C,sizeof(C)); + } + + Map(F); + iSize = EndOfFile; +} + /*}}}*/ +// DynamicMMap::DynamicMMap - Constructor for a non-file backed map /*{{{*/ +// --------------------------------------------------------------------- +/* We try here to use mmap to reserve some space - this is much more + cooler than the fallback solution to simply allocate a char array + and could come in handy later than we are able to grow such an mmap */ +DynamicMMap::DynamicMMap(unsigned long Flags,unsigned long const &WorkSpace, + unsigned long const &Grow, unsigned long const &Limit) : + MMap(Flags | NoImmMap | UnMapped), Fd(0), WorkSpace(WorkSpace), + GrowFactor(Grow), Limit(Limit) +{ + // disable Moveable if we don't grow + if (Grow == 0) + this->Flags &= ~Moveable; + +#ifndef __linux__ + // kfreebsd doesn't have mremap, so we use the fallback + if ((this->Flags & Moveable) == Moveable) + this->Flags |= Fallback; +#endif + +#ifdef _POSIX_MAPPED_FILES + if ((this->Flags & Fallback) != Fallback) { + // Set the permissions. + int Prot = PROT_READ; +#ifdef MAP_ANONYMOUS + int Map = MAP_PRIVATE | MAP_ANONYMOUS; +#else + int Map = MAP_PRIVATE | MAP_ANON; +#endif + if ((this->Flags & ReadOnly) != ReadOnly) + Prot |= PROT_WRITE; + if ((this->Flags & Public) == Public) +#ifdef MAP_ANONYMOUS + Map = MAP_SHARED | MAP_ANONYMOUS; +#else + Map = MAP_SHARED | MAP_ANON; +#endif + + // use anonymous mmap() to get the memory + Base = (unsigned char*) mmap(0, WorkSpace, Prot, Map, -1, 0); + + if(Base == MAP_FAILED) + _error->Errno("DynamicMMap",_("Couldn't make mmap of %lu bytes"),WorkSpace); + + iSize = 0; + return; + } +#endif + // fallback to a static allocated space + Base = calloc(WorkSpace, 1); + iSize = 0; +} + /*}}}*/ +// DynamicMMap::~DynamicMMap - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* We truncate the file to the size of the memory data set */ +DynamicMMap::~DynamicMMap() +{ + if (Fd == 0) + { + if (validData() == false) + return; +#ifdef _POSIX_MAPPED_FILES + munmap(Base, WorkSpace); +#else + free(Base); +#endif + return; + } + + unsigned long long EndOfFile = iSize; + iSize = WorkSpace; + Close(false); + if(ftruncate(Fd->Fd(),EndOfFile) < 0) + _error->Errno("ftruncate", _("Failed to truncate file")); +} + /*}}}*/ +// DynamicMMap::RawAllocate - Allocate a raw chunk of unaligned space /*{{{*/ +// --------------------------------------------------------------------- +/* This allocates a block of memory aligned to the given size */ +unsigned long DynamicMMap::RawAllocate(unsigned long long Size,unsigned long Aln) +{ + unsigned long long Result = iSize; + if (Aln != 0) + Result += Aln - (iSize%Aln); + + iSize = Result + Size; + + // try to grow the buffer + while(Result + Size > WorkSpace) + { + if(!Grow()) + { + _error->Fatal(_("Dynamic MMap ran out of room. Please increase the size " + "of APT::Cache-Start. Current value: %lu. (man 5 apt.conf)"), WorkSpace); + return 0; + } + } + return Result; +} + /*}}}*/ +// DynamicMMap::Allocate - Pooled aligned allocation /*{{{*/ +// --------------------------------------------------------------------- +/* This allocates an Item of size ItemSize so that it is aligned to its + size in the file. */ +unsigned long DynamicMMap::Allocate(unsigned long ItemSize) +{ + if (unlikely(ItemSize == 0)) + { + _error->Fatal("Can't allocate an item of size zero"); + return 0; + } + + // Look for a matching pool entry + Pool *I; + for (I = Pools; I != Pools + PoolCount; ++I) + { + if (I->ItemSize == ItemSize) + break; + } + // No pool is allocated, use an unallocated one. + if (unlikely(I == Pools + PoolCount)) + { + for (I = Pools; I != Pools + PoolCount; ++I) + { + if (I->ItemSize == 0) + break; + } + // Woops, we ran out, the calling code should allocate more. + if (I == Pools + PoolCount) + { + _error->Error("Ran out of allocation pools"); + return 0; + } + + I->ItemSize = ItemSize; + I->Count = 0; + } + + unsigned long Result = 0; + // Out of space, allocate some more + if (I->Count == 0) + { + const unsigned long size = 20*1024; + I->Count = size/ItemSize; + Pool* oldPools = Pools; + _error->PushToStack(); + Result = RawAllocate(size,ItemSize); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + if (Pools != oldPools) + I = Pools + (I - oldPools); + + // Does the allocation failed ? + if (Result == 0 && newError) + return 0; + I->Start = Result; + } + else + Result = I->Start; + + I->Count--; + I->Start += ItemSize; + return Result/ItemSize; +} + /*}}}*/ +// DynamicMMap::WriteString - Write a string to the file /*{{{*/ +// --------------------------------------------------------------------- +/* Strings are aligned to 16 bytes */ +unsigned long DynamicMMap::WriteString(const char *String, + unsigned long Len) +{ + if (Len == std::numeric_limits<unsigned long>::max()) + Len = strlen(String); + + _error->PushToStack(); + unsigned long Result = RawAllocate(Len+1+sizeof(uint16_t),sizeof(uint16_t)); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + + if (Base == NULL || (Result == 0 && newError)) + return 0; + + if (Len >= std::numeric_limits<uint16_t>::max()) + abort(); + + uint16_t LenToWrite = Len; + memcpy((char *)Base + Result, &LenToWrite, sizeof(LenToWrite)); + Result += + sizeof(LenToWrite); + + memcpy((char *)Base + Result,String,Len); + ((char *)Base)[Result + Len] = 0; + return Result; +} + /*}}}*/ +// DynamicMMap::Grow - Grow the mmap /*{{{*/ +// --------------------------------------------------------------------- +/* This method is a wrapper around different methods to (try to) grow + a mmap (or our char[]-fallback). Encounterable environments: + 1. Moveable + !Fallback + linux -> mremap with MREMAP_MAYMOVE + 2. Moveable + !Fallback + !linux -> not possible (forbidden by constructor) + 3. Moveable + Fallback -> realloc + 4. !Moveable + !Fallback + linux -> mremap alone - which will fail in 99,9% + 5. !Moveable + !Fallback + !linux -> not possible (forbidden by constructor) + 6. !Moveable + Fallback -> not possible + [ While Moveable and Fallback stands for the equally named flags and + "linux" indicates a linux kernel instead of a freebsd kernel. ] + So what you can see here is, that a MMAP which want to be growable need + to be moveable to have a real chance but that this method will at least try + the nearly impossible 4 to grow it before it finally give up: Never say never. */ +bool DynamicMMap::Grow() { + if (Limit != 0 && WorkSpace >= Limit) + return _error->Error(_("Unable to increase the size of the MMap as the " + "limit of %lu bytes is already reached."), Limit); + if (GrowFactor <= 0) + return _error->Error(_("Unable to increase size of the MMap as automatic growing is disabled by user.")); + + unsigned long long const newSize = WorkSpace + GrowFactor; + + if(Fd != 0) { + Fd->Seek(newSize - 1); + char C = 0; + Fd->Write(&C,sizeof(C)); + } + + unsigned long const poolOffset = Pools - ((Pool*) Base); + + if ((Flags & Fallback) != Fallback) { +#if defined(_POSIX_MAPPED_FILES) && defined(__linux__) + #ifdef MREMAP_MAYMOVE + + if ((Flags & Moveable) == Moveable) + Base = mremap(Base, WorkSpace, newSize, MREMAP_MAYMOVE); + else + #endif + Base = mremap(Base, WorkSpace, newSize, 0); + + if(Base == MAP_FAILED) + return false; +#else + return false; +#endif + } else { + if ((Flags & Moveable) != Moveable) + return false; + + Base = realloc(Base, newSize); + if (Base == NULL) + return false; + else + /* Set new memory to 0 */ + memset((char*)Base + WorkSpace, 0, newSize - WorkSpace); + } + + Pools =(Pool*) Base + poolOffset; + WorkSpace = newSize; + return true; +} + /*}}}*/ diff --git a/apt-pkg/contrib/mmap.h b/apt-pkg/contrib/mmap.h new file mode 100644 index 0000000..fe834d5 --- /dev/null +++ b/apt-pkg/contrib/mmap.h @@ -0,0 +1,120 @@ +// -*- mode: cpp; mode: fold -*- +// SPDX-License-Identifier: GPL-2.0+ +// Description /*{{{*/ +/* ###################################################################### + + MMap Class - Provides 'real' mmap or a faked mmap using read(). + + The purpose of this code is to provide a generic way for clients to + access the mmap function. In environments that do not support mmap + from file fd's this function will use read and normal allocated + memory. + + Writing to a public mmap will always fully commit all changes when the + class is deleted. Ie it will rewrite the file, unless it is readonly + + The DynamicMMap class is used to help the on-disk data structure + generators. It provides a large allocated workspace and members + to allocate space from the workspace in an efficient fashion. + + This file had this historic note, but now includes further changes + under the GPL-2.0+: + + This source is placed in the Public Domain, do with it what you will + It was originally written by Jason Gunthorpe. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_MMAP_H +#define PKGLIB_MMAP_H + +#include <string> +#include <limits> + +#include <sys/mman.h> + + +class FileFd; + +/* This should be a 32 bit type, larger types use too much ram and smaller + types are too small. Where ever possible 'unsigned long' should be used + instead of this internal type */ +typedef unsigned int map_ptrloc; + +class MMap +{ + protected: + + unsigned long Flags; + unsigned long long iSize; + void *Base; + + // In case mmap can not be used, we keep a dup of the file + // descriptor that should have been mmaped so that we can write to + // the file in Sync(). + FileFd *SyncToFd; + + bool Map(FileFd &Fd); + bool Close(bool DoSync = true); + + public: + + enum OpenFlags {NoImmMap = (1<<0),Public = (1<<1),ReadOnly = (1<<2), + UnMapped = (1<<3), Moveable = (1<<4), Fallback = (1 << 5)}; + + // Simple accessors + inline operator void *() {return Base;}; + inline void *Data() {return Base;}; + inline unsigned long long Size() {return iSize;}; + inline void AddSize(unsigned long long const size) {iSize += size;}; + inline bool validData() const { return Base != MAP_FAILED && Base != 0; }; + + // File manipulators + bool Sync(); + bool Sync(unsigned long Start,unsigned long Stop); + + MMap(FileFd &F,unsigned long Flags); + explicit MMap(unsigned long Flags); + virtual ~MMap(); +}; + +class DynamicMMap : public MMap +{ + public: + + // This is the allocation pool structure + struct Pool + { + unsigned long ItemSize; + unsigned long Start; + unsigned long Count; + }; + + protected: + + FileFd *Fd; + unsigned long WorkSpace; + unsigned long const GrowFactor; + unsigned long const Limit; + Pool *Pools; + unsigned int PoolCount; + + bool Grow(); + + public: + + // Allocation + unsigned long RawAllocate(unsigned long long Size,unsigned long Aln = 0); + unsigned long Allocate(unsigned long ItemSize); + unsigned long WriteString(const char *String,unsigned long Len = std::numeric_limits<unsigned long>::max()); + inline unsigned long WriteString(const std::string &S) {return WriteString(S.c_str(),S.length());}; + void UsePools(Pool &P,unsigned int Count) {Pools = &P; PoolCount = Count;}; + + DynamicMMap(FileFd &F,unsigned long Flags,unsigned long const &WorkSpace = 2*1024*1024, + unsigned long const &Grow = 1024*1024, unsigned long const &Limit = 0); + DynamicMMap(unsigned long Flags,unsigned long const &WorkSpace = 2*1024*1024, + unsigned long const &Grow = 1024*1024, unsigned long const &Limit = 0); + virtual ~DynamicMMap(); +}; + +#endif diff --git a/apt-pkg/contrib/netrc.cc b/apt-pkg/contrib/netrc.cc new file mode 100644 index 0000000..ec694da --- /dev/null +++ b/apt-pkg/contrib/netrc.cc @@ -0,0 +1,173 @@ +// -*- mode: cpp; mode: fold -*- +// SPDX-License-Identifier: GPL-2.0+ +// Description /*{{{*/ +/* ###################################################################### + + netrc file parser - returns the login and password of a give host in + a specified netrc-type file + + This file had this historic note, but now includes further changes + under the GPL-2.0+: + + Originally written by Daniel Stenberg, <daniel@haxx.se>, et al. and + placed into the Public Domain, do with it what you will. + + ##################################################################### */ + /*}}}*/ +#include <config.h> +#include <apti18n.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/strutl.h> + +#include <iostream> + +#include "netrc.h" + +/* Get user and password from .netrc when given a machine name */ +bool MaybeAddAuth(FileFd &NetRCFile, URI &Uri) +{ + if (Uri.User.empty() == false || Uri.Password.empty() == false) + return true; + if (NetRCFile.IsOpen() == false || NetRCFile.Failed()) + return false; + auto const Debug = _config->FindB("Debug::Acquire::netrc", false); + + std::string lookfor; + if (Uri.Port != 0) + strprintf(lookfor, "%s:%i%s", Uri.Host.c_str(), Uri.Port, Uri.Path.c_str()); + else + lookfor.append(Uri.Host).append(Uri.Path); + + enum + { + NO, + MACHINE, + GOOD_MACHINE, + LOGIN, + PASSWORD + } active_token = NO; + std::string line; + while (NetRCFile.Eof() == false || line.empty() == false) + { + bool protocolSpecified = false; + + if (line.empty()) + { + if (NetRCFile.ReadLine(line) == false) + break; + else if (line.empty()) + continue; + } + auto tokenend = line.find_first_of("\t "); + std::string token; + if (tokenend != std::string::npos) + { + token = line.substr(0, tokenend); + line.erase(0, tokenend + 1); + } + else + std::swap(line, token); + if (token.empty()) + continue; + switch (active_token) + { + case NO: + if (token == "machine") + active_token = MACHINE; + break; + case MACHINE: + // If token contains a protocol: Check it first, and strip it away if + // it matches. If it does not match, ignore this stanza. + // If there is no protocol, only allow https protocols. + protocolSpecified = token.find("://") != std::string::npos; + if (protocolSpecified) + { + if (not APT::String::Startswith(token, Uri.Access + "://")) + { + active_token = NO; + break; + } + token.erase(0, Uri.Access.length() + 3); + } + + if (token.find('/') == std::string::npos) + { + if (Uri.Port != 0 && Uri.Host == token) + active_token = GOOD_MACHINE; + else if (lookfor.compare(0, lookfor.length() - Uri.Path.length(), token) == 0) + active_token = GOOD_MACHINE; + else + active_token = NO; + } + else + { + if (APT::String::Startswith(lookfor, token)) + active_token = GOOD_MACHINE; + else + active_token = NO; + } + + if (active_token == GOOD_MACHINE && not protocolSpecified) + { + if (Uri.Access != "https" && Uri.Access != "tor+https") + { + _error->Warning(_("%s: Credentials for %s match, but the protocol is not encrypted. Annotate with %s:// to use."), NetRCFile.Name().c_str(), token.c_str(), Uri.Access.c_str()); + active_token = NO; + } + } + break; + case GOOD_MACHINE: + if (token == "login") + active_token = LOGIN; + else if (token == "password") + active_token = PASSWORD; + else if (token == "machine") + { + if (Debug) + std::clog << "MaybeAddAuth: Found matching host adding '" << Uri.User << "' and '" << Uri.Password << "' for " + << (std::string)Uri << " from " << NetRCFile.Name() << std::endl; + return true; + } + break; + case LOGIN: + std::swap(Uri.User, token); + active_token = GOOD_MACHINE; + break; + case PASSWORD: + std::swap(Uri.Password, token); + active_token = GOOD_MACHINE; + break; + } + } + if (active_token == GOOD_MACHINE) + { + if (Debug) + std::clog << "MaybeAddAuth: Found matching host adding '" << Uri.User << "' and '" << Uri.Password << "' for " + << (std::string)Uri << " from " << NetRCFile.Name() << std::endl; + return true; + } + else if (active_token == NO) + { + if (Debug) + std::clog << "MaybeAddAuth: Found no matching host for " + << (std::string)Uri << " from " << NetRCFile.Name() << std::endl; + return true; + } + else if (Debug) + { + std::clog << "MaybeAddAuth: Found no matching host (syntax error: token:"; + switch (active_token) + { + case NO: std::clog << "NO"; break; + case MACHINE: std::clog << "MACHINE"; break; + case GOOD_MACHINE: std::clog << "GOOD_MACHINE"; break; + case LOGIN: std::clog << "LOGIN"; break; + case PASSWORD: std::clog << "PASSWORD"; break; + } + std::clog << ") for " << (std::string)Uri << " from " << NetRCFile.Name() << std::endl; + } + return false; +} diff --git a/apt-pkg/contrib/netrc.h b/apt-pkg/contrib/netrc.h new file mode 100644 index 0000000..2d202d1 --- /dev/null +++ b/apt-pkg/contrib/netrc.h @@ -0,0 +1,30 @@ +// -*- mode: cpp; mode: fold -*- +// SPDX-License-Identifier: GPL-2.0+ +// Description /*{{{*/ +/* ###################################################################### + + netrc file parser - returns the login and password of a give host in + a specified netrc-type file + + This file had this historic note, but now includes further changes + under the GPL-2.0+: + + Originally written by Daniel Stenberg, <daniel@haxx.se>, et al. and + placed into the Public Domain, do with it what you will. + + ##################################################################### */ + /*}}}*/ +#ifndef NETRC_H +#define NETRC_H + +#include <string> + +#include <apt-pkg/macros.h> + + + +class URI; +class FileFd; + +APT_PUBLIC bool MaybeAddAuth(FileFd &NetRCFile, URI &Uri); +#endif diff --git a/apt-pkg/contrib/progress.cc b/apt-pkg/contrib/progress.cc new file mode 100644 index 0000000..eb688b9 --- /dev/null +++ b/apt-pkg/contrib/progress.cc @@ -0,0 +1,229 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + OpProgress - Operation Progress + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/progress.h> + +#include <cmath> +#include <chrono> +#include <cstring> +#include <iostream> +#include <string> +#include <stdio.h> +#include <sys/time.h> + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +// OpProgress::OpProgress - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +OpProgress::OpProgress() : Current(0), Total(0), Size(0), SubTotal(1), + LastPercent(0), Percent(0) +{ + memset(&LastTime,0,sizeof(LastTime)); +} + /*}}}*/ +// OpProgress::Progress - Sub progress with no state change /*{{{*/ +// --------------------------------------------------------------------- +/* Current is the Base Overall progress in units of Total. Cur is the sub + progress in units of SubTotal. Size is a scaling factor that says what + percent of Total SubTotal is. */ +void OpProgress::Progress(unsigned long long Cur) +{ + if (Total == 0 || Size == 0 || SubTotal == 0) + Percent = 0; + else + Percent = (Current + Cur/((double)SubTotal)*Size)*100.0/Total; + Update(); +} + /*}}}*/ +// OpProgress::OverallProgress - Set the overall progress /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void OpProgress::OverallProgress(unsigned long long Current, unsigned long long Total, + unsigned long long Size,const string &Op) +{ + this->Current = Current; + this->Total = Total; + this->Size = Size; + this->Op = Op; + SubOp = string(); + if (Total == 0) + Percent = 0; + else + Percent = Current*100.0/Total; + Update(); +} + /*}}}*/ +// OpProgress::SubProgress - Set the sub progress state /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void OpProgress::SubProgress(unsigned long long SubTotal,const string &Op, + float const Percent) +{ + this->SubTotal = SubTotal; + if (Op.empty() == false) + SubOp = Op; + if (Total == 0 || Percent == 0) + this->Percent = 0; + else if (Percent != -1) + this->Percent = this->Current += (Size*Percent)/SubTotal; + else + this->Percent = Current*100.0/Total; + Update(); +} + /*}}}*/ +// OpProgress::CheckChange - See if the display should be updated /*{{{*/ +// --------------------------------------------------------------------- +/* Progress calls are made so frequently that if every one resulted in + an update the display would be swamped and the system much slower. + This provides an upper bound on the update rate. */ +bool OpProgress::CheckChange(float Interval) +{ + // For absolute progress, we assume every call is relevant. + if (_config->FindB("APT::Internal::OpProgress::Absolute", false)) + return true; + // New major progress indication + if (Op != LastOp) + { + MajorChange = true; + LastOp = Op; + return true; + } + MajorChange = false; + + if (SubOp != LastSubOp) + { + LastSubOp = SubOp; + return true; + } + + if (std::lround(LastPercent) == std::lround(Percent)) + return false; + + LastPercent = Percent; + + if (Interval == 0) + return false; + + // Check time delta + auto const Now = std::chrono::steady_clock::now().time_since_epoch(); + auto const Now_sec = std::chrono::duration_cast<std::chrono::seconds>(Now); + auto const Now_usec = std::chrono::duration_cast<std::chrono::microseconds>(Now - Now_sec); + struct timeval NowTime = { static_cast<time_t>(Now_sec.count()), static_cast<suseconds_t>(Now_usec.count()) }; + + std::chrono::duration<decltype(Interval)> Delta = + std::chrono::seconds(NowTime.tv_sec - LastTime.tv_sec) + + std::chrono::microseconds(NowTime.tv_usec - LastTime.tv_usec); + + if (Delta.count() < Interval) + return false; + LastTime = NowTime; + return true; +} + /*}}}*/ +// OpTextProgress::OpTextProgress - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +OpTextProgress::OpTextProgress(Configuration &Config) : + NoUpdate(false), NoDisplay(false), LastLen(0) +{ + if (Config.FindI("quiet",0) >= 1 || Config.FindB("quiet::NoUpdate", false) == true) + NoUpdate = true; + if (Config.FindI("quiet",0) >= 2 || Config.FindB("quiet::NoProgress", false) == true) + NoDisplay = true; +} + /*}}}*/ +// OpTextProgress::Done - Clean up the display /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void OpTextProgress::Done() +{ + if (NoUpdate == false && OldOp.empty() == false) + { + char S[300]; + if (_error->PendingError() == true) + snprintf(S,sizeof(S),_("%c%s... Error!"),'\r',OldOp.c_str()); + else + snprintf(S,sizeof(S),_("%c%s... Done"),'\r',OldOp.c_str()); + Write(S); + cout << endl; + OldOp = string(); + } + + if (NoUpdate == true && NoDisplay == false && OldOp.empty() == false) + { + OldOp = string(); + cout << endl; + } +} + /*}}}*/ +// OpTextProgress::Update - Simple text spinner /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void OpTextProgress::Update() +{ + if (CheckChange((NoUpdate == true?0:0.7)) == false) + return; + + // No percent spinner + if (NoUpdate == true) + { + if (MajorChange == false) + return; + if (NoDisplay == false) + { + if (OldOp.empty() == false) + cout << endl; + OldOp = "a"; + cout << Op << _("...") << flush; + } + + return; + } + + // Erase the old text and 'log' the event + char S[300]; + if (MajorChange == true && OldOp.empty() == false) + { + snprintf(S,sizeof(S),"\r%s",OldOp.c_str()); + Write(S); + cout << endl; + } + + // Print the spinner. Absolute progress shows us a time progress. + if (_config->FindB("APT::Internal::OpProgress::Absolute", false) && Total != -1llu) + snprintf(S, sizeof(S), _("%c%s... %llu/%llus"), '\r', Op.c_str(), Current, Total); + else if (_config->FindB("APT::Internal::OpProgress::Absolute", false)) + snprintf(S, sizeof(S), _("%c%s... %llus"), '\r', Op.c_str(), Current); + else + snprintf(S, sizeof(S), _("%c%s... %u%%"), '\r', Op.c_str(), (unsigned int)Percent); + Write(S); + + OldOp = Op; +} + /*}}}*/ +// OpTextProgress::Write - Write the progress string /*{{{*/ +// --------------------------------------------------------------------- +/* This space fills the end to overwrite the previous text */ +void OpTextProgress::Write(const char *S) +{ + cout << S; + for (unsigned int I = strlen(S); I < LastLen; I++) + cout << ' '; + cout << '\r' << flush; + LastLen = strlen(S); +} + /*}}}*/ diff --git a/apt-pkg/contrib/progress.h b/apt-pkg/contrib/progress.h new file mode 100644 index 0000000..d6a698a --- /dev/null +++ b/apt-pkg/contrib/progress.h @@ -0,0 +1,87 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + OpProgress - Operation Progress + + This class allows lengthy operations to communicate their progress + to the GUI. The progress model is simple and is not designed to handle + the complex case of the multi-activity acquire class. + + The model is based on the concept of an overall operation consisting + of a series of small sub operations. Each sub operation has it's own + completion status and the overall operation has it's completion status. + The units of the two are not mixed and are completely independent. + + The UI is expected to subclass this to provide the visuals to the user. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_PROGRESS_H +#define PKGLIB_PROGRESS_H + +#include <apt-pkg/macros.h> +#include <string> +#include <sys/time.h> + + +class Configuration; +class APT_PUBLIC OpProgress +{ + friend class OpTextProgress; + unsigned long long Current; + unsigned long long Total; + unsigned long long Size; + unsigned long long SubTotal; + float LastPercent; + + // Change reduction code + struct timeval LastTime; + std::string LastOp; + std::string LastSubOp; + + protected: + + std::string Op; + std::string SubOp; + float Percent; + + bool MajorChange; + + bool CheckChange(float Interval = 0.7); + virtual void Update() {}; + + public: + + void Progress(unsigned long long Current); + void SubProgress(unsigned long long SubTotal, const std::string &Op = "", float const Percent = -1); + void OverallProgress(unsigned long long Current,unsigned long long Total, + unsigned long long Size,const std::string &Op); + virtual void Done() {}; + + OpProgress(); + virtual ~OpProgress() {}; +}; + +class APT_PUBLIC OpTextProgress : public OpProgress +{ + protected: + + std::string OldOp; + bool NoUpdate; + bool NoDisplay; + unsigned long LastLen; + virtual void Update() APT_OVERRIDE; + void Write(const char *S); + + public: + + virtual void Done() APT_OVERRIDE; + + explicit OpTextProgress(bool NoUpdate = false) : NoUpdate(NoUpdate), + NoDisplay(false), LastLen(0) {}; + explicit OpTextProgress(Configuration &Config); + virtual ~OpTextProgress() {Done();}; +}; + +#endif diff --git a/apt-pkg/contrib/proxy.cc b/apt-pkg/contrib/proxy.cc new file mode 100644 index 0000000..a99f44f --- /dev/null +++ b/apt-pkg/contrib/proxy.cc @@ -0,0 +1,99 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Proxy - Proxy related functions + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <iostream> +#include <fcntl.h> +#include <unistd.h> + +#include "proxy.h" + /*}}}*/ + +// AutoDetectProxy - auto detect proxy /*{{{*/ +// --------------------------------------------------------------------- +/* */ +static std::vector<std::string> CompatibleProxies(URI const &URL) +{ + if (URL.Access == "http" || URL.Access == "https") + return {"http", "https", "socks5h"}; + return {URL.Access}; +} + +bool AutoDetectProxy(URI &URL) +{ + // we support both http/https debug options + bool Debug = _config->FindB("Debug::Acquire::"+URL.Access,false); + + // the user already explicitly set a proxy for this host + if(_config->Find("Acquire::"+URL.Access+"::proxy::"+URL.Host, "") != "") + return true; + + // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old + // name without the dash ("-") + std::string AutoDetectProxyCmd = _config->Find("Acquire::"+URL.Access+"::Proxy-Auto-Detect", + _config->Find("Acquire::"+URL.Access+"::ProxyAutoDetect")); + + if (AutoDetectProxyCmd.empty()) + return true; + + if (Debug) + std::clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << std::endl; + + if (faccessat(AT_FDCWD, AutoDetectProxyCmd.c_str(), R_OK | X_OK, AT_EACCESS) != 0) + return _error->Errno("access", "ProxyAutoDetect command '%s' can not be executed!", AutoDetectProxyCmd.c_str()); + + std::string const urlstring = URL; + std::vector<const char *> Args; + Args.push_back(AutoDetectProxyCmd.c_str()); + Args.push_back(urlstring.c_str()); + Args.push_back(nullptr); + FileFd PipeFd; + pid_t Child; + if (Popen(&Args[0], PipeFd, Child, FileFd::ReadOnly, false, true) == false) + return _error->Error("ProxyAutoDetect command '%s' failed!", AutoDetectProxyCmd.c_str()); + char buf[512]; + bool const goodread = PipeFd.ReadLine(buf, sizeof(buf)) != nullptr; + PipeFd.Close(); + if (ExecWait(Child, "ProxyAutoDetect") == false) + return false; + // no output means the detector has no idea which proxy to use + // and apt will use the generic proxy settings + if (goodread == false) + return true; + auto const cleanedbuf = _strstrip(buf); + // We warn about this as the implementor probably meant to use DIRECT instead + if (cleanedbuf[0] == '\0') + { + _error->Warning("ProxyAutoDetect command returned an empty line"); + return true; + } + + if (Debug) + std::clog << "auto detect command returned: '" << cleanedbuf << "'" << std::endl; + + auto compatibleTypes = CompatibleProxies(URL); + bool compatible = strcmp(cleanedbuf, "DIRECT") == 0 || + compatibleTypes.end() != std::find_if(compatibleTypes.begin(), + compatibleTypes.end(), [cleanedbuf](std::string &compat) { + return strstr(cleanedbuf, compat.c_str()) == cleanedbuf; + }); + + if (compatible) + _config->Set("Acquire::"+URL.Access+"::proxy::"+URL.Host, cleanedbuf); + + return true; +} + /*}}}*/ diff --git a/apt-pkg/contrib/proxy.h b/apt-pkg/contrib/proxy.h new file mode 100644 index 0000000..f6d70ea --- /dev/null +++ b/apt-pkg/contrib/proxy.h @@ -0,0 +1,16 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Proxy - Proxy operations + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_PROXY_H +#define PKGLIB_PROXY_H + +class URI; +APT_PUBLIC bool AutoDetectProxy(URI &URL); + + +#endif diff --git a/apt-pkg/contrib/srvrec.cc b/apt-pkg/contrib/srvrec.cc new file mode 100644 index 0000000..3eb5f1d --- /dev/null +++ b/apt-pkg/contrib/srvrec.cc @@ -0,0 +1,211 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + SRV record support + + ##################################################################### */ + /*}}}*/ +#include <config.h> + +#include <netdb.h> + +#include <arpa/nameser.h> +#include <arpa/inet.h> +#include <netinet/in.h> +#include <resolv.h> +#include <time.h> + +#include <algorithm> +#include <memory> +#include <tuple> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/strutl.h> + +#include "srvrec.h" + +bool SrvRec::operator==(SrvRec const &other) const +{ + return (std::tie(target, priority, weight, port) == + std::tie(other.target, other.priority, other.weight, other.port)); +} + +bool GetSrvRecords(std::string host, int port, std::vector<SrvRec> &Result) +{ + // try SRV only for hostnames, not for IP addresses + { + struct in_addr addr4; + struct in6_addr addr6; + if (inet_pton(AF_INET, host.c_str(), &addr4) == 1 || + inet_pton(AF_INET6, host.c_str(), &addr6) == 1) + return true; + } + + std::string target; + int res; + struct servent s_ent_buf; + struct servent *s_ent = nullptr; + std::vector<char> buf(1024); + + res = getservbyport_r(htons(port), "tcp", &s_ent_buf, buf.data(), buf.size(), &s_ent); + if (res != 0 || s_ent == nullptr) + return false; + + strprintf(target, "_%s._tcp.%s", s_ent->s_name, host.c_str()); + return GetSrvRecords(target, Result); +} + +bool GetSrvRecords(std::string name, std::vector<SrvRec> &Result) +{ + unsigned char answer[PACKETSZ]; + int answer_len, compressed_name_len; + int answer_count; +#if __RES >= 19991006 + struct __res_state res; + + if (res_ninit(&res) != 0) + return _error->Errno("res_init", "Failed to init resolver"); + + // Close on return + std::shared_ptr<void> guard(&res, res_nclose); + + answer_len = res_nquery(&res, name.c_str(), C_IN, T_SRV, answer, sizeof(answer)); +#else + if (res_init() != 0) + return _error->Errno("res_init", "Failed to init resolver"); + + answer_len = res_query(name.c_str(), C_IN, T_SRV, answer, sizeof(answer)); +#endif //__RES >= 19991006 + if (answer_len == -1) + return false; + if (answer_len < (int)sizeof(HEADER)) + return _error->Warning("Not enough data from res_query (%i)", answer_len); + + // check the header + HEADER *header = (HEADER*)answer; + if (header->rcode != NOERROR) + return _error->Warning("res_query returned rcode %i", header->rcode); + answer_count = ntohs(header->ancount); + if (answer_count <= 0) + return _error->Warning("res_query returned no answers (%i) ", answer_count); + + // skip the header + compressed_name_len = dn_skipname(answer+sizeof(HEADER), answer+answer_len); + if(compressed_name_len < 0) + return _error->Warning("dn_skipname failed %i", compressed_name_len); + + // pt points to the first answer record, go over all of them now + unsigned char *pt = answer+sizeof(HEADER)+compressed_name_len+QFIXEDSZ; + while ((int)Result.size() < answer_count && pt < answer+answer_len) + { + u_int16_t type, klass, priority, weight, port, dlen; + char buf[MAXDNAME]; + + compressed_name_len = dn_skipname(pt, answer+answer_len); + if (compressed_name_len < 0) + return _error->Warning("dn_skipname failed (2): %i", + compressed_name_len); + pt += compressed_name_len; + if (((answer+answer_len) - pt) < 16) + return _error->Warning("packet too short"); + + // extract the data out of the result buffer + #define extract_u16(target, p) target = *p++ << 8; target |= *p++; + + extract_u16(type, pt); + if(type != T_SRV) + return _error->Warning("Unexpected type excepted %x != %x", + T_SRV, type); + extract_u16(klass, pt); + if(klass != C_IN) + return _error->Warning("Unexpected class excepted %x != %x", + C_IN, klass); + pt += 4; // ttl + extract_u16(dlen, pt); + extract_u16(priority, pt); + extract_u16(weight, pt); + extract_u16(port, pt); + + #undef extract_u16 + + compressed_name_len = dn_expand(answer, answer+answer_len, pt, buf, sizeof(buf)); + if(compressed_name_len < 0) + return _error->Warning("dn_expand failed %i", compressed_name_len); + pt += compressed_name_len; + + // add it to our class + Result.emplace_back(buf, priority, weight, port); + } + + // implement load balancing as specified in RFC-2782 + + // sort them by priority + std::stable_sort(Result.begin(), Result.end()); + + if (_config->FindB("Debug::Acquire::SrvRecs", false)) + for(auto const &R : Result) + std::cerr << "SrvRecs: got " << R.target + << " prio: " << R.priority + << " weight: " << R.weight + << '\n'; + + return true; +} + +SrvRec PopFromSrvRecs(std::vector<SrvRec> &Recs) +{ + // FIXME: instead of the simplistic shuffle below use the algorithm + // described in rfc2782 (with weights) + // and figure out how the weights need to be adjusted if + // a host refuses connections + +#if 0 // all code below is only needed for the weight adjusted selection + // assign random number ranges + int prev_weight = 0; + int prev_priority = 0; + for(std::vector<SrvRec>::iterator I = Result.begin(); + I != Result.end(); ++I) + { + if(prev_priority != I->priority) + prev_weight = 0; + I->random_number_range_start = prev_weight; + I->random_number_range_end = prev_weight + I->weight; + prev_weight = I->random_number_range_end; + prev_priority = I->priority; + + if (_config->FindB("Debug::Acquire::SrvRecs", false) == true) + std::cerr << "SrvRecs: got " << I->target + << " prio: " << I->priority + << " weight: " << I->weight + << std::endl; + } + + // go over the code in reverse order and note the max random range + int max = 0; + prev_priority = 0; + for(std::vector<SrvRec>::iterator I = Result.end(); + I != Result.begin(); --I) + { + if(prev_priority != I->priority) + max = I->random_number_range_end; + I->random_number_range_max = max; + } +#endif + + // shuffle in a very simplistic way for now (equal weights) + std::vector<SrvRec>::iterator I = Recs.begin(); + std::vector<SrvRec>::iterator const J = std::find_if(Recs.begin(), Recs.end(), + [&I](SrvRec const &J) { return I->priority != J.priority; }); + + // clock seems random enough. + I += std::max(static_cast<clock_t>(0), clock()) % std::distance(I, J); + SrvRec const selected = std::move(*I); + Recs.erase(I); + + if (_config->FindB("Debug::Acquire::SrvRecs", false) == true) + std::cerr << "PopFromSrvRecs: selecting " << selected.target << std::endl; + + return selected; +} diff --git a/apt-pkg/contrib/srvrec.h b/apt-pkg/contrib/srvrec.h new file mode 100644 index 0000000..1e981d3 --- /dev/null +++ b/apt-pkg/contrib/srvrec.h @@ -0,0 +1,57 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + SRV record support + + ##################################################################### */ + /*}}}*/ +#ifndef SRVREC_H +#define SRVREC_H + +#include <string> +#include <vector> +#include <arpa/nameser.h> +#include <sys/types.h> + +#include <apt-pkg/macros.h> + +class APT_PUBLIC SrvRec +{ + public: + std::string target; + u_int16_t priority; + u_int16_t weight; + u_int16_t port; + + // each server is assigned a interval [start, end] in the space of [0, max] + int random_number_range_start; + int random_number_range_end; + int random_number_range_max; + + bool operator<(SrvRec const &other) const { + return this->priority < other.priority; + } + bool operator==(SrvRec const &other) const; + + SrvRec(std::string const Target, u_int16_t const Priority, + u_int16_t const Weight, u_int16_t const Port) : + target(Target), priority(Priority), weight(Weight), port(Port), + random_number_range_start(0), random_number_range_end(0), + random_number_range_max(0) {} +}; + +/** \brief Get SRV records from host/port (builds the query string internally) + */ +APT_PUBLIC bool GetSrvRecords(std::string name, std::vector<SrvRec> &Result); + +/** \brief Get SRV records for query string like: _http._tcp.example.com + */ +APT_PUBLIC bool GetSrvRecords(std::string host, int port, std::vector<SrvRec> &Result); + +/** \brief Pop a single SRV record from the vector of SrvRec taking + * priority and weight into account + */ +APT_PUBLIC SrvRec PopFromSrvRecs(std::vector<SrvRec> &Recs); + +#endif diff --git a/apt-pkg/contrib/string_view.h b/apt-pkg/contrib/string_view.h new file mode 100644 index 0000000..04f6ff1 --- /dev/null +++ b/apt-pkg/contrib/string_view.h @@ -0,0 +1,163 @@ +/* + * Basic implementation of string_view + * + * (C) 2015 Julian Andres Klode <jak@debian.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#if !defined(APT_STRINGVIEW_H) +#define APT_STRINGVIEW_H +#include <apt-pkg/macros.h> +#include <string> +#include <string.h> + +namespace APT { + +/** + * \brief Simple subset of std::string_view from C++17 + * + * This is an internal implementation of the subset of std::string_view + * used by APT. It is not meant to be used in programs, only inside the + * library for performance critical paths. + */ +class StringView { + const char *data_; + size_t size_; + +public: + static constexpr size_t npos = static_cast<size_t>(-1); + static_assert(APT::StringView::npos == std::string::npos, "npos values are different"); + + /* Constructors */ + constexpr StringView() : data_(""), size_(0) {} + constexpr StringView(const char *data, size_t size) : data_(data), size_(size) {} + + StringView(const char *data) : data_(data), size_(strlen(data)) {} + StringView(std::string const & str): data_(str.data()), size_(str.size()) {} + + /* Modifiers */ + void remove_prefix(size_t n) { data_ += n; size_ -= n; } + void remove_suffix(size_t n) { size_ -= n; } + void clear() { size_ = 0; } + + /* Viewers */ + constexpr StringView substr(size_t pos, size_t n = npos) const { + return StringView(data_ + pos, n > (size_ - pos) ? (size_ - pos) : n); + } + + size_t find(int c, size_t pos) const { + if (pos == 0) + return find(c); + size_t const found = substr(pos).find(c); + if (found == npos) + return npos; + return pos + found; + } + size_t find(int c) const { + const char *found = static_cast<const char*>(memchr(data_, c, size_)); + + if (found == NULL) + return npos; + + return found - data_; + } + + size_t rfind(int c, size_t pos) const { + if (pos == npos) + return rfind(c); + return APT::StringView(data_, pos).rfind(c); + } + size_t rfind(int c) const { + const char *found = static_cast<const char*>(memrchr(data_, c, size_)); + + if (found == NULL) + return npos; + + return found - data_; + } + + size_t find(APT::StringView const needle) const { + if (needle.empty()) + return npos; + if (needle.length() == 1) + return find(*needle.data()); + size_t found = 0; + while ((found = find(*needle.data(), found)) != npos) { + if (compare(found, needle.length(), needle) == 0) + return found; + ++found; + } + return found; + } + size_t find(APT::StringView const needle, size_t pos) const { + if (pos == 0) + return find(needle); + size_t const found = substr(pos).find(needle); + if (found == npos) + return npos; + return pos + found; + } + + /* Conversions */ + std::string to_string() const { + return std::string(data_, size_); + } + + /* Comparisons */ + int compare(size_t pos, size_t n, StringView other) const { + return substr(pos, n).compare(other); + } + + int compare(StringView other) const { + int res; + + res = memcmp(data_, other.data_, std::min(size_, other.size_)); + if (res != 0) + return res; + if (size_ == other.size_) + return res; + + return (size_ > other.size_) ? 1 : -1; + } + + /* Optimization: If size not equal, string cannot be equal */ + bool operator ==(StringView other) const { return size_ == other.size_ && compare(other) == 0; } + bool operator !=(StringView other) const { return !(*this == other); } + + /* Accessors */ + constexpr bool empty() const { return size_ == 0; } + constexpr const char* data() const { return data_; } + constexpr const char* begin() const { return data_; } + constexpr const char* end() const { return data_ + size_; } + constexpr char operator [](size_t i) const { return data_[i]; } + constexpr size_t size() const { return size_; } + constexpr size_t length() const { return size_; } +}; + +/** + * \brief Faster comparison for string views (compare size before data) + * + * Still stable, but faster than the normal ordering. */ +static inline int StringViewCompareFast(StringView a, StringView b) { + if (a.size() != b.size()) + return a.size() - b.size(); + + return memcmp(a.data(), b.data(), a.size()); +} + +static constexpr inline APT::StringView operator""_sv(const char *data, size_t size) +{ + return APT::StringView(data, size); +} +} + +inline bool operator ==(const char *other, APT::StringView that); +inline bool operator ==(const char *other, APT::StringView that) { return that.operator==(other); } +inline bool operator ==(std::string const &other, APT::StringView that); +inline bool operator ==(std::string const &other, APT::StringView that) { return that.operator==(other); } + +#endif diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc new file mode 100644 index 0000000..67100f1 --- /dev/null +++ b/apt-pkg/contrib/strutl.cc @@ -0,0 +1,1820 @@ +// -*- mode: cpp; mode: fold -*- +// SPDX-License-Identifier: GPL-2.0+ +// Description /*{{{*/ +/* ###################################################################### + + String Util - Some useful string functions. + + These have been collected from here and there to do all sorts of useful + things to strings. They are useful in file parsers, URI handlers and + especially in APT methods. + + This file had this historic note, but now includes further changes + under the GPL-2.0+: + + This source is placed in the Public Domain, do with it what you will + It was originally written by Jason Gunthorpe <jgg@gpu.srv.ualberta.ca> + + ##################################################################### */ + /*}}}*/ +// Includes /*{{{*/ +#include <config.h> + +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <array> +#include <iomanip> +#include <limits> +#include <locale> +#include <sstream> +#include <memory> +#include <sstream> +#include <string> +#include <vector> + +#include <ctype.h> +#include <errno.h> +#include <iconv.h> +#include <regex.h> +#include <stdarg.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> +#include <wchar.h> + +#include <apti18n.h> + /*}}}*/ +using namespace std; + +// Strip - Remove white space from the front and back of a string /*{{{*/ +// --------------------------------------------------------------------- +namespace APT { + namespace String { +std::string Strip(const std::string &str) +{ + // ensure we have at least one character + if (str.empty() == true) + return str; + + char const * const s = str.c_str(); + size_t start = 0; + for (; isspace(s[start]) != 0; ++start) + ; // find the first not-space + + // string contains only whitespaces + if (s[start] == '\0') + return ""; + + size_t end = str.length() - 1; + for (; isspace(s[end]) != 0; --end) + ; // find the last not-space + + return str.substr(start, end - start + 1); +} + +bool Endswith(const std::string &s, const std::string &end) +{ + if (end.size() > s.size()) + return false; + return (s.compare(s.size() - end.size(), end.size(), end) == 0); +} + +bool Startswith(const std::string &s, const std::string &start) +{ + if (start.size() > s.size()) + return false; + return (s.compare(0, start.size(), start) == 0); +} + +std::string Join(std::vector<std::string> list, const std::string &sep) +{ + std::ostringstream oss; + for (auto it = list.begin(); it != list.end(); it++) + { + if (it != list.begin()) oss << sep; + oss << *it; + } + return oss.str(); +} + +// Returns string display length honoring multi-byte characters +size_t DisplayLength(StringView str) +{ + size_t len = 0; + + const char *p = str.data(); + const char *const end = str.end(); + + mbstate_t state{}; + while (p < end) + { + wchar_t wch; + size_t res = mbrtowc(&wch, p, end - p, &state); + switch (res) + { + case 0: + // Null wide character (i.e. L'\0') - stop + p = end; + break; + + case static_cast<size_t>(-1): + // Byte sequence is invalid. Assume that it's + // a single-byte single-width character. + len += 1; + p += 1; + + // state is undefined in this case - reset it + state = {}; + + break; + + case static_cast<size_t>(-2): + // Byte sequence is too short. Assume that it's + // an incomplete single-width character and stop. + len += 1; + p = end; + break; + + default: + len += wcwidth(wch); + p += res; + } + } + + return len; +} + +} +} + /*}}}*/ +// UTF8ToCodeset - Convert some UTF-8 string for some codeset /*{{{*/ +// --------------------------------------------------------------------- +/* This is handy to use before display some information for enduser */ +bool UTF8ToCodeset(const char *codeset, const string &orig, string *dest) +{ + iconv_t cd; + const char *inbuf; + char *inptr, *outbuf; + size_t insize, bufsize; + dest->clear(); + + cd = iconv_open(codeset, "UTF-8"); + if (cd == (iconv_t)(-1)) { + // Something went wrong + if (errno == EINVAL) + _error->Error("conversion from 'UTF-8' to '%s' not available", + codeset); + else + perror("iconv_open"); + + return false; + } + + insize = bufsize = orig.size(); + inbuf = orig.data(); + inptr = (char *)inbuf; + outbuf = new char[bufsize]; + size_t lastError = -1; + + while (insize != 0) + { + char *outptr = outbuf; + size_t outsize = bufsize; + size_t const err = iconv(cd, &inptr, &insize, &outptr, &outsize); + dest->append(outbuf, outptr - outbuf); + if (err == (size_t)(-1)) + { + switch (errno) + { + case EILSEQ: + insize--; + inptr++; + // replace a series of unknown multibytes with a single "?" + if (lastError != insize) { + lastError = insize - 1; + dest->append("?"); + } + break; + case EINVAL: + insize = 0; + break; + case E2BIG: + if (outptr == outbuf) + { + bufsize *= 2; + delete[] outbuf; + outbuf = new char[bufsize]; + } + break; + } + } + } + + delete[] outbuf; + + iconv_close(cd); + + return true; +} + /*}}}*/ +// strstrip - Remove white space from the front and back of a string /*{{{*/ +// --------------------------------------------------------------------- +/* This is handy to use when parsing a file. It also removes \n's left + over from fgets and company */ +char *_strstrip(char *String) +{ + for (;*String != 0 && (*String == ' ' || *String == '\t'); String++); + + if (*String == 0) + return String; + return _strrstrip(String); +} + /*}}}*/ +// strrstrip - Remove white space from the back of a string /*{{{*/ +// --------------------------------------------------------------------- +char *_strrstrip(char *String) +{ + char *End = String + strlen(String) - 1; + for (;End != String - 1 && (*End == ' ' || *End == '\t' || *End == '\n' || + *End == '\r'); End--); + End++; + *End = 0; + return String; +} + /*}}}*/ +// strtabexpand - Converts tabs into 8 spaces /*{{{*/ +// --------------------------------------------------------------------- +/* */ +char *_strtabexpand(char *String,size_t Len) +{ + for (char *I = String; I != I + Len && *I != 0; I++) + { + if (*I != '\t') + continue; + if (I + 8 > String + Len) + { + *I = 0; + return String; + } + + /* Assume the start of the string is 0 and find the next 8 char + division */ + int Len; + if (String == I) + Len = 1; + else + Len = 8 - ((String - I) % 8); + Len -= 2; + if (Len <= 0) + { + *I = ' '; + continue; + } + + memmove(I + Len,I + 1,strlen(I) + 1); + for (char *J = I; J + Len != I; *I = ' ', I++); + } + return String; +} + /*}}}*/ +// ParseQuoteWord - Parse a single word out of a string /*{{{*/ +// --------------------------------------------------------------------- +/* This grabs a single word, converts any % escaped characters to their + proper values and advances the pointer. Double quotes are understood + and striped out as well. This is for URI/URL parsing. It also can + understand [] brackets.*/ +bool ParseQuoteWord(const char *&String,string &Res) +{ + // Skip leading whitespace + const char *C = String; + for (; *C == ' '; C++) + ; + if (*C == 0) + return false; + + // Jump to the next word + for (;*C != 0 && isspace(*C) == 0; C++) + { + if (*C == '"') + { + C = strchr(C + 1, '"'); + if (C == NULL) + return false; + } + if (*C == '[') + { + C = strchr(C + 1, ']'); + if (C == NULL) + return false; + } + } + + // Now de-quote characters + Res.clear(); + Res.reserve(C - String); + char Tmp[3]; + const char *Start = String; + while (Start != C) + { + if (*Start == '%' && Start + 2 < C && + isxdigit(Start[1]) && isxdigit(Start[2])) + { + Tmp[0] = Start[1]; + Tmp[1] = Start[2]; + Tmp[2] = 0; + Res.push_back(static_cast<char>(strtol(Tmp, 0, 16))); + Start += 3; + continue; + } + if (*Start != '"') + Res.push_back(*Start); + ++Start; + } + + // Skip ending white space + for (; isspace(*C) != 0; C++) + ; + String = C; + return true; +} + /*}}}*/ +// ParseCWord - Parses a string like a C "" expression /*{{{*/ +// --------------------------------------------------------------------- +/* This expects a series of space separated strings enclosed in ""'s. + It concatenates the ""'s into a single string. */ +bool ParseCWord(const char *&String,string &Res) +{ + // Skip leading whitespace + const char *C = String; + for (; *C == ' '; C++) + ; + if (*C == 0) + return false; + + Res.clear(); + Res.reserve(strlen(String)); + for (; *C != 0; ++C) + { + if (*C == '"') + { + for (C++; *C != 0 && *C != '"'; C++) + Res.push_back(*C); + + if (*C == 0) + return false; + + continue; + } + + if (C != String && isspace(*C) != 0 && isspace(C[-1]) != 0) + continue; + if (isspace(*C) == 0) + return false; + Res.push_back(' '); + } + String = C; + return true; +} + /*}}}*/ +// QuoteString - Convert a string into quoted from /*{{{*/ +// --------------------------------------------------------------------- +/* */ +string QuoteString(const string &Str, const char *Bad) +{ + std::stringstream Res; + for (string::const_iterator I = Str.begin(); I != Str.end(); ++I) + { + if (strchr(Bad,*I) != 0 || isprint(*I) == 0 || + *I == 0x25 || // percent '%' char + *I <= 0x20 || *I >= 0x7F) // control chars + { + ioprintf(Res, "%%%02hhx", *I); + } + else + Res << *I; + } + return Res.str(); +} + /*}}}*/ +// DeQuoteString - Convert a string from quoted from /*{{{*/ +// --------------------------------------------------------------------- +/* This undoes QuoteString */ +string DeQuoteString(const string &Str) +{ + return DeQuoteString(Str.begin(),Str.end()); +} +string DeQuoteString(string::const_iterator const &begin, + string::const_iterator const &end) +{ + string Res; + for (string::const_iterator I = begin; I != end; ++I) + { + if (*I == '%' && I + 2 < end && + isxdigit(I[1]) && isxdigit(I[2])) + { + char Tmp[3]; + Tmp[0] = I[1]; + Tmp[1] = I[2]; + Tmp[2] = 0; + Res += (char)strtol(Tmp,0,16); + I += 2; + continue; + } + else + Res += *I; + } + return Res; +} + + /*}}}*/ +// SizeToStr - Convert a long into a human readable size /*{{{*/ +// --------------------------------------------------------------------- +/* A max of 4 digits are shown before conversion to the next highest unit. + The max length of the string will be 5 chars unless the size is > 10 + YottaBytes (E24) */ +string SizeToStr(double Size) +{ + double ASize; + if (Size >= 0) + ASize = Size; + else + ASize = -1*Size; + + /* bytes, KiloBytes, MegaBytes, GigaBytes, TeraBytes, PetaBytes, + ExaBytes, ZettaBytes, YottaBytes */ + char Ext[] = {'\0','k','M','G','T','P','E','Z','Y'}; + int I = 0; + while (I <= 8) + { + if (ASize < 100 && I != 0) + { + std::string S; + strprintf(S, "%'.1f %c", ASize, Ext[I]); + return S; + } + + if (ASize < 10000) + { + std::string S; + strprintf(S, "%'.0f %c", ASize, Ext[I]); + return S; + } + ASize /= 1000.0; + I++; + } + return ""; +} + /*}}}*/ +// TimeToStr - Convert the time into a string /*{{{*/ +// --------------------------------------------------------------------- +/* Converts a number of seconds to a hms format */ +string TimeToStr(unsigned long Sec) +{ + std::string S; + if (Sec > 60*60*24) + { + //TRANSLATOR: d means days, h means hours, min means minutes, s means seconds + strprintf(S,_("%lid %lih %limin %lis"),Sec/60/60/24,(Sec/60/60) % 24,(Sec/60) % 60,Sec % 60); + } + else if (Sec > 60*60) + { + //TRANSLATOR: h means hours, min means minutes, s means seconds + strprintf(S,_("%lih %limin %lis"),Sec/60/60,(Sec/60) % 60,Sec % 60); + } + else if (Sec > 60) + { + //TRANSLATOR: min means minutes, s means seconds + strprintf(S,_("%limin %lis"),Sec/60,Sec % 60); + } + else + { + //TRANSLATOR: s means seconds + strprintf(S,_("%lis"),Sec); + } + return S; +} + /*}}}*/ +// SubstVar - Substitute a string for another string /*{{{*/ +// --------------------------------------------------------------------- +/* This replaces all occurrences of Subst with Contents in Str. */ +string SubstVar(const string &Str,const string &Subst,const string &Contents) +{ + if (Subst.empty() == true) + return Str; + + string::size_type Pos = 0; + string::size_type OldPos = 0; + string Temp; + + while (OldPos < Str.length() && + (Pos = Str.find(Subst,OldPos)) != string::npos) + { + if (OldPos != Pos) + Temp.append(Str, OldPos, Pos - OldPos); + if (Contents.empty() == false) + Temp.append(Contents); + OldPos = Pos + Subst.length(); + } + + if (OldPos == 0) + return Str; + + if (OldPos >= Str.length()) + return Temp; + + Temp.append(Str, OldPos, string::npos); + return Temp; +} +string SubstVar(string Str,const struct SubstVar *Vars) +{ + for (; Vars->Subst != 0; Vars++) + Str = SubstVar(Str,Vars->Subst,*Vars->Contents); + return Str; +} + /*}}}*/ +// OutputInDepth - return a string with separator multiplied with depth /*{{{*/ +// --------------------------------------------------------------------- +/* Returns a string with the supplied separator depth + 1 times in it */ +std::string OutputInDepth(const unsigned long Depth, const char* Separator) +{ + std::string output = ""; + for(unsigned long d=Depth+1; d > 0; d--) + output.append(Separator); + return output; +} + /*}}}*/ +// URItoFileName - Convert the uri into a unique file name /*{{{*/ +// --------------------------------------------------------------------- +/* This converts a URI into a safe filename. It quotes all unsafe characters + and converts / to _ and removes the scheme identifier. The resulting + file name should be unique and never occur again for a different file */ +string URItoFileName(const string &URI) +{ + // Nuke 'sensitive' items + ::URI U(URI); + U.User.clear(); + U.Password.clear(); + U.Access.clear(); + + // "\x00-\x20{}|\\\\^\\[\\]<>\"\x7F-\xFF"; + string NewURI = QuoteString(U,"\\|{}[]<>\"^~_=!@#$%^&*"); + replace(NewURI.begin(),NewURI.end(),'/','_'); + return NewURI; +} + /*}}}*/ +// Base64Encode - Base64 Encoding routine for short strings /*{{{*/ +// --------------------------------------------------------------------- +/* This routine performs a base64 transformation on a string. It was ripped + from wget and then patched and bug fixed. + + This spec can be found in rfc2045 */ +string Base64Encode(const string &S) +{ + // Conversion table. + static char tbl[64] = {'A','B','C','D','E','F','G','H', + 'I','J','K','L','M','N','O','P', + 'Q','R','S','T','U','V','W','X', + 'Y','Z','a','b','c','d','e','f', + 'g','h','i','j','k','l','m','n', + 'o','p','q','r','s','t','u','v', + 'w','x','y','z','0','1','2','3', + '4','5','6','7','8','9','+','/'}; + + // Pre-allocate some space + string Final; + Final.reserve((4*S.length() + 2)/3 + 2); + + /* Transform the 3x8 bits to 4x6 bits, as required by + base64. */ + for (string::const_iterator I = S.begin(); I < S.end(); I += 3) + { + uint8_t Bits[3] = {0,0,0}; + Bits[0] = I[0]; + if (I + 1 < S.end()) + Bits[1] = I[1]; + if (I + 2 < S.end()) + Bits[2] = I[2]; + + Final += tbl[Bits[0] >> 2]; + Final += tbl[((Bits[0] & 3) << 4) + (Bits[1] >> 4)]; + + if (I + 1 >= S.end()) + break; + + Final += tbl[((Bits[1] & 0xf) << 2) + (Bits[2] >> 6)]; + + if (I + 2 >= S.end()) + break; + + Final += tbl[Bits[2] & 0x3f]; + } + + /* Apply the padding elements, this tells how many bytes the remote + end should discard */ + if (S.length() % 3 == 2) + Final += '='; + if (S.length() % 3 == 1) + Final += "=="; + + return Final; +} + /*}}}*/ +// stringcmp - Arbitrary string compare /*{{{*/ +// --------------------------------------------------------------------- +/* This safely compares two non-null terminated strings of arbitrary + length */ +int stringcmp(const char *A,const char *AEnd,const char *B,const char *BEnd) +{ + for (; A != AEnd && B != BEnd; A++, B++) + if (*A != *B) + break; + + if (A == AEnd && B == BEnd) + return 0; + if (A == AEnd) + return 1; + if (B == BEnd) + return -1; + if (*A < *B) + return -1; + return 1; +} + +#if __GNUC__ >= 3 +int stringcmp(string::const_iterator A,string::const_iterator AEnd, + const char *B,const char *BEnd) +{ + for (; A != AEnd && B != BEnd; A++, B++) + if (*A != *B) + break; + + if (A == AEnd && B == BEnd) + return 0; + if (A == AEnd) + return 1; + if (B == BEnd) + return -1; + if (*A < *B) + return -1; + return 1; +} +int stringcmp(string::const_iterator A,string::const_iterator AEnd, + string::const_iterator B,string::const_iterator BEnd) +{ + for (; A != AEnd && B != BEnd; A++, B++) + if (*A != *B) + break; + + if (A == AEnd && B == BEnd) + return 0; + if (A == AEnd) + return 1; + if (B == BEnd) + return -1; + if (*A < *B) + return -1; + return 1; +} +#endif + /*}}}*/ +// stringcasecmp - Arbitrary case insensitive string compare /*{{{*/ +// --------------------------------------------------------------------- +/* */ +int stringcasecmp(const char *A,const char *AEnd,const char *B,const char *BEnd) +{ + for (; A != AEnd && B != BEnd; A++, B++) + if (tolower_ascii(*A) != tolower_ascii(*B)) + break; + + if (A == AEnd && B == BEnd) + return 0; + if (A == AEnd) + return 1; + if (B == BEnd) + return -1; + if (tolower_ascii(*A) < tolower_ascii(*B)) + return -1; + return 1; +} +#if __GNUC__ >= 3 +int stringcasecmp(string::const_iterator A,string::const_iterator AEnd, + const char *B,const char *BEnd) +{ + for (; A != AEnd && B != BEnd; A++, B++) + if (tolower_ascii(*A) != tolower_ascii(*B)) + break; + + if (A == AEnd && B == BEnd) + return 0; + if (A == AEnd) + return 1; + if (B == BEnd) + return -1; + if (tolower_ascii(*A) < tolower_ascii(*B)) + return -1; + return 1; +} +int stringcasecmp(string::const_iterator A,string::const_iterator AEnd, + string::const_iterator B,string::const_iterator BEnd) +{ + for (; A != AEnd && B != BEnd; A++, B++) + if (tolower_ascii(*A) != tolower_ascii(*B)) + break; + + if (A == AEnd && B == BEnd) + return 0; + if (A == AEnd) + return 1; + if (B == BEnd) + return -1; + if (tolower_ascii(*A) < tolower_ascii(*B)) + return -1; + return 1; +} +#endif + /*}}}*/ +// LookupTag - Lookup the value of a tag in a tagged string /*{{{*/ +// --------------------------------------------------------------------- +/* The format is like those used in package files and the method + communication system */ +std::string LookupTag(const std::string &Message, const char *TagC, const char *Default) +{ + std::string tag = std::string("\n") + TagC + ":"; + if (Default == nullptr) + Default = ""; + if (Message.length() < tag.length()) + return Default; + std::transform(tag.begin(), tag.end(), tag.begin(), tolower_ascii); + auto valuestart = Message.cbegin(); + // maybe the message starts directly with tag + if (Message[tag.length() - 2] == ':') + { + std::string lowstart = std::string("\n") + Message.substr(0, tag.length() - 1); + std::transform(lowstart.begin(), lowstart.end(), lowstart.begin(), tolower_ascii); + if (lowstart == tag) + valuestart = std::next(valuestart, tag.length() - 1); + } + // the tag is somewhere in the message + if (valuestart == Message.cbegin()) + { + auto const tagbegin = std::search(Message.cbegin(), Message.cend(), tag.cbegin(), tag.cend(), + [](char const a, char const b) { return tolower_ascii(a) == b; }); + if (tagbegin == Message.cend()) + return Default; + valuestart = std::next(tagbegin, tag.length()); + } + auto const is_whitespace = [](char const c) { return isspace_ascii(c) != 0 && c != '\n'; }; + auto const is_newline = [](char const c) { return c == '\n'; }; + std::string result; + valuestart = std::find_if_not(valuestart, Message.cend(), is_whitespace); + // is the first line of the value empty? + if (valuestart != Message.cend() && *valuestart == '\n') + { + valuestart = std::next(valuestart); + if (valuestart != Message.cend() && *valuestart == ' ') + valuestart = std::next(valuestart); + } + // extract the value over multiple lines removing trailing whitespace + while (valuestart < Message.cend()) + { + auto const linebreak = std::find_if(valuestart, Message.cend(), is_newline); + auto valueend = std::prev(linebreak); + // skip spaces at the end of the line + while (valueend > valuestart && is_whitespace(*valueend)) + valueend = std::prev(valueend); + // append found line to result + { + std::string tmp(valuestart, std::next(valueend)); + if (tmp != ".") + { + if (result.empty()) + result.assign(std::move(tmp)); + else + result.append(tmp); + } + } + // see if the value is multiline + if (linebreak == Message.cend()) + break; + valuestart = std::next(linebreak); + if (valuestart == Message.cend() || *valuestart != ' ') + break; + result.append("\n"); + // skip the space leading a multiline (Keep all other whitespaces in the value) + valuestart = std::next(valuestart); + } + auto const valueend = result.find_last_not_of("\n"); + if (valueend == std::string::npos) + result.clear(); + else + result.erase(valueend + 1); + return result; +} + /*}}}*/ +// StringToBool - Converts a string into a boolean /*{{{*/ +// --------------------------------------------------------------------- +/* This inspects the string to see if it is true or if it is false and + then returns the result. Several variants on true/false are checked. */ +int StringToBool(const string &Text,int Default) +{ + char *ParseEnd; + int Res = strtol(Text.c_str(),&ParseEnd,0); + // ensure that the entire string was converted by strtol to avoid + // failures on "apt-cache show -a 0ad" where the "0" is converted + const char *TextEnd = Text.c_str()+Text.size(); + if (ParseEnd == TextEnd && Res >= 0 && Res <= 1) + return Res; + + // Check for positives + if (strcasecmp(Text.c_str(),"no") == 0 || + strcasecmp(Text.c_str(),"false") == 0 || + strcasecmp(Text.c_str(),"without") == 0 || + strcasecmp(Text.c_str(),"off") == 0 || + strcasecmp(Text.c_str(),"disable") == 0) + return 0; + + // Check for negatives + if (strcasecmp(Text.c_str(),"yes") == 0 || + strcasecmp(Text.c_str(),"true") == 0 || + strcasecmp(Text.c_str(),"with") == 0 || + strcasecmp(Text.c_str(),"on") == 0 || + strcasecmp(Text.c_str(),"enable") == 0) + return 1; + + return Default; +} + /*}}}*/ +// TimeRFC1123 - Convert a time_t into RFC1123 format /*{{{*/ +// --------------------------------------------------------------------- +/* This converts a time_t into a string time representation that is + year 2000 compliant and timezone neutral */ +string TimeRFC1123(time_t Date, bool const NumericTimezone) +{ + struct tm Conv; + if (gmtime_r(&Date, &Conv) == NULL) + return ""; + + auto const posix = std::locale::classic(); + std::ostringstream datestr; + datestr.imbue(posix); + APT::StringView const fmt("%a, %d %b %Y %H:%M:%S"); + std::use_facet<std::time_put<char>>(posix).put( + std::ostreambuf_iterator<char>(datestr), + datestr, ' ', &Conv, fmt.data(), fmt.data() + fmt.size()); + if (NumericTimezone) + datestr << " +0000"; + else + datestr << " GMT"; + return datestr.str(); +} + /*}}}*/ +// ReadMessages - Read messages from the FD /*{{{*/ +// --------------------------------------------------------------------- +/* This pulls full messages from the input FD into the message buffer. + It assumes that messages will not pause during transit so no + fancy buffering is used. + + In particular: this reads blocks from the input until it believes + that it's run out of input text. Each block is terminated by a + double newline ('\n' followed by '\n'). + */ +bool ReadMessages(int Fd, vector<string> &List) +{ + char Buffer[64000]; + // Represents any left-over from the previous iteration of the + // parse loop. (i.e., if a message is split across the end + // of the buffer, it goes here) + string PartialMessage; + + do { + int const Res = read(Fd, Buffer, sizeof(Buffer)); + if (Res < 0 && errno == EINTR) + continue; + + // process we read from has died + if (Res == 0) + return false; + + // No data +#if EAGAIN != EWOULDBLOCK + if (Res < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) +#else + if (Res < 0 && errno == EAGAIN) +#endif + return true; + if (Res < 0) + return false; + + // extract the message(s) from the buffer + char const *Start = Buffer; + char const * const End = Buffer + Res; + + char const * NL = (char const *) memchr(Start, '\n', End - Start); + if (NL == NULL) + { + // end of buffer: store what we have so far and read new data in + PartialMessage.append(Start, End - Start); + Start = End; + } + else + ++NL; + + if (PartialMessage.empty() == false && Start < End) + { + // if we start with a new line, see if the partial message we have ended with one + // so that we properly detect records ending between two read() runs + // cases are: \n|\n , \r\n|\r\n and \r\n\r|\n + // the case \r|\n\r\n is handled by the usual double-newline handling + if ((NL - Start) == 1 || ((NL - Start) == 2 && *Start == '\r')) + { + if (APT::String::Endswith(PartialMessage, "\n") || APT::String::Endswith(PartialMessage, "\r\n\r")) + { + PartialMessage.erase(PartialMessage.find_last_not_of("\r\n") + 1); + List.push_back(PartialMessage); + PartialMessage.clear(); + while (NL < End && (*NL == '\n' || *NL == '\r')) ++NL; + Start = NL; + } + } + } + + while (Start < End) { + char const * NL2 = (char const *) memchr(NL, '\n', End - NL); + if (NL2 == NULL) + { + // end of buffer: store what we have so far and read new data in + PartialMessage.append(Start, End - Start); + break; + } + ++NL2; + + // did we find a double newline? + if ((NL2 - NL) == 1 || ((NL2 - NL) == 2 && *NL == '\r')) + { + PartialMessage.append(Start, NL2 - Start); + PartialMessage.erase(PartialMessage.find_last_not_of("\r\n") + 1); + List.push_back(PartialMessage); + PartialMessage.clear(); + while (NL2 < End && (*NL2 == '\n' || *NL2 == '\r')) ++NL2; + Start = NL2; + } + NL = NL2; + } + + // we have read at least one complete message and nothing left + if (PartialMessage.empty() == true) + return true; + + if (WaitFd(Fd) == false) + return false; + } while (true); +} + /*}}}*/ +// MonthConv - Converts a month string into a number /*{{{*/ +// --------------------------------------------------------------------- +/* This was lifted from the boa webserver which lifted it from 'wn-v1.07' + Made it a bit more robust with a few tolower_ascii though. */ +static int MonthConv(char const * const Month) +{ + switch (tolower_ascii(*Month)) + { + case 'a': + return tolower_ascii(Month[1]) == 'p'?3:7; + case 'd': + return 11; + case 'f': + return 1; + case 'j': + if (tolower_ascii(Month[1]) == 'a') + return 0; + return tolower_ascii(Month[2]) == 'n'?5:6; + case 'm': + return tolower_ascii(Month[2]) == 'r'?2:4; + case 'n': + return 10; + case 'o': + return 9; + case 's': + return 8; + + // Pretend it is January.. + default: + return 0; + } +} + /*}}}*/ +// timegm - Internal timegm if the gnu version is not available /*{{{*/ +// --------------------------------------------------------------------- +/* Converts struct tm to time_t, assuming the data in tm is UTC rather + than local timezone (mktime assumes the latter). + + This function is a nonstandard GNU extension that is also present on + the BSDs and maybe other systems. For others we follow the advice of + the manpage of timegm and use his portable replacement. */ +#ifndef HAVE_TIMEGM +static time_t timegm(struct tm *t) +{ + char *tz = getenv("TZ"); + setenv("TZ", "", 1); + tzset(); + time_t ret = mktime(t); + if (tz) + setenv("TZ", tz, 1); + else + unsetenv("TZ"); + tzset(); + return ret; +} +#endif + /*}}}*/ +// RFC1123StrToTime - Converts an HTTP1.1 full date strings into a time_t /*{{{*/ +// --------------------------------------------------------------------- +/* tries to parses a full date as specified in RFC7231 §7.1.1.1 + with one exception: HTTP/1.1 valid dates need to have GMT as timezone. + As we encounter dates from UTC or with a numeric timezone in other places, + we allow them here to to be able to reuse the method. Either way, a date + must be in UTC or parsing will fail. Previous implementations of this + method used to ignore the timezone and assume always UTC. */ +bool RFC1123StrToTime(std::string const &str,time_t &time) +{ + unsigned short day = 0; + signed int year = 0; // yes, Y23K problem – we going to worry then… + std::string weekday, month, datespec, timespec, zone; + std::istringstream ss(str); + auto const &posix = std::locale::classic(); + ss.imbue(posix); + ss >> weekday; + // we only superficially check weekday, mostly to avoid accepting localized + // weekdays here and take only its length to decide which datetime format we + // encounter here. The date isn't stored. + std::transform(weekday.begin(), weekday.end(), weekday.begin(), ::tolower); + std::array<char const * const, 7> c_weekdays = {{ "sun", "mon", "tue", "wed", "thu", "fri", "sat" }}; + if (std::find(c_weekdays.begin(), c_weekdays.end(), weekday.substr(0,3)) == c_weekdays.end()) + return false; + + switch (weekday.length()) + { + case 4: + // Sun, 06 Nov 1994 08:49:37 GMT ; RFC 822, updated by RFC 1123 + if (weekday[3] != ',') + return false; + ss >> day >> month >> year >> timespec >> zone; + break; + case 3: + // Sun Nov 6 08:49:37 1994 ; ANSI C's asctime() format + ss >> month >> day >> timespec >> year; + zone = "UTC"; + break; + case 0: + case 1: + case 2: + return false; + default: + // Sunday, 06-Nov-94 08:49:37 GMT ; RFC 850, obsoleted by RFC 1036 + if (weekday[weekday.length() - 1] != ',') + return false; + ss >> datespec >> timespec >> zone; + auto const expldate = VectorizeString(datespec, '-'); + if (expldate.size() != 3) + return false; + try { + size_t pos; + day = std::stoi(expldate[0], &pos); + if (pos != expldate[0].length()) + return false; + year = 1900 + std::stoi(expldate[2], &pos); + if (pos != expldate[2].length()) + return false; + strprintf(datespec, "%.4d-%.2d-%.2d", year, MonthConv(expldate[1].c_str()) + 1, day); + } catch (...) { + return false; + } + break; + } + + if (ss.fail() || ss.bad() || !ss.eof()) + return false; + + if (zone != "GMT" && zone != "UTC" && zone != "Z") // RFC 822 + { + // numeric timezones as a should of RFC 1123 and generally preferred + try { + size_t pos; + auto const z = std::stoi(zone, &pos); + if (z != 0 || pos != zone.length()) + return false; + } catch (...) { + return false; + } + } + + if (datespec.empty()) + { + if (month.empty()) + return false; + strprintf(datespec, "%.4d-%.2d-%.2d", year, MonthConv(month.c_str()) + 1, day); + } + + std::string const datetime = datespec + ' ' + timespec; + struct tm Tm; + if (strptime(datetime.c_str(), "%Y-%m-%d %H:%M:%S", &Tm) == nullptr) + return false; + time = timegm(&Tm); + return true; +} + /*}}}*/ +// FTPMDTMStrToTime - Converts a ftp modification date into a time_t /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool FTPMDTMStrToTime(const char* const str,time_t &time) +{ + struct tm Tm; + // MDTM includes no whitespaces but recommend and ignored by strptime + if (strptime(str, "%Y %m %d %H %M %S", &Tm) == NULL) + return false; + + time = timegm(&Tm); + return true; +} + /*}}}*/ +// StrToNum - Convert a fixed length string to a number /*{{{*/ +// --------------------------------------------------------------------- +/* This is used in decoding the crazy fixed length string headers in + tar and ar files. */ +bool StrToNum(const char *Str,unsigned long &Res,unsigned Len,unsigned Base) +{ + unsigned long long BigRes; + if (not StrToNum(Str, BigRes, Len, Base)) + return false; + + if (std::numeric_limits<unsigned long>::max() < BigRes) + return false; + + Res = BigRes; + return true; +} + /*}}}*/ +// StrToNum - Convert a fixed length string to a number /*{{{*/ +// --------------------------------------------------------------------- +/* This is used in decoding the crazy fixed length string headers in + tar and ar files. */ +bool StrToNum(const char *Str,unsigned long long &Res,unsigned Len,unsigned Base) +{ + char S[30]; + if (Len >= sizeof(S)) + return false; + memcpy(S,Str,Len); + S[Len] = 0; + + // All spaces is a zero + Res = 0; + unsigned I; + for (I = 0; S[I] == ' '; ++I); + if (S[I] == 0) + return true; + if (S[I] == '-') + return false; + + char *End; + errno = 0; + Res = strtoull(S,&End,Base); + return not (End == S || errno != 0); +} + /*}}}*/ + +// Base256ToNum - Convert a fixed length binary to a number /*{{{*/ +// --------------------------------------------------------------------- +/* This is used in decoding the 256bit encoded fixed length fields in + tar files */ +bool Base256ToNum(const char *Str,unsigned long long &Res,unsigned int Len) +{ + if ((Str[0] & 0x80) == 0) + return false; + else + { + Res = Str[0] & 0x7F; + for(unsigned int i = 1; i < Len; ++i) + Res = (Res<<8) + Str[i]; + return true; + } +} + /*}}}*/ +// Base256ToNum - Convert a fixed length binary to a number /*{{{*/ +// --------------------------------------------------------------------- +/* This is used in decoding the 256bit encoded fixed length fields in + tar files */ +bool Base256ToNum(const char *Str,unsigned long &Res,unsigned int Len) +{ + unsigned long long Num = 0; + bool rc; + + rc = Base256ToNum(Str, Num, Len); + // rudimentary check for overflow (Res = ulong, Num = ulonglong) + Res = Num; + if (Res != Num) + return false; + + return rc; +} + /*}}}*/ +// HexDigit - Convert a hex character into an integer /*{{{*/ +// --------------------------------------------------------------------- +/* Helper for Hex2Num */ +static int HexDigit(int c) +{ + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + if (c >= 'A' && c <= 'F') + return c - 'A' + 10; + return -1; +} + /*}}}*/ +// Hex2Num - Convert a long hex number into a buffer /*{{{*/ +// --------------------------------------------------------------------- +/* The length of the buffer must be exactly 1/2 the length of the string. */ +bool Hex2Num(const APT::StringView Str,unsigned char *Num,unsigned int Length) +{ + if (Str.length() != Length*2) + return false; + + // Convert each digit. We store it in the same order as the string + int J = 0; + for (auto I = Str.begin(); I != Str.end();J++, I += 2) + { + int first_half = HexDigit(I[0]); + int second_half; + if (first_half < 0) + return false; + + second_half = HexDigit(I[1]); + if (second_half < 0) + return false; + Num[J] = first_half << 4; + Num[J] += second_half; + } + + return true; +} + /*}}}*/ +// TokSplitString - Split a string up by a given token /*{{{*/ +// --------------------------------------------------------------------- +/* This is intended to be a faster splitter, it does not use dynamic + memories. Input is changed to insert nulls at each token location. */ +bool TokSplitString(char Tok,char *Input,char **List, + unsigned long ListMax) +{ + // Strip any leading spaces + char *Start = Input; + char *Stop = Start + strlen(Start); + for (; *Start != 0 && isspace(*Start) != 0; Start++); + + unsigned long Count = 0; + char *Pos = Start; + while (Pos != Stop) + { + // Skip to the next Token + for (; Pos != Stop && *Pos != Tok; Pos++); + + // Back remove spaces + char *End = Pos; + for (; End > Start && (End[-1] == Tok || isspace(End[-1]) != 0); End--); + *End = 0; + + List[Count++] = Start; + if (Count >= ListMax) + { + List[Count-1] = 0; + return false; + } + + // Advance pos + for (; Pos != Stop && (*Pos == Tok || isspace(*Pos) != 0 || *Pos == 0); Pos++); + Start = Pos; + } + + List[Count] = 0; + return true; +} + /*}}}*/ +// VectorizeString - Split a string up into a vector of strings /*{{{*/ +// --------------------------------------------------------------------- +/* This can be used to split a given string up into a vector, so the + propose is the same as in the method above and this one is a bit slower + also, but the advantage is that we have an iteratable vector */ +vector<string> VectorizeString(string const &haystack, char const &split) +{ + vector<string> exploded; + if (haystack.empty() == true) + return exploded; + string::const_iterator start = haystack.begin(); + string::const_iterator end = start; + do { + for (; end != haystack.end() && *end != split; ++end); + exploded.push_back(string(start, end)); + start = end + 1; + } while (end != haystack.end() && (++end) != haystack.end()); + return exploded; +} + /*}}}*/ +// StringSplit - split a string into a string vector by token /*{{{*/ +// --------------------------------------------------------------------- +/* See header for details. + */ +vector<string> StringSplit(std::string const &s, std::string const &sep, + unsigned int maxsplit) +{ + vector<string> split; + size_t start, pos; + + // no separator given, this is bogus + if(sep.size() == 0) + return split; + + start = pos = 0; + while (pos != string::npos) + { + pos = s.find(sep, start); + split.push_back(s.substr(start, pos-start)); + + // if maxsplit is reached, the remaining string is the last item + if(split.size() >= maxsplit) + { + split[split.size()-1] = s.substr(start); + break; + } + start = pos+sep.size(); + } + return split; +} + /*}}}*/ +// RegexChoice - Simple regex list/list matcher /*{{{*/ +// --------------------------------------------------------------------- +/* */ +unsigned long RegexChoice(RxChoiceList *Rxs,const char **ListBegin, + const char **ListEnd) +{ + for (RxChoiceList *R = Rxs; R->Str != 0; R++) + R->Hit = false; + + unsigned long Hits = 0; + for (; ListBegin < ListEnd; ++ListBegin) + { + // Check if the name is a regex + const char *I; + bool Regex = true; + for (I = *ListBegin; *I != 0; I++) + if (*I == '.' || *I == '?' || *I == '*' || *I == '|') + break; + if (*I == 0) + Regex = false; + + // Compile the regex pattern + regex_t Pattern; + if (Regex == true) + if (regcomp(&Pattern,*ListBegin,REG_EXTENDED | REG_ICASE | + REG_NOSUB) != 0) + Regex = false; + + // Search the list + bool Done = false; + for (RxChoiceList *R = Rxs; R->Str != 0; R++) + { + if (R->Str[0] == 0) + continue; + + if (strcasecmp(R->Str,*ListBegin) != 0) + { + if (Regex == false) + continue; + if (regexec(&Pattern,R->Str,0,0,0) != 0) + continue; + } + Done = true; + + if (R->Hit == false) + Hits++; + + R->Hit = true; + } + + if (Regex == true) + regfree(&Pattern); + + if (Done == false) + _error->Warning(_("Selection %s not found"),*ListBegin); + } + + return Hits; +} + /*}}}*/ +// {str,io}printf - C format string outputter to C++ strings/iostreams /*{{{*/ +// --------------------------------------------------------------------- +/* This is used to make the internationalization strings easier to translate + and to allow reordering of parameters */ +bool iovprintf(std::ostream &out, const char *format, + va_list &args, ssize_t &size) { + auto S = std::unique_ptr<char,decltype(&free)>{static_cast<char*>(malloc(size)), &free}; + ssize_t const n = vsnprintf(S.get(), size, format, args); + if (n > -1 && n < size) { + out << S.get(); + return true; + } else { + if (n > -1) + size = n + 1; + else + size *= 2; + } + return false; +} +void ioprintf(ostream &out,const char *format,...) +{ + va_list args; + ssize_t size = 400; + while (true) { + bool ret; + va_start(args,format); + ret = iovprintf(out, format, args, size); + va_end(args); + if (ret == true) + return; + } +} +void strprintf(string &out,const char *format,...) +{ + va_list args; + ssize_t size = 400; + std::ostringstream outstr; + while (true) { + bool ret; + va_start(args,format); + ret = iovprintf(outstr, format, args, size); + va_end(args); + if (ret == true) + break; + } + out = outstr.str(); +} + /*}}}*/ +// safe_snprintf - Safer snprintf /*{{{*/ +// --------------------------------------------------------------------- +/* This is a snprintf that will never (ever) go past 'End' and returns a + pointer to the end of the new string. The returned string is always null + terminated unless Buffer == end. This is a better alterantive to using + consecutive snprintfs. */ +char *safe_snprintf(char *Buffer,char *End,const char *Format,...) +{ + va_list args; + int Did; + + if (End <= Buffer) + return End; + va_start(args,Format); + Did = vsnprintf(Buffer,End - Buffer,Format,args); + va_end(args); + + if (Did < 0 || Buffer + Did > End) + return End; + return Buffer + Did; +} + /*}}}*/ +// StripEpoch - Remove the version "epoch" from a version string /*{{{*/ +// --------------------------------------------------------------------- +string StripEpoch(const string &VerStr) +{ + size_t i = VerStr.find(":"); + if (i == string::npos) + return VerStr; + return VerStr.substr(i+1); +} + /*}}}*/ + +// tolower_ascii - tolower() function that ignores the locale /*{{{*/ +// --------------------------------------------------------------------- +/* This little function is the most called method we have and tries + therefore to do the absolute minimum - and is notable faster than + standard tolower/toupper and as a bonus avoids problems with different + locales - we only operate on ascii chars anyway. */ +#undef tolower_ascii +int tolower_ascii(int const c) APT_PURE APT_COLD; +int tolower_ascii(int const c) +{ + return tolower_ascii_inline(c); +} + /*}}}*/ + +// isspace_ascii - isspace() function that ignores the locale /*{{{*/ +// --------------------------------------------------------------------- +/* This little function is one of the most called methods we have and tries + therefore to do the absolute minimum - and is notable faster than + standard isspace() and as a bonus avoids problems with different + locales - we only operate on ascii chars anyway. */ +#undef isspace_ascii +int isspace_ascii(int const c) APT_PURE APT_COLD; +int isspace_ascii(int const c) +{ + return isspace_ascii_inline(c); +} + /*}}}*/ + +// CheckDomainList - See if Host is in a , separate list /*{{{*/ +// --------------------------------------------------------------------- +/* The domain list is a comma separate list of domains that are suffix + matched against the argument */ +bool CheckDomainList(const string &Host,const string &List) +{ + string::const_iterator Start = List.begin(); + for (string::const_iterator Cur = List.begin(); Cur <= List.end(); ++Cur) + { + if (Cur < List.end() && *Cur != ',') + continue; + + // Match the end of the string.. + if ((Host.size() >= (unsigned)(Cur - Start)) && + Cur - Start != 0 && + stringcasecmp(Host.end() - (Cur - Start),Host.end(),Start,Cur) == 0) + return true; + + Start = Cur + 1; + } + return false; +} + /*}}}*/ +// strv_length - Return the length of a NULL-terminated string array /*{{{*/ +// --------------------------------------------------------------------- +/* */ +size_t strv_length(const char **str_array) +{ + size_t i; + for (i=0; str_array[i] != NULL; i++) + /* nothing */ + ; + return i; +} + /*}}}*/ +// DeEscapeString - unescape (\0XX and \xXX) from a string /*{{{*/ +// --------------------------------------------------------------------- +/* */ +string DeEscapeString(const string &input) +{ + char tmp[3]; + string::const_iterator it; + string output; + for (it = input.begin(); it != input.end(); ++it) + { + // just copy non-escape chars + if (*it != '\\') + { + output += *it; + continue; + } + + // deal with double escape + if (*it == '\\' && + (it + 1 < input.end()) && it[1] == '\\') + { + // copy + output += *it; + // advance iterator one step further + ++it; + continue; + } + + // ensure we have a char to read + if (it + 1 == input.end()) + continue; + + // read it + ++it; + switch (*it) + { + case '0': + if (it + 2 < input.end()) { + tmp[0] = it[1]; + tmp[1] = it[2]; + tmp[2] = 0; + output += (char)strtol(tmp, 0, 8); + it += 2; + } else { + // FIXME: raise exception here? + } + break; + case 'x': + if (it + 2 < input.end()) { + tmp[0] = it[1]; + tmp[1] = it[2]; + tmp[2] = 0; + output += (char)strtol(tmp, 0, 16); + it += 2; + } else { + // FIXME: raise exception here? + } + break; + default: + // FIXME: raise exception here? + break; + } + } + return output; +} + /*}}}*/ +// URI::CopyFrom - Copy from an object /*{{{*/ +// --------------------------------------------------------------------- +/* This parses the URI into all of its components */ +void URI::CopyFrom(const string &U) +{ + string::const_iterator I = U.begin(); + + // Locate the first colon, this separates the scheme + for (; I < U.end() && *I != ':' ; ++I); + string::const_iterator FirstColon = I; + + /* Determine if this is a host type URI with a leading double // + and then search for the first single / */ + string::const_iterator SingleSlash = I; + if (I + 3 < U.end() && I[1] == '/' && I[2] == '/') + SingleSlash += 3; + + /* Find the / indicating the end of the hostname, ignoring /'s in the + square brackets */ + bool InBracket = false; + for (; SingleSlash < U.end() && (*SingleSlash != '/' || InBracket == true); ++SingleSlash) + { + if (*SingleSlash == '[') + InBracket = true; + if (InBracket == true && *SingleSlash == ']') + InBracket = false; + } + + if (SingleSlash > U.end()) + SingleSlash = U.end(); + + // We can now write the access and path specifiers + Access.assign(U.begin(),FirstColon); + if (SingleSlash != U.end()) + Path.assign(SingleSlash,U.end()); + if (Path.empty() == true) + Path = "/"; + + // Now we attempt to locate a user:pass@host fragment + if (FirstColon + 2 <= U.end() && FirstColon[1] == '/' && FirstColon[2] == '/') + FirstColon += 3; + else + FirstColon += 1; + if (FirstColon >= U.end()) + return; + + if (FirstColon > SingleSlash) + FirstColon = SingleSlash; + + // Find the colon... + I = FirstColon + 1; + if (I > SingleSlash) + I = SingleSlash; + + // Search for the @ separating user:pass from host + auto const RevAt = std::find( + std::string::const_reverse_iterator(SingleSlash), + std::string::const_reverse_iterator(I), '@'); + string::const_iterator const At = RevAt.base() == I ? SingleSlash : std::prev(RevAt.base()); + // and then look for the colon between user and pass + string::const_iterator const SecondColon = std::find(I, At, ':'); + + // Now write the host and user/pass + if (At == SingleSlash) + { + if (FirstColon < SingleSlash) + Host.assign(FirstColon,SingleSlash); + } + else + { + Host.assign(At+1,SingleSlash); + // username and password must be encoded (RFC 3986) + User.assign(DeQuoteString(FirstColon,SecondColon)); + if (SecondColon < At) + Password.assign(DeQuoteString(SecondColon+1,At)); + } + + // Now we parse the RFC 2732 [] hostnames. + unsigned long PortEnd = 0; + InBracket = false; + for (unsigned I = 0; I != Host.length();) + { + if (Host[I] == '[') + { + InBracket = true; + Host.erase(I,1); + continue; + } + + if (InBracket == true && Host[I] == ']') + { + InBracket = false; + Host.erase(I,1); + PortEnd = I; + continue; + } + I++; + } + + // Tsk, weird. + if (InBracket == true) + { + Host.clear(); + return; + } + + // Now we parse off a port number from the hostname + Port = 0; + string::size_type Pos = Host.rfind(':'); + if (Pos == string::npos || Pos < PortEnd) + return; + + Port = atoi(string(Host,Pos+1).c_str()); + Host.assign(Host,0,Pos); +} + /*}}}*/ +// URI::operator string - Convert the URI to a string /*{{{*/ +// --------------------------------------------------------------------- +/* */ +URI::operator string() +{ + std::stringstream Res; + + if (Access.empty() == false) + Res << Access << ':'; + + if (Host.empty() == false) + { + if (Access.empty() == false) + Res << "//"; + + if (User.empty() == false) + { + // FIXME: Technically userinfo is permitted even less + // characters than these, but this is not conveniently + // expressed with a denylist. + Res << QuoteString(User, ":/?#[]@"); + if (Password.empty() == false) + Res << ":" << QuoteString(Password, ":/?#[]@"); + Res << "@"; + } + + // Add RFC 2732 escaping characters + if (Access.empty() == false && Host.find_first_of("/:") != string::npos) + Res << '[' << Host << ']'; + else + Res << Host; + + if (Port != 0) + Res << ':' << std::to_string(Port); + } + + if (Path.empty() == false) + { + if (Path[0] != '/') + Res << "/" << Path; + else + Res << Path; + } + + return Res.str(); +} + /*}}}*/ +// URI::SiteOnly - Return the schema and site for the URI /*{{{*/ +string URI::SiteOnly(const string &URI) +{ + ::URI U(URI); + U.User.clear(); + U.Password.clear(); + U.Path.clear(); + return U; +} + /*}}}*/ +// URI::ArchiveOnly - Return the schema, site and cleaned path for the URI /*{{{*/ +string URI::ArchiveOnly(const string &URI) +{ + ::URI U(URI); + U.User.clear(); + U.Password.clear(); + if (U.Path.empty() == false && U.Path[U.Path.length() - 1] == '/') + U.Path.erase(U.Path.length() - 1); + return U; +} + /*}}}*/ +// URI::NoUserPassword - Return the schema, site and path for the URI /*{{{*/ +string URI::NoUserPassword(const string &URI) +{ + ::URI U(URI); + U.User.clear(); + U.Password.clear(); + return U; +} + /*}}}*/ diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h new file mode 100644 index 0000000..1fdc8dc --- /dev/null +++ b/apt-pkg/contrib/strutl.h @@ -0,0 +1,252 @@ +// -*- mode: cpp; mode: fold -*- +// SPDX-License-Identifier: GPL-2.0+ +// Description /*{{{*/ +/* ###################################################################### + + String Util - These are some useful string functions + + _strstrip is a function to remove whitespace from the front and end + of a string. + + This file had this historic note, but now includes further changes + under the GPL-2.0+: + + This source is placed in the Public Domain, do with it what you will + It was originally written by Jason Gunthorpe <jgg@gpu.srv.ualberta.ca> + + ##################################################################### */ + /*}}}*/ +#ifndef STRUTL_H +#define STRUTL_H + +#include <cstring> +#include <iostream> +#include <limits> +#include <string> +#include <vector> +#include <apt-pkg/string_view.h> +#include <stddef.h> +#include <time.h> + +#include "macros.h" + + +namespace APT { + namespace String { + APT_PUBLIC std::string Strip(const std::string &s); + APT_PUBLIC bool Endswith(const std::string &s, const std::string &ending); + APT_PUBLIC bool Startswith(const std::string &s, const std::string &starting); + APT_PUBLIC std::string Join(std::vector<std::string> list, const std::string &sep); + // Returns string display length honoring multi-byte characters + APT_PUBLIC size_t DisplayLength(StringView str); + } +} + + +APT_PUBLIC bool UTF8ToCodeset(const char *codeset, const std::string &orig, std::string *dest); +APT_PUBLIC char *_strstrip(char *String); +APT_PUBLIC char *_strrstrip(char *String); // right strip only +APT_DEPRECATED_MSG("Use SubstVar to avoid memory headaches") APT_PUBLIC char *_strtabexpand(char *String,size_t Len); +APT_PUBLIC bool ParseQuoteWord(const char *&String,std::string &Res); +APT_PUBLIC bool ParseCWord(const char *&String,std::string &Res); +APT_PUBLIC std::string QuoteString(const std::string &Str,const char *Bad); +APT_PUBLIC std::string DeQuoteString(const std::string &Str); +APT_PUBLIC std::string DeQuoteString(std::string::const_iterator const &begin, std::string::const_iterator const &end); + +// unescape (\0XX and \xXX) from a string +APT_PUBLIC std::string DeEscapeString(const std::string &input); + +APT_PUBLIC std::string SizeToStr(double Bytes); +APT_PUBLIC std::string TimeToStr(unsigned long Sec); +APT_PUBLIC std::string Base64Encode(const std::string &Str); +APT_PUBLIC std::string OutputInDepth(const unsigned long Depth, const char* Separator=" "); +APT_PUBLIC std::string URItoFileName(const std::string &URI); +/** returns a datetime string as needed by HTTP/1.1 and Debian files. + * + * Note: The date will always be represented in a UTC timezone + * + * @param Date to be represented as a string + * @param NumericTimezone is preferred in general, but HTTP/1.1 requires the use + * of GMT as timezone instead. \b true means that the timezone should be denoted + * as "+0000" while \b false uses "GMT". + */ +APT_PUBLIC std::string TimeRFC1123(time_t Date, bool const NumericTimezone); +/** parses time as needed by HTTP/1.1 and Debian files. + * + * HTTP/1.1 prefers dates in RFC1123 format (but the other two obsolete date formats + * are supported to) and e.g. Release files use the same format in Date & Valid-Until + * fields. + * + * Note: datetime strings need to be in UTC timezones (GMT, UTC, Z, +/-0000) to be + * parsed. Other timezones will be rejected as invalid. Previous implementations + * accepted other timezones, but treated them as UTC. + * + * @param str is the datetime string to parse + * @param[out] time will be the seconds since epoch of the given datetime if + * parsing is successful, undefined otherwise. + * @return \b true if parsing was successful, otherwise \b false. + */ +APT_PUBLIC bool RFC1123StrToTime(const std::string &str,time_t &time) APT_MUSTCHECK; +APT_PUBLIC bool FTPMDTMStrToTime(const char* const str,time_t &time) APT_MUSTCHECK; +APT_PUBLIC std::string LookupTag(const std::string &Message,const char *Tag,const char *Default = 0); +APT_PUBLIC int StringToBool(const std::string &Text,int Default = -1); +APT_PUBLIC bool ReadMessages(int Fd, std::vector<std::string> &List); +APT_PUBLIC bool StrToNum(const char *Str,unsigned long &Res,unsigned Len,unsigned Base = 0); +APT_PUBLIC bool StrToNum(const char *Str,unsigned long long &Res,unsigned Len,unsigned Base = 0); +APT_PUBLIC bool Base256ToNum(const char *Str,unsigned long &Res,unsigned int Len); +APT_PUBLIC bool Base256ToNum(const char *Str,unsigned long long &Res,unsigned int Len); +APT_PUBLIC bool Hex2Num(const APT::StringView Str,unsigned char *Num,unsigned int Length); +// input changing string split +APT_PUBLIC bool TokSplitString(char Tok,char *Input,char **List, + unsigned long ListMax); + +// split a given string by a char +APT_PUBLIC std::vector<std::string> VectorizeString(std::string const &haystack, char const &split) APT_PURE; + +/* \brief Return a vector of strings from string "input" where "sep" + * is used as the delimiter string. + * + * \param input The input string. + * + * \param sep The separator to use. + * + * \param maxsplit (optional) The maximum amount of splitting that + * should be done . + * + * The optional "maxsplit" argument can be used to limit the splitting, + * if used the string is only split on maxsplit places and the last + * item in the vector contains the remainder string. + */ +APT_PUBLIC std::vector<std::string> StringSplit(std::string const &input, + std::string const &sep, + unsigned int maxsplit=std::numeric_limits<unsigned int>::max()) APT_PURE; + + +APT_HIDDEN bool iovprintf(std::ostream &out, const char *format, va_list &args, ssize_t &size); +APT_PUBLIC void ioprintf(std::ostream &out,const char *format,...) APT_PRINTF(2); +APT_PUBLIC void strprintf(std::string &out,const char *format,...) APT_PRINTF(2); +APT_PUBLIC char *safe_snprintf(char *Buffer,char *End,const char *Format,...) APT_PRINTF(3); +APT_PUBLIC bool CheckDomainList(const std::string &Host, const std::string &List); + +/* Do some compat mumbo jumbo */ +#define tolower_ascii tolower_ascii_inline +#define isspace_ascii isspace_ascii_inline + +APT_PURE APT_HOT +static inline int tolower_ascii_unsafe(int const c) +{ + return c | 0x20; +} +APT_PURE APT_HOT +static inline int tolower_ascii_inline(int const c) +{ + return (c >= 'A' && c <= 'Z') ? c + 32 : c; +} +APT_PURE APT_HOT +static inline int isspace_ascii_inline(int const c) +{ + // 9='\t',10='\n',11='\v',12='\f',13='\r',32=' ' + return (c >= 9 && c <= 13) || c == ' '; +} +APT_PURE APT_HOT +static inline int islower_ascii(int const c) +{ + return c >= 'a' && c <= 'z'; +} +APT_PURE APT_HOT +static inline int isupper_ascii(int const c) +{ + return c >= 'A' && c <= 'Z'; +} +APT_PURE APT_HOT +static inline int isalpha_ascii(int const c) +{ + return isupper_ascii(c) || islower_ascii(c); +} + +APT_PUBLIC std::string StripEpoch(const std::string &VerStr); + +#define APT_MKSTRCMP(name,func) \ +inline APT_PURE int name(const char *A,const char *B) {return func(A,A+strlen(A),B,B+strlen(B));} \ +inline APT_PURE int name(const char *A,const char *AEnd,const char *B) {return func(A,AEnd,B,B+strlen(B));} \ +inline APT_PURE int name(const std::string& A,const char *B) {return func(A.c_str(),A.c_str()+A.length(),B,B+strlen(B));} \ +inline APT_PURE int name(const std::string& A,const std::string& B) {return func(A.c_str(),A.c_str()+A.length(),B.c_str(),B.c_str()+B.length());} \ +inline APT_PURE int name(const std::string& A,const char *B,const char *BEnd) {return func(A.c_str(),A.c_str()+A.length(),B,BEnd);} + +#define APT_MKSTRCMP2(name,func) \ +inline APT_PURE int name(const char *A,const char *AEnd,const char *B) {return func(A,AEnd,B,B+strlen(B));} \ +inline APT_PURE int name(const std::string& A,const char *B) {return func(A.begin(),A.end(),B,B+strlen(B));} \ +inline APT_PURE int name(const std::string& A,const std::string& B) {return func(A.begin(),A.end(),B.begin(),B.end());} \ +inline APT_PURE int name(const std::string& A,const char *B,const char *BEnd) {return func(A.begin(),A.end(),B,BEnd);} + +APT_PUBLIC int APT_PURE stringcmp(const char *A,const char *AEnd,const char *B,const char *BEnd); +APT_PUBLIC int APT_PURE stringcasecmp(const char *A,const char *AEnd,const char *B,const char *BEnd); + +/* We assume that GCC 3 indicates that libstdc++3 is in use too. In that + case the definition of string::const_iterator is not the same as + const char * and we need these extra functions */ +#if __GNUC__ >= 3 +APT_PUBLIC int APT_PURE stringcmp(std::string::const_iterator A,std::string::const_iterator AEnd, + const char *B,const char *BEnd); +APT_PUBLIC int APT_PURE stringcmp(std::string::const_iterator A,std::string::const_iterator AEnd, + std::string::const_iterator B,std::string::const_iterator BEnd); +APT_PUBLIC int APT_PURE stringcasecmp(std::string::const_iterator A,std::string::const_iterator AEnd, + const char *B,const char *BEnd); +APT_PUBLIC int APT_PURE stringcasecmp(std::string::const_iterator A,std::string::const_iterator AEnd, + std::string::const_iterator B,std::string::const_iterator BEnd); + +inline APT_PURE int stringcmp(std::string::const_iterator A,std::string::const_iterator Aend,const char *B) {return stringcmp(A,Aend,B,B+strlen(B));} +inline APT_PURE int stringcasecmp(std::string::const_iterator A,std::string::const_iterator Aend,const char *B) {return stringcasecmp(A,Aend,B,B+strlen(B));} +#endif + +APT_MKSTRCMP2(stringcmp,stringcmp) +APT_MKSTRCMP2(stringcasecmp,stringcasecmp) + +// Return the length of a NULL-terminated string array +APT_PUBLIC size_t APT_PURE strv_length(const char **str_array); + + +inline const char *DeNull(const char *s) {return (s == 0?"(null)":s);} + +class APT_PUBLIC URI +{ + void CopyFrom(const std::string &From); + + public: + + std::string Access; + std::string User; + std::string Password; + std::string Host; + std::string Path; + unsigned int Port; + + operator std::string(); + inline void operator =(const std::string &From) {CopyFrom(From);} + inline bool empty() {return Access.empty();}; + static std::string SiteOnly(const std::string &URI); + static std::string ArchiveOnly(const std::string &URI); + static std::string NoUserPassword(const std::string &URI); + + explicit URI(std::string Path) { CopyFrom(Path); } + URI() : Port(0) {} +}; + +struct SubstVar +{ + const char *Subst; + const std::string *Contents; +}; +APT_PUBLIC std::string SubstVar(std::string Str,const struct SubstVar *Vars); +APT_PUBLIC std::string SubstVar(const std::string &Str,const std::string &Subst,const std::string &Contents); + +struct RxChoiceList +{ + void *UserData; + const char *Str; + bool Hit; +}; +APT_PUBLIC unsigned long RegexChoice(RxChoiceList *Rxs,const char **ListBegin, + const char **ListEnd); + +#endif diff --git a/apt-pkg/contrib/weakptr.h b/apt-pkg/contrib/weakptr.h new file mode 100644 index 0000000..8de727d --- /dev/null +++ b/apt-pkg/contrib/weakptr.h @@ -0,0 +1,64 @@ +/* weakptr.h - An object which supports weak pointers. + * + * Copyright (C) 2010 Julian Andres Klode <jak@debian.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#ifndef WEAK_POINTER_H +#define WEAK_POINTER_H + +#include <set> +#include <stddef.h> + +/** + * Class for objects providing support for weak pointers. + * + * This class allows for the registration of certain pointers as weak, + * which will cause them to be set to NULL when the destructor of the + * object is called. + */ +class WeakPointable { +private: + std::set<WeakPointable**> pointers; + +public: + + /** + * Add a new weak pointer. + */ + inline void AddWeakPointer(WeakPointable** weakptr) { + pointers.insert(weakptr); + } + + /** + * Remove the weak pointer from the list of weak pointers. + */ + inline void RemoveWeakPointer(WeakPointable **weakptr) { + pointers.erase(weakptr); + } + + /** + * Deconstruct the object, set all weak pointers to NULL. + */ + ~WeakPointable() { + std::set<WeakPointable**>::iterator iter = pointers.begin(); + while (iter != pointers.end()) + **(iter++) = NULL; + } +}; + +#endif // WEAK_POINTER_H diff --git a/apt-pkg/deb/debfile.cc b/apt-pkg/deb/debfile.cc new file mode 100644 index 0000000..645a579 --- /dev/null +++ b/apt-pkg/deb/debfile.cc @@ -0,0 +1,274 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Archive File (.deb) + + .DEB archives are AR files containing two tars and an empty marker + member called 'debian-binary'. The two tars contain the meta data and + the actual archive contents. Thus this class is a very simple wrapper + around ar/tar to simply extract the right tar files. + + It also uses the deb package list parser to parse the control file + into the cache. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/arfile.h> +#include <apt-pkg/debfile.h> +#include <apt-pkg/dirstream.h> +#include <apt-pkg/error.h> +#include <apt-pkg/extracttar.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile.h> + +#include <algorithm> +#include <string> +#include <sstream> +#include <vector> +#include <string.h> +#include <sys/stat.h> + +#include <apti18n.h> + /*}}}*/ + +// DebFile::debDebFile - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Open the AR file and check for consistency */ +debDebFile::debDebFile(FileFd &File) : File(File), AR(File) +{ + if (_error->PendingError() == true) + return; + + if (!CheckMember("debian-binary")) { + _error->Error(_("This is not a valid DEB archive, missing '%s' member"), "debian-binary"); + return; + } + + if (!CheckMember("control.tar") && + !CheckMember("control.tar.gz") && + !CheckMember("control.tar.xz") && + !CheckMember("control.tar.zst")) + { + _error->Error(_("This is not a valid DEB archive, missing '%s' member"), "control.tar"); + return; + } + + if (!CheckMember("data.tar") && + !CheckMember("data.tar.gz") && + !CheckMember("data.tar.bz2") && + !CheckMember("data.tar.lzma") && + !CheckMember("data.tar.xz") && + !CheckMember("data.tar.zst")) + { + _error->Error(_("This is not a valid DEB archive, missing '%s' member"), "data.tar"); + return; + } +} + /*}}}*/ +// DebFile::CheckMember - Check if a named member is in the archive /*{{{*/ +// --------------------------------------------------------------------- +/* This is used to check for a correct deb and to give nicer error messages + for people playing around. */ +bool debDebFile::CheckMember(const char *Name) +{ + if (AR.FindMember(Name) == 0) + return false; + return true; +} + /*}}}*/ +// DebFile::GotoMember - Jump to a Member /*{{{*/ +// --------------------------------------------------------------------- +/* Jump in the file to the start of a named member and return the information + about that member. The caller can then read from the file up to the + returned size. Note, since this relies on the file position this is + a destructive operation, it also changes the last returned Member + structure - so don't nest them! */ +const ARArchive::Member *debDebFile::GotoMember(const char *Name) +{ + // Get the archive member and positition the file + const ARArchive::Member *Member = AR.FindMember(Name); + if (Member == 0) + { + return 0; + } + if (File.Seek(Member->Start) == false) + return 0; + + return Member; +} + /*}}}*/ +// DebFile::ExtractTarMember - Extract the contents of a tar member /*{{{*/ +// --------------------------------------------------------------------- +/* Simple wrapper around tar.. */ +bool debDebFile::ExtractTarMember(pkgDirStream &Stream,const char *Name) +{ + std::string Compressor; + auto const Compressors = APT::Configuration::getCompressors(); + + ARArchive::Member const *Member = AR.FindMember(Name); + if (Member != nullptr) + { + auto const found = std::find_if(Compressors.cbegin(), Compressors.cend(), [&](auto const &c) { + return not c.Extension.empty() && APT::String::Endswith(Name, c.Extension); + }); + if (found != Compressors.cend()) + Compressor = found->Name; + } + else + { + for (auto const &c : Compressors) + { + if (c.Extension.empty()) + continue; + Member = AR.FindMember(std::string(Name).append(c.Extension).c_str()); + if (Member == nullptr) + continue; + Compressor = c.Name; + break; + } + } + + if (Member == nullptr) + { + std::ostringstream ext; + ext << Name << '{'; + for (auto const &c : Compressors) + if (not c.Extension.empty()) + ext << c.Extension << ','; + ext << '}'; + return _error->Error(_("Internal error, could not locate member %s"), ext.str().c_str()); + } + + if (not File.Seek(Member->Start)) + return false; + + ExtractTar Tar(File,Member->Size,Compressor); + if (_error->PendingError()) + return false; + return Tar.Go(Stream); +} + /*}}}*/ +// DebFile::ExtractArchive - Extract the archive data itself /*{{{*/ +// --------------------------------------------------------------------- +/* Simple wrapper around DebFile::ExtractTarMember. */ +bool debDebFile::ExtractArchive(pkgDirStream &Stream) +{ + return ExtractTarMember(Stream, "data.tar"); +} + /*}}}*/ + +// DebFile::ControlExtract::DoItem - Control Tar Extraction /*{{{*/ +// --------------------------------------------------------------------- +/* This directory stream handler for the control tar handles extracting + it into the temporary meta directory. It only extracts files, it does + not create directories, links or anything else. */ +bool debDebFile::ControlExtract::DoItem(Item &Itm,int &Fd) +{ + if (Itm.Type != Item::File) + return true; + + /* Cleanse the file name, prevent people from trying to unpack into + absolute paths, .., etc */ + for (char *I = Itm.Name; *I != 0; I++) + if (*I == '/') + *I = '_'; + + /* Force the ownership to be root and ensure correct permissions, + go-w, the rest are left untouched */ + Itm.UID = 0; + Itm.GID = 0; + Itm.Mode &= ~(S_IWGRP | S_IWOTH); + + return pkgDirStream::DoItem(Itm,Fd); +} + /*}}}*/ + +// MemControlExtract::DoItem - Check if it is the control file /*{{{*/ +// --------------------------------------------------------------------- +/* This sets up to extract the control block member file into a memory + block of just the right size. All other files go into the bit bucket. */ + +// Upper size limit for control files. Two reasons for having a limit here: +// +// 1. We read those files into memory and want to avoid being killed by OOM +// +// 2. We allocate (Itm.Size+2)-large arrays, so this can overflow if Itm.Size +// becomes 2**64-2 or larger. This is obviously +// +// 64 MiB seems like a terribly large size that everyone should be happy with. +static const unsigned long long DEB_CONTROL_SIZE_LIMIT = 64 * 1024 * 1024; +bool debDebFile::MemControlExtract::DoItem(Item &Itm,int &Fd) +{ + // At the control file, allocate buffer memory. + if (Member == Itm.Name) + { + if (Itm.Size > DEB_CONTROL_SIZE_LIMIT) + return _error->Error("Control file too large: %llu > %llu bytes", Itm.Size, DEB_CONTROL_SIZE_LIMIT); + delete [] Control; + Control = new char[Itm.Size+2]; + IsControl = true; + Fd = -2; // Signal to pass to Process + Length = Itm.Size; + } + else + IsControl = false; + + return true; +} + /*}}}*/ +// MemControlExtract::Process - Process extracting the control file /*{{{*/ +// --------------------------------------------------------------------- +/* Just memcopy the block from the tar extractor and put it in the right + place in the pre-allocated memory block. */ +bool debDebFile::MemControlExtract::Process(Item &/*Itm*/,const unsigned char *Data, + unsigned long long Size,unsigned long long Pos) +{ + memcpy(Control + Pos, Data,Size); + return true; +} + /*}}}*/ +// MemControlExtract::Read - Read the control information from the deb /*{{{*/ +// --------------------------------------------------------------------- +/* This uses the internal tar extractor to fetch the control file, and then + it parses it into a tag section parser. */ +bool debDebFile::MemControlExtract::Read(debDebFile &Deb) +{ + if (Deb.ExtractTarMember(*this, "control.tar") == false) + return false; + + if (Control == 0) + return true; + + Control[Length] = '\n'; + Control[Length+1] = '\n'; + if (Section.Scan(Control,Length+2) == false) + return _error->Error(_("Unparsable control file")); + return true; +} + /*}}}*/ +// MemControlExtract::TakeControl - Parse a memory block /*{{{*/ +// --------------------------------------------------------------------- +/* The given memory block is loaded into the parser and parsed as a control + record. */ +bool debDebFile::MemControlExtract::TakeControl(const void *Data,unsigned long long Size) +{ + if (Size > DEB_CONTROL_SIZE_LIMIT) + return _error->Error("Control file too large: %llu > %llu bytes", Size, DEB_CONTROL_SIZE_LIMIT); + + delete [] Control; + Control = new char[Size+2]; + Length = Size; + memcpy(Control,Data,Size); + + Control[Length] = '\n'; + Control[Length+1] = '\n'; + return Section.Scan(Control,Length+2); +} + /*}}}*/ + diff --git a/apt-pkg/deb/debfile.h b/apt-pkg/deb/debfile.h new file mode 100644 index 0000000..48a75ae --- /dev/null +++ b/apt-pkg/deb/debfile.h @@ -0,0 +1,89 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Archive File (.deb) + + This Class handles all the operations performed directly on .deb + files. It makes use of the AR and TAR classes to give the necessary + external interface. + + There are only two things that can be done with a raw package, + extract it's control information and extract the contents itself. + + This should probably subclass an as-yet unwritten super class to + produce a generic archive mechanism. + + The memory control file extractor is useful to extract a single file + into memory from the control.tar.gz + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_DEBFILE_H +#define PKGLIB_DEBFILE_H + +#include <apt-pkg/arfile.h> +#include <apt-pkg/dirstream.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/tagfile.h> + +#include <string> + + +class FileFd; + +class APT_PUBLIC debDebFile +{ + protected: + + FileFd &File; + ARArchive AR; + + bool CheckMember(const char *Name); + + public: + class ControlExtract; + class MemControlExtract; + + bool ExtractTarMember(pkgDirStream &Stream, const char *Name); + bool ExtractArchive(pkgDirStream &Stream); + const ARArchive::Member *GotoMember(const char *Name); + inline FileFd &GetFile() {return File;}; + + explicit debDebFile(FileFd &File); +}; + +class APT_PUBLIC debDebFile::ControlExtract : public pkgDirStream +{ + public: + + virtual bool DoItem(Item &Itm,int &Fd) APT_OVERRIDE; +}; + +class APT_PUBLIC debDebFile::MemControlExtract : public pkgDirStream +{ + bool IsControl; + + public: + + char *Control; + pkgTagSection Section; + unsigned long Length; + std::string Member; + + // Members from DirStream + virtual bool DoItem(Item &Itm,int &Fd) APT_OVERRIDE; + virtual bool Process(Item &Itm,const unsigned char *Data, + unsigned long long Size,unsigned long long Pos) APT_OVERRIDE; + + // Helpers + bool Read(debDebFile &Deb); + bool TakeControl(const void *Data,unsigned long long Size); + + MemControlExtract() : IsControl(false), Control(0), Length(0), Member("control") {}; + explicit MemControlExtract(std::string Member) : IsControl(false), Control(0), Length(0), Member(Member) {}; + ~MemControlExtract() {delete [] Control;}; +}; + /*}}}*/ + +#endif diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc new file mode 100644 index 0000000..e1698e1 --- /dev/null +++ b/apt-pkg/deb/debindexfile.cc @@ -0,0 +1,421 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Specific sources.list types and the three sorts of Debian + index files. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apti18n.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/debfile.h> +#include <apt-pkg/debindexfile.h> +#include <apt-pkg/deblistparser.h> +#include <apt-pkg/debrecords.h> +#include <apt-pkg/debsrcrecords.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/indexfile.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgrecords.h> +#include <apt-pkg/srcrecords.h> +#include <apt-pkg/strutl.h> + +#include <iostream> +#include <memory> +#include <sstream> +#include <string> +#include <stdio.h> + +#include <sys/stat.h> +#include <unistd.h> + /*}}}*/ + +// Sources Index /*{{{*/ +debSourcesIndex::debSourcesIndex(IndexTarget const &Target,bool const Trusted) : + pkgDebianIndexTargetFile(Target, Trusted), d(NULL) +{ +} +std::string debSourcesIndex::SourceInfo(pkgSrcRecords::Parser const &Record, + pkgSrcRecords::File const &File) const +{ + // The result looks like: http://foo/debian/ stable/main src 1.1.1 (dsc) + std::string Res = Target.Description; + Res.erase(Target.Description.rfind(' ')); + + Res += " "; + Res += Record.Package(); + Res += " "; + Res += Record.Version(); + if (File.Type.empty() == false) + Res += " (" + File.Type + ")"; + return Res; +} +pkgSrcRecords::Parser *debSourcesIndex::CreateSrcParser() const +{ + std::string const SourcesURI = IndexFileName(); + if (FileExists(SourcesURI)) + return new debSrcRecordParser(SourcesURI, this); + return NULL; +} +bool debSourcesIndex::OpenListFile(FileFd &, std::string const &) +{ + return true; +} +pkgCacheListParser * debSourcesIndex::CreateListParser(FileFd &) +{ + return nullptr; +} +uint8_t debSourcesIndex::GetIndexFlags() const +{ + return 0; +} + /*}}}*/ +// Packages Index /*{{{*/ +debPackagesIndex::debPackagesIndex(IndexTarget const &Target, bool const Trusted) : + pkgDebianIndexTargetFile(Target, Trusted), d(NULL) +{ +} +std::string debPackagesIndex::ArchiveInfo(pkgCache::VerIterator const &Ver) const +{ + std::string Res = Target.Description; + { + auto const space = Target.Description.rfind(' '); + if (space != std::string::npos) + Res.erase(space); + } + + Res += " "; + Res += Ver.ParentPkg().Name(); + Res += " "; + std::string const Dist = Target.Option(IndexTarget::RELEASE); + if (Dist.empty() == false && Dist[Dist.size() - 1] != '/') + Res.append(Ver.Arch()).append(" "); + Res += Ver.VerStr(); + return Res; +} +uint8_t debPackagesIndex::GetIndexFlags() const +{ + return 0; +} + /*}}}*/ +// Translation-* Index /*{{{*/ +debTranslationsIndex::debTranslationsIndex(IndexTarget const &Target) : + pkgDebianIndexTargetFile(Target, true), d(NULL) +{} +bool debTranslationsIndex::HasPackages() const +{ + return Exists(); +} +bool debTranslationsIndex::OpenListFile(FileFd &Pkg, std::string const &FileName) +{ + if (FileExists(FileName)) + return pkgDebianIndexTargetFile::OpenListFile(Pkg, FileName); + return true; +} +uint8_t debTranslationsIndex::GetIndexFlags() const +{ + return pkgCache::Flag::NotSource | pkgCache::Flag::NoPackages; +} +std::string debTranslationsIndex::GetArchitecture() const +{ + return std::string(); +} +pkgCacheListParser * debTranslationsIndex::CreateListParser(FileFd &Pkg) +{ + if (Pkg.IsOpen() == false) + return nullptr; + _error->PushToStack(); + std::unique_ptr<pkgCacheListParser> Parser(new debTranslationsParser(&Pkg)); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + return newError ? nullptr : Parser.release(); +} + /*}}}*/ +// dpkg/status Index /*{{{*/ +debStatusIndex::debStatusIndex(std::string const &File) : pkgDebianIndexRealFile(File, true), d(NULL) +{ +} +std::string debStatusIndex::GetArchitecture() const +{ + return std::string(); +} +std::string debStatusIndex::GetComponent() const +{ + return "now"; +} +uint8_t debStatusIndex::GetIndexFlags() const +{ + return pkgCache::Flag::NotSource; +} + +pkgCacheListParser * debStatusIndex::CreateListParser(FileFd &Pkg) +{ + if (Pkg.IsOpen() == false) + return nullptr; + _error->PushToStack(); + std::unique_ptr<pkgCacheListParser> Parser(new debStatusListParser(&Pkg)); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + return newError ? nullptr : Parser.release(); +} + /*}}}*/ +// DebPkgFile Index - a single .deb file as an index /*{{{*/ +debDebPkgFileIndex::debDebPkgFileIndex(std::string const &DebFile) + : pkgDebianIndexRealFile(DebFile, true), d(NULL), DebFile(DebFile) +{ +} +bool debDebPkgFileIndex::GetContent(std::ostream &content, std::string const &debfile) +{ + struct stat Buf; + if (stat(debfile.c_str(), &Buf) != 0) + return false; + + FileFd debFd(debfile, FileFd::ReadOnly); + debDebFile deb(debFd); + debDebFile::MemControlExtract extractor("control"); + + if (not extractor.Read(deb)) + return _error->Error(_("Could not read meta data from %s"), debfile.c_str()); + + // trim off newlines + while (extractor.Control[extractor.Length] == '\n') + extractor.Control[extractor.Length--] = '\0'; + const char *Control = extractor.Control; + while (isspace_ascii(Control[0])) + Control++; + + content << Control << '\n'; + content << "Filename: " << debfile << "\n"; + content << "Size: " << std::to_string(Buf.st_size) << "\n"; + + return true; +} +bool debDebPkgFileIndex::OpenListFile(FileFd &Pkg, std::string const &FileName) +{ + // write the control data to a tempfile + if (GetTempFile("deb-file-" + flNotDir(FileName), true, &Pkg) == NULL) + return false; + std::ostringstream content; + if (GetContent(content, FileName) == false) + return false; + std::string const contentstr = content.str(); + if (contentstr.empty()) + return true; + if (Pkg.Write(contentstr.c_str(), contentstr.length()) == false || Pkg.Seek(0) == false) + return false; + return true; +} +pkgCacheListParser * debDebPkgFileIndex::CreateListParser(FileFd &Pkg) +{ + if (Pkg.IsOpen() == false) + return nullptr; + _error->PushToStack(); + std::unique_ptr<pkgCacheListParser> Parser(new debDebFileParser(&Pkg, DebFile)); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + return newError ? nullptr : Parser.release(); +} +uint8_t debDebPkgFileIndex::GetIndexFlags() const +{ + return pkgCache::Flag::LocalSource; +} +std::string debDebPkgFileIndex::GetArchitecture() const +{ + return std::string(); +} +std::string debDebPkgFileIndex::GetComponent() const +{ + return "local-deb"; +} +pkgCache::PkgFileIterator debDebPkgFileIndex::FindInCache(pkgCache &Cache) const +{ + std::string const FileName = IndexFileName(); + pkgCache::PkgFileIterator File = Cache.FileBegin(); + for (; File.end() == false; ++File) + { + if (File.FileName() == NULL || FileName != File.FileName()) + continue; + // we can't do size checks here as file size != content size + return File; + } + + return File; +} +std::string debDebPkgFileIndex::ArchiveInfo(pkgCache::VerIterator const &Ver) const +{ + std::string Res = IndexFileName() + " "; + Res.append(Ver.ParentPkg().Name()).append(" "); + Res.append(Ver.Arch()).append(" "); + Res.append(Ver.VerStr()); + return Res; +} + /*}}}*/ +// DscFile Index - a single .dsc file as an index /*{{{*/ +debDscFileIndex::debDscFileIndex(std::string const &DscFile) + : pkgDebianIndexRealFile(DscFile, true), d(NULL) +{ +} +pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const +{ + if (Exists() == false) + return NULL; + return new debDscRecordParser(File, this); +} +std::string debDscFileIndex::GetComponent() const +{ + return "local-dsc"; +} +std::string debDscFileIndex::GetArchitecture() const +{ + return "source"; +} +uint8_t debDscFileIndex::GetIndexFlags() const +{ + return pkgCache::Flag::LocalSource; +} + /*}}}*/ +// ControlFile Index - a directory with a debian/control file /*{{{*/ +std::string debDebianSourceDirIndex::GetComponent() const +{ + return "local-control"; +} + /*}}}*/ +// String Package Index - a string of Packages file content /*{{{*/ +std::string debStringPackageIndex::GetArchitecture() const +{ + return std::string(); +} +std::string debStringPackageIndex::GetComponent() const +{ + return "apt-tmp-index"; +} +uint8_t debStringPackageIndex::GetIndexFlags() const +{ + return pkgCache::Flag::NotSource; +} +const pkgIndexFile::Type *debStringPackageIndex::GetType() const +{ + return pkgIndexFile::Type::GetType("Debian Package Index"); +} +debStringPackageIndex::debStringPackageIndex(std::string const &content) : + pkgDebianIndexRealFile("", false), d(NULL) +{ + FileFd fd; + GetTempFile("apt-tmp-index", false, &fd); + fd.Write(content.data(), content.length()); + File = fd.Name(); +} +debStringPackageIndex::~debStringPackageIndex() +{ + RemoveFile("~debStringPackageIndex", File); +} + /*}}}*/ + +// Index File types for Debian /*{{{*/ +class APT_HIDDEN debIFTypeSrc : public pkgIndexFile::Type +{ + public: + debIFTypeSrc() {Label = "Debian Source Index";}; +}; +class APT_HIDDEN debIFTypePkg : public pkgIndexFile::Type +{ + public: + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator const &File) const APT_OVERRIDE + { + return new debRecordParser(File.FileName(),*File.Cache()); + }; + debIFTypePkg() {Label = "Debian Package Index";}; +}; +class APT_HIDDEN debIFTypeTrans : public debIFTypePkg +{ + public: + debIFTypeTrans() {Label = "Debian Translation Index";}; +}; +class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type +{ + public: + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator const &File) const APT_OVERRIDE + { + return new debRecordParser(File.FileName(),*File.Cache()); + }; + debIFTypeStatus() {Label = "Debian dpkg status file";}; +}; +class APT_HIDDEN debIFTypeDebPkgFile : public pkgIndexFile::Type +{ + public: + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator const &File) const APT_OVERRIDE + { + return new debDebFileRecordParser(File.FileName()); + }; + debIFTypeDebPkgFile() {Label = "Debian deb file";}; +}; +class APT_HIDDEN debIFTypeDscFile : public pkgIndexFile::Type +{ + public: + virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string const &DscFile) const APT_OVERRIDE + { + return new debDscRecordParser(DscFile, NULL); + }; + debIFTypeDscFile() {Label = "Debian dsc file";}; +}; +class APT_HIDDEN debIFTypeDebianSourceDir : public pkgIndexFile::Type +{ + public: + virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string const &SourceDir) const APT_OVERRIDE + { + return new debDscRecordParser(SourceDir + std::string("/debian/control"), NULL); + }; + debIFTypeDebianSourceDir() {Label = "Debian control file";}; +}; + +APT_HIDDEN debIFTypeSrc _apt_Src; +APT_HIDDEN debIFTypePkg _apt_Pkg; +APT_HIDDEN debIFTypeTrans _apt_Trans; +APT_HIDDEN debIFTypeStatus _apt_Status; +APT_HIDDEN debIFTypeDebPkgFile _apt_DebPkgFile; +// file based pseudo indexes +APT_HIDDEN debIFTypeDscFile _apt_DscFile; +APT_HIDDEN debIFTypeDebianSourceDir _apt_DebianSourceDir; + +const pkgIndexFile::Type *debSourcesIndex::GetType() const +{ + return &_apt_Src; +} +const pkgIndexFile::Type *debPackagesIndex::GetType() const +{ + return &_apt_Pkg; +} +const pkgIndexFile::Type *debTranslationsIndex::GetType() const +{ + return &_apt_Trans; +} +const pkgIndexFile::Type *debStatusIndex::GetType() const +{ + return &_apt_Status; +} +const pkgIndexFile::Type *debDebPkgFileIndex::GetType() const +{ + return &_apt_DebPkgFile; +} +const pkgIndexFile::Type *debDscFileIndex::GetType() const +{ + return &_apt_DscFile; +} +const pkgIndexFile::Type *debDebianSourceDirIndex::GetType() const +{ + return &_apt_DebianSourceDir; +} + /*}}}*/ + +debStatusIndex::~debStatusIndex() {} +debPackagesIndex::~debPackagesIndex() {} +debTranslationsIndex::~debTranslationsIndex() {} +debSourcesIndex::~debSourcesIndex() {} + +debDebPkgFileIndex::~debDebPkgFileIndex() {} +debDscFileIndex::~debDscFileIndex() {} diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h new file mode 100644 index 0000000..57b3738 --- /dev/null +++ b/apt-pkg/deb/debindexfile.h @@ -0,0 +1,196 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Index Files + + There are three sorts currently + + Package files that have File: tags + Package files that don't (/var/lib/dpkg/status) + Source files + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_DEBINDEXFILE_H +#define PKGLIB_DEBINDEXFILE_H + +#include <apt-pkg/indexfile.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/srcrecords.h> + +#include <string> + +class OpProgress; +class pkgAcquire; +class pkgCacheGenerator; + +class debStatusIndex : public pkgDebianIndexRealFile +{ + void * const d; +protected: + virtual std::string GetArchitecture() const APT_OVERRIDE; + virtual std::string GetComponent() const APT_OVERRIDE; + virtual uint8_t GetIndexFlags() const APT_OVERRIDE; + +public: + + virtual const Type *GetType() const APT_OVERRIDE APT_PURE; + + // Interface for the Cache Generator + virtual bool HasPackages() const APT_OVERRIDE {return true;}; + // Abort if the file does not exist. + virtual bool Exists() const APT_OVERRIDE {return true;}; + + virtual pkgCacheListParser * CreateListParser(FileFd &Pkg) APT_OVERRIDE; + + explicit debStatusIndex(std::string const &File); + virtual ~debStatusIndex(); +}; + +class debPackagesIndex : public pkgDebianIndexTargetFile +{ + void * const d; +protected: + virtual uint8_t GetIndexFlags() const APT_OVERRIDE; + +public: + virtual const Type *GetType() const APT_OVERRIDE APT_PURE; + + // Stuff for accessing files on remote items + virtual std::string ArchiveInfo(pkgCache::VerIterator const &Ver) const APT_OVERRIDE; + + // Interface for the Cache Generator + virtual bool HasPackages() const APT_OVERRIDE {return true;}; + + debPackagesIndex(IndexTarget const &Target, bool const Trusted); + virtual ~debPackagesIndex(); +}; + +class debTranslationsIndex : public pkgDebianIndexTargetFile +{ + void * const d; +protected: + virtual std::string GetArchitecture() const APT_OVERRIDE; + virtual uint8_t GetIndexFlags() const APT_OVERRIDE; + virtual bool OpenListFile(FileFd &Pkg, std::string const &FileName) APT_OVERRIDE; + APT_HIDDEN virtual pkgCacheListParser * CreateListParser(FileFd &Pkg) APT_OVERRIDE; + +public: + + virtual const Type *GetType() const APT_OVERRIDE APT_PURE; + + // Interface for the Cache Generator + virtual bool HasPackages() const APT_OVERRIDE; + + explicit debTranslationsIndex(IndexTarget const &Target); + virtual ~debTranslationsIndex(); +}; + +class debSourcesIndex : public pkgDebianIndexTargetFile +{ + void * const d; + virtual uint8_t GetIndexFlags() const APT_OVERRIDE; + virtual bool OpenListFile(FileFd &Pkg, std::string const &FileName) APT_OVERRIDE; + APT_HIDDEN virtual pkgCacheListParser * CreateListParser(FileFd &Pkg) APT_OVERRIDE; + + public: + + virtual const Type *GetType() const APT_OVERRIDE APT_PURE; + + // Stuff for accessing files on remote items + virtual std::string SourceInfo(pkgSrcRecords::Parser const &Record, + pkgSrcRecords::File const &File) const APT_OVERRIDE; + + // Interface for the record parsers + virtual pkgSrcRecords::Parser *CreateSrcParser() const APT_OVERRIDE; + + // Interface for the Cache Generator + virtual bool HasPackages() const APT_OVERRIDE {return false;}; + + debSourcesIndex(IndexTarget const &Target, bool const Trusted); + virtual ~debSourcesIndex(); +}; + +class debDebPkgFileIndex : public pkgDebianIndexRealFile +{ + void * const d; + std::string DebFile; + +protected: + virtual std::string GetComponent() const APT_OVERRIDE; + virtual std::string GetArchitecture() const APT_OVERRIDE; + virtual uint8_t GetIndexFlags() const APT_OVERRIDE; + virtual bool OpenListFile(FileFd &Pkg, std::string const &FileName) APT_OVERRIDE; + APT_HIDDEN virtual pkgCacheListParser * CreateListParser(FileFd &Pkg) APT_OVERRIDE; + +public: + virtual const Type *GetType() const APT_OVERRIDE APT_PURE; + + /** get the control (file) content of the deb file + * + * @param[out] content of the control file + * @param debfile is the filename of the .deb-file + * @return \b true if successful, otherwise \b false. + */ + static bool GetContent(std::ostream &content, std::string const &debfile); + + // Interface for the Cache Generator + virtual bool HasPackages() const APT_OVERRIDE {return true;} + virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const APT_OVERRIDE; + + // Interface for acquire + + explicit debDebPkgFileIndex(std::string const &DebFile); + virtual ~debDebPkgFileIndex(); + + std::string ArchiveInfo(pkgCache::VerIterator const &Ver) const override; +}; + +class APT_PUBLIC debDscFileIndex : public pkgDebianIndexRealFile +{ + void * const d; + +protected: + virtual std::string GetComponent() const APT_OVERRIDE; + virtual std::string GetArchitecture() const APT_OVERRIDE; + virtual uint8_t GetIndexFlags() const APT_OVERRIDE; + +public: + virtual const Type *GetType() const APT_OVERRIDE APT_PURE; + virtual pkgSrcRecords::Parser *CreateSrcParser() const APT_OVERRIDE; + virtual bool HasPackages() const APT_OVERRIDE {return false;}; + + explicit debDscFileIndex(std::string const &DscFile); + virtual ~debDscFileIndex(); +}; + +class debDebianSourceDirIndex : public debDscFileIndex +{ +protected: + virtual std::string GetComponent() const APT_OVERRIDE; + +public: + virtual const Type *GetType() const APT_OVERRIDE APT_PURE; +}; + +class APT_PUBLIC debStringPackageIndex : public pkgDebianIndexRealFile +{ + void * const d; +protected: + virtual std::string GetArchitecture() const APT_OVERRIDE; + virtual std::string GetComponent() const APT_OVERRIDE; + virtual uint8_t GetIndexFlags() const APT_OVERRIDE; + +public: + virtual const Type *GetType() const APT_OVERRIDE APT_PURE; + + // Interface for the Cache Generator + virtual bool HasPackages() const APT_OVERRIDE {return true;}; + // Abort if the file does not exist. + virtual bool Exists() const APT_OVERRIDE {return true;}; + + explicit debStringPackageIndex(std::string const &content); + virtual ~debStringPackageIndex(); +}; +#endif diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc new file mode 100644 index 0000000..6f47053 --- /dev/null +++ b/apt-pkg/deb/deblistparser.cc @@ -0,0 +1,1053 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Package Cache Generator - Generator for the cache structure. + + This builds the cache structure from the abstract package list parser. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/algorithms.h> +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/cachefilter.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/deblistparser.h> +#include <apt-pkg/error.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile-keys.h> +#include <apt-pkg/tagfile.h> + +#include <algorithm> +#include <string> +#include <vector> +#include <ctype.h> +#include <stddef.h> +#include <string.h> + /*}}}*/ + +using std::string; +using APT::StringView; + +static const debListParser::WordList PrioList[] = { + {"required",pkgCache::State::Required}, + {"important",pkgCache::State::Important}, + {"standard",pkgCache::State::Standard}, + {"optional",pkgCache::State::Optional}, + {"extra",pkgCache::State::Extra}, + {"", 0}}; + +// ListParser::debListParser - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Provide an architecture and only this one and "all" will be accepted + in Step(), if no Architecture is given we will accept every arch + we would accept in general with checkArchitecture() */ +debListParser::debListParser(FileFd *File) : + pkgCacheListParser(), Tags(File) +{ + // this dance allows an empty value to override the default + if (_config->Exists("pkgCacheGen::ForceEssential")) + { + forceEssential = _config->FindVector("pkgCacheGen::ForceEssential"); + if (forceEssential.empty() == false && _config->Find("pkgCacheGen::ForceEssential").empty()) + forceEssential.emplace_back("apt"); + } + else + forceEssential.emplace_back("apt"); + forceImportant = _config->FindVector("pkgCacheGen::ForceImportant"); + myArch = _config->Find("APT::Architecture"); +} + /*}}}*/ +// ListParser::Package - Return the package name /*{{{*/ +// --------------------------------------------------------------------- +/* This is to return the name of the package this section describes */ +string debListParser::Package() { + string Result = Section.Find(pkgTagSection::Key::Package).to_string(); + + // Normalize mixed case package names to lower case, like dpkg does + // See Bug#807012 for details. + // Only do this when the package name does not contain a / - as that + // indicates that the package name was derived from a filename given + // to install or build-dep or similar (Bug#854794) + if (likely(Result.find('/') == string::npos)) + { + for (char &c: Result) + { + char l = tolower_ascii_inline(c); + if (unlikely(l != c)) + c = l; + } + } + + if(unlikely(Result.empty() == true)) + _error->Error("Encountered a section with no Package: header"); + return Result; +} + /*}}}*/ +// ListParser::Architecture - Return the package arch /*{{{*/ +// --------------------------------------------------------------------- +/* This will return the Architecture of the package this section describes */ +APT::StringView debListParser::Architecture() { + auto const Arch = Section.Find(pkgTagSection::Key::Architecture); + return Arch.empty() ? "none" : Arch; +} + /*}}}*/ +// ListParser::ArchitectureAll /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool debListParser::ArchitectureAll() { + return Section.Find(pkgTagSection::Key::Architecture) == "all"; +} + /*}}}*/ +// ListParser::Version - Return the version string /*{{{*/ +// --------------------------------------------------------------------- +/* This is to return the string describing the version in debian form, + epoch:upstream-release. If this returns the blank string then the + entry is assumed to only describe package properties */ +APT::StringView debListParser::Version() +{ + return Section.Find(pkgTagSection::Key::Version); +} + /*}}}*/ +unsigned char debListParser::ParseMultiArch(bool const showErrors) /*{{{*/ +{ + unsigned char MA; + auto const MultiArch = Section.Find(pkgTagSection::Key::Multi_Arch); + if (MultiArch.empty() == true || MultiArch == "no") + MA = pkgCache::Version::No; + else if (MultiArch == "same") { + if (ArchitectureAll() == true) + { + if (showErrors == true) + _error->Warning("Architecture: all package '%s' can't be Multi-Arch: same", + Package().c_str()); + MA = pkgCache::Version::No; + } + else + MA = pkgCache::Version::Same; + } + else if (MultiArch == "foreign") + MA = pkgCache::Version::Foreign; + else if (MultiArch == "allowed") + MA = pkgCache::Version::Allowed; + else + { + if (showErrors == true) + _error->Warning("Unknown Multi-Arch type '%s' for package '%s'", + MultiArch.to_string().c_str(), Package().c_str()); + MA = pkgCache::Version::No; + } + + if (ArchitectureAll() == true) + MA |= pkgCache::Version::All; + + return MA; +} + /*}}}*/ +// ListParser::NewVersion - Fill in the version structure /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool debListParser::NewVersion(pkgCache::VerIterator &Ver) +{ + const char *Start; + const char *Stop; + + // Parse the section + if (Section.Find(pkgTagSection::Key::Section,Start,Stop) == true) + { + map_stringitem_t const idx = StoreString(pkgCacheGenerator::SECTION, Start, Stop - Start); + Ver->Section = idx; + } + // Parse the source package name + pkgCache::GrpIterator G = Ver.ParentPkg().Group(); + + // Setup the defaults + Ver->SourcePkgName = G->Name; + Ver->SourceVerStr = Ver->VerStr; + + // Parse the name and version str + if (Section.Find(pkgTagSection::Key::Source,Start,Stop) == true) + { + const char * const Space = static_cast<const char *>(memchr(Start, ' ', Stop - Start)); + pkgCache::VerIterator V; + + if (Space != NULL) + { + const char * const Open = static_cast<const char *>(memchr(Space, '(', Stop - Space)); + if (likely(Open != NULL)) + { + const char * const Close = static_cast<const char *>(memchr(Open, ')', Stop - Open)); + if (likely(Close != NULL)) + { + APT::StringView const version(Open + 1, (Close - Open) - 1); + if (version != Ver.VerStr()) + { + map_stringitem_t const idx = StoreString(pkgCacheGenerator::VERSIONNUMBER, version); + G = Ver.ParentPkg().Group(); + Ver->SourceVerStr = idx; + } + } + } + Stop = Space; + } + + APT::StringView const pkgname(Start, Stop - Start); + // Oh, our group is the wrong one for the source package. Make a new one. + if (pkgname != G.Name()) + { + if (not NewGroup(G, pkgname)) + return false; + } + } + + // Link into by source package group. + Ver->SourcePkgName = G->Name; + Ver->NextInSource = G->VersionsInSource; + G->VersionsInSource = Ver.MapPointer(); + + Ver->MultiArch = ParseMultiArch(true); + // Archive Size + Ver->Size = Section.FindULL(pkgTagSection::Key::Size); + // Unpacked Size (in K) + Ver->InstalledSize = Section.FindULL(pkgTagSection::Key::Installed_Size); + Ver->InstalledSize *= 1024; + + // Priority + if (Section.Find(pkgTagSection::Key::Priority,Start,Stop) == true) + { + if (GrabWord(StringView(Start,Stop-Start),PrioList,Ver->Priority) == false) + Ver->Priority = pkgCache::State::Extra; + } + + if (ParseDepends(Ver,pkgTagSection::Key::Pre_Depends,pkgCache::Dep::PreDepends) == false) + return false; + if (ParseDepends(Ver,pkgTagSection::Key::Depends,pkgCache::Dep::Depends) == false) + return false; + if (ParseDepends(Ver,pkgTagSection::Key::Conflicts,pkgCache::Dep::Conflicts) == false) + return false; + if (ParseDepends(Ver,pkgTagSection::Key::Breaks,pkgCache::Dep::DpkgBreaks) == false) + return false; + if (ParseDepends(Ver,pkgTagSection::Key::Recommends,pkgCache::Dep::Recommends) == false) + return false; + if (ParseDepends(Ver,pkgTagSection::Key::Suggests,pkgCache::Dep::Suggests) == false) + return false; + if (ParseDepends(Ver,pkgTagSection::Key::Replaces,pkgCache::Dep::Replaces) == false) + return false; + if (ParseDepends(Ver,pkgTagSection::Key::Enhances,pkgCache::Dep::Enhances) == false) + return false; + + if (ParseProvides(Ver) == false) + return false; + if (not APT::KernelAutoRemoveHelper::getUname(Ver.ParentPkg().Name()).empty()) + { + if (not NewProvides(Ver, "$kernel", "any", Ver.VerStr(), pkgCache::Flag::MultiArchImplicit)) + return false; + } + + return true; +} + /*}}}*/ +// ListParser::AvailableDescriptionLanguages /*{{{*/ +std::vector<std::string> debListParser::AvailableDescriptionLanguages() +{ + std::vector<std::string> avail; + static constexpr int prefixLen = 12; + char buf[32] = "Description-"; + if (Section.Exists(pkgTagSection::Key::Description)) + avail.emplace_back(); + for (auto const &lang : APT::Configuration::getLanguages(true)) + { + if (unlikely(lang.size() > sizeof(buf) - prefixLen)) { + _error->Warning("Ignoring translated description %s", lang.c_str()); + continue; + } + memcpy(buf + prefixLen, lang.c_str(), lang.size()); + if (Section.Exists(StringView(buf, prefixLen + lang.size())) == true) + avail.push_back(lang); + } + return avail; +} + /*}}}*/ +// ListParser::Description_md5 - Return the description_md5 MD5SumValue /*{{{*/ +// --------------------------------------------------------------------- +/* This is to return the md5 string to allow the check if it is the right + description. If no Description-md5 is found in the section it will be + calculated. + */ +APT::StringView debListParser::Description_md5() +{ + StringView const value = Section.Find(pkgTagSection::Key::Description_md5); + if (unlikely(value.empty() == true)) + { + StringView const desc = Section.Find(pkgTagSection::Key::Description); + if (desc == "\n") + return StringView(); + + Hashes md5(Hashes::MD5SUM); + md5.Add(desc.data(), desc.size()); + md5.Add("\n"); + MD5Buffer = md5.GetHashString(Hashes::MD5SUM).HashValue(); + return StringView(MD5Buffer); + } + else if (likely(value.size() == 32)) + { + return value; + } + _error->Error("Malformed Description-md5 line; doesn't have the required length (32 != %d) '%.*s'", (int)value.size(), (int)value.length(), value.data()); + return StringView(); +} + /*}}}*/ +// ListParser::UsePackage - Update a package structure /*{{{*/ +// --------------------------------------------------------------------- +/* This is called to update the package with any new information + that might be found in the section */ +bool debListParser::UsePackage(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver) +{ + string const static myArch = _config->Find("APT::Architecture"); + // Possible values are: "all", "native", "installed" and "none" + // The "installed" mode is handled by ParseStatus(), See #544481 and friends. + string const static essential = _config->Find("pkgCacheGen::Essential", "all"); + if (essential == "all" || + (essential == "native" && Pkg->Arch != 0 && myArch == Pkg.Arch())) + if (Section.FindFlag(pkgTagSection::Key::Essential,Pkg->Flags,pkgCache::Flag::Essential) == false) + return false; + if (Section.FindFlag(pkgTagSection::Key::Important,Pkg->Flags,pkgCache::Flag::Important) == false) + return false; + if (Section.FindFlag(pkgTagSection::Key::Protected, Pkg->Flags, pkgCache::Flag::Important) == false) + return false; + + if (std::find(forceEssential.begin(), forceEssential.end(), Pkg.Name()) != forceEssential.end()) + { + if ((essential == "native" && Pkg->Arch != 0 && myArch == Pkg.Arch()) || + essential == "all") + Pkg->Flags |= pkgCache::Flag::Essential | pkgCache::Flag::Important; + else + Pkg->Flags |= pkgCache::Flag::Important; + } + else if (std::find(forceImportant.begin(), forceImportant.end(), Pkg.Name()) != forceImportant.end()) + Pkg->Flags |= pkgCache::Flag::Important; + + auto phased = Section.FindULL(pkgTagSection::Key::Phased_Update_Percentage, 100); + if (phased != 100) + { + if (not Ver.PhasedUpdatePercentage(phased)) + _error->Warning("Ignoring invalid Phased-Update-Percentage value"); + } + + if (ParseStatus(Pkg,Ver) == false) + return false; + return true; +} + /*}}}*/ +// ListParser::VersionHash - Compute a unique hash for this version /*{{{*/ +// --------------------------------------------------------------------- +/* */ +uint32_t debListParser::VersionHash() +{ + static constexpr pkgTagSection::Key Sections[] ={ + pkgTagSection::Key::Installed_Size, + pkgTagSection::Key::Depends, + pkgTagSection::Key::Pre_Depends, +// pkgTagSection::Key::Suggests, +// pkgTagSection::Key::Recommends", + pkgTagSection::Key::Conflicts, + pkgTagSection::Key::Breaks, + pkgTagSection::Key::Replaces}; + unsigned long Result = 5381; + for (auto I : Sections) + { + const char *Start; + const char *End; + if (Section.Find(I,Start,End) == false) + continue; + + /* Strip out any spaces from the text, this undoes dpkgs reformatting + of certain fields. dpkg also has the rather interesting notion of + reformatting depends operators < -> <=, so we drop all = from the + string to make that not matter. */ + for (; Start != End; ++Start) + { + // Strip away 0: epochs from input + if (*Start == '0' && Start[1] == ':') { + Start++; // Skip the : + continue; // Skip the 0 + } + if (isspace_ascii(*Start) != 0 || *Start == '=') + continue; + Result = 33 * Result + tolower_ascii_unsafe(*Start); + } + + + } + + return Result; +} + /*}}}*/ +// StatusListParser::ParseStatus - Parse the status field /*{{{*/ +// --------------------------------------------------------------------- +/* Status lines are of the form, + Status: want flag status + want = unknown, install, hold, deinstall, purge + flag = ok, reinstreq + status = not-installed, config-files, half-installed, unpacked, + half-configured, triggers-awaited, triggers-pending, installed + */ +bool debListParser::ParseStatus(pkgCache::PkgIterator &, + pkgCache::VerIterator &Ver) +{ + // the status file has no info about the download size and + // usually this is fine as we will have picked that info up already – + // except if we have volatile sources which are parsed after the status file. + if (Ver->Size == 0) + Ver->Size = Section.FindULL(pkgTagSection::Key::Size); + return true; +} +bool debStatusListParser::ParseStatus(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver) +{ + const char *Start; + const char *Stop; + if (Section.Find(pkgTagSection::Key::Status,Start,Stop) == false) + return true; + + // UsePackage() is responsible for setting the flag in the default case + bool const static essential = _config->Find("pkgCacheGen::Essential", "") == "installed"; + if (essential == true && + Section.FindFlag(pkgTagSection::Key::Essential,Pkg->Flags,pkgCache::Flag::Essential) == false) + return false; + + // Isolate the first word + const char *I = Start; + for(; I < Stop && *I != ' '; I++); + if (I >= Stop || *I != ' ') + return _error->Error("Malformed Status line"); + + // Process the want field + WordList WantList[] = {{"unknown",pkgCache::State::Unknown}, + {"install",pkgCache::State::Install}, + {"hold",pkgCache::State::Hold}, + {"deinstall",pkgCache::State::DeInstall}, + {"purge",pkgCache::State::Purge}, + {"", 0}}; + if (GrabWord(StringView(Start,I-Start),WantList,Pkg->SelectedState) == false) + return _error->Error("Malformed 1st word in the Status line"); + + // Isloate the next word + I++; + Start = I; + for(; I < Stop && *I != ' '; I++); + if (I >= Stop || *I != ' ') + return _error->Error("Malformed status line, no 2nd word"); + + // Process the flag field + WordList FlagList[] = {{"ok",pkgCache::State::Ok}, + {"reinstreq",pkgCache::State::ReInstReq}, + {"hold",pkgCache::State::HoldInst}, + {"hold-reinstreq",pkgCache::State::HoldReInstReq}, + {"", 0}}; + if (GrabWord(StringView(Start,I-Start),FlagList,Pkg->InstState) == false) + return _error->Error("Malformed 2nd word in the Status line"); + + // Isloate the last word + I++; + Start = I; + for(; I < Stop && *I != ' '; I++); + if (I != Stop) + return _error->Error("Malformed Status line, no 3rd word"); + + // Process the flag field + WordList StatusList[] = {{"not-installed",pkgCache::State::NotInstalled}, + {"config-files",pkgCache::State::ConfigFiles}, + {"half-installed",pkgCache::State::HalfInstalled}, + {"unpacked",pkgCache::State::UnPacked}, + {"half-configured",pkgCache::State::HalfConfigured}, + {"triggers-awaited",pkgCache::State::TriggersAwaited}, + {"triggers-pending",pkgCache::State::TriggersPending}, + {"installed",pkgCache::State::Installed}, + {"", 0}}; + if (GrabWord(StringView(Start,I-Start),StatusList,Pkg->CurrentState) == false) + return _error->Error("Malformed 3rd word in the Status line"); + + /* A Status line marks the package as indicating the current + version as well. Only if it is actually installed.. Otherwise + the interesting dpkg handling of the status file creates bogus + entries. */ + if (!(Pkg->CurrentState == pkgCache::State::NotInstalled || + Pkg->CurrentState == pkgCache::State::ConfigFiles)) + { + if (Ver.end() == true) + _error->Warning("Encountered status field in a non-version description"); + else + Pkg->CurrentVer = Ver.MapPointer(); + } + + return true; +} + +const char *debListParser::ConvertRelation(const char *I,unsigned int &Op) +{ + // Determine the operator + switch (*I) + { + case '<': + I++; + if (*I == '=') + { + I++; + Op = pkgCache::Dep::LessEq; + break; + } + + if (*I == '<') + { + I++; + Op = pkgCache::Dep::Less; + break; + } + + // < is the same as <= and << is really Cs < for some reason + Op = pkgCache::Dep::LessEq; + break; + + case '>': + I++; + if (*I == '=') + { + I++; + Op = pkgCache::Dep::GreaterEq; + break; + } + + if (*I == '>') + { + I++; + Op = pkgCache::Dep::Greater; + break; + } + + // > is the same as >= and >> is really Cs > for some reason + Op = pkgCache::Dep::GreaterEq; + break; + + case '=': + Op = pkgCache::Dep::Equals; + I++; + break; + + // HACK around bad package definitions + default: + Op = pkgCache::Dep::Equals; + break; + } + return I; +} + /*}}}*/ +// ListParser::ParseDepends - Parse a dependency element /*{{{*/ +// --------------------------------------------------------------------- +/* This parses the dependency elements out of a standard string in place, + bit by bit. */ +const char *debListParser::ParseDepends(const char *Start,const char *Stop, + string &Package,string &Ver, + unsigned int &Op, bool const &ParseArchFlags, + bool const &StripMultiArch, + bool const &ParseRestrictionsList, + string const &Arch) +{ + StringView PackageView; + StringView VerView; + + auto res = ParseDepends(Start, Stop, PackageView, VerView, Op, (bool)ParseArchFlags, + (bool) StripMultiArch, (bool) ParseRestrictionsList, Arch); + Package = PackageView.to_string(); + Ver = VerView.to_string(); + + return res; +} + +const char *debListParser::ParseDepends(const char *Start, const char *Stop, + StringView &Package, StringView &Ver, + unsigned int &Op, bool ParseArchFlags, + bool StripMultiArch, + bool ParseRestrictionsList, string Arch) +{ + if (Arch.empty()) + Arch = _config->Find("APT::Architecture"); + // Strip off leading space + for (;Start != Stop && isspace_ascii(*Start) != 0; ++Start); + + // Parse off the package name + const char *I = Start; + for (;I != Stop && isspace_ascii(*I) == 0 && *I != '(' && *I != ')' && + *I != ',' && *I != '|' && *I != '[' && *I != ']' && + *I != '<' && *I != '>'; ++I); + + // Malformed, no '(' + if (I != Stop && *I == ')') + return 0; + + if (I == Start) + return 0; + + // Stash the package name + Package = StringView(Start, I - Start); + + // We don't want to confuse library users which can't handle MultiArch + if (StripMultiArch == true) { + size_t const found = Package.rfind(':'); + if (found != StringView::npos && + (Package.substr(found) == ":any" || + Package.substr(found) == ":native" || + Package.substr(found +1) == Arch)) + Package = Package.substr(0,found); + } + + // Skip white space to the '(' + for (;I != Stop && isspace_ascii(*I) != 0 ; I++); + + // Parse a version + if (I != Stop && *I == '(') + { + // Skip the '(' + for (I++; I != Stop && isspace_ascii(*I) != 0 ; I++); + if (I + 3 >= Stop) + return 0; + I = ConvertRelation(I,Op); + + // Skip whitespace + for (;I != Stop && isspace_ascii(*I) != 0; I++); + Start = I; + I = (const char*) memchr(I, ')', Stop - I); + if (I == NULL || Start == I) + return 0; + + // Skip trailing whitespace + const char *End = I; + for (; End > Start && isspace_ascii(End[-1]); End--); + + Ver = StringView(Start,End-Start); + I++; + } + else + { + Ver = StringView(); + Op = pkgCache::Dep::NoOp; + } + + // Skip whitespace + for (;I != Stop && isspace_ascii(*I) != 0; I++); + + if (unlikely(ParseArchFlags == true)) + { + APT::CacheFilter::PackageArchitectureMatchesSpecification matchesArch(Arch, false); + + // Parse an architecture + if (I != Stop && *I == '[') + { + ++I; + // malformed + if (unlikely(I == Stop)) + return 0; + + const char *End = I; + bool Found = false; + bool NegArch = false; + while (I != Stop) + { + // look for whitespace or ending ']' + for (;End != Stop && !isspace_ascii(*End) && *End != ']'; ++End); + + if (unlikely(End == Stop)) + return 0; + + if (*I == '!') + { + NegArch = true; + ++I; + } + + std::string const arch(I, End); + if (arch.empty() == false && matchesArch(arch.c_str()) == true) + { + Found = true; + if (I[-1] != '!') + NegArch = false; + // we found a match, so fast-forward to the end of the wildcards + for (; End != Stop && *End != ']'; ++End); + } + + if (*End++ == ']') { + I = End; + break; + } + + I = End; + for (;I != Stop && isspace_ascii(*I) != 0; I++); + } + + if (NegArch == true) + Found = !Found; + + if (Found == false) + Package = ""; /* not for this arch */ + } + + // Skip whitespace + for (;I != Stop && isspace_ascii(*I) != 0; I++); + } + + if (unlikely(ParseRestrictionsList == true)) + { + // Parse a restrictions formula which is in disjunctive normal form: + // (foo AND bar) OR (blub AND bla) + + std::vector<string> const profiles = APT::Configuration::getBuildProfiles(); + + // if the next character is a restriction list, then by default the + // dependency does not apply and the conditions have to be checked + // if the next character is not a restriction list, then by default the + // dependency applies + bool applies1 = (*I != '<'); + while (I != Stop) + { + if (*I != '<') + break; + + ++I; + // malformed + if (unlikely(I == Stop)) + return 0; + + const char *End = I; + + // if of the prior restriction list is already fulfilled, then + // we can just skip to the end of the current list + if (applies1) { + for (;End != Stop && *End != '>'; ++End); + I = ++End; + // skip whitespace + for (;I != Stop && isspace_ascii(*I) != 0; I++); + } else { + bool applies2 = true; + // all the conditions inside a restriction list have to be + // met so once we find one that is not met, we can skip to + // the end of this list + while (I != Stop) + { + // look for whitespace or ending '>' + // End now points to the character after the current term + for (;End != Stop && !isspace_ascii(*End) && *End != '>'; ++End); + + if (unlikely(End == Stop)) + return 0; + + bool NegRestriction = false; + if (*I == '!') + { + NegRestriction = true; + ++I; + } + + std::string const restriction(I, End); + if (restriction.empty() == false && profiles.empty() == false && + std::find(profiles.begin(), profiles.end(), restriction) != profiles.end()) + { + if (NegRestriction) { + applies2 = false; + // since one of the terms does not apply we don't have to check the others + for (; End != Stop && *End != '>'; ++End); + } + } else { + if (!NegRestriction) { + applies2 = false; + // since one of the terms does not apply we don't have to check the others + for (; End != Stop && *End != '>'; ++End); + } + } + + if (*End++ == '>') { + I = End; + // skip whitespace + for (;I != Stop && isspace_ascii(*I) != 0; I++); + break; + } + + I = End; + // skip whitespace + for (;I != Stop && isspace_ascii(*I) != 0; I++); + } + if (applies2) { + applies1 = true; + } + } + } + + if (applies1 == false) { + Package = ""; //not for this restriction + } + } + + if (I != Stop && *I == '|') + Op |= pkgCache::Dep::Or; + + if (I == Stop || *I == ',' || *I == '|') + { + if (I != Stop) + for (I++; I != Stop && isspace_ascii(*I) != 0; I++); + return I; + } + + return 0; +} + /*}}}*/ +// ListParser::ParseDepends - Parse a dependency list /*{{{*/ +// --------------------------------------------------------------------- +/* This is the higher level depends parser. It takes a tag and generates + a complete depends tree for the given version. */ +bool debListParser::ParseDepends(pkgCache::VerIterator &Ver, + pkgTagSection::Key Key,unsigned int Type) +{ + const char *Start; + const char *Stop; + if (Section.Find(Key,Start,Stop) == false || Start == Stop) + return true; + + string const pkgArch = Ver.Arch(); + bool const barbarianArch = not APT::Configuration::checkArchitecture(pkgArch); + + while (1) + { + StringView Package; + StringView Version; + unsigned int Op; + + Start = ParseDepends(Start, Stop, Package, Version, Op, false, false, false, myArch); + if (Start == 0) + return _error->Error("Problem parsing dependency %zu of %s:%s=%s", static_cast<size_t>(Key), // TODO + Ver.ParentPkg().Name(), Ver.Arch(), Ver.VerStr()); + size_t const found = Package.rfind(':'); + + if (found == string::npos) + { + if (NewDepends(Ver,Package,pkgArch,Version,Op,Type) == false) + return false; + } + else if (Package.substr(found) == ":any") + { + if (barbarianArch) + { + if (not NewDepends(Ver, Package, "any", Version, Op | pkgCache::Dep::Or, Type)) + return false; + if (not NewDepends(Ver, Package.substr(0, found), pkgArch, Version, Op, Type)) + return false; + } + else if (not NewDepends(Ver, Package, "any", Version, Op, Type)) + return false; + } + else + { + // Such dependencies are not supposed to be accepted … + // … but this is probably the best thing to do anyway + if (Package.substr(found + 1) == "native") + { + std::string const Pkg = Package.substr(0, found).to_string() + ':' + Ver.Cache()->NativeArch(); + if (NewDepends(Ver, Pkg, "any", Version, Op | pkgCache::Dep::ArchSpecific, Type) == false) + return false; + } + else if (NewDepends(Ver, Package, "any", Version, Op | pkgCache::Dep::ArchSpecific, Type) == false) + return false; + } + + if (Start == Stop) + break; + } + return true; +} + /*}}}*/ +// ListParser::ParseProvides - Parse the provides list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool debListParser::ParseProvides(pkgCache::VerIterator &Ver) +{ + /* it is unlikely, but while parsing dependencies, we might have already + picked up multi-arch implicit provides which we do not want to duplicate here */ + bool hasProvidesAlready = false; + std::string const spzName = Ver.ParentPkg().FullName(false); + { + for (pkgCache::PrvIterator Prv = Ver.ProvidesList(); Prv.end() == false; ++Prv) + { + if (Prv.IsMultiArchImplicit() == false || (Prv->Flags & pkgCache::Flag::ArchSpecific) == 0) + continue; + if (spzName != Prv.OwnerPkg().FullName(false)) + continue; + hasProvidesAlready = true; + break; + } + } + + string const Arch = Ver.Arch(); + bool const barbarianArch = not APT::Configuration::checkArchitecture(Arch); + const char *Start; + const char *Stop; + if (Section.Find(pkgTagSection::Key::Provides,Start,Stop) == true) + { + StringView Package; + StringView Version; + unsigned int Op; + + do + { + Start = ParseDepends(Start,Stop,Package,Version,Op, false, false, false); + const size_t archfound = Package.rfind(':'); + if (Start == 0) + return _error->Error("Problem parsing Provides line of %s:%s=%s", Ver.ParentPkg().Name(), Ver.Arch(), Ver.VerStr()); + if (unlikely(Op != pkgCache::Dep::NoOp && Op != pkgCache::Dep::Equals)) { + _error->Warning("Ignoring non-equal Provides for package %s in %s:%s=%s", Package.to_string().c_str(), Ver.ParentPkg().Name(), Ver.Arch(), Ver.VerStr()); + } else if (archfound != string::npos) { + StringView spzArch = Package.substr(archfound + 1); + if (spzArch != "any") + { + if (NewProvides(Ver, Package.substr(0, archfound), spzArch, Version, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false) + return false; + } + if (NewProvides(Ver, Package, "any", Version, pkgCache::Flag::ArchSpecific) == false) + return false; + } else if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign) { + if (not barbarianArch) + { + if (NewProvidesAllArch(Ver, Package, Version, 0) == false) + return false; + } + else if (NewProvides(Ver, Package, Arch, Version, 0) == false) + return false; + } else { + if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed && not barbarianArch) + { + if (NewProvides(Ver, Package.to_string().append(":any"), "any", Version, pkgCache::Flag::MultiArchImplicit) == false) + return false; + } + if (NewProvides(Ver, Package, Arch, Version, 0) == false) + return false; + } + if (archfound == std::string::npos) + { + string spzName = Package.to_string(); + spzName.push_back(':'); + spzName.append(Ver.ParentPkg().Arch()); + pkgCache::PkgIterator const spzPkg = Ver.Cache()->FindPkg(spzName, "any"); + if (spzPkg.end() == false) + { + if (NewProvides(Ver, spzName, "any", Version, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false) + return false; + } + } + } while (Start != Stop); + } + + if (not barbarianArch) + { + if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed) + { + string const Package = string(Ver.ParentPkg().Name()).append(":").append("any"); + if (NewProvides(Ver, Package, "any", Ver.VerStr(), pkgCache::Flag::MultiArchImplicit) == false) + return false; + } + else if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign) + { + if (NewProvidesAllArch(Ver, Ver.ParentPkg().Name(), Ver.VerStr(), pkgCache::Flag::MultiArchImplicit) == false) + return false; + } + } + + if (hasProvidesAlready == false) + { + pkgCache::PkgIterator const spzPkg = Ver.Cache()->FindPkg(spzName, "any"); + if (spzPkg.end() == false) + { + if (NewProvides(Ver, spzName, "any", Ver.VerStr(), pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false) + return false; + } + } + return true; +} + /*}}}*/ +// ListParser::GrabWord - Matches a word and returns /*{{{*/ +// --------------------------------------------------------------------- +/* Looks for a word in a list of words - for ParseStatus */ +bool debListParser::GrabWord(StringView Word, WordList const *List, unsigned char &Out) +{ + for (unsigned int C = 0; List[C].Str.empty() == false; C++) + { + if (Word.length() == List[C].Str.length() && + strncasecmp(Word.data(), List[C].Str.data(), Word.length()) == 0) + { + Out = List[C].Val; + return true; + } + } + return false; +} + /*}}}*/ +// ListParser::Step - Move to the next section in the file /*{{{*/ +// --------------------------------------------------------------------- +/* This has to be careful to only process the correct architecture */ +bool debListParser::Step() +{ + iOffset = Tags.Offset(); + return Tags.Step(Section); +} + /*}}}*/ +// ListParser::GetPrio - Convert the priority from a string /*{{{*/ +// --------------------------------------------------------------------- +/* */ +unsigned char debListParser::GetPrio(string Str) +{ + unsigned char Out; + if (GrabWord(Str,PrioList,Out) == false) + Out = pkgCache::State::Extra; + + return Out; +} + /*}}}*/ +bool debListParser::SameVersion(uint32_t Hash, /*{{{*/ + pkgCache::VerIterator const &Ver) +{ + if (pkgCacheListParser::SameVersion(Hash, Ver) == false) + return false; + // status file has no (Download)Size, but all others are fair game + // status file is parsed last, so the first version we encounter is + // probably also the version we have downloaded + unsigned long long const Size = Section.FindULL(pkgTagSection::Key::Size); + if (Size != 0 && Ver->Size != 0 && Size != Ver->Size) + return false; + // available everywhere, but easier to check here than to include in VersionHash + unsigned char MultiArch = ParseMultiArch(false); + if (MultiArch != Ver->MultiArch) + return false; + // for all practical proposes (we can check): same version + return true; +} + /*}}}*/ + +debDebFileParser::debDebFileParser(FileFd *File, std::string const &DebFile) + : debListParser(File), DebFile(DebFile) +{ +} + +bool debDebFileParser::UsePackage(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver) +{ + if (not debListParser::UsePackage(Pkg, Ver)) + return false; + // we use the full file path as a provides so that the file is found by its name + // using the MultiArchImplicit flag for this is a bit of a stretch + return NewProvides(Ver, DebFile, Pkg.Cache()->NativeArch(), Ver.VerStr(), pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific); +} + +debListParser::~debListParser() {} diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h new file mode 100644 index 0000000..eefce2a --- /dev/null +++ b/apt-pkg/deb/deblistparser.h @@ -0,0 +1,124 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Package List Parser - This implements the abstract parser + interface for Debian package files + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_DEBLISTPARSER_H +#define PKGLIB_DEBLISTPARSER_H + +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgcachegen.h> +#include <apt-pkg/tagfile.h> + +#include <string> +#include <vector> +#include <apt-pkg/string_view.h> + + +class FileFd; + +class APT_HIDDEN debListParser : public pkgCacheListParser +{ + public: + + // Parser Helper + struct WordList + { + APT::StringView Str; + unsigned char Val; + }; + + private: + std::vector<std::string> forceEssential; + std::vector<std::string> forceImportant; + std::string MD5Buffer; + std::string myArch; + + protected: + pkgTagFile Tags; + pkgTagSection Section; + map_filesize_t iOffset; + + virtual bool ParseStatus(pkgCache::PkgIterator &Pkg,pkgCache::VerIterator &Ver); + bool ParseDepends(pkgCache::VerIterator &Ver, pkgTagSection::Key Key, + unsigned int Type); + bool ParseProvides(pkgCache::VerIterator &Ver); + + APT_HIDDEN static bool GrabWord(APT::StringView Word,const WordList *List,unsigned char &Out); + APT_HIDDEN unsigned char ParseMultiArch(bool const showErrors); + + public: + + APT_PUBLIC static unsigned char GetPrio(std::string Str); + + // These all operate against the current section + virtual std::string Package() APT_OVERRIDE; + virtual bool ArchitectureAll() APT_OVERRIDE; + virtual APT::StringView Architecture() APT_OVERRIDE; + virtual APT::StringView Version() APT_OVERRIDE; + virtual bool NewVersion(pkgCache::VerIterator &Ver) APT_OVERRIDE; + virtual std::vector<std::string> AvailableDescriptionLanguages() APT_OVERRIDE; + virtual APT::StringView Description_md5() APT_OVERRIDE; + virtual uint32_t VersionHash() APT_OVERRIDE; + virtual bool SameVersion(uint32_t Hash, pkgCache::VerIterator const &Ver) APT_OVERRIDE; + virtual bool UsePackage(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver) APT_OVERRIDE; + virtual map_filesize_t Offset() APT_OVERRIDE {return iOffset;}; + virtual map_filesize_t Size() APT_OVERRIDE {return Section.size();}; + + virtual bool Step() APT_OVERRIDE; + + APT_PUBLIC static const char *ParseDepends(const char *Start, const char *Stop, + std::string &Package, std::string &Ver, unsigned int &Op, + bool const &ParseArchFlags = false, bool const &StripMultiArch = true, + bool const &ParseRestrictionsList = false, + std::string const &Arch = ""); + + APT_PUBLIC static const char *ParseDepends(const char *Start, const char *Stop, + APT::StringView &Package, + APT::StringView &Ver, unsigned int &Op, + bool const ParseArchFlags = false, bool StripMultiArch = true, + bool const ParseRestrictionsList = false, + std::string Arch = ""); + + APT_PUBLIC static const char *ConvertRelation(const char *I,unsigned int &Op); + + explicit debListParser(FileFd *File); + virtual ~debListParser(); +}; + +class APT_HIDDEN debDebFileParser : public debListParser +{ + private: + std::string DebFile; + + public: + debDebFileParser(FileFd *File, std::string const &DebFile); + virtual bool UsePackage(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver) APT_OVERRIDE; +}; + +class APT_HIDDEN debTranslationsParser : public debListParser +{ + public: + // a translation can never be a real package + virtual APT::StringView Architecture() APT_OVERRIDE { return ""; } + virtual APT::StringView Version() APT_OVERRIDE { return ""; } + + explicit debTranslationsParser(FileFd *File) + : debListParser(File) {}; +}; + +class APT_HIDDEN debStatusListParser : public debListParser +{ + public: + virtual bool ParseStatus(pkgCache::PkgIterator &Pkg,pkgCache::VerIterator &Ver) APT_OVERRIDE; + explicit debStatusListParser(FileFd *File) + : debListParser(File) {}; +}; +#endif diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc new file mode 100644 index 0000000..c6005f1 --- /dev/null +++ b/apt-pkg/deb/debmetaindex.cc @@ -0,0 +1,1346 @@ +#include <config.h> + +#include <apt-pkg/acquire-item.h> +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/debindexfile.h> +#include <apt-pkg/debmetaindex.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/gpgv.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/metaindex.h> +#include <apt-pkg/pkgcachegen.h> +#include <apt-pkg/sourcelist.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile.h> + +#include <algorithm> +#include <map> +#include <optional> +#include <sstream> +#include <string> +#include <utility> +#include <vector> + +#include <string.h> +#include <sys/stat.h> + +#include <apti18n.h> + +static std::string transformFingergrpints(std::string finger) /*{{{*/ +{ + std::transform(finger.begin(), finger.end(), finger.begin(), ::toupper); + if (finger.length() == 40) + { + if (finger.find_first_not_of("0123456789ABCDEF") == std::string::npos) + return finger; + } + else if (finger.length() == 41) + { + auto bang = finger.find_first_not_of("0123456789ABCDEF"); + if (bang == 40 && finger[bang] == '!') + return finger; + } + return ""; +} + /*}}}*/ +static std::string transformFingergrpintsWithFilenames(std::string const &finger) /*{{{*/ +{ + // no check for existence as we could be chrooting later or such things + if (finger.empty() == false && finger[0] == '/') + return finger; + return transformFingergrpints(finger); +} + /*}}}*/ +// Introducer is set if additional keys may be introduced, for example /*{{{*/ +// by setting it to a filename or a complete key +static std::string NormalizeSignedBy(std::string SignedBy, bool const Introducer) +{ + // This is an embedded public pgp key, normalize spaces inside it and empty "." lines + if (Introducer && SignedBy.find("-----BEGIN PGP PUBLIC KEY BLOCK-----") != std::string::npos) { + std::istringstream is(SignedBy); + std::ostringstream os; + std::string line; + + while (std::getline(is, line)) { + line = APT::String::Strip(line); + // The special encoding for empty lines in deb822 + if (line == ".") + line=""; + os << line << std::endl; + } + return os.str(); + } + + // we could go all fancy and allow short/long/string matches as gpgv/apt-key does, + // but fingerprints are harder to fake than the others and this option is set once, + // not interactively all the time so easy to type is not really a concern. + std::transform(SignedBy.begin(), SignedBy.end(), SignedBy.begin(), [](char const c) { + return (isspace_ascii(c) == 0) ? c : ','; + }); + auto fingers = VectorizeString(SignedBy, ','); + auto const isAnEmptyString = [](std::string const &s) { return s.empty(); }; + fingers.erase(std::remove_if(fingers.begin(), fingers.end(), isAnEmptyString), fingers.end()); + if (unlikely(fingers.empty())) + return ""; + if (Introducer) + std::transform(fingers.begin(), fingers.end(), fingers.begin(), transformFingergrpintsWithFilenames); + else + std::transform(fingers.begin(), fingers.end(), fingers.begin(), transformFingergrpints); + if (std::any_of(fingers.begin(), fingers.end(), isAnEmptyString)) + return ""; + std::stringstream os; + std::copy(fingers.begin(), fingers.end() - 1, std::ostream_iterator<std::string>(os, ",")); + os << *fingers.rbegin(); + return os.str(); +} + /*}}}*/ + +class APT_HIDDEN debReleaseIndexPrivate /*{{{*/ +{ + public: + struct APT_HIDDEN debSectionEntry + { + std::string const sourcesEntry; + std::string const Name; + std::vector<std::string> const Targets; + std::vector<std::string> const Architectures; + std::vector<std::string> const Languages; + bool const UsePDiffs; + std::string const UseByHash; + }; + + std::vector<debSectionEntry> DebEntries; + std::vector<debSectionEntry> DebSrcEntries; + + metaIndex::TriState CheckValidUntil; + time_t ValidUntilMin; + time_t ValidUntilMax; + + metaIndex::TriState CheckDate; + time_t DateMaxFuture; + time_t NotBefore; + + std::vector<std::string> Architectures; + std::vector<std::string> NoSupportForAll; + std::vector<std::string> SupportedComponents; + std::map<std::string, std::string> const ReleaseOptions; + + explicit debReleaseIndexPrivate(std::map<std::string, std::string> const &Options) : CheckValidUntil(metaIndex::TRI_UNSET), ValidUntilMin(0), ValidUntilMax(0), CheckDate(metaIndex::TRI_UNSET), DateMaxFuture(0), NotBefore(0), ReleaseOptions(Options) {} +}; + /*}}}*/ +// ReleaseIndex::MetaIndex* - display helpers /*{{{*/ +std::string debReleaseIndex::MetaIndexInfo(const char *Type) const +{ + std::string Info = ::URI::ArchiveOnly(URI) + ' '; + if (Dist[Dist.size() - 1] == '/') + { + if (Dist != "/") + Info += Dist; + } + else + Info += Dist; + Info += " "; + Info += Type; + return Info; +} +std::string debReleaseIndex::Describe() const +{ + return MetaIndexInfo("Release"); +} + +std::string debReleaseIndex::MetaIndexFile(const char *Type) const +{ + return _config->FindDir("Dir::State::lists") + + URItoFileName(MetaIndexURI(Type)); +} +static std::string constructMetaIndexURI(std::string URI, std::string const &Dist, char const * const Type) +{ + if (Dist == "/") + ; + else if (Dist[Dist.size()-1] == '/') + URI += pkgAcquire::URIEncode(Dist); + else + URI += "dists/" + pkgAcquire::URIEncode(Dist) + "/"; + return URI + pkgAcquire::URIEncode(Type); +} +std::string debReleaseIndex::MetaIndexURI(const char *Type) const +{ + return constructMetaIndexURI(URI, Dist, Type); +} + /*}}}*/ +// ReleaseIndex Con- and Destructors /*{{{*/ +debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist, std::map<std::string, std::string> const &Options) : + metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate(Options)) +{} +debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist, bool const pTrusted, std::map<std::string, std::string> const &Options) : + metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate(Options)) +{ + Trusted = pTrusted ? TRI_YES : TRI_NO; +} +debReleaseIndex::~debReleaseIndex() { + if (d != NULL) + delete d; +} + /*}}}*/ +// ReleaseIndex::GetIndexTargets /*{{{*/ +static void GetIndexTargetsFor(char const * const Type, std::string const &URI, std::string const &Dist, + std::vector<debReleaseIndexPrivate::debSectionEntry> const &entries, + std::vector<IndexTarget> &IndexTargets, std::map<std::string, std::string> const &ReleaseOptions) +{ + bool const flatArchive = (Dist[Dist.length() - 1] == '/'); + std::string const baseURI = constructMetaIndexURI(URI, Dist, ""); + + std::string DefCompressionTypes; + { + std::vector<std::string> types = APT::Configuration::getCompressionTypes(); + if (types.empty() == false) + { + std::ostringstream os; + std::copy(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " ")); + os << *types.rbegin(); + DefCompressionTypes = os.str(); + } + } + std::string DefKeepCompressedAs; + { + std::vector<APT::Configuration::Compressor> comps = APT::Configuration::getCompressors(); + if (comps.empty() == false) + { + std::sort(comps.begin(), comps.end(), + [](APT::Configuration::Compressor const &a, APT::Configuration::Compressor const &b) { return a.Cost < b.Cost; }); + std::ostringstream os; + for (auto const &c : comps) + if (c.Cost != 0) + os << c.Extension.substr(1) << ' '; + DefKeepCompressedAs = os.str(); + } + DefKeepCompressedAs += "uncompressed"; + } + + std::vector<std::string> const NativeArchs = { _config->Find("APT::Architecture"), "implicit:all" }; + bool const GzipIndex = _config->FindB("Acquire::GzipIndexes", false); + for (std::vector<debReleaseIndexPrivate::debSectionEntry>::const_iterator E = entries.begin(); E != entries.end(); ++E) + { + for (std::vector<std::string>::const_iterator T = E->Targets.begin(); T != E->Targets.end(); ++T) + { +#define APT_T_CONFIG_STR(X, Y) _config->Find(std::string("Acquire::IndexTargets::") + Type + "::" + *T + "::" + (X), (Y)) +#define APT_T_CONFIG_BOOL(X, Y) _config->FindB(std::string("Acquire::IndexTargets::") + Type + "::" + *T + "::" + (X), (Y)) + std::string const tplMetaKey = APT_T_CONFIG_STR(flatArchive ? "flatMetaKey" : "MetaKey", ""); + std::string const tplShortDesc = APT_T_CONFIG_STR("ShortDescription", ""); + std::string const tplLongDesc = "$(SITE) " + APT_T_CONFIG_STR(flatArchive ? "flatDescription" : "Description", ""); + std::string const tplIdentifier = APT_T_CONFIG_STR("Identifier", *T); + bool const IsOptional = APT_T_CONFIG_BOOL("Optional", true); + bool const KeepCompressed = APT_T_CONFIG_BOOL("KeepCompressed", GzipIndex); + bool const DefaultEnabled = APT_T_CONFIG_BOOL("DefaultEnabled", true); + bool const UsePDiffs = APT_T_CONFIG_BOOL("PDiffs", E->UsePDiffs); + std::string const UseByHash = APT_T_CONFIG_STR("By-Hash", E->UseByHash); + std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes); + std::string KeepCompressedAs = APT_T_CONFIG_STR("KeepCompressedAs", ""); + std::string const FallbackOf = APT_T_CONFIG_STR("Fallback-Of", ""); +#undef APT_T_CONFIG_BOOL +#undef APT_T_CONFIG_STR + if (tplMetaKey.empty()) + continue; + + if (KeepCompressedAs.empty()) + KeepCompressedAs = DefKeepCompressedAs; + else + { + std::vector<std::string> const defKeep = VectorizeString(DefKeepCompressedAs, ' '); + std::vector<std::string> const valKeep = VectorizeString(KeepCompressedAs, ' '); + std::vector<std::string> keep; + for (auto const &val : valKeep) + { + if (val.empty()) + continue; + if (std::find(defKeep.begin(), defKeep.end(), val) == defKeep.end()) + continue; + keep.push_back(val); + } + if (std::find(keep.begin(), keep.end(), "uncompressed") == keep.end()) + keep.push_back("uncompressed"); + std::ostringstream os; + std::copy(keep.begin(), keep.end()-1, std::ostream_iterator<std::string>(os, " ")); + os << *keep.rbegin(); + KeepCompressedAs = os.str(); + } + + for (std::vector<std::string>::const_iterator L = E->Languages.begin(); L != E->Languages.end(); ++L) + { + if (*L == "none" && tplMetaKey.find("$(LANGUAGE)") != std::string::npos) + continue; + + for (std::vector<std::string>::const_iterator A = E->Architectures.begin(); A != E->Architectures.end(); ++A) + { + for (auto const &NativeArch: NativeArchs) + { + constexpr static auto BreakPoint = "$(NATIVE_ARCHITECTURE)"; + // available in templates + std::map<std::string, std::string> Options; + Options.insert(ReleaseOptions.begin(), ReleaseOptions.end()); + if (tplMetaKey.find("$(COMPONENT)") != std::string::npos) + Options.emplace("COMPONENT", E->Name); + if (tplMetaKey.find("$(LANGUAGE)") != std::string::npos) + Options.emplace("LANGUAGE", *L); + if (tplMetaKey.find("$(ARCHITECTURE)") != std::string::npos) + Options.emplace("ARCHITECTURE", (*A == "implicit:all") ? "all" : *A); + else if (tplMetaKey.find("$(NATIVE_ARCHITECTURE)") != std::string::npos) + Options.emplace("ARCHITECTURE", (NativeArch == "implicit:all") ? "all" : NativeArch); + if (tplMetaKey.find("$(NATIVE_ARCHITECTURE)") != std::string::npos) + Options.emplace("NATIVE_ARCHITECTURE", (NativeArch == "implicit:all") ? "all" : NativeArch); + + std::string MetaKey = tplMetaKey; + std::string ShortDesc = tplShortDesc; + std::string LongDesc = tplLongDesc; + std::string Identifier = tplIdentifier; + for (std::map<std::string, std::string>::const_iterator O = Options.begin(); O != Options.end(); ++O) + { + std::string const varname = "$(" + O->first + ")"; + MetaKey = SubstVar(MetaKey, varname, O->second); + ShortDesc = SubstVar(ShortDesc, varname, O->second); + LongDesc = SubstVar(LongDesc, varname, O->second); + Identifier = SubstVar(Identifier, varname, O->second); + } + + { + auto const dup = std::find_if(IndexTargets.begin(), IndexTargets.end(), [&](IndexTarget const &IT) { + return MetaKey == IT.MetaKey && baseURI == IT.Option(IndexTarget::BASE_URI) && + E->sourcesEntry == IT.Option(IndexTarget::SOURCESENTRY) && *T == IT.Option(IndexTarget::CREATED_BY); + }); + if (dup != IndexTargets.end()) + { + if (tplMetaKey.find(BreakPoint) == std::string::npos) + break; + continue; + } + } + + { + auto const dup = std::find_if(IndexTargets.begin(), IndexTargets.end(), [&](IndexTarget const &IT) { + return MetaKey == IT.MetaKey && baseURI == IT.Option(IndexTarget::BASE_URI) && + E->sourcesEntry == IT.Option(IndexTarget::SOURCESENTRY) && *T != IT.Option(IndexTarget::CREATED_BY); + }); + if (dup != IndexTargets.end()) + { + std::string const dupT = dup->Option(IndexTarget::CREATED_BY); + std::string const dupEntry = dup->Option(IndexTarget::SOURCESENTRY); + //TRANSLATOR: an identifier like Packages; Releasefile key indicating + // a file like main/binary-amd64/Packages; another identifier like Contents; + // filename and linenumber of the sources.list entry currently parsed + _error->Warning(_("Target %s wants to acquire the same file (%s) as %s from source %s"), + T->c_str(), MetaKey.c_str(), dupT.c_str(), dupEntry.c_str()); + if (tplMetaKey.find(BreakPoint) == std::string::npos) + break; + continue; + } + } + + { + auto const dup = std::find_if(IndexTargets.begin(), IndexTargets.end(), [&](IndexTarget const &T) { + return MetaKey == T.MetaKey && baseURI == T.Option(IndexTarget::BASE_URI) && + E->sourcesEntry != T.Option(IndexTarget::SOURCESENTRY); + }); + if (dup != IndexTargets.end()) + { + std::string const dupEntry = dup->Option(IndexTarget::SOURCESENTRY); + if (T->find("legacy") == std::string::npos) + { + //TRANSLATOR: an identifier like Packages; Releasefile key indicating + // a file like main/binary-amd64/Packages; filename and linenumber of + // two sources.list entries + _error->Warning(_("Target %s (%s) is configured multiple times in %s and %s"), + T->c_str(), MetaKey.c_str(), dupEntry.c_str(), E->sourcesEntry.c_str()); + } + if (tplMetaKey.find(BreakPoint) == std::string::npos) + break; + continue; + } + } + + // not available in templates, but in the indextarget + Options.insert(std::make_pair("IDENTIFIER", Identifier)); + Options.insert(std::make_pair("TARGET_OF", Type)); + Options.insert(std::make_pair("CREATED_BY", *T)); + Options.insert(std::make_pair("FALLBACK_OF", FallbackOf)); + Options.insert(std::make_pair("PDIFFS", UsePDiffs ? "yes" : "no")); + Options.insert(std::make_pair("BY_HASH", UseByHash)); + Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no")); + Options.insert(std::make_pair("COMPRESSIONTYPES", CompressionTypes)); + Options.insert(std::make_pair("KEEPCOMPRESSEDAS", KeepCompressedAs)); + Options.insert(std::make_pair("SOURCESENTRY", E->sourcesEntry)); + + bool IsOpt = IsOptional; + { + auto const arch = Options.find("ARCHITECTURE"); + if (arch != Options.end() && arch->second == "all") + { + // one of them must be implicit:all then + if (*A != "all" && NativeArch != "all") + IsOpt = true; + else // user used arch=all explicitly + Options.emplace("Force-Support-For-All", "yes"); + } + } + + IndexTarget Target( + MetaKey, + ShortDesc, + LongDesc, + baseURI + MetaKey, + IsOpt, + KeepCompressed, + Options + ); + IndexTargets.push_back(Target); + + if (tplMetaKey.find(BreakPoint) == std::string::npos) + break; + } + + if (tplMetaKey.find("$(ARCHITECTURE)") == std::string::npos) + break; + + } + + if (tplMetaKey.find("$(LANGUAGE)") == std::string::npos) + break; + + } + + } + } +} +std::vector<IndexTarget> debReleaseIndex::GetIndexTargets() const +{ + std::vector<IndexTarget> IndexTargets; + GetIndexTargetsFor("deb-src", URI, Dist, d->DebSrcEntries, IndexTargets, d->ReleaseOptions); + GetIndexTargetsFor("deb", URI, Dist, d->DebEntries, IndexTargets, d->ReleaseOptions); + return IndexTargets; +} + /*}}}*/ +void debReleaseIndex::AddComponent(std::string const &sourcesEntry, /*{{{*/ + bool const isSrc, std::string const &Name, + std::vector<std::string> const &Targets, + std::vector<std::string> const &Architectures, + std::vector<std::string> Languages, + bool const usePDiffs, std::string const &useByHash) +{ + if (Languages.empty() == true) + Languages.push_back("none"); + debReleaseIndexPrivate::debSectionEntry const entry = { + sourcesEntry, Name, Targets, Architectures, Languages, usePDiffs, useByHash + }; + if (isSrc) + d->DebSrcEntries.push_back(entry); + else + d->DebEntries.push_back(entry); +} + /*}}}*/ +std::string debReleaseIndex::ArchiveURI(std::string const &File) const /*{{{*/ +{ + if (File.empty()) + return URI; + return URI + pkgAcquire::URIEncode(File); +} + /*}}}*/ + +bool debReleaseIndex::Load(std::string const &Filename, std::string * const ErrorText)/*{{{*/ +{ + LoadedSuccessfully = TRI_NO; + FileFd Fd; + if (OpenMaybeClearSignedFile(Filename, Fd) == false) + return false; + + pkgTagFile TagFile(&Fd, Fd.Size()); + if (Fd.IsOpen() == false || Fd.Failed()) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("Unable to parse Release file %s"),Filename.c_str()); + return false; + } + + pkgTagSection Section; + const char *Start, *End; + if (TagFile.Step(Section) == false) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("No sections in Release file %s"), Filename.c_str()); + return false; + } + // FIXME: find better tag name + SupportsAcquireByHash = Section.FindB("Acquire-By-Hash", false); + + Origin = Section.FindS("Origin"); + Label = Section.FindS("Label"); + Version = Section.FindS("Version"); + Suite = Section.FindS("Suite"); + Codename = Section.FindS("Codename"); + ReleaseNotes = Section.FindS("Release-Notes"); + { + std::string const archs = Section.FindS("Architectures"); + if (archs.empty() == false) + d->Architectures = VectorizeString(archs, ' '); + } + { + std::string const targets = Section.FindS("No-Support-for-Architecture-all"); + if (targets.empty() == false) + d->NoSupportForAll = VectorizeString(targets, ' '); + } + for (auto const &comp: VectorizeString(Section.FindS("Components"), ' ')) + { + if (comp.empty()) + continue; + auto const pos = comp.find_last_of('/'); + if (pos != std::string::npos) // e.g. security.debian.org uses this style + d->SupportedComponents.push_back(comp.substr(pos + 1)); + d->SupportedComponents.push_back(std::move(comp)); + } + { + decltype(pkgCache::ReleaseFile::Flags) flags = 0; + Section.FindFlag("NotAutomatic", flags, pkgCache::Flag::NotAutomatic); + signed short defaultpin = 500; + if ((flags & pkgCache::Flag::NotAutomatic) == pkgCache::Flag::NotAutomatic) + { + Section.FindFlag("ButAutomaticUpgrades", flags, pkgCache::Flag::ButAutomaticUpgrades); + if ((flags & pkgCache::Flag::ButAutomaticUpgrades) == pkgCache::Flag::ButAutomaticUpgrades) + defaultpin = 100; + else + defaultpin = 1; + } + DefaultPin = defaultpin; + } + + bool FoundHashSum = false; + bool FoundStrongHashSum = false; + for (auto const hashinfo : HashString::SupportedHashesInfo()) + { + if (not Section.Find(hashinfo.namekey, Start, End)) + continue; + + std::string Name; + std::string Hash; + unsigned long long Size; + while (Start < End) + { + if (!parseSumData(Start, End, Name, Hash, Size)) + return false; + + HashString const hs(hashinfo.name.to_string(), Hash); + if (Entries.find(Name) == Entries.end()) + { + metaIndex::checkSum *Sum = new metaIndex::checkSum; + Sum->MetaKeyFilename = Name; + Sum->Size = Size; + Sum->Hashes.FileSize(Size); + Entries[Name] = Sum; + } + Entries[Name]->Hashes.push_back(hs); + FoundHashSum = true; + if (FoundStrongHashSum == false && hs.usable() == true) + FoundStrongHashSum = true; + } + } + + bool AuthPossible = false; + if(FoundHashSum == false) + _error->Warning(_("No Hash entry in Release file %s"), Filename.c_str()); + else if(FoundStrongHashSum == false) + _error->Warning(_("No Hash entry in Release file %s which is considered strong enough for security purposes"), Filename.c_str()); + else + AuthPossible = true; + + std::string const StrDate = Section.FindS("Date"); + if (RFC1123StrToTime(StrDate, Date) == false) + { + _error->Warning( _("Invalid '%s' entry in Release file %s"), "Date", Filename.c_str()); + Date = 0; + } + + bool CheckDate = _config->FindB("Acquire::Check-Date", true); + if (d->CheckDate == metaIndex::TRI_NO) + CheckDate = false; + else if (d->CheckDate == metaIndex::TRI_YES) + CheckDate = true; + + if (CheckDate) + { + auto const Label = GetLabel(); + // get the user settings for this archive + time_t MaxFuture = d->DateMaxFuture; + if (MaxFuture == 0) + { + MaxFuture = _config->FindI("Acquire::Max-FutureTime", 10); + if (Label.empty() == false) + MaxFuture = _config->FindI(("Acquire::Max-FutureTime::" + Label).c_str(), MaxFuture); + } + + d->NotBefore = Date - MaxFuture; + + bool CheckValidUntil = _config->FindB("Acquire::Check-Valid-Until", true); + if (d->CheckValidUntil == metaIndex::TRI_NO) + CheckValidUntil = false; + else if (d->CheckValidUntil == metaIndex::TRI_YES) + CheckValidUntil = true; + + if (CheckValidUntil == true) + { + std::string const StrValidUntil = Section.FindS("Valid-Until"); + + // if we have a Valid-Until header in the Release file, use it as default + if (StrValidUntil.empty() == false) + { + if (RFC1123StrToTime(StrValidUntil, ValidUntil) == false) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("Invalid '%s' entry in Release file %s"), "Valid-Until", Filename.c_str()); + return false; + } + } + auto const Label = GetLabel(); + // get the user settings for this archive and use what expires earlier + time_t MaxAge = d->ValidUntilMax; + if (MaxAge == 0) + { + MaxAge = _config->FindI("Acquire::Max-ValidTime", 0); + if (Label.empty() == false) + MaxAge = _config->FindI(("Acquire::Max-ValidTime::" + Label).c_str(), MaxAge); + } + time_t MinAge = d->ValidUntilMin; + if (MinAge == 0) + { + MinAge = _config->FindI("Acquire::Min-ValidTime", 0); + if (Label.empty() == false) + MinAge = _config->FindI(("Acquire::Min-ValidTime::" + Label).c_str(), MinAge); + } + + if (MinAge != 0 || ValidUntil != 0 || MaxAge != 0) + { + if (MinAge != 0 && ValidUntil != 0) + { + time_t const min_date = Date + MinAge; + if (ValidUntil < min_date) + ValidUntil = min_date; + } + if (MaxAge != 0 && Date != 0) + { + time_t const max_date = Date + MaxAge; + if (ValidUntil == 0 || ValidUntil > max_date) + ValidUntil = max_date; + } + } + } + } + + /* as the Release file is parsed only after it was verified, the Signed-By field + does not effect the current, but the "next" Release file */ + auto Sign = Section.FindS("Signed-By"); + if (Sign.empty() == false) + { + SignedBy = NormalizeSignedBy(Sign, false); + if (SignedBy.empty() && ErrorText != NULL) + strprintf(*ErrorText, _("Invalid '%s' entry in Release file %s"), "Signed-By", Filename.c_str()); + } + + if (AuthPossible) + LoadedSuccessfully = TRI_YES; + return AuthPossible; +} + /*}}}*/ +time_t debReleaseIndex::GetNotBefore() const /*{{{*/ +{ + return d->NotBefore; +} + /*}}}*/ +metaIndex * debReleaseIndex::UnloadedClone() const /*{{{*/ +{ + if (Trusted == TRI_NO) + return new debReleaseIndex(URI, Dist, false, d->ReleaseOptions); + else if (Trusted == TRI_YES) + return new debReleaseIndex(URI, Dist, true, d->ReleaseOptions); + else + return new debReleaseIndex(URI, Dist, d->ReleaseOptions); +} + /*}}}*/ +bool debReleaseIndex::parseSumData(const char *&Start, const char *End, /*{{{*/ + std::string &Name, std::string &Hash, unsigned long long &Size) +{ + Name = ""; + Hash = ""; + Size = 0; + /* Skip over the first blank */ + while ((*Start == '\t' || *Start == ' ' || *Start == '\n' || *Start == '\r') + && Start < End) + Start++; + if (Start >= End) + return false; + + /* Move EntryEnd to the end of the first entry (the hash) */ + const char *EntryEnd = Start; + while ((*EntryEnd != '\t' && *EntryEnd != ' ') + && EntryEnd < End) + EntryEnd++; + if (EntryEnd == End) + return false; + + Hash.append(Start, EntryEnd-Start); + + /* Skip over intermediate blanks */ + Start = EntryEnd; + while (*Start == '\t' || *Start == ' ') + Start++; + if (Start >= End) + return false; + + EntryEnd = Start; + /* Find the end of the second entry (the size) */ + while ((*EntryEnd != '\t' && *EntryEnd != ' ' ) + && EntryEnd < End) + EntryEnd++; + if (EntryEnd == End) + return false; + + Size = strtoull (Start, NULL, 10); + + /* Skip over intermediate blanks */ + Start = EntryEnd; + while (*Start == '\t' || *Start == ' ') + Start++; + if (Start >= End) + return false; + + EntryEnd = Start; + /* Find the end of the third entry (the filename) */ + while ((*EntryEnd != '\t' && *EntryEnd != ' ' && + *EntryEnd != '\n' && *EntryEnd != '\r') + && EntryEnd < End) + EntryEnd++; + + Name.append(Start, EntryEnd-Start); + Start = EntryEnd; //prepare for the next round + return true; +} + /*}}}*/ + +bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/ +{ +#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, false, d->ReleaseOptions) + pkgAcqMetaClearSig * const TransactionManager = new pkgAcqMetaClearSig(Owner, + APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"), this); +#undef APT_TARGET + // special case for --print-uris + if (GetAll) + for (auto const &Target: GetIndexTargets()) + if (Target.Option(IndexTarget::FALLBACK_OF).empty()) + new pkgAcqIndex(Owner, TransactionManager, Target); + + return true; +} + /*}}}*/ +// ReleaseIndex::Set* TriState options /*{{{*/ +bool debReleaseIndex::SetTrusted(TriState const pTrusted) +{ + if (Trusted == TRI_UNSET) + Trusted = pTrusted; + else if (Trusted != pTrusted) + // TRANSLATOR: The first is an option name from sources.list manpage, the other two URI and Suite + return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), "Trusted", URI.c_str(), Dist.c_str()); + return true; +} +bool debReleaseIndex::SetCheckValidUntil(TriState const pCheckValidUntil) +{ + if (d->CheckValidUntil == TRI_UNSET) + d->CheckValidUntil = pCheckValidUntil; + else if (d->CheckValidUntil != pCheckValidUntil) + return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), "Check-Valid-Until", URI.c_str(), Dist.c_str()); + return true; +} +bool debReleaseIndex::SetValidUntilMin(time_t const Valid) +{ + if (d->ValidUntilMin == 0) + d->ValidUntilMin = Valid; + else if (d->ValidUntilMin != Valid) + return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), "Min-ValidTime", URI.c_str(), Dist.c_str()); + return true; +} +bool debReleaseIndex::SetValidUntilMax(time_t const Valid) +{ + if (d->ValidUntilMax == 0) + d->ValidUntilMax = Valid; + else if (d->ValidUntilMax != Valid) + return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), "Max-ValidTime", URI.c_str(), Dist.c_str()); + return true; +} +bool debReleaseIndex::SetCheckDate(TriState const pCheckDate) +{ + if (d->CheckDate == TRI_UNSET) + d->CheckDate = pCheckDate; + else if (d->CheckDate != pCheckDate) + return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), "Check-Date", URI.c_str(), Dist.c_str()); + return true; +} +bool debReleaseIndex::SetDateMaxFuture(time_t const DateMaxFuture) +{ + if (d->DateMaxFuture == 0) + d->DateMaxFuture = DateMaxFuture; + else if (d->DateMaxFuture != DateMaxFuture) + return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), "Date-Max-Future", URI.c_str(), Dist.c_str()); + return true; +} +bool debReleaseIndex::SetSignedBy(std::string const &pSignedBy) +{ + if (SignedBy.empty() == true && pSignedBy.empty() == false) + { + SignedBy = NormalizeSignedBy(pSignedBy, true); + if (SignedBy.empty()) + _error->Error(_("Invalid value set for option %s regarding source %s %s (%s)"), "Signed-By", URI.c_str(), Dist.c_str(), "not a fingerprint"); + } + else + { + auto const normalSignedBy = NormalizeSignedBy(pSignedBy, true); + if (normalSignedBy != SignedBy) + return _error->Error(_("Conflicting values set for option %s regarding source %s %s: %s != %s"), "Signed-By", URI.c_str(), Dist.c_str(), SignedBy.c_str(), normalSignedBy.c_str()); + } + return true; +} + /*}}}*/ +// ReleaseIndex::IsTrusted /*{{{*/ +bool debReleaseIndex::IsTrusted() const +{ + if (Trusted == TRI_YES) + return true; + else if (Trusted == TRI_NO) + return false; + + + if(_config->FindB("APT::Authentication::TrustCDROM", false)) + if(URI.substr(0,strlen("cdrom:")) == "cdrom:") + return true; + + if (FileExists(MetaIndexFile("Release.gpg"))) + return true; + + return FileExists(MetaIndexFile("InRelease")); +} + /*}}}*/ +bool debReleaseIndex::IsArchitectureSupported(std::string const &arch) const/*{{{*/ +{ + if (d->Architectures.empty()) + return true; + return std::find(d->Architectures.begin(), d->Architectures.end(), arch) != d->Architectures.end(); +} + /*}}}*/ +bool debReleaseIndex::IsArchitectureAllSupportedFor(IndexTarget const &target) const/*{{{*/ +{ + if (target.Options.find("Force-Support-For-All") != target.Options.end()) + return true; + if (IsArchitectureSupported("all") == false) + return false; + if (d->NoSupportForAll.empty()) + return true; + return std::find(d->NoSupportForAll.begin(), d->NoSupportForAll.end(), target.Option(IndexTarget::CREATED_BY)) == d->NoSupportForAll.end(); +} + /*}}}*/ +bool debReleaseIndex::HasSupportForComponent(std::string const &component) const/*{{{*/ +{ + if (d->SupportedComponents.empty()) + return true; + return std::find(d->SupportedComponents.begin(), d->SupportedComponents.end(), component) != d->SupportedComponents.end(); +} + /*}}}*/ +std::vector <pkgIndexFile *> *debReleaseIndex::GetIndexFiles() /*{{{*/ +{ + if (Indexes != NULL) + return Indexes; + + Indexes = new std::vector<pkgIndexFile*>(); + bool const istrusted = IsTrusted(); + for (auto const &T: GetIndexTargets()) + { + std::string const TargetName = T.Option(IndexTarget::CREATED_BY); + if (TargetName == "Packages") + Indexes->push_back(new debPackagesIndex(T, istrusted)); + else if (TargetName == "Sources") + Indexes->push_back(new debSourcesIndex(T, istrusted)); + else if (TargetName == "Translations") + Indexes->push_back(new debTranslationsIndex(T)); + } + return Indexes; +} + /*}}}*/ +std::map<std::string, std::string> debReleaseIndex::GetReleaseOptions() +{ + return d->ReleaseOptions; +} + +static bool ReleaseFileName(debReleaseIndex const * const That, std::string &ReleaseFile)/*{{{*/ +{ + ReleaseFile = That->MetaIndexFile("InRelease"); + bool releaseExists = false; + if (FileExists(ReleaseFile) == true) + releaseExists = true; + else + { + ReleaseFile = That->MetaIndexFile("Release"); + if (FileExists(ReleaseFile)) + releaseExists = true; + } + return releaseExists; +} + /*}}}*/ +bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/*{{{*/ +{ + std::string ReleaseFile; + bool const releaseExists = ReleaseFileName(this, ReleaseFile); + + ::URI Tmp(URI); + if (Gen.SelectReleaseFile(ReleaseFile, Tmp.Host) == false) + return _error->Error("Problem with SelectReleaseFile %s", ReleaseFile.c_str()); + + if (releaseExists == false) + return true; + + FileFd Rel; + // Beware: The 'Release' file might be clearsigned in case the + // signature for an 'InRelease' file couldn't be checked + if (OpenMaybeClearSignedFile(ReleaseFile, Rel) == false) + return false; + + // Store the IMS information + pkgCache::RlsFileIterator File = Gen.GetCurRlsFile(); + pkgCacheGenerator::Dynamic<pkgCache::RlsFileIterator> DynFile(File); + // Rel can't be used as this is potentially a temporary file + struct stat Buf; + if (stat(ReleaseFile.c_str(), &Buf) != 0) + return _error->Errno("fstat", "Unable to stat file %s", ReleaseFile.c_str()); + File->Size = Buf.st_size; + File->mtime = Buf.st_mtime; + + pkgTagFile TagFile(&Rel, Rel.Size()); + pkgTagSection Section; + if (Rel.IsOpen() == false || Rel.Failed() || TagFile.Step(Section) == false) + return false; + + std::string data; + #define APT_INRELEASE(TYPE, TAG, STORE) \ + data = Section.FindS(TAG); \ + if (data.empty() == false) \ + { \ + map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::TYPE, data); \ + if (storage == 0) return false; \ + STORE = storage; \ + } + APT_INRELEASE(MIXED, "Suite", File->Archive) + APT_INRELEASE(VERSIONNUMBER, "Version", File->Version) + APT_INRELEASE(MIXED, "Origin", File->Origin) + APT_INRELEASE(MIXED, "Codename", File->Codename) + APT_INRELEASE(MIXED, "Label", File->Label) + #undef APT_INRELEASE + Section.FindFlag("NotAutomatic", File->Flags, pkgCache::Flag::NotAutomatic); + Section.FindFlag("ButAutomaticUpgrades", File->Flags, pkgCache::Flag::ButAutomaticUpgrades); + + return true; +} + /*}}}*/ +// ReleaseIndex::FindInCache - Find this index /*{{{*/ +pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache, bool const ModifyCheck) const +{ + std::string ReleaseFile; + bool const releaseExists = ReleaseFileName(this, ReleaseFile); + + pkgCache::RlsFileIterator File = Cache.RlsFileBegin(); + for (; File.end() == false; ++File) + { + if (File->FileName == 0 || ReleaseFile != File.FileName()) + continue; + + // empty means the file does not exist by "design" + if (ModifyCheck == false || (releaseExists == false && File->Size == 0)) + return File; + + struct stat St; + if (stat(File.FileName(),&St) != 0) + { + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "ReleaseIndex::FindInCache - stat failed on " << File.FileName() << std::endl; + return pkgCache::RlsFileIterator(Cache); + } + if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime) + { + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "ReleaseIndex::FindInCache - size (" << St.st_size << " <> " << File->Size + << ") or mtime (" << St.st_mtime << " <> " << File->mtime + << ") doesn't match for " << File.FileName() << std::endl; + return pkgCache::RlsFileIterator(Cache); + } + return File; + } + + return File; +} + /*}}}*/ + +class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ +{ + static std::optional<std::vector<std::string>> getDefaultSetOf(std::string const &Name, + std::map<std::string, std::string> const &Options) + { + auto const val = Options.find(Name); + if (val != Options.end()) + return VectorizeString(val->second, ','); + return {}; + } + static std::vector<std::string> applyPlusMinusOptions(std::string const &Name, + std::map<std::string, std::string> const &Options, std::vector<std::string> &&Values) + { + auto val = Options.find(Name + "+"); + if (val != Options.end()) + { + std::vector<std::string> const plus = VectorizeString(val->second, ','); + std::copy_if(plus.begin(), plus.end(), std::back_inserter(Values), [&Values](std::string const &v) { + return std::find(Values.begin(), Values.end(), v) == Values.end(); + }); + } + if ((val = Options.find(Name + "-")) != Options.end()) + { + std::vector<std::string> const minus = VectorizeString(val->second, ','); + Values.erase(std::remove_if(Values.begin(), Values.end(), [&minus](std::string const &v) { + return std::find(minus.begin(), minus.end(), v) != minus.end(); + }), Values.end()); + } + return std::move(Values); + } + static std::vector<std::string> parsePlusMinusOptions(std::string const &Name, + std::map<std::string, std::string> const &Options, std::vector<std::string> const &defaultValues) + { + return applyPlusMinusOptions(Name, Options, getDefaultSetOf(Name, Options).value_or(defaultValues)); + } + static std::vector<std::string> parsePlusMinusArchOptions(std::string const &Name, + std::map<std::string, std::string> const &Options) + { + std::vector<std::string> Values; + if (auto opt = getDefaultSetOf(Name, Options); opt.has_value()) + Values = opt.value(); + else + { + Values = APT::Configuration::getArchitectures(); + auto veryforeign = _config->FindVector("APT::BarbarianArchitectures"); + Values.reserve(Values.size() + veryforeign.size()); + std::move(veryforeign.begin(), veryforeign.end(), std::back_inserter(Values)); + } + // all is a very special architecture users shouldn't be concerned with explicitly + // but if the user does, do not override the choice + auto const val = Options.find(Name + "-"); + if (val != Options.end()) + { + std::vector<std::string> const minus = VectorizeString(val->second, ','); + if (std::find(minus.begin(), minus.end(), "all") != minus.end()) + return applyPlusMinusOptions(Name, Options, std::move(Values)); + } + Values = applyPlusMinusOptions(Name, Options, std::move(Values)); + if (std::find(Values.begin(), Values.end(), "all") == Values.end()) + Values.push_back("implicit:all"); + return Values; + } + static std::vector<std::string> parsePlusMinusTargetOptions(char const * const Name, + std::map<std::string, std::string> const &Options) + { + std::vector<std::string> const alltargets = _config->FindVector(std::string("Acquire::IndexTargets::") + Name, "", true); + std::vector<std::string> deftargets; + deftargets.reserve(alltargets.size()); + std::copy_if(alltargets.begin(), alltargets.end(), std::back_inserter(deftargets), [&](std::string const &t) { + std::string c = "Acquire::IndexTargets::"; + c.append(Name).append("::").append(t).append("::DefaultEnabled"); + return _config->FindB(c, true); + }); + std::vector<std::string> mytargets = parsePlusMinusOptions("target", Options, deftargets); + for (auto const &target : alltargets) + { + std::map<std::string, std::string>::const_iterator const opt = Options.find(target); + if (opt == Options.end()) + continue; + auto const idMatch = [&](std::string const &t) { + return target == _config->Find(std::string("Acquire::IndexTargets::") + Name + "::" + t + "::Identifier", t); + }; + if (StringToBool(opt->second)) + std::copy_if(alltargets.begin(), alltargets.end(), std::back_inserter(mytargets), idMatch); + else + mytargets.erase(std::remove_if(mytargets.begin(), mytargets.end(), idMatch), mytargets.end()); + } + // if we can't order it in a 1000 steps we give up… probably a cycle + for (auto i = 0; i < 1000; ++i) + { + bool Changed = false; + for (auto t = mytargets.begin(); t != mytargets.end(); ++t) + { + std::string const fallback = _config->Find(std::string("Acquire::IndexTargets::") + Name + "::" + *t + "::Fallback-Of"); + if (fallback.empty()) + continue; + auto const faller = std::find(mytargets.begin(), mytargets.end(), fallback); + if (faller == mytargets.end() || faller < t) + continue; + Changed = true; + auto const tv = *t; + mytargets.erase(t); + mytargets.emplace_back(tv); + break; + } + if (Changed == false) + break; + } + // remove duplicates without changing the order (in first appearance) + { + std::set<std::string> seenOnce; + mytargets.erase(std::remove_if(mytargets.begin(), mytargets.end(), [&](std::string const &t) { + return seenOnce.insert(t).second == false; + }), mytargets.end()); + } + return mytargets; + } + + metaIndex::TriState GetTriStateOption(std::map<std::string, std::string>const &Options, char const * const name) const + { + std::map<std::string, std::string>::const_iterator const opt = Options.find(name); + if (opt != Options.end()) + return StringToBool(opt->second, false) ? metaIndex::TRI_YES : metaIndex::TRI_NO; + return metaIndex::TRI_DONTCARE; + } + + static time_t GetTimeOption(std::map<std::string, std::string>const &Options, char const * const name) + { + std::map<std::string, std::string>::const_iterator const opt = Options.find(name); + if (opt == Options.end()) + return 0; + return strtoull(opt->second.c_str(), NULL, 10); + } + + static bool GetBoolOption(std::map<std::string, std::string> const &Options, char const * const name, bool const defVal) + { + std::map<std::string, std::string>::const_iterator const opt = Options.find(name); + if (opt == Options.end()) + return defVal; + return StringToBool(opt->second, defVal); + } + + static std::vector<std::string> GetMapKeys(std::map<std::string, std::string> const &Options) + { + std::vector<std::string> ret; + ret.reserve(Options.size()); + std::transform(Options.begin(), Options.end(), std::back_inserter(ret), + [](auto &&O) { return O.first; }); + std::sort(ret.begin(), ret.end()); + return ret; + } + + static bool MapsAreEqual(std::map<std::string, std::string> const &OptionsA, + std::map<std::string, std::string> const &OptionsB, + std::string const &URI, std::string const &Dist) + { + auto const KeysA = GetMapKeys(OptionsA); + auto const KeysB = GetMapKeys(OptionsB); + auto const m = std::mismatch(KeysA.begin(), KeysA.end(), KeysB.begin()); + if (m.first != KeysA.end()) + { + if (std::find(KeysB.begin(), KeysB.end(), *m.first) == KeysB.end()) + return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), m.first->c_str(), "<set>", "<unset>"); + else + return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), m.second->c_str(), "<set>", "<unset>"); + } + if (m.second != KeysB.end()) + { + if (std::find(KeysA.begin(), KeysA.end(), *m.second) == KeysA.end()) + return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), m.first->c_str(), "<set>", "<unset>"); + else + return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), m.second->c_str(), "<set>", "<unset>"); + } + for (auto&& key: KeysA) + { + if (key == "BASE_URI" || key == "REPO_URI" || key == "SITE" || key == "RELEASE") + continue; + auto const a = OptionsA.find(key); + auto const b = OptionsB.find(key); + if (unlikely(a == OptionsA.end() || b == OptionsB.end()) || a->second != b->second) + return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), key.c_str(), URI.c_str(), Dist.c_str()); + } + return true; + } + + static debReleaseIndex * GetDebReleaseIndexBy(std::vector<metaIndex *> &List, std::string const &URI, + std::string const &Dist, std::map<std::string, std::string> const &Options) + { + std::map<std::string, std::string> ReleaseOptions{{ + {"BASE_URI", constructMetaIndexURI(URI, Dist, "")}, + {"REPO_URI", URI}, + {"SITE", ::URI::ArchiveOnly(URI)}, + {"RELEASE", (Dist == "/") ? "" : Dist}, + }}; + if (GetBoolOption(Options, "allow-insecure", _config->FindB("Acquire::AllowInsecureRepositories"))) + ReleaseOptions.emplace("ALLOW_INSECURE", "true"); + if (GetBoolOption(Options, "allow-weak", _config->FindB("Acquire::AllowWeakRepositories"))) + ReleaseOptions.emplace("ALLOW_WEAK", "true"); + if (GetBoolOption(Options, "allow-downgrade-to-insecure", _config->FindB("Acquire::AllowDowngradeToInsecureRepositories"))) + ReleaseOptions.emplace("ALLOW_DOWNGRADE_TO_INSECURE", "true"); + + auto InReleasePath = Options.find("inrelease-path"); + if (InReleasePath != Options.end()) + ReleaseOptions.emplace("INRELEASE_PATH", InReleasePath->second); + + debReleaseIndex * Deb = nullptr; + std::string const FileName = URItoFileName(constructMetaIndexURI(URI, Dist, "Release")); + for (auto const &I: List) + { + // We only worry about debian entries here + if (strcmp(I->GetType(), "deb") != 0) + continue; + + auto const D = dynamic_cast<debReleaseIndex*>(I); + if (unlikely(D == nullptr)) + continue; + + /* This check ensures that there will be only one Release file + queued for all the Packages files and Sources files it + corresponds to. */ + if (URItoFileName(D->MetaIndexURI("Release")) == FileName) + { + if (MapsAreEqual(ReleaseOptions, D->GetReleaseOptions(), URI, Dist) == false) + return nullptr; + Deb = D; + break; + } + } + + // No currently created Release file indexes this entry, so we create a new one. + if (Deb == nullptr) + { + Deb = new debReleaseIndex(URI, Dist, ReleaseOptions); + List.push_back(Deb); + } + return Deb; + } + + protected: + + bool CreateItemInternal(std::vector<metaIndex *> &List, std::string const &URI, + std::string const &Dist, std::string const &Section, + bool const &IsSrc, std::map<std::string, std::string> const &Options) const + { + auto const Deb = GetDebReleaseIndexBy(List, URI, Dist, Options); + if (Deb == nullptr) + return false; + + bool const UsePDiffs = GetBoolOption(Options, "pdiffs", _config->FindB("Acquire::PDiffs", true)); + + std::string UseByHash = _config->Find("APT::Acquire::By-Hash", "yes"); + UseByHash = _config->Find("Acquire::By-Hash", UseByHash); + { + std::string const host = ::URI(URI).Host; + if (host.empty() == false) + { + UseByHash = _config->Find("APT::Acquire::" + host + "::By-Hash", UseByHash); + UseByHash = _config->Find("Acquire::" + host + "::By-Hash", UseByHash); + } + std::map<std::string, std::string>::const_iterator const opt = Options.find("by-hash"); + if (opt != Options.end()) + UseByHash = opt->second; + } + + auto const entry = Options.find("sourceslist-entry"); + Deb->AddComponent( + entry->second, + IsSrc, + Section, + parsePlusMinusTargetOptions(Name, Options), + parsePlusMinusArchOptions("arch", Options), + parsePlusMinusOptions("lang", Options, APT::Configuration::getLanguages(true)), + UsePDiffs, + UseByHash + ); + + if (Deb->SetTrusted(GetTriStateOption(Options, "trusted")) == false || + Deb->SetCheckValidUntil(GetTriStateOption(Options, "check-valid-until")) == false || + Deb->SetValidUntilMax(GetTimeOption(Options, "valid-until-max")) == false || + Deb->SetValidUntilMin(GetTimeOption(Options, "valid-until-min")) == false || + Deb->SetCheckDate(GetTriStateOption(Options, "check-date")) == false || + Deb->SetDateMaxFuture(GetTimeOption(Options, "date-max-future")) == false) + return false; + + std::map<std::string, std::string>::const_iterator const signedby = Options.find("signed-by"); + if (signedby == Options.end()) + { + bool alreadySet = false; + std::string filename; + if (ReleaseFileName(Deb, filename)) + { + auto OldDeb = Deb->UnloadedClone(); + _error->PushToStack(); + OldDeb->Load(filename, nullptr); + bool const goodLoad = _error->PendingError() == false; + _error->RevertToStack(); + if (goodLoad) + { + if (OldDeb->GetValidUntil() > 0) + { + time_t const invalid_since = time(NULL) - OldDeb->GetValidUntil(); + if (invalid_since <= 0) + { + Deb->SetSignedBy(OldDeb->GetSignedBy()); + alreadySet = true; + } + } + } + delete OldDeb; + } + if (alreadySet == false && Deb->SetSignedBy("") == false) + return false; + } + else + { + if (Deb->SetSignedBy(signedby->second) == false) + return false; + } + + return true; + } + + debSLTypeDebian(char const * const Name, char const * const Label) : Type(Name, Label) + { + } +}; + /*}}}*/ +class APT_HIDDEN debSLTypeDeb : public debSLTypeDebian /*{{{*/ +{ + public: + + bool CreateItem(std::vector<metaIndex *> &List, std::string const &URI, + std::string const &Dist, std::string const &Section, + std::map<std::string, std::string> const &Options) const APT_OVERRIDE + { + return CreateItemInternal(List, URI, Dist, Section, false, Options); + } + + debSLTypeDeb() : debSLTypeDebian("deb", "Debian binary tree") + { + } +}; + /*}}}*/ +class APT_HIDDEN debSLTypeDebSrc : public debSLTypeDebian /*{{{*/ +{ + public: + + bool CreateItem(std::vector<metaIndex *> &List, std::string const &URI, + std::string const &Dist, std::string const &Section, + std::map<std::string, std::string> const &Options) const APT_OVERRIDE + { + return CreateItemInternal(List, URI, Dist, Section, true, Options); + } + + debSLTypeDebSrc() : debSLTypeDebian("deb-src", "Debian source tree") + { + } +}; + /*}}}*/ + +APT_HIDDEN debSLTypeDeb _apt_DebType; +APT_HIDDEN debSLTypeDebSrc _apt_DebSrcType; diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h new file mode 100644 index 0000000..717f08e --- /dev/null +++ b/apt-pkg/deb/debmetaindex.h @@ -0,0 +1,72 @@ +#ifndef PKGLIB_DEBMETAINDEX_H +#define PKGLIB_DEBMETAINDEX_H + +#include <apt-pkg/macros.h> +#include <apt-pkg/metaindex.h> + +#include <map> +#include <string> +#include <vector> + + +class pkgAcquire; +class pkgIndexFile; +class IndexTarget; +class pkgCacheGenerator; +class OpProgress; +class debReleaseIndexPrivate; + +class APT_HIDDEN debReleaseIndex : public metaIndex +{ + debReleaseIndexPrivate * const d; + + APT_HIDDEN bool parseSumData(const char *&Start, const char *End, std::string &Name, + std::string &Hash, unsigned long long &Size); + public: + + APT_HIDDEN std::string MetaIndexInfo(const char *Type) const; + APT_HIDDEN std::string MetaIndexFile(const char *Types) const; + APT_HIDDEN std::string MetaIndexURI(const char *Type) const; + + debReleaseIndex(std::string const &URI, std::string const &Dist, std::map<std::string,std::string> const &Options); + debReleaseIndex(std::string const &URI, std::string const &Dist, bool const Trusted, std::map<std::string,std::string> const &Options); + virtual ~debReleaseIndex(); + + virtual std::string ArchiveURI(std::string const &File) const APT_OVERRIDE; + virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) APT_OVERRIDE; + virtual std::vector<IndexTarget> GetIndexTargets() const APT_OVERRIDE; + + virtual std::string Describe() const APT_OVERRIDE; + virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const APT_OVERRIDE; + virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const APT_OVERRIDE; + + virtual bool Load(std::string const &Filename, std::string * const ErrorText) APT_OVERRIDE; + virtual metaIndex * UnloadedClone() const APT_OVERRIDE; + + virtual std::vector <pkgIndexFile *> *GetIndexFiles() APT_OVERRIDE; + + bool SetTrusted(TriState const Trusted); + bool SetCheckValidUntil(TriState const Trusted); + bool SetValidUntilMin(time_t const Valid); + bool SetValidUntilMax(time_t const Valid); + bool SetCheckDate(TriState const CheckDate); + bool SetDateMaxFuture(time_t const DateMaxFuture); + bool SetSignedBy(std::string const &SignedBy); + std::map<std::string, std::string> GetReleaseOptions(); + + virtual bool IsTrusted() const APT_OVERRIDE; + bool IsArchitectureSupported(std::string const &arch) const override; + bool IsArchitectureAllSupportedFor(IndexTarget const &target) const override; + bool HasSupportForComponent(std::string const &component) const override; + + APT_PURE time_t GetNotBefore() const override; + + void AddComponent(std::string const &sourcesEntry, + bool const isSrc, std::string const &Name, + std::vector<std::string> const &Targets, + std::vector<std::string> const &Architectures, + std::vector<std::string> Languages, + bool const usePDiffs, std::string const &useByHash); +}; + +#endif diff --git a/apt-pkg/deb/debrecords.cc b/apt-pkg/deb/debrecords.cc new file mode 100644 index 0000000..e2ffaef --- /dev/null +++ b/apt-pkg/deb/debrecords.cc @@ -0,0 +1,228 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Package Records - Parser for debian package records + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/debindexfile.h> +#include <apt-pkg/debrecords.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile-keys.h> +#include <apt-pkg/tagfile.h> + +#include <algorithm> +#include <sstream> +#include <string> +#include <vector> +#include <langinfo.h> +#include <string.h> + +#include <apti18n.h> + /*}}}*/ + +using std::string; + +// RecordParser::debRecordParser - Constructor /*{{{*/ +debRecordParser::debRecordParser(string FileName,pkgCache &Cache) : + debRecordParserBase(), d(NULL), File(FileName, FileFd::ReadOnly, FileFd::Extension), + Tags(&File, std::max(Cache.Head().MaxVerFileSize, Cache.Head().MaxDescFileSize) + 200) +{ +} + /*}}}*/ +// RecordParser::Jump - Jump to a specific record /*{{{*/ +bool debRecordParser::Jump(pkgCache::VerFileIterator const &Ver) +{ + if (Ver.end() == true) + return false; + return Tags.Jump(Section,Ver->Offset); +} +bool debRecordParser::Jump(pkgCache::DescFileIterator const &Desc) +{ + if (Desc.end() == true) + return false; + return Tags.Jump(Section,Desc->Offset); +} + /*}}}*/ +debRecordParser::~debRecordParser() {} + +debRecordParserBase::debRecordParserBase() : Parser(), d(NULL) {} +// RecordParserBase::FileName - Return the archive filename on the site /*{{{*/ +string debRecordParserBase::FileName() +{ + return Section.Find(pkgTagSection::Key::Filename).to_string(); +} + /*}}}*/ +// RecordParserBase::Name - Return the package name /*{{{*/ +string debRecordParserBase::Name() +{ + auto Result = Section.Find(pkgTagSection::Key::Package).to_string(); + + // Normalize mixed case package names to lower case, like dpkg does + // See Bug#807012 for details + std::transform(Result.begin(), Result.end(), Result.begin(), tolower_ascii); + + return Result; +} + /*}}}*/ +// RecordParserBase::Homepage - Return the package homepage /*{{{*/ +string debRecordParserBase::Homepage() +{ + return Section.Find(pkgTagSection::Key::Homepage).to_string(); +} + /*}}}*/ +// RecordParserBase::Hashes - return the available archive hashes /*{{{*/ +HashStringList debRecordParserBase::Hashes() const +{ + HashStringList hashes; + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + { + std::string const hash = Section.FindS(*type); + if (hash.empty() == false) + hashes.push_back(HashString(*type, hash)); + } + auto const size = Section.FindULL(pkgTagSection::Key::Size, 0); + if (size != 0) + hashes.FileSize(size); + return hashes; +} + /*}}}*/ +// RecordParserBase::Maintainer - Return the maintainer email /*{{{*/ +string debRecordParserBase::Maintainer() +{ + return Section.Find(pkgTagSection::Key::Maintainer).to_string(); +} + /*}}}*/ +// RecordParserBase::RecordField - Return the value of an arbitrary field /*{{*/ +string debRecordParserBase::RecordField(const char *fieldName) +{ + return Section.FindS(fieldName); +} + /*}}}*/ +// RecordParserBase::ShortDesc - Return a 1 line description /*{{{*/ +string debRecordParserBase::ShortDesc(std::string const &lang) +{ + string const Res = LongDesc(lang); + if (Res.empty() == true) + return ""; + string::size_type const Pos = Res.find('\n'); + if (Pos == string::npos) + return Res; + return string(Res,0,Pos); +} + /*}}}*/ +// RecordParserBase::LongDesc - Return a longer description /*{{{*/ +string debRecordParserBase::LongDesc(std::string const &lang) +{ + string orig; + if (lang.empty() == true) + { + std::vector<string> const lang = APT::Configuration::getLanguages(); + for (std::vector<string>::const_iterator l = lang.begin(); + l != lang.end(); ++l) + { + std::string const tagname = "Description-" + *l; + orig = Section.FindS(tagname); + if (orig.empty() == false) + break; + else if (*l == "en") + { + orig = Section.Find(pkgTagSection::Key::Description).to_string(); + if (orig.empty() == false) + break; + } + } + if (orig.empty() == true) + orig = Section.Find(pkgTagSection::Key::Description).to_string(); + } + else + { + std::string const tagname = "Description-" + lang; + orig = Section.FindS(tagname.c_str()); + if (orig.empty() == true && lang == "en") + orig = Section.Find(pkgTagSection::Key::Description).to_string(); + } + + char const * const codeset = nl_langinfo(CODESET); + if (strcmp(codeset,"UTF-8") != 0) { + string dest; + UTF8ToCodeset(codeset, orig, &dest); + return dest; + } + + return orig; +} + /*}}}*/ + +static const char * const SourceVerSeparators = " ()"; +// RecordParserBase::SourcePkg - Return the source package name if any /*{{{*/ +string debRecordParserBase::SourcePkg() +{ + auto Res = Section.Find(pkgTagSection::Key::Source).to_string(); + auto const Pos = Res.find_first_of(SourceVerSeparators); + if (Pos != std::string::npos) + Res.erase(Pos); + return Res; +} + /*}}}*/ +// RecordParserBase::SourceVer - Return the source version number if present /*{{{*/ +string debRecordParserBase::SourceVer() +{ + auto const Pkg = Section.Find(pkgTagSection::Key::Source).to_string(); + string::size_type Pos = Pkg.find_first_of(SourceVerSeparators); + if (Pos == string::npos) + return ""; + + string::size_type VerStart = Pkg.find_first_not_of(SourceVerSeparators, Pos); + if(VerStart == string::npos) + return ""; + + string::size_type VerEnd = Pkg.find_first_of(SourceVerSeparators, VerStart); + if(VerEnd == string::npos) + // Corresponds to the case of, e.g., "foo (1.2" without a closing + // paren. Be liberal and guess what it means. + return string(Pkg, VerStart); + else + return string(Pkg, VerStart, VerEnd - VerStart); +} + /*}}}*/ +// RecordParserBase::GetRec - Return the whole record /*{{{*/ +void debRecordParserBase::GetRec(const char *&Start,const char *&Stop) +{ + Section.GetSection(Start,Stop); +} + /*}}}*/ +debRecordParserBase::~debRecordParserBase() {} + +bool debDebFileRecordParser::LoadContent() +{ + // load content only once + if (controlContent.empty() == false) + return true; + + std::ostringstream content; + if (debDebPkgFileIndex::GetContent(content, debFileName) == false) + return false; + // add two newlines to make sure the scanner finds the section, + // which is usually done by pkgTagFile automatically if needed. + content << "\n\n"; + + controlContent = content.str(); + if (Section.Scan(controlContent.c_str(), controlContent.length()) == false) + return _error->Error(_("Unable to parse package file %s (%d)"), debFileName.c_str(), 3); + return true; +} +bool debDebFileRecordParser::Jump(pkgCache::VerFileIterator const &) { return LoadContent(); } +bool debDebFileRecordParser::Jump(pkgCache::DescFileIterator const &) { return LoadContent(); } +std::string debDebFileRecordParser::FileName() { return debFileName; } + +debDebFileRecordParser::debDebFileRecordParser(std::string FileName) : debRecordParserBase(), d(NULL), debFileName(FileName) {} +debDebFileRecordParser::~debDebFileRecordParser() {} diff --git a/apt-pkg/deb/debrecords.h b/apt-pkg/deb/debrecords.h new file mode 100644 index 0000000..10ef917 --- /dev/null +++ b/apt-pkg/deb/debrecords.h @@ -0,0 +1,89 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Package Records - Parser for debian package records + + This provides display-type parsing for the Packages file. This is + different than the list parser which provides cache generation + services. There should be no overlap between these two. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_DEBRECORDS_H +#define PKGLIB_DEBRECORDS_H + +#include <apt-pkg/fileutl.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgrecords.h> +#include <apt-pkg/tagfile.h> + +#include <string> + + +class APT_HIDDEN debRecordParserBase : public pkgRecords::Parser +{ + void * const d; + protected: + pkgTagSection Section; + + public: + // These refer to the archive file for the Version + virtual std::string FileName() APT_OVERRIDE; + virtual std::string SourcePkg() APT_OVERRIDE; + virtual std::string SourceVer() APT_OVERRIDE; + + virtual HashStringList Hashes() const APT_OVERRIDE; + + // These are some general stats about the package + virtual std::string Maintainer() APT_OVERRIDE; + virtual std::string ShortDesc(std::string const &lang) APT_OVERRIDE; + virtual std::string LongDesc(std::string const &lang) APT_OVERRIDE; + virtual std::string Name() APT_OVERRIDE; + virtual std::string Homepage() APT_OVERRIDE; + + // An arbitrary custom field + virtual std::string RecordField(const char *fieldName) APT_OVERRIDE; + + virtual void GetRec(const char *&Start,const char *&Stop) APT_OVERRIDE; + + debRecordParserBase(); + virtual ~debRecordParserBase(); +}; + +class APT_HIDDEN debRecordParser : public debRecordParserBase +{ + void * const d; + protected: + FileFd File; + pkgTagFile Tags; + + virtual bool Jump(pkgCache::VerFileIterator const &Ver) APT_OVERRIDE; + virtual bool Jump(pkgCache::DescFileIterator const &Desc) APT_OVERRIDE; + + public: + debRecordParser(std::string FileName,pkgCache &Cache); + virtual ~debRecordParser(); +}; + +// custom record parser that reads deb files directly +class APT_HIDDEN debDebFileRecordParser : public debRecordParserBase +{ + void * const d; + std::string debFileName; + std::string controlContent; + + APT_HIDDEN bool LoadContent(); + protected: + // single file files, so no jumping whatsoever + bool Jump(pkgCache::VerFileIterator const &) APT_OVERRIDE; + bool Jump(pkgCache::DescFileIterator const &) APT_OVERRIDE; + + public: + virtual std::string FileName() APT_OVERRIDE; + + explicit debDebFileRecordParser(std::string FileName); + virtual ~debDebFileRecordParser(); +}; + +#endif diff --git a/apt-pkg/deb/debsrcrecords.cc b/apt-pkg/deb/debsrcrecords.cc new file mode 100644 index 0000000..311bacf --- /dev/null +++ b/apt-pkg/deb/debsrcrecords.cc @@ -0,0 +1,281 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Source Package Records - Parser implementation for Debian style + source indexes + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/deblistparser.h> +#include <apt-pkg/debsrcrecords.h> +#include <apt-pkg/error.h> +#include <apt-pkg/gpgv.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/srcrecords.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile-keys.h> +#include <apt-pkg/tagfile.h> + +#include <algorithm> +#include <string> +#include <sstream> +#include <vector> +#include <ctype.h> +#include <stdlib.h> +#include <string.h> + /*}}}*/ + +using std::max; +using std::string; + +debSrcRecordParser::debSrcRecordParser(std::string const &File,pkgIndexFile const *Index) + : Parser(Index), d(NULL), Tags(&Fd), iOffset(0), Buffer(NULL) +{ + if (File.empty() == false) + { + if (Fd.Open(File, FileFd::ReadOnly, FileFd::Extension)) + Tags.Init(&Fd, 102400); + } +} +std::string debSrcRecordParser::Package() const /*{{{*/ +{ + auto const name = Sect.Find(pkgTagSection::Key::Package); + if (iIndex != nullptr || name.empty() == false) + return name.to_string(); + return Sect.Find(pkgTagSection::Key::Source).to_string(); +} + /*}}}*/ +// SrcRecordParser::Binaries - Return the binaries field /*{{{*/ +// --------------------------------------------------------------------- +/* This member parses the binaries field into a pair of class arrays and + returns a list of strings representing all of the components of the + binaries field. The returned array need not be freed and will be + reused by the next Binaries function call. This function is commonly + used during scanning to find the right package */ +const char **debSrcRecordParser::Binaries() +{ + const char *Start, *End; + if (Sect.Find(pkgTagSection::Key::Binary, Start, End) == false) + return NULL; + for (; isspace_ascii(*Start) != 0; ++Start); + if (Start >= End) + return NULL; + + StaticBinList.clear(); + free(Buffer); + Buffer = strndup(Start, End - Start); + + char* bin = Buffer; + do { + char* binStartNext = strchrnul(bin, ','); + // Found a comma, clean up any space before it + if (binStartNext > Buffer) { + char* binEnd = binStartNext - 1; + for (; binEnd > Buffer && isspace_ascii(*binEnd) != 0; --binEnd) + *binEnd = 0; + } + StaticBinList.push_back(bin); + if (*binStartNext != ',') + break; + *binStartNext = '\0'; + for (bin = binStartNext + 1; isspace_ascii(*bin) != 0; ++bin) + ; + } while (*bin != '\0'); + StaticBinList.push_back(NULL); + + return &StaticBinList[0]; +} + /*}}}*/ +// SrcRecordParser::BuildDepends - Return the Build-Depends information /*{{{*/ +// --------------------------------------------------------------------- +/* This member parses the build-depends information and returns a list of + package/version records representing the build dependency. The returned + array need not be freed and will be reused by the next call to this + function */ +bool debSrcRecordParser::BuildDepends(std::vector<pkgSrcRecords::Parser::BuildDepRec> &BuildDeps, + bool const &ArchOnly, bool const &StripMultiArch) +{ + BuildDeps.clear(); + + pkgTagSection::Key const fields[] = { + pkgTagSection::Key::Build_Depends, + pkgTagSection::Key::Build_Depends_Indep, + pkgTagSection::Key::Build_Conflicts, + pkgTagSection::Key::Build_Conflicts_Indep, + pkgTagSection::Key::Build_Depends_Arch, + pkgTagSection::Key::Build_Conflicts_Arch, + }; + for (unsigned short I = 0; I < sizeof(fields) / sizeof(fields[0]); ++I) + { + if (ArchOnly && (fields[I] == pkgTagSection::Key::Build_Depends_Indep || fields[I] == pkgTagSection::Key::Build_Conflicts_Indep)) + continue; + + const char *Start, *Stop; + if (Sect.Find(fields[I], Start, Stop) == false) + continue; + + if (Start == Stop) + continue; + + while (1) + { + // Strip off leading spaces (is done by ParseDepends, too) and + // superfluous commas (encountered in user-written dsc/control files) + do { + for (;Start != Stop && isspace_ascii(*Start) != 0; ++Start); + } while (*Start == ',' && ++Start != Stop); + if (Start == Stop) + break; + + BuildDepRec rec; + Start = debListParser::ParseDepends(Start, Stop, + rec.Package, rec.Version, rec.Op, true, StripMultiArch, true); + + if (Start == 0) + return _error->Error("Problem parsing dependency: %s", BuildDepType(I)); + rec.Type = I; + + // We parsed a package that was ignored (wrong architecture restriction + // or something). + if (rec.Package.empty()) + { + // If this was the last or-group member, close the or-group with the previous entry + if (not BuildDeps.empty() && (BuildDeps.back().Op & pkgCache::Dep::Or) == pkgCache::Dep::Or && (rec.Op & pkgCache::Dep::Or) != pkgCache::Dep::Or) + BuildDeps.back().Op &= ~pkgCache::Dep::Or; + } else { + BuildDeps.emplace_back(std::move(rec)); + } + } + } + + return true; +} + /*}}}*/ +// SrcRecordParser::Files - Return a list of files for this source /*{{{*/ +// --------------------------------------------------------------------- +/* This parses the list of files and returns it, each file is required to have + a complete source package */ +bool debSrcRecordParser::Files(std::vector<pkgSrcRecords::File> &List) +{ + List.clear(); + + // Stash the / terminated directory prefix + std::string Base = Sect.Find(pkgTagSection::Key::Directory).to_string(); + if (Base.empty() == false && Base[Base.length()-1] != '/') + Base += '/'; + + std::vector<std::string> const compExts = APT::Configuration::getCompressorExtensions(); + + auto const &posix = std::locale::classic(); + for (auto const hashinfo : HashString::SupportedHashesInfo()) + { + auto const Files = Sect.Find(hashinfo.chksumskey); + if (Files.empty() == true) + continue; + std::istringstream ss(Files.to_string()); + ss.imbue(posix); + + while (ss.good()) + { + std::string hash, path; + unsigned long long size; + if (iIndex == nullptr && hashinfo.chksumskey == pkgTagSection::Key::Files) + { + std::string ignore; + ss >> hash >> size >> ignore >> ignore >> path; + } + else + ss >> hash >> size >> path; + + if (ss.fail() || hash.empty() || path.empty()) + return _error->Error("Error parsing file record in %s of source package %s", hashinfo.chksumsname.to_string().c_str(), Package().c_str()); + + HashString const hashString(hashinfo.name.to_string(), hash); + if (Base.empty() == false) + path = Base + path; + + // look if we have a record for this file already + std::vector<pkgSrcRecords::File>::iterator file = List.begin(); + for (; file != List.end(); ++file) + if (file->Path == path) + break; + + // we have it already, store the new hash and be done + if (file != List.end()) + { + // an error here indicates that we have two different hashes for the same file + if (file->Hashes.push_back(hashString) == false) + return _error->Error("Error parsing checksum in %s of source package %s", hashinfo.chksumsname.to_string().c_str(), Package().c_str()); + continue; + } + + // we haven't seen this file yet + pkgSrcRecords::File F; + F.Path = path; + F.FileSize = size; + F.Hashes.push_back(hashString); + F.Hashes.FileSize(F.FileSize); + + // Try to guess what sort of file it is we are getting. + string::size_type Pos = F.Path.length()-1; + while (1) + { + string::size_type Tmp = F.Path.rfind('.',Pos); + if (Tmp == string::npos) + break; + if (F.Type == "tar") { + // source v3 has extension 'debian.tar.*' instead of 'diff.*' + if (string(F.Path, Tmp+1, Pos-Tmp) == "debian") + F.Type = "diff"; + break; + } + F.Type = string(F.Path,Tmp+1,Pos-Tmp); + + if (std::find(compExts.begin(), compExts.end(), std::string(".").append(F.Type)) != compExts.end() || + F.Type == "tar") + { + Pos = Tmp-1; + continue; + } + + break; + } + List.push_back(F); + } + } + + return true; +} + /*}}}*/ +// SrcRecordParser::~SrcRecordParser - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +debSrcRecordParser::~debSrcRecordParser() +{ + // was allocated via strndup() + free(Buffer); +} + /*}}}*/ + + +debDscRecordParser::debDscRecordParser(std::string const &DscFile, pkgIndexFile const *Index) + : debSrcRecordParser("", Index) +{ + // support clear signed files + if (OpenMaybeClearSignedFile(DscFile, Fd) == false) + { + _error->Error("Failed to open %s", DscFile.c_str()); + return; + } + + // re-init to ensure the updated Fd is used + Tags.Init(&Fd, pkgTagFile::SUPPORT_COMMENTS); + // read the first (and only) record + Step(); + +} diff --git a/apt-pkg/deb/debsrcrecords.h b/apt-pkg/deb/debsrcrecords.h new file mode 100644 index 0000000..1a0bbe0 --- /dev/null +++ b/apt-pkg/deb/debsrcrecords.h @@ -0,0 +1,68 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Source Package Records - Parser implementation for Debian style + source indexes + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_DEBSRCRECORDS_H +#define PKGLIB_DEBSRCRECORDS_H + +#include <apt-pkg/fileutl.h> +#include <apt-pkg/srcrecords.h> +#include <apt-pkg/tagfile-keys.h> +#include <apt-pkg/tagfile.h> + +#include <string> +#include <vector> +#include <stddef.h> + +class pkgIndexFile; + +class APT_HIDDEN debSrcRecordParser : public pkgSrcRecords::Parser +{ + /** \brief dpointer placeholder (for later in case we need it) */ + void * const d; + + protected: + FileFd Fd; + pkgTagFile Tags; + pkgTagSection Sect; + std::vector<const char*> StaticBinList; + unsigned long iOffset; + char *Buffer; + + public: + + virtual bool Restart() APT_OVERRIDE {return Jump(0);}; + virtual bool Step() APT_OVERRIDE {iOffset = Tags.Offset(); return Tags.Step(Sect);}; + virtual bool Jump(unsigned long const &Off) APT_OVERRIDE {iOffset = Off; return Tags.Jump(Sect,Off);}; + + virtual std::string Package() const APT_OVERRIDE; + virtual std::string Version() const APT_OVERRIDE {return Sect.Find(pkgTagSection::Key::Version).to_string();}; + virtual std::string Maintainer() const APT_OVERRIDE {return Sect.Find(pkgTagSection::Key::Maintainer).to_string();}; + virtual std::string Section() const APT_OVERRIDE {return Sect.Find(pkgTagSection::Key::Section).to_string();}; + virtual const char **Binaries() APT_OVERRIDE; + virtual bool BuildDepends(std::vector<BuildDepRec> &BuildDeps, bool const &ArchOnly, bool const &StripMultiArch = true) APT_OVERRIDE; + virtual unsigned long Offset() APT_OVERRIDE {return iOffset;}; + virtual std::string AsStr() APT_OVERRIDE + { + const char *Start=0,*Stop=0; + Sect.GetSection(Start,Stop); + return std::string(Start,Stop); + }; + virtual bool Files(std::vector<pkgSrcRecords::File> &F) APT_OVERRIDE; + + debSrcRecordParser(std::string const &File,pkgIndexFile const *Index); + virtual ~debSrcRecordParser(); +}; + +class APT_HIDDEN debDscRecordParser : public debSrcRecordParser +{ + public: + debDscRecordParser(std::string const &DscFile, pkgIndexFile const *Index); +}; + +#endif diff --git a/apt-pkg/deb/debsystem.cc b/apt-pkg/deb/debsystem.cc new file mode 100644 index 0000000..9c55e0a --- /dev/null +++ b/apt-pkg/deb/debsystem.cc @@ -0,0 +1,557 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + System - Abstraction for running on different systems. + + Basic general structure.. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/debindexfile.h> +#include <apt-pkg/debsystem.h> +#include <apt-pkg/debversion.h> +#include <apt-pkg/dpkgpm.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/progress.h> + +#include <algorithm> +#include <sstream> + +#include <string> +#include <vector> +#include <ctype.h> +#include <dirent.h> +#include <errno.h> +#include <fcntl.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ + +using std::string; + +debSystem debSys; + +class APT_HIDDEN debSystemPrivate { +public: + debSystemPrivate() : FrontendLockFD(-1), LockFD(-1), LockCount(0), StatusFile(0) + { + } + // For locking support + int FrontendLockFD; + int LockFD; + unsigned LockCount; + + debStatusIndex *StatusFile; +}; + +// System::debSystem - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +debSystem::debSystem() : pkgSystem("Debian dpkg interface", &debVS), d(new debSystemPrivate()) +{ +} + /*}}}*/ +// System::~debSystem - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +debSystem::~debSystem() +{ + delete d->StatusFile; + delete d; +} + /*}}}*/ +// System::Lock - Get the lock /*{{{*/ +// --------------------------------------------------------------------- +/* This mirrors the operations dpkg does when it starts up. Note the + checking of the updates directory. */ +static int GetLockMaybeWait(std::string const &file, OpProgress *Progress, int &timeoutSec) +{ + struct ScopedAbsoluteProgress + { + ScopedAbsoluteProgress() { _config->Set("APT::Internal::OpProgress::Absolute", true); } + ~ScopedAbsoluteProgress() { _config->Set("APT::Internal::OpProgress::Absolute", false); } + } _scopedAbsoluteProgress; + int fd = -1; + if (timeoutSec == 0 || Progress == nullptr) + return GetLock(file); + + if (_config->FindB("Debug::Locking", false)) + std::cerr << "Lock: " << file << " wait " << timeoutSec << std::endl; + + for (int i = 0; timeoutSec < 0 || i < timeoutSec; i++) + { + _error->PushToStack(); + fd = GetLock(file); + if (fd != -1 || errno == EPERM) + { + if (timeoutSec > 0) + timeoutSec -= i; + _error->MergeWithStack(); + return fd; + } + std::string poppedError; + std::string completeError; + _error->PopMessage(poppedError); + _error->RevertToStack(); + + strprintf(completeError, _("Waiting for cache lock: %s"), poppedError.c_str()); + sleep(1); + Progress->OverallProgress(i, timeoutSec, 0, completeError); + } + + if (timeoutSec > 0) + timeoutSec = 1; + return fd; +} + +bool debSystem::Lock(OpProgress *const Progress) +{ + // Disable file locking + if (_config->FindB("Debug::NoLocking",false) == true || d->LockCount > 0) + { + d->LockCount++; + return true; + } + + // This will count downwards. + int lockTimeOutSec = _config->FindI("DPkg::Lock::Timeout", 0); + // Create the lockfile + string AdminDir = flNotFile(_config->FindFile("Dir::State::status")); + string FrontendLockFile = AdminDir + "lock-frontend"; + d->FrontendLockFD = GetLockMaybeWait(FrontendLockFile, Progress, lockTimeOutSec); + if (d->FrontendLockFD == -1) + { + if (errno == EACCES || errno == EAGAIN) + return _error->Error(_("Unable to acquire the dpkg frontend lock (%s), " + "is another process using it?"),FrontendLockFile.c_str()); + else + return _error->Error(_("Unable to acquire the dpkg frontend lock (%s), " + "are you root?"),FrontendLockFile.c_str()); + } + if (LockInner(Progress, lockTimeOutSec) == false) + { + close(d->FrontendLockFD); + return false; + } + + // See if we need to abort with a dirty journal + if (CheckUpdates() == true) + { + close(d->LockFD); + close(d->FrontendLockFD); + d->FrontendLockFD = -1; + d->LockFD = -1; + const char *cmd; + if (getenv("SUDO_USER") != NULL) + cmd = "sudo dpkg --configure -a"; + else + cmd = "dpkg --configure -a"; + // TRANSLATORS: the %s contains the recovery command, usually + // dpkg --configure -a + return _error->Error(_("dpkg was interrupted, you must manually " + "run '%s' to correct the problem. "), cmd); + } + + d->LockCount++; + + return true; +} + +bool debSystem::LockInner(OpProgress *const Progress, int timeOutSec) +{ + string AdminDir = flNotFile(_config->FindFile("Dir::State::status")); + d->LockFD = GetLockMaybeWait(AdminDir + "lock", Progress, timeOutSec); + if (d->LockFD == -1) + { + if (errno == EACCES || errno == EAGAIN) + return _error->Error(_("Unable to lock the administration directory (%s), " + "is another process using it?"),AdminDir.c_str()); + else + return _error->Error(_("Unable to lock the administration directory (%s), " + "are you root?"),AdminDir.c_str()); + } + return true; +} + /*}}}*/ +// System::UnLock - Drop a lock /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool debSystem::UnLock(bool NoErrors) +{ + if (d->LockCount == 0 && NoErrors == true) + return false; + + if (d->LockCount < 1) + return _error->Error(_("Not locked")); + if (--d->LockCount == 0) + { + close(d->LockFD); + close(d->FrontendLockFD); + d->LockCount = 0; + } + + return true; +} +bool debSystem::UnLockInner(bool NoErrors) { + (void) NoErrors; + close(d->LockFD); + return true; +} + /*}}}*/ +// System::IsLocked - Check if system is locked /*{{{*/ +// --------------------------------------------------------------------- +/* This checks if the frontend lock is hold. The inner lock might be + * released. */ +bool debSystem::IsLocked() +{ + return d->LockCount > 0; +} + /*}}}*/ +// System::CheckUpdates - Check if the updates dir is dirty /*{{{*/ +// --------------------------------------------------------------------- +/* This does a check of the updates directory (dpkg journal) to see if it has + any entries in it. */ +bool debSystem::CheckUpdates() +{ + // Check for updates.. (dirty) + string File = flNotFile(_config->FindFile("Dir::State::status")) + "updates/"; + DIR *DirP = opendir(File.c_str()); + if (DirP == 0) + return false; + + /* We ignore any files that are not all digits, this skips .,.. and + some tmp files dpkg will leave behind.. */ + bool Damaged = false; + for (struct dirent *Ent = readdir(DirP); Ent != 0; Ent = readdir(DirP)) + { + Damaged = true; + for (unsigned int I = 0; Ent->d_name[I] != 0; I++) + { + // Check if its not a digit.. + if (isdigit(Ent->d_name[I]) == 0) + { + Damaged = false; + break; + } + } + if (Damaged == true) + break; + } + closedir(DirP); + + return Damaged; +} + /*}}}*/ +// System::CreatePM - Create the underlying package manager /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgPackageManager *debSystem::CreatePM(pkgDepCache *Cache) const +{ + return new pkgDPkgPM(Cache); +} + /*}}}*/ +// System::Initialize - Setup the configuration space.. /*{{{*/ +// --------------------------------------------------------------------- +/* These are the Debian specific configuration variables.. */ +static std::string getDpkgStatusLocation(Configuration const &Cnf) { + Configuration PathCnf; + PathCnf.Set("Dir", Cnf.Find("Dir", "/")); + PathCnf.Set("Dir::State::status", "status"); + auto const cnfstatedir = Cnf.Find("Dir::State", &STATE_DIR[1]); + // if the state dir ends in apt, replace it with dpkg - + // for the default this gives us the same as the fallback below. + // This can't be a ../dpkg as that would play bad with symlinks + std::string statedir; + if (APT::String::Endswith(cnfstatedir, "/apt/")) + statedir.assign(cnfstatedir, 0, cnfstatedir.length() - 5); + else if (APT::String::Endswith(cnfstatedir, "/apt")) + statedir.assign(cnfstatedir, 0, cnfstatedir.length() - 4); + if (statedir.empty()) + PathCnf.Set("Dir::State", "var/lib/dpkg"); + else + PathCnf.Set("Dir::State", flCombine(statedir, "dpkg")); + return PathCnf.FindFile("Dir::State::status"); +} +bool debSystem::Initialize(Configuration &Cnf) +{ + /* These really should be jammed into a generic 'Local Database' engine + which is yet to be determined. The functions in pkgcachegen should + be the only users of these */ + Cnf.CndSet("Dir::State::extended_states", "extended_states"); + if (Cnf.Exists("Dir::State::status") == false) + Cnf.Set("Dir::State::status", getDpkgStatusLocation(Cnf)); + Cnf.CndSet("Dir::Bin::dpkg",BIN_DIR"/dpkg"); + + if (d->StatusFile) { + delete d->StatusFile; + d->StatusFile = 0; + } + + return true; +} + /*}}}*/ +// System::ArchiveSupported - Is a file format supported /*{{{*/ +// --------------------------------------------------------------------- +/* The standard name for a deb is 'deb'.. There are no separate versions + of .deb to worry about.. */ +APT_PURE bool debSystem::ArchiveSupported(const char *Type) +{ + if (strcmp(Type,"deb") == 0) + return true; + return false; +} + /*}}}*/ +// System::Score - Determine how 'Debiany' this sys is.. /*{{{*/ +// --------------------------------------------------------------------- +/* We check some files that are sure tell signs of this being a Debian + System.. */ +signed debSystem::Score(Configuration const &Cnf) +{ + signed Score = 0; + if (FileExists(Cnf.FindFile("Dir::State::status",getDpkgStatusLocation(Cnf).c_str())) == true) + Score += 10; + if (FileExists(Cnf.Find("Dir::Bin::dpkg",BIN_DIR"/dpkg")) == true) + Score += 10; + if (FileExists("/etc/debian_version") == true) + Score += 10; + return Score; +} + /*}}}*/ +// System::AddStatusFiles - Register the status files /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool debSystem::AddStatusFiles(std::vector<pkgIndexFile *> &List) +{ + if (d->StatusFile == nullptr) + { + auto dpkgstatus = _config->FindFile("Dir::State::status"); + if (dpkgstatus.empty()) + return true; + // we ignore only if the file doesn't exist, not if it is inaccessible + // e.g. due to permissions on parent directories as FileExists would do + errno = 0; + if (access(dpkgstatus.c_str(), R_OK) != 0 && errno == ENOENT) + return true; + _error->PushToStack(); + d->StatusFile = new debStatusIndex(std::move(dpkgstatus)); + bool const errored = _error->PendingError(); + _error->MergeWithStack(); + if (errored) + { + delete d->StatusFile; + d->StatusFile = nullptr; + return false; + } + } + List.push_back(d->StatusFile); + return true; +} + /*}}}*/ +// System::FindIndex - Get an index file for status files /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool debSystem::FindIndex(pkgCache::PkgFileIterator File, + pkgIndexFile *&Found) const +{ + if (d->StatusFile == 0) + return false; + if (d->StatusFile->FindInCache(*File.Cache()) == File) + { + Found = d->StatusFile; + return true; + } + + return false; +} + /*}}}*/ +std::string debSystem::StripDpkgChrootDirectory(std::string const &File)/*{{{*/ +{ + // If the filename string begins with DPkg::Chroot-Directory, return the + // substr that is within the chroot so dpkg can access it. + std::string const chrootdir = _config->FindDir("DPkg::Chroot-Directory","/"); + size_t len = chrootdir.length(); + if (chrootdir == "/" || File.compare(0, len, chrootdir) != 0) + return File; + if (chrootdir.at(len - 1) == '/') + --len; + return File.substr(len); +} + /*}}}*/ +std::string debSystem::GetDpkgExecutable() /*{{{*/ +{ + return StripDpkgChrootDirectory(_config->Find("Dir::Bin::dpkg","dpkg")); +} + /*}}}*/ +std::vector<std::string> debSystem::GetDpkgBaseCommand() /*{{{*/ +{ + // Generate the base argument list for dpkg + std::vector<std::string> Args = { GetDpkgExecutable() }; + // Stick in any custom dpkg options + Configuration::Item const *Opts = _config->Tree("DPkg::Options"); + if (Opts != 0) + { + Opts = Opts->Child; + for (; Opts != 0; Opts = Opts->Next) + { + if (Opts->Value.empty() == true) + continue; + Args.push_back(Opts->Value); + } + } + return Args; +} + /*}}}*/ +void debSystem::DpkgChrootDirectory() /*{{{*/ +{ + std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); + if (chrootDir == "/") + return; + std::cerr << "Chrooting into " << chrootDir << std::endl; + if (chroot(chrootDir.c_str()) != 0) + _exit(100); + if (chdir("/") != 0) + _exit(100); +} + /*}}}*/ +pid_t debSystem::ExecDpkg(std::vector<std::string> const &sArgs, int * const inputFd, int * const outputFd, bool const DiscardOutput)/*{{{*/ +{ + std::vector<const char *> Args(sArgs.size(), NULL); + std::transform(sArgs.begin(), sArgs.end(), Args.begin(), [](std::string const &s) { return s.c_str(); }); + Args.push_back(NULL); + + int external[2] = {-1, -1}; + if (inputFd != nullptr || outputFd != nullptr) + if (pipe(external) != 0) + { + _error->WarningE("dpkg", "Can't create IPC pipe for dpkg call"); + return -1; + } + + pid_t const dpkg = ExecFork(); + if (dpkg == 0) { + int const nullfd = open("/dev/null", O_RDWR); + if (inputFd == nullptr) + dup2(nullfd, STDIN_FILENO); + else + { + close(external[1]); + dup2(external[0], STDIN_FILENO); + } + if (outputFd == nullptr) + dup2(nullfd, STDOUT_FILENO); + else + { + close(external[0]); + dup2(external[1], STDOUT_FILENO); + } + if (DiscardOutput == true) + dup2(nullfd, STDERR_FILENO); + debSystem::DpkgChrootDirectory(); + + if (_system != nullptr && _system->IsLocked() == true) + { + setenv("DPKG_FRONTEND_LOCKED", "true", 1); + } + + if (_config->Find("DPkg::Path", "").empty() == false) + setenv("PATH", _config->Find("DPkg::Path", "").c_str(), 1); + + execvp(Args[0], (char**) &Args[0]); + _error->WarningE("dpkg", "Can't execute dpkg!"); + _exit(100); + } + if (outputFd != nullptr) + { + close(external[1]); + *outputFd = external[0]; + } + else if (inputFd != nullptr) + { + close(external[0]); + *inputFd = external[1]; + } + return dpkg; +} + /*}}}*/ +bool debSystem::MultiArchSupported() const /*{{{*/ +{ + return AssertFeature("multi-arch"); +} + /*}}}*/ +bool debSystem::AssertFeature(std::string const &feature) /*{{{*/ +{ + std::vector<std::string> Args = GetDpkgBaseCommand(); + Args.push_back("--assert-" + feature); + pid_t const dpkgAssertMultiArch = ExecDpkg(Args, nullptr, nullptr, true); + if (dpkgAssertMultiArch > 0) + { + int Status = 0; + while (waitpid(dpkgAssertMultiArch, &Status, 0) != dpkgAssertMultiArch) + { + if (errno == EINTR) + continue; + _error->WarningE("dpkgGo", _("Waited for %s but it wasn't there"), "dpkg --assert-multi-arch"); + break; + } + if (WIFEXITED(Status) == true && WEXITSTATUS(Status) == 0) + return true; + } + return false; +} + /*}}}*/ +std::vector<std::string> debSystem::ArchitecturesSupported() const /*{{{*/ +{ + std::vector<std::string> archs; + { + string const arch = _config->Find("APT::Architecture"); + if (arch.empty() == false) + archs.push_back(std::move(arch)); + } + + std::vector<std::string> sArgs = GetDpkgBaseCommand(); + sArgs.push_back("--print-foreign-architectures"); + int outputFd = -1; + pid_t const dpkgMultiArch = ExecDpkg(sArgs, nullptr, &outputFd, true); + if (dpkgMultiArch == -1) + return archs; + + FILE *dpkg = fdopen(outputFd, "r"); + if(dpkg != NULL) { + char* buf = NULL; + size_t bufsize = 0; + while (getline(&buf, &bufsize, dpkg) != -1) + { + char* tok_saveptr; + char* arch = strtok_r(buf, " ", &tok_saveptr); + while (arch != NULL) { + for (; isspace_ascii(*arch) != 0; ++arch); + if (arch[0] != '\0') { + char const* archend = arch; + for (; isspace_ascii(*archend) == 0 && *archend != '\0'; ++archend); + string a(arch, (archend - arch)); + if (std::find(archs.begin(), archs.end(), a) == archs.end()) + archs.push_back(a); + } + arch = strtok_r(NULL, " ", &tok_saveptr); + } + } + free(buf); + fclose(dpkg); + } + ExecWait(dpkgMultiArch, "dpkg --print-foreign-architectures", true); + return archs; +} + /*}}}*/ diff --git a/apt-pkg/deb/debsystem.h b/apt-pkg/deb/debsystem.h new file mode 100644 index 0000000..c426faf --- /dev/null +++ b/apt-pkg/deb/debsystem.h @@ -0,0 +1,59 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + System - Debian version of the System Class + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_DEBSYSTEM_H +#define PKGLIB_DEBSYSTEM_H + +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgsystem.h> + +#include <vector> +class Configuration; +class pkgIndexFile; +class pkgPackageManager; +class debSystemPrivate; +class pkgDepCache; + + +class debSystem : public pkgSystem +{ + // private d-pointer + debSystemPrivate * const d; + APT_HIDDEN bool CheckUpdates(); + + public: + virtual bool Lock(OpProgress *const Progress) APT_OVERRIDE; + virtual bool UnLock(bool NoErrors = false) APT_OVERRIDE; + virtual pkgPackageManager *CreatePM(pkgDepCache *Cache) const APT_OVERRIDE; + virtual bool Initialize(Configuration &Cnf) APT_OVERRIDE; + virtual bool ArchiveSupported(const char *Type) APT_OVERRIDE; + virtual signed Score(Configuration const &Cnf) APT_OVERRIDE; + virtual bool AddStatusFiles(std::vector<pkgIndexFile *> &List) APT_OVERRIDE; + virtual bool FindIndex(pkgCache::PkgFileIterator File, + pkgIndexFile *&Found) const APT_OVERRIDE; + + debSystem(); + virtual ~debSystem(); + + APT_HIDDEN static std::string GetDpkgExecutable(); + APT_HIDDEN static std::vector<std::string> GetDpkgBaseCommand(); + APT_HIDDEN static void DpkgChrootDirectory(); + APT_HIDDEN static std::string StripDpkgChrootDirectory(std::string const &File); + APT_HIDDEN static pid_t ExecDpkg(std::vector<std::string> const &sArgs, int * const inputFd, int * const outputFd, bool const DiscardOutput); + bool MultiArchSupported() const override; + static bool AssertFeature(std::string const &Feature); + std::vector<std::string> ArchitecturesSupported() const override; + + bool LockInner(OpProgress *const Progress, int timeoutSec) override; + bool UnLockInner(bool NoErrors=false) override; + bool IsLocked() override; +}; + +extern debSystem debSys; + +#endif diff --git a/apt-pkg/deb/debversion.cc b/apt-pkg/deb/debversion.cc new file mode 100644 index 0000000..005f1bc --- /dev/null +++ b/apt-pkg/deb/debversion.cc @@ -0,0 +1,279 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Version - Versioning system for Debian + + This implements the standard Debian versioning system. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/debversion.h> +#include <apt-pkg/pkgcache.h> + +#include <ctype.h> +#include <stdlib.h> +#include <string.h> + /*}}}*/ + +debVersioningSystem debVS; + +// debVS::debVersioningSystem - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +debVersioningSystem::debVersioningSystem() +{ + Label = "Standard .deb"; +} + /*}}}*/ + +// debVS::CmpFragment - Compare versions /*{{{*/ +// --------------------------------------------------------------------- +/* This compares a fragment of the version. This is a slightly adapted + version of what dpkg uses in dpkg/lib/dpkg/version.c. + In particular, the a | b = NULL check is removed as we check this in the + caller, we use an explicit end for a | b strings and we check ~ explicit. */ +static int order(char c) +{ + if (isdigit(c)) + return 0; + else if (isalpha_ascii(c)) + return c; + else if (c == '~') + return -1; + else if (c) + return c + 256; + else + return 0; +} +int debVersioningSystem::CmpFragment(const char *A,const char *AEnd, + const char *B,const char *BEnd) +{ + /* Iterate over the whole string + What this does is to split the whole string into groups of + numeric and non numeric portions. For instance: + a67bhgs89 + Has 4 portions 'a', '67', 'bhgs', '89'. A more normal: + 2.7.2-linux-1 + Has '2', '.', '7', '.' ,'-linux-','1' */ + const char *lhs = A; + const char *rhs = B; + while (lhs != AEnd && rhs != BEnd) + { + int first_diff = 0; + + while (lhs != AEnd && rhs != BEnd && + (!isdigit(*lhs) || !isdigit(*rhs))) + { + int vc = order(*lhs); + int rc = order(*rhs); + if (vc != rc) + return vc - rc; + ++lhs; ++rhs; + } + + while (*lhs == '0') + ++lhs; + while (*rhs == '0') + ++rhs; + while (isdigit(*lhs) && isdigit(*rhs)) + { + if (!first_diff) + first_diff = *lhs - *rhs; + ++lhs; + ++rhs; + } + + if (isdigit(*lhs)) + return 1; + if (isdigit(*rhs)) + return -1; + if (first_diff) + return first_diff; + } + + // The strings must be equal + if (lhs == AEnd && rhs == BEnd) + return 0; + + // lhs is shorter + if (lhs == AEnd) + { + if (*rhs == '~') return 1; + return -1; + } + + // rhs is shorter + if (rhs == BEnd) + { + if (*lhs == '~') return -1; + return 1; + } + + // Shouldn't happen + return 1; +} + /*}}}*/ +// debVS::CmpVersion - Comparison for versions /*{{{*/ +// --------------------------------------------------------------------- +/* This fragments the version into E:V-R triples and compares each + portion separately. */ +int debVersioningSystem::DoCmpVersion(const char *A,const char *AEnd, + const char *B,const char *BEnd) +{ + // Strip off the epoch and compare it + const char *lhs = (const char*) memchr(A, ':', AEnd - A); + const char *rhs = (const char*) memchr(B, ':', BEnd - B); + if (lhs == NULL) + lhs = A; + if (rhs == NULL) + rhs = B; + + // Special case: a zero epoch is the same as no epoch, + // so remove it. + if (lhs != A) + { + for (; *A == '0'; ++A); + if (A == lhs) + { + ++A; + ++lhs; + } + } + if (rhs != B) + { + for (; *B == '0'; ++B); + if (B == rhs) + { + ++B; + ++rhs; + } + } + + // Compare the epoch + int Res = CmpFragment(A,lhs,B,rhs); + if (Res != 0) + return Res; + + // Skip the : + if (lhs != A) + lhs++; + if (rhs != B) + rhs++; + + // Find the last - + const char *dlhs = (const char*) memrchr(lhs, '-', AEnd - lhs); + const char *drhs = (const char*) memrchr(rhs, '-', BEnd - rhs); + if (dlhs == NULL) + dlhs = AEnd; + if (drhs == NULL) + drhs = BEnd; + + // Compare the main version + Res = CmpFragment(lhs,dlhs,rhs,drhs); + if (Res != 0) + return Res; + + // Skip the - + if (dlhs != lhs) + dlhs++; + if (drhs != rhs) + drhs++; + + // no debian revision need to be treated like -0 + if (*(dlhs-1) == '-' && *(drhs-1) == '-') + return CmpFragment(dlhs,AEnd,drhs,BEnd); + else if (*(dlhs-1) == '-') + { + const char* null = "0"; + return CmpFragment(dlhs,AEnd,null, null+1); + } + else if (*(drhs-1) == '-') + { + const char* null = "0"; + return CmpFragment(null, null+1, drhs, BEnd); + } + else + return 0; +} + /*}}}*/ +// debVS::CheckDep - Check a single dependency /*{{{*/ +// --------------------------------------------------------------------- +/* This simply performs the version comparison and switch based on + operator. If DepVer is 0 then we are comparing against a provides + with no version. */ +bool debVersioningSystem::CheckDep(const char *PkgVer, + int Op,const char *DepVer) +{ + if (DepVer == 0 || DepVer[0] == 0) + return true; + if (PkgVer == 0 || PkgVer[0] == 0) + return false; + Op &= 0x0F; + + // fast track for (equal) strings [by location] which are by definition equal versions + if (PkgVer == DepVer) + return Op == pkgCache::Dep::Equals || Op == pkgCache::Dep::LessEq || Op == pkgCache::Dep::GreaterEq; + + // Perform the actual comparison. + int const Res = CmpVersion(PkgVer, DepVer); + switch (Op) + { + case pkgCache::Dep::LessEq: + if (Res <= 0) + return true; + break; + + case pkgCache::Dep::GreaterEq: + if (Res >= 0) + return true; + break; + + case pkgCache::Dep::Less: + if (Res < 0) + return true; + break; + + case pkgCache::Dep::Greater: + if (Res > 0) + return true; + break; + + case pkgCache::Dep::Equals: + if (Res == 0) + return true; + break; + + case pkgCache::Dep::NotEquals: + if (Res != 0) + return true; + break; + } + + return false; +} + /*}}}*/ +// debVS::UpstreamVersion - Return the upstream version string /*{{{*/ +// --------------------------------------------------------------------- +/* This strips all the debian specific information from the version number */ +std::string debVersioningSystem::UpstreamVersion(const char *Ver) +{ + // Strip off the bit before the first colon + const char *I = Ver; + for (; *I != 0 && *I != ':'; I++); + if (*I == ':') + Ver = I + 1; + + // Chop off the trailing - + I = Ver; + unsigned Last = strlen(Ver); + for (; *I != 0; I++) + if (*I == '-') + Last = I - Ver; + + return std::string(Ver,Last); +} + /*}}}*/ diff --git a/apt-pkg/deb/debversion.h b/apt-pkg/deb/debversion.h new file mode 100644 index 0000000..5c328a9 --- /dev/null +++ b/apt-pkg/deb/debversion.h @@ -0,0 +1,41 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Version - Versioning system for Debian + + This implements the standard Debian versioning system. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_DEBVERSION_H +#define PKGLIB_DEBVERSION_H + +#include <apt-pkg/version.h> + +#include <string> + +class APT_PUBLIC debVersioningSystem : public pkgVersioningSystem +{ + public: + + static int CmpFragment(const char *A, const char *AEnd, const char *B, + const char *BEnd) APT_PURE; + + // Compare versions.. + virtual int DoCmpVersion(const char *A,const char *Aend, + const char *B,const char *Bend) APT_OVERRIDE APT_PURE; + virtual bool CheckDep(const char *PkgVer,int Op,const char *DepVer) APT_OVERRIDE APT_PURE; + virtual APT_PURE int DoCmpReleaseVer(const char *A,const char *Aend, + const char *B,const char *Bend) APT_OVERRIDE + { + return DoCmpVersion(A,Aend,B,Bend); + } + virtual std::string UpstreamVersion(const char *A) APT_OVERRIDE; + + debVersioningSystem(); +}; + +extern APT_PUBLIC debVersioningSystem debVS; + +#endif diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc new file mode 100644 index 0000000..93effa9 --- /dev/null +++ b/apt-pkg/deb/dpkgpm.cc @@ -0,0 +1,2491 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + DPKG Package Manager - Provide an interface to dpkg + + ##################################################################### */ + /*}}}*/ +// Includes /*{{{*/ +#include <config.h> + +#include <apt-pkg/cachefile.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/debsystem.h> +#include <apt-pkg/depcache.h> +#include <apt-pkg/dpkgpm.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/install-progress.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/packagemanager.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/statechanges.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/version.h> + +#include <dirent.h> +#include <errno.h> +#include <fcntl.h> +#include <grp.h> +#include <pwd.h> +#include <signal.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/ioctl.h> +#include <sys/select.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <termios.h> +#include <time.h> +#include <unistd.h> + +#include <algorithm> +#include <array> +#include <cstring> +#include <iostream> +#include <map> +#include <numeric> +#include <set> +#include <sstream> +#include <string> +#include <type_traits> +#include <unordered_set> +#include <utility> +#include <vector> + +#include <apti18n.h> + /*}}}*/ + +extern char **environ; + +using namespace std; + +APT_PURE static string AptHistoryRequestingUser() /*{{{*/ +{ + const char* EnvKeys[]{"SUDO_UID", "PKEXEC_UID", "PACKAGEKIT_CALLER_UID"}; + + for (const auto &Key: EnvKeys) + { + if (getenv(Key) != nullptr) + { + int uid = atoi(getenv(Key)); + if (uid > 0) { + struct passwd pwd; + struct passwd *result; + char buf[255]; + if (getpwuid_r(uid, &pwd, buf, sizeof(buf), &result) == 0 && result != NULL) { + std::string res; + strprintf(res, "%s (%d)", pwd.pw_name, uid); + return res; + } + } + } + } + return ""; +} + /*}}}*/ +APT_PURE static unsigned int EnvironmentSize() /*{{{*/ +{ + unsigned int size = 0; + char **envp = environ; + + while (*envp != NULL) + size += strlen (*envp++) + 1; + + return size; +} + /*}}}*/ +class pkgDPkgPMPrivate /*{{{*/ +{ +public: + pkgDPkgPMPrivate() : stdin_is_dev_null(false), status_fd_reached_end_of_file(false), + dpkgbuf_pos(0), term_out(NULL), history_out(NULL), + progress(NULL), tt_is_valid(false), master(-1), + slave(NULL), protect_slave_from_dying(-1), + direct_stdin(false) + { + dpkgbuf[0] = '\0'; + } + ~pkgDPkgPMPrivate() + { + } + bool stdin_is_dev_null; + bool status_fd_reached_end_of_file; + // the buffer we use for the dpkg status-fd reading + char dpkgbuf[1024]; + size_t dpkgbuf_pos; + FILE *term_out; + FILE *history_out; + string dpkg_error; + APT::Progress::PackageManager *progress; + + // pty stuff + struct termios tt; + bool tt_is_valid; + int master; + char * slave; + int protect_slave_from_dying; + + // signals + sigset_t sigmask; + sigset_t original_sigmask; + + bool direct_stdin; +}; + /*}}}*/ +namespace +{ + // Maps the dpkg "processing" info to human readable names. Entry 0 + // of each array is the key, entry 1 is the value. + const std::pair<const char *, const char *> PackageProcessingOps[] = { + std::make_pair("install", N_("Preparing %s")), + // we don't care for the difference + std::make_pair("upgrade", N_("Preparing %s")), + std::make_pair("configure", N_("Preparing to configure %s")), + std::make_pair("remove", N_("Preparing for removal of %s")), + std::make_pair("purge", N_("Preparing to completely remove %s")), + std::make_pair("disappear", N_("Noting disappearance of %s")), + std::make_pair("trigproc", N_("Running post-installation trigger %s")) + }; + + const std::pair<const char *, const char *> * const PackageProcessingOpsBegin = PackageProcessingOps; + const std::pair<const char *, const char *> * const PackageProcessingOpsEnd = PackageProcessingOps + sizeof(PackageProcessingOps) / sizeof(PackageProcessingOps[0]); + + // Predicate to test whether an entry in the PackageProcessingOps + // array matches a string. + class MatchProcessingOp + { + const char *target; + + public: + explicit MatchProcessingOp(const char *the_target) + : target(the_target) + { + } + + bool operator()(const std::pair<const char *, const char *> &pair) const + { + return strcmp(pair.first, target) == 0; + } + }; +} + +// ionice - helper function to ionice the given PID /*{{{*/ +/* there is no C header for ionice yet - just the syscall interface + so we use the binary from util-linux */ +static bool ionice(int PID) +{ + if (!FileExists("/usr/bin/ionice")) + return false; + pid_t Process = ExecFork(); + if (Process == 0) + { + char buf[32]; + snprintf(buf, sizeof(buf), "-p%d", PID); + const char *Args[4]; + Args[0] = "/usr/bin/ionice"; + Args[1] = "-c3"; + Args[2] = buf; + Args[3] = 0; + execv(Args[0], (char **)Args); + } + return ExecWait(Process, "ionice"); +} + /*}}}*/ +// FindNowVersion - Helper to find a Version in "now" state /*{{{*/ +// --------------------------------------------------------------------- +/* This is helpful when a package is no longer installed but has residual + * config files + */ +static +pkgCache::VerIterator FindNowVersion(const pkgCache::PkgIterator &Pkg) +{ + pkgCache::VerIterator Ver; + for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver) + for (pkgCache::VerFileIterator Vf = Ver.FileList(); Vf.end() == false; ++Vf) + for (pkgCache::PkgFileIterator F = Vf.File(); F.end() == false; ++F) + { + if (F.Archive() != 0 && strcmp(F.Archive(), "now") == 0) + return Ver; + } + return Ver; +} + /*}}}*/ +static pkgCache::VerIterator FindToBeRemovedVersion(pkgCache::PkgIterator const &Pkg)/*{{{*/ +{ + auto const PV = Pkg.CurrentVer(); + if (PV.end() == false) + return PV; + return FindNowVersion(Pkg); +} + /*}}}*/ + +// DPkgPM::pkgDPkgPM - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgDPkgPM::pkgDPkgPM(pkgDepCache *Cache) + : pkgPackageManager(Cache),d(new pkgDPkgPMPrivate()), pkgFailures(0), PackagesDone(0), PackagesTotal(0) +{ +} + /*}}}*/ +// DPkgPM::pkgDPkgPM - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgDPkgPM::~pkgDPkgPM() +{ + delete d; +} + /*}}}*/ +// DPkgPM::Install - Install a package /*{{{*/ +// --------------------------------------------------------------------- +/* Add an install operation to the sequence list */ +bool pkgDPkgPM::Install(PkgIterator Pkg,string File) +{ + if (File.empty() == true || Pkg.end() == true) + return _error->Error("Internal Error, No file name for %s",Pkg.FullName().c_str()); + + List.emplace_back(Item::Install, Pkg, debSystem::StripDpkgChrootDirectory(File)); + return true; +} + /*}}}*/ +// DPkgPM::Configure - Configure a package /*{{{*/ +// --------------------------------------------------------------------- +/* Add a configure operation to the sequence list */ +bool pkgDPkgPM::Configure(PkgIterator Pkg) +{ + if (Pkg.end() == true) + return false; + + List.push_back(Item(Item::Configure, Pkg)); + + // Use triggers for config calls if we configure "smart" + // as otherwise Pre-Depends will not be satisfied, see #526774 + if (_config->FindB("DPkg::TriggersPending", false) == true) + List.push_back(Item(Item::TriggersPending, PkgIterator())); + + return true; +} + /*}}}*/ +// DPkgPM::Remove - Remove a package /*{{{*/ +// --------------------------------------------------------------------- +/* Add a remove operation to the sequence list */ +bool pkgDPkgPM::Remove(PkgIterator Pkg,bool Purge) +{ + if (Pkg.end() == true) + return false; + + if (Purge == true) + List.push_back(Item(Item::Purge,Pkg)); + else + List.push_back(Item(Item::Remove,Pkg)); + return true; +} + /*}}}*/ +// DPkgPM::SendPkgInfo - Send info for install-pkgs hook /*{{{*/ +// --------------------------------------------------------------------- +/* This is part of the helper script communication interface, it sends + very complete information down to the other end of the pipe.*/ +bool pkgDPkgPM::SendPkgsInfo(FILE * const F, unsigned int const &Version) +{ + // This version of APT supports only v3, so don't sent higher versions + if (Version <= 3) + fprintf(F,"VERSION %u\n", Version); + else + fprintf(F,"VERSION 3\n"); + + /* Write out all of the configuration directives by walking the + configuration tree */ + const Configuration::Item *Top = _config->Tree(0); + for (; Top != 0;) + { + if (Top->Value.empty() == false) + { + fprintf(F,"%s=%s\n", + QuoteString(Top->FullTag(),"=\"\n").c_str(), + QuoteString(Top->Value,"\n").c_str()); + } + + if (Top->Child != 0) + { + Top = Top->Child; + continue; + } + + while (Top != 0 && Top->Next == 0) + Top = Top->Parent; + if (Top != 0) + Top = Top->Next; + } + fprintf(F,"\n"); + + // Write out the package actions in order. + for (vector<Item>::iterator I = List.begin(); I != List.end(); ++I) + { + if(I->Pkg.end() == true) + continue; + + pkgDepCache::StateCache &S = Cache[I->Pkg]; + + fprintf(F,"%s ",I->Pkg.Name()); + + // Current version which we are going to replace + pkgCache::VerIterator CurVer = I->Pkg.CurrentVer(); + if (CurVer.end() == true && (I->Op == Item::Remove || I->Op == Item::Purge)) + CurVer = FindNowVersion(I->Pkg); + + if (CurVer.end() == true) + { + if (Version <= 2) + fprintf(F, "- "); + else + fprintf(F, "- - none "); + } + else + { + fprintf(F, "%s ", CurVer.VerStr()); + if (Version >= 3) + fprintf(F, "%s %s ", CurVer.Arch(), CurVer.MultiArchType()); + } + + // Show the compare operator between current and install version + if (S.InstallVer != 0) + { + pkgCache::VerIterator const InstVer = S.InstVerIter(Cache); + int Comp = 2; + if (CurVer.end() == false) + Comp = InstVer.CompareVer(CurVer); + if (Comp < 0) + fprintf(F,"> "); + else if (Comp == 0) + fprintf(F,"= "); + else if (Comp > 0) + fprintf(F,"< "); + fprintf(F, "%s ", InstVer.VerStr()); + if (Version >= 3) + fprintf(F, "%s %s ", InstVer.Arch(), InstVer.MultiArchType()); + } + else + { + if (Version <= 2) + fprintf(F, "> - "); + else + fprintf(F, "> - - none "); + } + + // Show the filename/operation + if (I->Op == Item::Install) + { + // No errors here.. + if (I->File[0] != '/') + fprintf(F,"**ERROR**\n"); + else + fprintf(F,"%s\n",I->File.c_str()); + } + else if (I->Op == Item::Configure) + fprintf(F,"**CONFIGURE**\n"); + else if (I->Op == Item::Remove || + I->Op == Item::Purge) + fprintf(F,"**REMOVE**\n"); + + if (ferror(F) != 0) + return false; + } + return true; +} + /*}}}*/ +// DPkgPM::RunScriptsWithPkgs - Run scripts with package names on stdin /*{{{*/ +// --------------------------------------------------------------------- +/* This looks for a list of scripts to run from the configuration file + each one is run and is fed on standard input a list of all .deb files + that are due to be installed. */ +bool pkgDPkgPM::RunScriptsWithPkgs(const char *Cnf) +{ + bool result = true; + + Configuration::Item const *Opts = _config->Tree(Cnf); + if (Opts == 0 || Opts->Child == 0) + return true; + Opts = Opts->Child; + + sighandler_t old_sigpipe = signal(SIGPIPE, SIG_IGN); + sighandler_t old_sigint = signal(SIGINT, SIG_IGN); + sighandler_t old_sigquit = signal(SIGQUIT, SIG_IGN); + + unsigned int Count = 1; + for (; Opts != 0; Opts = Opts->Next, Count++) + { + if (Opts->Value.empty() == true) + continue; + + if(_config->FindB("Debug::RunScripts", false) == true) + std::clog << "Running external script with list of all .deb file: '" + << Opts->Value << "'" << std::endl; + + // Determine the protocol version + string OptSec = Opts->Value; + string::size_type Pos; + if ((Pos = OptSec.find(' ')) == string::npos || Pos == 0) + Pos = OptSec.length(); + OptSec = "DPkg::Tools::Options::" + string(Opts->Value.c_str(),Pos); + + unsigned int Version = _config->FindI(OptSec+"::Version",1); + unsigned int InfoFD = _config->FindI(OptSec + "::InfoFD", STDIN_FILENO); + + // Create the pipes + std::set<int> KeepFDs; + MergeKeepFdsFromConfiguration(KeepFDs); + int Pipes[2]; + if (pipe(Pipes) != 0) { + result = _error->Errno("pipe","Failed to create IPC pipe to subprocess"); + break; + } + if (InfoFD != (unsigned)Pipes[0]) + SetCloseExec(Pipes[0],true); + else + KeepFDs.insert(Pipes[0]); + + + SetCloseExec(Pipes[1],true); + + // Purified Fork for running the script + pid_t Process = ExecFork(KeepFDs); + if (Process == 0) + { + // Setup the FDs + dup2(Pipes[0], InfoFD); + SetCloseExec(STDOUT_FILENO,false); + SetCloseExec(STDIN_FILENO,false); + SetCloseExec(STDERR_FILENO,false); + + string hookfd; + strprintf(hookfd, "%d", InfoFD); + setenv("APT_HOOK_INFO_FD", hookfd.c_str(), 1); + + if (_system != nullptr && _system->IsLocked() == true && stringcasecmp(Cnf, "DPkg::Pre-Install-Pkgs") == 0) + setenv("DPKG_FRONTEND_LOCKED", "true", 1); + + debSystem::DpkgChrootDirectory(); + const char *Args[4]; + Args[0] = "/bin/sh"; + Args[1] = "-c"; + Args[2] = Opts->Value.c_str(); + Args[3] = 0; + execv(Args[0],(char **)Args); + _exit(100); + } + close(Pipes[0]); + FILE *F = fdopen(Pipes[1],"w"); + if (F == 0) { + result = _error->Errno("fdopen","Failed to open new FD"); + break; + } + + // Feed it the filenames. + if (Version <= 1) + { + for (vector<Item>::iterator I = List.begin(); I != List.end(); ++I) + { + // Only deal with packages to be installed from .deb + if (I->Op != Item::Install) + continue; + + // No errors here.. + if (I->File[0] != '/') + continue; + + /* Feed the filename of each package that is pending install + into the pipe. */ + fprintf(F,"%s\n",I->File.c_str()); + if (ferror(F) != 0) + break; + } + } + else + SendPkgsInfo(F, Version); + + fclose(F); + + // Clean up the sub process + if (ExecWait(Process,Opts->Value.c_str()) == false) { + result = _error->Error("Failure running script %s",Opts->Value.c_str()); + break; + } + } + signal(SIGINT, old_sigint); + signal(SIGPIPE, old_sigpipe); + signal(SIGQUIT, old_sigquit); + + return result; +} + /*}}}*/ +// DPkgPM::DoStdin - Read stdin and pass to master pty /*{{{*/ +// --------------------------------------------------------------------- +/* +*/ +void pkgDPkgPM::DoStdin(int master) +{ + unsigned char input_buf[256] = {0,}; + ssize_t len = read(STDIN_FILENO, input_buf, sizeof(input_buf)); + if (len) + FileFd::Write(master, input_buf, len); + else + d->stdin_is_dev_null = true; +} + /*}}}*/ +// DPkgPM::DoTerminalPty - Read the terminal pty and write log /*{{{*/ +// --------------------------------------------------------------------- +/* + * read the terminal pty and write log + */ +void pkgDPkgPM::DoTerminalPty(int master) +{ + unsigned char term_buf[1024] = {0,0, }; + + ssize_t len=read(master, term_buf, sizeof(term_buf)); + if(len == -1 && errno == EIO) + { + // this happens when the child is about to exit, we + // give it time to actually exit, otherwise we run + // into a race so we sleep for half a second. + struct timespec sleepfor = { 0, 500000000 }; + nanosleep(&sleepfor, NULL); + return; + } + if(len <= 0) + return; + FileFd::Write(1, term_buf, len); + if(d->term_out) + fwrite(term_buf, len, sizeof(char), d->term_out); +} + /*}}}*/ +// DPkgPM::ProcessDpkgStatusBuf /*{{{*/ +void pkgDPkgPM::ProcessDpkgStatusLine(char *line) +{ + bool const Debug = _config->FindB("Debug::pkgDPkgProgressReporting",false); + if (Debug == true) + std::clog << "got from dpkg '" << line << "'" << std::endl; + + /* dpkg sends strings like this: + 'status: <pkg>: <pkg qstate>' + 'status: <pkg>:<arch>: <pkg qstate>' + + 'processing: {install,upgrade,configure,remove,purge,disappear,trigproc}: pkg' + 'processing: {install,upgrade,configure,remove,purge,disappear,trigproc}: trigger' + */ + + // we need to split on ": " (note the appended space) as the ':' is + // part of the pkgname:arch information that dpkg sends + // + // A dpkg error message may contain additional ":" (like + // "failed in buffer_write(fd) (10, ret=-1): backend dpkg-deb ..." + // so we need to ensure to not split too much + std::vector<std::string> list = StringSplit(line, ": ", 4); + if(list.size() < 3) + { + if (Debug == true) + std::clog << "ignoring line: not enough ':'" << std::endl; + return; + } + + // build the (prefix, pkgname, action) tuple, position of this + // is different for "processing" or "status" messages + std::string prefix = APT::String::Strip(list[0]); + std::string pkgname; + std::string action; + + // "processing" has the form "processing: action: pkg or trigger" + // with action = ["install", "upgrade", "configure", "remove", "purge", + // "disappear", "trigproc"] + if (prefix == "processing") + { + pkgname = APT::String::Strip(list[2]); + action = APT::String::Strip(list[1]); + } + // "status" has the form: "status: pkg: state" + // with state in ["half-installed", "unpacked", "half-configured", + // "installed", "config-files", "not-installed"] + else if (prefix == "status") + { + pkgname = APT::String::Strip(list[1]); + action = APT::String::Strip(list[2]); + + /* handle the special cases first: + + errors look like this: + 'status: /var/cache/apt/archives/krecipes_0.8.1-0ubuntu1_i386.deb : error : trying to overwrite `/usr/share/doc/kde/HTML/en/krecipes/krectip.png', which is also in package krecipes-data + and conffile-prompt like this + 'status:/etc/compiz.conf/compiz.conf : conffile-prompt: 'current-conffile' 'new-conffile' useredited distedited + */ + if(action == "error") + { + d->progress->Error(pkgname, PackagesDone, PackagesTotal, list[3]); + ++pkgFailures; + WriteApportReport(pkgname.c_str(), list[3].c_str()); + return; + } + else if(action == "conffile-prompt") + { + d->progress->ConffilePrompt(pkgname, PackagesDone, PackagesTotal, list[3]); + return; + } + } else { + if (Debug == true) + std::clog << "unknown prefix '" << prefix << "'" << std::endl; + return; + } + + // At this point we have a pkgname, but it might not be arch-qualified ! + if (pkgname.find(":") == std::string::npos) + { + pkgCache::GrpIterator const Grp = Cache.FindGrp(pkgname); + if (unlikely(Grp.end()== true)) + { + if (Debug == true) + std::clog << "unable to figure out which package is dpkg referring to with '" << pkgname << "'! (0)" << std::endl; + return; + } + /* No arch means that dpkg believes there can only be one package + this can refer to so lets see what could be candidates here: */ + std::vector<pkgCache::PkgIterator> candset; + for (auto P = Grp.PackageList(); P.end() != true; P = Grp.NextPkg(P)) + { + if (PackageOps.find(P.FullName()) != PackageOps.end()) + candset.push_back(P); + // packages can disappear without them having any interaction itself + // so we have to consider these as candidates, too + else if (P->CurrentVer != 0 && action == "disappear") + candset.push_back(P); + } + if (unlikely(candset.empty())) + { + if (Debug == true) + std::clog << "unable to figure out which package is dpkg referring to with '" << pkgname << "'! (1)" << std::endl; + return; + } + else if (candset.size() == 1) // we are lucky + pkgname = candset.cbegin()->FullName(); + else + { + /* here be dragons^Wassumptions about dpkg: + - an M-A:same version is always arch-qualified + - a package from a foreign arch is (in newer versions) */ + size_t installedInstances = 0, wannabeInstances = 0; + for (auto const &P: candset) + { + if (P->CurrentVer != 0) + { + ++installedInstances; + if (Cache[P].Delete() == false) + ++wannabeInstances; + } + else if (Cache[P].Install()) + ++wannabeInstances; + } + // the package becomes M-A:same, so we are still talking about current + if (installedInstances == 1 && wannabeInstances >= 2) + { + for (auto const &P: candset) + { + if (P->CurrentVer == 0) + continue; + pkgname = P.FullName(); + break; + } + } + // the package was M-A:same, it isn't now, so we can only talk about that + else if (installedInstances >= 2 && wannabeInstances == 1) + { + for (auto const &P: candset) + { + auto const IV = Cache[P].InstVerIter(Cache); + if (IV.end()) + continue; + pkgname = P.FullName(); + break; + } + } + // that is a crossgrade + else if (installedInstances == 1 && wannabeInstances == 1 && candset.size() == 2) + { + auto const PkgHasCurrentVersion = [](pkgCache::PkgIterator const &P) { return P->CurrentVer != 0; }; + auto const P = std::find_if(candset.begin(), candset.end(), PkgHasCurrentVersion); + if (unlikely(P == candset.end())) + { + if (Debug == true) + std::clog << "situation for '" << pkgname << "' looked like a crossgrade, but no current version?!" << std::endl; + return; + } + auto fullname = P->FullName(); + if (PackageOps[fullname].size() != PackageOpsDone[fullname]) + pkgname = std::move(fullname); + else + { + auto const pkgi = std::find_if_not(candset.begin(), candset.end(), PkgHasCurrentVersion); + if (unlikely(pkgi == candset.end())) + { + if (Debug == true) + std::clog << "situation for '" << pkgname << "' looked like a crossgrade, but all are installed?!" << std::endl; + return; + } + pkgname = pkgi->FullName(); + } + } + // we are desperate: so "just" take the native one, but that might change mid-air, + // so we have to ask dpkg what it believes native is at the moment… all the time + else + { + std::vector<std::string> sArgs = debSystem::GetDpkgBaseCommand(); + sArgs.push_back("--print-architecture"); + int outputFd = -1; + pid_t const dpkgNativeArch = debSystem::ExecDpkg(sArgs, nullptr, &outputFd, true); + if (unlikely(dpkgNativeArch == -1)) + { + if (Debug == true) + std::clog << "calling dpkg failed to ask it for its current native architecture to expand '" << pkgname << "'!" << std::endl; + return; + } + FILE *dpkg = fdopen(outputFd, "r"); + if(dpkg != NULL) + { + char* buf = NULL; + size_t bufsize = 0; + if (getline(&buf, &bufsize, dpkg) != -1) + pkgname += ':' + bufsize; + free(buf); + fclose(dpkg); + } + ExecWait(dpkgNativeArch, "dpkg --print-architecture", true); + if (pkgname.find(':') != std::string::npos) + { + if (Debug == true) + std::clog << "unable to figure out which package is dpkg referring to with '" << pkgname << "'! (2)" << std::endl; + return; + } + } + } + } + + std::string arch = ""; + if (pkgname.find(":") != string::npos) + arch = StringSplit(pkgname, ":")[1]; + std::string i18n_pkgname = pkgname; + if (arch.size() != 0) + strprintf(i18n_pkgname, "%s (%s)", StringSplit(pkgname, ":")[0].c_str(), arch.c_str()); + + // 'processing' from dpkg looks like + // 'processing: action: pkg' + if(prefix == "processing") + { + auto const iter = std::find_if(PackageProcessingOpsBegin, PackageProcessingOpsEnd, MatchProcessingOp(action.c_str())); + if(iter == PackageProcessingOpsEnd) + { + if (Debug == true) + std::clog << "ignoring unknown action: " << action << std::endl; + return; + } + std::string msg; + strprintf(msg, _(iter->second), i18n_pkgname.c_str()); + d->progress->StatusChanged(pkgname, PackagesDone, PackagesTotal, msg); + + // FIXME: this needs a muliarch testcase + // FIXME2: is "pkgname" here reliable with dpkg only sending us + // short pkgnames? + if (action == "disappear") + handleDisappearAction(pkgname); + else if (action == "upgrade") + handleCrossUpgradeAction(pkgname); + return; + } + + if (prefix == "status") + { + std::vector<struct DpkgState> &states = PackageOps[pkgname]; + if(PackageOpsDone[pkgname] < states.size()) + { + char const * next_action = states[PackageOpsDone[pkgname]].state; + if (next_action) + { + /* + if (action == "half-installed" && strcmp("half-configured", next_action) == 0 && + PackageOpsDone[pkg] + 2 < states.size() && action == states[PackageOpsDone[pkg] + 2].state) + { + if (Debug == true) + std::clog << "(parsed from dpkg) pkg: " << short_pkgname << " action: " << action + << " pending trigger defused by unpack" << std::endl; + // unpacking a package defuses the pending trigger + PackageOpsDone[pkg] += 2; + PackagesDone += 2; + next_action = states[PackageOpsDone[pkg]].state; + } + */ + if (Debug == true) + std::clog << "(parsed from dpkg) pkg: " << pkgname + << " action: " << action << " (expected: '" << next_action << "' " + << PackageOpsDone[pkgname] << " of " << states.size() << ")" << endl; + + // check if the package moved to the next dpkg state + if(action == next_action) + { + // only read the translation if there is actually a next action + char const * const translation = _(states[PackageOpsDone[pkgname]].str); + + // we moved from one dpkg state to a new one, report that + ++PackageOpsDone[pkgname]; + ++PackagesDone; + + std::string msg; + strprintf(msg, translation, i18n_pkgname.c_str()); + d->progress->StatusChanged(pkgname, PackagesDone, PackagesTotal, msg); + } + } + } + else if (action == "triggers-pending") + { + if (Debug == true) + std::clog << "(parsed from dpkg) pkg: " << pkgname + << " action: " << action << " (prefix 2 to " + << PackageOpsDone[pkgname] << " of " << states.size() << ")" << endl; + + states.insert(states.begin(), {"installed", N_("Installed %s")}); + states.insert(states.begin(), {"half-configured", N_("Configuring %s")}); + PackagesTotal += 2; + } + } +} + /*}}}*/ +// DPkgPM::handleDisappearAction /*{{{*/ +void pkgDPkgPM::handleDisappearAction(string const &pkgname) +{ + pkgCache::PkgIterator Pkg = Cache.FindPkg(pkgname); + if (unlikely(Pkg.end() == true)) + return; + + // a disappeared package has no further actions + auto const ROps = PackageOps[Pkg.FullName()].size(); + auto && ROpsDone = PackageOpsDone[Pkg.FullName()]; + PackagesDone += ROps - ROpsDone; + ROpsDone = ROps; + + // record the package name for display and stuff later + disappearedPkgs.insert(Pkg.FullName(true)); + + // the disappeared package was auto-installed - nothing to do + if ((Cache[Pkg].Flags & pkgCache::Flag::Auto) == pkgCache::Flag::Auto) + return; + pkgCache::VerIterator PkgVer = Cache[Pkg].InstVerIter(Cache); + if (unlikely(PkgVer.end() == true)) + return; + /* search in the list of dependencies for (Pre)Depends, + check if this dependency has a Replaces on our package + and if so transfer the manual installed flag to it */ + for (pkgCache::DepIterator Dep = PkgVer.DependsList(); Dep.end() != true; ++Dep) + { + if (Dep->Type != pkgCache::Dep::Depends && + Dep->Type != pkgCache::Dep::PreDepends) + continue; + pkgCache::PkgIterator Tar = Dep.TargetPkg(); + if (unlikely(Tar.end() == true)) + continue; + // the package is already marked as manual + if ((Cache[Tar].Flags & pkgCache::Flag::Auto) != pkgCache::Flag::Auto) + continue; + pkgCache::VerIterator TarVer = Cache[Tar].InstVerIter(Cache); + if (TarVer.end() == true) + continue; + for (pkgCache::DepIterator Rep = TarVer.DependsList(); Rep.end() != true; ++Rep) + { + if (Rep->Type != pkgCache::Dep::Replaces) + continue; + if (Pkg != Rep.TargetPkg()) + continue; + // okay, they are strongly connected - transfer manual-bit + if (Debug == true) + std::clog << "transfer manual-bit from disappeared »" << pkgname << "« to »" << Tar.FullName() << "«" << std::endl; + Cache[Tar].Flags &= ~Flag::Auto; + break; + } + } +} + /*}}}*/ +void pkgDPkgPM::handleCrossUpgradeAction(string const &pkgname) /*{{{*/ +{ + // in a crossgrade what looked like a remove first is really an unpack over it + auto const Pkg = Cache.FindPkg(pkgname); + if (likely(Pkg.end() == false) && Cache[Pkg].Delete()) + { + auto const Grp = Pkg.Group(); + if (likely(Grp.end() == false)) + { + for (auto P = Grp.PackageList(); P.end() != true; P = Grp.NextPkg(P)) + if(Cache[P].Install()) + { + auto && Ops = PackageOps[P.FullName()]; + auto const unpackOp = std::find_if(Ops.cbegin(), Ops.cend(), [](DpkgState const &s) { return strcmp(s.state, "unpacked") == 0; }); + if (unpackOp != Ops.cend()) + { + // skip ahead in the crossgraded packages + auto const skipped = std::distance(Ops.cbegin(), unpackOp); + PackagesDone += skipped; + PackageOpsDone[P.FullName()] += skipped; + // finish the crossremoved package + auto const ROps = PackageOps[Pkg.FullName()].size(); + auto && ROpsDone = PackageOpsDone[Pkg.FullName()]; + PackagesDone += ROps - ROpsDone; + ROpsDone = ROps; + break; + } + } + } + } +} + /*}}}*/ +// DPkgPM::DoDpkgStatusFd /*{{{*/ +void pkgDPkgPM::DoDpkgStatusFd(int statusfd) +{ + auto const remainingBuffer = (sizeof(d->dpkgbuf) / sizeof(d->dpkgbuf[0])) - d->dpkgbuf_pos; + if (likely(remainingBuffer > 0) && d->status_fd_reached_end_of_file == false) + { + auto const len = read(statusfd, &d->dpkgbuf[d->dpkgbuf_pos], remainingBuffer); + if (len < 0) + return; + else if (len == 0 && d->dpkgbuf_pos == 0) + { + d->status_fd_reached_end_of_file = true; + return; + } + d->dpkgbuf_pos += (len / sizeof(d->dpkgbuf[0])); + } + + // process line by line from the buffer + char *p = d->dpkgbuf, *q = nullptr; + while((q=(char*)memchr(p, '\n', (d->dpkgbuf + d->dpkgbuf_pos) - p)) != nullptr) + { + *q = '\0'; + ProcessDpkgStatusLine(p); + p = q + 1; // continue with next line + } + + // check if we stripped the buffer clean + if (p > (d->dpkgbuf + d->dpkgbuf_pos)) + { + d->dpkgbuf_pos = 0; + return; + } + + // otherwise move the unprocessed tail to the start and update pos + memmove(d->dpkgbuf, p, (p - d->dpkgbuf)); + d->dpkgbuf_pos = (d->dpkgbuf + d->dpkgbuf_pos) - p; +} + /*}}}*/ +// DPkgPM::WriteHistoryTag /*{{{*/ +void pkgDPkgPM::WriteHistoryTag(string const &tag, string value) +{ + size_t const length = value.length(); + if (length == 0) + return; + // poor mans rstrip(", ") + if (value[length-2] == ',' && value[length-1] == ' ') + value.erase(length - 2, 2); + fprintf(d->history_out, "%s: %s\n", tag.c_str(), value.c_str()); +} /*}}}*/ +// DPkgPM::OpenLog /*{{{*/ +bool pkgDPkgPM::OpenLog() +{ + string const logfile_name = _config->FindFile("Dir::Log::Terminal", "/dev/null"); + string logdir = flNotFile(logfile_name); + if(CreateAPTDirectoryIfNeeded(logdir, logdir) == false) + // FIXME: use a better string after freeze + return _error->Error(_("Directory '%s' missing"), logdir.c_str()); + + // get current time + char timestr[200]; + time_t const t = time(NULL); + struct tm tm_buf; + struct tm const * const tmp = localtime_r(&t, &tm_buf); + strftime(timestr, sizeof(timestr), "%F %T", tmp); + + // open terminal log + if (logfile_name != "/dev/null") + { + d->term_out = fopen(logfile_name.c_str(),"a"); + if (d->term_out == NULL) + return _error->WarningE("OpenLog", _("Could not open file '%s'"), logfile_name.c_str()); + setvbuf(d->term_out, NULL, _IONBF, 0); + SetCloseExec(fileno(d->term_out), true); + if (getuid() == 0) // if we aren't root, we can't chown a file, so don't try it + { + struct passwd *pw = getpwnam("root"); + struct group *gr = getgrnam("adm"); + if (pw != NULL && gr != NULL && chown(logfile_name.c_str(), pw->pw_uid, gr->gr_gid) != 0) + _error->WarningE("OpenLog", "chown to root:adm of file %s failed", logfile_name.c_str()); + } + if (chmod(logfile_name.c_str(), 0640) != 0) + _error->WarningE("OpenLog", "chmod 0640 of file %s failed", logfile_name.c_str()); + fprintf(d->term_out, "\nLog started: %s\n", timestr); + } + + // write your history + string const history_name = _config->FindFile("Dir::Log::History", "/dev/null"); + string logdir2 = flNotFile(logfile_name); + if(logdir != logdir2 && CreateAPTDirectoryIfNeeded(logdir2, logdir2) == false) + return _error->Error(_("Directory '%s' missing"), logdir.c_str()); + if (history_name != "/dev/null") + { + d->history_out = fopen(history_name.c_str(),"a"); + if (d->history_out == NULL) + return _error->WarningE("OpenLog", _("Could not open file '%s'"), history_name.c_str()); + SetCloseExec(fileno(d->history_out), true); + chmod(history_name.c_str(), 0644); + fprintf(d->history_out, "\nStart-Date: %s\n", timestr); + string remove, purge, install, reinstall, upgrade, downgrade; + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + enum { CANDIDATE, CANDIDATE_AUTO, CURRENT_CANDIDATE, CURRENT } infostring; + string *line = NULL; + #define HISTORYINFO(X, Y) { line = &X; infostring = Y; } + if (Cache[I].NewInstall() == true) + HISTORYINFO(install, CANDIDATE_AUTO) + else if (Cache[I].ReInstall() == true) + HISTORYINFO(reinstall, CANDIDATE) + else if (Cache[I].Upgrade() == true) + HISTORYINFO(upgrade, CURRENT_CANDIDATE) + else if (Cache[I].Downgrade() == true) + HISTORYINFO(downgrade, CURRENT_CANDIDATE) + else if (Cache[I].Delete() == true) + HISTORYINFO((Cache[I].Purge() ? purge : remove), CURRENT) + else + continue; + #undef HISTORYINFO + line->append(I.FullName(false)).append(" ("); + switch (infostring) { + case CANDIDATE: line->append(Cache[I].CandVersion); break; + case CANDIDATE_AUTO: + line->append(Cache[I].CandVersion); + if ((Cache[I].Flags & pkgCache::Flag::Auto) == pkgCache::Flag::Auto) + line->append(", automatic"); + break; + case CURRENT_CANDIDATE: line->append(Cache[I].CurVersion).append(", ").append(Cache[I].CandVersion); break; + case CURRENT: line->append(Cache[I].CurVersion); break; + } + line->append("), "); + } + if (_config->Exists("Commandline::AsString") == true) + WriteHistoryTag("Commandline", _config->Find("Commandline::AsString")); + std::string RequestingUser = AptHistoryRequestingUser(); + if (RequestingUser != "") + WriteHistoryTag("Requested-By", RequestingUser); + WriteHistoryTag("Install", install); + WriteHistoryTag("Reinstall", reinstall); + WriteHistoryTag("Upgrade", upgrade); + WriteHistoryTag("Downgrade",downgrade); + WriteHistoryTag("Remove",remove); + WriteHistoryTag("Purge",purge); + fflush(d->history_out); + } + + return true; +} + /*}}}*/ +// DPkg::CloseLog /*{{{*/ +bool pkgDPkgPM::CloseLog() +{ + char timestr[200]; + time_t t = time(NULL); + struct tm tm_buf; + struct tm *tmp = localtime_r(&t, &tm_buf); + strftime(timestr, sizeof(timestr), "%F %T", tmp); + + if(d->term_out) + { + fprintf(d->term_out, "Log ended: "); + fprintf(d->term_out, "%s", timestr); + fprintf(d->term_out, "\n"); + fclose(d->term_out); + } + d->term_out = NULL; + + if(d->history_out) + { + if (disappearedPkgs.empty() == false) + { + string disappear; + for (std::set<std::string>::const_iterator d = disappearedPkgs.begin(); + d != disappearedPkgs.end(); ++d) + { + pkgCache::PkgIterator P = Cache.FindPkg(*d); + disappear.append(*d); + if (P.end() == true) + disappear.append(", "); + else + disappear.append(" (").append(Cache[P].CurVersion).append("), "); + } + WriteHistoryTag("Disappeared", disappear); + } + if (d->dpkg_error.empty() == false) + fprintf(d->history_out, "Error: %s\n", d->dpkg_error.c_str()); + fprintf(d->history_out, "End-Date: %s\n", timestr); + fclose(d->history_out); + } + d->history_out = NULL; + + return true; +} + /*}}}*/ + +// DPkgPM::BuildPackagesProgressMap /*{{{*/ +void pkgDPkgPM::BuildPackagesProgressMap() +{ + // map the dpkg states to the operations that are performed + // (this is sorted in the same way as Item::Ops) + static const std::array<std::array<DpkgState, 2>, 4> DpkgStatesOpMap = {{ + // Install operation + {{ + {"half-installed", N_("Unpacking %s")}, + {"unpacked", N_("Installing %s") }, + }}, + // Configure operation + {{ + {"half-configured", N_("Configuring %s") }, + { "installed", N_("Installed %s")}, + }}, + // Remove operation + {{ + {"half-configured", N_("Removing %s")}, + {"half-installed", N_("Removing %s")}, + }}, + // Purge operation + {{ + {"config-files", N_("Completely removing %s")}, + {"not-installed", N_("Completely removed %s")}, + }}, + }}; + static_assert(Item::Purge == 3, "Enum item has unexpected index for mapping array"); + + // init the PackageOps map, go over the list of packages that + // that will be [installed|configured|removed|purged] and add + // them to the PackageOps map (the dpkg states it goes through) + // and the PackageOpsTranslations (human readable strings) + for (auto &&I : List) + { + if(I.Pkg.end() == true) + continue; + + string const name = I.Pkg.FullName(); + PackageOpsDone[name] = 0; + auto AddToPackageOps = [&](decltype(I.Op) const Op) { + auto const DpkgOps = DpkgStatesOpMap[Op]; + std::copy(DpkgOps.begin(), DpkgOps.end(), std::back_inserter(PackageOps[name])); + PackagesTotal += DpkgOps.size(); + }; + // purging a package which is installed first passes through remove states + if (I.Op == Item::Purge && I.Pkg->CurrentVer != 0) + AddToPackageOps(Item::Remove); + AddToPackageOps(I.Op); + + if ((I.Op == Item::Remove || I.Op == Item::Purge) && I.Pkg->CurrentVer != 0) + { + if (I.Pkg->CurrentState == pkgCache::State::UnPacked || + I.Pkg->CurrentState == pkgCache::State::HalfInstalled) + { + if (likely(strcmp(PackageOps[name][0].state, "half-configured") == 0)) + { + ++PackageOpsDone[name]; + --PackagesTotal; + } + } + } + } + /* one extra: We don't want the progress bar to reach 100%, especially not + if we call dpkg --configure --pending and process a bunch of triggers + while showing 100%. Also, spindown takes a while, so never reaching 100% + is way more correct than reaching 100% while still doing stuff even if + doing it this way is slightly bending the rules */ + ++PackagesTotal; +} + /*}}}*/ +void pkgDPkgPM::StartPtyMagic() /*{{{*/ +{ + if (_config->FindB("Dpkg::Use-Pty", true) == false) + { + d->master = -1; + if (d->slave != NULL) + free(d->slave); + d->slave = NULL; + return; + } + + if (isatty(STDIN_FILENO) == 0) + d->direct_stdin = true; + + _error->PushToStack(); + + d->master = posix_openpt(O_RDWR | O_NOCTTY); + if (d->master == -1) + _error->Errno("posix_openpt", _("Can not write log (%s)"), _("Is /dev/pts mounted?")); + else if (unlockpt(d->master) == -1) + _error->Errno("unlockpt", "Unlocking the slave of master fd %d failed!", d->master); + else + { +#ifdef HAVE_PTSNAME_R + char slave_name[64]; // 64 is used by bionic + if (ptsname_r(d->master, slave_name, sizeof(slave_name)) != 0) +#else + char const * const slave_name = ptsname(d->master); + if (slave_name == NULL) +#endif + _error->Errno("ptsname", "Getting name for slave of master fd %d failed!", d->master); + else + { + d->slave = strdup(slave_name); + if (d->slave == NULL) + _error->Errno("strdup", "Copying name %s for slave of master fd %d failed!", slave_name, d->master); + else if (grantpt(d->master) == -1) + _error->Errno("grantpt", "Granting access to slave %s based on master fd %d failed!", slave_name, d->master); + else if (tcgetattr(STDIN_FILENO, &d->tt) == 0) + { + d->tt_is_valid = true; + struct termios raw_tt; + // copy window size of stdout if its a 'good' terminal + if (tcgetattr(STDOUT_FILENO, &raw_tt) == 0) + { + struct winsize win; + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &win) < 0) + _error->Errno("ioctl", "Getting TIOCGWINSZ from stdout failed!"); + if (ioctl(d->master, TIOCSWINSZ, &win) < 0) + _error->Errno("ioctl", "Setting TIOCSWINSZ for master fd %d failed!", d->master); + } + if (tcsetattr(d->master, TCSANOW, &d->tt) == -1) + _error->Errno("tcsetattr", "Setting in Start via TCSANOW for master fd %d failed!", d->master); + + raw_tt = d->tt; + cfmakeraw(&raw_tt); + raw_tt.c_lflag &= ~ECHO; + raw_tt.c_lflag |= ISIG; + // block SIGTTOU during tcsetattr to prevent a hang if + // the process is a member of the background process group + // http://www.opengroup.org/onlinepubs/000095399/functions/tcsetattr.html + sigemptyset(&d->sigmask); + sigaddset(&d->sigmask, SIGTTOU); + sigprocmask(SIG_BLOCK,&d->sigmask, &d->original_sigmask); + if (tcsetattr(STDIN_FILENO, TCSAFLUSH, &raw_tt) == -1) + _error->Errno("tcsetattr", "Setting in Start via TCSAFLUSH for stdin failed!"); + sigprocmask(SIG_SETMASK, &d->original_sigmask, NULL); + + } + if (d->slave != NULL) + { + /* on linux, closing (and later reopening) all references to the slave + makes the slave a death end, so we open it here to have one open all + the time. We could use this fd in SetupSlavePtyMagic() for linux, but + on kfreebsd we get an incorrect ("step like") output then while it has + no problem with closing all references… so to avoid platform specific + code here we combine both and be happy once more */ + d->protect_slave_from_dying = open(d->slave, O_RDWR | O_CLOEXEC | O_NOCTTY); + } + } + } + + if (_error->PendingError() == true) + { + if (d->master != -1) + { + close(d->master); + d->master = -1; + } + if (d->slave != NULL) + { + free(d->slave); + d->slave = NULL; + } + _error->DumpErrors(std::cerr, GlobalError::DEBUG, false); + } + _error->RevertToStack(); +} + /*}}}*/ +void pkgDPkgPM::SetupSlavePtyMagic() /*{{{*/ +{ + if(d->master == -1 || d->slave == NULL) + return; + + if (close(d->master) == -1) + _error->FatalE("close", "Closing master %d in child failed!", d->master); + d->master = -1; + if (setsid() == -1) + _error->FatalE("setsid", "Starting a new session for child failed!"); + + int const slaveFd = open(d->slave, O_RDWR | O_NOCTTY); + if (slaveFd == -1) + _error->FatalE("open", _("Can not write log (%s)"), _("Is /dev/pts mounted?")); + else if (ioctl(slaveFd, TIOCSCTTY, 0) < 0) + _error->FatalE("ioctl", "Setting TIOCSCTTY for slave fd %d failed!", slaveFd); + else + { + unsigned short i = 0; + if (d->direct_stdin == true) + ++i; + for (; i < 3; ++i) + if (dup2(slaveFd, i) == -1) + _error->FatalE("dup2", "Dupping %d to %d in child failed!", slaveFd, i); + + if (d->tt_is_valid == true && tcsetattr(STDIN_FILENO, TCSANOW, &d->tt) < 0) + _error->FatalE("tcsetattr", "Setting in Setup via TCSANOW for slave fd %d failed!", slaveFd); + } + + if (slaveFd != -1) + close(slaveFd); +} + /*}}}*/ +void pkgDPkgPM::StopPtyMagic() /*{{{*/ +{ + if (d->slave != NULL) + free(d->slave); + d->slave = NULL; + if (d->protect_slave_from_dying != -1) + { + close(d->protect_slave_from_dying); + d->protect_slave_from_dying = -1; + } + if(d->master >= 0) + { + if (d->tt_is_valid == true && tcsetattr(STDIN_FILENO, TCSAFLUSH, &d->tt) == -1) + _error->FatalE("tcsetattr", "Setting in Stop via TCSAFLUSH for stdin failed!"); + close(d->master); + d->master = -1; + } +} + /*}}}*/ +static void cleanUpTmpDir(char * const tmpdir) /*{{{*/ +{ + if (tmpdir == nullptr) + return; + DIR * const D = opendir(tmpdir); + if (D == nullptr) + _error->Errno("opendir", _("Unable to read %s"), tmpdir); + else + { + auto const dfd = dirfd(D); + for (struct dirent *Ent = readdir(D); Ent != nullptr; Ent = readdir(D)) + { + if (Ent->d_name[0] == '.') + continue; +#ifdef _DIRENT_HAVE_D_TYPE + if (unlikely(Ent->d_type != DT_LNK && Ent->d_type != DT_UNKNOWN)) + continue; +#endif + if (unlikely(unlinkat(dfd, Ent->d_name, 0) != 0)) + break; + } + closedir(D); + rmdir(tmpdir); + } + free(tmpdir); +} + /*}}}*/ + +// DPkgPM::Go - Run the sequence /*{{{*/ +// --------------------------------------------------------------------- +/* This globs the operations and calls dpkg + * + * If it is called with a progress object apt will report the install + * progress to this object. It maps the dpkg states a package goes + * through to human readable (and i10n-able) + * names and calculates a percentage for each step. + */ +static bool ItemIsEssential(pkgDPkgPM::Item const &I) +{ + static auto const cachegen = _config->Find("pkgCacheGen::Essential"); + if (cachegen == "none" || cachegen == "native") + return true; + if (unlikely(I.Pkg.end())) + return true; + return (I.Pkg->Flags & pkgCache::Flag::Essential) != 0; +} +static bool ItemIsProtected(pkgDPkgPM::Item const &I) +{ + static auto const cachegen = _config->Find("pkgCacheGen::Protected"); + if (cachegen == "none" || cachegen == "native") + return true; + if (unlikely(I.Pkg.end())) + return true; + return (I.Pkg->Flags & pkgCache::Flag::Important) != 0; +} +bool pkgDPkgPM::ExpandPendingCalls(std::vector<Item> &List, pkgDepCache &Cache) +{ + { + std::unordered_set<decltype(pkgCache::Package::ID)> alreadyRemoved; + for (auto && I : List) + if (I.Op == Item::Remove || I.Op == Item::Purge) + alreadyRemoved.insert(I.Pkg->ID); + std::remove_reference<decltype(List)>::type AppendList; + for (auto Pkg = Cache.PkgBegin(); Pkg.end() == false; ++Pkg) + if (Cache[Pkg].Delete() && alreadyRemoved.insert(Pkg->ID).second == true) + AppendList.emplace_back(Cache[Pkg].Purge() ? Item::Purge : Item::Remove, Pkg); + std::move(AppendList.begin(), AppendList.end(), std::back_inserter(List)); + } + { + std::unordered_set<decltype(pkgCache::Package::ID)> alreadyConfigured; + for (auto && I : List) + if (I.Op == Item::Configure) + alreadyConfigured.insert(I.Pkg->ID); + std::remove_reference<decltype(List)>::type AppendList; + for (auto && I : List) + if (I.Op == Item::Install && alreadyConfigured.insert(I.Pkg->ID).second == true) + AppendList.emplace_back(Item::Configure, I.Pkg); + for (auto Pkg = Cache.PkgBegin(); Pkg.end() == false; ++Pkg) + if (Pkg.State() == pkgCache::PkgIterator::NeedsConfigure && + Cache[Pkg].Delete() == false && alreadyConfigured.insert(Pkg->ID).second == true) + AppendList.emplace_back(Item::Configure, Pkg); + std::move(AppendList.begin(), AppendList.end(), std::back_inserter(List)); + } + return true; +} +class APT_HIDDEN BuildDpkgCall { + std::vector<char*> args; + std::vector<bool> to_free; + size_t baseArguments = 0; + size_t baseArgumentsLen = 0; + size_t len = 0; +public: + void clearCallArguments() { + for (size_t i = baseArguments; i < args.size(); ++i) + if (to_free[i]) + std::free(args[i]); + args.erase(args.begin() + baseArguments, args.end()); + to_free.erase(to_free.begin() + baseArguments, to_free.end()); + len = baseArgumentsLen; + } + void reserve(size_t const n) { + args.reserve(n); + to_free.reserve(n); + } + void push_back(char const * const str) { + args.push_back(const_cast<char*>(str)); + to_free.push_back(false); + len += strlen(args.back()); + } + void push_back(std::string &&str) { + args.push_back(strdup(str.c_str())); + to_free.push_back(true); + len += str.length(); + } + auto bytes() const { return len; } + auto data() const { return args.data(); } + auto begin() const { return args.cbegin(); } + auto end() const { return args.cend(); } + auto& front() const { return args.front(); } + APT_NORETURN void execute(char const *const errmsg) { + args.push_back(nullptr); + execvp(args.front(), &args.front()); + std::cerr << errmsg << std::endl; + _exit(100); + } + BuildDpkgCall() { + for (auto &&arg : debSystem::GetDpkgBaseCommand()) + push_back(std::move(arg)); + baseArguments = args.size(); + baseArgumentsLen = len; + } + BuildDpkgCall(BuildDpkgCall const &) = delete; + BuildDpkgCall(BuildDpkgCall &&) = delete; + BuildDpkgCall& operator=(BuildDpkgCall const &) = delete; + BuildDpkgCall& operator=(BuildDpkgCall &&) = delete; + ~BuildDpkgCall() { + baseArguments = 0; + clearCallArguments(); + } +}; +bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress) +{ + struct Inhibitor + { + int Fd = -1; + Inhibitor() + { + if (_config->FindB("DPkg::Inhibit-Shutdown", true)) + Fd = Inhibit("shutdown", "APT", "APT is installing or removing packages", "block"); + } + ~Inhibitor() + { + if (Fd > 0) + close(Fd); + } + } inhibitor; + + // explicitly remove&configure everything for hookscripts and progress building + // we need them only temporarily through, so keep the length and erase afterwards + decltype(List)::const_iterator::difference_type explicitIdx = + std::distance(List.cbegin(), List.cend()); + ExpandPendingCalls(List, Cache); + + /* if dpkg told us that it has already done everything to the package we wanted it to do, + we shouldn't ask it for "more" later. That can e.g. happen if packages without conffiles + are purged as they will have pass through the purge states on remove already */ + auto const StripAlreadyDoneFrom = [&](APT::VersionVector & Pending) { + Pending.erase(std::remove_if(Pending.begin(), Pending.end(), [&](pkgCache::VerIterator const &Ver) { + auto const PN = Ver.ParentPkg().FullName(); + auto const POD = PackageOpsDone.find(PN); + if (POD == PackageOpsDone.end()) + return false; + return PackageOps[PN].size() <= POD->second; + }), Pending.end()); + }; + + pkgPackageManager::SigINTStop = false; + d->progress = progress; + + // try to figure out the max environment size + int OSArgMax = sysconf(_SC_ARG_MAX); + if(OSArgMax < 0) + OSArgMax = 32*1024; + OSArgMax -= EnvironmentSize() - 2*1024; + unsigned int const MaxArgBytes = _config->FindI("Dpkg::MaxArgBytes", OSArgMax); + bool const NoTriggers = _config->FindB("DPkg::NoTriggers", true); + + if (RunScripts("DPkg::Pre-Invoke") == false) + return false; + + if (RunScriptsWithPkgs("DPkg::Pre-Install-Pkgs") == false) + return false; + + auto const noopDPkgInvocation = _config->FindB("Debug::pkgDPkgPM",false); + // store auto-bits as they are supposed to be after dpkg is run + if (noopDPkgInvocation == false) + Cache.writeStateFile(NULL); + + bool dpkg_recursive_install = _config->FindB("dpkg::install::recursive", false); + if (_config->FindB("dpkg::install::recursive::force", false) == false) + { + // dpkg uses a sorted treewalk since that version which enables the workaround to work + auto const dpkgpkg = Cache.FindPkg("dpkg"); + if (likely(dpkgpkg.end() == false && dpkgpkg->CurrentVer != 0)) + dpkg_recursive_install = Cache.VS().CmpVersion("1.18.5", dpkgpkg.CurrentVer().VerStr()) <= 0; + } + // no point in doing this dance for a handful of packages only + unsigned int const dpkg_recursive_install_min = _config->FindI("dpkg::install::recursive::minimum", 5); + // FIXME: workaround for dpkg bug, see our ./test-bug-740843-versioned-up-down-breaks test + bool const dpkg_recursive_install_numbered = _config->FindB("dpkg::install::recursive::numbered", true); + + // for the progress + BuildPackagesProgressMap(); + + APT::StateChanges approvedStates; + if (_config->FindB("dpkg::selection::remove::approved", true)) + { + for (auto && I : List) + if (I.Op == Item::Purge) + approvedStates.Purge(FindToBeRemovedVersion(I.Pkg)); + else if (I.Op == Item::Remove) + approvedStates.Remove(FindToBeRemovedVersion(I.Pkg)); + } + + // Skip removes if we install another architecture of this package soon (crossgrade) + // We can't just skip them all the time as it could be an ordering requirement [of another package] + if ((approvedStates.Remove().empty() == false || approvedStates.Purge().empty() == false) && + _config->FindB("dpkg::remove::crossgrade::implicit", true) == true) + { + std::unordered_set<decltype(pkgCache::Package::ID)> crossgraded; + std::vector<std::pair<Item*, std::string>> toCrossgrade; + auto const PlanedEnd = std::next(List.begin(), explicitIdx); + for (auto I = List.begin(); I != PlanedEnd; ++I) + { + if (I->Op != Item::Remove && I->Op != Item::Purge) + continue; + + auto const Grp = I->Pkg.Group(); + size_t installedInstances = 0, wannabeInstances = 0; + bool multiArchInstances = false; + for (auto Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg)) + { + if (Pkg->CurrentVer != 0) + { + ++installedInstances; + if (Cache[Pkg].Delete() == false) + ++wannabeInstances; + } + else if (PackageOps.find(Pkg.FullName()) != PackageOps.end()) + ++wannabeInstances; + if (multiArchInstances == false) + { + auto const V = Cache[Pkg].InstVerIter(Cache); + if (V.end() == false && (Pkg->CurrentVer == 0 || V != Pkg.CurrentVer())) + multiArchInstances = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same); + } + } + /* theoretically the installed check would be enough as some wannabe will + be first and hence be the crossgrade we were looking for, but #844300 + prevents this so we keep these situations explicit removes. + It is also the reason why neither of them can be a M-A:same package */ + if (installedInstances == 1 && wannabeInstances == 1 && multiArchInstances == false) + { + auto const FirstInstall = std::find_if_not(I, List.end(), + [](Item const &i) { return i.Op == Item::Remove || i.Op == Item::Purge; }); + auto const LastInstall = std::find_if_not(FirstInstall, List.end(), + [](Item const &i) { return i.Op == Item::Install; }); + auto const crosser = std::find_if(FirstInstall, LastInstall, + [&I](Item const &i) { return i.Pkg->Group == I->Pkg->Group; }); + if (crosser != LastInstall) + { + crossgraded.insert(I->Pkg->ID); + toCrossgrade.emplace_back(&(*I), crosser->Pkg.FullName()); + } + } + } + for (auto I = PlanedEnd; I != List.end(); ++I) + { + if (I->Op != Item::Remove && I->Op != Item::Purge) + continue; + + auto const Grp = I->Pkg.Group(); + for (auto Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg)) + { + if (Pkg == I->Pkg || Cache[Pkg].Install() == false) + continue; + toCrossgrade.emplace_back(&(*I), Pkg.FullName()); + break; + } + } + for (auto C : toCrossgrade) + { + // we never do purges on packages which are crossgraded, even if "requested" + if (C.first->Op == Item::Purge) + { + C.first->Op = Item::Remove; // crossgrades should never be purged + auto && Purges = approvedStates.Purge(); + auto const Ver = std::find_if( +#if __GNUC__ >= 5 || (__GNUC_MINOR__ >= 9 && __GNUC__ >= 4) + Purges.cbegin(), Purges.cend(), +#else + Purges.begin(), Purges.end(), +#endif + [&C](pkgCache::VerIterator const &V) { return V.ParentPkg() == C.first->Pkg; }); + approvedStates.Remove(*Ver); + Purges.erase(Ver); + auto && RemOp = PackageOps[C.first->Pkg.FullName()]; + if (RemOp.size() == 4) + { + RemOp.erase(std::next(RemOp.begin(), 2), RemOp.end()); + PackagesTotal -= 2; + } + else + _error->Warning("Unexpected amount of planned ops for package %s: %lu", C.first->Pkg.FullName().c_str(), RemOp.size()); + } + } + if (crossgraded.empty() == false) + { + auto const oldsize = List.size(); + List.erase(std::remove_if(List.begin(), PlanedEnd, + [&crossgraded](Item const &i){ + return (i.Op == Item::Remove || i.Op == Item::Purge) && + crossgraded.find(i.Pkg->ID) != crossgraded.end(); + }), PlanedEnd); + explicitIdx -= (oldsize - List.size()); + } + } + + APT::StateChanges currentStates; + if (_config->FindB("dpkg::selection::current::saveandrestore", true)) + { + for (auto Pkg = Cache.PkgBegin(); Pkg.end() == false; ++Pkg) + if (Pkg->CurrentVer == 0) + continue; + else if (Pkg->SelectedState == pkgCache::State::Purge) + currentStates.Purge(FindToBeRemovedVersion(Pkg)); + else if (Pkg->SelectedState == pkgCache::State::DeInstall) + currentStates.Remove(FindToBeRemovedVersion(Pkg)); + if (currentStates.empty() == false) + { + APT::StateChanges cleanStates; + for (auto && P: currentStates.Remove()) + cleanStates.Install(P); + for (auto && P: currentStates.Purge()) + cleanStates.Install(P); + if (cleanStates.Save(false) == false) + return _error->Error("Couldn't clean the currently selected dpkg states"); + } + } + + if (_config->FindB("dpkg::selection::remove::approved", true)) + { + if (approvedStates.Save(false) == false) + { + _error->Error("Couldn't record the approved state changes as dpkg selection states"); + if (currentStates.Save(false) == false) + _error->Error("Couldn't restore dpkg selection states which were present before this interaction!"); + return false; + } + + List.erase(std::next(List.begin(), explicitIdx), List.end()); + + std::vector<bool> toBeRemoved(Cache.Head().PackageCount, false); + for (auto && I: approvedStates.Remove()) + toBeRemoved[I.ParentPkg()->ID] = true; + for (auto && I: approvedStates.Purge()) + toBeRemoved[I.ParentPkg()->ID] = true; + + for (auto && I: List) + if (I.Op == Item::Remove || I.Op == Item::Purge) + toBeRemoved[I.Pkg->ID] = false; + + bool const RemovePending = std::find(toBeRemoved.begin(), toBeRemoved.end(), true) != toBeRemoved.end(); + bool const PurgePending = approvedStates.Purge().empty() == false; + if (RemovePending != false || PurgePending != false) + List.emplace_back(Item::ConfigurePending, pkgCache::PkgIterator()); + if (RemovePending) + List.emplace_back(Item::RemovePending, pkgCache::PkgIterator()); + if (PurgePending) + List.emplace_back(Item::PurgePending, pkgCache::PkgIterator()); + + // support subpressing of triggers processing for special + // cases like d-i that runs the triggers handling manually + if (_config->FindB("DPkg::ConfigurePending", true)) + List.emplace_back(Item::ConfigurePending, pkgCache::PkgIterator()); + } + bool const TriggersPending = _config->FindB("DPkg::TriggersPending", false); + + d->stdin_is_dev_null = false; + + // create log + OpenLog(); + + bool dpkgMultiArch = _system->MultiArchSupported(); + bool dpkgProtectedField = debSystem::AssertFeature("protected-field"); + + // start pty magic before the loop + StartPtyMagic(); + + // Tell the progress that its starting and fork dpkg + d->progress->Start(d->master); + + // this loop is runs once per dpkg operation + vector<Item>::const_iterator I = List.cbegin(); + BuildDpkgCall Args; + while (I != List.end()) + { + // Do all actions with the same Op in one run + vector<Item>::const_iterator J = I; + if (TriggersPending == true) + for (; J != List.end(); ++J) + { + if (J->Op == I->Op) + continue; + if (J->Op != Item::TriggersPending) + break; + vector<Item>::const_iterator T = J + 1; + if (T != List.end() && T->Op == I->Op) + continue; + break; + } + else if (J->Op == Item::Remove || J->Op == Item::Purge) + J = std::find_if(J, List.cend(), [](Item const &I) { return I.Op != Item::Remove && I.Op != Item::Purge; }); + else + J = std::find_if(J, List.cend(), [&J](Item const &I) { return I.Op != J->Op; }); + + Args.clearCallArguments(); + Args.reserve((J - I) + 10); + + int fd[2]; + if (pipe(fd) != 0) + return _error->Errno("pipe","Failed to create IPC pipe to dpkg"); + + Args.push_back("--status-fd"); + Args.push_back(std::to_string(fd[1])); + unsigned long const Op = I->Op; + + if (NoTriggers == true && I->Op != Item::TriggersPending && + (I->Op != Item::ConfigurePending || std::next(I) != List.end())) + Args.push_back("--no-triggers"); + + switch (I->Op) + { + case Item::Remove: + case Item::Purge: + Args.push_back("--force-depends"); + Args.push_back("--abort-after=1"); + if (std::any_of(I, J, ItemIsEssential)) + Args.push_back("--force-remove-essential"); + if (dpkgProtectedField && std::any_of(I, J, ItemIsProtected)) + Args.push_back("--force-remove-protected"); + Args.push_back("--remove"); + break; + + case Item::Configure: + Args.push_back("--configure"); + break; + + case Item::ConfigurePending: + Args.push_back("--configure"); + Args.push_back("--pending"); + break; + + case Item::TriggersPending: + Args.push_back("--triggers-only"); + Args.push_back("--pending"); + break; + + case Item::RemovePending: + Args.push_back("--remove"); + Args.push_back("--pending"); + break; + + case Item::PurgePending: + Args.push_back("--purge"); + Args.push_back("--pending"); + break; + + case Item::Install: + Args.push_back("--unpack"); + Args.push_back("--auto-deconfigure"); + break; + } + + std::unique_ptr<char, decltype(&cleanUpTmpDir)> tmpdir_for_dpkg_recursive{nullptr, &cleanUpTmpDir}; + std::string const dpkg_chroot_dir = _config->FindDir("DPkg::Chroot-Directory", "/"); + + // Write in the file or package names + if (I->Op == Item::Install) + { + auto const installsToDo = J - I; + if (dpkg_recursive_install == true && dpkg_recursive_install_min < installsToDo) + { + { + std::string basetmpdir = (dpkg_chroot_dir == "/") ? GetTempDir() : flCombine(dpkg_chroot_dir, "tmp"); + std::string tmpdir; + strprintf(tmpdir, "%s/apt-dpkg-install-XXXXXX", basetmpdir.c_str()); + tmpdir_for_dpkg_recursive.reset(strndup(tmpdir.data(), tmpdir.length())); + if (mkdtemp(tmpdir_for_dpkg_recursive.get()) == nullptr) + return _error->Errno("DPkg::Go", "mkdtemp of %s failed in preparation of calling dpkg unpack", tmpdir_for_dpkg_recursive.get()); + } + + char p = 1; + for (auto c = installsToDo - 1; (c = c/10) != 0; ++p); + for (unsigned long n = 0; I != J; ++n, ++I) + { + if (I->File[0] != '/') + return _error->Error("Internal Error, Pathname to install is not absolute '%s'",I->File.c_str()); + auto file = flNotDir(I->File); + if (flExtension(file) != "deb") + file.append(".deb"); + std::string linkpath; + if (dpkg_recursive_install_numbered) + strprintf(linkpath, "%s/%.*lu-%s", tmpdir_for_dpkg_recursive.get(), p, n, file.c_str()); + else + strprintf(linkpath, "%s/%s", tmpdir_for_dpkg_recursive.get(), file.c_str()); + std::string linktarget = I->File; + if (dpkg_chroot_dir != "/") { + char * fakechroot = getenv("FAKECHROOT"); + if (fakechroot != nullptr && strcmp(fakechroot, "true") == 0) { + // if apt is run with DPkg::Chroot-Directory under + // fakechroot, absolulte symbolic links must be prefixed + // with the chroot path to be valid inside fakechroot + strprintf(linktarget, "%s/%s", dpkg_chroot_dir.c_str(), I->File.c_str()); + } + } + if (symlink(linktarget.c_str(), linkpath.c_str()) != 0) + return _error->Errno("DPkg::Go", "Symlinking %s to %s failed!", linktarget.c_str(), linkpath.c_str()); + } + Args.push_back("--recursive"); + Args.push_back(debSystem::StripDpkgChrootDirectory(tmpdir_for_dpkg_recursive.get())); + } + else + { + for (;I != J && Args.bytes() < MaxArgBytes; ++I) + { + if (I->File[0] != '/') + return _error->Error("Internal Error, Pathname to install is not absolute '%s'",I->File.c_str()); + Args.push_back(I->File.c_str()); + } + } + } + else if (I->Op == Item::RemovePending) + { + ++I; + StripAlreadyDoneFrom(approvedStates.Remove()); + if (approvedStates.Remove().empty()) + continue; + } + else if (I->Op == Item::PurgePending) + { + ++I; + // explicit removes of packages without conffiles passthrough the purge states instantly, too. + // Setting these non-installed packages up for purging generates 'unknown pkg' warnings from dpkg + StripAlreadyDoneFrom(approvedStates.Purge()); + if (approvedStates.Purge().empty()) + continue; + std::remove_reference<decltype(approvedStates.Remove())>::type approvedRemoves; + std::swap(approvedRemoves, approvedStates.Remove()); + // we apply it again here as an explicit remove in the ordering will have cleared the purge state + if (approvedStates.Save(false) == false) + { + _error->Error("Couldn't record the approved purges as dpkg selection states"); + if (currentStates.Save(false) == false) + _error->Error("Couldn't restore dpkg selection states which were present before this interaction!"); + return false; + } + std::swap(approvedRemoves, approvedStates.Remove()); + } + else + { + string const nativeArch = _config->Find("APT::Architecture"); + auto const oldSize = I->Pkg.end() ? 0ull : Args.bytes(); + for (;I != J && Args.bytes() < MaxArgBytes; ++I) + { + if((*I).Pkg.end() == true) + continue; + if (I->Op == Item::Configure && disappearedPkgs.find(I->Pkg.FullName(true)) != disappearedPkgs.end()) + continue; + // We keep this here to allow "smooth" transitions from e.g. multiarch dpkg/ubuntu to dpkg/debian + if (dpkgMultiArch == false && (I->Pkg.Arch() == nativeArch || + strcmp(I->Pkg.Arch(), "all") == 0 || + strcmp(I->Pkg.Arch(), "none") == 0)) + Args.push_back(I->Pkg.Name()); + else if (Op == Item::Purge && I->Pkg->CurrentVer == 0) + continue; // we purge later with --purge --pending, so if it isn't installed (aka rc-only), skip it here + else if (strcmp(I->Pkg.Arch(), "none") == 0) + Args.push_back(I->Pkg.Name()); // never arch-qualify a package without an arch + else + { + pkgCache::VerIterator PkgVer; + if (Op == Item::Remove || Op == Item::Purge) + PkgVer = I->Pkg.CurrentVer(); + else + PkgVer = Cache[I->Pkg].InstVerIter(Cache); + if (PkgVer.end()) + { + _error->Warning("Can not find PkgVer for '%s'", I->Pkg.Name()); + Args.push_back(I->Pkg.Name()); + continue; + } + Args.push_back(std::string(I->Pkg.Name()) + ":" + PkgVer.Arch()); + } + } + // skip configure action if all scheduled packages disappeared + if (oldSize == Args.bytes()) + continue; + } + + J = I; + + if (noopDPkgInvocation == true) + { + for (auto const a : Args) + clog << a << ' '; + clog << endl; + close(fd[0]); + close(fd[1]); + continue; + } + cout << flush; + clog << flush; + cerr << flush; + + /* Mask off sig int/quit. We do this because dpkg also does when + it forks scripts. What happens is that when you hit ctrl-c it sends + it to all processes in the group. Since dpkg ignores the signal + it doesn't die but we do! So we must also ignore it */ + sighandler_t old_SIGQUIT = signal(SIGQUIT,SIG_IGN); + sighandler_t old_SIGINT = signal(SIGINT,SigINT); + + // Check here for any SIGINT + if (pkgPackageManager::SigINTStop && (Op == Item::Remove || Op == Item::Purge || Op == Item::Install)) + break; + + // ignore SIGHUP as well (debian #463030) + sighandler_t old_SIGHUP = signal(SIGHUP,SIG_IGN); + + // now run dpkg + d->progress->StartDpkg(); + std::set<int> KeepFDs; + KeepFDs.insert(fd[1]); + MergeKeepFdsFromConfiguration(KeepFDs); + pid_t Child = ExecFork(KeepFDs); + if (Child == 0) + { + // This is the child + SetupSlavePtyMagic(); + close(fd[0]); // close the read end of the pipe + + debSystem::DpkgChrootDirectory(); + + if (chdir(_config->FindDir("DPkg::Run-Directory","/").c_str()) != 0) + _exit(100); + + if (_config->FindB("DPkg::FlushSTDIN",true) == true && isatty(STDIN_FILENO)) + { + int Flags; + int dummy = 0; + if ((Flags = fcntl(STDIN_FILENO,F_GETFL,dummy)) < 0) + _exit(100); + + // Discard everything in stdin before forking dpkg + if (fcntl(STDIN_FILENO,F_SETFL,Flags | O_NONBLOCK) < 0) + _exit(100); + + while (read(STDIN_FILENO,&dummy,1) == 1); + + if (fcntl(STDIN_FILENO,F_SETFL,Flags & (~(long)O_NONBLOCK)) < 0) + _exit(100); + } + + // if color support isn't enabled/disabled explicitly tell + // dpkg to use the same state apt is using for its color support + if (_config->FindB("APT::Color", false) == true) + setenv("DPKG_COLORS", "always", 0); + else + setenv("DPKG_COLORS", "never", 0); + + if (_system->IsLocked() == true) { + setenv("DPKG_FRONTEND_LOCKED", "true", 1); + } + if (_config->Find("DPkg::Path", "").empty() == false) + setenv("PATH", _config->Find("DPkg::Path", "").c_str(), 1); + + Args.execute("Could not exec dpkg!"); + } + + // we read from dpkg here + int const _dpkgin = fd[0]; + close(fd[1]); // close the write end of the pipe + d->status_fd_reached_end_of_file = false; + + // apply ionice + if (_config->FindB("DPkg::UseIoNice", false) == true) + ionice(Child); + + // setups fds + sigemptyset(&d->sigmask); + sigprocmask(SIG_BLOCK,&d->sigmask,&d->original_sigmask); + + // the result of the waitpid call + int Status = 0; + int res; + bool waitpid_failure = false; + bool dpkg_finished = false; + do + { + if (dpkg_finished == false) + { + if ((res = waitpid(Child, &Status, WNOHANG)) == Child) + dpkg_finished = true; + else if (res < 0) + { + // error handling, waitpid returned -1 + if (errno == EINTR) + continue; + waitpid_failure = true; + break; + } + } + if (dpkg_finished && d->status_fd_reached_end_of_file) + break; + + // wait for input or output here + fd_set rfds; + FD_ZERO(&rfds); + if (d->master >= 0 && d->direct_stdin == false && d->stdin_is_dev_null == false) + FD_SET(STDIN_FILENO, &rfds); + FD_SET(_dpkgin, &rfds); + if(d->master >= 0) + FD_SET(d->master, &rfds); + struct timespec tv; + tv.tv_sec = 0; + tv.tv_nsec = d->progress->GetPulseInterval(); + auto const select_ret = pselect(max(d->master, _dpkgin)+1, &rfds, NULL, NULL, + &tv, &d->original_sigmask); + d->progress->Pulse(); + if (select_ret == 0) + continue; + else if (select_ret < 0 && errno == EINTR) + continue; + else if (select_ret < 0) + { + perror("select() returned error"); + continue; + } + + if(d->master >= 0 && FD_ISSET(d->master, &rfds)) + DoTerminalPty(d->master); + if(d->master >= 0 && FD_ISSET(0, &rfds)) + DoStdin(d->master); + if(FD_ISSET(_dpkgin, &rfds)) + DoDpkgStatusFd(_dpkgin); + + } while (true); + close(_dpkgin); + + // Restore sig int/quit + signal(SIGQUIT,old_SIGQUIT); + signal(SIGINT,old_SIGINT); + signal(SIGHUP,old_SIGHUP); + + if (waitpid_failure == true) + { + strprintf(d->dpkg_error, "Sub-process %s couldn't be waited for.",Args.front()); + _error->Error("%s", d->dpkg_error.c_str()); + break; + } + + // Check for an error code. + if (WIFEXITED(Status) == 0 || WEXITSTATUS(Status) != 0) + { + // if it was set to "keep-dpkg-running" then we won't return + // here but keep the loop going and just report it as a error + // for later + bool const stopOnError = _config->FindB("Dpkg::StopOnError",true); + + if (WIFSIGNALED(Status) != 0 && WTERMSIG(Status) == SIGSEGV) + strprintf(d->dpkg_error, "Sub-process %s received a segmentation fault.",Args.front()); + else if (WIFEXITED(Status) != 0) + strprintf(d->dpkg_error, "Sub-process %s returned an error code (%u)",Args.front(),WEXITSTATUS(Status)); + else + strprintf(d->dpkg_error, "Sub-process %s exited unexpectedly",Args.front()); + _error->Error("%s", d->dpkg_error.c_str()); + + if(stopOnError) + break; + } + } + // dpkg is done at this point + StopPtyMagic(); + CloseLog(); + + if (d->dpkg_error.empty() == false) + { + // no point in resetting packages we already completed removal for + StripAlreadyDoneFrom(approvedStates.Remove()); + StripAlreadyDoneFrom(approvedStates.Purge()); + APT::StateChanges undo; + auto && undoRem = approvedStates.Remove(); + std::move(undoRem.begin(), undoRem.end(), std::back_inserter(undo.Install())); + auto && undoPur = approvedStates.Purge(); + std::move(undoPur.begin(), undoPur.end(), std::back_inserter(undo.Install())); + approvedStates.clear(); + if (undo.Save(false) == false) + _error->Error("Couldn't revert dpkg selection for approved remove/purge after an error was encountered!"); + } + + StripAlreadyDoneFrom(currentStates.Remove()); + StripAlreadyDoneFrom(currentStates.Purge()); + if (currentStates.Save(false) == false) + _error->Error("Couldn't restore dpkg selection states which were present before this interaction!"); + + if (pkgPackageManager::SigINTStop) + _error->Warning(_("Operation was interrupted before it could finish")); + + if (noopDPkgInvocation == false) + { + if (d->dpkg_error.empty() && (PackagesDone + 1) != PackagesTotal) + { + std::string pkglist; + for (auto const &PO: PackageOps) + if (PO.second.size() != PackageOpsDone[PO.first]) + { + if (pkglist.empty() == false) + pkglist.append(" "); + pkglist.append(PO.first); + } + /* who cares about correct progress? As we depend on it for skipping actions + our parsing should be correct. People will no doubt be confused if they see + this message, but the dpkg warning about unknown packages isn't much better + from a user POV and combined we might have a chance to figure out what is wrong */ + _error->Warning("APT had planned for dpkg to do more than it reported back (%u vs %u).\n" + "Affected packages: %s", PackagesDone, PackagesTotal, pkglist.c_str()); + } + + std::string const oldpkgcache = _config->FindFile("Dir::cache::pkgcache"); + if (oldpkgcache.empty() == false && RealFileExists(oldpkgcache) == true && + RemoveFile("pkgDPkgPM::Go", oldpkgcache)) + { + std::string const srcpkgcache = _config->FindFile("Dir::cache::srcpkgcache"); + if (srcpkgcache.empty() == false && RealFileExists(srcpkgcache) == true) + { + _error->PushToStack(); + pkgCacheFile CacheFile; + CacheFile.BuildCaches(NULL, true); + _error->RevertToStack(); + } + } + } + + // disappearing packages can forward their auto-bit + if (disappearedPkgs.empty() == false) + Cache.writeStateFile(NULL); + + d->progress->Stop(); + + if (RunScripts("DPkg::Post-Invoke") == false) + return false; + + return d->dpkg_error.empty(); +} + +void SigINT(int /*sig*/) { + pkgPackageManager::SigINTStop = true; +} + /*}}}*/ +// pkgDpkgPM::Reset - Dump the contents of the command list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgDPkgPM::Reset() +{ + List.erase(List.begin(),List.end()); +} + /*}}}*/ +// pkgDpkgPM::WriteApportReport - write out error report pkg failure /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) +{ + // If apport doesn't exist or isn't installed do nothing + // This e.g. prevents messages in 'universes' without apport + pkgCache::PkgIterator apportPkg = Cache.FindPkg("apport"); + if (apportPkg.end() == true || apportPkg->CurrentVer == 0) + return; + + string pkgname, reportfile, pkgver, arch; + string::size_type pos; + FILE *report; + + if (_config->FindB("Dpkg::ApportFailureReport", true) == false) + { + std::clog << "configured to not write apport reports" << std::endl; + return; + } + + // only report the first errors + if(pkgFailures > _config->FindI("APT::Apport::MaxReports", 3)) + { + std::clog << _("No apport report written because MaxReports is reached already") << std::endl; + return; + } + + // check if its not a follow up error + const char *needle = dgettext("dpkg", "dependency problems - leaving unconfigured"); + if(strstr(errormsg, needle) != NULL) { + std::clog << _("No apport report written because the error message indicates its a followup error from a previous failure.") << std::endl; + return; + } + + // do not report disk-full failures + if(strstr(errormsg, strerror(ENOSPC)) != NULL) { + std::clog << _("No apport report written because the error message indicates a disk full error") << std::endl; + return; + } + + // do not report out-of-memory failures + if(strstr(errormsg, strerror(ENOMEM)) != NULL || + strstr(errormsg, "failed to allocate memory") != NULL) { + std::clog << _("No apport report written because the error message indicates a out of memory error") << std::endl; + return; + } + + // do not report bugs regarding inaccessible local files + if(strstr(errormsg, strerror(ENOENT)) != NULL || + strstr(errormsg, "cannot access archive") != NULL) { + std::clog << _("No apport report written because the error message indicates an issue on the local system") << std::endl; + return; + } + + // do not report errors encountered when decompressing packages + if(strstr(errormsg, "--fsys-tarfile returned error exit status 2") != NULL) { + std::clog << _("No apport report written because the error message indicates an issue on the local system") << std::endl; + return; + } + + // do not report dpkg I/O errors, this is a format string, so we compare + // the prefix and the suffix of the error with the dpkg error message + vector<string> io_errors; + io_errors.push_back(string("failed to read")); + io_errors.push_back(string("failed to write")); + io_errors.push_back(string("failed to seek")); + io_errors.push_back(string("unexpected end of file or stream")); + + for (vector<string>::iterator I = io_errors.begin(); I != io_errors.end(); ++I) + { + vector<string> list = VectorizeString(dgettext("dpkg", (*I).c_str()), '%'); + if (list.size() > 1) { + // we need to split %s, VectorizeString only allows char so we need + // to kill the "s" manually + if (list[1].size() > 1) { + list[1].erase(0, 1); + if(strstr(errormsg, list[0].c_str()) && + strstr(errormsg, list[1].c_str())) { + std::clog << _("No apport report written because the error message indicates a dpkg I/O error") << std::endl; + return; + } + } + } + } + + // get the pkgname and reportfile + pkgname = flNotDir(pkgpath); + pos = pkgname.find('_'); + if(pos != string::npos) + pkgname = pkgname.substr(0, pos); + + // find the package version and source package name + pkgCache::PkgIterator Pkg = Cache.FindPkg(pkgname); + if (Pkg.end() == true) + { + if (pos == std::string::npos || _config->FindB("dpkg::install::recursive::numbered", true) == false) + return; + auto const dash = pkgname.find_first_not_of("0123456789"); + if (dash == std::string::npos || pkgname[dash] != '-') + return; + pkgname.erase(0, dash + 1); + Pkg = Cache.FindPkg(pkgname); + if (Pkg.end() == true) + return; + } + pkgCache::VerIterator Ver = Cache.GetCandidateVersion(Pkg); + if (Ver.end() == true) + return; + pkgver = Ver.VerStr() == NULL ? "unknown" : Ver.VerStr(); + + // if the file exists already, we check: + // - if it was reported already (touched by apport). + // If not, we do nothing, otherwise + // we overwrite it. This is the same behaviour as apport + // - if we have a report with the same pkgversion already + // then we skip it + _config->CndSet("Dir::Apport", "var/crash"); + reportfile = flCombine(_config->FindDir("Dir::Apport", "var/crash"), pkgname+".0.crash"); + if(FileExists(reportfile)) + { + struct stat buf; + char strbuf[255]; + + // check atime/mtime + stat(reportfile.c_str(), &buf); + if(buf.st_mtime > buf.st_atime) + return; + + // check if the existing report is the same version + report = fopen(reportfile.c_str(),"r"); + while(fgets(strbuf, sizeof(strbuf), report) != NULL) + { + if(strstr(strbuf,"Package:") == strbuf) + { + char pkgname[255], version[255]; + if(sscanf(strbuf, "Package: %254s %254s", pkgname, version) == 2) + if(strcmp(pkgver.c_str(), version) == 0) + { + fclose(report); + return; + } + } + } + fclose(report); + } + + // now write the report + arch = _config->Find("APT::Architecture"); + report = fopen(reportfile.c_str(),"w"); + if(report == NULL) + return; + if(_config->FindB("DPkgPM::InitialReportOnly",false) == true) + chmod(reportfile.c_str(), 0); + else + chmod(reportfile.c_str(), 0600); + fprintf(report, "ProblemType: Package\n"); + fprintf(report, "Architecture: %s\n", arch.c_str()); + time_t now = time(NULL); + char ctime_buf[26]; // need at least 26 bytes according to ctime(3) + fprintf(report, "Date: %s" , ctime_r(&now, ctime_buf)); + fprintf(report, "Package: %s %s\n", pkgname.c_str(), pkgver.c_str()); + fprintf(report, "SourcePackage: %s\n", Ver.SourcePkgName()); + fprintf(report, "ErrorMessage:\n %s\n", errormsg); + + // ensure that the log is flushed + if(d->term_out) + fflush(d->term_out); + + // attach terminal log it if we have it + string logfile_name = _config->FindFile("Dir::Log::Terminal", "/dev/null"); + if (logfile_name != "/dev/null") + { + FILE *log = NULL; + + fprintf(report, "DpkgTerminalLog:\n"); + log = fopen(logfile_name.c_str(),"r"); + if(log != NULL) + { + char buf[1024]; + while( fgets(buf, sizeof(buf), log) != NULL) + fprintf(report, " %s", buf); + fprintf(report, " \n"); + fclose(log); + } + } + + // attach history log it if we have it + string histfile_name = _config->FindFile("Dir::Log::History", "/dev/null"); + if (histfile_name != "/dev/null") + { + fprintf(report, "DpkgHistoryLog:\n"); + FILE* log = fopen(histfile_name.c_str(),"r"); + if(log != NULL) + { + char buf[1024]; + while( fgets(buf, sizeof(buf), log) != NULL) + fprintf(report, " %s", buf); + fclose(log); + } + } + + // log the ordering, see dpkgpm.h and the "Ops" enum there + fprintf(report, "AptOrdering:\n"); + for (auto && I : List) + { + char const * opstr = nullptr; + switch (I.Op) + { + case Item::Install: opstr = "Install"; break; + case Item::Configure: opstr = "Configure"; break; + case Item::Remove: opstr = "Remove"; break; + case Item::Purge: opstr = "Purge"; break; + case Item::ConfigurePending: opstr = "ConfigurePending"; break; + case Item::TriggersPending: opstr = "TriggersPending"; break; + case Item::RemovePending: opstr = "RemovePending"; break; + case Item::PurgePending: opstr = "PurgePending"; break; + } + auto const pkgname = I.Pkg.end() ? "NULL" : I.Pkg.FullName(); + fprintf(report, " %s: %s\n", pkgname.c_str(), opstr); + } + + // attach dmesg log (to learn about segfaults) + if (FileExists("/bin/dmesg")) + { + fprintf(report, "Dmesg:\n"); + FILE *log = popen("/bin/dmesg","r"); + if(log != NULL) + { + char buf[1024]; + while( fgets(buf, sizeof(buf), log) != NULL) + fprintf(report, " %s", buf); + pclose(log); + } + } + + // attach df -l log (to learn about filesystem status) + if (FileExists("/bin/df")) + { + + fprintf(report, "Df:\n"); + FILE *log = popen("/bin/df -l -x squashfs","r"); + if(log != NULL) + { + char buf[1024]; + while( fgets(buf, sizeof(buf), log) != NULL) + fprintf(report, " %s", buf); + pclose(log); + } + } + + fclose(report); + +} + /*}}}*/ diff --git a/apt-pkg/deb/dpkgpm.h b/apt-pkg/deb/dpkgpm.h new file mode 100644 index 0000000..caf7d35 --- /dev/null +++ b/apt-pkg/deb/dpkgpm.h @@ -0,0 +1,135 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + DPKG Package Manager - Provide an interface to dpkg + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_DPKGPM_H +#define PKGLIB_DPKGPM_H + +#include <apt-pkg/macros.h> +#include <apt-pkg/packagemanager.h> +#include <apt-pkg/pkgcache.h> + +#include <map> +#include <string> +#include <vector> +#include <stdio.h> + + +class pkgDepCache; +namespace APT { namespace Progress { class PackageManager; } } + + +class pkgDPkgPMPrivate; + + +class APT_PUBLIC pkgDPkgPM : public pkgPackageManager +{ + private: + pkgDPkgPMPrivate * const d; + + /** \brief record the disappear action and handle accordingly + + dpkg let packages disappear then they have no files any longer and + nothing depends on them. We need to collect this as dpkg as well as + APT doesn't know beforehand that the package will disappear, so the + only possible option is to tell the user afterwards about it. + To enhance the experience we also try to forward the auto-install + flag so the disappear-causer(s) are not autoremoved next time - + for the transfer to happen the disappeared version needs to depend + on the package the flag should be forwarded to and this package + needs to declare a Replaces on the disappeared package. + \param pkgname Name of the package that disappeared + */ + APT_HIDDEN void handleDisappearAction(std::string const &pkgname); + APT_HIDDEN void handleCrossUpgradeAction(std::string const &pkgname); + + protected: + int pkgFailures; + + // progress reporting + struct DpkgState + { + const char *state; // the dpkg state (e.g. "unpack") + const char *str; // the human readable translation of the state + }; + + // the dpkg states that the pkg will run through, the string is + // the package, the vector contains the dpkg states that the package + // will go through + std::map<std::string,std::vector<struct DpkgState> > PackageOps; + // the dpkg states that are already done; the string is the package + // the int is the state that is already done (e.g. a package that is + // going to be install is already in state "half-installed") + std::map<std::string,unsigned int> PackageOpsDone; + + // progress reporting + unsigned int PackagesDone; + unsigned int PackagesTotal; + + public: + struct Item + { + enum Ops {Install, Configure, Remove, Purge, ConfigurePending, TriggersPending, + RemovePending, PurgePending } Op; + std::string File; + PkgIterator Pkg; + Item(Ops Op,PkgIterator Pkg,std::string File = "") : Op(Op), + File(File), Pkg(Pkg) {}; + Item() {}; + }; + protected: + std::vector<Item> List; + + // Helpers + bool RunScriptsWithPkgs(const char *Cnf); + bool SendPkgsInfo(FILE * const F, unsigned int const &Version); + void WriteHistoryTag(std::string const &tag, std::string value); + std::string ExpandShortPackageName(pkgDepCache &Cache, + const std::string &short_pkgname); + + // Terminal progress + void SendTerminalProgress(float percentage); + + // apport integration + void WriteApportReport(const char *pkgpath, const char *errormsg); + + // dpkg log + bool OpenLog(); + bool CloseLog(); + + // helper + void BuildPackagesProgressMap(); + void StartPtyMagic(); + void SetupSlavePtyMagic(); + void StopPtyMagic(); + + // input processing + void DoStdin(int master); + void DoTerminalPty(int master); + void DoDpkgStatusFd(int statusfd); + void ProcessDpkgStatusLine(char *line); + + // The Actual installation implementation + virtual bool Install(PkgIterator Pkg,std::string File) APT_OVERRIDE; + virtual bool Configure(PkgIterator Pkg) APT_OVERRIDE; + virtual bool Remove(PkgIterator Pkg,bool Purge = false) APT_OVERRIDE; + + virtual bool Go(APT::Progress::PackageManager *progress) APT_OVERRIDE; + + virtual void Reset() APT_OVERRIDE; + + public: + + explicit pkgDPkgPM(pkgDepCache *Cache); + virtual ~pkgDPkgPM(); + + APT_HIDDEN static bool ExpandPendingCalls(std::vector<Item> &List, pkgDepCache &Cache); +}; + +void SigINT(int sig); + +#endif diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc new file mode 100644 index 0000000..322bf1b --- /dev/null +++ b/apt-pkg/depcache.cc @@ -0,0 +1,2564 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Dependency Cache - Caches Dependency information. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/algorithms.h> +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/cachefile.h> +#include <apt-pkg/cachefilter.h> +#include <apt-pkg/cacheset.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/depcache.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/prettyprinters.h> +#include <apt-pkg/progress.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile-keys.h> +#include <apt-pkg/tagfile.h> +#include <apt-pkg/version.h> +#include <apt-pkg/versionmatch.h> + +#include <algorithm> +#include <iostream> +#include <memory> +#include <sstream> +#include <iterator> +#include <list> +#include <set> +#include <string> +#include <unordered_map> +#include <utility> +#include <vector> +#include <stdio.h> +#include <string.h> + +#include <sys/stat.h> + +#include <apti18n.h> + /*}}}*/ + +using std::string; + +// helper for kernel autoremoval /*{{{*/ + +/** \brief Returns \b true for packages matching a regular + * expression in APT::NeverAutoRemove. + */ +class DefaultRootSetFunc2 : public pkgDepCache::DefaultRootSetFunc +{ + std::unique_ptr<APT::CacheFilter::Matcher> Kernels; + + public: + DefaultRootSetFunc2(pkgCache *cache) : Kernels(APT::KernelAutoRemoveHelper::GetProtectedKernelsFilter(cache)){}; + virtual ~DefaultRootSetFunc2(){}; + + bool InRootSet(const pkgCache::PkgIterator &pkg) APT_OVERRIDE { return pkg.end() == false && ((*Kernels)(pkg) || DefaultRootSetFunc::InRootSet(pkg)); }; +}; + + /*}}}*/ +// helper for Install-Recommends-Sections and Never-MarkAuto-Sections /*{{{*/ +// FIXME: Has verbatim copy in cmdline/apt-mark.cc +static bool ConfigValueInSubTree(const char* SubTree, std::string_view const needle) +{ + if (needle.empty()) + return false; + Configuration::Item const *Opts = _config->Tree(SubTree); + if (Opts != nullptr && Opts->Child != nullptr) + { + Opts = Opts->Child; + for (; Opts != nullptr; Opts = Opts->Next) + { + if (Opts->Value.empty()) + continue; + if (needle == Opts->Value) + return true; + } + } + return false; +} +static bool SectionInSubTree(char const * const SubTree, std::string_view Needle) +{ + if (ConfigValueInSubTree(SubTree, Needle)) + return true; + auto const sub = Needle.rfind('/'); + if (sub == std::string_view::npos) + { + std::string special{"/"}; + special.append(Needle); + return ConfigValueInSubTree(SubTree, special); + } + return ConfigValueInSubTree(SubTree, Needle.substr(sub + 1)); +} + /*}}}*/ +pkgDepCache::ActionGroup::ActionGroup(pkgDepCache &cache) : /*{{{*/ + d(NULL), cache(cache), released(false) +{ + cache.IncreaseActionGroupLevel(); +} + +void pkgDepCache::ActionGroup::release() +{ + if(released) + return; + released = true; + if (cache.DecreaseActionGroupLevel() == 0) + cache.MarkAndSweep(); +} + +pkgDepCache::ActionGroup::~ActionGroup() +{ + release(); +} +int pkgDepCache::IncreaseActionGroupLevel() +{ + return ++group_level; +} +int pkgDepCache::DecreaseActionGroupLevel() +{ + if(group_level == 0) + { + std::cerr << "W: Unbalanced action groups, expect badness\n"; + return -1; + } + return --group_level; +} + /*}}}*/ +// DepCache::pkgDepCache - Constructors /*{{{*/ +// --------------------------------------------------------------------- +/* */ + +struct pkgDepCache::Private +{ + std::unique_ptr<InRootSetFunc> inRootSetFunc; + std::unique_ptr<APT::CacheFilter::Matcher> IsAVersionedKernelPackage, IsProtectedKernelPackage; +}; +pkgDepCache::pkgDepCache(pkgCache *const pCache, Policy *const Plcy) : group_level(0), Cache(pCache), PkgState(0), DepState(0), + iUsrSize(0), iDownloadSize(0), iInstCount(0), iDelCount(0), iKeepCount(0), + iBrokenCount(0), iPolicyBrokenCount(0), iBadCount(0), d(new Private) +{ + DebugMarker = _config->FindB("Debug::pkgDepCache::Marker", false); + DebugAutoInstall = _config->FindB("Debug::pkgDepCache::AutoInstall", false); + delLocalPolicy = 0; + LocalPolicy = Plcy; + if (LocalPolicy == 0) + delLocalPolicy = LocalPolicy = new Policy; +} + /*}}}*/ +// DepCache::~pkgDepCache - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgDepCache::~pkgDepCache() +{ + delete [] PkgState; + delete [] DepState; + delete delLocalPolicy; + delete d; +} + /*}}}*/ +bool pkgDepCache::CheckConsistency(char const *const msgtag) /*{{{*/ +{ + auto const OrigPkgState = PkgState; + auto const OrigDepState = DepState; + + PkgState = new StateCache[Head().PackageCount]; + DepState = new unsigned char[Head().DependsCount]; + memset(PkgState,0,sizeof(*PkgState)*Head().PackageCount); + memset(DepState,0,sizeof(*DepState)*Head().DependsCount); + + auto const origUsrSize = iUsrSize; + auto const origDownloadSize = iDownloadSize; + auto const origInstCount = iInstCount; + auto const origDelCount = iDelCount; + auto const origKeepCount = iKeepCount; + auto const origBrokenCount = iBrokenCount; + auto const origPolicyBrokenCount = iPolicyBrokenCount; + auto const origBadCount = iBadCount; + + for (PkgIterator I = PkgBegin(); not I.end(); ++I) + { + auto &State = PkgState[I->ID]; + auto const &OrigState = OrigPkgState[I->ID]; + State.iFlags = OrigState.iFlags; + + State.CandidateVer = OrigState.CandidateVer; + State.InstallVer = OrigState.InstallVer; + State.Mode = OrigState.Mode; + State.Update(I,*this); + State.Status = OrigState.Status; + } + PerformDependencyPass(nullptr); + + _error->PushToStack(); +#define APT_CONSISTENCY_CHECK(VAR,STR) \ + if (orig##VAR != i##VAR) \ + _error->Warning("Internal Inconsistency in pkgDepCache: " #VAR " " STR " vs " STR " (%s)", i##VAR, orig##VAR, msgtag) + APT_CONSISTENCY_CHECK(UsrSize, "%lld"); + APT_CONSISTENCY_CHECK(DownloadSize, "%lld"); + APT_CONSISTENCY_CHECK(InstCount, "%lu"); + APT_CONSISTENCY_CHECK(DelCount, "%lu"); + APT_CONSISTENCY_CHECK(KeepCount, "%lu"); + APT_CONSISTENCY_CHECK(BrokenCount, "%lu"); + APT_CONSISTENCY_CHECK(PolicyBrokenCount, "%lu"); + APT_CONSISTENCY_CHECK(BadCount, "%lu"); +#undef APT_CONSISTENCY_CHECK + + for (PkgIterator P = PkgBegin(); not P.end(); ++P) + { + auto const &State = PkgState[P->ID]; + auto const &OrigState = OrigPkgState[P->ID]; + if (State.Status != OrigState.Status) + _error->Warning("Internal Inconsistency in pkgDepCache: Status of %s is %d vs %d (%s)", P.FullName().c_str(), State.Status, OrigState.Status, msgtag); + if (State.NowBroken() != OrigState.NowBroken()) + _error->Warning("Internal Inconsistency in pkgDepCache: Now broken for %s is %d vs %d (%s)", P.FullName().c_str(), static_cast<int>(State.DepState), static_cast<int>(OrigState.DepState), msgtag); + if (State.NowPolicyBroken() != OrigState.NowPolicyBroken()) + _error->Warning("Internal Inconsistency in pkgDepCache: Now policy broken for %s is %d vs %d (%s)", P.FullName().c_str(), static_cast<int>(State.DepState), static_cast<int>(OrigState.DepState), msgtag); + if (State.InstBroken() != OrigState.InstBroken()) + _error->Warning("Internal Inconsistency in pkgDepCache: Install broken for %s is %d vs %d (%s)", P.FullName().c_str(), static_cast<int>(State.DepState), static_cast<int>(OrigState.DepState), msgtag); + if (State.InstPolicyBroken() != OrigState.InstPolicyBroken()) + _error->Warning("Internal Inconsistency in pkgDepCache: Install broken for %s is %d vs %d (%s)", P.FullName().c_str(), static_cast<int>(State.DepState), static_cast<int>(OrigState.DepState), msgtag); + } + + auto inconsistent = _error->PendingError(); + _error->MergeWithStack(); + + delete[] PkgState; + delete[] DepState; + PkgState = OrigPkgState; + DepState = OrigDepState; + iUsrSize = origUsrSize; + iDownloadSize = origDownloadSize; + iInstCount = origInstCount; + iDelCount = origDelCount; + iKeepCount = origKeepCount; + iBrokenCount = origBrokenCount; + iPolicyBrokenCount = origPolicyBrokenCount; + iBadCount = origBadCount; + + return not inconsistent; +} + /*}}}*/ +// DepCache::Init - Generate the initial extra structures. /*{{{*/ +// --------------------------------------------------------------------- +/* This allocats the extension buffers and initializes them. */ +bool pkgDepCache::Init(OpProgress * const Prog) +{ + // Suppress mark updates during this operation (just in case) and + // run a mark operation when Init terminates. + ActionGroup actions(*this); + + delete [] PkgState; + delete [] DepState; + PkgState = new StateCache[Head().PackageCount]; + DepState = new unsigned char[Head().DependsCount]; + memset(PkgState,0,sizeof(*PkgState)*Head().PackageCount); + memset(DepState,0,sizeof(*DepState)*Head().DependsCount); + + if (Prog != 0) + { + Prog->OverallProgress(0,2*Head().PackageCount,Head().PackageCount, + _("Building dependency tree")); + Prog->SubProgress(Head().PackageCount,_("Candidate versions")); + } + + /* Set the current state of everything. In this state all of the + packages are kept exactly as is. See AllUpgrade */ + int Done = 0; + for (PkgIterator I = PkgBegin(); I.end() != true; ++I, ++Done) + { + if (Prog != 0 && Done%20 == 0) + Prog->Progress(Done); + + // Find the proper cache slot + StateCache &State = PkgState[I->ID]; + State.iFlags = 0; + + // Figure out the install version + State.CandidateVer = LocalPolicy->GetCandidateVer(I); + State.InstallVer = I.CurrentVer(); + State.Mode = ModeKeep; + + State.Update(I,*this); + } + + if (Prog != 0) + { + Prog->OverallProgress(Head().PackageCount,2*Head().PackageCount, + Head().PackageCount, + _("Building dependency tree")); + Prog->SubProgress(Head().PackageCount,_("Dependency generation")); + } + + Update(Prog); + + if(Prog != 0) + Prog->Done(); + + return true; +} + /*}}}*/ +bool pkgDepCache::readStateFile(OpProgress * const Prog) /*{{{*/ +{ + FileFd state_file; + string const state = _config->FindFile("Dir::State::extended_states"); + if(RealFileExists(state)) { + state_file.Open(state, FileFd::ReadOnly, FileFd::Extension); + off_t const file_size = state_file.Size(); + if(Prog != NULL) + { + Prog->Done(); + Prog->OverallProgress(0, file_size, 1, + _("Reading state information")); + } + + pkgTagFile tagfile(&state_file); + pkgTagSection section; + off_t amt = 0; + bool const debug_autoremove = _config->FindB("Debug::pkgAutoRemove",false); + while(tagfile.Step(section)) { + auto const pkgname = section.Find(pkgTagSection::Key::Package); + auto pkgarch = section.Find(pkgTagSection::Key::Architecture); + if (pkgarch.empty() == true) + pkgarch = "any"; + pkgCache::PkgIterator pkg = Cache->FindPkg(pkgname, pkgarch); + // Silently ignore unknown packages and packages with no actual version. + if(pkg.end() == true || pkg->VersionList == 0) + continue; + + short const reason = section.FindI("Auto-Installed", 0); + if(reason > 0) + { + PkgState[pkg->ID].Flags |= Flag::Auto; + if (unlikely(debug_autoremove)) + std::clog << "Auto-Installed : " << pkg.FullName() << std::endl; + if (pkgarch == "any") + { + pkgCache::GrpIterator G = pkg.Group(); + for (pkg = G.NextPkg(pkg); pkg.end() != true; pkg = G.NextPkg(pkg)) + if (pkg->VersionList != 0) + PkgState[pkg->ID].Flags |= Flag::Auto; + } + } + amt += section.size(); + if(Prog != NULL) + Prog->OverallProgress(amt, file_size, 1, + _("Reading state information")); + } + if(Prog != NULL) + Prog->OverallProgress(file_size, file_size, 1, + _("Reading state information")); + } + + return true; +} + /*}}}*/ +bool pkgDepCache::writeStateFile(OpProgress * const /*prog*/, bool const InstalledOnly) /*{{{*/ +{ + bool const debug_autoremove = _config->FindB("Debug::pkgAutoRemove",false); + + if(debug_autoremove) + std::clog << "pkgDepCache::writeStateFile()" << std::endl; + + FileFd StateFile; + string const state = _config->FindFile("Dir::State::extended_states"); + if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::State"), flNotFile(state)) == false) + return false; + + // if it does not exist, create a empty one + if(!RealFileExists(state)) + { + StateFile.Open(state, FileFd::WriteAtomic, FileFd::Extension); + StateFile.Close(); + } + + // open it + if (!StateFile.Open(state, FileFd::ReadOnly, FileFd::Extension)) + return _error->Error(_("Failed to open StateFile %s"), + state.c_str()); + + FileFd OutFile(state, FileFd::ReadWrite | FileFd::Atomic, FileFd::Extension); + if (OutFile.IsOpen() == false || OutFile.Failed() == true) + return _error->Error(_("Failed to write temporary StateFile %s"), state.c_str()); + + // first merge with the existing sections + pkgTagFile tagfile(&StateFile); + pkgTagSection section; + std::set<string> pkgs_seen; + while(tagfile.Step(section)) { + auto const pkgname = section.Find(pkgTagSection::Key::Package); + auto pkgarch = section.Find(pkgTagSection::Key::Architecture); + if (pkgarch.empty() == true) + pkgarch = "native"; + // Silently ignore unknown packages and packages with no actual + // version. + pkgCache::PkgIterator pkg = Cache->FindPkg(pkgname, pkgarch); + if(pkg.end() || pkg.VersionList().end()) + continue; + StateCache const &P = PkgState[pkg->ID]; + bool newAuto = (P.Flags & Flag::Auto); + // reset to default (=manual) not installed or now-removed ones if requested + if (InstalledOnly && ( + (pkg->CurrentVer == 0 && P.Mode != ModeInstall) || + (pkg->CurrentVer != 0 && P.Mode == ModeDelete))) + newAuto = false; + if (newAuto == false) + { + // The section is obsolete if it contains no other tag + auto const count = section.Count(); + if (count < 2 || + (count == 2 && section.Exists("Auto-Installed")) || + (count == 3 && section.Exists("Auto-Installed") && section.Exists("Architecture"))) + { + if(debug_autoremove) + std::clog << "Drop obsolete section with " << count << " fields for " << APT::PrettyPkg(this, pkg) << std::endl; + continue; + } + } + + if(debug_autoremove) + std::clog << "Update existing AutoInstall to " << newAuto << " for " << APT::PrettyPkg(this, pkg) << std::endl; + + std::vector<pkgTagSection::Tag> rewrite; + rewrite.push_back(pkgTagSection::Tag::Rewrite("Architecture", pkg.Arch())); + rewrite.push_back(pkgTagSection::Tag::Rewrite("Auto-Installed", newAuto ? "1" : "0")); + section.Write(OutFile, NULL, rewrite); + if (OutFile.Write("\n", 1) == false) + return false; + pkgs_seen.insert(pkg.FullName()); + } + + // then write the ones we have not seen yet + for(pkgCache::PkgIterator pkg=Cache->PkgBegin(); !pkg.end(); ++pkg) { + StateCache const &P = PkgState[pkg->ID]; + if(P.Flags & Flag::Auto) { + if (pkgs_seen.find(pkg.FullName()) != pkgs_seen.end()) { + if(debug_autoremove) + std::clog << "Skipping already written " << APT::PrettyPkg(this, pkg) << std::endl; + continue; + } + // skip not installed ones if requested + if (InstalledOnly && ( + (pkg->CurrentVer == 0 && P.Mode != ModeInstall) || + (pkg->CurrentVer != 0 && P.Mode == ModeDelete))) + continue; + if(debug_autoremove) + std::clog << "Writing new AutoInstall: " << APT::PrettyPkg(this, pkg) << std::endl; + std::string stanza = "Package: "; + stanza.append(pkg.Name()) + .append("\nArchitecture: ").append(pkg.Arch()) + .append("\nAuto-Installed: 1\n\n"); + if (OutFile.Write(stanza.c_str(), stanza.length()) == false) + return false; + } + } + if (StateFile.Failed()) + { + OutFile.OpFail(); + return false; + } + if (OutFile.Close() == false) + return false; + chmod(state.c_str(), 0644); + return true; +} + /*}}}*/ +// DepCache::CheckDep - Checks a single dependency /*{{{*/ +// --------------------------------------------------------------------- +/* This first checks the dependency against the main target package and + then walks along the package provides list and checks if each provides + will be installed then checks the provides against the dep. Res will be + set to the package which was used to satisfy the dep. */ +bool pkgDepCache::CheckDep(DepIterator const &Dep,int const Type,PkgIterator &Res) +{ + Res = Dep.TargetPkg(); + + /* Check simple depends. A depends -should- never self match but + we allow it anyhow because dpkg does. Technically it is a packaging + bug. Conflicts may never self match */ + if (Dep.IsIgnorable(Res) == false) + { + // Check the base package + if (Type == NowVersion) + { + if (Res->CurrentVer != 0 && Dep.IsSatisfied(Res.CurrentVer()) == true) + return true; + } + else if (Type == InstallVersion) + { + if (PkgState[Res->ID].InstallVer != 0 && + Dep.IsSatisfied(PkgState[Res->ID].InstVerIter(*this)) == true) + return true; + } + else if (Type == CandidateVersion) + if (PkgState[Res->ID].CandidateVer != 0 && + Dep.IsSatisfied(PkgState[Res->ID].CandidateVerIter(*this)) == true) + return true; + } + + if (Dep->Type == Dep::Obsoletes) + return false; + + // Check the providing packages + PrvIterator P = Dep.TargetPkg().ProvidesList(); + for (; P.end() != true; ++P) + { + if (Dep.IsIgnorable(P) == true) + continue; + + // Check if the provides is a hit + if (Type == NowVersion) + { + if (P.OwnerPkg().CurrentVer() != P.OwnerVer()) + continue; + } + else if (Type == InstallVersion) + { + StateCache &State = PkgState[P.OwnerPkg()->ID]; + if (State.InstallVer != (Version *)P.OwnerVer()) + continue; + } + else if (Type == CandidateVersion) + { + StateCache &State = PkgState[P.OwnerPkg()->ID]; + if (State.CandidateVer != (Version *)P.OwnerVer()) + continue; + } + + // Compare the versions. + if (Dep.IsSatisfied(P) == true) + { + Res = P.OwnerPkg(); + return true; + } + } + + return false; +} + /*}}}*/ +// DepCache::AddSizes - Add the packages sizes to the counters /*{{{*/ +// --------------------------------------------------------------------- +/* Call with Inverse = true to perform the inverse operation */ +void pkgDepCache::AddSizes(const PkgIterator &Pkg, bool const Inverse) +{ + StateCache &P = PkgState[Pkg->ID]; + + if (Pkg->VersionList == 0) + return; + + if (Pkg.State() == pkgCache::PkgIterator::NeedsConfigure && + P.Keep() == true) + return; + + // Compute the size data + if (P.NewInstall() == true) + { + if (Inverse == false) { + iUsrSize += P.InstVerIter(*this)->InstalledSize; + iDownloadSize += P.InstVerIter(*this)->Size; + } else { + iUsrSize -= P.InstVerIter(*this)->InstalledSize; + iDownloadSize -= P.InstVerIter(*this)->Size; + } + return; + } + + // Upgrading + if (Pkg->CurrentVer != 0 && + (P.InstallVer != (Version *)Pkg.CurrentVer() || + (P.iFlags & ReInstall) == ReInstall) && P.InstallVer != 0) + { + if (Inverse == false) { + iUsrSize -= Pkg.CurrentVer()->InstalledSize; + iUsrSize += P.InstVerIter(*this)->InstalledSize; + iDownloadSize += P.InstVerIter(*this)->Size; + } else { + iUsrSize -= P.InstVerIter(*this)->InstalledSize; + iUsrSize += Pkg.CurrentVer()->InstalledSize; + iDownloadSize -= P.InstVerIter(*this)->Size; + } + return; + } + + // Reinstall + if (Pkg.State() == pkgCache::PkgIterator::NeedsUnpack && + P.Delete() == false) + { + if (Inverse == false) + iDownloadSize += P.InstVerIter(*this)->Size; + else + iDownloadSize -= P.InstVerIter(*this)->Size; + return; + } + + // Removing + if (Pkg->CurrentVer != 0 && P.InstallVer == 0) + { + if (Inverse == false) + iUsrSize -= Pkg.CurrentVer()->InstalledSize; + else + iUsrSize += Pkg.CurrentVer()->InstalledSize; + return; + } +} + /*}}}*/ +// DepCache::AddStates - Add the package to the state counter /*{{{*/ +// --------------------------------------------------------------------- +/* This routine is tricky to use, you must make sure that it is never + called twice for the same package. This means the Remove/Add section + should be as short as possible and not encompass any code that will + call Remove/Add itself. Remember, dependencies can be circular so + while processing a dep for Pkg it is possible that Add/Remove + will be called on Pkg */ +void pkgDepCache::AddStates(const PkgIterator &Pkg, bool const Invert) +{ + signed char const Add = (Invert == false) ? 1 : -1; + StateCache &State = PkgState[Pkg->ID]; + + // The Package is broken (either minimal dep or policy dep) + if ((State.DepState & DepInstMin) != DepInstMin) + iBrokenCount += Add; + if ((State.DepState & DepInstPolicy) != DepInstPolicy) + iPolicyBrokenCount += Add; + + // Bad state + if (Pkg.State() != PkgIterator::NeedsNothing) + iBadCount += Add; + + // Not installed + if (Pkg->CurrentVer == 0) + { + if (State.Mode == ModeDelete && + (State.iFlags & Purge) == Purge && Pkg.Purge() == false) + iDelCount += Add; + + if (State.Mode == ModeInstall) + iInstCount += Add; + return; + } + + // Installed, no upgrade + if (State.Status == 0) + { + if (State.Mode == ModeDelete) + iDelCount += Add; + else + if ((State.iFlags & ReInstall) == ReInstall) + iInstCount += Add; + return; + } + + // Alll 3 are possible + if (State.Mode == ModeDelete) + iDelCount += Add; + else if (State.Mode == ModeKeep) + iKeepCount += Add; + else if (State.Mode == ModeInstall) + iInstCount += Add; +} + /*}}}*/ +// DepCache::BuildGroupOrs - Generate the Or group dep data /*{{{*/ +// --------------------------------------------------------------------- +/* The or group results are stored in the last item of the or group. This + allows easy detection of the state of a whole or'd group. */ +void pkgDepCache::BuildGroupOrs(VerIterator const &V) +{ + unsigned char Group = 0; + for (DepIterator D = V.DependsList(); D.end() != true; ++D) + { + // Build the dependency state. + unsigned char &State = DepState[D->ID]; + + /* Invert for Conflicts. We have to do this twice to get the + right sense for a conflicts group */ + if (D.IsNegative() == true) + State = ~State; + + // Add to the group if we are within an or.. + State &= 0x7; + Group |= State; + State |= Group << 3; + if ((D->CompareOp & Dep::Or) != Dep::Or) + Group = 0; + + // Invert for Conflicts + if (D.IsNegative() == true) + State = ~State; + } +} + /*}}}*/ +// DepCache::VersionState - Perform a pass over a dependency list /*{{{*/ +// --------------------------------------------------------------------- +/* This is used to run over a dependency list and determine the dep + state of the list, filtering it through both a Min check and a Policy + check. The return result will have SetMin/SetPolicy low if a check + fails. It uses the DepState cache for it's computations. */ +unsigned char pkgDepCache::VersionState(DepIterator D, unsigned char const Check, + unsigned char const SetMin, + unsigned char const SetPolicy) const +{ + unsigned char Dep = 0xFF; + while (D.end() != true) + { + // the last or-dependency has the state of all previous or'ed + DepIterator Start, End; + D.GlobOr(Start, End); + // ignore if we are called with Dep{Install,…} or DepG{Install,…} + // the later would be more correct, but the first is what we get + unsigned char const State = DepState[End->ID] | (DepState[End->ID] >> 3); + + // Minimum deps that must be satisfied to have a working package + if (Start.IsCritical() == true) + { + if ((State & Check) != Check) + return Dep &= ~(SetMin | SetPolicy); + } + // Policy deps that must be satisfied to install the package + else if (IsImportantDep(Start) == true && + (State & Check) != Check) + Dep &= ~SetPolicy; + } + return Dep; +} + /*}}}*/ +// DepCache::DependencyState - Compute the 3 results for a dep /*{{{*/ +// --------------------------------------------------------------------- +/* This is the main dependency computation bit. It computes the 3 main + results for a dependency: Now, Install and Candidate. Callers must + invert the result if dealing with conflicts. */ +unsigned char pkgDepCache::DependencyState(DepIterator const &D) +{ + unsigned char State = 0; + + if (CheckDep(D,NowVersion) == true) + State |= DepNow; + if (CheckDep(D,InstallVersion) == true) + State |= DepInstall; + if (CheckDep(D,CandidateVersion) == true) + State |= DepCVer; + + return State; +} + /*}}}*/ +// DepCache::UpdateVerState - Compute the Dep member of the state /*{{{*/ +// --------------------------------------------------------------------- +/* This determines the combined dependency representation of a package + for its two states now and install. This is done by using the pre-generated + dependency information. */ +void pkgDepCache::UpdateVerState(PkgIterator const &Pkg) +{ + // Empty deps are always true + StateCache &State = PkgState[Pkg->ID]; + State.DepState = 0xFF; + + // Check the Current state + if (Pkg->CurrentVer != 0) + { + DepIterator D = Pkg.CurrentVer().DependsList(); + State.DepState &= VersionState(D,DepNow,DepNowMin,DepNowPolicy); + } + + /* Check the candidate state. We do not compare against the whole as + a candidate state but check the candidate version against the + install states */ + if (State.CandidateVer != 0) + { + DepIterator D = State.CandidateVerIter(*this).DependsList(); + State.DepState &= VersionState(D,DepInstall,DepCandMin,DepCandPolicy); + } + + // Check target state which can only be current or installed + if (State.InstallVer != 0) + { + DepIterator D = State.InstVerIter(*this).DependsList(); + State.DepState &= VersionState(D,DepInstall,DepInstMin,DepInstPolicy); + } +} + /*}}}*/ +// DepCache::Update - Figure out all the state information /*{{{*/ +// --------------------------------------------------------------------- +/* This will figure out the state of all the packages and all the + dependencies based on the current policy. */ +void pkgDepCache::PerformDependencyPass(OpProgress * const Prog) +{ + iUsrSize = 0; + iDownloadSize = 0; + iInstCount = 0; + iDelCount = 0; + iKeepCount = 0; + iBrokenCount = 0; + iPolicyBrokenCount = 0; + iBadCount = 0; + + int Done = 0; + for (PkgIterator I = PkgBegin(); I.end() != true; ++I, ++Done) + { + if (Prog != 0 && Done%20 == 0) + Prog->Progress(Done); + for (VerIterator V = I.VersionList(); V.end() != true; ++V) + { + unsigned char Group = 0; + + for (DepIterator D = V.DependsList(); D.end() != true; ++D) + { + // Build the dependency state. + unsigned char &State = DepState[D->ID]; + State = DependencyState(D); + + // Add to the group if we are within an or.. + Group |= State; + State |= Group << 3; + if ((D->CompareOp & Dep::Or) != Dep::Or) + Group = 0; + + // Invert for Conflicts + if (D.IsNegative() == true) + State = ~State; + } + } + + // Compute the package dependency state and size additions + AddSizes(I); + UpdateVerState(I); + AddStates(I); + } + if (Prog != 0) + Prog->Progress(Done); +} +void pkgDepCache::Update(OpProgress * const Prog) +{ + PerformDependencyPass(Prog); + readStateFile(Prog); +} + /*}}}*/ +// DepCache::Update - Update the deps list of a package /*{{{*/ +// --------------------------------------------------------------------- +/* This is a helper for update that only does the dep portion of the scan. + It is mainly meant to scan reverse dependencies. */ +void pkgDepCache::Update(DepIterator D) +{ + // Update the reverse deps + for (;D.end() != true; ++D) + { + unsigned char &State = DepState[D->ID]; + State = DependencyState(D); + + // Invert for Conflicts + if (D.IsNegative() == true) + State = ~State; + + RemoveStates(D.ParentPkg()); + BuildGroupOrs(D.ParentVer()); + UpdateVerState(D.ParentPkg()); + AddStates(D.ParentPkg()); + } +} + /*}}}*/ +// DepCache::Update - Update the related deps of a package /*{{{*/ +// --------------------------------------------------------------------- +/* This is called whenever the state of a package changes. It updates + all cached dependencies related to this package. */ +void pkgDepCache::Update(PkgIterator const &Pkg) +{ + // Recompute the dep of the package + RemoveStates(Pkg); + UpdateVerState(Pkg); + AddStates(Pkg); + + // Update the reverse deps + Update(Pkg.RevDependsList()); + + // Update the provides map for the current ver + auto const CurVer = Pkg.CurrentVer(); + if (not CurVer.end()) + for (PrvIterator P = CurVer.ProvidesList(); not P.end(); ++P) + Update(P.ParentPkg().RevDependsList()); + + // Update the provides map for the candidate ver + auto const CandVer = PkgState[Pkg->ID].CandidateVerIter(*this); + if (not CandVer.end() && CandVer != CurVer) + for (PrvIterator P = CandVer.ProvidesList(); not P.end(); ++P) + Update(P.ParentPkg().RevDependsList()); +} + /*}}}*/ +// DepCache::IsModeChangeOk - check if it is ok to change the mode /*{{{*/ +// --------------------------------------------------------------------- +/* this is used by all Mark methods on the very first line to check sanity + and prevents mode changes for packages on hold for example. + If you want to check Mode specific stuff you can use the virtual public + Is<Mode>Ok methods instead */ +static char const* PrintMode(char const mode) +{ + switch (mode) + { + case pkgDepCache::ModeInstall: return "Install"; + case pkgDepCache::ModeKeep: return "Keep"; + case pkgDepCache::ModeDelete: return "Delete"; + case pkgDepCache::ModeGarbage: return "Garbage"; + default: return "UNKNOWN"; + } +} +static bool IsModeChangeOk(pkgDepCache &Cache, pkgDepCache::ModeList const mode, pkgCache::PkgIterator const &Pkg, + unsigned long const Depth, bool const FromUser, bool const DebugMarker) +{ + // we are not trying too hard… + if (unlikely(Depth > 3000)) + return false; + + // general sanity + if (unlikely(Pkg.end() == true || Pkg->VersionList == 0)) + return false; + + // the user is always right + if (FromUser == true) + return true; + + auto &P = Cache[Pkg]; + // not changing the mode is obviously also fine as we might want to call + // e.g. MarkInstall multiple times with different arguments for the same package + if (P.Mode == mode) + return true; + + // if previous state was set by user only user can reset it + if ((P.iFlags & pkgDepCache::Protected) == pkgDepCache::Protected) + { + if (unlikely(DebugMarker == true)) + std::clog << OutputInDepth(Depth) << "Ignore Mark" << PrintMode(mode) + << " of " << APT::PrettyPkg(&Cache, Pkg) << " as its mode (" << PrintMode(P.Mode) + << ") is protected" << std::endl; + return false; + } + // enforce dpkg holds + else if (mode != pkgDepCache::ModeKeep && Pkg->SelectedState == pkgCache::State::Hold && + _config->FindB("APT::Ignore-Hold",false) == false) + { + if (unlikely(DebugMarker == true)) + std::clog << OutputInDepth(Depth) << "Hold prevents Mark" << PrintMode(mode) + << " of " << APT::PrettyPkg(&Cache, Pkg) << std::endl; + return false; + } + // Do not allow removals of essential packages not explicitly triggered by the user + else if (mode == pkgDepCache::ModeDelete && (Pkg->Flags & pkgCache::Flag::Essential) == pkgCache::Flag::Essential && + not _config->FindB("APT::Get::Allow-Solver-Remove-Essential", false)) + { + if (unlikely(DebugMarker == true)) + std::clog << OutputInDepth(Depth) << "Essential prevents Mark" << PrintMode(mode) + << " of " << APT::PrettyPkg(&Cache, Pkg) << std::endl; + return false; + } + // Do not allow removals of essential packages not explicitly triggered by the user + else if (mode == pkgDepCache::ModeDelete && (Pkg->Flags & pkgCache::Flag::Important) == pkgCache::Flag::Important && + not _config->FindB("APT::Get::Allow-Solver-Remove-Essential", false)) + { + if (unlikely(DebugMarker == true)) + std::clog << OutputInDepth(Depth) << "Protected prevents Mark" << PrintMode(mode) + << " of " << APT::PrettyPkg(&Cache, Pkg) << std::endl; + return false; + } + + return true; +} + /*}}}*/ +// DepCache::MarkKeep - Put the package in the keep state /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgDepCache::MarkKeep(PkgIterator const &Pkg, bool Soft, bool FromUser, + unsigned long Depth) +{ + if (not IsModeChangeOk(*this, ModeKeep, Pkg, Depth, FromUser, DebugMarker)) + return false; + + /* Reject an attempt to keep a non-source broken installed package, those + must be upgraded */ + if (Pkg.State() == PkgIterator::NeedsUnpack && + Pkg.CurrentVer().Downloadable() == false) + return false; + + /* We changed the soft state all the time so the UI is a bit nicer + to use */ + StateCache &P = PkgState[Pkg->ID]; + + // Check that it is not already kept + if (P.Mode == ModeKeep) + return true; + + if (Soft == true) + P.iFlags |= AutoKept; + else + P.iFlags &= ~AutoKept; + + ActionGroup group(*this); + +#if 0 // resetting the autoflag here means we lose the + // auto-mark information if a user selects a package for removal + // but changes his mind then and sets it for keep again + // - this makes sense as default when all Garbage dependencies + // are automatically marked for removal (as aptitude does). + // setting a package for keep then makes it no longer autoinstalled + // for all other use-case this action is rather surprising + if(FromUser && !P.Marked) + P.Flags &= ~Flag::Auto; +#endif + + if (DebugMarker == true) + std::clog << OutputInDepth(Depth) << "MarkKeep " << APT::PrettyPkg(this, Pkg) << " FU=" << FromUser << std::endl; + + RemoveSizes(Pkg); + RemoveStates(Pkg); + + P.Mode = ModeKeep; + if (Pkg->CurrentVer == 0) + P.InstallVer = 0; + else + P.InstallVer = Pkg.CurrentVer(); + + AddStates(Pkg); + Update(Pkg); + AddSizes(Pkg); + + return true; +} + /*}}}*/ +// DepCache::MarkDelete - Put the package in the delete state /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgDepCache::MarkDelete(PkgIterator const &Pkg, bool rPurge, + unsigned long Depth, bool FromUser) +{ + if (not IsModeChangeOk(*this, ModeDelete, Pkg, Depth, FromUser, DebugMarker)) + return false; + + StateCache &P = PkgState[Pkg->ID]; + + // Check that it is not already marked for delete + if ((P.Mode == ModeDelete || P.InstallVer == 0) && + (Pkg.Purge() == true || rPurge == false)) + return true; + + // check if we are allowed to remove the package + if (IsDeleteOk(Pkg,rPurge,Depth,FromUser) == false) + return false; + + P.iFlags &= ~(AutoKept | Purge); + if (rPurge == true) + P.iFlags |= Purge; + + ActionGroup group(*this); + + if (FromUser == false) + { + VerIterator const PV = P.InstVerIter(*this); + if (PV.end() == false) + { + // removed metapackages mark their dependencies as manual to prevent in "desktop depends browser, texteditor" + // the removal of browser to suggest the removal of desktop and texteditor. + // We ignore the auto-bit here as we can't deal with metapackage cascardes otherwise. + // We do not check for or-groups here as we don't know which package takes care of + // providing the feature the user likes e.g.: browser1 | browser2 | browser3 + // Temporary removals are effected by this as well, which is bad, but unlikely in practice + bool const PinNeverMarkAutoSection = (PV->Section != 0 && SectionInSubTree("APT::Never-MarkAuto-Sections", PV.Section())); + if (PinNeverMarkAutoSection) + { + for (DepIterator D = PV.DependsList(); D.end() != true; ++D) + { + if (D.IsMultiArchImplicit() == true || D.IsNegative() == true || IsImportantDep(D) == false) + continue; + + pkgCacheFile CacheFile(this); + APT::VersionList verlist = APT::VersionList::FromDependency(CacheFile, D, APT::CacheSetHelper::INSTALLED); + for (auto const &V : verlist) + { + PkgIterator const DP = V.ParentPkg(); + if(DebugAutoInstall == true) + std::clog << OutputInDepth(Depth) << "Setting " << DP.FullName(false) << " NOT as auto-installed (direct " + << D.DepType() << " of " << Pkg.FullName(false) << " which is in APT::Never-MarkAuto-Sections)" << std::endl; + + MarkAuto(DP, false); + } + } + } + } + } + + if (DebugMarker == true) + std::clog << OutputInDepth(Depth) << (rPurge ? "MarkPurge " : "MarkDelete ") << APT::PrettyPkg(this, Pkg) << " FU=" << FromUser << std::endl; + + RemoveSizes(Pkg); + RemoveStates(Pkg); + + if (Pkg->CurrentVer == 0 && (Pkg.Purge() == true || rPurge == false)) + P.Mode = ModeKeep; + else + P.Mode = ModeDelete; + P.InstallVer = 0; + + AddStates(Pkg); + Update(Pkg); + AddSizes(Pkg); + + return true; +} + /*}}}*/ +// DepCache::IsDeleteOk - check if it is ok to remove this package /*{{{*/ +// --------------------------------------------------------------------- +/* The default implementation tries to prevent deletion of install requests. + dpkg holds are enforced by the private IsModeChangeOk */ +bool pkgDepCache::IsDeleteOk(PkgIterator const &Pkg,bool rPurge, + unsigned long Depth, bool FromUser) +{ + return IsDeleteOkProtectInstallRequests(Pkg, rPurge, Depth, FromUser); +} +bool pkgDepCache::IsDeleteOkProtectInstallRequests(PkgIterator const &Pkg, + bool const /*rPurge*/, unsigned long const Depth, bool const FromUser) +{ + if (FromUser == false && Pkg->CurrentVer == 0) + { + StateCache &P = PkgState[Pkg->ID]; + if (P.InstallVer != 0 && P.Status == 2 && (P.Flags & Flag::Auto) != Flag::Auto) + { + if (DebugMarker == true) + std::clog << OutputInDepth(Depth) << "Manual install request prevents MarkDelete of " << APT::PrettyPkg(this, Pkg) << std::endl; + return false; + } + } + return true; +} + /*}}}*/ +struct CompareProviders /*{{{*/ +{ + pkgDepCache const &Cache; + pkgCache::PkgIterator const Pkg; + explicit CompareProviders(pkgDepCache const &pCache, pkgCache::DepIterator const &Dep) : Cache{pCache}, Pkg{Dep.TargetPkg()} {} + bool operator() (pkgCache::VerIterator const &AV, pkgCache::VerIterator const &BV) + { + pkgCache::PkgIterator const A = AV.ParentPkg(); + pkgCache::PkgIterator const B = BV.ParentPkg(); + // Deal with protected first as if they don't work we usually have a problem + if (Cache[A].Protect() != Cache[B].Protect()) + return Cache[A].Protect(); + // Prefer MA:same packages if other architectures for it are installed + if ((AV->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same || + (BV->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same) + { + bool instA = false; + if ((AV->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same) + { + pkgCache::GrpIterator Grp = A.Group(); + for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P)) + if (P->CurrentVer != 0) + { + instA = true; + break; + } + } + bool instB = false; + if ((BV->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same) + { + pkgCache::GrpIterator Grp = B.Group(); + for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P)) + { + if (P->CurrentVer != 0) + { + instB = true; + break; + } + } + } + if (instA != instB) + return instA; + } + if ((A->CurrentVer == 0 || B->CurrentVer == 0) && A->CurrentVer != B->CurrentVer) + return A->CurrentVer != 0; + // Prefer packages in the same group as the target; e.g. foo:i386, foo:amd64 + if (A->Group != B->Group) + { + if (A->Group == Pkg->Group && B->Group != Pkg->Group) + return true; + else if (B->Group == Pkg->Group && A->Group != Pkg->Group) + return false; + } + // we like essentials + if ((A->Flags & pkgCache::Flag::Essential) != (B->Flags & pkgCache::Flag::Essential)) + { + if ((A->Flags & pkgCache::Flag::Essential) == pkgCache::Flag::Essential) + return true; + else if ((B->Flags & pkgCache::Flag::Essential) == pkgCache::Flag::Essential) + return false; + } + if ((A->Flags & pkgCache::Flag::Important) != (B->Flags & pkgCache::Flag::Important)) + { + if ((A->Flags & pkgCache::Flag::Important) == pkgCache::Flag::Important) + return true; + else if ((B->Flags & pkgCache::Flag::Important) == pkgCache::Flag::Important) + return false; + } + // prefer native architecture + if (strcmp(A.Arch(), B.Arch()) != 0) + { + if (strcmp(A.Arch(), A.Cache()->NativeArch()) == 0) + return true; + else if (strcmp(B.Arch(), B.Cache()->NativeArch()) == 0) + return false; + std::vector<std::string> archs = APT::Configuration::getArchitectures(); + for (std::vector<std::string>::const_iterator a = archs.begin(); a != archs.end(); ++a) + if (*a == A.Arch()) + return true; + else if (*a == B.Arch()) + return false; + } + // higher priority seems like a good idea + if (AV->Priority != BV->Priority) + return AV->Priority < BV->Priority; + // unable to decide… + return A->ID > B->ID; + } +}; + /*}}}*/ +bool pkgDepCache::MarkInstall_StateChange(pkgCache::PkgIterator const &Pkg, bool AutoInst, bool FromUser) /*{{{*/ +{ + bool AlwaysMarkAsAuto = _config->FindB("APT::Get::Mark-Auto", false) == true; + auto &P = (*this)[Pkg]; + if (P.Protect() && P.InstallVer == P.CandidateVer) + return true; + + P.iFlags &= ~pkgDepCache::AutoKept; + + /* Target the candidate version and remove the autoflag. We reset the + autoflag below if this was called recursively. Otherwise the user + should have the ability to de-auto a package by changing its state */ + RemoveSizes(Pkg); + RemoveStates(Pkg); + + P.Mode = pkgDepCache::ModeInstall; + P.InstallVer = P.CandidateVer; + + if(FromUser && !AlwaysMarkAsAuto) + { + // Set it to manual if it's a new install or already installed, + // but only if its not marked by the autoremover (aptitude depend on this behavior) + // or if we do automatic installation (aptitude never does it) + if(P.Status == 2 || (Pkg->CurrentVer != 0 && (AutoInst == true || P.Marked == false))) + P.Flags &= ~pkgCache::Flag::Auto; + } + else + { + // Set it to auto if this is a new install. + if(P.Status == 2) + P.Flags |= pkgCache::Flag::Auto; + } + if (P.CandidateVer == (pkgCache::Version *)Pkg.CurrentVer()) + P.Mode = pkgDepCache::ModeKeep; + + AddStates(Pkg); + Update(Pkg); + AddSizes(Pkg); + return true; +} + /*}}}*/ +static bool MarkInstall_DiscardCandidate(pkgDepCache &Cache, pkgCache::PkgIterator const &Pkg) /*{{{*/ +{ + auto &State = Cache[Pkg]; + State.CandidateVer = State.InstallVer; + auto const oldStatus = State.Status; + State.Update(Pkg, Cache); + State.Status = oldStatus; + return true; +} + /*}}}*/ +bool pkgDepCache::MarkInstall_DiscardInstall(pkgCache::PkgIterator const &Pkg) /*{{{*/ +{ + StateCache &State = PkgState[Pkg->ID]; + if (State.Mode == ModeKeep && State.InstallVer == State.CandidateVer && State.CandidateVer == Pkg.CurrentVer()) + return true; + RemoveSizes(Pkg); + RemoveStates(Pkg); + if (Pkg->CurrentVer != 0) + State.InstallVer = Pkg.CurrentVer(); + else + State.InstallVer = nullptr; + State.Mode = ModeKeep; + AddStates(Pkg); + Update(Pkg); + AddSizes(Pkg); + return MarkInstall_DiscardCandidate(*this, Pkg); +} + /*}}}*/ +static bool MarkInstall_CollectDependencies(pkgDepCache const &Cache, pkgCache::VerIterator const &PV, std::vector<pkgCache::DepIterator> &toInstall, std::vector<pkgCache::DepIterator> &toRemove) /*{{{*/ +{ + auto const propagateProtected = Cache[PV.ParentPkg()].Protect(); + for (auto Dep = PV.DependsList(); not Dep.end();) + { + auto const Start = Dep; + // check if an installed package satisfies the dependency (and get the extend of the or-group) + bool foundSolution = false; + for (bool LastOR = true; not Dep.end() && LastOR; ++Dep) + { + LastOR = (Dep->CompareOp & pkgCache::Dep::Or) == pkgCache::Dep::Or; + if ((Cache[Dep] & pkgDepCache::DepInstall) == pkgDepCache::DepInstall) + foundSolution = true; + } + if (foundSolution && not propagateProtected) + continue; + + /* Check if this dep should be consider for install. + (Pre-)Depends, Conflicts and Breaks for sure. + Recommends & Suggests depending on configuration */ + if (not Cache.IsImportantDep(Start)) + continue; + + if (Start.IsNegative()) + { + if (Start->Type != pkgCache::Dep::Obsoletes) + toRemove.push_back(Start); + } + else + toInstall.push_back(Start); + } + return true; +} + /*}}}*/ +static APT::VersionVector getAllPossibleSolutions(pkgDepCache &Cache, pkgCache::DepIterator Start, pkgCache::DepIterator const &End, APT::CacheSetHelper::VerSelector const selector, bool const sorted) /*{{{*/ +{ + pkgCacheFile CacheFile{&Cache}; + APT::VersionVector toUpgrade, toNewInstall; + do + { + APT::VersionVector verlist = APT::VersionVector::FromDependency(CacheFile, Start, selector); + if (not sorted) + { + std::move(verlist.begin(), verlist.end(), std::back_inserter(toUpgrade)); + continue; + } + std::sort(verlist.begin(), verlist.end(), CompareProviders{Cache, Start}); + for (auto &&Ver : verlist) + { + auto P = Ver.ParentPkg(); + if (P->CurrentVer != 0) + toUpgrade.emplace_back(std::move(Ver)); + else + toNewInstall.emplace_back(std::move(Ver)); + } + } while (Start++ != End); + if (toUpgrade.empty()) + toUpgrade = std::move(toNewInstall); + else + std::move(toNewInstall.begin(), toNewInstall.end(), std::back_inserter(toUpgrade)); + + if (not sorted) + std::sort(toUpgrade.begin(), toUpgrade.end(), [](pkgCache::VerIterator const &A, pkgCache::VerIterator const &B) { return A->ID < B->ID; }); + toUpgrade.erase(std::unique(toUpgrade.begin(), toUpgrade.end()), toUpgrade.end()); + + if (not End.IsNegative()) + toUpgrade.erase(std::remove_if(toUpgrade.begin(), toUpgrade.end(), [&Cache](pkgCache::VerIterator const &V) { + auto const P = V.ParentPkg(); + auto const &State = Cache[P]; + return State.Protect() && (State.Delete() || (State.Keep() && P->CurrentVer == 0)); + }), + toUpgrade.end()); + + return toUpgrade; +} + /*}}}*/ +static bool MarkInstall_MarkDeleteForNotUpgradeable(pkgDepCache &Cache, bool const DebugAutoInstall, pkgCache::VerIterator const &PV, unsigned long const Depth, pkgCache::PkgIterator const &Pkg, bool const propagateProtected, APT::PackageVector &delayedRemove)/*{{{*/ +{ + auto &State = Cache[Pkg]; + if (not propagateProtected) + { + if (State.Delete()) + return true; + if(DebugAutoInstall) + std::clog << OutputInDepth(Depth) << " Delayed Removing: " << Pkg.FullName() << " as upgrade is not an option for " << PV.ParentPkg().FullName() << " (" << PV.VerStr() << ")\n"; + if (not IsModeChangeOk(Cache, pkgDepCache::ModeDelete, Pkg, Depth, false, DebugAutoInstall) || + not Cache.IsDeleteOk(Pkg, false, Depth, false)) + return false; + delayedRemove.push_back(Pkg); + return true; + } + + if (not State.Delete()) + { + if(DebugAutoInstall) + std::clog << OutputInDepth(Depth) << " Removing: " << Pkg.FullName() << " as upgrade is not an option for " << PV.ParentPkg().FullName() << " (" << PV.VerStr() << ")\n"; + if (not Cache.MarkDelete(Pkg, false, Depth + 1, false)) + return false; + } + MarkInstall_DiscardCandidate(Cache, Pkg); + Cache.MarkProtected(Pkg); + return true; +} + /*}}}*/ +static bool MarkInstall_RemoveConflictsIfNotUpgradeable(pkgDepCache &Cache, bool const DebugAutoInstall, pkgCache::VerIterator const &PV, unsigned long Depth, std::vector<pkgCache::DepIterator> &toRemove, APT::PackageVector &toUpgrade, APT::PackageVector &delayedRemove, bool const propagateProtected, bool const FromUser) /*{{{*/ +{ + /* Negative dependencies have no or-group + If the candidate is effected try to keep current and discard candidate + If the current is effected try upgrading to candidate or remove it */ + bool failedToRemoveSomething = false; + APT::PackageVector badCandidate; + for (auto const &D : toRemove) + { + for (auto const &Ver : getAllPossibleSolutions(Cache, D, D, APT::CacheSetHelper::CANDIDATE, true)) + { + auto const Pkg = Ver.ParentPkg(); + auto &State = Cache[Pkg]; + if (State.CandidateVer != Ver) + continue; + if (Pkg.CurrentVer() != Ver) + { + if (State.Install() && not Cache.MarkKeep(Pkg, false, false, Depth)) + { + failedToRemoveSomething = true; + if (not propagateProtected && not FromUser) + break; + } + else if (propagateProtected) + { + MarkInstall_DiscardCandidate(Cache, Pkg); + if (Pkg->CurrentVer == 0) + Cache.MarkProtected(Pkg); + } + else + badCandidate.push_back(Pkg); + } + else if (not MarkInstall_MarkDeleteForNotUpgradeable(Cache, DebugAutoInstall, PV, Depth, Pkg, propagateProtected, delayedRemove)) + { + failedToRemoveSomething = true; + if (not propagateProtected && not FromUser) + break; + } + } + if (failedToRemoveSomething && not propagateProtected && not FromUser) + break; + for (auto const &Ver : getAllPossibleSolutions(Cache, D, D, APT::CacheSetHelper::INSTALLED, true)) + { + auto const Pkg = Ver.ParentPkg(); + auto &State = Cache[Pkg]; + if (State.CandidateVer != Ver && State.CandidateVer != nullptr && + std::find(badCandidate.cbegin(), badCandidate.cend(), Pkg) == badCandidate.end()) + toUpgrade.push_back(Pkg); + else if (State.CandidateVer == Pkg.CurrentVer()) + ; // already done in the first loop above + else if (not MarkInstall_MarkDeleteForNotUpgradeable(Cache, DebugAutoInstall, PV, Depth, Pkg, propagateProtected, delayedRemove)) + { + failedToRemoveSomething = true; + if (not propagateProtected && not FromUser) + break; + } + } + if (failedToRemoveSomething && not propagateProtected && not FromUser) + break; + } + toRemove.clear(); + return not failedToRemoveSomething; +} + /*}}}*/ +static bool MarkInstall_CollectReverseDepends(pkgDepCache &Cache, bool const DebugAutoInstall, pkgCache::VerIterator const &PV, unsigned long Depth, APT::PackageVector &toUpgrade) /*{{{*/ +{ + auto CurrentVer = PV.ParentPkg().CurrentVer(); + if (CurrentVer.end()) + return true; + for (pkgCache::DepIterator D = PV.ParentPkg().RevDependsList(); D.end() == false; ++D) + { + auto ParentPkg = D.ParentPkg(); + // Skip non-installed versions and packages already marked for upgrade + if (ParentPkg.CurrentVer() != D.ParentVer() || Cache[ParentPkg].Install()) + continue; + // We only handle important positive dependencies, RemoveConflictsIfNotUpgradeable handles negative + if (not Cache.IsImportantDep(D) || D.IsNegative()) + continue; + // The dependency was previously not satisfied (e.g. part of an or group) or will be satisfied, so it's OK + if (not D.IsSatisfied(CurrentVer) || D.IsSatisfied(PV)) + continue; + if (std::find(toUpgrade.begin(), toUpgrade.end(), ParentPkg) != toUpgrade.end()) + continue; + + if (DebugAutoInstall) + std::clog << OutputInDepth(Depth) << " Upgrading: " << APT::PrettyPkg(&Cache, ParentPkg) << " due to " << APT::PrettyDep(&Cache, D) << "\n"; + + toUpgrade.push_back(ParentPkg); + } + return true; +} + /*}}}*/ +static bool MarkInstall_UpgradeOrRemoveConflicts(pkgDepCache &Cache, bool const DebugAutoInstall, unsigned long Depth, bool const ForceImportantDeps, APT::PackageVector &toUpgrade, bool const propagateProtected, bool const FromUser) /*{{{*/ +{ + bool failedToRemoveSomething = false; + for (auto const &InstPkg : toUpgrade) + if (not Cache[InstPkg].Install() && not Cache.MarkInstall(InstPkg, true, Depth + 1, false, ForceImportantDeps)) + { + if (DebugAutoInstall) + std::clog << OutputInDepth(Depth) << " Removing: " << InstPkg.FullName() << " as upgrade is not possible\n"; + if (not Cache.MarkDelete(InstPkg, false, Depth + 1, false)) + { + failedToRemoveSomething = true; + if (not propagateProtected && not FromUser) + break; + } + else if (propagateProtected) + Cache.MarkProtected(InstPkg); + } + toUpgrade.clear(); + return not failedToRemoveSomething; +} + /*}}}*/ +static bool MarkInstall_UpgradeOtherBinaries(pkgDepCache &Cache, bool const DebugAutoInstall, unsigned long Depth, bool const ForceImportantDeps, pkgCache::PkgIterator Pkg, pkgCache::VerIterator Ver) /*{{{*/ +{ + APT::PackageSet toUpgrade; + + if (not _config->FindB("APT::Get::Upgrade-By-Source-Package", true)) + return true; + + auto SrcGrp = Cache.FindGrp(Ver.SourcePkgName()); + for (auto OtherBinary = SrcGrp.VersionsInSource(); not OtherBinary.end(); OtherBinary = OtherBinary.NextInSource()) + { + auto OtherPkg = OtherBinary.ParentPkg(); + auto OtherState = Cache[OtherPkg]; + if (OtherPkg == Pkg) + continue; + // Package is not installed or at right version, don't need to upgrade + if (OtherPkg->CurrentVer == 0 || OtherPkg.CurrentVer() == OtherBinary) + continue; + // Package is to be installed at right version, don't need to upgrade + if (OtherState.Install() && OtherState.InstallVer == OtherBinary) + continue; + // Package has a different source version than us, so it's not relevant + if (strcmp(OtherBinary.SourceVerStr(), Ver.SourceVerStr()) != 0 || OtherState.CandidateVer != OtherBinary) + continue; + if (DebugAutoInstall) + std::clog << OutputInDepth(Depth) << "Upgrading " << APT::PrettyPkg(&Cache, OtherPkg) << " due to " << Pkg.FullName() << '\n'; + + toUpgrade.insert(OtherPkg); + } + for (auto &OtherPkg : toUpgrade) + Cache.MarkInstall(OtherPkg, false, Depth + 1, false, ForceImportantDeps); + for (auto &OtherPkg : toUpgrade) + Cache.MarkInstall(OtherPkg, true, Depth + 1, false, ForceImportantDeps); + return true; +} + /*}}}*/ +static bool MarkInstall_InstallDependencies(pkgDepCache &Cache, bool const DebugAutoInstall, bool const DebugMarker, pkgCache::PkgIterator const &Pkg, unsigned long Depth, bool const ForceImportantDeps, std::vector<pkgCache::DepIterator> &toInstall, APT::PackageVector *const toMoveAuto, bool const propagateProtected, bool const FromUser) /*{{{*/ +{ + auto const IsSatisfiedByInstalled = [&](auto &D) { return (Cache[pkgCache::DepIterator{Cache, &D}] & pkgDepCache::DepInstall) == pkgDepCache::DepInstall; }; + bool failedToInstallSomething = false; + for (auto &&Dep : toInstall) + { + auto const Copy = Dep; + pkgCache::DepIterator Start, End; + Dep.GlobOr(Start, End); + bool foundSolution = std::any_of(Start, Dep, IsSatisfiedByInstalled); + if (foundSolution && not propagateProtected) + continue; + bool const IsCriticalDep = Start.IsCritical(); + if (foundSolution) + { + // try propagating protected to this satisfied dependency + if (not IsCriticalDep) + continue; + auto const possibleSolutions = getAllPossibleSolutions(Cache, Start, End, APT::CacheSetHelper::CANDANDINST, false); + if (possibleSolutions.size() != 1) + continue; + auto const InstPkg = possibleSolutions.begin().ParentPkg(); + if (Cache[InstPkg].Protect()) + continue; + Cache.MarkProtected(InstPkg); + if (not Cache.MarkInstall(InstPkg, true, Depth + 1, false, ForceImportantDeps)) + failedToInstallSomething = true; + continue; + } + + /* Check if any ImportantDep() (but not Critical) were added + * since we installed the package. Also check for deps that + * were satisfied in the past: for instance, if a version + * restriction in a Recommends was tightened, upgrading the + * package should follow that Recommends rather than causing the + * dependency to be removed. (bug #470115) + */ + if (Pkg->CurrentVer != 0 && not ForceImportantDeps && not IsCriticalDep) + { + bool isNewImportantDep = true; + bool isPreviouslySatisfiedImportantDep = false; + for (pkgCache::DepIterator D = Pkg.CurrentVer().DependsList(); D.end() != true; ++D) + { + //FIXME: Should we handle or-group better here? + // We do not check if the package we look for is part of the same or-group + // we might find while searching, but could that really be a problem? + if (D.IsCritical() || not Cache.IsImportantDep(D) || + Start.TargetPkg() != D.TargetPkg()) + continue; + + isNewImportantDep = false; + + while ((D->CompareOp & pkgCache::Dep::Or) != 0) + ++D; + + isPreviouslySatisfiedImportantDep = ((Cache[D] & pkgDepCache::DepGNow) != 0); + if (isPreviouslySatisfiedImportantDep) + break; + } + + if (isNewImportantDep) + { + if (DebugAutoInstall) + std::clog << OutputInDepth(Depth) << "new important dependency: " + << Start.TargetPkg().FullName() << '\n'; + } + else if (isPreviouslySatisfiedImportantDep) + { + if (DebugAutoInstall) + std::clog << OutputInDepth(Depth) << "previously satisfied important dependency on " + << Start.TargetPkg().FullName() << '\n'; + } + else + { + if (DebugAutoInstall) + std::clog << OutputInDepth(Depth) << "ignore old unsatisfied important dependency on " + << Start.TargetPkg().FullName() << '\n'; + continue; + } + } + + auto const possibleSolutions = getAllPossibleSolutions(Cache, Start, End, APT::CacheSetHelper::CANDIDATE, true); + for (auto const &InstVer : possibleSolutions) + { + auto const InstPkg = InstVer.ParentPkg(); + if (Cache[InstPkg].CandidateVer != InstVer) + continue; + if (DebugAutoInstall) + std::clog << OutputInDepth(Depth) << "Installing " << InstPkg.FullName() + << " as " << End.DepType() << " of " << Pkg.FullName() << '\n'; + if (propagateProtected && IsCriticalDep && possibleSolutions.size() == 1) + { + if (not Cache.MarkInstall(InstPkg, false, Depth + 1, false, ForceImportantDeps)) + continue; + Cache.MarkProtected(InstPkg); + } + if (not Cache.MarkInstall(InstPkg, true, Depth + 1, false, ForceImportantDeps)) + continue; + + if (toMoveAuto != nullptr && InstPkg->CurrentVer == 0) + toMoveAuto->push_back(InstPkg); + + foundSolution = true; + break; + } + if (DebugMarker && not foundSolution) + std::clog << OutputInDepth(Depth+1) << APT::PrettyDep(&Cache, Copy) << " can't be satisfied! (dep)\n"; + if (not foundSolution && IsCriticalDep) + { + failedToInstallSomething = true; + if (not propagateProtected && not FromUser) + break; + } + } + toInstall.clear(); + return not failedToInstallSomething; +} + /*}}}*/ +// DepCache::MarkInstall - Put the package in the install state /*{{{*/ +bool pkgDepCache::MarkInstall(PkgIterator const &Pkg, bool AutoInst, + unsigned long Depth, bool FromUser, + bool ForceImportantDeps) +{ + StateCache &P = PkgState[Pkg->ID]; + if (P.Protect() && P.Keep() && P.CandidateVer != nullptr && P.CandidateVer == Pkg.CurrentVer()) + ; // we are here to mark our dependencies as protected, no state is changed + else if (not IsModeChangeOk(*this, ModeInstall, Pkg, Depth, FromUser, DebugMarker)) + return false; + + + // See if there is even any possible installation candidate + if (P.CandidateVer == 0) + return false; + + // Check that it is not already marked for install and that it can be installed + if (not P.Protect() && not P.InstPolicyBroken() && not P.InstBroken()) + { + if (P.CandidateVer == Pkg.CurrentVer()) + { + if (P.InstallVer == 0) + return MarkKeep(Pkg, false, FromUser, Depth + 1); + return true; + } + else if (P.Mode == ModeInstall) + return true; + } + + // check if we are allowed to install the package + if (not IsInstallOk(Pkg, AutoInst, Depth, FromUser)) + return false; + + ActionGroup group(*this); + if (FromUser && not MarkInstall_StateChange(Pkg, AutoInst, FromUser)) + return false; + + bool const AutoSolve = AutoInst && _config->Find("APT::Solver", "internal") == "internal"; + bool const failEarly = not P.Protect() && not FromUser; + bool hasFailed = false; + + std::vector<pkgCache::DepIterator> toInstall, toRemove; + APT::PackageVector toUpgrade, delayedRemove; + if (AutoSolve) + { + VerIterator const PV = P.CandidateVerIter(*this); + if (unlikely(PV.end())) + return false; + if (not MarkInstall_CollectDependencies(*this, PV, toInstall, toRemove)) + return false; + + if (not MarkInstall_RemoveConflictsIfNotUpgradeable(*this, DebugAutoInstall, PV, Depth, toRemove, toUpgrade, delayedRemove, P.Protect(), FromUser)) + { + if (failEarly) + return false; + hasFailed = true; + } + if (not MarkInstall_CollectReverseDepends(*this, DebugAutoInstall, PV, Depth, toUpgrade)) + { + if (failEarly) + return false; + hasFailed = true; + } + } + + if (not FromUser && not MarkInstall_StateChange(Pkg, AutoInst, FromUser)) + return false; + + if (not AutoSolve) + return not hasFailed; + + if (DebugMarker) + std::clog << OutputInDepth(Depth) << "MarkInstall " << APT::PrettyPkg(this, Pkg) << " FU=" << FromUser << '\n'; + + class ScopedProtected + { + pkgDepCache::StateCache &P; + bool const already; + + public: + ScopedProtected(pkgDepCache::StateCache &p) : P{p}, already{P.Protect()} + { + if (not already) + P.iFlags |= Protected; + } + ~ScopedProtected() + { + if (not already) + P.iFlags &= (~Protected); + } + operator bool() noexcept { return already; } + } propagateProtected{PkgState[Pkg->ID]}; + + if (not MarkInstall_UpgradeOtherBinaries(*this, DebugAutoInstall, Depth, ForceImportantDeps, Pkg, P.CandidateVerIter(*this))) + return false; + if (not MarkInstall_UpgradeOrRemoveConflicts(*this, DebugAutoInstall, Depth, ForceImportantDeps, toUpgrade, propagateProtected, FromUser)) + { + if (failEarly) + { + MarkInstall_DiscardInstall(Pkg); + return false; + } + hasFailed = true; + } + + bool const MoveAutoBitToDependencies = [&]() { + VerIterator const PV = P.InstVerIter(*this); + if (unlikely(PV.end())) + return false; + if (PV->Section == 0 || (P.Flags & Flag::Auto) == Flag::Auto) + return false; + VerIterator const CurVer = Pkg.CurrentVer(); + if (not CurVer.end() && CurVer->Section != 0 && strcmp(CurVer.Section(), PV.Section()) != 0) + { + bool const CurVerInMoveSection = SectionInSubTree("APT::Move-Autobit-Sections", CurVer.Section()); + bool const InstVerInMoveSection = SectionInSubTree("APT::Move-Autobit-Sections", PV.Section()); + return (not CurVerInMoveSection && InstVerInMoveSection); + } + return false; + }(); + + APT::PackageVector toMoveAuto; + if (not MarkInstall_InstallDependencies(*this, DebugAutoInstall, DebugMarker, Pkg, Depth, ForceImportantDeps, toInstall, + MoveAutoBitToDependencies ? &toMoveAuto : nullptr, propagateProtected, FromUser)) + { + if (failEarly) + { + MarkInstall_DiscardInstall(Pkg); + return false; + } + hasFailed = true; + } + + for (auto const &R : delayedRemove) + { + if (not MarkDelete(R, false, Depth, false)) + { + if (failEarly) + { + MarkInstall_DiscardInstall(Pkg); + return false; + } + hasFailed = true; + } + } + + if (MoveAutoBitToDependencies) + { + if (DebugAutoInstall) + std::clog << OutputInDepth(Depth) << "Setting " << Pkg.FullName(false) << " as auto-installed, moving manual to its dependencies" << std::endl; + MarkAuto(Pkg, true); + for (auto const &InstPkg : toMoveAuto) + { + if (DebugAutoInstall) + std::clog << OutputInDepth(Depth) << "Setting " << InstPkg.FullName(false) << " NOT as auto-installed (dependency" + << " of " << Pkg.FullName(false) << " which is manual and in APT::Move-Autobit-Sections)\n"; + MarkAuto(InstPkg, false); + } + } + return not hasFailed; +} + /*}}}*/ +// DepCache::IsInstallOk - check if it is ok to install this package /*{{{*/ +// --------------------------------------------------------------------- +/* The default implementation checks if the installation of an M-A:same + package would lead us into a version-screw and if so forbids it. + dpkg holds are enforced by the private IsModeChangeOk */ +bool pkgDepCache::IsInstallOk(PkgIterator const &Pkg,bool AutoInst, + unsigned long Depth, bool FromUser) +{ + return IsInstallOkMultiArchSameVersionSynced(Pkg,AutoInst, Depth, FromUser) && + IsInstallOkDependenciesSatisfiableByCandidates(Pkg,AutoInst, Depth, FromUser); +} +bool pkgDepCache::IsInstallOkMultiArchSameVersionSynced(PkgIterator const &Pkg, + bool const /*AutoInst*/, unsigned long const Depth, bool const FromUser) +{ + if (FromUser == true) // as always: user is always right + return true; + + // if we have checked before and it was okay, it will still be okay + if (PkgState[Pkg->ID].Mode == ModeInstall && + PkgState[Pkg->ID].InstallVer == PkgState[Pkg->ID].CandidateVer) + return true; + + // ignore packages with none-M-A:same candidates + VerIterator const CandVer = PkgState[Pkg->ID].CandidateVerIter(*this); + if (unlikely(CandVer.end() == true) || CandVer == Pkg.CurrentVer() || + (CandVer->MultiArch & pkgCache::Version::Same) != pkgCache::Version::Same) + return true; + + GrpIterator const Grp = Pkg.Group(); + for (PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P)) + { + // not installed or self-check: fine by definition + if (P->CurrentVer == 0 || P == Pkg) + continue; + + // not having a candidate or being in sync + // (simple string-compare as stuff like '1' == '0:1-0' can't happen here) + VerIterator CV = PkgState[P->ID].CandidateVerIter(*this); + if (CV.end() == true || strcmp(CandVer.VerStr(), CV.VerStr()) == 0) + continue; + + // packages losing M-A:same can be out-of-sync + if ((CV->MultiArch & pkgCache::Version::Same) != pkgCache::Version::Same) + continue; + + // not downloadable means the package is obsolete, so allow out-of-sync + if (CV.Downloadable() == false) + continue; + + PkgState[Pkg->ID].iFlags |= AutoKept; + if (unlikely(DebugMarker == true)) + std::clog << OutputInDepth(Depth) << "Ignore MarkInstall of " << APT::PrettyPkg(this, Pkg) + << " as it is not in sync with its M-A:same sibling " << APT::PrettyPkg(this, P) + << " (" << CandVer.VerStr() << " != " << CV.VerStr() << ")" << std::endl; + return false; + } + + return true; +} +bool pkgDepCache::IsInstallOkDependenciesSatisfiableByCandidates(PkgIterator const &Pkg, + bool const AutoInst, unsigned long const Depth, bool const /*FromUser*/) +{ + if (AutoInst == false) + return true; + + VerIterator const CandVer = PkgState[Pkg->ID].CandidateVerIter(*this); + if (unlikely(CandVer.end() == true) || CandVer == Pkg.CurrentVer()) + return true; + + for (DepIterator Dep = CandVer.DependsList(); Dep.end() != true;) + { + DepIterator Start = Dep; + bool foundSolution = false; + unsigned Ors = 0; + // Is it possible to satisfy this dependency? + for (bool LastOR = true; not Dep.end() && LastOR; ++Dep, ++Ors) + { + LastOR = (Dep->CompareOp & Dep::Or) == Dep::Or; + + if ((DepState[Dep->ID] & (DepInstall | DepCVer)) != 0) + foundSolution = true; + } + + if (foundSolution || not Start.IsCritical() || Start.IsNegative()) + continue; + + if (DebugAutoInstall == true) + std::clog << OutputInDepth(Depth) << APT::PrettyDep(this, Start) << " can't be satisfied!" << std::endl; + + // the dependency is critical, but can't be installed, so discard the candidate + // as the problemresolver will trip over it otherwise trying to install it (#735967) + StateCache &State = PkgState[Pkg->ID]; + if (not State.Protect()) + { + if (Pkg->CurrentVer != 0) + SetCandidateVersion(Pkg.CurrentVer()); + else + State.CandidateVer = nullptr; + if (not State.Delete()) + { + State.Mode = ModeKeep; + State.Update(Pkg, *this); + } + } + return false; + } + + return true; +} + /*}}}*/ +// DepCache::SetReInstall - Set the reinstallation flag /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgDepCache::SetReInstall(PkgIterator const &Pkg,bool To) +{ + if (unlikely(Pkg.end() == true)) + return; + + APT::PackageList pkglist; + if (Pkg->CurrentVer != 0 && + (Pkg.CurrentVer()-> MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same) + { + pkgCache::GrpIterator Grp = Pkg.Group(); + for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P)) + { + if (P->CurrentVer != 0) + pkglist.insert(P); + } + } + else + pkglist.insert(Pkg); + + ActionGroup group(*this); + + for (APT::PackageList::const_iterator Pkg = pkglist.begin(); Pkg != pkglist.end(); ++Pkg) + { + RemoveSizes(Pkg); + RemoveStates(Pkg); + + StateCache &P = PkgState[Pkg->ID]; + if (To == true) + P.iFlags |= ReInstall; + else + P.iFlags &= ~ReInstall; + + AddStates(Pkg); + AddSizes(Pkg); + } +} + /*}}}*/ +pkgCache::VerIterator pkgDepCache::GetCandidateVersion(PkgIterator const &Pkg)/*{{{*/ +{ + return PkgState[Pkg->ID].CandidateVerIter(*this); +} + /*}}}*/ +// DepCache::SetCandidateVersion - Change the candidate version /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgDepCache::SetCandidateVersion(VerIterator TargetVer) +{ + pkgCache::PkgIterator Pkg = TargetVer.ParentPkg(); + StateCache &P = PkgState[Pkg->ID]; + + if (P.CandidateVer == TargetVer) + return; + + ActionGroup group(*this); + + RemoveSizes(Pkg); + RemoveStates(Pkg); + + if (P.CandidateVer == P.InstallVer && P.Install() == true) + P.InstallVer = (Version *)TargetVer; + P.CandidateVer = (Version *)TargetVer; + P.Update(Pkg,*this); + + AddStates(Pkg); + Update(Pkg); + AddSizes(Pkg); + +} + /*}}}*/ +// DepCache::SetCandidateRelease - Change the candidate version /*{{{*/ +// --------------------------------------------------------------------- +/* changes the candidate of a package and walks over all its dependencies + to check if it needs to change the candidate of the dependency, too, + to reach a installable versionstate */ +bool pkgDepCache::SetCandidateRelease(pkgCache::VerIterator TargetVer, + std::string const &TargetRel) +{ + std::list<std::pair<pkgCache::VerIterator, pkgCache::VerIterator> > Changed; + return SetCandidateRelease(TargetVer, TargetRel, Changed); +} +bool pkgDepCache::SetCandidateRelease(pkgCache::VerIterator TargetVer, + std::string const &TargetRel, + std::list<std::pair<pkgCache::VerIterator, pkgCache::VerIterator> > &Changed) +{ + ActionGroup group(*this); + SetCandidateVersion(TargetVer); + + if (TargetRel == "installed" || TargetRel == "candidate") // both doesn't make sense in this context + return true; + + pkgVersionMatch Match(TargetRel, pkgVersionMatch::Release); + // save the position of the last element we will not undo - if we have to + std::list<std::pair<pkgCache::VerIterator, pkgCache::VerIterator> >::iterator newChanged = --(Changed.end()); + + for (pkgCache::DepIterator D = TargetVer.DependsList(); D.end() == false; ++D) + { + if (D->Type != pkgCache::Dep::PreDepends && D->Type != pkgCache::Dep::Depends && + ((D->Type != pkgCache::Dep::Recommends && D->Type != pkgCache::Dep::Suggests) || + IsImportantDep(D) == false)) + continue; + + // walk over an or-group and check if we need to do anything + // for simpilicity no or-group is handled as a or-group including one dependency + pkgCache::DepIterator Start = D; + bool itsFine = false; + for (bool stillOr = true; stillOr == true; ++Start) + { + stillOr = (Start->CompareOp & Dep::Or) == Dep::Or; + pkgCache::PkgIterator const P = Start.TargetPkg(); + // virtual packages can't be a solution + if (P.end() == true || (P->ProvidesList == 0 && P->VersionList == 0)) + continue; + // if its already installed, check if this one is good enough + pkgCache::VerIterator const Now = P.CurrentVer(); + if (Now.end() == false && Start.IsSatisfied(Now)) + { + itsFine = true; + break; + } + pkgCache::VerIterator const Cand = PkgState[P->ID].CandidateVerIter(*this); + // no versioned dependency - but is it installable? + if (Start.TargetVer() == 0 || Start.TargetVer()[0] == '\0') + { + // Check if one of the providers is installable + if (P->ProvidesList != 0) + { + pkgCache::PrvIterator Prv = P.ProvidesList(); + for (; Prv.end() == false; ++Prv) + { + pkgCache::VerIterator const C = PkgState[Prv.OwnerPkg()->ID].CandidateVerIter(*this); + if (C.end() == true || C != Prv.OwnerVer() || + (VersionState(C.DependsList(), DepInstall, DepCandMin, DepCandPolicy) & DepCandMin) != DepCandMin) + continue; + break; + } + if (Prv.end() == true) + continue; + } + // no providers, so check if we have an installable candidate version + else if (Cand.end() == true || + (VersionState(Cand.DependsList(), DepInstall, DepCandMin, DepCandPolicy) & DepCandMin) != DepCandMin) + continue; + itsFine = true; + break; + } + if (Cand.end() == true) + continue; + // check if the current candidate is enough for the versioned dependency - and installable? + if (Start.IsSatisfied(Cand) == true && + (VersionState(Cand.DependsList(), DepInstall, DepCandMin, DepCandPolicy) & DepCandMin) == DepCandMin) + { + itsFine = true; + break; + } + } + + if (itsFine == true) { + // something in the or-group was fine, skip all other members + for (; (D->CompareOp & Dep::Or) == Dep::Or; ++D); + continue; + } + + // walk again over the or-group and check each if a candidate switch would help + itsFine = false; + for (bool stillOr = true; stillOr == true; ++D) + { + stillOr = (D->CompareOp & Dep::Or) == Dep::Or; + // changing candidate will not help if the dependency is not versioned + if (D.TargetVer() == 0 || D.TargetVer()[0] == '\0') + { + if (stillOr == true) + continue; + break; + } + + pkgCache::VerIterator V; + if (TargetRel == "newest") + V = D.TargetPkg().VersionList(); + else + V = Match.Find(D.TargetPkg()); + + // check if the version from this release could satisfy the dependency + if (V.end() == true || D.IsSatisfied(V) == false) + { + if (stillOr == true) + continue; + break; + } + + pkgCache::VerIterator oldCand = PkgState[D.TargetPkg()->ID].CandidateVerIter(*this); + if (V == oldCand) + { + // Do we already touched this Version? If so, their versioned dependencies are okay, no need to check again + for (std::list<std::pair<pkgCache::VerIterator, pkgCache::VerIterator> >::const_iterator c = Changed.begin(); + c != Changed.end(); ++c) + { + if (c->first->ParentPkg != V->ParentPkg) + continue; + itsFine = true; + break; + } + } + + if (itsFine == false) + { + // change the candidate + Changed.emplace_back(V, TargetVer); + if (SetCandidateRelease(V, TargetRel, Changed) == false) + { + if (stillOr == false) + break; + // undo the candidate changing + SetCandidateVersion(oldCand); + Changed.pop_back(); + continue; + } + itsFine = true; + } + + // something in the or-group was fine, skip all other members + for (; (D->CompareOp & Dep::Or) == Dep::Or; ++D); + break; + } + + if (itsFine == false && (D->Type == pkgCache::Dep::PreDepends || D->Type == pkgCache::Dep::Depends)) + { + // undo all changes which aren't lead to a solution + for (std::list<std::pair<pkgCache::VerIterator, pkgCache::VerIterator> >::const_iterator c = ++newChanged; + c != Changed.end(); ++c) + SetCandidateVersion(c->first); + Changed.erase(newChanged, Changed.end()); + return false; + } + } + return true; +} + /*}}}*/ +// DepCache::MarkAuto - set the Auto flag for a package /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgDepCache::MarkAuto(const PkgIterator &Pkg, bool Auto) +{ + StateCache &state = PkgState[Pkg->ID]; + + ActionGroup group(*this); + + if(Auto) + state.Flags |= Flag::Auto; + else + state.Flags &= ~Flag::Auto; +} + /*}}}*/ +// StateCache::Update - Compute the various static display things /*{{{*/ +// --------------------------------------------------------------------- +/* This is called whenever the Candidate version changes. */ +void pkgDepCache::StateCache::Update(PkgIterator Pkg,pkgCache &Cache) +{ + // Some info + VerIterator Ver = CandidateVerIter(Cache); + + // Use a null string or the version string + if (Ver.end() == true) + CandVersion = ""; + else + CandVersion = Ver.VerStr(); + + // Find the current version + if (Pkg->CurrentVer != 0) + CurVersion = Pkg.CurrentVer().VerStr(); + else + CurVersion = ""; + + // Figure out if its up or down or equal + if (Pkg->CurrentVer == 0 || Pkg->VersionList == 0 || CandidateVer == 0) + Status = 2; + else + Status = Ver.CompareVer(Pkg.CurrentVer()); +} + /*}}}*/ +// Policy::GetCandidateVer - Returns the Candidate install version /*{{{*/ +// --------------------------------------------------------------------- +/* The default just returns the highest available version that is not + a source and automatic. */ +pkgCache::VerIterator pkgDepCache::Policy::GetCandidateVer(PkgIterator const &Pkg) +{ + /* Not source/not automatic versions cannot be a candidate version + unless they are already installed */ + VerIterator Last; + + for (VerIterator I = Pkg.VersionList(); I.end() == false; ++I) + { + if (Pkg.CurrentVer() == I) + return I; + + for (VerFileIterator J = I.FileList(); J.end() == false; ++J) + { + if (J.File().Flagged(Flag::NotSource)) + continue; + + /* Stash the highest version of a not-automatic source, we use it + if there is nothing better */ + if (J.File().Flagged(Flag::NotAutomatic) || + J.File().Flagged(Flag::ButAutomaticUpgrades)) + { + if (Last.end() == true) + Last = I; + continue; + } + + return I; + } + } + + return Last; +} + /*}}}*/ +// Policy::IsImportantDep - True if the dependency is important /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgDepCache::Policy::IsImportantDep(DepIterator const &Dep) const +{ + if(Dep.IsCritical()) + return true; + else if(Dep->Type == pkgCache::Dep::Recommends) + { + if (InstallRecommends) + return true; + // we support a special mode to only install-recommends for certain + // sections + // FIXME: this is a meant as a temporary solution until the + // recommends are cleaned up + const char *sec = Dep.ParentVer().Section(); + if (sec && SectionInSubTree("APT::Install-Recommends-Sections", sec)) + return true; + } + else if(Dep->Type == pkgCache::Dep::Suggests) + return InstallSuggests; + + return false; +} + /*}}}*/ +// Policy::GetPriority - Get the priority of the package pin /*{{{*/ +APT_PURE signed short pkgDepCache::Policy::GetPriority(pkgCache::PkgIterator const &/*Pkg*/) +{ return 0; } +APT_PURE signed short pkgDepCache::Policy::GetPriority(pkgCache::VerIterator const &/*Ver*/, bool /*ConsiderFiles*/) +{ return 0; } +APT_PURE signed short pkgDepCache::Policy::GetPriority(pkgCache::PkgFileIterator const &/*File*/) +{ return 0; } + /*}}}*/ +pkgDepCache::InRootSetFunc *pkgDepCache::GetRootSetFunc() /*{{{*/ +{ + DefaultRootSetFunc *f = new DefaultRootSetFunc2(&GetCache()); + if (f->wasConstructedSuccessfully()) + return f; + else + { + delete f; + return NULL; + } +} + +pkgDepCache::InRootSetFunc *pkgDepCache::GetCachedRootSetFunc() +{ + if (d->inRootSetFunc == nullptr) + d->inRootSetFunc.reset(GetRootSetFunc()); + return d->inRootSetFunc.get(); +} + /*}}}*/ +bool pkgDepCache::MarkFollowsRecommends() /*{{{*/ +{ + return _config->FindB("APT::AutoRemove::RecommendsImportant", true); +} + /*}}}*/ +bool pkgDepCache::MarkFollowsSuggests() /*{{{*/ +{ + return _config->FindB("APT::AutoRemove::SuggestsImportant", true); +} + /*}}}*/ +static bool IsPkgInBoringState(pkgCache::PkgIterator const &Pkg, pkgDepCache::StateCache const * const PkgState)/*{{{*/ +{ + if (Pkg->CurrentVer == 0) + { + if (PkgState[Pkg->ID].Keep()) + return true; + } + else + { + if (PkgState[Pkg->ID].Delete()) + return true; + } + return false; +} + /*}}}*/ +// MarkPackage - mark a single package in Mark-and-Sweep /*{{{*/ +static bool MarkPackage(pkgCache::PkgIterator const &Pkg, + pkgCache::VerIterator const &Ver, + bool const follow_recommends, + bool const follow_suggests, + bool const debug_autoremove, + std::string_view const reason, + size_t const Depth, + pkgCache &Cache, + pkgDepCache &DepCache, + pkgDepCache::StateCache *const PkgState, + std::vector<bool> &fullyExplored, + std::unique_ptr<APT::CacheFilter::Matcher> &IsAVersionedKernelPackage, + std::unique_ptr<APT::CacheFilter::Matcher> &IsProtectedKernelPackage) +{ + if (Ver.end() || PkgState[Pkg->ID].Marked) + return true; + + if (IsPkgInBoringState(Pkg, PkgState)) + { + fullyExplored[Pkg->ID] = true; + return true; + } + + // we are not trying too hard… + if (unlikely(Depth > 3000)) + return false; + + PkgState[Pkg->ID].Marked = true; + if(debug_autoremove) + std::clog << "Marking: " << Pkg.FullName() << " " << Ver.VerStr() + << " (" << reason << ")" << std::endl; + + auto const sort_by_source_version = [](pkgCache::VerIterator const &A, pkgCache::VerIterator const &B) { + auto const verret = A.Cache()->VS->CmpVersion(A.SourceVerStr(), B.SourceVerStr()); + if (verret != 0) + return verret < 0; + return A->ID < B->ID; + }; + + for (auto D = Ver.DependsList(); not D.end(); ++D) + { + auto const T = D.TargetPkg(); + if (T.end() || fullyExplored[T->ID]) + continue; + + if (D->Type != pkgCache::Dep::Depends && + D->Type != pkgCache::Dep::PreDepends && + (not follow_recommends || D->Type != pkgCache::Dep::Recommends) && + (not follow_suggests || D->Type != pkgCache::Dep::Suggests)) + continue; + + bool unsatisfied_choice = false; + std::unordered_map<std::string, APT::VersionVector> providers_by_source; + // collect real part + if (not IsPkgInBoringState(T, PkgState)) + { + auto const TV = (PkgState[T->ID].Install()) ? PkgState[T->ID].InstVerIter(DepCache) : T.CurrentVer(); + if (likely(not TV.end())) + { + if (not D.IsSatisfied(TV)) + unsatisfied_choice = true; + else + providers_by_source[TV.SourcePkgName()].push_back(TV); + } + } + if (providers_by_source.empty() && not unsatisfied_choice) + PkgState[T->ID].Marked = true; + // collect virtual part + for (auto Prv = T.ProvidesList(); not Prv.end(); ++Prv) + { + auto const PP = Prv.OwnerPkg(); + if (IsPkgInBoringState(PP, PkgState)) + continue; + + // we want to ignore provides from uninteresting versions + auto const PV = (PkgState[PP->ID].Install()) ? + PkgState[PP->ID].InstVerIter(DepCache) : PP.CurrentVer(); + if (unlikely(PV.end()) || PV != Prv.OwnerVer()) + continue; + + if (not D.IsSatisfied(Prv)) + unsatisfied_choice = true; + else + providers_by_source[PV.SourcePkgName()].push_back(PV); + } + // only latest binary package of a source package is marked instead of all + for (auto &providers : providers_by_source) + { + auto const highestSrcVer = (*std::max_element(providers.second.begin(), providers.second.end(), sort_by_source_version)).SourceVerStr(); + providers.second.erase(std::remove_if(providers.second.begin(), providers.second.end(), [&](auto const &V) { return strcmp(highestSrcVer, V.SourceVerStr()) != 0; }), providers.second.end()); + // if the provider is a versioned kernel package mark them only for protected kernels + if (providers.second.size() == 1) + continue; + if (not IsAVersionedKernelPackage) + IsAVersionedKernelPackage = [&]() -> std::unique_ptr<APT::CacheFilter::Matcher> { + auto const patterns = _config->FindVector("APT::VersionedKernelPackages"); + if (patterns.empty()) + return std::make_unique<APT::CacheFilter::FalseMatcher>(); + std::ostringstream regex; + regex << '^'; + std::copy(patterns.begin(), patterns.end() - 1, std::ostream_iterator<std::string>(regex, "-.*$|^")); + regex << patterns.back() << "-.*$"; + return std::make_unique<APT::CacheFilter::PackageNameMatchesRegEx>(regex.str()); + }(); + if (not std::all_of(providers.second.begin(), providers.second.end(), [&](auto const &Prv) { return (*IsAVersionedKernelPackage)(Prv.ParentPkg()); })) + continue; + // … if there is at least one for protected kernels installed + if (not IsProtectedKernelPackage) + IsProtectedKernelPackage = APT::KernelAutoRemoveHelper::GetProtectedKernelsFilter(&Cache); + if (not std::any_of(providers.second.begin(), providers.second.end(), [&](auto const &Prv) { return (*IsProtectedKernelPackage)(Prv.ParentPkg()); })) + continue; + providers.second.erase(std::remove_if(providers.second.begin(), providers.second.end(), + [&](auto const &Prv) { return not((*IsProtectedKernelPackage)(Prv.ParentPkg())); }), + providers.second.end()); + } + + if (not unsatisfied_choice) + fullyExplored[T->ID] = true; + for (auto const &providers : providers_by_source) + { + for (auto const &PV : providers.second) + { + auto const PP = PV.ParentPkg(); + if (debug_autoremove) + std::clog << "Following dep: " << APT::PrettyDep(&DepCache, D) + << ", provided by " << PP.FullName() << " " << PV.VerStr() + << " (" << providers_by_source.size() << "/" << providers.second.size() << ")\n"; + if (not MarkPackage(PP, PV, follow_recommends, follow_suggests, debug_autoremove, + "Dependency", Depth + 1, Cache, DepCache, PkgState, fullyExplored, + IsAVersionedKernelPackage, IsProtectedKernelPackage)) + return false; + } + } + } + return true; +} + /*}}}*/ +// pkgDepCache::MarkRequired - the main mark algorithm /*{{{*/ +bool pkgDepCache::MarkRequired(InRootSetFunc &userFunc) +{ + if (_config->Find("APT::Solver", "internal") != "internal") + return true; + + // init the states + auto const PackagesCount = Head().PackageCount; + for(auto i = decltype(PackagesCount){0}; i < PackagesCount; ++i) + { + PkgState[i].Marked = false; + PkgState[i].Garbage = false; + } + std::vector<bool> fullyExplored(PackagesCount, false); + + bool const debug_autoremove = _config->FindB("Debug::pkgAutoRemove", false); + if (debug_autoremove) + for(PkgIterator p = PkgBegin(); !p.end(); ++p) + if(PkgState[p->ID].Flags & Flag::Auto) + std::clog << "AutoDep: " << p.FullName() << std::endl; + + bool const follow_recommends = MarkFollowsRecommends(); + bool const follow_suggests = MarkFollowsSuggests(); + + // do the mark part, this is the core bit of the algorithm + for (PkgIterator P = PkgBegin(); !P.end(); ++P) + { + if (PkgState[P->ID].Marked || IsPkgInBoringState(P, PkgState)) + continue; + + std::string_view reason; + if ((PkgState[P->ID].Flags & Flag::Auto) == 0) + reason = "Manual-Installed"; + else if (P->Flags & Flag::Essential) + reason = "Essential"; + else if (P->Flags & Flag::Important) + reason = "Important"; + else if (P->CurrentVer != 0 && P.CurrentVer()->Priority == pkgCache::State::Required) + reason = "Required"; + else if (userFunc.InRootSet(P)) + reason = "Blacklisted [APT::NeverAutoRemove]"; + else if (not IsModeChangeOk(*this, ModeGarbage, P, 0, false, DebugMarker)) + reason = "Hold"; + else + continue; + + pkgCache::VerIterator const PV = (PkgState[P->ID].Install()) ? PkgState[P->ID].InstVerIter(*this) : P.CurrentVer(); + if (not MarkPackage(P, PV, follow_recommends, follow_suggests, debug_autoremove, + reason, 0, *Cache, *this, PkgState, fullyExplored, + d->IsAVersionedKernelPackage, d->IsProtectedKernelPackage)) + return false; + } + return true; +} + /*}}}*/ +bool pkgDepCache::Sweep() /*{{{*/ +{ + bool debug_autoremove = _config->FindB("Debug::pkgAutoRemove",false); + + // do the sweep + for(PkgIterator p=PkgBegin(); !p.end(); ++p) + { + StateCache &state=PkgState[p->ID]; + + // skip required packages + if (!p.CurrentVer().end() && + (p.CurrentVer()->Priority == pkgCache::State::Required)) + continue; + + // if it is not marked and it is installed, it's garbage + if(!state.Marked && (!p.CurrentVer().end() || state.Install())) + { + state.Garbage=true; + if(debug_autoremove) + std::clog << "Garbage: " << p.FullName() << std::endl; + } + } + + return true; +} + /*}}}*/ +// DepCache::MarkAndSweep /*{{{*/ +bool pkgDepCache::MarkAndSweep(InRootSetFunc &rootFunc) +{ + return MarkRequired(rootFunc) && Sweep(); +} +bool pkgDepCache::MarkAndSweep() +{ + InRootSetFunc *f(GetCachedRootSetFunc()); + if (f != NULL) + return MarkAndSweep(*f); + else + return false; +} + /*}}}*/ diff --git a/apt-pkg/depcache.h b/apt-pkg/depcache.h new file mode 100644 index 0000000..be27b1d --- /dev/null +++ b/apt-pkg/depcache.h @@ -0,0 +1,509 @@ +// -*- mode: c++; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + DepCache - Dependency Extension data for the cache + + This class stores the cache data and a set of extension structures for + monitoring the current state of all the packages. It also generates and + caches the 'install' state of many things. This refers to the state of the + package after an install has been run. + + The StateCache::State field can be -1,0,1,2 which is <,=,>,no current. + StateCache::Mode is which of the 3 fields is active. + + This structure is important to support the readonly status of the cache + file. When the data is saved the cache will be refereshed from our + internal rep and written to disk. Then the actual persistent data + files will be put on the disk. + + Each dependency is compared against 3 target versions to produce to + 3 dependency results. + Now - Compared using the Currently install version + Install - Compared using the install version (final state) + CVer - (Candidate Version) Compared using the Candidate Version + The candidate and now results are used to decide whether a package + should be automatically installed or if it should be left alone. + + Remember, the Candidate Version is selected based on the distribution + settings for the Package. The Install Version is selected based on the + state (Delete, Keep, Install) field and can be either the Current Version + or the Candidate version. + + The Candidate version is what is shown the 'Install Version' field. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_DEPCACHE_H +#define PKGLIB_DEPCACHE_H + +#include <apt-pkg/configuration.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> + +#include <stddef.h> + +#include <list> +#include <memory> +#include <string> +#include <utility> + + +class OpProgress; +class pkgVersioningSystem; +namespace APT +{ +template <class Container> +class PackageContainer; +using PackageVector = PackageContainer<std::vector<pkgCache::PkgIterator>>; +} // namespace APT + +class APT_PUBLIC pkgDepCache : protected pkgCache::Namespace +{ + public: + + /** \brief An arbitrary predicate on packages. */ + class APT_PUBLIC InRootSetFunc + { + public: + virtual bool InRootSet(const pkgCache::PkgIterator &/*pkg*/) {return false;}; + virtual ~InRootSetFunc() {}; + }; + + private: + /** \brief Update the Marked field of all packages. + * + * Each package's StateCache::Marked field will be set to \b true + * if and only if it can be reached from the root set. By + * default, the root set consists of the set of manually installed + * or essential packages, but it can be extended using the + * parameter #rootFunc. + * + * \param rootFunc A callback that can be used to add extra + * packages to the root set. + * + * \return \b false if an error occurred. + */ + bool MarkRequired(InRootSetFunc &rootFunc); + + /** \brief Set the StateCache::Garbage flag on all packages that + * should be removed. + * + * Packages that were not marked by the last call to #MarkRequired + * are tested to see whether they are actually garbage. If so, + * they are marked as such. + * + * \return \b false if an error occurred. + */ + bool Sweep(); + + public: + + // These flags are used in DepState + enum DepFlags {DepNow = (1 << 0),DepInstall = (1 << 1),DepCVer = (1 << 2), + DepGNow = (1 << 3),DepGInstall = (1 << 4),DepGCVer = (1 << 5)}; + + // These flags are used in StateCache::DepState + enum DepStateFlags {DepNowPolicy = (1 << 0), DepNowMin = (1 << 1), + DepInstPolicy = (1 << 2), DepInstMin = (1 << 3), + DepCandPolicy = (1 << 4), DepCandMin = (1 << 5)}; + + // These flags are used in StateCache::iFlags + enum InternalFlags {AutoKept = (1 << 0), Purge = (1 << 1), ReInstall = (1 << 2), Protected = (1 << 3)}; + + enum VersionTypes {NowVersion, InstallVersion, CandidateVersion}; + enum ModeList {ModeDelete = 0, ModeKeep = 1, ModeInstall = 2, ModeGarbage = 3}; + + /** \brief Represents an active action group. + * + * An action group is a group of actions that are currently being + * performed. While an active group is active, certain routine + * clean-up actions that would normally be performed after every + * cache operation are delayed until the action group is + * completed. This is necessary primarily to avoid inefficiencies + * when modifying a large number of packages at once. + * + * This class represents an active action group. Creating an + * instance will create an action group; destroying one will + * destroy the corresponding action group. + * + * The following operations are suppressed by this class: + * + * - Keeping the Marked and Garbage flags up to date. + * + * \note This can be used in the future to easily accumulate + * atomic actions for undo or to display "what apt did anyway"; + * e.g., change the counter of how many action groups are active + * to a std::set of pointers to them and use those to store + * information about what happened in a group in the group. + */ + class APT_PUBLIC ActionGroup + { + void * const d; + pkgDepCache &cache; + + bool released; + + /** Action groups are noncopyable. */ + APT_HIDDEN ActionGroup(const ActionGroup &other); + public: + /** \brief Create a new ActionGroup. + * + * \param cache The cache that this ActionGroup should + * manipulate. + * + * As long as this object exists, no automatic cleanup + * operations will be undertaken. + */ + explicit ActionGroup(pkgDepCache &cache); + + /** \brief Clean up the action group before it is destroyed. + * + * If it is destroyed later, no second cleanup will be run. + */ + void release(); + + /** \brief Destroy the action group. + * + * If this is the last action group, the automatic cache + * cleanup operations will be undertaken. + */ + virtual ~ActionGroup(); + }; + + /** \brief Returns \b true for packages matching a regular + * expression in APT::NeverAutoRemove. + */ + class APT_PUBLIC DefaultRootSetFunc : public InRootSetFunc, public Configuration::MatchAgainstConfig + { + public: + DefaultRootSetFunc() : Configuration::MatchAgainstConfig("APT::NeverAutoRemove") {}; + virtual ~DefaultRootSetFunc() {}; + + bool InRootSet(const pkgCache::PkgIterator &pkg) APT_OVERRIDE { return pkg.end() == false && Match(pkg.Name()); }; + }; + + struct APT_PUBLIC StateCache + { + // text versions of the two version fields + const char *CandVersion; + const char *CurVersion; + + // Pointer to the candidate install version. + Version *CandidateVer; + + // Pointer to the install version. + Version *InstallVer; + + // Copy of Package::Flags + unsigned short Flags; + unsigned short iFlags; // Internal flags + + /** \brief \b true if this package can be reached from the root set. */ + bool Marked; + + /** \brief \b true if this package is unused and should be removed. + * + * This differs from !#Marked, because it is possible that some + * unreachable packages will be protected from becoming + * garbage. + */ + bool Garbage; + + // Various tree indicators + signed char Status; // -1,0,1,2 + unsigned char Mode; // ModeList + unsigned char DepState; // DepState Flags + + // Update of candidate version + void Update(PkgIterator Pkg,pkgCache &Cache); + + // Various test members for the current status of the package + inline bool NewInstall() const {return Status == 2 && Mode == ModeInstall;}; + inline bool Delete() const {return Mode == ModeDelete;}; + inline bool Purge() const {return Delete() == true && (iFlags & pkgDepCache::Purge) == pkgDepCache::Purge; }; + inline bool Keep() const {return Mode == ModeKeep;}; + inline bool Protect() const {return (iFlags & Protected) == Protected;}; + inline bool Upgrade() const {return Status > 0 && Mode == ModeInstall;}; + inline bool Upgradable() const {return Status >= 1 && CandidateVer != NULL;}; + inline bool Downgrade() const {return Status < 0 && Mode == ModeInstall;}; + inline bool Held() const {return Status != 0 && Keep();}; + inline bool NowBroken() const {return (DepState & DepNowMin) != DepNowMin;}; + inline bool NowPolicyBroken() const {return (DepState & DepNowPolicy) != DepNowPolicy;}; + inline bool InstBroken() const {return (DepState & DepInstMin) != DepInstMin;}; + inline bool InstPolicyBroken() const {return (DepState & DepInstPolicy) != DepInstPolicy;}; + inline bool Install() const {return Mode == ModeInstall;}; + inline bool ReInstall() const {return Delete() == false && (iFlags & pkgDepCache::ReInstall) == pkgDepCache::ReInstall;}; + inline VerIterator InstVerIter(pkgCache &Cache) + {return VerIterator(Cache,InstallVer);}; + inline VerIterator CandidateVerIter(pkgCache &Cache) + {return VerIterator(Cache,CandidateVer);}; + }; + + // Helper functions + void BuildGroupOrs(VerIterator const &V); + void UpdateVerState(PkgIterator const &Pkg); + + // User Policy control + class APT_PUBLIC Policy + { + public: + Policy() { + InstallRecommends = _config->FindB("APT::Install-Recommends", false); + InstallSuggests = _config->FindB("APT::Install-Suggests", false); + } + + virtual VerIterator GetCandidateVer(PkgIterator const &Pkg); + virtual bool IsImportantDep(DepIterator const &Dep) const; + virtual signed short GetPriority(PkgIterator const &Pkg); + virtual signed short GetPriority(VerIterator const &Ver, bool ConsiderFiles=true); + virtual signed short GetPriority(PkgFileIterator const &File); + + virtual ~Policy() {}; + + private: + bool InstallRecommends; + bool InstallSuggests; + }; + + private: + /** The number of open "action groups"; certain post-action + * operations are suppressed if this number is > 0. + */ + int group_level; + + friend class ActionGroup; + public: + int IncreaseActionGroupLevel(); + int DecreaseActionGroupLevel(); + + protected: + + // State information + pkgCache *Cache; + StateCache *PkgState; + unsigned char *DepState; + + /** Stores the space changes after installation */ + signed long long iUsrSize; + /** Stores how much we need to download to get the packages */ + unsigned long long iDownloadSize; + unsigned long iInstCount; + unsigned long iDelCount; + unsigned long iKeepCount; + unsigned long iBrokenCount; + unsigned long iPolicyBrokenCount; + unsigned long iBadCount; + + bool DebugMarker; + bool DebugAutoInstall; + + Policy *delLocalPolicy; // For memory clean up.. + Policy *LocalPolicy; + + // Check for a matching provides + bool CheckDep(DepIterator const &Dep,int const Type,PkgIterator &Res); + inline bool CheckDep(DepIterator const &Dep,int const Type) + { + PkgIterator Res(*this,0); + return CheckDep(Dep,Type,Res); + } + + // Computes state information for deps and versions (w/o storing) + unsigned char DependencyState(DepIterator const &D); + unsigned char VersionState(DepIterator D,unsigned char const Check, + unsigned char const SetMin, + unsigned char const SetPolicy) const; + + // Recalculates various portions of the cache, call after changing something + void Update(DepIterator Dep); // Mostly internal + void Update(PkgIterator const &P); + + // Count manipulators + void AddSizes(const PkgIterator &Pkg, bool const Invert = false); + inline void RemoveSizes(const PkgIterator &Pkg) {AddSizes(Pkg, true);}; + void AddStates(const PkgIterator &Pkg, bool const Invert = false); + inline void RemoveStates(const PkgIterator &Pkg) {AddStates(Pkg,true);}; + + public: + + // Legacy.. We look like a pkgCache + inline operator pkgCache &() {return *Cache;}; + inline Header &Head() {return *Cache->HeaderP;}; + inline GrpIterator GrpBegin() {return Cache->GrpBegin();}; + inline PkgIterator PkgBegin() {return Cache->PkgBegin();}; + inline GrpIterator FindGrp(APT::StringView Name) {return Cache->FindGrp(Name);}; + inline PkgIterator FindPkg(APT::StringView Name) {return Cache->FindPkg(Name);}; + inline PkgIterator FindPkg(APT::StringView Name, APT::StringView Arch) {return Cache->FindPkg(Name, Arch);}; + + inline pkgCache &GetCache() {return *Cache;}; + inline pkgVersioningSystem &VS() {return *Cache->VS;}; + + inline bool IsImportantDep(DepIterator Dep) const {return LocalPolicy->IsImportantDep(Dep);}; + inline Policy &GetPolicy() {return *LocalPolicy;}; + + // Accessors + inline StateCache &operator [](PkgIterator const &I) {return PkgState[I->ID];}; + inline StateCache &operator [](PkgIterator const &I) const {return PkgState[I->ID];}; + inline unsigned char &operator [](DepIterator const &I) {return DepState[I->ID];}; + inline unsigned char const &operator [](DepIterator const &I) const {return DepState[I->ID];}; + + /** \return A function identifying packages in the root set other + * than manually installed packages and essential packages, or \b + * NULL if an error occurs. + * + * \todo Is this the best place for this function? Perhaps the + * settings for mark-and-sweep should be stored in a single + * external class? + */ + virtual InRootSetFunc *GetRootSetFunc(); + + /** This should return const really - do not delete. */ + InRootSetFunc *GetCachedRootSetFunc() APT_HIDDEN; + + /** \return \b true if the garbage collector should follow recommendations. + */ + virtual bool MarkFollowsRecommends(); + + /** \return \b true if the garbage collector should follow suggestions. + */ + virtual bool MarkFollowsSuggests(); + + /** \brief Update the Marked and Garbage fields of all packages. + * + * This routine is implicitly invoked after all state manipulators + * and when an ActionGroup is destroyed. It invokes the private + * MarkRequired() and Sweep() to do its dirty work. + * + * \param rootFunc A predicate that returns \b true for packages + * that should be added to the root set. + */ + bool MarkAndSweep(InRootSetFunc &rootFunc); + bool MarkAndSweep(); + + /** \name State Manipulators + */ + // @{ + bool MarkKeep(PkgIterator const &Pkg, bool Soft = false, + bool FromUser = true, unsigned long Depth = 0); + bool MarkDelete(PkgIterator const &Pkg, bool MarkPurge = false, + unsigned long Depth = 0, bool FromUser = true); + bool MarkInstall(PkgIterator const &Pkg,bool AutoInst = true, + unsigned long Depth = 0, bool FromUser = true, + bool ForceImportantDeps = false); + void MarkProtected(PkgIterator const &Pkg) { PkgState[Pkg->ID].iFlags |= Protected; }; + + void SetReInstall(PkgIterator const &Pkg,bool To); + + /** @return 'the' candidate version of a package + * + * The version returned is the version previously set explicitly via + * SetCandidate* methods like #SetCandidateVersion or if there wasn't one + * set the version as chosen via #Policy. + * + * @param Pkg is the package to return the candidate for + */ + pkgCache::VerIterator GetCandidateVersion(pkgCache::PkgIterator const &Pkg); + void SetCandidateVersion(VerIterator TargetVer); + bool SetCandidateRelease(pkgCache::VerIterator TargetVer, + std::string const &TargetRel); + /** Set the candidate version for dependencies too if needed. + * + * Sets not only the candidate version as SetCandidateVersion does, + * but walks also down the dependency tree and checks if it is required + * to set the candidate of the dependency to a version from the given + * release, too. + * + * \param TargetVer new candidate version of the package + * \param TargetRel try to switch to this release if needed + * \param[out] Changed a list of pairs consisting of the \b old + * version of the changed package and the version which + * required the switch of this dependency + * \return \b true if the switch was successful, \b false otherwise + */ + bool SetCandidateRelease(pkgCache::VerIterator TargetVer, + std::string const &TargetRel, + std::list<std::pair<pkgCache::VerIterator, pkgCache::VerIterator> > &Changed); + + /** Set the "is automatically installed" flag of Pkg. */ + void MarkAuto(const PkgIterator &Pkg, bool Auto); + // @} + + /** \return \b true if it's OK for MarkInstall to install + * the given package. + * + * The default implementation simply calls all IsInstallOk* + * method mentioned below. + * + * Overriding implementations should use the hold-state-flag to + * cache results from previous checks of this package - if possible. + * + * The parameters are the same as in the calling MarkInstall: + * \param Pkg the package that MarkInstall wants to install. + * \param AutoInst install this and all its dependencies + * \param Depth recursive deep of this Marker call + * \param FromUser was the install requested by the user? + */ + virtual bool IsInstallOk(const PkgIterator &Pkg,bool AutoInst = true, + unsigned long Depth = 0, bool FromUser = true); + + /** \return \b true if it's OK for MarkDelete to remove + * the given package. + * + * The default implementation simply calls all IsDeleteOk* + * method mentioned below, see also #IsInstallOk. + * + * The parameters are the same as in the calling MarkDelete: + * \param Pkg the package that MarkDelete wants to remove. + * \param MarkPurge should we purge instead of "only" remove? + * \param Depth recursive deep of this Marker call + * \param FromUser was the remove requested by the user? + */ + virtual bool IsDeleteOk(const PkgIterator &Pkg,bool MarkPurge = false, + unsigned long Depth = 0, bool FromUser = true); + + // read persistent states + bool readStateFile(OpProgress * const prog); + bool writeStateFile(OpProgress * const prog, bool const InstalledOnly=true); + + // Size queries + inline signed long long UsrSize() {return iUsrSize;}; + inline unsigned long long DebSize() {return iDownloadSize;}; + inline unsigned long DelCount() {return iDelCount;}; + inline unsigned long KeepCount() {return iKeepCount;}; + inline unsigned long InstCount() {return iInstCount;}; + inline unsigned long BrokenCount() {return iBrokenCount;}; + inline unsigned long PolicyBrokenCount() {return iPolicyBrokenCount;}; + inline unsigned long BadCount() {return iBadCount;}; + + bool Init(OpProgress * const Prog); + // Generate all state information + void Update(OpProgress * const Prog = 0); + + pkgDepCache(pkgCache * const Cache,Policy * const Plcy = 0); + virtual ~pkgDepCache(); + + bool CheckConsistency(char const *const msgtag = ""); + + protected: + // methods call by IsInstallOk + bool IsInstallOkMultiArchSameVersionSynced(PkgIterator const &Pkg, + bool const AutoInst, unsigned long const Depth, bool const FromUser); + bool IsInstallOkDependenciesSatisfiableByCandidates(PkgIterator const &Pkg, + bool const AutoInst, unsigned long const Depth, bool const FromUser); + + // methods call by IsDeleteOk + bool IsDeleteOkProtectInstallRequests(PkgIterator const &Pkg, + bool const rPurge, unsigned long const Depth, bool const FromUser); + + private: + struct Private; + Private *const d; + + APT_HIDDEN bool MarkInstall_StateChange(PkgIterator const &Pkg, bool AutoInst, bool FromUser); + APT_HIDDEN bool MarkInstall_DiscardInstall(PkgIterator const &Pkg); + + APT_HIDDEN void PerformDependencyPass(OpProgress * const Prog); +}; + +#endif diff --git a/apt-pkg/dirstream.cc b/apt-pkg/dirstream.cc new file mode 100644 index 0000000..d6cf0ab --- /dev/null +++ b/apt-pkg/dirstream.cc @@ -0,0 +1,118 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Directory Stream + + This class provides a simple basic extractor that can be used for + a number of purposes. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/dirstream.h> +#include <apt-pkg/error.h> + +#include <errno.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <unistd.h> +#include <apti18n.h> + /*}}}*/ + +// DirStream::DoItem - Process an item /*{{{*/ +// --------------------------------------------------------------------- +/* This is a very simple extractor, it does not deal with things like + overwriting directories with files and so on. */ +bool pkgDirStream::DoItem(Item &Itm,int &Fd) +{ + switch (Itm.Type) + { + case Item::File: + { + /* Open the output file, NDELAY is used to prevent this from + blowing up on device special files.. */ + int iFd = open(Itm.Name,O_NDELAY|O_WRONLY|O_CREAT|O_TRUNC|O_APPEND, + Itm.Mode); + if (iFd < 0) + return _error->Errno("open",_("Failed to write file %s"), + Itm.Name); + + // fchmod deals with umask and fchown sets the ownership + if (fchmod(iFd,Itm.Mode) != 0) + { + close(iFd); + return _error->Errno("fchmod",_("Failed to write file %s"), Itm.Name); + } + if (fchown(iFd,Itm.UID,Itm.GID) != 0 && errno != EPERM) + { + close(iFd); + return _error->Errno("fchown",_("Failed to write file %s"), Itm.Name); + } + Fd = iFd; + return true; + } + + case Item::HardLink: + case Item::SymbolicLink: + case Item::CharDevice: + case Item::BlockDevice: + case Item::Directory: + { + struct stat Buf; + // check if the dir is already there, if so return true + if (stat(Itm.Name,&Buf) == 0) + { + if(S_ISDIR(Buf.st_mode)) + return true; + // something else is there already, return false + return false; + } + // nothing here, create the dir + if(mkdir(Itm.Name,Itm.Mode) < 0) + return false; + return true; + } + case Item::FIFO: + break; + } + + return true; +} + /*}}}*/ +// DirStream::FinishedFile - Finished processing a file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgDirStream::FinishedFile(Item &Itm,int Fd) +{ + if (Fd < 0) + return true; + + /* Set the modification times. The only way it can fail is if someone + has futzed with our file, which is intolerable :> */ + struct timeval times[2]; + times[0].tv_sec = times[1].tv_sec = Itm.MTime; + times[0].tv_usec = times[1].tv_usec = 0; + if (utimes(Itm.Name, times) != 0) + _error->Errno("utimes", "Failed to set modification time for %s",Itm.Name); + + if (close(Fd) != 0) + return _error->Errno("close",_("Failed to close file %s"),Itm.Name); + return true; +} + /*}}}*/ +// DirStream::Fail - Failed processing a file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgDirStream::Fail(Item &/*Itm*/, int Fd) +{ + if (Fd < 0) + return true; + + close(Fd); + return false; +} + /*}}}*/ diff --git a/apt-pkg/dirstream.h b/apt-pkg/dirstream.h new file mode 100644 index 0000000..e5b226e --- /dev/null +++ b/apt-pkg/dirstream.h @@ -0,0 +1,57 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Directory Stream + + When unpacking the contents of the archive are passed into a directory + stream class for analysis and processing. The class controls all aspects + of actually writing the directory stream from disk. The low level + archive handlers are only responsible for decoding the archive format + and sending events (via method calls) to the specified directory + stream. + + When unpacking a real file the archive handler is passed back a file + handle to write the data to, this is to support strange + archives+unpacking methods. If that fd is -1 then the file data is + simply ignored. + + The provided defaults do the 'Right Thing' for a normal unpacking + process (ie 'tar') + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_DIRSTREAM_H +#define PKGLIB_DIRSTREAM_H + +#include <apt-pkg/macros.h> + +class APT_PUBLIC pkgDirStream +{ + public: + + // All possible information about a component + struct Item + { + enum Type_t {File, HardLink, SymbolicLink, CharDevice, BlockDevice, + Directory, FIFO} Type; + char *Name; + char *LinkTarget; + unsigned long Mode; + unsigned long UID; + unsigned long GID; + unsigned long long Size; + unsigned long MTime; + unsigned long Major; + unsigned long Minor; + }; + + virtual bool DoItem(Item &Itm,int &Fd); + virtual bool Fail(Item &Itm,int Fd); + virtual bool FinishedFile(Item &Itm,int Fd); + virtual bool Process(Item &/*Itm*/,const unsigned char * /*Data*/, + unsigned long long /*Size*/,unsigned long long /*Pos*/) {return true;}; + virtual ~pkgDirStream() {}; +}; + +#endif diff --git a/apt-pkg/edsp.cc b/apt-pkg/edsp.cc new file mode 100644 index 0000000..9d196ee --- /dev/null +++ b/apt-pkg/edsp.cc @@ -0,0 +1,1195 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + Set of methods to help writing and reading everything needed for EDSP + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/algorithms.h> +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/cacheset.h> +#include <apt-pkg/depcache.h> +#include <apt-pkg/edsp.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/packagemanager.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgsystem.h> +#include <apt-pkg/prettyprinters.h> +#include <apt-pkg/progress.h> +#include <apt-pkg/string_view.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile.h> + +#include <ctype.h> +#include <stddef.h> +#include <stdio.h> +#include <string.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <algorithm> +#include <array> +#include <limits> +#include <sstream> +#include <string> + +#include <apti18n.h> + /*}}}*/ + +using std::string; + +// we could use pkgCache::DepType and ::Priority, but these would be localized strings… +constexpr char const * const PrioMap[] = { + nullptr, "important", "required", "standard", + "optional", "extra" +}; +constexpr char const * const DepMap[] = { + nullptr, "Depends", "Pre-Depends", "Suggests", + "Recommends" , "Conflicts", "Replaces", + "Obsoletes", "Breaks", "Enhances" +}; + +// WriteOkay - varaidic helper to easily Write to a FileFd /*{{{*/ +static bool WriteOkay_fn(FileFd &) { return true; } +template<typename... Tail> static bool WriteOkay_fn(FileFd &output, APT::StringView data, Tail... more_data) +{ + return likely(output.Write(data.data(), data.length()) && WriteOkay_fn(output, more_data...)); +} +template<typename... Tail> static bool WriteOkay_fn(FileFd &output, unsigned int data, Tail... more_data) +{ + std::string number; + strprintf(number, "%d", data); + return likely(output.Write(number.data(), number.length()) && WriteOkay_fn(output, more_data...)); +} +template<typename... Data> static bool WriteOkay(bool &Okay, FileFd &output, Data&&... data) +{ + Okay = likely(Okay && WriteOkay_fn(output, std::forward<Data>(data)...)); + return Okay; +} +template<typename... Data> static bool WriteOkay(FileFd &output, Data&&... data) +{ + bool Okay = likely(output.Failed() == false); + return WriteOkay(Okay, output, std::forward<Data>(data)...); +} + /*}}}*/ +// WriteScenarioVersion /*{{{*/ +static bool WriteScenarioVersion(FileFd &output, pkgCache::PkgIterator const &Pkg, + pkgCache::VerIterator const &Ver) +{ + bool Okay = WriteOkay(output, "Package: ", Pkg.Name(), + "\nArchitecture: ", Ver.Arch(), + "\nVersion: ", Ver.VerStr()); + WriteOkay(Okay, output, "\nAPT-ID: ", Ver->ID); + if (Ver.PhasedUpdatePercentage() != 100) + WriteOkay(Okay, output, "\nPhased-Update-Percentage: ", Ver.PhasedUpdatePercentage()); + if ((Pkg->Flags & pkgCache::Flag::Essential) == pkgCache::Flag::Essential) + WriteOkay(Okay, output, "\nEssential: yes"); + if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed) + WriteOkay(Okay, output, "\nMulti-Arch: allowed"); + else if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign) + WriteOkay(Okay, output, "\nMulti-Arch: foreign"); + else if ((Ver->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same) + WriteOkay(Okay, output, "\nMulti-Arch: same"); + return Okay; +} + /*}}}*/ +// WriteScenarioDependency /*{{{*/ +static bool WriteScenarioDependency(FileFd &output, pkgCache::VerIterator const &Ver, bool const OnlyCritical) +{ + std::array<std::string, APT_ARRAY_SIZE(DepMap)> dependencies; + bool orGroup = false; + for (pkgCache::DepIterator Dep = Ver.DependsList(); Dep.end() == false; ++Dep) + { + if (Dep.IsImplicit() == true) + continue; + if (OnlyCritical && Dep.IsCritical() == false) + continue; + if (orGroup == false && dependencies[Dep->Type].empty() == false) + dependencies[Dep->Type].append(", "); + dependencies[Dep->Type].append(Dep.TargetPkg().Name()); + if (Dep->Version != 0) + dependencies[Dep->Type].append(" (").append(pkgCache::CompTypeDeb(Dep->CompareOp)).append(" ").append(Dep.TargetVer()).append(")"); + if ((Dep->CompareOp & pkgCache::Dep::Or) == pkgCache::Dep::Or) + { + dependencies[Dep->Type].append(" | "); + orGroup = true; + } + else + orGroup = false; + } + bool Okay = output.Failed() == false; + for (size_t i = 1; i < dependencies.size(); ++i) + if (dependencies[i].empty() == false) + WriteOkay(Okay, output, "\n", DepMap[i], ": ", dependencies[i]); + std::vector<std::string> provides; + for (auto Prv = Ver.ProvidesList(); not Prv.end(); ++Prv) + { + if (Prv.IsMultiArchImplicit()) + continue; + std::string provide = Prv.Name(); + if (Prv->ProvideVersion != 0) + provide.append(" (= ").append(Prv.ProvideVersion()).append(")"); + if ((Ver->MultiArch & pkgCache::Version::Foreign) != 0 && std::find(provides.cbegin(), provides.cend(), provide) != provides.cend()) + continue; + provides.emplace_back(std::move(provide)); + } + if (not provides.empty()) + { + std::ostringstream out; + std::copy(provides.begin(), provides.end() - 1, std::ostream_iterator<std::string>(out, ", ")); + out << provides.back(); + WriteOkay(Okay, output, "\nProvides: ", out.str()); + } + return WriteOkay(Okay, output, "\n"); +} + /*}}}*/ +// WriteScenarioLimitedDependency /*{{{*/ +static bool WriteScenarioLimitedDependency(FileFd &output, + pkgCache::VerIterator const &Ver, + std::vector<bool> const &pkgset, + bool const OnlyCritical) +{ + std::array<std::string, APT_ARRAY_SIZE(DepMap)> dependencies; + bool orGroup = false; + for (pkgCache::DepIterator Dep = Ver.DependsList(); Dep.end() == false; ++Dep) + { + if (Dep.IsImplicit() == true) + continue; + if (OnlyCritical && Dep.IsCritical() == false) + continue; + if (orGroup == false) + { + if (pkgset[Dep.TargetPkg()->ID] == false) + continue; + if (dependencies[Dep->Type].empty() == false) + dependencies[Dep->Type].append(", "); + } + else if (pkgset[Dep.TargetPkg()->ID] == false) + { + if ((Dep->CompareOp & pkgCache::Dep::Or) == pkgCache::Dep::Or) + continue; + dependencies[Dep->Type].erase(dependencies[Dep->Type].end()-3, dependencies[Dep->Type].end()); + orGroup = false; + continue; + } + dependencies[Dep->Type].append(Dep.TargetPkg().Name()); + if (Dep->Version != 0) + dependencies[Dep->Type].append(" (").append(pkgCache::CompTypeDeb(Dep->CompareOp)).append(" ").append(Dep.TargetVer()).append(")"); + if ((Dep->CompareOp & pkgCache::Dep::Or) == pkgCache::Dep::Or) + { + dependencies[Dep->Type].append(" | "); + orGroup = true; + } + else + orGroup = false; + } + bool Okay = output.Failed() == false; + for (size_t i = 1; i < dependencies.size(); ++i) + if (dependencies[i].empty() == false) + WriteOkay(Okay, output, "\n", DepMap[i], ": ", dependencies[i]); + string provides; + for (pkgCache::PrvIterator Prv = Ver.ProvidesList(); Prv.end() == false; ++Prv) + { + if (Prv.IsMultiArchImplicit() == true) + continue; + if (pkgset[Prv.ParentPkg()->ID] == false) + continue; + if (provides.empty() == false) + provides.append(", "); + provides.append(Prv.Name()); + if (Prv->ProvideVersion != 0) + provides.append(" (= ").append(Prv.ProvideVersion()).append(")"); + } + if (provides.empty() == false) + WriteOkay(Okay, output, "\nProvides: ", provides); + return WriteOkay(Okay, output, "\n"); +} + /*}}}*/ +static bool checkKnownArchitecture(std::string const &arch) /*{{{*/ +{ + if (APT::Configuration::checkArchitecture(arch)) + return true; + static auto const veryforeign = _config->FindVector("APT::BarbarianArchitectures"); + return std::find(veryforeign.begin(), veryforeign.end(), arch) != veryforeign.end(); +} + /*}}}*/ +static bool WriteGenericRequestHeaders(FileFd &output, APT::StringView const head)/*{{{*/ +{ + bool Okay = WriteOkay(output, head, "Architecture: ", _config->Find("APT::Architecture"), "\n", + "Architectures:"); + for (auto const &a : APT::Configuration::getArchitectures()) + WriteOkay(Okay, output, " ", a); + for (auto const &a : _config->FindVector("APT::BarbarianArchitectures")) + WriteOkay(Okay, output, " ", a); + return WriteOkay(Okay, output, "\n"); +} + /*}}}*/ +static bool SkipUnavailableVersions(pkgDepCache &Cache, pkgCache::PkgIterator const &Pkg, pkgCache::VerIterator const &Ver)/*{{{*/ +{ + /* versions which aren't current and aren't available in + any "online" source file are bad, expect if they are the chosen + candidate: The exception is for build-dep implementation as it creates + such pseudo (package) versions and removes them later on again. + We filter out versions at all so packages in 'rc' state only available + in dpkg/status aren't passed to solvers as they can't be installed. */ + if (Pkg->CurrentVer != 0) + return false; + if (Cache.GetCandidateVersion(Pkg) == Ver) + return false; + for (pkgCache::VerFileIterator I = Ver.FileList(); I.end() == false; ++I) + if (I.File().Flagged(pkgCache::Flag::NotSource) == false) + return false; + return true; +} + /*}}}*/ +static bool WriteScenarioEDSPVersion(pkgDepCache &Cache, FileFd &output, pkgCache::PkgIterator const &Pkg,/*{{{*/ + pkgCache::VerIterator const &Ver) +{ + bool Okay = WriteOkay(output, "\nSource: ", Ver.SourcePkgName(), + "\nSource-Version: ", Ver.SourceVerStr()); + if (PrioMap[Ver->Priority] != nullptr) + WriteOkay(Okay, output, "\nPriority: ", PrioMap[Ver->Priority]); + if (Ver->Section != 0) + WriteOkay(Okay, output, "\nSection: ", Ver.Section()); + if (Pkg.CurrentVer() == Ver) + WriteOkay(Okay, output, "\nInstalled: yes"); + if (Pkg->SelectedState == pkgCache::State::Hold || + (Cache[Pkg].Keep() == true && Cache[Pkg].Protect() == true)) + WriteOkay(Okay, output, "\nHold: yes"); + std::set<string> Releases; + for (pkgCache::VerFileIterator I = Ver.FileList(); I.end() == false; ++I) { + pkgCache::PkgFileIterator File = I.File(); + if (File.Flagged(pkgCache::Flag::NotSource) == false) { + string Release = File.RelStr(); + if (!Release.empty()) + Releases.insert(Release); + } + } + if (!Releases.empty()) { + WriteOkay(Okay, output, "\nAPT-Release:"); + for (std::set<string>::iterator R = Releases.begin(); R != Releases.end(); ++R) + WriteOkay(Okay, output, "\n ", *R); + } + WriteOkay(Okay, output, "\nAPT-Pin: ", Cache.GetPolicy().GetPriority(Ver)); + if (Cache.GetCandidateVersion(Pkg) == Ver) + WriteOkay(Okay, output, "\nAPT-Candidate: yes"); + if ((Cache[Pkg].Flags & pkgCache::Flag::Auto) == pkgCache::Flag::Auto) + WriteOkay(Okay, output, "\nAPT-Automatic: yes"); + return Okay; +} + /*}}}*/ +// EDSP::WriteScenario - to the given file descriptor /*{{{*/ +bool EDSP::WriteScenario(pkgDepCache &Cache, FileFd &output, OpProgress *Progress) +{ + if (Progress != NULL) + Progress->SubProgress(Cache.Head().VersionCount, _("Send scenario to solver")); + decltype(Cache.Head().VersionCount) p = 0; + bool Okay = output.Failed() == false; + for (pkgCache::PkgIterator Pkg = Cache.PkgBegin(); Pkg.end() == false && likely(Okay); ++Pkg) + { + if (Pkg->CurrentVer == 0 && not checkKnownArchitecture(Pkg.Arch())) + continue; + for (pkgCache::VerIterator Ver = Pkg.VersionList(); Ver.end() == false && likely(Okay); ++Ver, ++p) + { + if (SkipUnavailableVersions(Cache, Pkg, Ver)) + continue; + Okay &= WriteScenarioVersion(output, Pkg, Ver); + Okay &= WriteScenarioEDSPVersion(Cache, output, Pkg, Ver); + Okay &= WriteScenarioDependency(output, Ver, false); + WriteOkay(Okay, output, "\n"); + if (Progress != NULL && p % 100 == 0) + Progress->Progress(p); + } + } + return Okay; +} + /*}}}*/ +// EDSP::WriteLimitedScenario - to the given file descriptor /*{{{*/ +bool EDSP::WriteLimitedScenario(pkgDepCache &Cache, FileFd &output, + std::vector<bool> const &pkgset, + OpProgress *Progress) +{ + if (Progress != NULL) + Progress->SubProgress(Cache.Head().VersionCount, _("Send scenario to solver")); + decltype(Cache.Head().PackageCount) p = 0; + bool Okay = output.Failed() == false; + for (auto Pkg = Cache.PkgBegin(); Pkg.end() == false && likely(Okay); ++Pkg, ++p) + { + if (pkgset[Pkg->ID] == false) + continue; + for (pkgCache::VerIterator Ver = Pkg.VersionList(); Ver.end() == false && likely(Okay); ++Ver) + { + if (SkipUnavailableVersions(Cache, Pkg, Ver)) + continue; + Okay &= WriteScenarioVersion(output, Pkg, Ver); + Okay &= WriteScenarioEDSPVersion(Cache, output, Pkg, Ver); + Okay &= WriteScenarioLimitedDependency(output, Ver, pkgset, false); + WriteOkay(Okay, output, "\n"); + if (Progress != NULL && p % 100 == 0) + Progress->Progress(p); + } + } + if (Progress != NULL) + Progress->Done(); + return Okay; +} + /*}}}*/ +// EDSP::WriteRequest - to the given file descriptor /*{{{*/ +bool EDSP::WriteRequest(pkgDepCache &Cache, FileFd &output, + unsigned int const flags, + OpProgress *Progress) +{ + if (Progress != NULL) + Progress->SubProgress(Cache.Head().PackageCount, _("Send request to solver")); + decltype(Cache.Head().PackageCount) p = 0; + string del, inst; + for (pkgCache::PkgIterator Pkg = Cache.PkgBegin(); Pkg.end() == false; ++Pkg, ++p) + { + if (Progress != NULL && p % 100 == 0) + Progress->Progress(p); + string* req; + pkgDepCache::StateCache &P = Cache[Pkg]; + if (P.Delete() == true) + req = &del; + else if (P.NewInstall() == true || P.Upgrade() == true || P.ReInstall() == true || + (P.Mode == pkgDepCache::ModeKeep && (P.iFlags & pkgDepCache::Protected) == pkgDepCache::Protected)) + req = &inst; + else + continue; + req->append(" ").append(Pkg.FullName()); + } + + bool Okay = WriteGenericRequestHeaders(output, "Request: EDSP 0.5\n"); + string machineID = APT::Configuration::getMachineID(); + if (not machineID.empty()) + WriteOkay(Okay, output, "Machine-ID: ", machineID, "\n"); + if (del.empty() == false) + WriteOkay(Okay, output, "Remove:", del, "\n"); + if (inst.empty() == false) + WriteOkay(Okay, output, "Install:", inst, "\n"); + if (flags & Request::AUTOREMOVE) + WriteOkay(Okay, output, "Autoremove: yes\n"); + if (flags & Request::UPGRADE_ALL) + { + WriteOkay(Okay, output, "Upgrade-All: yes\n"); + if (flags & (Request::FORBID_NEW_INSTALL | Request::FORBID_REMOVE)) + WriteOkay(Okay, output, "Upgrade: yes\n"); + else + WriteOkay(Okay, output, "Dist-Upgrade: yes\n"); + } + if (flags & Request::FORBID_NEW_INSTALL) + WriteOkay(Okay, output, "Forbid-New-Install: yes\n"); + if (flags & Request::FORBID_REMOVE) + WriteOkay(Okay, output, "Forbid-Remove: yes\n"); + auto const solver = _config->Find("APT::Solver", "internal"); + WriteOkay(Okay, output, "Solver: ", solver, "\n"); + if (_config->FindB("APT::Solver::Strict-Pinning", true) == false) + WriteOkay(Okay, output, "Strict-Pinning: no\n"); + string solverpref("APT::Solver::"); + solverpref.append(solver).append("::Preferences"); + if (_config->Exists(solverpref) == true) + WriteOkay(Okay, output, "Preferences: ", _config->Find(solverpref,""), "\n"); + return WriteOkay(Okay, output, "\n"); +} + /*}}}*/ +// EDSP::ReadResponse - from the given file descriptor /*{{{*/ +bool EDSP::ReadResponse(int const input, pkgDepCache &Cache, OpProgress *Progress) { + /* We build an map id to mmap offset here + In theory we could use the offset as ID, but then VersionCount + couldn't be used to create other versionmappings anymore and it + would be too easy for a (buggy) solver to segfault APT… */ + auto VersionCount = Cache.Head().VersionCount; + decltype(VersionCount) VerIdx[VersionCount]; + for (pkgCache::PkgIterator P = Cache.PkgBegin(); P.end() == false; ++P) { + for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V) + VerIdx[V->ID] = V.Index(); + Cache[P].Marked = true; + Cache[P].Garbage = false; + } + + FileFd in; + in.OpenDescriptor(input, FileFd::ReadOnly, true); + pkgTagFile response(&in, 100); + pkgTagSection section; + + std::set<decltype(Cache.PkgBegin()->ID)> seenOnce; + while (response.Step(section) == true) { + std::string type; + if (section.Exists("Install") == true) + type = "Install"; + else if (section.Exists("Remove") == true) + type = "Remove"; + else if (section.Exists("Progress") == true) { + if (Progress != NULL) { + string msg = section.FindS("Message"); + if (msg.empty() == true) + msg = _("Prepare for receiving solution"); + Progress->SubProgress(100, msg, section.FindI("Percentage", 0)); + } + continue; + } else if (section.Exists("Error") == true) { + if (_error->PendingError()) { + if (Progress != nullptr) + Progress->Done(); + Progress = nullptr; + _error->DumpErrors(std::cerr, GlobalError::DEBUG, false); + } + std::string msg = SubstVar(SubstVar(section.FindS("Message"), "\n .\n", "\n\n"), "\n ", "\n"); + if (msg.empty() == true) { + msg = _("External solver failed without a proper error message"); + _error->Error("%s", msg.c_str()); + } else + _error->Error("External solver failed with: %s", msg.substr(0,msg.find('\n')).c_str()); + if (Progress != nullptr) + Progress->Done(); + std::cerr << "The solver encountered an error of type: " << section.FindS("Error") << std::endl; + std::cerr << "The following information might help you to understand what is wrong:" << std::endl; + std::cerr << msg << std::endl << std::endl; + return false; + } else if (section.Exists("Autoremove") == true) + type = "Autoremove"; + else { + char const *Start, *End; + section.GetSection(Start, End); + _error->Warning("Encountered an unexpected section with %d fields: %s", section.Count(), std::string(Start, End).c_str()); + continue; + } + + decltype(VersionCount) const id = section.FindULL(type, VersionCount); + if (id == VersionCount) { + _error->Warning("Unable to parse %s request with id value '%s'!", type.c_str(), section.FindS(type.c_str()).c_str()); + continue; + } else if (id > VersionCount) { + _error->Warning("ID value '%s' in %s request stanza is to high to refer to a known version!", section.FindS(type.c_str()).c_str(), type.c_str()); + continue; + } + + pkgCache::VerIterator Ver(Cache.GetCache(), Cache.GetCache().VerP + VerIdx[id]); + auto const Pkg = Ver.ParentPkg(); + if (type == "Autoremove") { + Cache[Pkg].Marked = false; + Cache[Pkg].Garbage = true; + } else if (seenOnce.emplace(Pkg->ID).second == false) { + _error->Warning("Ignoring %s stanza received for package %s which already had a previous stanza effecting it!", type.c_str(), Pkg.FullName(false).c_str()); + } else if (type == "Install") { + if (Pkg.CurrentVer() == Ver) { + _error->Warning("Ignoring Install stanza received for version %s of package %s which is already installed!", + Ver.VerStr(), Pkg.FullName(false).c_str()); + } else { + Cache.SetCandidateVersion(Ver); + Cache.MarkInstall(Pkg, false, 0, false); + } + } else if (type == "Remove") { + if (Pkg->CurrentVer == 0) + _error->Warning("Ignoring Remove stanza received for version %s of package %s which isn't installed!", + Ver.VerStr(), Pkg.FullName(false).c_str()); + else if (Pkg.CurrentVer() != Ver) + _error->Warning("Ignoring Remove stanza received for version %s of package %s which isn't the installed version %s!", + Ver.VerStr(), Pkg.FullName(false).c_str(), Pkg.CurrentVer().VerStr()); + else + Cache.MarkDelete(Ver.ParentPkg(), false); + } + } + return true; +} + /*}}}*/ +// ReadLine - first line from the given file descriptor /*{{{*/ +// --------------------------------------------------------------------- +/* Little helper method to read a complete line into a string. Similar to + fgets but we need to use the low-level read() here as otherwise the + listparser will be confused later on as mixing of fgets and read isn't + a supported action according to the manpages and results are undefined */ +static bool ReadLine(int const input, std::string &line) { + char one; + ssize_t data = 0; + line.erase(); + line.reserve(100); + while ((data = read(input, &one, sizeof(one))) != -1) { + if (data != 1) + continue; + if (one == '\n') + return true; + if (one == '\r') + continue; + if (line.empty() == true && isblank(one) != 0) + continue; + line += one; + } + return false; +} + /*}}}*/ +// StringToBool - convert yes/no to bool /*{{{*/ +// --------------------------------------------------------------------- +/* we are not as lazy as we are in the global StringToBool as we really + only accept yes/no here */ +static bool localStringToBool(std::string answer, bool const defValue) { + std::transform(answer.begin(), answer.end(), answer.begin(), ::tolower); + if (answer == "yes") + return true; + else if (answer == "no") + return false; + else + _error->Warning("Value '%s' is not a boolean 'yes' or 'no'!", answer.c_str()); + return defValue; +} + /*}}}*/ +static bool LineStartsWithAndStrip(std::string &line, APT::StringView const with)/*{{{*/ +{ + if (line.compare(0, with.size(), with.data()) != 0) + return false; + line = APT::String::Strip(line.substr(with.length())); + return true; +} + /*}}}*/ +static bool ReadFlag(unsigned int &flags, std::string &line, APT::StringView const name, unsigned int const setflag)/*{{{*/ +{ + if (LineStartsWithAndStrip(line, name) == false) + return false; + if (localStringToBool(line, false)) + flags |= setflag; + else + flags &= ~setflag; + return true; +} + /*}}}*/ +// EDSP::ReadRequest - first stanza from the given file descriptor /*{{{*/ +bool EDSP::ReadRequest(int const input, std::list<std::string> &install, + std::list<std::string> &remove, unsigned int &flags) +{ + install.clear(); + remove.clear(); + flags = 0; + std::string line; + while (ReadLine(input, line) == true) + { + // Skip empty lines before request + if (line.empty() == true) + continue; + // The first Tag must be a request, so search for it + if (LineStartsWithAndStrip(line, "Request:")) + continue; + + while (ReadLine(input, line) == true) + { + // empty lines are the end of the request + if (line.empty() == true) + return true; + + std::list<std::string> *request = NULL; + if (LineStartsWithAndStrip(line, "Install:")) + request = &install; + else if (LineStartsWithAndStrip(line, "Remove:")) + request = &remove; + else if (ReadFlag(flags, line, "Upgrade:", (Request::UPGRADE_ALL | Request::FORBID_REMOVE | Request::FORBID_NEW_INSTALL)) || + ReadFlag(flags, line, "Dist-Upgrade:", Request::UPGRADE_ALL) || + ReadFlag(flags, line, "Upgrade-All:", Request::UPGRADE_ALL) || + ReadFlag(flags, line, "Forbid-New-Install:", Request::FORBID_NEW_INSTALL) || + ReadFlag(flags, line, "Forbid-Remove:", Request::FORBID_REMOVE) || + ReadFlag(flags, line, "Autoremove:", Request::AUTOREMOVE)) + ; + else if (LineStartsWithAndStrip(line, "Architecture:")) + _config->Set("APT::Architecture", line); + else if (LineStartsWithAndStrip(line, "Architectures:")) + _config->Set("APT::Architectures", SubstVar(line, " ", ",")); + else if (LineStartsWithAndStrip(line, "Machine-ID")) + _config->Set("APT::Machine-ID", line); + else if (LineStartsWithAndStrip(line, "Solver:")) + ; // purely informational line + else + _error->Warning("Unknown line in EDSP Request stanza: %s", line.c_str()); + + if (request == NULL) + continue; + auto const pkgs = VectorizeString(line, ' '); + std::move(pkgs.begin(), pkgs.end(), std::back_inserter(*request)); + } + } + return false; +} /*}}}*/ +// EDSP::ApplyRequest - first stanza from the given file descriptor /*{{{*/ +bool EDSP::ApplyRequest(std::list<std::string> const &install, + std::list<std::string> const &remove, + pkgDepCache &Cache) +{ + for (std::list<std::string>::const_iterator i = install.begin(); + i != install.end(); ++i) { + pkgCache::PkgIterator P = Cache.FindPkg(*i); + if (P.end() == true) + _error->Warning("Package %s is not known, so can't be installed", i->c_str()); + else + Cache.MarkInstall(P, false); + } + + for (std::list<std::string>::const_iterator i = remove.begin(); + i != remove.end(); ++i) { + pkgCache::PkgIterator P = Cache.FindPkg(*i); + if (P.end() == true) + _error->Warning("Package %s is not known, so can't be installed", i->c_str()); + else + Cache.MarkDelete(P); + } + return true; +} + /*}}}*/ +// EDSP::WriteSolutionStanza - to the given file descriptor /*{{{*/ +bool EDSP::WriteSolutionStanza(FileFd &output, char const * const Type, pkgCache::VerIterator const &Ver) +{ + bool Okay = output.Failed() == false; + WriteOkay(Okay, output, Type, ": ", _system->GetVersionMapping(Ver->ID)); + if (_config->FindB("Debug::EDSP::WriteSolution", false) == true) + WriteOkay(Okay, output, "\nPackage: ", Ver.ParentPkg().FullName(), "\nVersion: ", Ver.VerStr()); + return WriteOkay(Okay, output, "\n\n"); +} + /*}}}*/ +// EDSP::WriteProgess - pulse to the given file descriptor /*{{{*/ +bool EDSP::WriteProgress(unsigned short const percent, const char* const message, FileFd &output) { + return WriteOkay(output, "Progress: ", TimeRFC1123(time(NULL), true), "\n", + "Percentage: ", percent, "\n", + "Message: ", message, "\n\n") && output.Flush(); +} + /*}}}*/ +// EDSP::WriteError - format an error message to be send to file descriptor /*{{{*/ +static std::string formatMessage(std::string const &msg) +{ + return SubstVar(SubstVar(APT::String::Strip(msg), "\n\n", "\n.\n"), "\n", "\n "); +} +bool EDSP::WriteError(char const * const uuid, std::string const &message, FileFd &output) { + return WriteOkay(output, "Error: ", uuid, "\n", + "Message: ", formatMessage(message), + "\n\n"); +} + /*}}}*/ +static std::string findExecutable(std::vector<std::string> const &dirs, char const * const binary) {/*{{{*/ + for (auto && dir : dirs) { + std::string const file = flCombine(dir, binary); + if (RealFileExists(file) == true) + return file; + } + return ""; +} + /*}}}*/ +static pid_t ExecuteExternal(char const* const type, char const * const binary, char const * const configdir, int * const solver_in, int * const solver_out) {/*{{{*/ + auto const solverDirs = _config->FindVector(configdir); + auto const file = findExecutable(solverDirs, binary); + std::string dumper; + { + dumper = findExecutable(solverDirs, "apt-dump-solver"); + if (dumper.empty()) + dumper = findExecutable(solverDirs, "dump"); + } + + if (file.empty() == true) + { + _error->Error("Can't call external %s '%s' as it is not in a configured directory!", type, binary); + return 0; + } + int external[4] = {-1, -1, -1, -1}; + if (pipe(external) != 0 || pipe(external + 2) != 0) + { + _error->Errno("Resolve", "Can't create needed IPC pipes for EDSP"); + return 0; + } + for (int i = 0; i < 4; ++i) + SetCloseExec(external[i], true); + + pid_t Solver = ExecFork(); + if (Solver == 0) { + dup2(external[0], STDIN_FILENO); + dup2(external[3], STDOUT_FILENO); + auto const dumpfile = _config->FindFile((std::string("Dir::Log::") + type).c_str()); + auto const dumpdir = flNotFile(dumpfile); + auto const runasuser = _config->Find(std::string("APT::") + type + "::" + binary + "::RunAsUser", + _config->Find(std::string("APT::") + type + "::RunAsUser", + _config->Find("APT::Sandbox::User"))); + if (dumper.empty() || dumpfile.empty() || dumper == file || CreateAPTDirectoryIfNeeded(dumpdir, dumpdir) == false) + { + _config->Set("APT::Sandbox::User", runasuser); + DropPrivileges(); + char const * const calling[] = { file.c_str(), nullptr }; + execv(calling[0], const_cast<char**>(calling)); + } + else + { + char const * const calling[] = { dumper.c_str(), "--user", runasuser.c_str(), dumpfile.c_str(), file.c_str(), nullptr }; + execv(calling[0], const_cast<char**>(calling)); + } + std::cerr << "Failed to execute " << type << " '" << binary << "'!" << std::endl; + _exit(100); + } + close(external[0]); + close(external[3]); + + if (WaitFd(external[1], true, 5) == false) + { + _error->Errno("Resolve", "Timed out while Waiting on availability of %s stdin", type); + return 0; + } + + *solver_in = external[1]; + *solver_out = external[2]; + return Solver; +} + /*}}}*/ +// EDSP::ExecuteSolver - fork requested solver and setup ipc pipes {{{*/ +pid_t EDSP::ExecuteSolver(const char* const solver, int * const solver_in, int * const solver_out, bool) { + return ExecuteExternal("solver", solver, "Dir::Bin::Solvers", solver_in, solver_out); +} + /*}}}*/ +static bool CreateDumpFile(char const * const id, char const * const type, FileFd &output)/*{{{*/ +{ + auto const dumpfile = _config->FindFile((std::string("Dir::Log::") + type).c_str()); + if (dumpfile.empty()) + return false; + auto const dumpdir = flNotFile(dumpfile); + _error->PushToStack(); + bool errored_out = CreateAPTDirectoryIfNeeded(dumpdir, dumpdir) == false || + output.Open(dumpfile, FileFd::WriteOnly | FileFd::Exclusive | FileFd::Create, FileFd::Extension, 0644) == false; + std::vector<std::string> downgrademsgs; + while (_error->empty() == false) + { + std::string msg; + _error->PopMessage(msg); + downgrademsgs.emplace_back(std::move(msg)); + } + _error->RevertToStack(); + for (auto && msg : downgrademsgs) + _error->Warning("%s", msg.c_str()); + if (errored_out) + return _error->WarningE(id, _("Could not open file '%s'"), dumpfile.c_str()); + return true; +} + /*}}}*/ +// EDSP::ResolveExternal - resolve problems by asking external for help {{{*/ +bool EDSP::ResolveExternal(const char* const solver, pkgDepCache &Cache, + unsigned int const flags, OpProgress *Progress) { + if (strcmp(solver, "internal") == 0) + { + FileFd output; + bool Okay = CreateDumpFile("EDSP::Resolve", "solver", output); + Okay &= EDSP::WriteRequest(Cache, output, flags, nullptr); + return Okay && EDSP::WriteScenario(Cache, output, nullptr); + } + _error->PushToStack(); + int solver_in, solver_out; + pid_t const solver_pid = ExecuteSolver(solver, &solver_in, &solver_out, true); + if (solver_pid == 0) + return false; + + FileFd output; + if (output.OpenDescriptor(solver_in, FileFd::WriteOnly | FileFd::BufferedWrite, true) == false) + return _error->Errno("ResolveExternal", "Opening solver %s stdin on fd %d for writing failed", solver, solver_in); + + bool Okay = output.Failed() == false; + if (Okay && Progress != NULL) + Progress->OverallProgress(0, 100, 5, _("Execute external solver")); + Okay &= EDSP::WriteRequest(Cache, output, flags, Progress); + if (Okay && Progress != NULL) + Progress->OverallProgress(5, 100, 20, _("Execute external solver")); + Okay &= EDSP::WriteScenario(Cache, output, Progress); + output.Close(); + + if (Okay && Progress != NULL) + Progress->OverallProgress(25, 100, 75, _("Execute external solver")); + bool const ret = EDSP::ReadResponse(solver_out, Cache, Progress); + _error->MergeWithStack(); + if (ExecWait(solver_pid, solver)) + return ret; + return false; +} /*}}}*/ + +bool EIPP::OrderInstall(char const * const solver, pkgPackageManager * const PM, /*{{{*/ + unsigned int const flags, OpProgress * const Progress) +{ + if (strcmp(solver, "internal") == 0) + { + FileFd output; + _error->PushToStack(); + bool Okay = CreateDumpFile("EIPP::OrderInstall", "planner", output); + if (Okay == false && dynamic_cast<pkgSimulate*>(PM) != nullptr) + { + _error->RevertToStack(); + return false; + } + _error->MergeWithStack(); + Okay &= EIPP::WriteRequest(PM->Cache, output, flags, nullptr); + return Okay && EIPP::WriteScenario(PM->Cache, output, nullptr); + } + _error->PushToStack(); + int solver_in, solver_out; + pid_t const solver_pid = ExecuteExternal("planner", solver, "Dir::Bin::Planners", &solver_in, &solver_out); + if (solver_pid == 0) + return false; + + FileFd output; + if (output.OpenDescriptor(solver_in, FileFd::WriteOnly | FileFd::BufferedWrite, true) == false) + return _error->Errno("EIPP::OrderInstall", "Opening planner %s stdin on fd %d for writing failed", solver, solver_in); + + bool Okay = output.Failed() == false; + if (Okay && Progress != NULL) + Progress->OverallProgress(0, 100, 5, _("Execute external planner")); + Okay &= EIPP::WriteRequest(PM->Cache, output, flags, Progress); + if (Okay && Progress != NULL) + Progress->OverallProgress(5, 100, 20, _("Execute external planner")); + Okay &= EIPP::WriteScenario(PM->Cache, output, Progress); + output.Close(); + + if (Okay) + { + if (Progress != nullptr) + Progress->OverallProgress(25, 100, 75, _("Execute external planner")); + + // we don't tell the external planners about boring things + for (auto Pkg = PM->Cache.PkgBegin(); Pkg.end() == false; ++Pkg) + { + if (Pkg->CurrentState == pkgCache::State::ConfigFiles && PM->Cache[Pkg].Purge() == true) + PM->Remove(Pkg, true); + } + } + bool const ret = EIPP::ReadResponse(solver_out, PM, Progress); + _error->MergeWithStack(); + if (ExecWait(solver_pid, solver)) + return ret; + return false; +} + /*}}}*/ +bool EIPP::WriteRequest(pkgDepCache &Cache, FileFd &output, /*{{{*/ + unsigned int const flags, + OpProgress * const Progress) +{ + if (Progress != NULL) + Progress->SubProgress(Cache.Head().PackageCount, _("Send request to planner")); + decltype(Cache.Head().PackageCount) p = 0; + string del, inst, reinst; + for (pkgCache::PkgIterator Pkg = Cache.PkgBegin(); Pkg.end() == false; ++Pkg, ++p) + { + if (Progress != NULL && p % 100 == 0) + Progress->Progress(p); + string* req; + pkgDepCache::StateCache &P = Cache[Pkg]; + if (P.Purge() == true && Pkg->CurrentState == pkgCache::State::ConfigFiles) + continue; + if (P.Delete() == true) + req = &del; + else if (P.NewInstall() == true || P.Upgrade() == true || P.Downgrade() == true) + req = &inst; + else if (P.ReInstall() == true) + req = &reinst; + else + continue; + req->append(" ").append(Pkg.FullName()); + } + + bool Okay = WriteGenericRequestHeaders(output, "Request: EIPP 0.1\n"); + if (del.empty() == false) + WriteOkay(Okay, output, "Remove:", del, "\n"); + if (inst.empty() == false) + WriteOkay(Okay, output, "Install:", inst, "\n"); + if (reinst.empty() == false) + WriteOkay(Okay, output, "ReInstall:", reinst, "\n"); + WriteOkay(Okay, output, "Planner: ", _config->Find("APT::Planner", "internal"), "\n"); + if ((flags & Request::IMMEDIATE_CONFIGURATION_ALL) != 0) + WriteOkay(Okay, output, "Immediate-Configuration: yes\n"); + else if ((flags & Request::NO_IMMEDIATE_CONFIGURATION) != 0) + WriteOkay(Okay, output, "Immediate-Configuration: no\n"); + else if ((flags & Request::ALLOW_TEMPORARY_REMOVE_OF_ESSENTIALS) != 0) + WriteOkay(Okay, output, "Allow-Temporary-Remove-of-Essentials: yes\n"); + return WriteOkay(Okay, output, "\n"); +} + /*}}}*/ +static bool WriteScenarioEIPPVersion(pkgDepCache &, FileFd &output, pkgCache::PkgIterator const &Pkg,/*{{{*/ + pkgCache::VerIterator const &Ver) +{ + bool Okay = true; + if (Pkg.CurrentVer() == Ver) + switch (Pkg->CurrentState) + { + case pkgCache::State::NotInstalled: WriteOkay(Okay, output, "\nStatus: not-installed"); break; + case pkgCache::State::ConfigFiles: WriteOkay(Okay, output, "\nStatus: config-files"); break; + case pkgCache::State::HalfInstalled: WriteOkay(Okay, output, "\nStatus: half-installed"); break; + case pkgCache::State::UnPacked: WriteOkay(Okay, output, "\nStatus: unpacked"); break; + case pkgCache::State::HalfConfigured: WriteOkay(Okay, output, "\nStatus: half-configured"); break; + case pkgCache::State::TriggersAwaited: WriteOkay(Okay, output, "\nStatus: triggers-awaited"); break; + case pkgCache::State::TriggersPending: WriteOkay(Okay, output, "\nStatus: triggers-pending"); break; + case pkgCache::State::Installed: WriteOkay(Okay, output, "\nStatus: installed"); break; + } + return Okay; +} + /*}}}*/ +// EIPP::WriteScenario - to the given file descriptor /*{{{*/ +template<typename forVersion> void forAllInterestingVersions(pkgDepCache &Cache, pkgCache::PkgIterator const &Pkg, forVersion const &func) +{ + if (Pkg->CurrentState == pkgCache::State::NotInstalled) + { + auto P = Cache[Pkg]; + if (P.Install() == false) + return; + func(Pkg, P.InstVerIter(Cache)); + } + else + { + if (Pkg->CurrentVer != 0) + func(Pkg, Pkg.CurrentVer()); + auto P = Cache[Pkg]; + auto const V = P.InstVerIter(Cache); + if (P.Delete() == false && Pkg.CurrentVer() != V) + func(Pkg, V); + } +} + +bool EIPP::WriteScenario(pkgDepCache &Cache, FileFd &output, OpProgress * const Progress) +{ + if (Progress != NULL) + Progress->SubProgress(Cache.Head().PackageCount, _("Send scenario to planner")); + decltype(Cache.Head().PackageCount) p = 0; + bool Okay = output.Failed() == false; + std::vector<bool> pkgset(Cache.Head().PackageCount, false); + auto const MarkVersion = [&](pkgCache::PkgIterator const &Pkg, pkgCache::VerIterator const &Ver) { + pkgset[Pkg->ID] = true; + for (auto D = Ver.DependsList(); D.end() == false; ++D) + { + if (D.IsCritical() == false) + continue; + auto const P = D.TargetPkg(); + for (auto Prv = P.ProvidesList(); Prv.end() == false; ++Prv) + { + auto const V = Prv.OwnerVer(); + auto const PV = V.ParentPkg(); + if (V == PV.CurrentVer() || V == Cache[PV].InstVerIter(Cache)) + pkgset[PV->ID] = true; + } + pkgset[P->ID] = true; + if (strcmp(P.Arch(), "any") == 0) + { + APT::StringView const pkgname(P.Name()); + auto const idxColon = pkgname.find(':'); + if (idxColon != APT::StringView::npos) + { + pkgCache::PkgIterator PA; + if (pkgname.substr(idxColon + 1) == "any") + { + auto const GA = Cache.FindGrp(pkgname.substr(0, idxColon).to_string()); + for (auto PA = GA.PackageList(); PA.end() == false; PA = GA.NextPkg(PA)) + { + pkgset[PA->ID] = true; + } + } + else + { + auto const PA = Cache.FindPkg(pkgname.to_string()); + if (PA.end() == false) + pkgset[PA->ID] = true; + } + } + } + else + { + auto const PA = Cache.FindPkg(P.FullName(false), "any"); + if (PA.end() == false) + pkgset[PA->ID] = true; + } + } + }; + for (pkgCache::PkgIterator Pkg = Cache.PkgBegin(); Pkg.end() == false; ++Pkg) + forAllInterestingVersions(Cache, Pkg, MarkVersion); + auto const WriteVersion = [&](pkgCache::PkgIterator const &Pkg, pkgCache::VerIterator const &Ver) { + Okay &= WriteScenarioVersion(output, Pkg, Ver); + Okay &= WriteScenarioEIPPVersion(Cache, output, Pkg, Ver); + Okay &= WriteScenarioLimitedDependency(output, Ver, pkgset, true); + WriteOkay(Okay, output, "\n"); + if (Progress != NULL && p % 100 == 0) + Progress->Progress(p); + }; + for (pkgCache::PkgIterator Pkg = Cache.PkgBegin(); Pkg.end() == false && likely(Okay); ++Pkg, ++p) + { + if (pkgset[Pkg->ID] == false || Pkg->VersionList == 0) + continue; + forAllInterestingVersions(Cache, Pkg, WriteVersion); + } + return Okay; +} + /*}}}*/ +// EIPP::ReadResponse - from the given file descriptor /*{{{*/ +bool EIPP::ReadResponse(int const input, pkgPackageManager * const PM, OpProgress *Progress) { + /* We build an map id to mmap offset here + In theory we could use the offset as ID, but then VersionCount + couldn't be used to create other versionmappings anymore and it + would be too easy for a (buggy) solver to segfault APT… */ + auto VersionCount = PM->Cache.Head().VersionCount; + decltype(VersionCount) VerIdx[VersionCount]; + for (pkgCache::PkgIterator P = PM->Cache.PkgBegin(); P.end() == false; ++P) { + for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V) + VerIdx[V->ID] = V.Index(); + } + + FileFd in; + in.OpenDescriptor(input, FileFd::ReadOnly); + pkgTagFile response(&in, 100); + pkgTagSection section; + + while (response.Step(section) == true) { + char const * type = nullptr; + if (section.Exists("Progress") == true) { + if (Progress != NULL) { + string msg = section.FindS("Message"); + if (msg.empty() == true) + msg = _("Prepare for receiving solution"); + Progress->SubProgress(100, msg, section.FindI("Percentage", 0)); + } + continue; + } else if (section.Exists("Error") == true) { + if (_error->PendingError()) { + if (Progress != nullptr) + Progress->Done(); + Progress = nullptr; + _error->DumpErrors(std::cerr, GlobalError::DEBUG, false); + } + std::string msg = SubstVar(SubstVar(section.FindS("Message"), "\n .\n", "\n\n"), "\n ", "\n"); + if (msg.empty() == true) { + msg = _("External planner failed without a proper error message"); + _error->Error("%s", msg.c_str()); + } else + _error->Error("External planner failed with: %s", msg.substr(0,msg.find('\n')).c_str()); + if (Progress != nullptr) + Progress->Done(); + std::cerr << "The planner encountered an error of type: " << section.FindS("Error") << std::endl; + std::cerr << "The following information might help you to understand what is wrong:" << std::endl; + std::cerr << msg << std::endl << std::endl; + return false; + } else if (section.Exists("Unpack") == true) + type = "Unpack"; + else if (section.Exists("Configure") == true) + type = "Configure"; + else if (section.Exists("Remove") == true) + type = "Remove"; + else { + char const *Start, *End; + section.GetSection(Start, End); + _error->Warning("Encountered an unexpected section with %d fields: %s", section.Count(), std::string(Start, End).c_str()); + continue; + } + + if (type == nullptr) + continue; + decltype(VersionCount) const id = section.FindULL(type, VersionCount); + if (id == VersionCount) { + _error->Warning("Unable to parse %s request with id value '%s'!", type, section.FindS(type).c_str()); + continue; + } else if (id > VersionCount) { + _error->Warning("ID value '%s' in %s request stanza is to high to refer to a known version!", section.FindS(type).c_str(), type); + continue; + } + + pkgCache::VerIterator Ver(PM->Cache.GetCache(), PM->Cache.GetCache().VerP + VerIdx[id]); + auto const Pkg = Ver.ParentPkg(); + if (strcmp(type, "Unpack") == 0) + PM->Install(Pkg, PM->FileNames[Pkg->ID]); + else if (strcmp(type, "Configure") == 0) + PM->Configure(Pkg); + else if (strcmp(type, "Remove") == 0) + PM->Remove(Pkg, PM->Cache[Pkg].Purge()); + } + return in.Failed() == false; +} + /*}}}*/ +bool EIPP::ReadRequest(int const input, std::list<std::pair<std::string,PKG_ACTION>> &actions,/*{{{*/ + unsigned int &flags) +{ + actions.clear(); + flags = 0; + std::string line; + while (ReadLine(input, line) == true) + { + // Skip empty lines before request + if (line.empty() == true) + continue; + // The first Tag must be a request, so search for it + if (line.compare(0, 8, "Request:") != 0) + continue; + + while (ReadLine(input, line) == true) + { + // empty lines are the end of the request + if (line.empty() == true) + return true; + + PKG_ACTION pkgact = PKG_ACTION::NOOP; + if (LineStartsWithAndStrip(line, "Install:")) + pkgact = PKG_ACTION::INSTALL; + else if (LineStartsWithAndStrip(line, "ReInstall:")) + pkgact = PKG_ACTION::REINSTALL; + else if (LineStartsWithAndStrip(line, "Remove:")) + pkgact = PKG_ACTION::REMOVE; + else if (LineStartsWithAndStrip(line, "Architecture:")) + _config->Set("APT::Architecture", line); + else if (LineStartsWithAndStrip(line, "Architectures:")) + _config->Set("APT::Architectures", SubstVar(line, " ", ",")); + else if (LineStartsWithAndStrip(line, "Planner:")) + ; // purely informational line + else if (LineStartsWithAndStrip(line, "Immediate-Configuration:")) + { + if (localStringToBool(line, true)) + flags |= Request::IMMEDIATE_CONFIGURATION_ALL; + else + flags |= Request::NO_IMMEDIATE_CONFIGURATION; + } + else if (ReadFlag(flags, line, "Allow-Temporary-Remove-of-Essentials:", Request::ALLOW_TEMPORARY_REMOVE_OF_ESSENTIALS)) + ; + else + _error->Warning("Unknown line in EIPP Request stanza: %s", line.c_str()); + + if (pkgact == PKG_ACTION::NOOP) + continue; + for (auto && p: VectorizeString(line, ' ')) + actions.emplace_back(std::move(p), pkgact); + } + } + return false; +} + /*}}}*/ +bool EIPP::ApplyRequest(std::list<std::pair<std::string,PKG_ACTION>> &actions,/*{{{*/ + pkgDepCache &Cache) +{ + for (auto Pkg = Cache.PkgBegin(); Pkg.end() == false; ++Pkg) + { + short versions = 0; + for (auto Ver = Pkg.VersionList(); Ver.end() == false; ++Ver) + { + ++versions; + if (Pkg.CurrentVer() == Ver) + continue; + Cache.SetCandidateVersion(Ver); + } + if (unlikely(versions > 2)) + _error->Warning("Package %s has %d versions, but should have at most 2!", Pkg.FullName().c_str(), versions); + } + for (auto && a: actions) + { + pkgCache::PkgIterator P = Cache.FindPkg(a.first); + if (P.end() == true) + { + _error->Warning("Package %s is not known, so can't be acted on", a.first.c_str()); + continue; + } + switch (a.second) + { + case PKG_ACTION::NOOP: + _error->Warning("Package %s has NOOP as action?!?", a.first.c_str()); + break; + case PKG_ACTION::INSTALL: + Cache.MarkInstall(P, false); + break; + case PKG_ACTION::REINSTALL: + Cache.MarkInstall(P, false); + Cache.SetReInstall(P, true); + break; + case PKG_ACTION::REMOVE: + Cache.MarkDelete(P); + break; + } + } + return true; +} + /*}}}*/ diff --git a/apt-pkg/edsp.h b/apt-pkg/edsp.h new file mode 100644 index 0000000..434010d --- /dev/null +++ b/apt-pkg/edsp.h @@ -0,0 +1,252 @@ +// -*- mode: cpp; mode: fold -*- +/** Description \file edsp.h {{{ + ###################################################################### + Set of methods to help writing and reading everything needed for EDSP + with the notable exception of reading a scenario for conversion into + a Cache as this is handled by edsp interface for listparser and friends + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_EDSP_H +#define PKGLIB_EDSP_H + +#include <apt-pkg/cacheset.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> + +#include <stdio.h> + +#include <list> +#include <string> +#include <vector> + + +class pkgDepCache; +class OpProgress; + +namespace EDSP /*{{{*/ +{ + namespace Request + { + enum Flags + { + AUTOREMOVE = (1 << 0), /*!< removal of unneeded packages should be performed */ + UPGRADE_ALL = (1 << 1), /*!< upgrade all installed packages, like 'apt-get full-upgrade' without forbid flags */ + FORBID_NEW_INSTALL = (1 << 2), /*!< forbid the resolver to install new packages */ + FORBID_REMOVE = (1 << 3), /*!< forbid the resolver to remove packages */ + }; + } + /** \brief creates the EDSP request stanza + * + * In the EDSP protocol the first thing send to the resolver is a stanza + * encoding the request. This method will write this stanza by looking at + * the given Cache and requests the installation of all packages which were + * marked for installation in it (equally for remove). + * + * \param Cache in which the request is encoded + * \param output is written to this "file" + * \param flags effecting the request documented in #EDSP::Request::Flags + * \param Progress is an instance to report progress to + * + * \return true if request was composed successfully, otherwise false + */ + APT_PUBLIC bool WriteRequest(pkgDepCache &Cache, FileFd &output, + unsigned int const flags = 0, + OpProgress *Progress = NULL); + + /** \brief creates the scenario representing the package universe + * + * After the request all known information about a package are send + * to the solver. The output looks similar to a Packages or status file + * + * All packages and version included in this Cache are send, even if + * it doesn't make sense from an APT resolver point of view like versions + * with a negative pin to enable the solver to propose even that as a + * solution or at least to be able to give a hint what can be done to + * satisfy a request. + * + * \param Cache is the known package universe + * \param output is written to this "file" + * \param Progress is an instance to report progress to + * + * \return true if universe was composed successfully, otherwise false + */ + APT_PUBLIC bool WriteScenario(pkgDepCache &Cache, FileFd &output, OpProgress *Progress = NULL); + + /** \brief creates a limited scenario representing the package universe + * + * This method works similar to #WriteScenario as it works in the same + * way but doesn't send the complete universe to the solver but only + * packages included in the pkgset which will have only dependencies + * on packages which are in the given set. All other dependencies will + * be removed, so that this method can be used to create testcases + * + * \param Cache is the known package universe + * \param output is written to this "file" + * \param pkgset is a set of packages the universe should be limited to + * \param Progress is an instance to report progress to + * + * \return true if universe was composed successfully, otherwise false + */ + APT_PUBLIC bool WriteLimitedScenario(pkgDepCache &Cache, FileFd &output, + std::vector<bool> const &pkgset, + OpProgress *Progress = NULL); + + /** \brief waits and acts on the information returned from the solver + * + * This method takes care of interpreting whatever the solver sends + * through the standard output like a solution, progress or an error. + * The main thread should hand his control over to this method to + * wait for the solver to finish the given task. The file descriptor + * used as input is completely consumed and closed by the method. + * + * \param input file descriptor with the response from the solver + * \param Cache the solution should be applied on if any + * \param Progress is an instance to report progress to + * + * \return true if a solution is found and applied correctly, otherwise false + */ + APT_PUBLIC bool ReadResponse(int const input, pkgDepCache &Cache, OpProgress *Progress = NULL); + + /** \brief search and read the request stanza for action later + * + * This method while ignore the input up to the point it finds the + * Request: line as an indicator for the Request stanza. + * The request is stored in the parameters install and remove then, + * as the cache isn't build yet as the scenario follows the request. + * + * \param input file descriptor with the edsp input for the solver + * \param[out] install is a list which gets populated with requested installs + * \param[out] remove is a list which gets populated with requested removals + * \param[out] upgrade is true if it is a request like apt-get upgrade + * \param[out] distUpgrade is true if it is a request like apt-get dist-upgrade + * \param[out] autoRemove is true if removal of unneeded packages should be performed + * + * \return true if the request could be found and worked on, otherwise false + */ + APT_PUBLIC bool ReadRequest(int const input, std::list<std::string> &install, + std::list<std::string> &remove, unsigned int &flags); + + /** \brief takes the request lists and applies it on the cache + * + * The lists as created by #ReadRequest will be used to find the + * packages in question and mark them for install/remove. + * No solving is done and no auto-install/-remove. + * + * \param install is a list of packages to mark for installation + * \param remove is a list of packages to mark for removal + * \param Cache is there the markers should be set + * + * \return false if the request couldn't be applied, true otherwise + */ + APT_PUBLIC bool ApplyRequest(std::list<std::string> const &install, + std::list<std::string> const &remove, + pkgDepCache &Cache); + + /** \brief formats a solution stanza for the given version + * + * EDSP uses a simple format for reporting solutions: + * A single required field name with an ID as value. + * Additional fields might appear as debug aids. + * + * \param output to write the stanza forming the solution to + * \param Type of the stanza, used as field name + * \param Ver this stanza applies to + * + * \return true if stanza could be written, otherwise false + */ + APT_PUBLIC bool WriteSolutionStanza(FileFd &output, char const * const Type, pkgCache::VerIterator const &Ver); + + /** \brief sends a progress report + * + * \param percent of the solving completed + * \param message the solver wants the user to see + * \param output the front-end listens for progress report + */ + APT_PUBLIC bool WriteProgress(unsigned short const percent, const char* const message, FileFd &output); + + /** \brief sends an error report + * + * Solvers are expected to execute successfully even if + * they were unable to calculate a solution for a given task. + * Obviously they can't send a solution through, so this + * methods deals with formatting an error message correctly + * so that the front-ends can receive and display it. + * + * The first line of the message should be a short description + * of the error so it can be used for dialog titles or alike + * + * \param uuid of this error message + * \param message is free form text to describe the error + * \param output the front-end listens for error messages + */ + APT_PUBLIC bool WriteError(char const * const uuid, std::string const &message, FileFd &output); + + + /** \brief executes the given solver and returns the pipe ends + * + * The given solver is executed if it can be found in one of the + * configured directories and setup for it is performed. + * + * \param solver to execute + * \param[out] solver_in will be the stdin of the solver + * \param[out] solver_out will be the stdout of the solver + * + * \return PID of the started solver or 0 if failure occurred + */ + APT_PUBLIC pid_t ExecuteSolver(const char* const solver, int * const solver_in, int * const solver_out, bool /*overload*/); + + /** \brief call an external resolver to handle the request + * + * This method wraps all the methods above to call an external solver + * + * \param solver to execute + * \param Cache with the problem and as universe to work in + * \param flags effecting the request documented in #EDSP::Request::Flags + * \param Progress is an instance to report progress to + * + * \return true if the solver has successfully solved the problem, + * otherwise false + */ + APT_PUBLIC bool ResolveExternal(const char* const solver, pkgDepCache &Cache, + unsigned int const flags = 0, + OpProgress *Progress = NULL); +} + /*}}}*/ +class pkgPackageManager; +namespace EIPP /*{{{*/ +{ + namespace Request + { + enum Flags + { + IMMEDIATE_CONFIGURATION_ALL = (1 << 0), /*!< try to keep the least amount of packages unconfigured as possible at all times */ + NO_IMMEDIATE_CONFIGURATION = (1 << 1), /*!< do not perform immediate configuration at all */ + ALLOW_TEMPORARY_REMOVE_OF_ESSENTIALS = (1 << 2), /*!< just as the name suggests, very special case and dangerous! */ + }; + } + + APT_HIDDEN bool WriteRequest(pkgDepCache &Cache, FileFd &output, + unsigned int const flags, OpProgress * const Progress); + APT_HIDDEN bool WriteScenario(pkgDepCache &Cache, FileFd &output, + OpProgress * const Progress); + + APT_HIDDEN bool OrderInstall(char const * const planner, pkgPackageManager * const PM, + unsigned int const version, OpProgress * const Progress); + APT_HIDDEN bool ReadResponse(int const input, pkgPackageManager * const PM, + OpProgress * const Progress); + + enum class PKG_ACTION + { + NOOP, + INSTALL, + REINSTALL, + REMOVE + }; + APT_PUBLIC bool ReadRequest(int const input, + std::list<std::pair<std::string,PKG_ACTION>> &actions, + unsigned int &flags); + APT_PUBLIC bool ApplyRequest(std::list<std::pair<std::string,PKG_ACTION>> &actions, + pkgDepCache &Cache); +} + /*}}}*/ +#endif diff --git a/apt-pkg/edsp/edspindexfile.cc b/apt-pkg/edsp/edspindexfile.cc new file mode 100644 index 0000000..faade6e --- /dev/null +++ b/apt-pkg/edsp/edspindexfile.cc @@ -0,0 +1,129 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + The scenario file is designed to work as an intermediate file between + APT and the resolver. Its on propose very similar to a dpkg status file + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/edspindexfile.h> +#include <apt-pkg/edsplistparser.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/indexfile.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgrecords.h> + +#include <memory> +#include <string> +#include <stddef.h> +#include <unistd.h> + /*}}}*/ + +// EDSP-like Index /*{{{*/ +edspLikeIndex::edspLikeIndex(std::string const &File) : pkgDebianIndexRealFile(File, true) +{ +} +std::string edspLikeIndex::GetArchitecture() const +{ + return std::string(); +} +bool edspLikeIndex::HasPackages() const +{ + return true; +} +bool edspLikeIndex::Exists() const +{ + return true; +} +uint8_t edspLikeIndex::GetIndexFlags() const +{ + return 0; +} +bool edspLikeIndex::OpenListFile(FileFd &Pkg, std::string const &FileName) +{ + if (FileName.empty() == false && FileName != "/nonexistent/stdin") + return pkgDebianIndexRealFile::OpenListFile(Pkg, FileName); + if (Pkg.OpenDescriptor(STDIN_FILENO, FileFd::ReadOnly) == false) + return _error->Error("Problem opening %s",FileName.c_str()); + return true; +} + /*}}}*/ +// EDSP Index /*{{{*/ +edspIndex::edspIndex(std::string const &File) : edspLikeIndex(File) +{ +} +std::string edspIndex::GetComponent() const +{ + return "edsp"; +} +pkgCacheListParser * edspIndex::CreateListParser(FileFd &Pkg) +{ + if (Pkg.IsOpen() == false) + return nullptr; + _error->PushToStack(); + std::unique_ptr<pkgCacheListParser> Parser(new edspListParser(&Pkg)); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + return newError ? nullptr : Parser.release(); +} + /*}}}*/ +// EIPP Index /*{{{*/ +eippIndex::eippIndex(std::string const &File) : edspLikeIndex(File) +{ +} +std::string eippIndex::GetComponent() const +{ + return "eipp"; +} +pkgCacheListParser * eippIndex::CreateListParser(FileFd &Pkg) +{ + if (Pkg.IsOpen() == false) + return nullptr; + _error->PushToStack(); + std::unique_ptr<pkgCacheListParser> Parser(new eippListParser(&Pkg)); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + return newError ? nullptr : Parser.release(); +} + /*}}}*/ + +// Index File types for APT /*{{{*/ +class APT_HIDDEN edspIFType: public pkgIndexFile::Type +{ + public: + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator const &) const APT_OVERRIDE + { + // we don't have a record parser for this type as the file is not persistent + return NULL; + }; + edspIFType() {Label = "EDSP scenario file";}; +}; +APT_HIDDEN edspIFType _apt_Edsp; +const pkgIndexFile::Type *edspIndex::GetType() const +{ + return &_apt_Edsp; +} + +class APT_HIDDEN eippIFType: public pkgIndexFile::Type +{ + public: + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator const &) const APT_OVERRIDE + { + // we don't have a record parser for this type as the file is not persistent + return NULL; + }; + eippIFType() {Label = "EIPP scenario file";}; +}; +APT_HIDDEN eippIFType _apt_Eipp; +const pkgIndexFile::Type *eippIndex::GetType() const +{ + return &_apt_Eipp; +} + /*}}}*/ + +edspLikeIndex::~edspLikeIndex() {} +edspIndex::~edspIndex() {} +eippIndex::~eippIndex() {} diff --git a/apt-pkg/edsp/edspindexfile.h b/apt-pkg/edsp/edspindexfile.h new file mode 100644 index 0000000..42ef3fe --- /dev/null +++ b/apt-pkg/edsp/edspindexfile.h @@ -0,0 +1,59 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + The scenario file is designed to work as an intermediate file between + APT and the resolver. Its on propose very similar to a dpkg status file + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_EDSPINDEXFILE_H +#define PKGLIB_EDSPINDEXFILE_H + +#include <apt-pkg/debindexfile.h> +#include <string> + + +class OpProgress; +class pkgCacheGenerator; + +class APT_HIDDEN edspLikeIndex : public pkgDebianIndexRealFile +{ +protected: + virtual bool OpenListFile(FileFd &Pkg, std::string const &File) APT_OVERRIDE; + virtual uint8_t GetIndexFlags() const APT_OVERRIDE; + virtual std::string GetArchitecture() const APT_OVERRIDE; + +public: + virtual bool Exists() const APT_OVERRIDE; + virtual bool HasPackages() const APT_OVERRIDE; + + explicit edspLikeIndex(std::string const &File); + virtual ~edspLikeIndex(); +}; + +class APT_HIDDEN edspIndex : public edspLikeIndex +{ +protected: + APT_HIDDEN virtual pkgCacheListParser * CreateListParser(FileFd &Pkg) APT_OVERRIDE; + virtual std::string GetComponent() const APT_OVERRIDE; + +public: + virtual const Type *GetType() const APT_OVERRIDE APT_PURE; + + explicit edspIndex(std::string const &File); + virtual ~edspIndex(); +}; + +class APT_HIDDEN eippIndex : public edspLikeIndex +{ +protected: + APT_HIDDEN virtual pkgCacheListParser * CreateListParser(FileFd &Pkg) APT_OVERRIDE; + virtual std::string GetComponent() const APT_OVERRIDE; + +public: + virtual const Type *GetType() const APT_OVERRIDE APT_PURE; + + explicit eippIndex(std::string const &File); + virtual ~eippIndex(); +}; + +#endif diff --git a/apt-pkg/edsp/edsplistparser.cc b/apt-pkg/edsp/edsplistparser.cc new file mode 100644 index 0000000..5419069 --- /dev/null +++ b/apt-pkg/edsp/edsplistparser.cc @@ -0,0 +1,177 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Package Cache Generator - Generator for the cache structure. + + This builds the cache structure from the abstract package list parser. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/deblistparser.h> +#include <apt-pkg/edsplistparser.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgsystem.h> +#include <apt-pkg/string_view.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile-keys.h> +#include <apt-pkg/tagfile.h> + +#include <array> + + /*}}}*/ + +// ListParser::edspListParser - Constructor /*{{{*/ +edspLikeListParser::edspLikeListParser(FileFd * const File) : debListParser(File) +{ +} +edspListParser::edspListParser(FileFd * const File) : edspLikeListParser(File) +{ + std::string const states = _config->FindFile("Dir::State::extended_states"); + RemoveFile("edspListParserPrivate", states); + extendedstates.Open(states, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive, 0600); + std::string const prefs = _config->FindFile("Dir::Etc::preferences"); + RemoveFile("edspListParserPrivate", prefs); + preferences.Open(prefs, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive, 0600); +} + /*}}}*/ +// ListParser::NewVersion - Fill in the version structure /*{{{*/ +bool edspLikeListParser::NewVersion(pkgCache::VerIterator &Ver) +{ + _system->SetVersionMapping(Ver->ID, Section.FindI("APT-ID", Ver->ID)); + return debListParser::NewVersion(Ver); +} + /*}}}*/ +// ListParser::Description - Return the description string /*{{{*/ +// --------------------------------------------------------------------- +/* Sorry, no description for the resolvers… */ +std::vector<std::string> edspLikeListParser::AvailableDescriptionLanguages() +{ + return {}; +} +APT::StringView edspLikeListParser::Description_md5() +{ + return APT::StringView(); +} + /*}}}*/ +// ListParser::VersionHash - Compute a unique hash for this version /*{{{*/ +uint32_t edspLikeListParser::VersionHash() +{ + if (Section.Exists("APT-ID") == true) + return Section.FindI("APT-ID"); + return 0; +} + /*}}}*/ +// ListParser::ParseStatus - Parse the status field /*{{{*/ +// --------------------------------------------------------------------- +/* The Status: line here is not a normal dpkg one but just one which tells + use if the package is installed or not, where missing means not. */ +bool edspListParser::ParseStatus(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver) +{ + unsigned long state = 0; + if (Section.FindFlag("Hold",state,pkgCache::State::Hold) == false) + return false; + if (state != 0) + Pkg->SelectedState = pkgCache::State::Hold; + + state = 0; + if (Section.FindFlag("Installed",state,pkgCache::State::Installed) == false) + return false; + if (state != 0) + { + Pkg->CurrentState = pkgCache::State::Installed; + Pkg->CurrentVer = Ver.MapPointer(); + } + + if (Section.FindB("APT-Automatic", false)) + { + std::string out; + strprintf(out, "Package: %s\nArchitecture: %s\nAuto-Installed: 1\n\n", Pkg.Name(), Pkg.Arch()); + if (extendedstates.Write(out.c_str(), out.length()) == false) + return false; + } + + // FIXME: Using an overriding pin is wrong. + if (Section.FindB("APT-Candidate", false)) + { + std::string out; + strprintf(out, "Package: %s\nPin: version %s\nPin-Priority: 9999\n\n", Pkg.FullName().c_str(), Ver.VerStr()); + if (preferences.Write(out.c_str(), out.length()) == false) + return false; + } + + signed short const pinvalue = Section.FindI("APT-Pin", 500); + if (pinvalue != 500) + { + std::string out; + strprintf(out, "Package: %s\nPin: version %s\nPin-Priority: %d\n\n", Pkg.FullName().c_str(), Ver.VerStr(), pinvalue); + if (preferences.Write(out.c_str(), out.length()) == false) + return false; + } + + return true; +} + /*}}}*/ + +// ListParser::eippListParser - Constructor /*{{{*/ +eippListParser::eippListParser(FileFd *File) : edspLikeListParser(File) +{ +} + /*}}}*/ +// ListParser::ParseStatus - Parse the status field /*{{{*/ +// --------------------------------------------------------------------- +/* The Status: line here is not a normal dpkg one but just one which tells + use if the package is installed or not, where missing means not. */ +bool eippListParser::ParseStatus(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver) +{ + // Process the flag field + static std::array<WordList, 8> const statusvalues = {{ + {"not-installed",pkgCache::State::NotInstalled}, + {"config-files",pkgCache::State::ConfigFiles}, + {"half-installed",pkgCache::State::HalfInstalled}, + {"unpacked",pkgCache::State::UnPacked}, + {"half-configured",pkgCache::State::HalfConfigured}, + {"triggers-awaited",pkgCache::State::TriggersAwaited}, + {"triggers-pending",pkgCache::State::TriggersPending}, + {"installed",pkgCache::State::Installed}, + }}; + auto const status = Section.Find(pkgTagSection::Key::Status); + if (not status.empty()) + { + for (auto && sv: statusvalues) + { + if (status != sv.Str) + continue; + Pkg->CurrentState = sv.Val; + switch (Pkg->CurrentState) + { + case pkgCache::State::NotInstalled: + case pkgCache::State::ConfigFiles: + break; + case pkgCache::State::HalfInstalled: + case pkgCache::State::UnPacked: + case pkgCache::State::HalfConfigured: + case pkgCache::State::TriggersAwaited: + case pkgCache::State::TriggersPending: + case pkgCache::State::Installed: + Pkg->CurrentVer = Ver.MapPointer(); + break; + } + break; + } + } + + return true; +} + /*}}}*/ + +edspLikeListParser::~edspLikeListParser() {} +edspListParser::~edspListParser() {} +eippListParser::~eippListParser() {} diff --git a/apt-pkg/edsp/edsplistparser.h b/apt-pkg/edsp/edsplistparser.h new file mode 100644 index 0000000..41bfd1f --- /dev/null +++ b/apt-pkg/edsp/edsplistparser.h @@ -0,0 +1,58 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + EDSP Package List Parser - This implements the abstract parser + interface for the APT specific intermediate format which is passed + to external resolvers + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_EDSPLISTPARSER_H +#define PKGLIB_EDSPLISTPARSER_H + +#include <apt-pkg/deblistparser.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/pkgcache.h> + +#include <string> + + +namespace APT { + class StringView; +} +class APT_HIDDEN edspLikeListParser : public debListParser +{ + public: + virtual bool NewVersion(pkgCache::VerIterator &Ver) APT_OVERRIDE; + virtual std::vector<std::string> AvailableDescriptionLanguages() APT_OVERRIDE; + virtual APT::StringView Description_md5() APT_OVERRIDE; + virtual uint32_t VersionHash() APT_OVERRIDE; + + explicit edspLikeListParser(FileFd *File); + virtual ~edspLikeListParser(); +}; + +class APT_HIDDEN edspListParser : public edspLikeListParser +{ + FileFd extendedstates; + FileFd preferences; + +protected: + virtual bool ParseStatus(pkgCache::PkgIterator &Pkg,pkgCache::VerIterator &Ver) APT_OVERRIDE; + +public: + explicit edspListParser(FileFd *File); + virtual ~edspListParser(); +}; + +class APT_HIDDEN eippListParser : public edspLikeListParser +{ +protected: + virtual bool ParseStatus(pkgCache::PkgIterator &Pkg,pkgCache::VerIterator &Ver) APT_OVERRIDE; + +public: + explicit eippListParser(FileFd *File); + virtual ~eippListParser(); +}; +#endif diff --git a/apt-pkg/edsp/edspsystem.cc b/apt-pkg/edsp/edspsystem.cc new file mode 100644 index 0000000..c86f1ed --- /dev/null +++ b/apt-pkg/edsp/edspsystem.cc @@ -0,0 +1,166 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + This system provides the abstraction to use the scenario file as the + only source of package information to be able to feed the created file + back to APT for its own consumption (eat your own dogfood). + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/debversion.h> +#include <apt-pkg/edspindexfile.h> +#include <apt-pkg/edspsystem.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/pkgcache.h> + +#include <stddef.h> +#include <stdlib.h> +#include <unistd.h> + +#include <string> +#include <vector> + + /*}}}*/ + +// System::System - Constructor /*{{{*/ +edspLikeSystem::edspLikeSystem(char const * const Label) : pkgSystem(Label, &debVS) +{ +} +edspSystem::edspSystem() : edspLikeSystem("Debian APT solver interface") +{ +} +eippSystem::eippSystem() : edspLikeSystem("Debian APT planner interface") +{ +} + /*}}}*/ +// System::Lock - Get the lock /*{{{*/ +bool edspLikeSystem::Lock(OpProgress *) +{ + return true; +} + /*}}}*/ +// System::UnLock - Drop a lock /*{{{*/ +bool edspLikeSystem::UnLock(bool /*NoErrors*/) +{ + return true; +} + /*}}}*/ +// System::CreatePM - Create the underlying package manager /*{{{*/ +// --------------------------------------------------------------------- +/* we can't use edsp input as input for real installations - just a + simulation can work, but everything else will fail bigtime */ +pkgPackageManager *edspLikeSystem::CreatePM(pkgDepCache * /*Cache*/) const +{ + return nullptr; +} + /*}}}*/ +// System::Initialize - Setup the configuration space.. /*{{{*/ +bool edspLikeSystem::Initialize(Configuration &Cnf) +{ + Cnf.Set("Dir::Log", "/dev/null"); + // state is included completely in the input files + Cnf.Set("Dir::Etc::preferences", "/dev/null"); + Cnf.Set("Dir::Etc::preferencesparts", "/dev/null"); + Cnf.Set("Dir::State::status","/dev/null"); + Cnf.Set("Dir::State::extended_states","/dev/null"); + Cnf.Set("Dir::State::lists","/dev/null"); + // do not store an mmap cache + Cnf.Set("Dir::Cache::pkgcache", ""); + Cnf.Set("Dir::Cache::srcpkgcache", ""); + // the protocols only propose actions, not do them + Cnf.Set("Debug::NoLocking", "true"); + Cnf.Set("APT::Get::Simulate", "true"); + + StatusFile.reset(nullptr); + return true; +} +bool edspSystem::Initialize(Configuration &Cnf) +{ + if (edspLikeSystem::Initialize(Cnf) == false) + return false; + std::string const tmp = GetTempDir(); + char tmpname[300]; + snprintf(tmpname, sizeof(tmpname), "%s/apt-edsp-solver-XXXXXX", tmp.c_str()); + if (nullptr == mkdtemp(tmpname)) + return false; + tempDir = tmpname; + tempStatesFile = flCombine(tempDir, "extended_states"); + Cnf.Set("Dir::State::extended_states", tempStatesFile); + tempPrefsFile = flCombine(tempDir, "apt_preferences"); + Cnf.Set("Dir::Etc::preferences", tempPrefsFile); + return true; +} + /*}}}*/ +// System::ArchiveSupported - Is a file format supported /*{{{*/ +bool edspLikeSystem::ArchiveSupported(const char * /*Type*/) +{ + return false; +} + /*}}}*/ +// System::Score - Never use the EDSP system automatically /*{{{*/ +signed edspLikeSystem::Score(Configuration const &) +{ + return -1000; +} + /*}}}*/ +// System::FindIndex - Get an index file for status files /*{{{*/ +bool edspLikeSystem::FindIndex(pkgCache::PkgFileIterator File, + pkgIndexFile *&Found) const +{ + if (StatusFile == 0) + return false; + if (StatusFile->FindInCache(*File.Cache()) == File) + { + Found = StatusFile.get(); + return true; + } + + return false; +} + /*}}}*/ +bool edspSystem::AddStatusFiles(std::vector<pkgIndexFile *> &List) /*{{{*/ +{ + if (StatusFile == nullptr) + { + if (_config->Find("edsp::scenario", "") == "/nonexistent/stdin") + StatusFile.reset(new edspIndex("/nonexistent/stdin")); + else + StatusFile.reset(new edspIndex(_config->FindFile("edsp::scenario"))); + } + List.push_back(StatusFile.get()); + return true; +} + /*}}}*/ +bool eippSystem::AddStatusFiles(std::vector<pkgIndexFile *> &List) /*{{{*/ +{ + if (StatusFile == nullptr) + { + if (_config->Find("eipp::scenario", "") == "/nonexistent/stdin") + StatusFile.reset(new eippIndex("/nonexistent/stdin")); + else + StatusFile.reset(new eippIndex(_config->FindFile("eipp::scenario"))); + } + List.push_back(StatusFile.get()); + return true; +} + /*}}}*/ + +edspLikeSystem::~edspLikeSystem() {} +edspSystem::~edspSystem() +{ + if (tempDir.empty()) + return; + + RemoveFile("~edspSystem", tempStatesFile); + RemoveFile("~edspSystem", tempPrefsFile); + rmdir(tempDir.c_str()); +} +eippSystem::~eippSystem() {} + +APT_HIDDEN edspSystem edspSys; +APT_HIDDEN eippSystem eippSys; diff --git a/apt-pkg/edsp/edspsystem.h b/apt-pkg/edsp/edspsystem.h new file mode 100644 index 0000000..97c2d66 --- /dev/null +++ b/apt-pkg/edsp/edspsystem.h @@ -0,0 +1,75 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + System - Debian version of the System Class + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_EDSPSYSTEM_H +#define PKGLIB_EDSPSYSTEM_H + +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgsystem.h> +#include <apt-pkg/error.h> + +#include <memory> +#include <vector> + +#include <apt-pkg/macros.h> + +class Configuration; +class pkgDepCache; +class pkgIndexFile; +class pkgPackageManager; + +class APT_HIDDEN edspLikeSystem : public pkgSystem +{ +protected: + std::unique_ptr<pkgIndexFile> StatusFile; + +public: + virtual bool Lock(OpProgress * const Progress) APT_OVERRIDE APT_PURE; + virtual bool UnLock(bool NoErrors = false) APT_OVERRIDE APT_PURE; + virtual pkgPackageManager *CreatePM(pkgDepCache *Cache) const APT_OVERRIDE APT_PURE; + virtual bool Initialize(Configuration &Cnf) APT_OVERRIDE; + virtual bool ArchiveSupported(const char *Type) APT_OVERRIDE APT_PURE; + virtual signed Score(Configuration const &Cnf) APT_OVERRIDE; + virtual bool FindIndex(pkgCache::PkgFileIterator File, + pkgIndexFile *&Found) const APT_OVERRIDE; + + bool MultiArchSupported() const override { return true; } + std::vector<std::string> ArchitecturesSupported() const override { return {}; }; + + bool LockInner(OpProgress * const, int) override { return _error->Error("LockInner is not implemented"); }; + bool UnLockInner(bool) override { return _error->Error("UnLockInner is not implemented"); }; + bool IsLocked() override { return true; }; + + explicit edspLikeSystem(char const * const Label); + virtual ~edspLikeSystem(); +}; + +class APT_HIDDEN edspSystem : public edspLikeSystem +{ + std::string tempDir; + std::string tempStatesFile; + std::string tempPrefsFile; + +public: + virtual bool Initialize(Configuration &Cnf) APT_OVERRIDE; + virtual bool AddStatusFiles(std::vector<pkgIndexFile *> &List) APT_OVERRIDE; + + edspSystem(); + virtual ~edspSystem(); +}; + +class APT_HIDDEN eippSystem : public edspLikeSystem +{ + public: + virtual bool AddStatusFiles(std::vector<pkgIndexFile *> &List) APT_OVERRIDE; + + eippSystem(); + virtual ~eippSystem(); +}; + +#endif diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc new file mode 100644 index 0000000..0d56926 --- /dev/null +++ b/apt-pkg/indexcopy.cc @@ -0,0 +1,775 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Index Copying - Aid for copying and verifying the index files + + This class helps apt-cache reconstruct a damaged index files. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/cdrom.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/debmetaindex.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/gpgv.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/metaindex.h> +#include <apt-pkg/progress.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile-keys.h> +#include <apt-pkg/tagfile.h> + +#include <iostream> +#include <sstream> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <unistd.h> + +#include "indexcopy.h" +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +// IndexCopy::CopyPackages - Copy the package files from the CD /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, + pkgCdromStatus *log) +{ + OpProgress *Progress = NULL; + if (List.empty() == true) + return true; + + if(log) + Progress = log->GetOpProgress(); + + bool NoStat = _config->FindB("APT::CDROM::Fast",false); + bool Debug = _config->FindB("Debug::aptcdrom",false); + + // Prepare the progress indicator + off_t TotalSize = 0; + std::vector<APT::Configuration::Compressor> const compressor = APT::Configuration::getCompressors(); + for (auto const &F : List) + { + struct stat Buf; + bool found = false; + auto const file = F + GetFileName(); + for (auto const &c : compressor) + { + if (stat((file + c.Extension).c_str(), &Buf) != 0) + continue; + found = true; + break; + } + + if (found == false) + return _error->Errno("stat", "Stat failed for %s", file.c_str()); + TotalSize += Buf.st_size; + } + + off_t CurrentSize = 0; + unsigned int NotFound = 0; + unsigned int WrongSize = 0; + unsigned int Packages = 0; + for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) + { + std::string OrigPath(*I,CDROM.length()); + + // Open the package file + FileFd Pkg(*I + GetFileName(), FileFd::ReadOnly, FileFd::Auto); + off_t const FileSize = Pkg.Size(); + + pkgTagFile Parser(&Pkg); + if (Pkg.IsOpen() == false || Pkg.Failed()) + return false; + + // Open the output file + char S[400]; + snprintf(S,sizeof(S),"cdrom:[%s]/%s%s",Name.c_str(), + (*I).c_str() + CDROM.length(),GetFileName()); + string TargetF = _config->FindDir("Dir::State::lists") + "partial/"; + TargetF += URItoFileName(S); + FileFd Target; + if (_config->FindB("APT::CDROM::NoAct",false) == true) + { + TargetF = "/dev/null"; + Target.Open(TargetF,FileFd::WriteExists); + } else { + Target.Open(TargetF,FileFd::WriteAtomic); + } + if (Target.IsOpen() == false || Target.Failed()) + return false; + + // Setup the progress meter + if(Progress) + Progress->OverallProgress(CurrentSize,TotalSize,FileSize, + string("Reading ") + Type() + " Indexes"); + + // Parse + if(Progress) + Progress->SubProgress(Pkg.Size()); + pkgTagSection Section; + this->Section = &Section; + string Prefix; + unsigned long Hits = 0; + unsigned long Chop = 0; + while (Parser.Step(Section) == true) + { + if(Progress) + Progress->Progress(Parser.Offset()); + string File; + unsigned long long Size; + if (GetFile(File,Size) == false) + return false; + + if (Chop != 0) + File = OrigPath + ChopDirs(File,Chop); + + // See if the file exists + if (NoStat == false || Hits < 10) + { + // Attempt to fix broken structure + if (Hits == 0) + { + if (ReconstructPrefix(Prefix,OrigPath,CDROM,File) == false && + ReconstructChop(Chop,*I,File) == false) + { + if (Debug == true) + clog << "Missed: " << File << endl; + NotFound++; + continue; + } + if (Chop != 0) + File = OrigPath + ChopDirs(File,Chop); + } + + // Get the size + struct stat Buf; + if (stat((CDROM + Prefix + File).c_str(),&Buf) != 0 || + Buf.st_size == 0) + { + bool Mangled = false; + // Attempt to fix busted symlink support for one instance + string OrigFile = File; + string::size_type Start = File.find("binary-"); + string::size_type End = File.find("/",Start+3); + if (Start != string::npos && End != string::npos) + { + File.replace(Start,End-Start,"binary-all"); + Mangled = true; + } + + if (Mangled == false || + stat((CDROM + Prefix + File).c_str(),&Buf) != 0) + { + if (Debug == true) + clog << "Missed(2): " << OrigFile << endl; + NotFound++; + continue; + } + } + + // Size match + if ((unsigned long long)Buf.st_size != Size) + { + if (Debug == true) + clog << "Wrong Size: " << File << endl; + WrongSize++; + continue; + } + } + + Packages++; + Hits++; + + if (RewriteEntry(Target, File) == false) + return false; + } + + if (Debug == true) + cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl; + + if (_config->FindB("APT::CDROM::NoAct",false) == false) + { + // Move out of the partial directory + Target.Close(); + string FinalF = _config->FindDir("Dir::State::lists"); + FinalF += URItoFileName(S); + if (rename(TargetF.c_str(),FinalF.c_str()) != 0) + return _error->Errno("rename","Failed to rename"); + ChangeOwnerAndPermissionOfFile("CopyPackages", FinalF.c_str(), "root", ROOT_GROUP, 0644); + } + + /* Mangle the source to be in the proper notation with + prefix dist [component] */ + *I = string(*I,Prefix.length()); + ConvertToSourceList(CDROM,*I); + *I = Prefix + ' ' + *I; + + CurrentSize += FileSize; + } + if(Progress) + Progress->Done(); + + // Some stats + if(log) { + stringstream msg; + if(NotFound == 0 && WrongSize == 0) + ioprintf(msg, _("Wrote %i records.\n"), Packages); + else if (NotFound != 0 && WrongSize == 0) + ioprintf(msg, _("Wrote %i records with %i missing files.\n"), + Packages, NotFound); + else if (NotFound == 0 && WrongSize != 0) + ioprintf(msg, _("Wrote %i records with %i mismatched files\n"), + Packages, WrongSize); + if (NotFound != 0 && WrongSize != 0) + ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize); + } + + if (Packages == 0) + _error->Warning("No valid records were found."); + + if (NotFound + WrongSize > 10) + _error->Warning("A lot of entries were discarded, something may be wrong.\n"); + + return true; +} + /*}}}*/ +// IndexCopy::ChopDirs - Chop off the leading directory components /*{{{*/ +// --------------------------------------------------------------------- +/* */ +string IndexCopy::ChopDirs(string Path,unsigned int Depth) +{ + string::size_type I = 0; + do + { + I = Path.find('/',I+1); + Depth--; + } + while (I != string::npos && Depth != 0); + + if (I == string::npos) + return string(); + + return string(Path,I+1); +} + /*}}}*/ +// IndexCopy::ReconstructPrefix - Fix strange prefixing /*{{{*/ +// --------------------------------------------------------------------- +/* This prepends dir components from the path to the package files to + the path to the deb until it is found */ +bool IndexCopy::ReconstructPrefix(string &Prefix,string OrigPath,string CD, + string File) +{ + bool Debug = _config->FindB("Debug::aptcdrom",false); + unsigned int Depth = 1; + string MyPrefix = Prefix; + while (1) + { + struct stat Buf; + if (stat((CD + MyPrefix + File).c_str(),&Buf) != 0) + { + if (Debug == true) + cout << "Failed, " << CD + MyPrefix + File << endl; + if (GrabFirst(OrigPath,MyPrefix,Depth++) == true) + continue; + + return false; + } + else + { + Prefix = MyPrefix; + return true; + } + } + return false; +} + /*}}}*/ +// IndexCopy::ReconstructChop - Fixes bad source paths /*{{{*/ +// --------------------------------------------------------------------- +/* This removes path components from the filename and prepends the location + of the package files until a file is found */ +bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File) +{ + // Attempt to reconstruct the filename + unsigned long Depth = 0; + while (1) + { + struct stat Buf; + if (stat((Dir + File).c_str(),&Buf) != 0) + { + File = ChopDirs(File,1); + Depth++; + if (File.empty() == false) + continue; + return false; + } + else + { + Chop = Depth; + return true; + } + } + return false; +} + /*}}}*/ +// IndexCopy::ConvertToSourceList - Convert a Path to a sourcelist /*{{{*/ +// --------------------------------------------------------------------- +/* We look for things in dists/ notation and convert them to + <dist> <component> form otherwise it is left alone. This also strips + the CD path. + + This implements a regex sort of like: + (.*)/dists/([^/]*)/(.*)/binary-* + ^ ^ ^- Component + | |-------- Distribution + |------------------- Path + + It was deciced to use only a single word for dist (rather than say + unstable/non-us) to increase the chance that each CD gets a single + line in sources.list. + */ +void IndexCopy::ConvertToSourceList(string CD,string &Path) +{ + // Strip the cdrom base path + Path = string(Path,CD.length()); + if (Path.empty() == true) + Path = "/"; + + // Too short to be a dists/ type + if (Path.length() < strlen("dists/")) + return; + + // Not a dists type. + if (stringcmp(Path.c_str(),Path.c_str()+strlen("dists/"),"dists/") != 0) + return; + + // Isolate the dist + string::size_type Slash = strlen("dists/"); + string::size_type Slash2 = Path.find('/',Slash + 1); + if (Slash2 == string::npos || Slash2 + 2 >= Path.length()) + return; + string Dist = string(Path,Slash,Slash2 - Slash); + + // Isolate the component + Slash = Slash2; + for (unsigned I = 0; I != 10; I++) + { + Slash = Path.find('/',Slash+1); + if (Slash == string::npos || Slash + 2 >= Path.length()) + return; + string Comp = string(Path,Slash2+1,Slash - Slash2-1); + + // Verify the trailing binary- bit + string::size_type BinSlash = Path.find('/',Slash + 1); + if (Slash == string::npos) + return; + string Binary = string(Path,Slash+1,BinSlash - Slash-1); + + if (strncmp(Binary.c_str(), "binary-", strlen("binary-")) == 0) + { + Binary.erase(0, strlen("binary-")); + if (APT::Configuration::checkArchitecture(Binary) == false) + continue; + } + else if (Binary != "source") + continue; + + Path = Dist + ' ' + Comp; + return; + } +} + /*}}}*/ +// IndexCopy::GrabFirst - Return the first Depth path components /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth) +{ + string::size_type I = 0; + do + { + I = Path.find('/',I+1); + Depth--; + } + while (I != string::npos && Depth != 0); + + if (I == string::npos) + return false; + + To = string(Path,0,I+1); + return true; +} + /*}}}*/ +// PackageCopy::GetFile - Get the file information from the section /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool PackageCopy::GetFile(string &File,unsigned long long &Size) +{ + File = Section->Find(pkgTagSection::Key::Filename).to_string(); + Size = Section->FindULL(pkgTagSection::Key::Size); + if (File.empty() || Size == 0) + return _error->Error("Cannot find filename or size tag"); + return true; +} + /*}}}*/ +// PackageCopy::RewriteEntry - Rewrite the entry with a new filename /*{{{*/ +bool PackageCopy::RewriteEntry(FileFd &Target,string const &File) +{ + std::vector<pkgTagSection::Tag> Changes; + Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", File)); + + if (Section->Write(Target, TFRewritePackageOrder, Changes) == false) + return false; + return Target.Write("\n", 1); +} + /*}}}*/ +// SourceCopy::GetFile - Get the file information from the section /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool SourceCopy::GetFile(string &File,unsigned long long &Size) +{ + std::string Files; + for (auto hashinfo : HashString::SupportedHashesInfo()) + { + Files = Section->Find(hashinfo.chksumskey).to_string(); + if (not Files.empty()) + break; + } + if (Files.empty()) + return false; + + // Read the first file triplet + const char *C = Files.c_str(); + string sSize; + string MD5Hash; + + // Parse each of the elements + if (ParseQuoteWord(C,MD5Hash) == false || + ParseQuoteWord(C,sSize) == false || + ParseQuoteWord(C,File) == false) + return _error->Error("Error parsing file record"); + + // Parse the size and append the directory + Size = strtoull(sSize.c_str(), NULL, 10); + auto const Base = Section->Find(pkgTagSection::Key::Directory); + File = flCombine(Base.to_string(), File); + return true; +} + /*}}}*/ +// SourceCopy::RewriteEntry - Rewrite the entry with a new filename /*{{{*/ +bool SourceCopy::RewriteEntry(FileFd &Target, std::string const &File) +{ + string const Dir(File,0,File.rfind('/')); + std::vector<pkgTagSection::Tag> Changes; + Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Dir)); + + if (Section->Write(Target, TFRewriteSourceOrder, Changes) == false) + return false; + return Target.Write("\n", 1); +} + /*}}}*/ +// SigVerify::Verify - Verify a files md5sum against its metaindex /*{{{*/ +bool SigVerify::Verify(string prefix, string file, metaIndex *MetaIndex) +{ + const metaIndex::checkSum *Record = MetaIndex->Lookup(file); + bool const Debug = _config->FindB("Debug::aptcdrom",false); + + // we skip non-existing files in the verifcation of the Release file + // as non-existing files do not harm, but a warning scares people and + // makes it hard to strip unneeded files from an ISO like uncompressed + // indexes as it is done on the mirrors (see also LP: #255545 ) + if(!RealFileExists(prefix+file)) + { + if (Debug == true) + cout << "Skipping nonexistent in " << prefix << " file " << file << std::endl; + return true; + } + + if (!Record) + { + _error->Warning(_("Can't find authentication record for: %s"), file.c_str()); + return false; + } + + if (!Record->Hashes.VerifyFile(prefix+file)) + { + _error->Warning(_("Hash mismatch for: %s"),file.c_str()); + return false; + } + + if(Debug == true) + { + cout << "File: " << prefix+file << endl + << "Expected Hash " << endl; + for (HashStringList::const_iterator hs = Record->Hashes.begin(); hs != Record->Hashes.end(); ++hs) + std::cout << "\t- " << hs->toStr() << std::endl; + } + + return true; +} + /*}}}*/ +bool SigVerify::CopyMetaIndex(string CDROM, string CDName, /*{{{*/ + string prefix, string file) +{ + char S[400]; + snprintf(S,sizeof(S),"cdrom:[%s]/%s%s",CDName.c_str(), + (prefix).c_str() + CDROM.length(),file.c_str()); + string TargetF = _config->FindDir("Dir::State::lists"); + TargetF += URItoFileName(S); + + FileFd Target; + FileFd Rel; + Target.Open(TargetF,FileFd::WriteAtomic); + Rel.Open(prefix + file,FileFd::ReadOnly); + if (CopyFile(Rel,Target) == false || Target.Close() == false) + return _error->Error("Copying of '%s' for '%s' from '%s' failed", file.c_str(), CDName.c_str(), prefix.c_str()); + ChangeOwnerAndPermissionOfFile("CopyPackages", TargetF.c_str(), "root", ROOT_GROUP, 0644); + + return true; +} + /*}}}*/ +bool SigVerify::CopyAndVerify(string CDROM,string Name,vector<string> &SigList, /*{{{*/ + vector<string> /*PkgList*/,vector<string> /*SrcList*/) +{ + if (SigList.empty() == true) + return true; + + bool Debug = _config->FindB("Debug::aptcdrom",false); + + // Read all Release files + for (vector<string>::iterator I = SigList.begin(); I != SigList.end(); ++I) + { + if(Debug) + cout << "Signature verify for: " << *I << endl; + + metaIndex *MetaIndex = new debReleaseIndex("","", {}); + string prefix = *I; + + string const releasegpg = *I+"Release.gpg"; + string const release = *I+"Release"; + string const inrelease = *I+"InRelease"; + bool useInRelease = true; + + // a Release.gpg without a Release should never happen + if (RealFileExists(inrelease) == true) + ; + else if(RealFileExists(release) == false || RealFileExists(releasegpg) == false) + { + delete MetaIndex; + continue; + } + else + useInRelease = false; + + pid_t pid = ExecFork(); + if(pid < 0) { + _error->Error("Fork failed"); + return false; + } + if(pid == 0) + { + if (useInRelease == true) + ExecGPGV(inrelease, inrelease); + else + ExecGPGV(release, releasegpg); + } + + if(!ExecWait(pid, "gpgv")) { + _error->Warning("Signature verification failed for: %s", + (useInRelease ? inrelease.c_str() : releasegpg.c_str())); + // something went wrong, don't copy the Release.gpg + // FIXME: delete any existing gpg file? + delete MetaIndex; + continue; + } + + // Open the Release file and add it to the MetaIndex + std::string ErrorText; + if(MetaIndex->Load(release, &ErrorText) == false) + { + _error->Error("%s", ErrorText.c_str()); + return false; + } + + // go over the Indexfiles and see if they verify + // if so, remove them from our copy of the lists + vector<string> keys = MetaIndex->MetaKeys(); + for (vector<string>::iterator I = keys.begin(); I != keys.end(); ++I) + { + if(!Verify(prefix,*I, MetaIndex)) { + // something went wrong, don't copy the Release.gpg + // FIXME: delete any existing gpg file? + _error->Discard(); + continue; + } + } + + // we need a fresh one for the Release.gpg + delete MetaIndex; + + // everything was fine, copy the Release and Release.gpg file + if (useInRelease == true) + CopyMetaIndex(CDROM, Name, prefix, "InRelease"); + else + { + CopyMetaIndex(CDROM, Name, prefix, "Release"); + CopyMetaIndex(CDROM, Name, prefix, "Release.gpg"); + } + } + + return true; +} + /*}}}*/ +bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/ + vector<string> &List, pkgCdromStatus *log) +{ + OpProgress *Progress = NULL; + if (List.empty() == true) + return true; + + if(log) + Progress = log->GetOpProgress(); + + bool Debug = _config->FindB("Debug::aptcdrom",false); + + // Prepare the progress indicator + off_t TotalSize = 0; + std::vector<APT::Configuration::Compressor> const compressor = APT::Configuration::getCompressors(); + for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) + { + struct stat Buf; + bool found = false; + std::string file = *I; + for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) + { + if (stat((file + c->Extension).c_str(), &Buf) != 0) + continue; + found = true; + break; + } + + if (found == false) + return _error->Errno("stat", "Stat failed for %s", file.c_str()); + TotalSize += Buf.st_size; + } + + off_t CurrentSize = 0; + unsigned int NotFound = 0; + unsigned int WrongSize = 0; + unsigned int Packages = 0; + for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) + { + // Open the package file + FileFd Pkg(*I, FileFd::ReadOnly, FileFd::Auto); + off_t const FileSize = Pkg.Size(); + + pkgTagFile Parser(&Pkg); + if (Pkg.IsOpen() == false || Pkg.Failed()) + return false; + + // Open the output file + char S[400]; + snprintf(S,sizeof(S),"cdrom:[%s]/%s",Name.c_str(), + (*I).c_str() + CDROM.length()); + string TargetF = _config->FindDir("Dir::State::lists") + "partial/"; + TargetF += URItoFileName(S); + FileFd Target; + if (_config->FindB("APT::CDROM::NoAct",false) == true) + { + TargetF = "/dev/null"; + Target.Open(TargetF,FileFd::WriteExists); + } else { + Target.Open(TargetF,FileFd::WriteAtomic); + } + if (Pkg.IsOpen() == false || Pkg.Failed()) + return false; + + // Setup the progress meter + if(Progress) + Progress->OverallProgress(CurrentSize,TotalSize,FileSize, + string("Reading Translation Indexes")); + + // Parse + if(Progress) + Progress->SubProgress(Pkg.Size()); + pkgTagSection Section; + this->Section = &Section; + string Prefix; + unsigned long Hits = 0; + while (Parser.Step(Section) == true) + { + if(Progress) + Progress->Progress(Parser.Offset()); + + if (Section.Write(Target) == false || Target.Write("\n", 1) == false) + return false; + + Packages++; + Hits++; + } + + if (Debug == true) + cout << " Processed by using Prefix '" << Prefix << "' and chop " << endl; + + if (_config->FindB("APT::CDROM::NoAct",false) == false) + { + // Move out of the partial directory + Target.Close(); + string FinalF = _config->FindDir("Dir::State::lists"); + FinalF += URItoFileName(S); + if (rename(TargetF.c_str(),FinalF.c_str()) != 0) + return _error->Errno("rename","Failed to rename"); + ChangeOwnerAndPermissionOfFile("CopyTranslations", FinalF.c_str(), "root", ROOT_GROUP, 0644); + } + + CurrentSize += FileSize; + } + if(Progress) + Progress->Done(); + + // Some stats + if(log) { + stringstream msg; + if(NotFound == 0 && WrongSize == 0) + ioprintf(msg, _("Wrote %i records.\n"), Packages); + else if (NotFound != 0 && WrongSize == 0) + ioprintf(msg, _("Wrote %i records with %i missing files.\n"), + Packages, NotFound); + else if (NotFound == 0 && WrongSize != 0) + ioprintf(msg, _("Wrote %i records with %i mismatched files\n"), + Packages, WrongSize); + if (NotFound != 0 && WrongSize != 0) + ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize); + } + + if (Packages == 0) + _error->Warning("No valid records were found."); + + if (NotFound + WrongSize > 10) + _error->Warning("A lot of entries were discarded, something may be wrong.\n"); + + return true; +} + /*}}}*/ + +IndexCopy::IndexCopy() : d(nullptr), Section(nullptr) {} +IndexCopy::~IndexCopy() {} + +PackageCopy::PackageCopy() : IndexCopy(), d(NULL) {} +PackageCopy::~PackageCopy() {} +SourceCopy::SourceCopy() : IndexCopy(), d(NULL) {} +SourceCopy::~SourceCopy() {} +TranslationsCopy::TranslationsCopy() : d(nullptr), Section(nullptr) {} +TranslationsCopy::~TranslationsCopy() {} +SigVerify::SigVerify() : d(NULL) {} +SigVerify::~SigVerify() {} diff --git a/apt-pkg/indexcopy.h b/apt-pkg/indexcopy.h new file mode 100644 index 0000000..41be154 --- /dev/null +++ b/apt-pkg/indexcopy.h @@ -0,0 +1,116 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Index Copying - Aid for copying and verifying the index files + + ##################################################################### */ + /*}}}*/ +#ifndef INDEXCOPY_H +#define INDEXCOPY_H + +#include <vector> +#ifndef APT_11_CLEAN_HEADERS +#include <string> +#include <stdio.h> +#endif + +#include <apt-pkg/macros.h> + + +class pkgTagSection; +class pkgCdromStatus; +class FileFd; +class metaIndex; + +class APT_PUBLIC IndexCopy /*{{{*/ +{ + /** \brief dpointer placeholder (for later in case we need it) */ + void * const d; + + protected: + + pkgTagSection *Section; + + std::string ChopDirs(std::string Path,unsigned int Depth); + bool ReconstructPrefix(std::string &Prefix,std::string OrigPath,std::string CD, + std::string File); + bool ReconstructChop(unsigned long &Chop,std::string Dir,std::string File); + void ConvertToSourceList(std::string CD,std::string &Path); + bool GrabFirst(std::string Path,std::string &To,unsigned int Depth); + virtual bool GetFile(std::string &Filename,unsigned long long &Size) = 0; + virtual bool RewriteEntry(FileFd &Target, std::string const &File) = 0; + virtual const char *GetFileName() = 0; + virtual const char *Type() = 0; + + public: + + bool CopyPackages(std::string CDROM,std::string Name,std::vector<std::string> &List, + pkgCdromStatus *log); + IndexCopy(); + virtual ~IndexCopy(); +}; + /*}}}*/ +class APT_PUBLIC PackageCopy : public IndexCopy /*{{{*/ +{ + void * const d; + protected: + + virtual bool GetFile(std::string &Filename,unsigned long long &Size) APT_OVERRIDE; + virtual bool RewriteEntry(FileFd &Target, std::string const &File) APT_OVERRIDE; + virtual const char *GetFileName() APT_OVERRIDE {return "Packages";}; + virtual const char *Type() APT_OVERRIDE {return "Package";}; + + public: + PackageCopy(); + virtual ~PackageCopy(); +}; + /*}}}*/ +class APT_PUBLIC SourceCopy : public IndexCopy /*{{{*/ +{ + void * const d; + protected: + + virtual bool GetFile(std::string &Filename,unsigned long long &Size) APT_OVERRIDE; + virtual bool RewriteEntry(FileFd &Target, std::string const &File) APT_OVERRIDE; + virtual const char *GetFileName() APT_OVERRIDE {return "Sources";}; + virtual const char *Type() APT_OVERRIDE {return "Source";}; + + public: + SourceCopy(); + virtual ~SourceCopy(); +}; + /*}}}*/ +class APT_PUBLIC TranslationsCopy /*{{{*/ +{ + void * const d; + protected: + pkgTagSection *Section; + + public: + bool CopyTranslations(std::string CDROM,std::string Name,std::vector<std::string> &List, + pkgCdromStatus *log); + + TranslationsCopy(); + virtual ~TranslationsCopy(); +}; + /*}}}*/ +class APT_PUBLIC SigVerify /*{{{*/ +{ + /** \brief dpointer placeholder (for later in case we need it) */ + void * const d; + + APT_HIDDEN bool Verify(std::string prefix,std::string file, metaIndex *records); + APT_HIDDEN bool CopyMetaIndex(std::string CDROM, std::string CDName, + std::string prefix, std::string file); + + public: + bool CopyAndVerify(std::string CDROM,std::string Name,std::vector<std::string> &SigList, + std::vector<std::string> PkgList,std::vector<std::string> SrcList); + + SigVerify(); + virtual ~SigVerify(); +}; + /*}}}*/ + +#endif diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc new file mode 100644 index 0000000..f13b529 --- /dev/null +++ b/apt-pkg/indexfile.cc @@ -0,0 +1,384 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Index File - Abstraction for an index of archive/source file. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/acquire.h> +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/deblistparser.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/indexfile.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgcachegen.h> +#include <apt-pkg/progress.h> +#include <apt-pkg/srcrecords.h> +#include <apt-pkg/strutl.h> + +#include <apt-pkg/debindexfile.h> + +#include <sys/stat.h> + +#include <clocale> +#include <cstring> +#include <memory> +#include <string> +#include <vector> + /*}}}*/ + +// Global list of Item supported +static pkgIndexFile::Type *ItmList[10]; +pkgIndexFile::Type **pkgIndexFile::Type::GlobalList = ItmList; +unsigned long pkgIndexFile::Type::GlobalListLen = 0; + +// Type::Type - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgIndexFile::Type::Type() +{ + ItmList[GlobalListLen] = this; + GlobalListLen++; + Label = NULL; +} + /*}}}*/ +// Type::GetType - Locate the type by name /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgIndexFile::Type *pkgIndexFile::Type::GetType(const char *Type) +{ + for (unsigned I = 0; I != GlobalListLen; I++) + if (strcmp(GlobalList[I]->Label,Type) == 0) + return GlobalList[I]; + return 0; +} + /*}}}*/ +pkgIndexFile::pkgIndexFile(bool const Trusted) : /*{{{*/ + d(NULL), Trusted(Trusted) +{ +} + /*}}}*/ +// IndexFile::ArchiveInfo - Stub /*{{{*/ +std::string pkgIndexFile::ArchiveInfo(pkgCache::VerIterator const &) const +{ + return std::string(); +} + /*}}}*/ +// IndexFile::FindInCache - Stub /*{{{*/ +pkgCache::PkgFileIterator pkgIndexFile::FindInCache(pkgCache &Cache) const +{ + return pkgCache::PkgFileIterator(Cache); +} + /*}}}*/ +// IndexFile::SourceIndex - Stub /*{{{*/ +std::string pkgIndexFile::SourceInfo(pkgSrcRecords::Parser const &/*Record*/, + pkgSrcRecords::File const &/*File*/) const +{ + return std::string(); +} + /*}}}*/ + +// IndexTarget - Constructor /*{{{*/ +IndexTarget::IndexTarget(std::string const &MetaKey, std::string const &ShortDesc, + std::string const &LongDesc, std::string const &URI, bool const IsOptional, + bool const KeepCompressed, std::map<std::string, std::string> const &Options) : + URI(URI), Description(LongDesc), ShortDesc(ShortDesc), MetaKey(MetaKey), + IsOptional(IsOptional), KeepCompressed(KeepCompressed), Options(Options) +{ +} + /*}}}*/ +std::string IndexTarget::Option(OptionKeys const EnumKey) const /*{{{*/ +{ + std::string Key; + switch (EnumKey) + { +#define APT_CASE(X) case X: Key = #X; break + APT_CASE(SITE); + APT_CASE(RELEASE); + APT_CASE(COMPONENT); + APT_CASE(LANGUAGE); + APT_CASE(ARCHITECTURE); + APT_CASE(BASE_URI); + APT_CASE(REPO_URI); + APT_CASE(IDENTIFIER); + APT_CASE(TARGET_OF); + APT_CASE(CREATED_BY); + APT_CASE(FALLBACK_OF); + APT_CASE(PDIFFS); + APT_CASE(DEFAULTENABLED); + APT_CASE(COMPRESSIONTYPES); + APT_CASE(SOURCESENTRY); + APT_CASE(BY_HASH); + APT_CASE(KEEPCOMPRESSEDAS); + APT_CASE(ALLOW_INSECURE); + APT_CASE(ALLOW_WEAK); + APT_CASE(ALLOW_DOWNGRADE_TO_INSECURE); + APT_CASE(INRELEASE_PATH); +#undef APT_CASE + case FILENAME: + { + auto const M = Options.find("FILENAME"); + if (M == Options.end()) + return _config->FindDir("Dir::State::lists") + URItoFileName(URI); + return M->second; + } + case EXISTING_FILENAME: + std::string const filename = Option(FILENAME); + std::vector<std::string> const types = VectorizeString(Option(COMPRESSIONTYPES), ' '); + for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t) + { + if (t->empty()) + continue; + std::string const file = (*t == "uncompressed") ? filename : (filename + "." + *t); + if (FileExists(file)) + return file; + } + return ""; + } + std::map<std::string,std::string>::const_iterator const M = Options.find(Key); + if (M == Options.end()) + return ""; + return M->second; +} + /*}}}*/ +bool IndexTarget::OptionBool(OptionKeys const EnumKey) const /*{{{*/ +{ + return StringToBool(Option(EnumKey), false); +} + /*}}}*/ +std::string IndexTarget::Format(std::string format) const /*{{{*/ +{ + for (std::map<std::string, std::string>::const_iterator O = Options.begin(); O != Options.end(); ++O) + { + format = SubstVar(format, std::string("$(") + O->first + ")", O->second); + } + format = SubstVar(format, "$(METAKEY)", MetaKey); + format = SubstVar(format, "$(SHORTDESC)", ShortDesc); + format = SubstVar(format, "$(DESCRIPTION)", Description); + format = SubstVar(format, "$(URI)", URI); + format = SubstVar(format, "$(FILENAME)", Option(IndexTarget::FILENAME)); + return format; +} + /*}}}*/ + +pkgDebianIndexTargetFile::pkgDebianIndexTargetFile(IndexTarget const &Target, bool const Trusted) :/*{{{*/ + pkgDebianIndexFile(Trusted), d(NULL), Target(Target) +{ +} + /*}}}*/ +std::string pkgDebianIndexTargetFile::ArchiveURI(std::string const &File) const/*{{{*/ +{ + return Target.Option(IndexTarget::REPO_URI) + pkgAcquire::URIEncode(File); +} + /*}}}*/ +std::string pkgDebianIndexTargetFile::Describe(bool const Short) const /*{{{*/ +{ + if (Short) + return Target.Description; + return Target.Description + " (" + IndexFileName() + ")"; +} + /*}}}*/ +std::string pkgDebianIndexTargetFile::IndexFileName() const /*{{{*/ +{ + std::string const s = Target.Option(IndexTarget::FILENAME); + if (FileExists(s)) + return s; + + std::vector<std::string> const types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' '); + for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t) + { + std::string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; +} + /*}}}*/ +unsigned long pkgDebianIndexTargetFile::Size() const /*{{{*/ +{ + unsigned long size = 0; + + /* we need to ignore errors here; if the lists are absent, just return 0 */ + _error->PushToStack(); + + FileFd f(IndexFileName(), FileFd::ReadOnly, FileFd::Extension); + if (!f.Failed()) + size = f.Size(); + + if (_error->PendingError() == true) + size = 0; + _error->RevertToStack(); + + return size; +} + /*}}}*/ +bool pkgDebianIndexTargetFile::Exists() const /*{{{*/ +{ + return FileExists(IndexFileName()); +} + /*}}}*/ +std::string pkgDebianIndexTargetFile::GetArchitecture() const /*{{{*/ +{ + return Target.Option(IndexTarget::ARCHITECTURE); +} + /*}}}*/ +std::string pkgDebianIndexTargetFile::GetComponent() const /*{{{*/ +{ + return Target.Option(IndexTarget::COMPONENT); +} + /*}}}*/ +bool pkgDebianIndexTargetFile::OpenListFile(FileFd &Pkg, std::string const &FileName)/*{{{*/ +{ + if (Pkg.Open(FileName, FileFd::ReadOnly, FileFd::Extension) == false) + return _error->Error("Problem opening %s",FileName.c_str()); + return true; +} + /*}}}*/ +std::string pkgDebianIndexTargetFile::GetProgressDescription() const +{ + return Target.Description; +} +IndexTarget pkgDebianIndexTargetFile::GetIndexTarget() const +{ + return Target; +} + +pkgDebianIndexRealFile::pkgDebianIndexRealFile(std::string const &pFile, bool const Trusted) :/*{{{*/ + pkgDebianIndexFile(Trusted), d(NULL) +{ + if (pFile.empty()) + ; + else if (pFile == "/nonexistent/stdin") + File = pFile; + else + File = flAbsPath(pFile); +} + /*}}}*/ +// IndexRealFile::Size - Return the size of the index /*{{{*/ +unsigned long pkgDebianIndexRealFile::Size() const +{ + struct stat S; + if (stat(File.c_str(),&S) != 0) + return 0; + return S.st_size; +} + /*}}}*/ +bool pkgDebianIndexRealFile::Exists() const /*{{{*/ +{ + return FileExists(File); +} + /*}}}*/ +std::string pkgDebianIndexRealFile::Describe(bool const /*Short*/) const/*{{{*/ +{ + return File; +} + /*}}}*/ +std::string pkgDebianIndexRealFile::ArchiveURI(std::string const &/*File*/) const/*{{{*/ +{ + return "file:" + pkgAcquire::URIEncode(File); +} + /*}}}*/ +std::string pkgDebianIndexRealFile::IndexFileName() const /*{{{*/ +{ + return File; +} + /*}}}*/ +std::string pkgDebianIndexRealFile::GetProgressDescription() const +{ + return File; +} +bool pkgDebianIndexRealFile::OpenListFile(FileFd &Pkg, std::string const &FileName)/*{{{*/ +{ + if (Pkg.Open(FileName, FileFd::ReadOnly, FileFd::Extension) == false) + return _error->Error("Problem opening %s",FileName.c_str()); + return true; +} + /*}}}*/ + +pkgDebianIndexFile::pkgDebianIndexFile(bool const Trusted) : pkgIndexFile(Trusted) +{ +} +pkgDebianIndexFile::~pkgDebianIndexFile() +{ +} +pkgCacheListParser * pkgDebianIndexFile::CreateListParser(FileFd &Pkg) +{ + if (Pkg.IsOpen() == false) + return nullptr; + _error->PushToStack(); + std::unique_ptr<pkgCacheListParser> Parser(new debListParser(&Pkg)); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + return newError ? nullptr : Parser.release(); +} +bool pkgDebianIndexFile::Merge(pkgCacheGenerator &Gen,OpProgress * const Prog) +{ + std::string const PackageFile = IndexFileName(); + FileFd Pkg; + if (OpenListFile(Pkg, PackageFile) == false) + return false; + _error->PushToStack(); + std::unique_ptr<pkgCacheListParser> Parser(CreateListParser(Pkg)); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + if (newError == false && Parser == nullptr) + return true; + if (Parser == NULL) + return false; + + if (Prog != NULL) + Prog->SubProgress(0, GetProgressDescription()); + + if (Gen.SelectFile(PackageFile, *this, GetArchitecture(), GetComponent(), GetIndexFlags()) == false) + return _error->Error("Problem with SelectFile %s",PackageFile.c_str()); + + // Store the IMS information + pkgCache::PkgFileIterator File = Gen.GetCurFile(); + pkgCacheGenerator::Dynamic<pkgCache::PkgFileIterator> DynFile(File); + File->Size = Pkg.FileSize(); + File->mtime = Pkg.ModificationTime(); + + if (Gen.MergeList(*Parser) == false) + return _error->Error("Problem with MergeList %s",PackageFile.c_str()); + return true; +} +pkgCache::PkgFileIterator pkgDebianIndexFile::FindInCache(pkgCache &Cache) const +{ + std::string const FileName = IndexFileName(); + pkgCache::PkgFileIterator File = Cache.FileBegin(); + for (; File.end() == false; ++File) + { + if (File.FileName() == NULL || FileName != File.FileName()) + continue; + + struct stat St; + if (stat(File.FileName(),&St) != 0) + { + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "DebianIndexFile::FindInCache - stat failed on " << File.FileName() << std::endl; + return pkgCache::PkgFileIterator(Cache); + } + if ((map_filesize_t)St.st_size != File->Size || St.st_mtime != File->mtime) + { + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "DebianIndexFile::FindInCache - size (" << St.st_size << " <> " << File->Size + << ") or mtime (" << St.st_mtime << " <> " << File->mtime + << ") doesn't match for " << File.FileName() << std::endl; + return pkgCache::PkgFileIterator(Cache); + } + return File; + } + + return File; +} + +pkgIndexFile::~pkgIndexFile() {} +pkgDebianIndexTargetFile::~pkgDebianIndexTargetFile() {} +pkgDebianIndexRealFile::~pkgDebianIndexRealFile() {} diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h new file mode 100644 index 0000000..d0b045a --- /dev/null +++ b/apt-pkg/indexfile.h @@ -0,0 +1,214 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Index File - Abstraction for an index of archive/source file. + + There are 4 primary sorts of index files, all represented by this + class: + + Binary index files + Binary translation files + Binary index files describing the local system + Source index files + + They are all bundled together here, and the interfaces for + sources.list, acquire, cache gen and record parsing all use this class + to access the underlying representation. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_INDEXFILE_H +#define PKGLIB_INDEXFILE_H + +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgrecords.h> +#include <apt-pkg/srcrecords.h> + +#include <map> +#include <string> + + +class pkgCacheGenerator; +class pkgCacheListParser; +class OpProgress; + +class APT_PUBLIC IndexTarget /*{{{*/ +/** \brief Information about an index file. */ +{ + public: + /** \brief A URI from which the index file can be downloaded. */ + std::string URI; + + /** \brief A description of the index file. */ + std::string Description; + + /** \brief A shorter description of the index file. */ + std::string ShortDesc; + + /** \brief The key by which this index file should be + looked up within the meta index file. */ + std::string MetaKey; + + /** \brief Is it okay if the file isn't found in the meta index */ + bool IsOptional; + + /** \brief If the file is downloaded compressed, do not unpack it */ + bool KeepCompressed; + + /** \brief options with which this target was created + Prefer the usage of #Option if at all possible. + Beware: Not all of these options are intended for public use */ + std::map<std::string, std::string> Options; + + IndexTarget(std::string const &MetaKey, std::string const &ShortDesc, + std::string const &LongDesc, std::string const &URI, bool const IsOptional, + bool const KeepCompressed, std::map<std::string, std::string> const &Options); + + enum OptionKeys + { + SITE, + RELEASE, + COMPONENT, + LANGUAGE, + ARCHITECTURE, + BASE_URI, + REPO_URI, + CREATED_BY, + TARGET_OF, + FILENAME, + EXISTING_FILENAME, + PDIFFS, + COMPRESSIONTYPES, + DEFAULTENABLED, + SOURCESENTRY, + BY_HASH, + KEEPCOMPRESSEDAS, + FALLBACK_OF, + IDENTIFIER, + ALLOW_INSECURE, + ALLOW_WEAK, + ALLOW_DOWNGRADE_TO_INSECURE, + INRELEASE_PATH, + }; + std::string Option(OptionKeys const Key) const; + bool OptionBool(OptionKeys const Key) const; + std::string Format(std::string format) const; +}; + /*}}}*/ + +class APT_PUBLIC pkgIndexFile +{ + void * const d; + protected: + bool Trusted; + + public: + + class APT_PUBLIC Type + { + public: + + // Global list of Items supported + static Type **GlobalList; + static unsigned long GlobalListLen; + static Type *GetType(const char * const Type) APT_PURE; + + const char *Label; + + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator const &/*File*/) const {return 0;}; + virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string const &/*File*/) const {return 0;}; + Type(); + virtual ~Type() {}; + }; + + virtual const Type *GetType() const = 0; + + // Return descriptive strings of various sorts + virtual std::string ArchiveInfo(pkgCache::VerIterator const &Ver) const; + virtual std::string SourceInfo(pkgSrcRecords::Parser const &Record, + pkgSrcRecords::File const &File) const; + virtual std::string Describe(bool const Short = false) const = 0; + + // Interface for acquire + virtual std::string ArchiveURI(std::string const &/*File*/) const {return std::string();}; + + // Interface for the record parsers + virtual pkgSrcRecords::Parser *CreateSrcParser() const {return 0;}; + + // Interface for the Cache Generator + virtual bool Exists() const = 0; + virtual bool HasPackages() const = 0; + virtual unsigned long Size() const = 0; + virtual bool Merge(pkgCacheGenerator &/*Gen*/, OpProgress* const /*Prog*/) { return true; }; + virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const; + + bool IsTrusted() const { return Trusted; }; + + explicit pkgIndexFile(bool const Trusted); + virtual ~pkgIndexFile(); +}; + +class APT_PUBLIC pkgDebianIndexFile : public pkgIndexFile +{ +protected: + virtual std::string IndexFileName() const = 0; + virtual std::string GetComponent() const = 0; + virtual std::string GetArchitecture() const = 0; + virtual std::string GetProgressDescription() const = 0; + virtual uint8_t GetIndexFlags() const = 0; + virtual bool OpenListFile(FileFd &Pkg, std::string const &FileName) = 0; + APT_HIDDEN virtual pkgCacheListParser * CreateListParser(FileFd &Pkg); + +public: + virtual bool Merge(pkgCacheGenerator &Gen, OpProgress* const Prog) APT_OVERRIDE; + virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const APT_OVERRIDE; + + explicit pkgDebianIndexFile(bool const Trusted); + virtual ~pkgDebianIndexFile(); +}; + +class APT_PUBLIC pkgDebianIndexTargetFile : public pkgDebianIndexFile +{ + void * const d; +protected: + IndexTarget const Target; + + virtual std::string IndexFileName() const APT_OVERRIDE; + virtual std::string GetComponent() const APT_OVERRIDE; + virtual std::string GetArchitecture() const APT_OVERRIDE; + virtual std::string GetProgressDescription() const APT_OVERRIDE; + virtual bool OpenListFile(FileFd &Pkg, std::string const &FileName) APT_OVERRIDE; + +public: + virtual std::string ArchiveURI(std::string const &File) const APT_OVERRIDE; + virtual std::string Describe(bool const Short = false) const APT_OVERRIDE; + virtual bool Exists() const APT_OVERRIDE; + virtual unsigned long Size() const APT_OVERRIDE; + IndexTarget GetIndexTarget() const APT_HIDDEN; + + pkgDebianIndexTargetFile(IndexTarget const &Target, bool const Trusted); + virtual ~pkgDebianIndexTargetFile(); +}; + +class APT_PUBLIC pkgDebianIndexRealFile : public pkgDebianIndexFile +{ + void * const d; +protected: + std::string File; + + virtual std::string IndexFileName() const APT_OVERRIDE; + virtual std::string GetProgressDescription() const APT_OVERRIDE; + virtual bool OpenListFile(FileFd &Pkg, std::string const &FileName) APT_OVERRIDE; +public: + virtual std::string Describe(bool const /*Short*/ = false) const APT_OVERRIDE; + virtual bool Exists() const APT_OVERRIDE; + virtual unsigned long Size() const APT_OVERRIDE; + virtual std::string ArchiveURI(std::string const &/*File*/) const APT_OVERRIDE; + + pkgDebianIndexRealFile(std::string const &File, bool const Trusted); + virtual ~pkgDebianIndexRealFile(); +}; + +#endif diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc new file mode 100644 index 0000000..b9d9b15 --- /dev/null +++ b/apt-pkg/init.cc @@ -0,0 +1,289 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Init - Initialize the package library + + ##################################################################### */ + /*}}}*/ +// Include files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/init.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgsystem.h> +#include <apt-pkg/strutl.h> + +#include <cstdlib> +#include <fstream> +#include <sstream> +#include <string> +#include <unordered_map> +#include <vector> +#include <string.h> + +#include <apti18n.h> + /*}}}*/ + +#define Stringfy_(x) # x +#define Stringfy(x) Stringfy_(x) +const char *pkgVersion = PACKAGE_VERSION; +const char *pkgLibVersion = Stringfy(APT_PKG_MAJOR) "." + Stringfy(APT_PKG_MINOR) "." + Stringfy(APT_PKG_RELEASE); +namespace APT { + APT_HIDDEN extern std::unordered_map<std::string, std::vector<std::string>> ArchToTupleMap; +} + +// Splits by whitespace. There may be continuous spans of whitespace - they +// will be considered as one. +static std::vector<std::string> split(std::string const & s) +{ + std::vector<std::string> vec; + std::istringstream iss(s); + iss.imbue(std::locale::classic()); + for(std::string current_s; iss >> current_s; ) + vec.push_back(current_s); + return vec; +} + + +// pkgInitArchTupleMap - Initialize the architecture tuple map /*{{{*/ +// --------------------------------------------------------------------- +/* This initializes */ +static bool pkgInitArchTupleMap() +{ + auto tuplepath = _config->FindFile("Dir::dpkg::tupletable", DPKG_DATADIR "/tupletable"); + auto tripletpath = _config->FindFile("Dir::dpkg::triplettable", DPKG_DATADIR "/triplettable"); + auto cpupath = _config->FindFile("Dir::dpkg::cputable", DPKG_DATADIR "/cputable"); + + // Load a list of CPUs + std::vector<std::string> cpus; + std::ifstream cputable(cpupath); + for (std::string cpuline; std::getline(cputable, cpuline); ) + { + if (cpuline[0] == '#' || cpuline[0] == '\0') + continue; + auto cpurow = split(cpuline); + auto cpu = APT::String::Strip(cpurow.at(0)); + + cpus.push_back(cpu); + } + if (!cputable.eof()) + return _error->Error("Error reading the CPU table"); + + // Load the architecture tuple + std::ifstream tupletable; + if (FileExists(tuplepath)) + tupletable.open(tuplepath); + else if (FileExists(tripletpath)) + tupletable.open(tripletpath); + else + return _error->Error("Cannot find dpkg tuplet or triplet table"); + + APT::ArchToTupleMap.clear(); + for (std::string tupleline; std::getline(tupletable, tupleline); ) + { + if (tupleline[0] == '#' || tupleline[0] == '\0') + continue; + + std::vector<std::string> tuplerow = split(tupleline); + + auto tuple = APT::String::Strip(tuplerow.at(0)); + auto arch = APT::String::Strip(tuplerow.at(1)); + + if (tuple.find("<cpu>") == tuple.npos && arch.find("<cpu>") == arch.npos) + { + APT::ArchToTupleMap.insert({arch, VectorizeString(tuple, '-')}); + } + else + { + for (auto && cpu : cpus) + { + auto mytuple = SubstVar(tuple, std::string("<cpu>"), cpu); + auto myarch = SubstVar(arch, std::string("<cpu>"), cpu); + + APT::ArchToTupleMap.insert({myarch, VectorizeString(mytuple, '-')}); + } + } + } + if (!tupletable.eof()) + return _error->Error("Error reading the Tuple table"); + + return true; +} + /*}}}*/ + + +// pkgInitConfig - Initialize the configuration class /*{{{*/ +// --------------------------------------------------------------------- +/* Directories are specified in such a way that the FindDir function will + understand them. That is, if they don't start with a / then their parent + is prepended, this allows a fair degree of flexibility. */ +bool pkgInitConfig(Configuration &Cnf) +{ + // General APT things + Cnf.CndSet("APT::Architecture", COMMON_ARCH); + if (Cnf.Exists("APT::Build-Essential") == false) + Cnf.Set("APT::Build-Essential::", "build-essential"); + Cnf.CndSet("APT::Install-Recommends", true); + Cnf.CndSet("APT::Install-Suggests", false); + Cnf.CndSet("Dir","/"); + + // State + Cnf.CndSet("Dir::State", &STATE_DIR[1]); + Cnf.CndSet("Dir::State::lists","lists/"); + Cnf.CndSet("Dir::State::cdroms","cdroms.list"); + + // Cache + Cnf.CndSet("Dir::Cache", &CACHE_DIR[1]); + Cnf.CndSet("Dir::Cache::archives","archives/"); + Cnf.CndSet("Dir::Cache::srcpkgcache","srcpkgcache.bin"); + Cnf.CndSet("Dir::Cache::pkgcache","pkgcache.bin"); + + // Configuration + Cnf.CndSet("Dir::Etc", &CONF_DIR[1]); + Cnf.CndSet("Dir::Etc::sourcelist","sources.list"); + Cnf.CndSet("Dir::Etc::sourceparts","sources.list.d"); + Cnf.CndSet("Dir::Etc::main","apt.conf"); + Cnf.CndSet("Dir::Etc::netrc", "auth.conf"); + Cnf.CndSet("Dir::Etc::netrcparts", "auth.conf.d"); + Cnf.CndSet("Dir::Etc::parts","apt.conf.d"); + Cnf.CndSet("Dir::Etc::preferences","preferences"); + Cnf.CndSet("Dir::Etc::preferencesparts","preferences.d"); + Cnf.CndSet("Dir::Etc::trusted", "trusted.gpg"); + Cnf.CndSet("Dir::Etc::trustedparts","trusted.gpg.d"); + Cnf.CndSet("Dir::Bin::methods", LIBEXEC_DIR "/methods"); + Cnf.CndSet("Dir::Bin::solvers::",LIBEXEC_DIR "/solvers"); + Cnf.CndSet("Dir::Bin::planners::",LIBEXEC_DIR "/planners"); + Cnf.CndSet("Dir::Media::MountPath","/media/apt"); + + // State + Cnf.CndSet("Dir::Log", &LOG_DIR[1]); + Cnf.CndSet("Dir::Log::Terminal","term.log"); + Cnf.CndSet("Dir::Log::History","history.log"); + Cnf.CndSet("Dir::Log::Planner","eipp.log.xz"); + + Cnf.Set("Dir::Ignore-Files-Silently::", "~$"); + Cnf.Set("Dir::Ignore-Files-Silently::", "\\.disabled$"); + Cnf.Set("Dir::Ignore-Files-Silently::", "\\.bak$"); + Cnf.Set("Dir::Ignore-Files-Silently::", "\\.dpkg-[a-z]+$"); + Cnf.Set("Dir::Ignore-Files-Silently::", "\\.ucf-[a-z]+$"); + Cnf.Set("Dir::Ignore-Files-Silently::", "\\.save$"); + Cnf.Set("Dir::Ignore-Files-Silently::", "\\.orig$"); + Cnf.Set("Dir::Ignore-Files-Silently::", "\\.distUpgrade$"); + + // Repository security + Cnf.CndSet("Acquire::AllowInsecureRepositories", false); + Cnf.CndSet("Acquire::AllowWeakRepositories", false); + Cnf.CndSet("Acquire::AllowDowngradeToInsecureRepositories", false); + + // Default cdrom mount point + Cnf.CndSet("Acquire::cdrom::mount", "/media/cdrom/"); + + // The default user we drop to in the methods + Cnf.CndSet("APT::Sandbox::User", "_apt"); + + Cnf.CndSet("Acquire::IndexTargets::deb::Packages::MetaKey", "$(COMPONENT)/binary-$(ARCHITECTURE)/Packages"); + Cnf.CndSet("Acquire::IndexTargets::deb::Packages::flatMetaKey", "Packages"); + Cnf.CndSet("Acquire::IndexTargets::deb::Packages::ShortDescription", "Packages"); + Cnf.CndSet("Acquire::IndexTargets::deb::Packages::Description", "$(RELEASE)/$(COMPONENT) $(ARCHITECTURE) Packages"); + Cnf.CndSet("Acquire::IndexTargets::deb::Packages::flatDescription", "$(RELEASE) Packages"); + Cnf.CndSet("Acquire::IndexTargets::deb::Packages::Optional", false); + Cnf.CndSet("Acquire::IndexTargets::deb::Translations::MetaKey", "$(COMPONENT)/i18n/Translation-$(LANGUAGE)"); + Cnf.CndSet("Acquire::IndexTargets::deb::Translations::flatMetaKey", "$(LANGUAGE)"); + Cnf.CndSet("Acquire::IndexTargets::deb::Translations::ShortDescription", "Translation-$(LANGUAGE)"); + Cnf.CndSet("Acquire::IndexTargets::deb::Translations::Description", "$(RELEASE)/$(COMPONENT) Translation-$(LANGUAGE)"); + Cnf.CndSet("Acquire::IndexTargets::deb::Translations::flatDescription", "$(RELEASE) Translation-$(LANGUAGE)"); + Cnf.CndSet("Acquire::IndexTargets::deb-src::Sources::MetaKey", "$(COMPONENT)/source/Sources"); + Cnf.CndSet("Acquire::IndexTargets::deb-src::Sources::flatMetaKey", "Sources"); + Cnf.CndSet("Acquire::IndexTargets::deb-src::Sources::ShortDescription", "Sources"); + Cnf.CndSet("Acquire::IndexTargets::deb-src::Sources::Description", "$(RELEASE)/$(COMPONENT) Sources"); + Cnf.CndSet("Acquire::IndexTargets::deb-src::Sources::flatDescription", "$(RELEASE) Sources"); + Cnf.CndSet("Acquire::IndexTargets::deb-src::Sources::Optional", false); + + Cnf.CndSet("Acquire::Changelogs::URI::Origin::Debian", "https://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog"); + Cnf.CndSet("Acquire::Changelogs::URI::Origin::Ubuntu", "https://changelogs.ubuntu.com/changelogs/pool/@CHANGEPATH@/changelog"); + Cnf.CndSet("Acquire::Changelogs::AlwaysOnline::Origin::Ubuntu", true); + + + Cnf.CndSet("DPkg::Path", "/usr/sbin:/usr/bin:/sbin:/bin"); + + // Read an alternate config file + _error->PushToStack(); + const char *Cfg = getenv("APT_CONFIG"); + if (Cfg != 0 && strlen(Cfg) != 0) + { + if (RealFileExists(Cfg) == true) + ReadConfigFile(Cnf, Cfg); + else + _error->WarningE("RealFileExists",_("Unable to read %s"),Cfg); + } + + // Read the configuration parts dir + std::string const Parts = Cnf.FindDir("Dir::Etc::parts", "/dev/null"); + if (DirectoryExists(Parts) == true) + ReadConfigDir(Cnf, Parts); + else if (APT::String::Endswith(Parts, "/dev/null") == false) + _error->WarningE("DirectoryExists",_("Unable to read %s"),Parts.c_str()); + + // Read the main config file + std::string const FName = Cnf.FindFile("Dir::Etc::main", "/dev/null"); + if (RealFileExists(FName) == true) + ReadConfigFile(Cnf, FName); + + if (Cnf.FindB("Debug::pkgInitConfig",false) == true) + Cnf.Dump(); + +#ifdef APT_DOMAIN + if (Cnf.Exists("Dir::Locale")) + { + bindtextdomain(APT_DOMAIN,Cnf.FindDir("Dir::Locale").c_str()); + bindtextdomain(textdomain(0),Cnf.FindDir("Dir::Locale").c_str()); + } +#endif + + auto const good = _error->PendingError() == false; + _error->MergeWithStack(); + return good; +} + /*}}}*/ +// pkgInitSystem - Initialize the _system class /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgInitSystem(Configuration &Cnf,pkgSystem *&Sys) +{ + Sys = 0; + std::string Label = Cnf.Find("Apt::System",""); + if (Label.empty() == false) + { + Sys = pkgSystem::GetSystem(Label.c_str()); + if (Sys == 0) + return _error->Error(_("Packaging system '%s' is not supported"),Label.c_str()); + } + else + { + signed MaxScore = 0; + for (unsigned I = 0; I != pkgSystem::GlobalListLen; I++) + { + signed Score = pkgSystem::GlobalList[I]->Score(Cnf); + if (Score > MaxScore) + { + MaxScore = Score; + Sys = pkgSystem::GlobalList[I]; + } + } + + if (Sys == 0) + return _error->Error(_("Unable to determine a suitable packaging system type")); + } + + if (pkgInitArchTupleMap() == false) + return false; + + return Sys->Initialize(Cnf); +} + /*}}}*/ diff --git a/apt-pkg/init.h b/apt-pkg/init.h new file mode 100644 index 0000000..431e8a5 --- /dev/null +++ b/apt-pkg/init.h @@ -0,0 +1,26 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Init - Initialize the package library + + This function must be called to configure the config class before + calling many APT library functions. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_INIT_H +#define PKGLIB_INIT_H + +#include <apt-pkg/macros.h> + +class pkgSystem; +class Configuration; + +APT_PUBLIC extern const char *pkgVersion; +APT_PUBLIC extern const char *pkgLibVersion; + +APT_PUBLIC bool pkgInitConfig(Configuration &Cnf); +APT_PUBLIC bool pkgInitSystem(Configuration &Cnf,pkgSystem *&Sys); + +#endif diff --git a/apt-pkg/install-progress.cc b/apt-pkg/install-progress.cc new file mode 100644 index 0000000..c7f7573 --- /dev/null +++ b/apt-pkg/install-progress.cc @@ -0,0 +1,443 @@ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/install-progress.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <cmath> +#include <iostream> +#include <sstream> +#include <vector> +#include <fcntl.h> +#include <signal.h> +#include <stdio.h> +#include <sys/ioctl.h> +#include <unistd.h> + +#include <apti18n.h> + +namespace APT { +namespace Progress { + +PackageManager::PackageManager() : d(NULL), percentage(0.0), last_reported_progress(-1) {} +PackageManager::~PackageManager() {} + +/* Return a APT::Progress::PackageManager based on the global + * apt configuration (i.e. APT::Status-Fd and APT::Status-deb822-Fd) + */ +PackageManager* PackageManagerProgressFactory() +{ + // select the right progress + int status_fd = _config->FindI("APT::Status-Fd", -1); + int status_deb822_fd = _config->FindI("APT::Status-deb822-Fd", -1); + + APT::Progress::PackageManager *progress = NULL; + if (status_deb822_fd > 0) + progress = new APT::Progress::PackageManagerProgressDeb822Fd( + status_deb822_fd); + else if (status_fd > 0) + progress = new APT::Progress::PackageManagerProgressFd(status_fd); + else if(_config->FindB("Dpkg::Progress-Fancy", false) == true) + progress = new APT::Progress::PackageManagerFancy(); + else if (_config->FindB("Dpkg::Progress", + _config->FindB("DpkgPM::Progress", false)) == true) + progress = new APT::Progress::PackageManagerText(); + else + progress = new APT::Progress::PackageManager(); + return progress; +} + +bool PackageManager::StatusChanged(std::string /*PackageName*/, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string /*HumanReadableAction*/) +{ + int reporting_steps = _config->FindI("DpkgPM::Reporting-Steps", 1); + percentage = StepsDone/(double)TotalSteps * 100.0; + strprintf(progress_str, _("Progress: [%3li%%]"), std::lround(percentage)); + + if(percentage < (last_reported_progress + reporting_steps)) + return false; + + return true; +} + +PackageManagerProgressFd::PackageManagerProgressFd(int progress_fd) + : d(NULL), StepsDone(0), StepsTotal(1) +{ + OutStatusFd = progress_fd; +} +PackageManagerProgressFd::~PackageManagerProgressFd() {} + +void PackageManagerProgressFd::WriteToStatusFd(std::string s) +{ + if(OutStatusFd <= 0) + return; + FileFd::Write(OutStatusFd, s.c_str(), s.size()); +} + +static std::string GetProgressFdString(char const * const status, + char const * const pkg, unsigned long long Done, + unsigned long long Total, char const * const msg) +{ + float const progress{Done / static_cast<float>(Total) * 100}; + std::ostringstream str; + str.imbue(std::locale::classic()); + str.precision(4); + str << status << ':' << pkg << ':' << std::fixed << progress << ':' << msg << '\n'; + return str.str(); +} + +void PackageManagerProgressFd::StartDpkg() +{ + if(OutStatusFd <= 0) + return; + + // FIXME: use SetCloseExec here once it taught about throwing + // exceptions instead of doing _exit(100) on failure + fcntl(OutStatusFd,F_SETFD,FD_CLOEXEC); + + // send status information that we are about to fork dpkg + WriteToStatusFd(GetProgressFdString("pmstatus", "dpkg-exec", StepsDone, StepsTotal, _("Running dpkg"))); +} + +void PackageManagerProgressFd::Stop() +{ +} + +void PackageManagerProgressFd::Error(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string ErrorMessage) +{ + WriteToStatusFd(GetProgressFdString("pmerror", PackageName.c_str(), + StepsDone, TotalSteps, ErrorMessage.c_str())); +} + +void PackageManagerProgressFd::ConffilePrompt(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string ConfMessage) +{ + WriteToStatusFd(GetProgressFdString("pmconffile", PackageName.c_str(), + StepsDone, TotalSteps, ConfMessage.c_str())); +} + + +bool PackageManagerProgressFd::StatusChanged(std::string PackageName, + unsigned int xStepsDone, + unsigned int xTotalSteps, + std::string pkg_action) +{ + StepsDone = xStepsDone; + StepsTotal = xTotalSteps; + + WriteToStatusFd(GetProgressFdString("pmstatus", StringSplit(PackageName, ":")[0].c_str(), + StepsDone, StepsTotal, pkg_action.c_str())); + + if(_config->FindB("Debug::APT::Progress::PackageManagerFd", false) == true) + std::cerr << "progress: " << PackageName << " " << xStepsDone + << " " << xTotalSteps << " " << pkg_action + << std::endl; + + + return true; +} + + +PackageManagerProgressDeb822Fd::PackageManagerProgressDeb822Fd(int progress_fd) + : d(NULL), StepsDone(0), StepsTotal(1) +{ + OutStatusFd = progress_fd; +} +PackageManagerProgressDeb822Fd::~PackageManagerProgressDeb822Fd() {} + +void PackageManagerProgressDeb822Fd::WriteToStatusFd(std::string s) +{ + FileFd::Write(OutStatusFd, s.c_str(), s.size()); +} + +static std::string GetProgressDeb822String(char const * const status, + char const * const pkg, unsigned long long Done, + unsigned long long Total, char const * const msg) +{ + float const progress{Done / static_cast<float>(Total) * 100}; + std::ostringstream str; + str.imbue(std::locale::classic()); + str.precision(4); + str << "Status: " << status << '\n'; + if (pkg != nullptr) + str << "Package: " << pkg << '\n'; + str << "Percent: " << std::fixed << progress << '\n' + << "Message: " << msg << "\n\n"; + return str.str(); +} + +void PackageManagerProgressDeb822Fd::StartDpkg() +{ + // FIXME: use SetCloseExec here once it taught about throwing + // exceptions instead of doing _exit(100) on failure + fcntl(OutStatusFd,F_SETFD,FD_CLOEXEC); + + WriteToStatusFd(GetProgressDeb822String("progress", nullptr, StepsDone, StepsTotal, _("Running dpkg"))); +} + +void PackageManagerProgressDeb822Fd::Stop() +{ +} + +void PackageManagerProgressDeb822Fd::Error(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string ErrorMessage) +{ + WriteToStatusFd(GetProgressDeb822String("Error", PackageName.c_str(), StepsDone, TotalSteps, ErrorMessage.c_str())); +} + +void PackageManagerProgressDeb822Fd::ConffilePrompt(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string ConfMessage) +{ + WriteToStatusFd(GetProgressDeb822String("ConfFile", PackageName.c_str(), StepsDone, TotalSteps, ConfMessage.c_str())); +} + + +bool PackageManagerProgressDeb822Fd::StatusChanged(std::string PackageName, + unsigned int xStepsDone, + unsigned int xTotalSteps, + std::string message) +{ + StepsDone = xStepsDone; + StepsTotal = xTotalSteps; + + WriteToStatusFd(GetProgressDeb822String("progress", PackageName.c_str(), StepsDone, StepsTotal, message.c_str())); + return true; +} + + +PackageManagerFancy::PackageManagerFancy() + : d(NULL), child_pty(-1) +{ + // setup terminal size + if (instances.empty()) + SIGWINCH_orig = signal(SIGWINCH, PackageManagerFancy::staticSIGWINCH); + instances.push_back(this); +} +std::vector<PackageManagerFancy*> PackageManagerFancy::instances; +sighandler_t PackageManagerFancy::SIGWINCH_orig; +volatile sig_atomic_t PackageManagerFancy::SIGWINCH_flag = 0; + +PackageManagerFancy::~PackageManagerFancy() +{ + instances.erase(find(instances.begin(), instances.end(), this)); + if (instances.empty()) + signal(SIGWINCH, SIGWINCH_orig); +} + +void PackageManagerFancy::staticSIGWINCH(int /*signum*/) +{ + SIGWINCH_flag = 1; +} + +void PackageManagerFancy::CheckSIGWINCH() +{ + if (SIGWINCH_flag) + { + SIGWINCH_flag = 0; + int errsv = errno; + int const nr_terminal_rows = GetTerminalSize().rows; + SetupTerminalScrollArea(nr_terminal_rows); + DrawStatusLine(); + errno = errsv; + } +} + +void PackageManagerFancy::Pulse() +{ + CheckSIGWINCH(); +} + +PackageManagerFancy::TermSize +PackageManagerFancy::GetTerminalSize() +{ + struct winsize win; + PackageManagerFancy::TermSize s = { 0, 0 }; + + // FIXME: get from "child_pty" instead? + if(ioctl(STDOUT_FILENO, TIOCGWINSZ, (char *)&win) != 0) + return s; + + if(_config->FindB("Debug::InstallProgress::Fancy", false) == true) + std::cerr << "GetTerminalSize: " << win.ws_row << " x " << win.ws_col << std::endl; + + s.rows = win.ws_row; + s.columns = win.ws_col; + return s; +} + +void PackageManagerFancy::SetupTerminalScrollArea(int nr_rows) +{ + if(_config->FindB("Debug::InstallProgress::Fancy", false) == true) + std::cerr << "SetupTerminalScrollArea: " << nr_rows << std::endl; + + if (unlikely(nr_rows <= 1)) + return; + + // scroll down a bit to avoid visual glitch when the screen + // area shrinks by one row + std::cout << "\n"; + + // save cursor + std::cout << "\0337"; + + // set scroll region (this will place the cursor in the top left) + std::cout << "\033[0;" << std::to_string(nr_rows - 1) << "r"; + + // restore cursor but ensure its inside the scrolling area + std::cout << "\0338"; + static const char *move_cursor_up = "\033[1A"; + std::cout << move_cursor_up; + + // ensure its flushed + std::flush(std::cout); + + // setup tty size to ensure xterm/linux console are working properly too + // see bug #731738 + struct winsize win; + if (ioctl(child_pty, TIOCGWINSZ, (char *)&win) != -1) + { + win.ws_row = nr_rows - 1; + ioctl(child_pty, TIOCSWINSZ, (char *)&win); + } +} + +void PackageManagerFancy::HandleSIGWINCH(int) +{ + // for abi compatibility, do not use +} + +void PackageManagerFancy::Start(int a_child_pty) +{ + child_pty = a_child_pty; + int const nr_terminal_rows = GetTerminalSize().rows; + SetupTerminalScrollArea(nr_terminal_rows); +} + +void PackageManagerFancy::Stop() +{ + int const nr_terminal_rows = GetTerminalSize().rows; + if (nr_terminal_rows > 0) + { + SetupTerminalScrollArea(nr_terminal_rows + 1); + + // override the progress line (sledgehammer) + static const char* clear_screen_below_cursor = "\033[J"; + std::cout << clear_screen_below_cursor; + std::flush(std::cout); + } + child_pty = -1; +} + +std::string +PackageManagerFancy::GetTextProgressStr(float Percent, int OutputSize) +{ + std::string output; + if (unlikely(OutputSize < 3)) + return output; + + int const BarSize = OutputSize - 2; // bar without the leading "[" and trailing "]" + int const BarDone = std::max(0, std::min(BarSize, static_cast<int>(std::floor(Percent * BarSize)))); + output.append("["); + std::fill_n(std::fill_n(std::back_inserter(output), BarDone, '#'), BarSize - BarDone, '.'); + output.append("]"); + return output; +} + +bool PackageManagerFancy::StatusChanged(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string HumanReadableAction) +{ + if (!PackageManager::StatusChanged(PackageName, StepsDone, TotalSteps, + HumanReadableAction)) + return false; + + return DrawStatusLine(); +} +bool PackageManagerFancy::DrawStatusLine() +{ + PackageManagerFancy::TermSize const size = GetTerminalSize(); + if (unlikely(size.rows < 1 || size.columns < 1)) + return false; + + static std::string save_cursor = "\0337"; + static std::string restore_cursor = "\0338"; + + // green + static std::string set_bg_color = DeQuoteString( + _config->Find("Dpkg::Progress-Fancy::Progress-fg", "%1b[42m")); + // black + static std::string set_fg_color = DeQuoteString( + _config->Find("Dpkg::Progress-Fancy::Progress-bg", "%1b[30m")); + + static std::string restore_bg = "\033[49m"; + static std::string restore_fg = "\033[39m"; + + std::cout << save_cursor + // move cursor position to last row + << "\033[" << std::to_string(size.rows) << ";0f" + << set_bg_color + << set_fg_color + << progress_str + << restore_bg + << restore_fg; + std::flush(std::cout); + + // draw text progress bar + if (_config->FindB("Dpkg::Progress-Fancy::Progress-Bar", true)) + { + int padding = 4; + auto const progressbar_size = size.columns - padding - String::DisplayLength(progress_str); + auto const current_percent = percentage / 100.0f; + std::cout << " " + << GetTextProgressStr(current_percent, progressbar_size) + << " "; + std::flush(std::cout); + } + + // restore + std::cout << restore_cursor; + std::flush(std::cout); + + last_reported_progress = percentage; + + return true; +} + +bool PackageManagerText::StatusChanged(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string HumanReadableAction) +{ + if (!PackageManager::StatusChanged(PackageName, StepsDone, TotalSteps, HumanReadableAction)) + return false; + + std::cout << progress_str << "\r\n"; + std::flush(std::cout); + + last_reported_progress = percentage; + + return true; +} + +PackageManagerText::PackageManagerText() : PackageManager(), d(NULL) {} +PackageManagerText::~PackageManagerText() {} + + + + +} // namespace progress +} // namespace apt diff --git a/apt-pkg/install-progress.h b/apt-pkg/install-progress.h new file mode 100644 index 0000000..617ce2a --- /dev/null +++ b/apt-pkg/install-progress.h @@ -0,0 +1,180 @@ +#ifndef PKGLIB_IPROGRESS_H +#define PKGLIB_IPROGRESS_H + +#include <apt-pkg/macros.h> + +#include <string> +#include <vector> +#include <signal.h> +#include <unistd.h> + +namespace APT { +namespace Progress { + + class PackageManager; + APT_PUBLIC PackageManager* PackageManagerProgressFactory(); + + class APT_PUBLIC PackageManager + { + private: + /** \brief dpointer placeholder */ + void * const d; + + protected: + std::string progress_str; + float percentage; + int last_reported_progress; + + public: + PackageManager(); + virtual ~PackageManager(); + + /* Global Start/Stop */ + virtual void Start(int /*child_pty*/=-1) {}; + virtual void Stop() {}; + + /* When dpkg is invoked (may happen multiple times for each + * install/remove block + */ + virtual void StartDpkg() {}; + + virtual pid_t fork() {return ::fork(); }; + + virtual void Pulse() {}; + virtual long GetPulseInterval() { + return 50000000; + }; + + virtual bool StatusChanged(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string HumanReadableAction); + virtual void Error(std::string /*PackageName*/, + unsigned int /*StepsDone*/, + unsigned int /*TotalSteps*/, + std::string /*ErrorMessage*/) {} + virtual void ConffilePrompt(std::string /*PackageName*/, + unsigned int /*StepsDone*/, + unsigned int /*TotalSteps*/, + std::string /*ConfMessage*/) {} + }; + + class APT_PUBLIC PackageManagerProgressFd : public PackageManager + { + void * const d; + protected: + int OutStatusFd; + int StepsDone; + int StepsTotal; + void WriteToStatusFd(std::string msg); + + public: + explicit PackageManagerProgressFd(int progress_fd); + virtual ~PackageManagerProgressFd(); + + virtual void StartDpkg() APT_OVERRIDE; + virtual void Stop() APT_OVERRIDE; + + virtual bool StatusChanged(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string HumanReadableAction) APT_OVERRIDE; + virtual void Error(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string ErrorMessage) APT_OVERRIDE; + virtual void ConffilePrompt(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string ConfMessage) APT_OVERRIDE; + + }; + + class APT_PUBLIC PackageManagerProgressDeb822Fd : public PackageManager + { + void * const d; + protected: + int OutStatusFd; + int StepsDone; + int StepsTotal; + void WriteToStatusFd(std::string msg); + + public: + explicit PackageManagerProgressDeb822Fd(int progress_fd); + virtual ~PackageManagerProgressDeb822Fd(); + + virtual void StartDpkg() APT_OVERRIDE; + virtual void Stop() APT_OVERRIDE; + + virtual bool StatusChanged(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string HumanReadableAction) APT_OVERRIDE; + virtual void Error(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string ErrorMessage) APT_OVERRIDE; + virtual void ConffilePrompt(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string ConfMessage) APT_OVERRIDE; + }; + + class APT_PUBLIC PackageManagerFancy : public PackageManager + { + void * const d; + private: + APT_HIDDEN static void staticSIGWINCH(int); + static std::vector<PackageManagerFancy*> instances; + static sighandler_t SIGWINCH_orig; + static volatile sig_atomic_t SIGWINCH_flag; + APT_HIDDEN void CheckSIGWINCH(); + APT_HIDDEN bool DrawStatusLine(); + + protected: + void SetupTerminalScrollArea(int nr_rows); + void HandleSIGWINCH(int); // for abi compatibility, do not use + + typedef struct { + int rows; + int columns; + } TermSize; + TermSize GetTerminalSize(); + + sighandler_t old_SIGWINCH; // for abi compatibility, do not use + int child_pty; + + public: + PackageManagerFancy(); + virtual ~PackageManagerFancy(); + virtual void Pulse() APT_OVERRIDE; + virtual void Start(int child_pty=-1) APT_OVERRIDE; + virtual void Stop() APT_OVERRIDE; + virtual bool StatusChanged(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string HumanReadableAction) APT_OVERRIDE; + + // return a progress bar of the given size for the given progress + // percent between 0.0 and 1.0 in the form "[####...]" + static std::string GetTextProgressStr(float percent, int OutputSize); + }; + + class APT_PUBLIC PackageManagerText : public PackageManager + { + void * const d; + public: + virtual bool StatusChanged(std::string PackageName, + unsigned int StepsDone, + unsigned int TotalSteps, + std::string HumanReadableAction) APT_OVERRIDE; + + PackageManagerText(); + virtual ~PackageManagerText(); + }; + + +} // namespace Progress +} // namespace APT + +#endif diff --git a/apt-pkg/metaindex.cc b/apt-pkg/metaindex.cc new file mode 100644 index 0000000..97996b3 --- /dev/null +++ b/apt-pkg/metaindex.cc @@ -0,0 +1,150 @@ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/indexfile.h> +#include <apt-pkg/metaindex.h> +#include <apt-pkg/pkgcachegen.h> + +#include <apt-pkg/debmetaindex.h> + +#include <string> +#include <vector> + /*}}}*/ + +class metaIndexPrivate /*{{{*/ +{ +}; + /*}}}*/ + +std::string metaIndex::Describe() const +{ + return "Release"; +} + +pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache, bool const) const +{ + return pkgCache::RlsFileIterator(Cache); +} + +bool metaIndex::Merge(pkgCacheGenerator &Gen,OpProgress *) const +{ + return Gen.SelectReleaseFile("", ""); +} + +metaIndex::metaIndex(std::string const &URI, std::string const &Dist, + char const * const Type) +: d(new metaIndexPrivate()), Indexes(NULL), Type(Type), URI(URI), Dist(Dist), Trusted(TRI_UNSET), + Date(0), ValidUntil(0), SupportsAcquireByHash(false), LoadedSuccessfully(TRI_UNSET) +{ + /* nothing */ +} + +metaIndex::~metaIndex() +{ + if (Indexes != 0) + { + for (std::vector<pkgIndexFile *>::iterator I = (*Indexes).begin(); + I != (*Indexes).end(); ++I) + delete *I; + delete Indexes; + } + for (auto const &E: Entries) + delete E.second; + delete d; +} + +// one line Getters for public fields /*{{{*/ +APT_PURE std::string metaIndex::GetURI() const { return URI; } +APT_PURE std::string metaIndex::GetDist() const { return Dist; } +APT_PURE const char* metaIndex::GetType() const { return Type; } +APT_PURE metaIndex::TriState metaIndex::GetTrusted() const { return Trusted; } +APT_PURE std::string metaIndex::GetSignedBy() const { return SignedBy; } +APT_PURE std::string metaIndex::GetOrigin() const { return Origin; } +APT_PURE std::string metaIndex::GetLabel() const { return Label; } +APT_PURE std::string metaIndex::GetVersion() const { return Version; } +APT_PURE std::string metaIndex::GetCodename() const { return Codename; } +APT_PURE std::string metaIndex::GetSuite() const { return Suite; } +APT_PURE std::string metaIndex::GetReleaseNotes() const { return ReleaseNotes; } +APT_PURE signed short metaIndex::GetDefaultPin() const { return DefaultPin; } +APT_PURE bool metaIndex::GetSupportsAcquireByHash() const { return SupportsAcquireByHash; } +APT_PURE time_t metaIndex::GetValidUntil() const { return ValidUntil; } +APT_PURE time_t metaIndex::GetDate() const { return this->Date; } +APT_PURE metaIndex::TriState metaIndex::GetLoadedSuccessfully() const { return LoadedSuccessfully; } +APT_PURE std::string metaIndex::GetExpectedDist() const { return Dist; } + /*}}}*/ +bool metaIndex::CheckDist(std::string const &MaybeDist) const /*{{{*/ +{ + if (MaybeDist.empty() || this->Codename == MaybeDist || this->Suite == MaybeDist) + return true; + + std::string Transformed = MaybeDist; + if (Transformed == "../project/experimental") + Transformed = "experimental"; + + auto const pos = Transformed.rfind('/'); + if (pos != std::string::npos) + Transformed = Transformed.substr(0, pos); + + if (Transformed == ".") + Transformed.clear(); + + return Transformed.empty() || this->Codename == Transformed || this->Suite == Transformed; +} + /*}}}*/ +APT_PURE metaIndex::checkSum *metaIndex::Lookup(std::string const &MetaKey) const /*{{{*/ +{ + std::map<std::string, metaIndex::checkSum* >::const_iterator sum = Entries.find(MetaKey); + if (sum == Entries.end()) + return NULL; + return sum->second; +} + /*}}}*/ +APT_PURE bool metaIndex::Exists(std::string const &MetaKey) const /*{{{*/ +{ + return Entries.find(MetaKey) != Entries.end(); +} + /*}}}*/ +std::vector<std::string> metaIndex::MetaKeys() const /*{{{*/ +{ + std::vector<std::string> keys; + std::map<std::string, checkSum *>::const_iterator I = Entries.begin(); + while(I != Entries.end()) { + keys.push_back((*I).first); + ++I; + } + return keys; +} + /*}}}*/ +void metaIndex::swapLoad(metaIndex * const OldMetaIndex) /*{{{*/ +{ + std::swap(SignedBy, OldMetaIndex->SignedBy); + std::swap(Suite, OldMetaIndex->Suite); + std::swap(Codename, OldMetaIndex->Codename); + std::swap(Date, OldMetaIndex->Date); + std::swap(ValidUntil, OldMetaIndex->ValidUntil); + std::swap(SupportsAcquireByHash, OldMetaIndex->SupportsAcquireByHash); + std::swap(Entries, OldMetaIndex->Entries); + std::swap(LoadedSuccessfully, OldMetaIndex->LoadedSuccessfully); + + OldMetaIndex->Origin = Origin; + OldMetaIndex->Label = Label; + OldMetaIndex->Version =Version; + OldMetaIndex->DefaultPin = DefaultPin; +} + /*}}}*/ + +bool metaIndex::IsArchitectureSupported(std::string const &) const /*{{{*/ +{ + return true; +} + /*}}}*/ +bool metaIndex::IsArchitectureAllSupportedFor(IndexTarget const &) const/*{{{*/ +{ + return true; +} + /*}}}*/ +bool metaIndex::HasSupportForComponent(std::string const &) const/*{{{*/ +{ + return true; +} + /*}}}*/ diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h new file mode 100644 index 0000000..b8db217 --- /dev/null +++ b/apt-pkg/metaindex.h @@ -0,0 +1,115 @@ +#ifndef PKGLIB_METAINDEX_H +#define PKGLIB_METAINDEX_H + +#include <apt-pkg/indexfile.h> +#include <apt-pkg/init.h> + +#include <stddef.h> + +#include <string> +#include <vector> + + +class pkgAcquire; +class IndexTarget; +class pkgCacheGenerator; +class OpProgress; + +class metaIndexPrivate; + +class APT_PUBLIC metaIndex +{ +public: + struct checkSum + { + std::string MetaKeyFilename; + HashStringList Hashes; + unsigned long long Size; + }; + + enum APT_HIDDEN TriState { + TRI_YES, TRI_DONTCARE, TRI_NO, TRI_UNSET + }; +private: + metaIndexPrivate * const d; +protected: + std::vector <pkgIndexFile *> *Indexes; + // parsed from the sources.list + const char *Type; + std::string URI; + std::string Dist; + TriState Trusted; + std::string SignedBy; + + // parsed from a file + std::string Suite; + std::string Codename; + std::string Origin; + std::string Label; + std::string Version; + signed short DefaultPin; + std::string ReleaseNotes; + time_t Date; + time_t ValidUntil; + bool SupportsAcquireByHash; + std::map<std::string, checkSum *> Entries; + TriState LoadedSuccessfully; + +public: + // Various accessors + std::string GetURI() const; + std::string GetDist() const; + const char* GetType() const; + TriState GetTrusted() const; + std::string GetSignedBy() const; + + std::string GetOrigin() const; + std::string GetLabel() const; + std::string GetVersion() const; + std::string GetCodename() const; + std::string GetSuite() const; + std::string GetReleaseNotes() const; + signed short GetDefaultPin() const; + bool GetSupportsAcquireByHash() const; + time_t GetValidUntil() const; + time_t GetDate() const; + virtual time_t GetNotBefore() const = 0; + + std::string GetExpectedDist() const; + bool CheckDist(std::string const &MaybeDist) const; + + // Interface for acquire + virtual std::string Describe() const; + virtual std::string ArchiveURI(std::string const& File) const = 0; + virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) = 0; + virtual std::vector<IndexTarget> GetIndexTargets() const = 0; + virtual std::vector<pkgIndexFile *> *GetIndexFiles() = 0; + virtual bool IsTrusted() const = 0; + virtual bool Load(std::string const &Filename, std::string * const ErrorText) = 0; + /** @return a new metaIndex object based on this one, but without information from #Load */ + virtual metaIndex * UnloadedClone() const = 0; + // the given metaIndex is potentially invalid after this call and should be deleted + void swapLoad(metaIndex * const OldMetaIndex); + + // Lookup functions for parsed Hashes + checkSum *Lookup(std::string const &MetaKey) const; + /** \brief tests if a checksum for this file is available */ + bool Exists(std::string const &MetaKey) const; + std::vector<std::string> MetaKeys() const; + TriState GetLoadedSuccessfully() const; + + // Interfaces for pkgCacheGen + virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const; + virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const; + + + metaIndex(std::string const &URI, std::string const &Dist, + char const * const Type); + virtual ~metaIndex(); + + virtual bool IsArchitectureSupported(std::string const &arch) const; + virtual bool IsArchitectureAllSupportedFor(IndexTarget const &target) const; + virtual bool HasSupportForComponent(std::string const &component) const; +}; + +#endif diff --git a/apt-pkg/orderlist.cc b/apt-pkg/orderlist.cc new file mode 100644 index 0000000..49a3126 --- /dev/null +++ b/apt-pkg/orderlist.cc @@ -0,0 +1,1138 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Order List - Represents and Manipulates an ordered list of packages. + + A list of packages can be ordered by a number of conflicting criteria + each given a specific priority. Each package also has a set of flags + indicating some useful things about it that are derived in the + course of sorting. The pkgPackageManager class uses this class for + all of it's installation ordering needs. + + This is a modified version of Manoj's Routine B. It consists of four + independent ordering algorithms that can be applied at for different + points in the ordering. By applying progressively fewer ordering + operations it is possible to give each consideration it's own + priority and create an order that satisfies the lowest applicable + consideration. + + The rules for unpacking ordering are: + 1) Unpacking ignores Depends: on all packages + 2) Unpacking requires Conflicts: on -ALL- packages to be satisfied + 3) Unpacking requires PreDepends: on this package only to be satisfied + 4) Removing requires that no packages depend on the package to be + removed. + + And the rule for configuration ordering is: + 1) Configuring requires that the Depends: of the package be satisfied + Conflicts+PreDepends are ignored because unpacking says they are + already correct [exageration, it does check but we need not be + concerned] + + And some features that are valuable for unpacking ordering. + f1) Unpacking a new package should advoid breaking dependencies of + configured packages + f2) Removal should not require a force, corrolory of f1 + f3) Unpacking should order by depends rather than fall back to random + ordering. + + Each of the features can be enabled in the sorting routine at an + arbitrary priority to give quite abit of control over the final unpacking + order. + + The rules listed above may never be violated and are called Critical. + When a critical rule is violated then a loop condition is recorded + and will have to be delt with in the caller. + + The ordering keeps two lists, the main list and the 'After List'. The + purpose of the after list is to allow packages to be delayed. This is done + by setting the after flag on the package. Any package which requires this + package to be ordered before will inherit the after flag and so on. This + is used for CD swap ordering where all packages on a second CD have the + after flag set. This forces them and all their dependents to be ordered + toward the end. + + There are complications in this algorithm when presented with cycles. + For all known practical cases it works, all cases where it doesn't work + is fixable by tweaking the package descriptions. However, it should be + possible to improve this further to make some better choices when + presented with cycles. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/depcache.h> +#include <apt-pkg/error.h> +#include <apt-pkg/orderlist.h> +#include <apt-pkg/pkgcache.h> + +#include <algorithm> +#include <iostream> +#include <stdlib.h> +#include <string.h> + /*}}}*/ + +using namespace std; + +// OrderList::pkgOrderList - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgOrderList::pkgOrderList(pkgDepCache *pCache) : d(NULL), Cache(*pCache), + Primary(NULL), Secondary(NULL), + RevDepends(NULL), Remove(NULL), + AfterEnd(NULL), FileList(NULL), + LoopCount(-1), Depth(0) +{ + Debug = _config->FindB("Debug::pkgOrderList",false); + + /* Construct the arrays, egcs 1.0.1 bug requires the package count + hack */ + auto const Size = Cache.Head().PackageCount; + Flags = new unsigned short[Size]; + End = List = new Package *[Size]; + memset(Flags,0,sizeof(*Flags)*Size); +} + /*}}}*/ +// OrderList::~pkgOrderList - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgOrderList::~pkgOrderList() +{ + delete [] List; + delete [] Flags; +} + /*}}}*/ +// OrderList::IsMissing - Check if a file is missing /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgOrderList::IsMissing(PkgIterator Pkg) +{ + // Skip packages to erase + if (Cache[Pkg].Delete() == true) + return false; + + // Skip Packages that need configure only. + if ((Pkg.State() == pkgCache::PkgIterator::NeedsConfigure || + Pkg.State() == pkgCache::PkgIterator::NeedsNothing) && + Cache[Pkg].Keep() == true) + return false; + + if (FileList == 0) + return false; + + if (FileList[Pkg->ID].empty() == false) + return false; + + return true; +} + /*}}}*/ +// OrderList::DoRun - Does an order run /*{{{*/ +// --------------------------------------------------------------------- +/* The caller is expected to have setup the desired probe state */ +bool pkgOrderList::DoRun() +{ + // Temp list + unsigned long Size = Cache.Head().PackageCount; + std::unique_ptr<Package *[]> NList(new Package *[Size]); + std::unique_ptr<Package *[]> AfterList(new Package *[Size]); + AfterEnd = AfterList.get(); + + Depth = 0; + WipeFlags(Added | AddPending | Loop | InList); + + for (iterator I = List; I != End; ++I) + Flag(*I,InList); + + // Rebuild the main list into the temp list. + iterator OldEnd = End; + End = NList.get(); + for (iterator I = List; I != OldEnd; ++I) + if (VisitNode(PkgIterator(Cache,*I), "DoRun") == false) + { + End = OldEnd; + return false; + } + + // Copy the after list to the end of the main list + for (Package **I = AfterList.get(); I != AfterEnd; I++) + *End++ = *I; + + // Swap the main list to the new list + delete [] List; + List = NList.release(); + return true; +} + /*}}}*/ +// OrderList::OrderCritical - Perform critical unpacking ordering /*{{{*/ +// --------------------------------------------------------------------- +/* This performs predepends and immediate configuration ordering only. + This is termed critical unpacking ordering. Any loops that form are + fatal and indicate that the packages cannot be installed. */ +bool pkgOrderList::OrderCritical() +{ + FileList = 0; + + Primary = &pkgOrderList::DepUnPackPreD; + Secondary = 0; + RevDepends = 0; + Remove = 0; + LoopCount = 0; + + // Sort + std::sort(List,End, [this](Package *a, Package *b) { return OrderCompareB(a, b) < 0; } ); + + if (DoRun() == false) + return false; + + if (LoopCount != 0) + return _error->Error("Fatal, predepends looping detected"); + + if (Debug == true) + { + clog << "** Critical Unpack ordering done" << endl; + + for (iterator I = List; I != End; ++I) + { + PkgIterator P(Cache,*I); + if (IsNow(P) == true) + clog << " " << P.FullName() << ' ' << IsMissing(P) << ',' << IsFlag(P,After) << endl; + } + } + + return true; +} + /*}}}*/ +// OrderList::OrderUnpack - Perform complete unpacking ordering /*{{{*/ +// --------------------------------------------------------------------- +/* This performs complete unpacking ordering and creates an order that is + suitable for unpacking */ +bool pkgOrderList::OrderUnpack(string *FileList) +{ + this->FileList = FileList; + + // Setup the after flags + if (FileList != 0) + { + WipeFlags(After); + + // Set the inlist flag + for (iterator I = List; I != End; ++I) + { + PkgIterator P(Cache,*I); + if (IsMissing(P) == true && IsNow(P) == true) + Flag(*I,After); + } + } + + Primary = &pkgOrderList::DepUnPackCrit; + Secondary = &pkgOrderList::DepConfigure; + RevDepends = &pkgOrderList::DepUnPackDep; + Remove = &pkgOrderList::DepRemove; + LoopCount = -1; + + // Sort + std::sort(List,End, [this](Package *a, Package *b) { return OrderCompareA(a, b) < 0; }); + + if (Debug == true) + clog << "** Pass A" << endl; + if (DoRun() == false) + return false; + + if (Debug == true) + clog << "** Pass B" << endl; + Secondary = 0; + if (DoRun() == false) + return false; + + if (Debug == true) + clog << "** Pass C" << endl; + LoopCount = 0; + RevDepends = 0; + Remove = 0; // Otherwise the libreadline remove problem occurs + if (DoRun() == false) + return false; + + if (Debug == true) + clog << "** Pass D" << endl; + LoopCount = 0; + Primary = &pkgOrderList::DepUnPackPre; + if (DoRun() == false) + return false; + + if (Debug == true) + { + clog << "** Unpack ordering done" << endl; + + for (iterator I = List; I != End; ++I) + { + PkgIterator P(Cache,*I); + if (IsNow(P) == true) + clog << " " << P.FullName() << ' ' << IsMissing(P) << ',' << IsFlag(P,After) << endl; + } + } + + return true; +} + /*}}}*/ +// OrderList::OrderConfigure - Perform configuration ordering /*{{{*/ +// --------------------------------------------------------------------- +/* This orders by depends only and produces an order which is suitable + for configuration */ +bool pkgOrderList::OrderConfigure() +{ + FileList = 0; + Primary = &pkgOrderList::DepConfigure; + Secondary = 0; + RevDepends = 0; + Remove = 0; + LoopCount = -1; + return DoRun(); +} + /*}}}*/ +// OrderList::Score - Score the package for sorting /*{{{*/ +// --------------------------------------------------------------------- +/* Higher scores order earlier */ +int pkgOrderList::Score(PkgIterator Pkg) +{ + // Removals should be done after we dealt with essentials + static int const ScoreDelete = _config->FindI("OrderList::Score::Delete", 100); + if (Cache[Pkg].Delete() == true) + return ScoreDelete; + + // This should never happen.. + if (Cache[Pkg].InstVerIter(Cache).end() == true) + return -1; + + static int const ScoreEssential = _config->FindI("OrderList::Score::Essential", 200); + static int const ScoreImmediate = _config->FindI("OrderList::Score::Immediate", 10); + static int const ScorePreDepends = _config->FindI("OrderList::Score::PreDepends", 50); + + int Score = 0; + if ((Pkg->Flags & pkgCache::Flag::Essential) == pkgCache::Flag::Essential) + Score += ScoreEssential; + + if (IsFlag(Pkg,Immediate) == true) + Score += ScoreImmediate; + + for (DepIterator D = Cache[Pkg].InstVerIter(Cache).DependsList(); + D.end() == false; ++D) + if (D->Type == pkgCache::Dep::PreDepends) + { + Score += ScorePreDepends; + break; + } + + // Required Important Standard Optional Extra + if (Cache[Pkg].InstVerIter(Cache)->Priority <= 5) + { + signed short PrioMap[] = {0,5,4,3,1,0}; + Score += PrioMap[Cache[Pkg].InstVerIter(Cache)->Priority]; + } + return Score; +} + /*}}}*/ +// OrderList::FileCmp - Compare by package file /*{{{*/ +// --------------------------------------------------------------------- +/* This compares by the package file that the install version is in. */ +int pkgOrderList::FileCmp(PkgIterator A,PkgIterator B) +{ + if (Cache[A].Delete() == true && Cache[B].Delete() == true) + return 0; + if (Cache[A].Delete() == true) + return -1; + if (Cache[B].Delete() == true) + return 1; + + if (Cache[A].InstVerIter(Cache).FileList().end() == true) + return -1; + if (Cache[B].InstVerIter(Cache).FileList().end() == true) + return 1; + + pkgCache::PackageFile *FA = Cache[A].InstVerIter(Cache).FileList().File(); + pkgCache::PackageFile *FB = Cache[B].InstVerIter(Cache).FileList().File(); + if (FA < FB) + return -1; + if (FA > FB) + return 1; + return 0; +} + /*}}}*/ +// BoolCompare - Comparison function for two booleans /*{{{*/ +// --------------------------------------------------------------------- +/* */ +static int BoolCompare(bool A,bool B) +{ + if (A == B) + return 0; + if (A == false) + return -1; + return 1; +} + /*}}}*/ +// OrderList::OrderCompareA - Order the installation by op /*{{{*/ +// --------------------------------------------------------------------- +/* This provides a first-pass sort of the list and gives a decent starting + point for further complete ordering. It is used by OrderUnpack only */ +int pkgOrderList::OrderCompareA(Package *a, Package *b) +{ + PkgIterator A(Cache,a); + PkgIterator B(Cache,b); + + // We order packages with a set state toward the front + int Res; + if ((Res = BoolCompare(IsNow(A),IsNow(B))) != 0) + return -1*Res; + + // We order missing files to toward the end +/* if (FileList != 0) + { + if ((Res = BoolCompare(IsMissing(A), + IsMissing(B))) != 0) + return Res; + }*/ + + if (A.State() != pkgCache::PkgIterator::NeedsNothing && + B.State() == pkgCache::PkgIterator::NeedsNothing) + return -1; + + if (A.State() == pkgCache::PkgIterator::NeedsNothing && + B.State() != pkgCache::PkgIterator::NeedsNothing) + return 1; + + int ScoreA = Score(A); + int ScoreB = Score(B); + + if (ScoreA > ScoreB) + return -1; + + if (ScoreA < ScoreB) + return 1; + + return strcmp(A.Name(),B.Name()); +} + /*}}}*/ +// OrderList::OrderCompareB - Order the installation by source /*{{{*/ +// --------------------------------------------------------------------- +/* This orders by installation source. This is useful to handle + inter-source breaks */ +int pkgOrderList::OrderCompareB(Package *a, Package *b) +{ + PkgIterator A(Cache,a); + PkgIterator B(Cache,b); + + if (A.State() != pkgCache::PkgIterator::NeedsNothing && + B.State() == pkgCache::PkgIterator::NeedsNothing) + return -1; + + if (A.State() == pkgCache::PkgIterator::NeedsNothing && + B.State() != pkgCache::PkgIterator::NeedsNothing) + return 1; + + int F = FileCmp(A,B); + if (F != 0) + { + if (F > 0) + return -1; + return 1; + } + + int ScoreA = Score(A); + int ScoreB = Score(B); + + if (ScoreA > ScoreB) + return -1; + + if (ScoreA < ScoreB) + return 1; + + return strcmp(A.Name(),B.Name()); +} + /*}}}*/ +// OrderList::VisitDeps - Visit forward install dependencies /*{{{*/ +// --------------------------------------------------------------------- +/* This calls the dependency function for the normal forwards dependencies + of the package */ +bool pkgOrderList::VisitDeps(DepFunc F,PkgIterator Pkg) +{ + if (F == 0 || Pkg.end() == true || Cache[Pkg].InstallVer == 0) + return true; + + return (this->*F)(Cache[Pkg].InstVerIter(Cache).DependsList()); +} + /*}}}*/ +// OrderList::VisitRDeps - Visit reverse dependencies /*{{{*/ +// --------------------------------------------------------------------- +/* This calls the dependency function for all of the normal reverse depends + of the package */ +bool pkgOrderList::VisitRDeps(DepFunc F,PkgIterator Pkg) +{ + if (F == 0 || Pkg.end() == true) + return true; + + return (this->*F)(Pkg.RevDependsList()); +} + /*}}}*/ +// OrderList::VisitRProvides - Visit provides reverse dependencies /*{{{*/ +// --------------------------------------------------------------------- +/* This calls the dependency function for all reverse dependencies + generated by the provides line on the package. */ +bool pkgOrderList::VisitRProvides(DepFunc F,VerIterator Ver) +{ + if (F == 0 || Ver.end() == true) + return true; + + bool Res = true; + for (PrvIterator P = Ver.ProvidesList(); P.end() == false; ++P) + Res &= (this->*F)(P.ParentPkg().RevDependsList()); + return Res; +} + /*}}}*/ +// OrderList::VisitProvides - Visit all of the providing packages /*{{{*/ +// --------------------------------------------------------------------- +/* This routine calls visit on all providing packages. + + If the dependency is negative it first visits packages which are + intended to be removed and after that all other packages. + It does so to avoid situations in which this package is used to + satisfy a (or-group/provides) dependency of another package which + could have been satisfied also by upgrading another package - + otherwise we have more broken packages dpkg needs to auto- + deconfigure and in very complicated situations it even decides + against it! */ +bool pkgOrderList::VisitProvides(DepIterator D,bool Critical) +{ + std::unique_ptr<Version *[]> List(D.AllTargets()); + for (Version **I = List.get(); *I != 0; ++I) + { + VerIterator Ver(Cache,*I); + PkgIterator Pkg = Ver.ParentPkg(); + + if (D.IsNegative() == true && Cache[Pkg].Delete() == false) + continue; + + if (Cache[Pkg].Keep() == true && Pkg.State() == PkgIterator::NeedsNothing) + continue; + + if (D.IsNegative() == false && + Cache[Pkg].InstallVer != *I) + continue; + + if (D.IsNegative() == true && + (Version *)Pkg.CurrentVer() != *I) + continue; + + // Skip over missing files + if (Critical == false && IsMissing(D.ParentPkg()) == true) + continue; + + if (VisitNode(Pkg, "Provides-1") == false) + return false; + } + if (D.IsNegative() == false) + return true; + for (Version **I = List.get(); *I != 0; ++I) + { + VerIterator Ver(Cache,*I); + PkgIterator Pkg = Ver.ParentPkg(); + + if (Cache[Pkg].Delete() == true) + continue; + + if (Cache[Pkg].Keep() == true && Pkg.State() == PkgIterator::NeedsNothing) + continue; + + if ((Version *)Pkg.CurrentVer() != *I) + continue; + + // Skip over missing files + if (Critical == false && IsMissing(D.ParentPkg()) == true) + continue; + + if (VisitNode(Pkg, "Provides-2") == false) + return false; + } + + return true; +} + /*}}}*/ +// OrderList::VisitNode - Recursive ordering director /*{{{*/ +// --------------------------------------------------------------------- +/* This is the core ordering routine. It calls the set dependency + consideration functions which then potentially call this again. Finite + depth is achieved through the colouring mechanism. */ +bool pkgOrderList::VisitNode(PkgIterator Pkg, char const* from) +{ + // Looping or irrelevant. + // This should probably trancend not installed packages + if (Pkg.end() == true || IsFlag(Pkg,Added) == true || + IsFlag(Pkg,AddPending) == true || IsFlag(Pkg,InList) == false) + return true; + + if (Debug == true) + { + for (int j = 0; j != Depth; j++) clog << ' '; + clog << "Visit " << Pkg.FullName() << " from " << from << endl; + } + + Depth++; + + // Color grey + Flag(Pkg,AddPending); + + DepFunc Old = Primary; + + // Perform immediate configuration of the package if so flagged. + if (IsFlag(Pkg,Immediate) == true && Primary != &pkgOrderList::DepUnPackPre) + Primary = &pkgOrderList::DepUnPackPreD; + + if (IsNow(Pkg) == true) + { + bool Res = true; + if (Cache[Pkg].Delete() == false) + { + // Primary + Res &= Res && VisitDeps(Primary,Pkg); + Res &= Res && VisitRDeps(Primary,Pkg); + Res &= Res && VisitRProvides(Primary,Pkg.CurrentVer()); + Res &= Res && VisitRProvides(Primary,Cache[Pkg].InstVerIter(Cache)); + + // RevDep + Res &= Res && VisitRDeps(RevDepends,Pkg); + Res &= Res && VisitRProvides(RevDepends,Pkg.CurrentVer()); + Res &= Res && VisitRProvides(RevDepends,Cache[Pkg].InstVerIter(Cache)); + + // Secondary + Res &= Res && VisitDeps(Secondary,Pkg); + Res &= Res && VisitRDeps(Secondary,Pkg); + Res &= Res && VisitRProvides(Secondary,Pkg.CurrentVer()); + Res &= Res && VisitRProvides(Secondary,Cache[Pkg].InstVerIter(Cache)); + } + else + { + // RevDep + Res &= Res && VisitRDeps(Remove,Pkg); + Res &= Res && VisitRProvides(Remove,Pkg.CurrentVer()); + } + } + + if (IsFlag(Pkg,Added) == false) + { + Flag(Pkg,Added,Added | AddPending); + if (IsFlag(Pkg,After) == true) + *AfterEnd++ = Pkg; + else + *End++ = Pkg; + } + + Primary = Old; + Depth--; + + if (Debug == true) + { + for (int j = 0; j != Depth; j++) clog << ' '; + clog << "Leave " << Pkg.FullName() << ' ' << IsFlag(Pkg,Added) << ',' << IsFlag(Pkg,AddPending) << endl; + } + + return true; +} + /*}}}*/ +// OrderList::DepUnPackCrit - Critical UnPacking ordering /*{{{*/ +// --------------------------------------------------------------------- +/* Critical unpacking ordering strives to satisfy Conflicts: and + PreDepends: only. When a prdepends is encountered the Primary + DepFunc is changed to be DepUnPackPreD. + + Loops are preprocessed and logged. */ +bool pkgOrderList::DepUnPackCrit(DepIterator D) +{ + for (; D.end() == false; ++D) + { + if (D.Reverse() == true) + { + /* Reverse depenanices are only interested in conflicts, + predepend breakage is ignored here */ + if (D->Type != pkgCache::Dep::Conflicts && + D->Type != pkgCache::Dep::Obsoletes) + continue; + + // Duplication elimination, consider only the current version + if (D.ParentPkg().CurrentVer() != D.ParentVer()) + continue; + + /* For reverse dependencies we wish to check if the + dependency is satisfied in the install state. The + target package (caller) is going to be in the installed + state. */ + if (CheckDep(D) == true) + continue; + + if (VisitNode(D.ParentPkg(), "UnPackCrit") == false) + return false; + } + else + { + /* Forward critical dependencies MUST be correct before the + package can be unpacked. */ + if (D.IsNegative() == false && + D->Type != pkgCache::Dep::PreDepends) + continue; + + /* We wish to check if the dep is okay in the now state of the + target package against the install state of this package. */ + if (CheckDep(D) == true) + { + /* We want to catch predepends loops with the code below. + Conflicts loops that are Dep OK are ignored */ + if (IsFlag(D.TargetPkg(),AddPending) == false || + D->Type != pkgCache::Dep::PreDepends) + continue; + } + + // This is the loop detection + if (IsFlag(D.TargetPkg(),Added) == true || + IsFlag(D.TargetPkg(),AddPending) == true) + { + if (IsFlag(D.TargetPkg(),AddPending) == true) + AddLoop(D); + continue; + } + + /* Predepends require a special ordering stage, they must have + all dependents installed as well */ + DepFunc Old = Primary; + bool Res = false; + if (D->Type == pkgCache::Dep::PreDepends) + Primary = &pkgOrderList::DepUnPackPreD; + Res = VisitProvides(D,true); + Primary = Old; + if (Res == false) + return false; + } + } + return true; +} + /*}}}*/ +// OrderList::DepUnPackPreD - Critical UnPacking ordering with depends /*{{{*/ +// --------------------------------------------------------------------- +/* Critical PreDepends (also configure immediate and essential) strives to + ensure not only that all conflicts+predepends are met but that this + package will be immediately configurable when it is unpacked. + Loops are preprocessed and logged. */ +bool pkgOrderList::DepUnPackPreD(DepIterator D) +{ + if (D.Reverse() == true) + return DepUnPackCrit(D); + + for (; D.end() == false; ++D) + { + if (D.IsCritical() == false) + continue; + + /* We wish to check if the dep is okay in the now state of the + target package against the install state of this package. */ + if (CheckDep(D) == true) + { + /* We want to catch predepends loops with the code below. + Conflicts loops that are Dep OK are ignored */ + if (IsFlag(D.TargetPkg(),AddPending) == false || + D->Type != pkgCache::Dep::PreDepends) + continue; + } + + // This is the loop detection + if (IsFlag(D.TargetPkg(),Added) == true || + IsFlag(D.TargetPkg(),AddPending) == true) + { + if (IsFlag(D.TargetPkg(),AddPending) == true) + AddLoop(D); + continue; + } + + if (VisitProvides(D,true) == false) + return false; + } + return true; +} + /*}}}*/ +// OrderList::DepUnPackPre - Critical Predepends ordering /*{{{*/ +// --------------------------------------------------------------------- +/* Critical PreDepends (also configure immediate and essential) strives to + ensure not only that all conflicts+predepends are met but that this + package will be immediately configurable when it is unpacked. + + Loops are preprocessed and logged. All loops will be fatal. */ +bool pkgOrderList::DepUnPackPre(DepIterator D) +{ + if (D.Reverse() == true) + return true; + + for (; D.end() == false; ++D) + { + /* Only consider the PreDepends or Depends. Depends are only + considered at the lowest depth or in the case of immediate + configure */ + if (D->Type != pkgCache::Dep::PreDepends) + { + if (D->Type == pkgCache::Dep::Depends) + { + if (Depth == 1 && IsFlag(D.ParentPkg(),Immediate) == false) + continue; + } + else + continue; + } + + /* We wish to check if the dep is okay in the now state of the + target package against the install state of this package. */ + if (CheckDep(D) == true) + { + /* We want to catch predepends loops with the code below. + Conflicts loops that are Dep OK are ignored */ + if (IsFlag(D.TargetPkg(),AddPending) == false) + continue; + } + + // This is the loop detection + if (IsFlag(D.TargetPkg(),Added) == true || + IsFlag(D.TargetPkg(),AddPending) == true) + { + if (IsFlag(D.TargetPkg(),AddPending) == true) + AddLoop(D); + continue; + } + + if (VisitProvides(D,true) == false) + return false; + } + return true; +} + /*}}}*/ +// OrderList::DepUnPackDep - Reverse dependency considerations /*{{{*/ +// --------------------------------------------------------------------- +/* Reverse dependencies are considered to determine if unpacking this + package will break any existing dependencies. If so then those + packages are ordered before this one so that they are in the + UnPacked state. + + The forwards depends loop is designed to bring the packages dependents + close to the package. This helps reduce deconfigure time. + + Loops are irrelevant to this. */ +bool pkgOrderList::DepUnPackDep(DepIterator D) +{ + + for (; D.end() == false; ++D) + if (D.IsCritical() == true) + { + if (D.Reverse() == true) + { + /* Duplication prevention. We consider rev deps only on + the current version, a not installed package + cannot break */ + if (D.ParentPkg()->CurrentVer == 0 || + D.ParentPkg().CurrentVer() != D.ParentVer()) + continue; + + // The dep will not break so it is irrelevant. + if (CheckDep(D) == true) + continue; + + // Skip over missing files + if (IsMissing(D.ParentPkg()) == true) + continue; + + if (VisitNode(D.ParentPkg(), "UnPackDep-Parent") == false) + return false; + } + else + { + if (D->Type == pkgCache::Dep::Depends) + if (VisitProvides(D,false) == false) + return false; + + if (D->Type == pkgCache::Dep::DpkgBreaks) + { + if (CheckDep(D) == true) + continue; + + if (VisitNode(D.TargetPkg(), "UnPackDep-Target") == false) + return false; + } + } + } + return true; +} + /*}}}*/ +// OrderList::DepConfigure - Configuration ordering /*{{{*/ +// --------------------------------------------------------------------- +/* Configuration only ordering orders by the Depends: line only. It + orders configuration so that when a package comes to be configured it's + dependents are configured. + + Loops are ignored. Depends loop entry points are chaotic. */ +bool pkgOrderList::DepConfigure(DepIterator D) +{ + // Never consider reverse configuration dependencies. + if (D.Reverse() == true) + return true; + + for (; D.end() == false; ++D) + if (D->Type == pkgCache::Dep::Depends) + if (VisitProvides(D,false) == false) + return false; + return true; +} + /*}}}*/ +// OrderList::DepRemove - Removal ordering /*{{{*/ +// --------------------------------------------------------------------- +/* Checks all given dependencies if they are broken by the removal of a + package and if so fix it by visiting another provider or or-group + member to ensure that the dependee keeps working which is especially + important for Immediate packages like e.g. those depending on an + awk implementation. If the dependency can't be fixed with another + package this means an upgrade of the package will solve the problem. */ +bool pkgOrderList::DepRemove(DepIterator Broken) +{ + if (Broken.Reverse() == false) + return true; + + for (; Broken.end() == false; ++Broken) + { + if (Broken->Type != pkgCache::Dep::Depends && + Broken->Type != pkgCache::Dep::PreDepends) + continue; + + PkgIterator BrokenPkg = Broken.ParentPkg(); + // uninstalled packages can't break via a remove + if (BrokenPkg->CurrentVer == 0) + continue; + + // if its already added, we can't do anything useful + if (IsFlag(BrokenPkg, AddPending) == true || IsFlag(BrokenPkg, Added) == true) + continue; + + // if the dependee is going to be removed, visit it now + if (Cache[BrokenPkg].Delete() == true) + return VisitNode(BrokenPkg, "Remove-Dependee"); + + // The package stays around, so find out how this is possible + for (DepIterator D = BrokenPkg.CurrentVer().DependsList(); D.end() == false;) + { + // only important or-groups need fixing + if (D->Type != pkgCache::Dep::Depends && + D->Type != pkgCache::Dep::PreDepends) + { + ++D; + continue; + } + + // Start is the beginning of the or-group, D is the first one after or + DepIterator Start = D; + bool foundBroken = false; + for (bool LastOR = true; D.end() == false && LastOR == true; ++D) + { + LastOR = (D->CompareOp & pkgCache::Dep::Or) == pkgCache::Dep::Or; + if (D == Broken) + foundBroken = true; + } + + // this or-group isn't the broken one: keep searching + if (foundBroken == false) + continue; + + // iterate over all members of the or-group searching for a ready replacement + bool readyReplacement = false; + for (DepIterator OrMember = Start; OrMember != D && readyReplacement == false; ++OrMember) + { + Version ** Replacements = OrMember.AllTargets(); + for (Version **R = Replacements; *R != 0; ++R) + { + VerIterator Ver(Cache,*R); + // only currently installed packages can be a replacement + PkgIterator RPkg = Ver.ParentPkg(); + if (RPkg.CurrentVer() != Ver) + continue; + + // packages going to be removed can't be a replacement + if (Cache[RPkg].Delete() == true) + continue; + + readyReplacement = true; + break; + } + delete[] Replacements; + } + + // something else is ready to take over, do nothing + if (readyReplacement == true) + continue; + + // see if we can visit a replacement + bool visitReplacement = false; + for (DepIterator OrMember = Start; OrMember != D && visitReplacement == false; ++OrMember) + { + Version ** Replacements = OrMember.AllTargets(); + for (Version **R = Replacements; *R != 0; ++R) + { + VerIterator Ver(Cache,*R); + // consider only versions we plan to install + PkgIterator RPkg = Ver.ParentPkg(); + if (Cache[RPkg].Install() == false || Cache[RPkg].InstallVer != Ver) + continue; + + // loops are not going to help us, so don't create them + if (IsFlag(RPkg, AddPending) == true) + continue; + + if (IsMissing(RPkg) == true) + continue; + + visitReplacement = true; + if (IsFlag(BrokenPkg, Immediate) == false) + { + if (VisitNode(RPkg, "Remove-Rep") == true) + break; + } + else + { + Flag(RPkg, Immediate); + if (VisitNode(RPkg, "Remove-ImmRep") == true) + break; + } + visitReplacement = false; + } + delete[] Replacements; + } + if (visitReplacement == true) + continue; + + // the broken package in current version can't be fixed, so install new version + if (IsMissing(BrokenPkg) == true) + break; + + if (VisitNode(BrokenPkg, "Remove-Upgrade") == false) + return false; + } + } + + return true; +} + /*}}}*/ +// OrderList::AddLoop - Add a loop to the loop list /*{{{*/ +// --------------------------------------------------------------------- +/* We record the loops. This is a relic since loop breaking is done + genericaly as part of the safety routines. */ +bool pkgOrderList::AddLoop(DepIterator D) +{ + if (LoopCount < 0 || LoopCount >= 20) + return false; + + // Skip dups + if (LoopCount != 0) + { + if (Loops[LoopCount - 1].ParentPkg() == D.ParentPkg() || + Loops[LoopCount - 1].TargetPkg() == D.ParentPkg()) + return true; + } + + Loops[LoopCount++] = D; + + // Mark the packages as being part of a loop. + //Flag(D.TargetPkg(),Loop); + //Flag(D.ParentPkg(),Loop); + /* This is currently disabled because the Loop flag is being used for + loop management in the package manager. Check the orderlist.h file for more info */ + return true; +} + /*}}}*/ +// OrderList::WipeFlags - Unset the given flags from all packages /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgOrderList::WipeFlags(unsigned long F) +{ + auto Size = Cache.Head().PackageCount; + for (decltype(Size) I = 0; I != Size; ++I) + Flags[I] &= ~F; +} + /*}}}*/ +// OrderList::CheckDep - Check a dependency for truth /*{{{*/ +// --------------------------------------------------------------------- +/* This performs a complete analysis of the dependency wrt to the + current add list. It returns true if after all events are + performed it is still true. This sort of routine can be approximated + by examining the DepCache, however in convoluted cases of provides + this fails to produce a suitable result. */ +bool pkgOrderList::CheckDep(DepIterator D) +{ + std::unique_ptr<Version *[]> List(D.AllTargets()); + bool Hit = false; + for (Version **I = List.get(); *I != 0; I++) + { + VerIterator Ver(Cache,*I); + PkgIterator Pkg = Ver.ParentPkg(); + + /* The meaning of Added and AddPending is subtle. AddPending is + an indication that the package is looping. Because of the + way ordering works Added means the package will be unpacked + before this one and AddPending means after. It is therefore + correct to ignore AddPending in all cases, but that exposes + reverse-ordering loops which should be ignored. */ + if (IsFlag(Pkg,Added) == true || + (IsFlag(Pkg,AddPending) == true && D.Reverse() == true)) + { + if (Cache[Pkg].InstallVer != *I) + continue; + } + else + if ((Version *)Pkg.CurrentVer() != *I || + Pkg.State() != PkgIterator::NeedsNothing) + continue; + + /* Conflicts requires that all versions are not present, depends + just needs one */ + if (D.IsNegative() == false) + { + // ignore provides by older versions of this package + if (((D.Reverse() == false && Pkg == D.ParentPkg()) || + (D.Reverse() == true && Pkg == D.TargetPkg())) && + Cache[Pkg].InstallVer != *I) + continue; + + /* Try to find something that does not have the after flag set + if at all possible */ + if (IsFlag(Pkg,After) == true) + { + Hit = true; + continue; + } + + return true; + } + else + { + if (IsFlag(Pkg,After) == true) + Flag(D.ParentPkg(),After); + + return false; + } + } + + // We found a hit, but it had the after flag set + if (Hit == true && D->Type == pkgCache::Dep::PreDepends) + { + Flag(D.ParentPkg(),After); + return true; + } + + /* Conflicts requires that all versions are not present, depends + just needs one */ + if (D->Type == pkgCache::Dep::Conflicts || + D->Type == pkgCache::Dep::Obsoletes) + return true; + return false; +} + /*}}}*/ diff --git a/apt-pkg/orderlist.h b/apt-pkg/orderlist.h new file mode 100644 index 0000000..559bf98 --- /dev/null +++ b/apt-pkg/orderlist.h @@ -0,0 +1,126 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Order List - Represents and Manipulates an ordered list of packages. + + A list of packages can be ordered by a number of conflicting criteria + each given a specific priority. Each package also has a set of flags + indicating some useful things about it that are derived in the + course of sorting. The pkgPackageManager class uses this class for + all of it's installation ordering needs. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_ORDERLIST_H +#define PKGLIB_ORDERLIST_H + +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> + +#include <string> + +class pkgDepCache; +class APT_PUBLIC pkgOrderList : protected pkgCache::Namespace +{ + void * const d; + protected: + + pkgDepCache &Cache; + typedef bool (pkgOrderList::*DepFunc)(DepIterator D); + + // These are the currently selected ordering functions + DepFunc Primary; + DepFunc Secondary; + DepFunc RevDepends; + DepFunc Remove; + + // State + Package **End; + Package **List; + Package **AfterEnd; + std::string *FileList; + DepIterator Loops[20]; + int LoopCount; + int Depth; + unsigned short *Flags; + bool Debug; + + // Main visit function + bool VisitNode(PkgIterator Pkg, char const* from); + bool VisitDeps(DepFunc F,PkgIterator Pkg); + bool VisitRDeps(DepFunc F,PkgIterator Pkg); + bool VisitRProvides(DepFunc F,VerIterator Ver); + bool VisitProvides(DepIterator Pkg,bool Critical); + + // Dependency checking functions. + bool DepUnPackCrit(DepIterator D); + bool DepUnPackPreD(DepIterator D); + bool DepUnPackPre(DepIterator D); + bool DepUnPackDep(DepIterator D); + bool DepConfigure(DepIterator D); + bool DepRemove(DepIterator D); + + // Analysis helpers + bool AddLoop(DepIterator D); + bool CheckDep(DepIterator D); + bool DoRun(); + + // For pre sorting + int OrderCompareA(Package *a, Package *b) APT_PURE; + int OrderCompareB(Package *a, Package *b) APT_PURE; + int FileCmp(PkgIterator A,PkgIterator B) APT_PURE; + + public: + + typedef Package **iterator; + + /* State flags + The Loop flag can be set on a package that is currently being processed by either SmartConfigure or + SmartUnPack. This allows the package manager to tell when a loop has been formed as it will try to + SmartUnPack or SmartConfigure a package with the Loop flag set. It will then either stop (as it knows + that the operation is unnecessary as its already in process), or in the case of the conflicts resolution + in SmartUnPack, use EarlyRemove to resolve the situation. */ + enum Flags {Added = (1 << 0), AddPending = (1 << 1), + Immediate = (1 << 2), Loop = (1 << 3), + UnPacked = (1 << 4), Configured = (1 << 5), + Removed = (1 << 6), // Early Remove + InList = (1 << 7), + After = (1 << 8), + States = (UnPacked | Configured | Removed)}; + + // Flag manipulators + inline bool IsFlag(PkgIterator Pkg,unsigned long F) {return (Flags[Pkg->ID] & F) == F;}; + inline bool IsFlag(Package *Pkg,unsigned long F) {return (Flags[Pkg->ID] & F) == F;}; + void Flag(PkgIterator Pkg,unsigned long State, unsigned long F) {Flags[Pkg->ID] = (Flags[Pkg->ID] & (~F)) | State;}; + inline void Flag(PkgIterator Pkg,unsigned long F) {Flags[Pkg->ID] |= F;}; + inline void Flag(Package *Pkg,unsigned long F) {Flags[Pkg->ID] |= F;}; + // RmFlag removes a flag from a package + inline void RmFlag(Package *Pkg,unsigned long F) {Flags[Pkg->ID] &= ~F;}; + // IsNow will return true if the Pkg has been not been either configured or unpacked + inline bool IsNow(PkgIterator Pkg) {return (Flags[Pkg->ID] & (States & (~Removed))) == 0;}; + bool IsMissing(PkgIterator Pkg) APT_PURE; + void WipeFlags(unsigned long F); + void SetFileList(std::string *FileList) {this->FileList = FileList;}; + + // Accessors + inline iterator begin() {return List;}; + inline iterator end() {return End;}; + inline void push_back(Package *Pkg) {*(End++) = Pkg;}; + inline void push_back(PkgIterator Pkg) {*(End++) = Pkg;}; + inline void pop_back() {End--;}; + inline bool empty() {return End == List;}; + inline unsigned int size() {return End - List;}; + + // Ordering modes + bool OrderCritical(); + bool OrderUnpack(std::string *FileList = 0); + bool OrderConfigure(); + + int Score(PkgIterator Pkg); + + explicit pkgOrderList(pkgDepCache *Cache); + virtual ~pkgOrderList(); +}; + +#endif diff --git a/apt-pkg/packagemanager.cc b/apt-pkg/packagemanager.cc new file mode 100644 index 0000000..48dd03f --- /dev/null +++ b/apt-pkg/packagemanager.cc @@ -0,0 +1,1172 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Package Manager - Abstracts the package manager + + More work is needed in the area of transitioning provides, ie exim + replacing smail. This can cause interesting side effects. + + Other cases involving conflicts+replaces should be tested. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/acquire-item.h> +#include <apt-pkg/algorithms.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/depcache.h> +#include <apt-pkg/edsp.h> +#include <apt-pkg/error.h> +#include <apt-pkg/install-progress.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/orderlist.h> +#include <apt-pkg/packagemanager.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/prettyprinters.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/version.h> + +#include <iostream> +#include <list> +#include <string> +#include <stddef.h> + +#include <apti18n.h> + /*}}}*/ +using namespace std; + +bool pkgPackageManager::SigINTStop = false; + +// PM::PackageManager - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgPackageManager::pkgPackageManager(pkgDepCache *pCache) : Cache(*pCache), + List(NULL), Res(Incomplete), d(NULL) +{ + FileNames = new string[Cache.Head().PackageCount]; + Debug = _config->FindB("Debug::pkgPackageManager",false); + NoImmConfigure = !_config->FindB("APT::Immediate-Configure",true); + ImmConfigureAll = _config->FindB("APT::Immediate-Configure-All",false); +} + /*}}}*/ +// PM::PackageManager - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgPackageManager::~pkgPackageManager() +{ + delete List; + delete [] FileNames; +} + /*}}}*/ +// PM::GetArchives - Queue the archives for download /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgPackageManager::GetArchives(pkgAcquire *Owner,pkgSourceList *Sources, + pkgRecords *Recs) +{ + if (CreateOrderList() == false) + return false; + + bool const ordering = + _config->FindB("PackageManager::UnpackAll",true) ? + List->OrderUnpack() : List->OrderCritical(); + if (ordering == false) + return _error->Error("Internal ordering error"); + + for (pkgOrderList::iterator I = List->begin(); I != List->end(); ++I) + { + PkgIterator Pkg(Cache,*I); + FileNames[Pkg->ID] = string(); + + // Skip packages to erase + if (Cache[Pkg].Delete() == true) + continue; + + // Skip Packages that need configure only. + if (Pkg.State() == pkgCache::PkgIterator::NeedsConfigure && + Cache[Pkg].Keep() == true) + continue; + + // Skip already processed packages + if (List->IsNow(Pkg) == false) + continue; + + new pkgAcqArchive(Owner,Sources,Recs,Cache[Pkg].InstVerIter(Cache), + FileNames[Pkg->ID]); + } + + return true; +} + /*}}}*/ +// PM::FixMissing - Keep all missing packages /*{{{*/ +// --------------------------------------------------------------------- +/* This is called to correct the installation when packages could not + be downloaded. */ +bool pkgPackageManager::FixMissing() +{ + pkgDepCache::ActionGroup group(Cache); + pkgProblemResolver Resolve(&Cache); + List->SetFileList(FileNames); + + bool Bad = false; + for (PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + if (List->IsMissing(I) == false) + continue; + + // Okay, this file is missing and we need it. Mark it for keep + Bad = true; + Cache.MarkKeep(I, false, false); + } + + // We have to empty the list otherwise it will not have the new changes + delete List; + List = 0; + + if (Bad == false) + return true; + + // Now downgrade everything that is broken + return Resolve.ResolveByKeep() == true && Cache.BrokenCount() == 0; +} + /*}}}*/ +// PM::ImmediateAdd - Add the immediate flag recursively /*{{{*/ +// --------------------------------------------------------------------- +/* This adds the immediate flag to the pkg and recursively to the + dependencies + */ +void pkgPackageManager::ImmediateAdd(PkgIterator I, bool UseInstallVer, unsigned const int &Depth) +{ + DepIterator D; + + if(UseInstallVer) + { + if(Cache[I].InstallVer == 0) + return; + D = Cache[I].InstVerIter(Cache).DependsList(); + } else { + if (I->CurrentVer == 0) + return; + D = I.CurrentVer().DependsList(); + } + + for ( /* nothing */ ; D.end() == false; ++D) + if (D->Type == pkgCache::Dep::Depends || D->Type == pkgCache::Dep::PreDepends) + { + if(!List->IsFlag(D.TargetPkg(), pkgOrderList::Immediate)) + { + if(Debug) + clog << OutputInDepth(Depth) << "ImmediateAdd(): Adding Immediate flag to " << APT::PrettyPkg(&Cache, D.TargetPkg()) << " cause of " << D.DepType() << " " << I.FullName() << endl; + List->Flag(D.TargetPkg(),pkgOrderList::Immediate); + ImmediateAdd(D.TargetPkg(), UseInstallVer, Depth + 1); + } + } + return; +} + /*}}}*/ +// PM::CreateOrderList - Create the ordering class /*{{{*/ +// --------------------------------------------------------------------- +/* This populates the ordering list with all the packages that are + going to change. */ +bool pkgPackageManager::CreateOrderList() +{ + if (List != 0) + return true; + + delete List; + List = new pkgOrderList(&Cache); + + if (Debug && ImmConfigureAll) + clog << "CreateOrderList(): Adding Immediate flag for all packages because of APT::Immediate-Configure-All" << endl; + + // Generate the list of affected packages and sort it + for (PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + // Ignore no-version packages + if (I->VersionList == 0) + continue; + + // Mark the package and its dependents for immediate configuration + if ((((I->Flags & pkgCache::Flag::Essential) == pkgCache::Flag::Essential) && + NoImmConfigure == false) || ImmConfigureAll) + { + if(Debug && !ImmConfigureAll) + clog << "CreateOrderList(): Adding Immediate flag for " << I.FullName() << endl; + List->Flag(I,pkgOrderList::Immediate); + + if (!ImmConfigureAll) { + // Look for other install packages to make immediate configurea + ImmediateAdd(I, true); + + // And again with the current version. + ImmediateAdd(I, false); + } + } + + // Not interesting + if ((Cache[I].Keep() == true || + Cache[I].InstVerIter(Cache) == I.CurrentVer()) && + I.State() == pkgCache::PkgIterator::NeedsNothing && + (Cache[I].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall && + (I.Purge() != false || Cache[I].Mode != pkgDepCache::ModeDelete || + (Cache[I].iFlags & pkgDepCache::Purge) != pkgDepCache::Purge)) + continue; + + // Append it to the list + List->push_back(I); + } + + return true; +} + /*}}}*/ +// PM::DepAlwaysTrue - Returns true if this dep is irrelevant /*{{{*/ +// --------------------------------------------------------------------- +/* The restriction on provides is to eliminate the case when provides + are transitioning between valid states [ie exim to smail] */ +bool pkgPackageManager::DepAlwaysTrue(DepIterator D) +{ + if (D.TargetPkg()->ProvidesList != 0) + return false; + + if ((Cache[D] & pkgDepCache::DepInstall) != 0 && + (Cache[D] & pkgDepCache::DepNow) != 0) + return true; + return false; +} + /*}}}*/ +// PM::CheckRConflicts - Look for reverse conflicts /*{{{*/ +// --------------------------------------------------------------------- +/* This looks over the reverses for a conflicts line that needs early + removal. */ +bool pkgPackageManager::CheckRConflicts(PkgIterator Pkg,DepIterator D, + const char *Ver) +{ + for (;D.end() == false; ++D) + { + if (D->Type != pkgCache::Dep::Conflicts && + D->Type != pkgCache::Dep::Obsoletes) + continue; + + // The package hasn't been changed + if (List->IsNow(Pkg) == false) + continue; + + // Ignore self conflicts, ignore conflicts from irrelevant versions + if (D.IsIgnorable(Pkg) || D.ParentVer() != D.ParentPkg().CurrentVer()) + continue; + + if (Cache.VS().CheckDep(Ver,D->CompareOp,D.TargetVer()) == false) + continue; + + if (EarlyRemove(D.ParentPkg(), &D) == false) + return _error->Error("Reverse conflicts early remove for package '%s' failed", + Pkg.FullName().c_str()); + } + return true; +} + /*}}}*/ +// PM::CheckRBreaks - Look for reverse breaks /*{{{*/ +bool pkgPackageManager::CheckRBreaks(PkgIterator const &Pkg, DepIterator D, + const char * const Ver) +{ + for (;D.end() == false; ++D) + { + if (D->Type != pkgCache::Dep::DpkgBreaks) + continue; + + PkgIterator const DP = D.ParentPkg(); + if (Cache[DP].Delete() == false) + continue; + + // Ignore self conflicts, ignore conflicts from irrelevant versions + if (D.IsIgnorable(Pkg) || D.ParentVer() != DP.CurrentVer()) + continue; + + if (Cache.VS().CheckDep(Ver, D->CompareOp, D.TargetVer()) == false) + continue; + + // no earlyremove() here as user has already agreed to the permanent removal + if (SmartRemove(DP) == false) + return _error->Error("Internal Error, Could not early remove %s (%d)",DP.FullName().c_str(), 4); + } + return true; +} + /*}}}*/ +// PM::ConfigureAll - Run the all out configuration /*{{{*/ +// --------------------------------------------------------------------- +/* This configures every package. It is assumed they are all unpacked and + that the final configuration is valid. This is also used to catch packages + that have not been configured when using ImmConfigureAll */ +bool pkgPackageManager::ConfigureAll() +{ + pkgOrderList OList(&Cache); + + // Populate the order list + for (pkgOrderList::iterator I = List->begin(); I != List->end(); ++I) + if (List->IsFlag(pkgCache::PkgIterator(Cache,*I), + pkgOrderList::UnPacked) == true) + OList.push_back(*I); + + if (OList.OrderConfigure() == false) + return false; + + std::string const conf = _config->Find("PackageManager::Configure", "smart"); + bool const ConfigurePkgs = (ImmConfigureAll || conf == "all"); + + // Perform the configuring + for (pkgOrderList::iterator I = OList.begin(); I != OList.end(); ++I) + { + PkgIterator Pkg(Cache,*I); + + /* Check if the package has been configured, this can happen if SmartConfigure + calls its self */ + if (List->IsFlag(Pkg,pkgOrderList::Configured)) continue; + + if (ConfigurePkgs == true && SmartConfigure(Pkg, 0) == false) { + if (ImmConfigureAll) + _error->Error(_("Could not perform immediate configuration on '%s'. " + "Please see man 5 apt.conf under APT::Immediate-Configure for details. (%d)"),Pkg.FullName().c_str(),1); + else + _error->Error("Internal error, packages left unconfigured. %s",Pkg.FullName().c_str()); + return false; + } + + List->Flag(Pkg,pkgOrderList::Configured,pkgOrderList::States); + } + + return true; +} + /*}}}*/ +// PM::NonLoopingSmart - helper to avoid loops while calling Smart methods /*{{{*/ +// ----------------------------------------------------------------------- +/* ensures that a loop of the form A depends B, B depends A (and similar) + is not leading us down into infinite recursion segfault land */ +bool pkgPackageManager::NonLoopingSmart(SmartAction const action, pkgCache::PkgIterator &Pkg, + pkgCache::PkgIterator DepPkg, int const Depth, bool const PkgLoop, + bool * const Bad, bool * const Changed) +{ + if (PkgLoop == false) + List->Flag(Pkg,pkgOrderList::Loop); + bool success = false; + switch(action) + { + case UNPACK_IMMEDIATE: success = SmartUnPack(DepPkg, true, Depth + 1); break; + case UNPACK: success = SmartUnPack(DepPkg, false, Depth + 1); break; + case CONFIGURE: success = SmartConfigure(DepPkg, Depth + 1); break; + } + if (PkgLoop == false) + List->RmFlag(Pkg,pkgOrderList::Loop); + + if (success == false) + return false; + + if (Bad != NULL) + *Bad = false; + if (Changed != NULL && List->IsFlag(DepPkg,pkgOrderList::Loop) == false) + *Changed = true; + return true; +} + /*}}}*/ +// PM::SmartConfigure - Perform immediate configuration of the pkg /*{{{*/ +// --------------------------------------------------------------------- +/* This function tries to put the system in a state where Pkg can be configured. + This involves checking each of Pkg's dependencies and unpacking and + configuring packages where needed. */ +bool pkgPackageManager::SmartConfigure(PkgIterator Pkg, int const Depth) +{ + // If this is true, only check and correct and dependencies without the Loop flag + bool const PkgLoop = List->IsFlag(Pkg,pkgOrderList::Loop); + + if (Debug) { + VerIterator InstallVer = VerIterator(Cache,Cache[Pkg].InstallVer); + clog << OutputInDepth(Depth) << "SmartConfigure " << Pkg.FullName() << " (" << InstallVer.VerStr() << ")"; + if (PkgLoop) + clog << " (Only Correct Dependencies)"; + clog << endl; + } + + VerIterator const instVer = Cache[Pkg].InstVerIter(Cache); + + /* Because of the ordered list, most dependencies should be unpacked, + however if there is a loop (A depends on B, B depends on A) this will not + be the case, so check for dependencies before configuring. */ + bool Bad = false, Changed = false; + const unsigned int max_loops = _config->FindI("APT::pkgPackageManager::MaxLoopCount", 5000); + unsigned int i=0; + std::list<DepIterator> needConfigure; + do + { + // Check each dependency and see if anything needs to be done + // so that it can be configured + Changed = false; + for (DepIterator D = instVer.DependsList(); D.end() == false; ) + { + // Compute a single dependency element (glob or) + pkgCache::DepIterator Start, End; + D.GlobOr(Start,End); + + if (End->Type != pkgCache::Dep::Depends && End->Type != pkgCache::Dep::PreDepends) + continue; + Bad = true; + + // the first pass checks if we its all good, i.e. if we have + // to do anything at all + for (DepIterator Cur = Start; true; ++Cur) + { + std::unique_ptr<Version *[]> VList(Cur.AllTargets()); + + for (Version **I = VList.get(); *I != 0; ++I) + { + VerIterator Ver(Cache,*I); + PkgIterator DepPkg = Ver.ParentPkg(); + + // Check if the current version of the package is available and will satisfy this dependency + if (DepPkg.CurrentVer() == Ver && List->IsNow(DepPkg) == true && + List->IsFlag(DepPkg,pkgOrderList::Removed) == false && + DepPkg.State() == PkgIterator::NeedsNothing && + (Cache[DepPkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall) + { + Bad = false; + break; + } + + // Check if the version that is going to be installed will satisfy the dependency + if (Cache[DepPkg].InstallVer != *I || List->IsNow(DepPkg) == false) + continue; + + if (PkgLoop == true) + { + if (Debug) + std::clog << OutputInDepth(Depth) << "Package " << APT::PrettyPkg(&Cache, Pkg) << " loops in SmartConfigure"; + if (List->IsFlag(DepPkg,pkgOrderList::UnPacked)) + Bad = false; + else if (Debug) + std::clog << ", but it isn't unpacked yet"; + if (Debug) + std::clog << std::endl; + } + } + + if (Cur == End || Bad == false) + break; + } + + // this dependency is in a good state, so we can stop + if (Bad == false) + { + if (Debug) + std::clog << OutputInDepth(Depth) << "Found ok dep " << APT::PrettyPkg(&Cache, Start.TargetPkg()) << std::endl; + continue; + } + + // Check for dependencies that have not been unpacked, + // probably due to loops. + for (DepIterator Cur = Start; true; ++Cur) + { + std::unique_ptr<Version *[]> VList(Cur.AllTargets()); + + for (Version **I = VList.get(); *I != 0; ++I) + { + VerIterator Ver(Cache,*I); + PkgIterator DepPkg = Ver.ParentPkg(); + + // Check if the current version of the package is available and will satisfy this dependency + if (DepPkg.CurrentVer() == Ver && List->IsNow(DepPkg) == true && + List->IsFlag(DepPkg,pkgOrderList::Removed) == false && + DepPkg.State() == PkgIterator::NeedsNothing && + (Cache[DepPkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall) + continue; + + // Check if the version that is going to be installed will satisfy the dependency + if (Cache[DepPkg].InstallVer != *I || List->IsNow(DepPkg) == false) + continue; + + if (PkgLoop == true) + { + if (Debug) + std::clog << OutputInDepth(Depth) << "Package " << APT::PrettyPkg(&Cache, Pkg) << " loops in SmartConfigure"; + if (List->IsFlag(DepPkg,pkgOrderList::UnPacked)) + Bad = false; + else if (Debug) + std::clog << ", but it isn't unpacked yet"; + if (Debug) + std::clog << std::endl; + } + else + { + if (Debug) + clog << OutputInDepth(Depth) << "Unpacking " << DepPkg.FullName() << " to avoid loop " << APT::PrettyDep(&Cache, Cur) << endl; + if (NonLoopingSmart(UNPACK_IMMEDIATE, Pkg, DepPkg, Depth, PkgLoop, &Bad, &Changed) == false) + return false; + } + // at this point we either unpacked a Dep or we are in a loop, + // no need to unpack a second one + break; + } + + if (Cur == End || Bad == false) + break; + } + + if (Bad == false) + continue; + + needConfigure.push_back(Start); + } + if (i++ > max_loops) + return _error->Error("Internal error: MaxLoopCount reached in SmartUnPack (1) for %s, aborting", Pkg.FullName().c_str()); + } while (Changed == true); + + // now go over anything that needs configuring + Bad = false, Changed = false, i = 0; + do + { + Changed = false; + for (std::list<DepIterator>::const_iterator D = needConfigure.begin(); D != needConfigure.end(); ++D) + { + // Compute a single dependency element (glob or) without modifying D + pkgCache::DepIterator Start, End; + { + pkgCache::DepIterator Discard = *D; + Discard.GlobOr(Start,End); + } + + if (End->Type != pkgCache::Dep::Depends && End->Type != pkgCache::Dep::PreDepends) + continue; + Bad = true; + + // Search for dependencies which are unpacked but aren't configured yet (maybe loops) + for (DepIterator Cur = Start; true; ++Cur) + { + std::unique_ptr<Version *[]> VList(Cur.AllTargets()); + + for (Version **I = VList.get(); *I != 0; ++I) + { + VerIterator Ver(Cache,*I); + PkgIterator DepPkg = Ver.ParentPkg(); + + // Check if the version that is going to be installed will satisfy the dependency + if (Cache[DepPkg].InstallVer != *I) + continue; + + if (List->IsFlag(DepPkg,pkgOrderList::UnPacked)) + { + if (List->IsFlag(DepPkg,pkgOrderList::Loop) && PkgLoop) + { + // This dependency has already been dealt with by another SmartConfigure on Pkg + Bad = false; + break; + } + if (Debug) + std::clog << OutputInDepth(Depth) << "Configure already unpacked " << APT::PrettyPkg(&Cache, DepPkg) << std::endl; + if (NonLoopingSmart(CONFIGURE, Pkg, DepPkg, Depth, PkgLoop, &Bad, &Changed) == false) + return false; + break; + + } + else if (List->IsFlag(DepPkg,pkgOrderList::Configured)) + { + Bad = false; + break; + } + } + if (Cur == End || Bad == false) + break; + } + + + if (Bad == true && Changed == false && Debug == true) + std::clog << OutputInDepth(Depth) << "Could not satisfy " << APT::PrettyDep(&Cache, *D) << std::endl; + } + if (i++ > max_loops) + return _error->Error("Internal error: MaxLoopCount reached in SmartUnPack (2) for %s, aborting", Pkg.FullName().c_str()); + } while (Changed == true); + + if (Bad == true) + return _error->Error(_("Could not configure '%s'. "),Pkg.FullName().c_str()); + + // Check for reverse conflicts. + if (CheckRBreaks(Pkg,Pkg.RevDependsList(), instVer.VerStr()) == false) + return false; + + for (PrvIterator P = instVer.ProvidesList(); P.end() == false; ++P) + if (Pkg->Group != P.OwnerPkg()->Group) + CheckRBreaks(Pkg,P.ParentPkg().RevDependsList(),P.ProvideVersion()); + + if (PkgLoop) return true; + + static std::string const conf = _config->Find("PackageManager::Configure", "smart"); + static bool const ConfigurePkgs = (conf == "all" || conf == "smart"); + + if (List->IsFlag(Pkg,pkgOrderList::Configured)) + return _error->Error("Internal configure error on '%s'.", Pkg.FullName().c_str()); + + if (ConfigurePkgs == true && Configure(Pkg) == false) + return false; + + List->Flag(Pkg,pkgOrderList::Configured,pkgOrderList::States); + + if ((Cache[Pkg].InstVerIter(Cache)->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same && + not List->IsFlag(Pkg, pkgOrderList::Immediate)) + for (PkgIterator P = Pkg.Group().PackageList(); + P.end() == false; P = Pkg.Group().NextPkg(P)) + { + if (Pkg == P || List->IsFlag(P,pkgOrderList::Configured) == true || + List->IsFlag(P,pkgOrderList::UnPacked) == false || + Cache[P].InstallVer == 0 || (P.CurrentVer() == Cache[P].InstallVer && + (Cache[Pkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall)) + continue; + if (SmartConfigure(P, (Depth +1)) == false) + return false; + } + + // Sanity Check + if (List->IsFlag(Pkg,pkgOrderList::Configured) == false) + return _error->Error(_("Could not configure '%s'. "),Pkg.FullName().c_str()); + + return true; +} + /*}}}*/ +// PM::EarlyRemove - Perform removal of packages before their time /*{{{*/ +// --------------------------------------------------------------------- +/* This is called to deal with conflicts arising from unpacking */ +bool pkgPackageManager::EarlyRemove(PkgIterator Pkg, DepIterator const * const Dep) +{ + if (List->IsNow(Pkg) == false) + return true; + + // Already removed it + if (List->IsFlag(Pkg,pkgOrderList::Removed) == true) + return true; + + // Woops, it will not be re-installed! + if (List->IsFlag(Pkg,pkgOrderList::InList) == false) + return false; + + // these breaks on M-A:same packages can be dealt with. They 'loop' by design + if (Dep != NULL && (*Dep)->Type == pkgCache::Dep::DpkgBreaks && Dep->IsMultiArchImplicit() == true) + return true; + + // Essential packages get special treatment + bool IsEssential = false; + if ((Pkg->Flags & pkgCache::Flag::Essential) != 0) + IsEssential = true; + bool IsProtected = false; + if ((Pkg->Flags & pkgCache::Flag::Important) != 0) + IsProtected = true; + + /* Check for packages that are the dependents of essential packages and + promote them too */ + if (Pkg->CurrentVer != 0) + { + for (pkgCache::DepIterator D = Pkg.RevDependsList(); D.end() == false && + IsEssential == false; ++D) + if (D->Type == pkgCache::Dep::Depends || D->Type == pkgCache::Dep::PreDepends) { + if ((D.ParentPkg()->Flags & pkgCache::Flag::Essential) != 0) + IsEssential = true; + if ((D.ParentPkg()->Flags & pkgCache::Flag::Important) != 0) + IsProtected = true; + } + } + + // dpkg will auto-deconfigure it, no need for the big remove hammer + if (Dep != NULL && (*Dep)->Type == pkgCache::Dep::DpkgBreaks) + return true; + else if (IsEssential == true) + { + // FIXME: Unify messaging with Protected below. + if (_config->FindB("APT::Force-LoopBreak",false) == false) + return _error->Error(_("This installation run will require temporarily " + "removing the essential package %s due to a " + "Conflicts/Pre-Depends loop. This is often bad, " + "but if you really want to do it, activate the " + "APT::Force-LoopBreak option."),Pkg.FullName().c_str()); + } + else if (IsProtected == true) + { + // FIXME: Message should talk about Protected, not Essential, and unified. + if (_config->FindB("APT::Force-LoopBreak",false) == false) + return _error->Error(_("This installation run will require temporarily " + "removing the essential package %s due to a " + "Conflicts/Pre-Depends loop. This is often bad, " + "but if you really want to do it, activate the " + "APT::Force-LoopBreak option."),Pkg.FullName().c_str()); + } + + bool Res = SmartRemove(Pkg); + if (Cache[Pkg].Delete() == false) + List->Flag(Pkg,pkgOrderList::Removed,pkgOrderList::States); + + return Res; +} + /*}}}*/ +// PM::SmartRemove - Removal Helper /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgPackageManager::SmartRemove(PkgIterator Pkg) +{ + if (List->IsNow(Pkg) == false) + return true; + + List->Flag(Pkg,pkgOrderList::Configured,pkgOrderList::States); + + return Remove(Pkg,(Cache[Pkg].iFlags & pkgDepCache::Purge) == pkgDepCache::Purge); +} + /*}}}*/ +// PM::SmartUnPack - Install helper /*{{{*/ +// --------------------------------------------------------------------- +/* This puts the system in a state where it can Unpack Pkg, if Pkg is already + unpacked, or when it has been unpacked, if Immediate==true it configures it. */ +bool pkgPackageManager::SmartUnPack(PkgIterator Pkg, bool const Immediate, int const Depth) +{ + bool PkgLoop = List->IsFlag(Pkg,pkgOrderList::Loop); + + if (Debug) { + clog << OutputInDepth(Depth) << "SmartUnPack " << Pkg.FullName(); + VerIterator InstallVer = VerIterator(Cache,Cache[Pkg].InstallVer); + if (Pkg.CurrentVer() == 0) + clog << " (install version " << InstallVer.VerStr() << ")"; + else + clog << " (replace version " << Pkg.CurrentVer().VerStr() << " with " << InstallVer.VerStr() << ")"; + if (PkgLoop) + clog << " (Only Perform PreUnpack Checks)"; + if (Immediate) + clog << " immediately"; + clog << endl; + } + + VerIterator const instVer = Cache[Pkg].InstVerIter(Cache); + + /* PreUnpack Checks: This loop checks and attempts to rectify any problems that would prevent the package being unpacked. + It addresses: PreDepends, Conflicts, Obsoletes and Breaks (DpkgBreaks). Any resolutions that do not require it should + avoid configuration (calling SmartUnpack with Immediate=true), this is because when unpacking some packages with + complex dependency structures, trying to configure some packages while breaking the loops can complicate things. + This will be either dealt with if the package is configured as a dependency of Pkg (if and when Pkg is configured), + or by the ConfigureAll call at the end of the for loop in OrderInstall. */ + bool SomethingBad = false, Changed = false; + bool couldBeTemporaryRemoved = Depth != 0 && List->IsFlag(Pkg,pkgOrderList::Removed) == false; + const unsigned int max_loops = _config->FindI("APT::pkgPackageManager::MaxLoopCount", 5000); + unsigned int i = 0; + do + { + Changed = false; + for (DepIterator D = instVer.DependsList(); D.end() == false; ) + { + // Compute a single dependency element (glob or) + pkgCache::DepIterator Start, End; + D.GlobOr(Start,End); + + if (End->Type == pkgCache::Dep::PreDepends) + { + bool Bad = true; + if (Debug) + clog << OutputInDepth(Depth) << "PreDepends order for " << Pkg.FullName() << std::endl; + + // Look for easy targets: packages that are already okay + for (DepIterator Cur = Start; Bad == true; ++Cur) + { + std::unique_ptr<Version *[]> VList(Cur.AllTargets()); + for (Version **I = VList.get(); *I != 0; ++I) + { + VerIterator Ver(Cache,*I); + PkgIterator Pkg = Ver.ParentPkg(); + + // See if the current version is ok + if (Pkg.CurrentVer() == Ver && List->IsNow(Pkg) == true && + Pkg.State() == PkgIterator::NeedsNothing && + (Cache[Pkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall) + { + Bad = false; + if (Debug) + clog << OutputInDepth(Depth) << "Found ok package " << Pkg.FullName() << endl; + break; + } + } + if (Cur == End) + break; + } + + // Look for something that could be configured. + for (DepIterator Cur = Start; Bad == true && Cur.end() == false; ++Cur) + { + std::unique_ptr<Version *[]> VList(Cur.AllTargets()); + for (Version **I = VList.get(); *I != 0; ++I) + { + VerIterator Ver(Cache,*I); + PkgIterator DepPkg = Ver.ParentPkg(); + + // Not the install version + if (Cache[DepPkg].InstallVer != *I) + continue; + + if (Cache[DepPkg].Keep() == true && DepPkg.State() == PkgIterator::NeedsNothing && + (Cache[DepPkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall) + continue; + + if (List->IsFlag(DepPkg,pkgOrderList::Configured)) + { + Bad = false; + break; + } + + // check if it needs unpack or if configure is enough + if (List->IsFlag(DepPkg,pkgOrderList::UnPacked) == false) + { + // two packages pre-depending on each other can't be handled sanely + if (List->IsFlag(DepPkg,pkgOrderList::Loop) && PkgLoop) + { + // this isn't an error as there is potential for something else to satisfy it + // (like a provides or an or-group member) + if (Debug) + clog << OutputInDepth(Depth) << "Unpack loop detected between " << DepPkg.FullName() << " and " << Pkg.FullName() << endl; + continue; + } + + if (Debug) + clog << OutputInDepth(Depth) << "Trying to SmartUnpack " << DepPkg.FullName() << endl; + if (NonLoopingSmart(UNPACK_IMMEDIATE, Pkg, DepPkg, Depth, PkgLoop, &Bad, &Changed) == false) + return false; + } + else + { + if (Debug) + clog << OutputInDepth(Depth) << "Trying to SmartConfigure " << DepPkg.FullName() << endl; + if (NonLoopingSmart(CONFIGURE, Pkg, DepPkg, Depth, PkgLoop, &Bad, &Changed) == false) + return false; + } + break; + } + } + + if (Bad == true) + SomethingBad = true; + } + else if (End->Type == pkgCache::Dep::Conflicts || + End->Type == pkgCache::Dep::Obsoletes || + End->Type == pkgCache::Dep::DpkgBreaks) + { + std::unique_ptr<Version *[]> VList(End.AllTargets()); + for (Version **I = VList.get(); *I != 0; ++I) + { + VerIterator Ver(Cache,*I); + PkgIterator ConflictPkg = Ver.ParentPkg(); + if (ConflictPkg.CurrentVer() != Ver) + { + if (Debug) + std::clog << OutputInDepth(Depth) << "Ignore not-installed version " << Ver.VerStr() << " of " << ConflictPkg.FullName() << " for " << APT::PrettyDep(&Cache, End) << std::endl; + continue; + } + + if (List->IsNow(ConflictPkg) == false) + { + if (Debug) + std::clog << OutputInDepth(Depth) << "Ignore already dealt-with version " << Ver.VerStr() << " of " << ConflictPkg.FullName() << " for " << APT::PrettyDep(&Cache, End) << std::endl; + continue; + } + + if (List->IsFlag(ConflictPkg,pkgOrderList::Removed) == true) + { + if (Debug) + clog << OutputInDepth(Depth) << "Ignoring " << APT::PrettyDep(&Cache, End) << " as " << ConflictPkg.FullName() << "was temporarily removed" << endl; + continue; + } + + if (List->IsFlag(ConflictPkg,pkgOrderList::Loop) && PkgLoop) + { + if (End->Type == pkgCache::Dep::DpkgBreaks && End.IsMultiArchImplicit() == true) + { + if (Debug) + clog << OutputInDepth(Depth) << "Because dependency is MultiArchImplicit we ignored looping on: " << APT::PrettyPkg(&Cache, ConflictPkg) << endl; + continue; + } + if (Debug) + { + if (End->Type == pkgCache::Dep::DpkgBreaks) + clog << OutputInDepth(Depth) << "Because of breaks knot, deconfigure " << ConflictPkg.FullName() << " temporarily" << endl; + else + clog << OutputInDepth(Depth) << "Because of conflict knot, removing " << ConflictPkg.FullName() << " temporarily" << endl; + } + if (EarlyRemove(ConflictPkg, &End) == false) + return _error->Error("Internal Error, Could not early remove %s (%d)",ConflictPkg.FullName().c_str(), 3); + SomethingBad = true; + continue; + } + + if (Cache[ConflictPkg].Delete() == false) + { + if (Debug) + { + clog << OutputInDepth(Depth) << "Unpacking " << ConflictPkg.FullName() << " to avoid " << APT::PrettyDep(&Cache, End); + if (PkgLoop == true) + clog << " (Looping)"; + clog << std::endl; + } + // we would like to avoid temporary removals and all that at best via a simple unpack + _error->PushToStack(); + if (NonLoopingSmart(UNPACK, Pkg, ConflictPkg, Depth, PkgLoop, NULL, &Changed) == false) + { + // but if it fails ignore this failure and look for alternative ways of solving + if (Debug) + { + clog << OutputInDepth(Depth) << "Avoidance unpack of " << ConflictPkg.FullName() << " failed for " << APT::PrettyDep(&Cache, End) << " ignoring:" << std::endl; + _error->DumpErrors(std::clog, GlobalError::DEBUG, false); + } + _error->RevertToStack(); + // ignorance can only happen if a) one of the offenders is already gone + if (List->IsFlag(ConflictPkg,pkgOrderList::Removed) == true) + { + if (Debug) + clog << OutputInDepth(Depth) << "But " << ConflictPkg.FullName() << " was temporarily removed in the meantime to satisfy " << APT::PrettyDep(&Cache, End) << endl; + } + else if (List->IsFlag(Pkg,pkgOrderList::Removed) == true) + { + if (Debug) + clog << OutputInDepth(Depth) << "But " << Pkg.FullName() << " was temporarily removed in the meantime to satisfy " << APT::PrettyDep(&Cache, End) << endl; + } + // or b) we can make one go (removal or dpkg auto-deconfigure) + else + { + if (Debug) + clog << OutputInDepth(Depth) << "So temporary remove/deconfigure " << ConflictPkg.FullName() << " to satisfy " << APT::PrettyDep(&Cache, End) << endl; + if (EarlyRemove(ConflictPkg, &End) == false) + return _error->Error("Internal Error, Could not early remove %s (%d)",ConflictPkg.FullName().c_str(), 2); + } + } + else + _error->MergeWithStack(); + } + else + { + if (Debug) + clog << OutputInDepth(Depth) << "Removing " << ConflictPkg.FullName() << " now to avoid " << APT::PrettyDep(&Cache, End) << endl; + // no earlyremove() here as user has already agreed to the permanent removal + if (SmartRemove(ConflictPkg) == false) + return _error->Error("Internal Error, Could not early remove %s (%d)",ConflictPkg.FullName().c_str(), 1); + } + } + } + } + if (i++ > max_loops) + return _error->Error("Internal error: APT::pkgPackageManager::MaxLoopCount reached in SmartConfigure for %s, aborting", Pkg.FullName().c_str()); + } while (Changed == true); + + if (SomethingBad == true) + return _error->Error("Couldn't configure %s, probably a dependency cycle.", Pkg.FullName().c_str()); + + if (couldBeTemporaryRemoved == true && List->IsFlag(Pkg,pkgOrderList::Removed) == true) + { + if (Debug) + std::clog << OutputInDepth(Depth) << "Prevent unpack as " << APT::PrettyPkg(&Cache, Pkg) << " is currently temporarily removed" << std::endl; + return true; + } + + // Check for reverse conflicts. + if (CheckRConflicts(Pkg,Pkg.RevDependsList(), + instVer.VerStr()) == false) + return false; + + for (PrvIterator P = instVer.ProvidesList(); + P.end() == false; ++P) + if (Pkg->Group != P.OwnerPkg()->Group) + CheckRConflicts(Pkg,P.ParentPkg().RevDependsList(),P.ProvideVersion()); + + if (PkgLoop) + return true; + + List->Flag(Pkg,pkgOrderList::UnPacked,pkgOrderList::States); + + if (Immediate == true && (instVer->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same) + { + /* Do lockstep M-A:same unpacking in two phases: + First unpack all installed architectures, then the not installed. + This way we avoid that M-A: enabled packages are installed before + their older non-M-A enabled packages are replaced by newer versions */ + bool const installed = Pkg->CurrentVer != 0; + if (installed == true && + (instVer != Pkg.CurrentVer() || + ((Cache[Pkg].iFlags & pkgDepCache::ReInstall) == pkgDepCache::ReInstall)) && + Install(Pkg,FileNames[Pkg->ID]) == false) + return false; + for (PkgIterator P = Pkg.Group().PackageList(); + P.end() == false; P = Pkg.Group().NextPkg(P)) + { + if (P->CurrentVer == 0 || P == Pkg || List->IsFlag(P,pkgOrderList::UnPacked) == true || + Cache[P].InstallVer == 0 || (P.CurrentVer() == Cache[P].InstallVer && + (Cache[Pkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall)) + continue; + if (SmartUnPack(P, false, Depth + 1) == false) + return false; + } + if (installed == false && Install(Pkg,FileNames[Pkg->ID]) == false) + return false; + for (PkgIterator P = Pkg.Group().PackageList(); + P.end() == false; P = Pkg.Group().NextPkg(P)) + { + if (P->CurrentVer != 0 || P == Pkg || List->IsFlag(P,pkgOrderList::UnPacked) == true || + List->IsFlag(P,pkgOrderList::Configured) == true || + Cache[P].InstallVer == 0 || (P.CurrentVer() == Cache[P].InstallVer && + (Cache[Pkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall)) + continue; + if (SmartUnPack(P, false, Depth + 1) == false) + return false; + } + } + // packages which are already unpacked don't need to be unpacked again + else if ((instVer != Pkg.CurrentVer() || + ((Cache[Pkg].iFlags & pkgDepCache::ReInstall) == pkgDepCache::ReInstall)) && + Install(Pkg,FileNames[Pkg->ID]) == false) + return false; + + if (Immediate == true) { + // Perform immediate configuration of the package. + _error->PushToStack(); + bool configured = SmartConfigure(Pkg, Depth + 1); + _error->RevertToStack(); + + if (not configured && Debug) { + clog << OutputInDepth(Depth); + ioprintf(clog, _("Could not perform immediate configuration on '%s'. " + "Please see man 5 apt.conf under APT::Immediate-Configure for details. (%d)"), + Pkg.FullName().c_str(), 2); + clog << endl; + } + } + + return true; +} + /*}}}*/ +// PM::OrderInstall - Installation ordering routine /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgPackageManager::OrderResult pkgPackageManager::OrderInstall() +{ + if (CreateOrderList() == false) + return Failed; + + Reset(); + + if (Debug == true) + clog << "Beginning to order" << endl; + + std::string const planner = _config->Find("APT::Planner", "internal"); + unsigned int flags = 0; + if (_config->FindB("APT::Immediate-Configure", true) == false) + flags |= EIPP::Request::NO_IMMEDIATE_CONFIGURATION; + else if (_config->FindB("APT::Immediate-Configure-All", false)) + flags |= EIPP::Request::IMMEDIATE_CONFIGURATION_ALL; + else if (_config->FindB("APT::Force-LoopBreak", false)) + flags |= EIPP::Request::ALLOW_TEMPORARY_REMOVE_OF_ESSENTIALS; + auto const ret = EIPP::OrderInstall(planner.c_str(), this, flags, nullptr); + if (planner != "internal") + return ret ? Completed : Failed; + + bool const ordering = + _config->FindB("PackageManager::UnpackAll",true) ? + List->OrderUnpack(FileNames) : List->OrderCritical(); + if (ordering == false) + { + _error->Error("Internal ordering error"); + return Failed; + } + + if (Debug == true) + clog << "Done ordering" << endl; + + bool DoneSomething = false; + for (pkgOrderList::iterator I = List->begin(); I != List->end(); ++I) + { + PkgIterator Pkg(Cache,*I); + + if (List->IsNow(Pkg) == false) + { + if (Debug == true) + clog << "Skipping already done " << Pkg.FullName() << endl; + continue; + } + + if (List->IsMissing(Pkg) == true) + { + if (Debug == true) + clog << "Sequence completed at " << Pkg.FullName() << endl; + if (DoneSomething == false) + { + _error->Error("Internal Error, ordering was unable to handle the media swap"); + return Failed; + } + return Incomplete; + } + + // Sanity check + if (Cache[Pkg].Keep() == true && + Pkg.State() == pkgCache::PkgIterator::NeedsNothing && + (Cache[Pkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall) + { + _error->Error("Internal Error, trying to manipulate a kept package (%s)",Pkg.FullName().c_str()); + return Failed; + } + + // Perform a delete or an install + if (Cache[Pkg].Delete() == true) + { + if (SmartRemove(Pkg) == false) + return Failed; + } + else + if (SmartUnPack(Pkg,List->IsFlag(Pkg,pkgOrderList::Immediate),0) == false) + return Failed; + DoneSomething = true; + + if (ImmConfigureAll) { + /* ConfigureAll here to pick up and packages left unconfigured because they were unpacked in the + "PreUnpack Checks" section */ + if (!ConfigureAll()) + return Failed; + } + } + + // Final run through the configure phase + if (ConfigureAll() == false) + return Failed; + + // Sanity check + for (pkgOrderList::iterator I = List->begin(); I != List->end(); ++I) + { + if (List->IsFlag(*I,pkgOrderList::Configured) == false) + { + _error->Error("Internal error, packages left unconfigured. %s", + PkgIterator(Cache,*I).FullName().c_str()); + return Failed; + } + } + + return Completed; +} +// PM::DoInstallPostFork - Does install part that happens after the fork /*{{{*/ +// --------------------------------------------------------------------- +pkgPackageManager::OrderResult +pkgPackageManager::DoInstallPostFork(APT::Progress::PackageManager *progress) +{ + bool goResult; + goResult = Go(progress); + if(goResult == false) + return Failed; + + return Res; +} + /*}}}*/ +// PM::DoInstall - Does the installation /*{{{*/ +// --------------------------------------------------------------------- +/* This uses the filenames in FileNames and the information in the + DepCache to perform the installation of packages.*/ +pkgPackageManager::OrderResult +pkgPackageManager::DoInstall(APT::Progress::PackageManager *progress) +{ + if(DoInstallPreFork() == Failed) + return Failed; + + return DoInstallPostFork(progress); +} + /*}}}*/ diff --git a/apt-pkg/packagemanager.h b/apt-pkg/packagemanager.h new file mode 100644 index 0000000..d4cc3c6 --- /dev/null +++ b/apt-pkg/packagemanager.h @@ -0,0 +1,138 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Package Manager - Abstracts the package manager + + Three steps are + - Aquiration of archives (stores the list of final file names) + - Sorting of operations + - Invocation of package manager + + This is the final stage when the package cache entities get converted + into file names and the state stored in a DepCache is transformed + into a series of operations. + + In the final scheme of things this may serve as a director class to + access the actual install methods based on the file type being + installed. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_PACKAGEMANAGER_H +#define PKGLIB_PACKAGEMANAGER_H + +#include <apt-pkg/edsp.h> +#include <apt-pkg/init.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> + +#include <set> +#include <string> + + +class pkgAcquire; +class pkgDepCache; +class pkgSourceList; +class pkgOrderList; +class pkgRecords; +class OpProgress; +class pkgPackageManager; +namespace APT { + namespace Progress { + class PackageManager; + } +} + +class APT_PUBLIC pkgPackageManager : protected pkgCache::Namespace +{ + public: + + enum OrderResult {Completed,Failed,Incomplete}; + static bool SigINTStop; + + protected: + std::string *FileNames; + pkgDepCache &Cache; + pkgOrderList *List; + bool Debug; + bool NoImmConfigure; + bool ImmConfigureAll; + + /** \brief saves packages dpkg let disappear + + This way APT can retreat from trying to configure these + packages later on and a front-end can choose to display a + notice to inform the user about these disappears. + */ + std::set<std::string> disappearedPkgs; + + void ImmediateAdd(PkgIterator P, bool UseInstallVer, unsigned const int &Depth = 0); + virtual OrderResult OrderInstall(); + bool CheckRConflicts(PkgIterator Pkg,DepIterator Dep,const char *Ver); + bool CheckRBreaks(PkgIterator const &Pkg,DepIterator Dep,const char * const Ver); + bool CreateOrderList(); + + // Analysis helpers + bool DepAlwaysTrue(DepIterator D) APT_PURE; + + // Install helpers + bool ConfigureAll(); + bool SmartConfigure(PkgIterator Pkg, int const Depth) APT_MUSTCHECK; + bool SmartUnPack(PkgIterator Pkg, bool const Immediate = true, int const Depth = 0) APT_MUSTCHECK; + bool SmartRemove(PkgIterator Pkg) APT_MUSTCHECK; + bool EarlyRemove(PkgIterator Pkg, DepIterator const * const Dep) APT_MUSTCHECK; + + // The Actual installation implementation + virtual bool Install(PkgIterator /*Pkg*/,std::string /*File*/) {return false;}; + virtual bool Configure(PkgIterator /*Pkg*/) {return false;}; + virtual bool Remove(PkgIterator /*Pkg*/,bool /*Purge*/=false) {return false;}; + virtual bool Go(APT::Progress::PackageManager * /*progress*/) {return true;}; + + virtual void Reset() {}; + + // the result of the operation + OrderResult Res; + + public: + + // Main action members + bool GetArchives(pkgAcquire *Owner,pkgSourceList *Sources, + pkgRecords *Recs); + + // Do the installation + OrderResult DoInstall(APT::Progress::PackageManager *progress); + + friend bool EIPP::OrderInstall(char const * const planner, pkgPackageManager * const PM, + unsigned int const version, OpProgress * const Progress); + friend bool EIPP::ReadResponse(int const input, pkgPackageManager * const PM, + OpProgress * const Progress); + + // stuff that needs to be done before the fork() of a library that + // uses apt + OrderResult DoInstallPreFork() { + Res = OrderInstall(); + return Res; + }; + // stuff that needs to be done after the fork + OrderResult DoInstallPostFork(APT::Progress::PackageManager *progress); + // compat + + // ? + bool FixMissing(); + + /** \brief returns all packages dpkg let disappear */ + inline std::set<std::string> GetDisappearedPackages() { return disappearedPkgs; }; + + explicit pkgPackageManager(pkgDepCache *Cache); + virtual ~pkgPackageManager(); + + private: + void * const d; + enum APT_HIDDEN SmartAction { UNPACK_IMMEDIATE, UNPACK, CONFIGURE }; + APT_HIDDEN bool NonLoopingSmart(SmartAction const action, pkgCache::PkgIterator &Pkg, + pkgCache::PkgIterator DepPkg, int const Depth, bool const PkgLoop, + bool * const Bad, bool * const Changed) APT_MUSTCHECK; +}; + +#endif diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc new file mode 100644 index 0000000..0d18c6c --- /dev/null +++ b/apt-pkg/pkgcache.cc @@ -0,0 +1,1011 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Package Cache - Accessor code for the cache + + Please see doc/apt-pkg/cache.sgml for a more detailed description of + this format. Also be sure to keep that file up-to-date!! + + This is the general utility functions for cache management. They provide + a complete set of accessor functions for the cache. The cacheiterators + header contains the STL-like iterators that can be used to easially + navigate the cache as well as seamlessly dereference the mmap'd + indexes. Use these always. + + The main class provides for ways to get package indexes and some + general lookup functions to start the iterators. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/mmap.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/policy.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/version.h> + +#include <algorithm> +#include <sstream> +#include <string> +#include <vector> +#include <stddef.h> +#include <string.h> +#include <sys/stat.h> +#include <xxhash.h> + +#include <apti18n.h> + /*}}}*/ + +using std::string; +using APT::StringView; + + +// Cache::Header::Header - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Simply initialize the header */ +pkgCache::Header::Header() +{ +#define APT_HEADER_SET(X,Y) X = Y; static_assert(std::numeric_limits<decltype(X)>::max() > Y, "Size violation detected in pkgCache::Header") + APT_HEADER_SET(Signature, 0x98FE76DC); + + /* Whenever the structures change the major version should be bumped, + whenever the generator changes the minor version should be bumped. */ + APT_HEADER_SET(MajorVersion, 16); + APT_HEADER_SET(MinorVersion, 0); + APT_HEADER_SET(Dirty, false); + + APT_HEADER_SET(HeaderSz, sizeof(pkgCache::Header)); + APT_HEADER_SET(GroupSz, sizeof(pkgCache::Group)); + APT_HEADER_SET(PackageSz, sizeof(pkgCache::Package)); + APT_HEADER_SET(ReleaseFileSz, sizeof(pkgCache::ReleaseFile)); + APT_HEADER_SET(PackageFileSz, sizeof(pkgCache::PackageFile)); + APT_HEADER_SET(VersionSz, sizeof(pkgCache::Version)); + APT_HEADER_SET(DescriptionSz, sizeof(pkgCache::Description)); + APT_HEADER_SET(DependencySz, sizeof(pkgCache::Dependency)); + APT_HEADER_SET(DependencyDataSz, sizeof(pkgCache::DependencyData)); + APT_HEADER_SET(ProvidesSz, sizeof(pkgCache::Provides)); + APT_HEADER_SET(VerFileSz, sizeof(pkgCache::VerFile)); + APT_HEADER_SET(DescFileSz, sizeof(pkgCache::DescFile)); +#undef APT_HEADER_SET + + GroupCount = 0; + PackageCount = 0; + VersionCount = 0; + DescriptionCount = 0; + DependsCount = 0; + DependsDataCount = 0; + ReleaseFileCount = 0; + PackageFileCount = 0; + VerFileCount = 0; + DescFileCount = 0; + ProvidesCount = 0; + MaxVerFileSize = 0; + MaxDescFileSize = 0; + + FileList = 0; + RlsFileList = 0; + VerSysName = 0; + Architecture = 0; + SetArchitectures(0); + SetHashTableSize(_config->FindI("APT::Cache-HashTableSize", 196613)); + memset(Pools,0,sizeof(Pools)); + + CacheFileSize = 0; +} + /*}}}*/ +// Cache::Header::CheckSizes - Check if the two headers have same *sz /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgCache::Header::CheckSizes(Header &Against) const +{ + if (HeaderSz == Against.HeaderSz && + GroupSz == Against.GroupSz && + PackageSz == Against.PackageSz && + ReleaseFileSz == Against.ReleaseFileSz && + PackageFileSz == Against.PackageFileSz && + VersionSz == Against.VersionSz && + DescriptionSz == Against.DescriptionSz && + DependencySz == Against.DependencySz && + DependencyDataSz == Against.DependencyDataSz && + VerFileSz == Against.VerFileSz && + DescFileSz == Against.DescFileSz && + ProvidesSz == Against.ProvidesSz) + return true; + return false; +} + /*}}}*/ + +// Cache::pkgCache - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgCache::pkgCache(MMap *Map, bool DoMap) : Map(*Map), VS(nullptr), d(NULL) +{ + // call getArchitectures() with cached=false to ensure that the + // architectures cache is re-evaluated. this is needed in cases + // when the APT::Architecture field changes between two cache creations + APT::Configuration::getArchitectures(false); + MultiArchEnabled = true; + if (DoMap == true) + ReMap(); +} + /*}}}*/ +// Cache::ReMap - Reopen the cache file /*{{{*/ +// --------------------------------------------------------------------- +/* If the file is already closed then this will open it open it. */ +bool pkgCache::ReMap(bool const &Errorchecks) +{ + // Apply the typecasts. + HeaderP = (Header *)Map.Data(); + GrpP = (Group *)Map.Data(); + PkgP = (Package *)Map.Data(); + VerFileP = (VerFile *)Map.Data(); + DescFileP = (DescFile *)Map.Data(); + RlsFileP = (ReleaseFile *)Map.Data(); + PkgFileP = (PackageFile *)Map.Data(); + VerP = (Version *)Map.Data(); + DescP = (Description *)Map.Data(); + ProvideP = (Provides *)Map.Data(); + DepP = (Dependency *)Map.Data(); + DepDataP = (DependencyData *)Map.Data(); + StrP = (char *)Map.Data(); + + if (Errorchecks == false) + return true; + + if (Map.Size() == 0 || HeaderP == 0) + return _error->Error(_("Empty package cache")); + + // Check the header + Header DefHeader; + if (HeaderP->Signature != DefHeader.Signature || + HeaderP->Dirty == true) + return _error->Error(_("The package cache file is corrupted")); + + if (HeaderP->MajorVersion != DefHeader.MajorVersion || + HeaderP->MinorVersion != DefHeader.MinorVersion || + HeaderP->CheckSizes(DefHeader) == false) + return _error->Error(_("The package cache file is an incompatible version")); + + if (HeaderP->VerSysName == 0 || HeaderP->Architecture == 0 || HeaderP->GetArchitectures() == 0) + return _error->Error(_("The package cache file is corrupted")); + + // Locate our VS.. + if ((VS = pkgVersioningSystem::GetVS(StrP + HeaderP->VerSysName)) == 0) + return _error->Error(_("This APT does not support the versioning system '%s'"),StrP + HeaderP->VerSysName); + + // Check the architecture + std::vector<std::string> archs = APT::Configuration::getArchitectures(); + std::string list = ""; + for (auto const & arch : archs) { + if (!list.empty()) + list.append(","); + list.append(arch); + } + if (_config->Find("APT::Architecture") != StrP + HeaderP->Architecture || + list != StrP + HeaderP->GetArchitectures()) + return _error->Error(_("The package cache was built for different architectures: %s vs %s"), StrP + HeaderP->GetArchitectures(), list.c_str()); + + + auto hash = CacheHash(); + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "Opened cache with hash " << hash << ", expecting " << HeaderP->CacheFileSize << "\n"; + if (hash != HeaderP->CacheFileSize) + return _error->Error(_("The package cache file is corrupted, it has the wrong hash")); + + return true; +} + /*}}}*/ +// Cache::Hash - Hash a string /*{{{*/ +// --------------------------------------------------------------------- +/* This is used to generate the hash entries for the HashTable. With my + package list from bo this function gets 94% table usage on a 512 item + table (480 used items) */ +map_id_t pkgCache::sHash(StringView Str) const +{ + uint32_t Hash = 5381; + auto I = Str.begin(); + auto End = Str.end(); + for (; I + 7 < End; I += 8) + { + Hash = (33u * 33u * 33u * 33u * 33u * 33u * 33u * 33u * Hash + + 33u * 33u * 33u * 33u * 33u * 33u * 33u * tolower_ascii_unsafe(I[0]) + + 33u * 33u * 33u * 33u * 33u * 33u * tolower_ascii_unsafe(I[1]) + + 33u * 33u * 33u * 33u * 33u * tolower_ascii_unsafe(I[2]) + + 33u * 33u * 33u * 33u * tolower_ascii_unsafe(I[3]) + + 33u * 33u * 33u * tolower_ascii_unsafe(I[4]) + + 33u * 33u * tolower_ascii_unsafe(I[5]) + + 33u * tolower_ascii_unsafe(I[6]) + + tolower_ascii_unsafe(I[7])); + } + for (; I != End; ++I) + Hash = 33u * Hash + tolower_ascii_unsafe(*I); + return Hash % HeaderP->GetHashTableSize(); +} +uint32_t pkgCache::CacheHash() +{ + pkgCache::Header header = {}; + XXH3_state_t *state = XXH3_createState(); + + if (Map.Size() < sizeof(header)) + return 0; + + XXH3_64bits_reset(state); + memcpy(&header, GetMap().Data(), sizeof(header)); + + header.Dirty = false; + header.CacheFileSize = 0; + + XXH3_64bits_update(state, + reinterpret_cast<const unsigned char *>(PACKAGE_VERSION), + APT_ARRAY_SIZE(PACKAGE_VERSION)); + + XXH3_64bits_update(state, + reinterpret_cast<const unsigned char *>(&header), + sizeof(header)); + + if (Map.Size() > sizeof(header)) { + XXH3_64bits_update(state, + static_cast<const unsigned char *>(GetMap().Data()) + sizeof(header), + GetMap().Size() - sizeof(header)); + } + + auto const digest = XXH3_64bits_digest(state); + XXH3_freeState(state); + return digest & 0xFFFFFFFF; +} + /*}}}*/ +// Cache::FindPkg - Locate a package by name /*{{{*/ +// --------------------------------------------------------------------- +/* Returns 0 on error, pointer to the package otherwise */ +pkgCache::PkgIterator pkgCache::FindPkg(StringView Name) { + auto const found = Name.rfind(':'); + if (found == string::npos) + return FindPkg(Name, "native"); + auto const Arch = Name.substr(found+1); + /* Beware: This is specialcased to handle pkg:any in dependencies + as these are linked to virtual pkg:any named packages. + If you want any arch from a pkg, use FindPkg(pkg,"any") */ + if (Arch == "any") + return FindPkg(Name, "any"); + return FindPkg(Name.substr(0, found), Arch); +} + /*}}}*/ +// Cache::FindPkg - Locate a package by name /*{{{*/ +// --------------------------------------------------------------------- +/* Returns 0 on error, pointer to the package otherwise */ +pkgCache::PkgIterator pkgCache::FindPkg(StringView Name, StringView Arch) { + /* We make a detour via the GrpIterator here as + on a multi-arch environment a group is easier to + find than a package (less entries in the buckets) */ + pkgCache::GrpIterator Grp = FindGrp(Name); + if (Grp.end() == true) + return PkgIterator(*this,0); + + return Grp.FindPkg(Arch); +} + /*}}}*/ +// Cache::FindGrp - Locate a group by name /*{{{*/ +// --------------------------------------------------------------------- +/* Returns End-Pointer on error, pointer to the group otherwise */ +pkgCache::GrpIterator pkgCache::FindGrp(StringView Name) { + if (unlikely(Name.empty() == true)) + return GrpIterator(*this,0); + + // Look at the hash bucket for the group + Group *Grp = GrpP + HeaderP->GrpHashTableP()[sHash(Name)]; + for (; Grp != GrpP; Grp = GrpP + Grp->Next) { + int const cmp = StringViewCompareFast(Name, ViewString(Grp->Name)); + if (cmp == 0) + return GrpIterator(*this, Grp); + else if (cmp < 0) + break; + } + + return GrpIterator(*this,0); +} + /*}}}*/ +// Cache::CompTypeDeb - Return a string describing the compare type /*{{{*/ +// --------------------------------------------------------------------- +/* This returns a string representation of the dependency compare + type in the weird debian style.. */ +const char *pkgCache::CompTypeDeb(unsigned char Comp) +{ + const char * const Ops[] = {"","<=",">=","<<",">>","=","!="}; + if (unlikely((unsigned)(Comp & 0xF) >= sizeof(Ops)/sizeof(Ops[0]))) + return ""; + return Ops[Comp & 0xF]; +} + /*}}}*/ +// Cache::CompType - Return a string describing the compare type /*{{{*/ +// --------------------------------------------------------------------- +/* This returns a string representation of the dependency compare + type */ +const char *pkgCache::CompType(unsigned char Comp) +{ + const char * const Ops[] = {"","<=",">=","<",">","=","!="}; + if (unlikely((unsigned)(Comp & 0xF) >= sizeof(Ops)/sizeof(Ops[0]))) + return ""; + return Ops[Comp & 0xF]; +} + /*}}}*/ +// Cache::DepType - Return a string describing the dep type /*{{{*/ +// --------------------------------------------------------------------- +/* */ +const char *pkgCache::DepType(unsigned char Type) +{ + const char *Types[] = {"",_("Depends"),_("PreDepends"),_("Suggests"), + _("Recommends"),_("Conflicts"),_("Replaces"), + _("Obsoletes"),_("Breaks"), _("Enhances")}; + if (Type < sizeof(Types)/sizeof(*Types)) + return Types[Type]; + return ""; +} + /*}}}*/ +// Cache::Priority - Convert a priority value to a string /*{{{*/ +// --------------------------------------------------------------------- +/* */ +const char *pkgCache::Priority(unsigned char Prio) +{ + const char *Mapping[] = {0,_("required"),_("important"),_("standard"), + _("optional"),_("extra")}; + if (Prio < APT_ARRAY_SIZE(Mapping)) + return Mapping[Prio]; + return 0; +} + /*}}}*/ +// GrpIterator::FindPkg - Locate a package by arch /*{{{*/ +// --------------------------------------------------------------------- +/* Returns an End-Pointer on error, pointer to the package otherwise */ +pkgCache::PkgIterator pkgCache::GrpIterator::FindPkg(StringView Arch) const { + if (unlikely(IsGood() == false || S->FirstPackage == 0)) + return PkgIterator(*Owner, 0); + + /* If we accept any package we simply return the "first" + package in this group */ + if (Arch == "any") + return PkgIterator(*Owner, Owner->PkgP + S->FirstPackage); + if (Arch == "native" || Arch == "all") + Arch = Owner->NativeArch(); + + // Iterate over the list to find the matching arch + for (pkgCache::Package *Pkg = PackageList(); Pkg != Owner->PkgP; + Pkg = Owner->PkgP + Pkg->NextPackage) { + if (Arch == Owner->ViewString(Pkg->Arch)) + return PkgIterator(*Owner, Pkg); + if ((Owner->PkgP + S->LastPackage) == Pkg) + break; + } + + return PkgIterator(*Owner, 0); +} + /*}}}*/ +// GrpIterator::FindPreferredPkg - Locate the "best" package /*{{{*/ +// --------------------------------------------------------------------- +/* Returns an End-Pointer on error, pointer to the package otherwise */ +pkgCache::PkgIterator pkgCache::GrpIterator::FindPreferredPkg(bool const &PreferNonVirtual) const { + pkgCache::PkgIterator Pkg = FindPkg(StringView("native", 6)); + if (Pkg.end() == false && (PreferNonVirtual == false || Pkg->VersionList != 0)) + return Pkg; + + std::vector<std::string> const archs = APT::Configuration::getArchitectures(); + for (std::vector<std::string>::const_iterator a = archs.begin(); + a != archs.end(); ++a) { + Pkg = FindPkg(*a); + if (Pkg.end() == false && (PreferNonVirtual == false || Pkg->VersionList != 0)) + return Pkg; + } + // packages without an architecture + Pkg = FindPkg(StringView("none", 4)); + if (Pkg.end() == false && (PreferNonVirtual == false || Pkg->VersionList != 0)) + return Pkg; + + if (PreferNonVirtual == true) + return FindPreferredPkg(false); + return PkgIterator(*Owner, 0); +} + /*}}}*/ +// GrpIterator::NextPkg - Locate the next package in the group /*{{{*/ +// --------------------------------------------------------------------- +/* Returns an End-Pointer on error, pointer to the package otherwise. + We can't simply ++ to the next as the next package of the last will + be from a different group (with the same hash value) */ +pkgCache::PkgIterator pkgCache::GrpIterator::NextPkg(pkgCache::PkgIterator const &LastPkg) const { + if (unlikely(IsGood() == false || S->FirstPackage == 0 || + LastPkg.end() == true)) + return PkgIterator(*Owner, 0); + + if (S->LastPackage == LastPkg.MapPointer()) + return PkgIterator(*Owner, 0); + + return PkgIterator(*Owner, Owner->PkgP + LastPkg->NextPackage); +} + /*}}}*/ +// GrpIterator::operator++ - Prefix incr /*{{{*/ +// --------------------------------------------------------------------- +/* This will advance to the next logical group in the hash table. */ +pkgCache::GrpIterator& pkgCache::GrpIterator::operator++() +{ + // Follow the current links + if (S != Owner->GrpP) + S = Owner->GrpP + S->Next; + + // Follow the hash table + while (S == Owner->GrpP && (HashIndex+1) < (signed)Owner->HeaderP->GetHashTableSize()) + { + ++HashIndex; + S = Owner->GrpP + Owner->HeaderP->GrpHashTableP()[HashIndex]; + } + return *this; +} + /*}}}*/ +// PkgIterator::operator++ - Prefix incr /*{{{*/ +// --------------------------------------------------------------------- +/* This will advance to the next logical package in the hash table. */ +pkgCache::PkgIterator& pkgCache::PkgIterator::operator++() +{ + // Follow the current links + if (S != Owner->PkgP) + S = Owner->PkgP + S->NextPackage; + + // Follow the hash table + while (S == Owner->PkgP && (HashIndex+1) < (signed)Owner->HeaderP->GetHashTableSize()) + { + ++HashIndex; + S = Owner->PkgP + Owner->HeaderP->PkgHashTableP()[HashIndex]; + } + return *this; +} + /*}}}*/ +pkgCache::DepIterator& pkgCache::DepIterator::operator++() /*{{{*/ +{ + if (S == Owner->DepP) + return *this; + S = Owner->DepP + (Type == DepVer ? S->NextDepends : S->NextRevDepends); + if (S == Owner->DepP) + S2 = Owner->DepDataP; + else + S2 = Owner->DepDataP + S->DependencyData; + return *this; +} + /*}}}*/ +// PkgIterator::State - Check the State of the package /*{{{*/ +// --------------------------------------------------------------------- +/* By this we mean if it is either cleanly installed or cleanly removed. */ +pkgCache::PkgIterator::OkState pkgCache::PkgIterator::State() const +{ + if (S->InstState == pkgCache::State::ReInstReq || + S->InstState == pkgCache::State::HoldReInstReq) + return NeedsUnpack; + + if (S->CurrentState == pkgCache::State::UnPacked || + S->CurrentState == pkgCache::State::HalfConfigured) + // we leave triggers alone completely. dpkg deals with + // them in a hard-to-predict manner and if they get + // resolved by dpkg before apt run dpkg --configure on + // the TriggersPending package dpkg returns a error + //Pkg->CurrentState == pkgCache::State::TriggersAwaited + //Pkg->CurrentState == pkgCache::State::TriggersPending) + return NeedsConfigure; + + if (S->CurrentState == pkgCache::State::HalfInstalled || + S->InstState != pkgCache::State::Ok) + return NeedsUnpack; + + return NeedsNothing; +} + /*}}}*/ +// PkgIterator::CurVersion - Returns the current version string /*{{{*/ +// --------------------------------------------------------------------- +/* Return string representing of the current version. */ +const char * +pkgCache::PkgIterator::CurVersion() const +{ + VerIterator version = CurrentVer(); + if (version.IsGood()) + return CurrentVer().VerStr(); + return 0; +} + /*}}}*/ +// ostream operator to handle string representation of a package /*{{{*/ +// --------------------------------------------------------------------- +/* Output name < cur.rent.version -> candid.ate.version | new.est.version > (section) + Note that the characters <|>() are all literal above. Versions will be omitted + if they provide no new information (e.g. there is no newer version than candidate) + If no version and/or section can be found "none" is used. */ +std::ostream& +operator<<(std::ostream& out, pkgCache::PkgIterator Pkg) +{ + if (Pkg.end() == true) + return out << "invalid package"; + + string current = string(Pkg.CurVersion() == 0 ? "none" : Pkg.CurVersion()); + string newest = string(Pkg.VersionList().end() ? "none" : Pkg.VersionList().VerStr()); + + out << Pkg.Name() << " [ " << Pkg.Arch() << " ] < " << current; + if ( newest != "none") + out << " | " << newest; + if (Pkg->VersionList == 0) + out << " > ( none )"; + else + out << " > ( " << string(Pkg.VersionList().Section()==0?"unknown":Pkg.VersionList().Section()) << " )"; + return out; +} + /*}}}*/ +// PkgIterator::FullName - Returns Name and (maybe) Architecture /*{{{*/ +// --------------------------------------------------------------------- +/* Returns a name:arch string */ +std::string pkgCache::PkgIterator::FullName(bool const &Pretty) const +{ + string fullname = Name(); + if (Pretty == false || + (strcmp(Arch(), "all") != 0 && strcmp(Arch(), "any") != 0 && + strcmp(Owner->NativeArch(), Arch()) != 0)) + return fullname.append(":").append(Arch()); + return fullname; +} + /*}}}*/ +// DepIterator::IsCritical - Returns true if the dep is important /*{{{*/ +// --------------------------------------------------------------------- +/* Currently critical deps are defined as depends, predepends and + conflicts (including dpkg's Breaks fields). */ +bool pkgCache::DepIterator::IsCritical() const +{ + if (IsNegative() == true || + S2->Type == pkgCache::Dep::Depends || + S2->Type == pkgCache::Dep::PreDepends) + return true; + return false; +} + /*}}}*/ +// DepIterator::IsNegative - Returns true if the dep is a negative one /*{{{*/ +// --------------------------------------------------------------------- +/* Some dependencies are positive like Depends and Recommends, others + are negative like Conflicts which can and should be handled differently */ +bool pkgCache::DepIterator::IsNegative() const +{ + return S2->Type == Dep::DpkgBreaks || + S2->Type == Dep::Conflicts || + S2->Type == Dep::Obsoletes; +} + /*}}}*/ +// DepIterator::SmartTargetPkg - Resolve dep target pointers w/provides /*{{{*/ +// --------------------------------------------------------------------- +/* This intellegently looks at dep target packages and tries to figure + out which package should be used. This is needed to nicely handle + provide mapping. If the target package has no other providing packages + then it returned. Otherwise the providing list is looked at to + see if there is one unique providing package if so it is returned. + Otherwise true is returned and the target package is set. The return + result indicates whether the node should be expandable + + In Conjunction with the DepCache the value of Result may not be + super-good since the policy may have made it uninstallable. Using + AllTargets is better in this case. */ +bool pkgCache::DepIterator::SmartTargetPkg(PkgIterator &Result) const +{ + Result = TargetPkg(); + + // No provides at all + if (Result->ProvidesList == 0) + return false; + + // There is the Base package and the providing ones which is at least 2 + if (Result->VersionList != 0) + return true; + + /* We have to skip over indirect provisions of the package that + owns the dependency. For instance, if libc5-dev depends on the + virtual package libc-dev which is provided by libc5-dev and libc6-dev + we must ignore libc5-dev when considering the provides list. */ + PrvIterator PStart = Result.ProvidesList(); + for (; PStart.end() != true && PStart.OwnerPkg() == ParentPkg(); ++PStart); + + // Nothing but indirect self provides + if (PStart.end() == true) + return false; + + // Check for single packages in the provides list + PrvIterator P = PStart; + for (; P.end() != true; ++P) + { + // Skip over self provides + if (P.OwnerPkg() == ParentPkg()) + continue; + if (PStart.OwnerPkg() != P.OwnerPkg()) + break; + } + + Result = PStart.OwnerPkg(); + + // Check for non dups + if (P.end() != true) + return true; + + return false; +} + /*}}}*/ +// DepIterator::AllTargets - Returns the set of all possible targets /*{{{*/ +// --------------------------------------------------------------------- +/* This is a more useful version of TargetPkg() that follows versioned + provides. It includes every possible package-version that could satisfy + the dependency. The last item in the list has a 0. The resulting pointer + must be delete [] 'd */ +pkgCache::Version **pkgCache::DepIterator::AllTargets() const +{ + Version **Res = 0; + unsigned long Size =0; + while (1) + { + Version **End = Res; + PkgIterator DPkg = TargetPkg(); + + // Walk along the actual package providing versions + for (VerIterator I = DPkg.VersionList(); I.end() == false; ++I) + { + if (IsIgnorable(I.ParentPkg()) == true) + continue; + if (IsSatisfied(I) == false) + continue; + + Size++; + if (Res != 0) + *End++ = I; + } + + // Follow all provides + for (PrvIterator I = DPkg.ProvidesList(); I.end() == false; ++I) + { + if (IsIgnorable(I) == true) + continue; + if (IsSatisfied(I) == false) + continue; + + Size++; + if (Res != 0) + *End++ = I.OwnerVer(); + } + + // Do it again and write it into the array + if (Res == 0) + { + Res = new Version *[Size+1]; + Size = 0; + } + else + { + *End = 0; + break; + } + } + + return Res; +} + /*}}}*/ +// DepIterator::GlobOr - Compute an OR group /*{{{*/ +// --------------------------------------------------------------------- +/* This Takes an iterator, iterates past the current dependency grouping + and returns Start and End so that so End is the final element + in the group, if End == Start then D is End++ and End is the + dependency D was pointing to. Use in loops to iterate sensibly. */ +void pkgCache::DepIterator::GlobOr(DepIterator &Start,DepIterator &End) +{ + // Compute a single dependency element (glob or) + Start = *this; + End = *this; + for (bool LastOR = true; end() == false && LastOR == true;) + { + LastOR = (S2->CompareOp & pkgCache::Dep::Or) == pkgCache::Dep::Or; + ++(*this); + if (LastOR == true) + End = (*this); + } +} + /*}}}*/ +// DepIterator::IsIgnorable - should this packag/providr be ignored? /*{{{*/ +// --------------------------------------------------------------------- +/* Deps like self-conflicts should be ignored as well as implicit conflicts + on virtual packages. */ +bool pkgCache::DepIterator::IsIgnorable(PkgIterator const &PT) const +{ + if (IsNegative() == false) + return false; + + pkgCache::PkgIterator const PP = ParentPkg(); + if (PP->Group != PT->Group) + return false; + // self-conflict + if (PP == PT) + return true; + pkgCache::VerIterator const PV = ParentVer(); + // ignore group-conflict on a M-A:same package - but not our implicit dependencies + // so that we can have M-A:same packages conflicting with their own real name + if ((PV->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same) + return IsMultiArchImplicit() == false; + + return false; +} +bool pkgCache::DepIterator::IsIgnorable(PrvIterator const &Prv) const +{ + if (IsNegative() == false) + return false; + + PkgIterator const Pkg = ParentPkg(); + /* Provides may never be applied against the same package (or group) + if it is a conflicts. See the comment above. */ + if (Prv.OwnerPkg()->Group == Pkg->Group) + return true; + // Implicit group-conflicts should not be applied on providers of other groups + if (IsMultiArchImplicit() && Prv.OwnerPkg()->Group != Pkg->Group) + return true; + + return false; +} + /*}}}*/ +// DepIterator::IsSatisfied - check if a version satisfied the dependency /*{{{*/ +bool pkgCache::DepIterator::IsSatisfied(VerIterator const &Ver) const +{ + return Owner->VS->CheckDep(Ver.VerStr(),S2->CompareOp,TargetVer()); +} +bool pkgCache::DepIterator::IsSatisfied(PrvIterator const &Prv) const +{ + return Owner->VS->CheckDep(Prv.ProvideVersion(),S2->CompareOp,TargetVer()); +} + /*}}}*/ +// DepIterator::IsImplicit - added by the cache generation /*{{{*/ +bool pkgCache::DepIterator::IsImplicit() const +{ + if (IsMultiArchImplicit() == true) + return true; + if (IsNegative() || S2->Type == pkgCache::Dep::Replaces) + { + if ((S2->CompareOp & pkgCache::Dep::ArchSpecific) != pkgCache::Dep::ArchSpecific && + strcmp(ParentPkg().Arch(), TargetPkg().Arch()) != 0) + return true; + } + return false; +} + /*}}}*/ +// ostream operator to handle string representation of a dependency /*{{{*/ +// --------------------------------------------------------------------- +/* */ +std::ostream& operator<<(std::ostream& out, pkgCache::DepIterator D) +{ + if (D.end() == true) + return out << "invalid dependency"; + + pkgCache::PkgIterator P = D.ParentPkg(); + pkgCache::PkgIterator T = D.TargetPkg(); + + out << (P.end() ? "invalid pkg" : P.FullName(false)) << " " << D.DepType() + << " on "; +APT_IGNORE_DEPRECATED_PUSH + if (T.end() == true) + out << "invalid pkg"; + else + out << T; +APT_IGNORE_DEPRECATED_POP + + if (D->Version != 0) + out << " (" << D.CompType() << " " << D.TargetVer() << ")"; + + return out; +} + /*}}}*/ +// VerIterator::CompareVer - Fast version compare for same pkgs /*{{{*/ +// --------------------------------------------------------------------- +/* This just looks over the version list to see if B is listed before A. In + most cases this will return in under 4 checks, ver lists are short. */ +int pkgCache::VerIterator::CompareVer(const VerIterator &B) const +{ + // Check if they are equal + if (*this == B) + return 0; + if (end() == true) + return -1; + if (B.end() == true) + return 1; + + /* Start at A and look for B. If B is found then A > B otherwise + B was before A so A < B */ + VerIterator I = *this; + for (;I.end() == false; ++I) + if (I == B) + return 1; + return -1; +} + /*}}}*/ +// VerIterator::Downloadable - Checks if the version is downloadable /*{{{*/ +// --------------------------------------------------------------------- +/* */ +APT_PURE bool pkgCache::VerIterator::Downloadable() const +{ + VerFileIterator Files = FileList(); + for (; Files.end() == false; ++Files) + if (Files.File().Flagged(pkgCache::Flag::NotSource) == false) + return true; + return false; +} + /*}}}*/ +// VerIterator::Automatic - Check if this version is 'automatic' /*{{{*/ +// --------------------------------------------------------------------- +/* This checks to see if any of the versions files are not NotAutomatic. + True if this version is selectable for automatic installation. */ +APT_PURE bool pkgCache::VerIterator::Automatic() const +{ + VerFileIterator Files = FileList(); + for (; Files.end() == false; ++Files) + // Do not check ButAutomaticUpgrades here as it is kind of automatic… + if (Files.File().Flagged(pkgCache::Flag::NotAutomatic) == false) + return true; + return false; +} + /*}}}*/ +// VerIterator::NewestFile - Return the newest file version relation /*{{{*/ +// --------------------------------------------------------------------- +/* This looks at the version numbers associated with all of the sources + this version is in and returns the highest.*/ +pkgCache::VerFileIterator pkgCache::VerIterator::NewestFile() const +{ + VerFileIterator Files = FileList(); + VerFileIterator Highest = Files; + for (; Files.end() == false; ++Files) + { + if (Owner->VS->CmpReleaseVer(Files.File().Version(),Highest.File().Version()) > 0) + Highest = Files; + } + + return Highest; +} + /*}}}*/ +// VerIterator::RelStr - Release description string /*{{{*/ +// --------------------------------------------------------------------- +/* This describes the version from a release-centric manner. The output is a + list of Label:Version/Archive */ +static std::string PkgFileIteratorToRelString(pkgCache::PkgFileIterator const &File) +{ + std::string Res; + if (File.Label() != 0) + Res = Res + File.Label() + ':'; + + if (File.Archive() != 0) + { + if (File.Version() == 0) + Res += File.Archive(); + else + Res = Res + File.Version() + '/' + File.Archive(); + } + else + { + // No release file, print the host name that this came from + if (File.Site() == 0 || File.Site()[0] == 0) + Res += "localhost"; + else + Res += File.Site(); + } + return Res; +} +string pkgCache::VerIterator::RelStr() const +{ + std::vector<std::string> RelStrs; + for (pkgCache::VerFileIterator I = this->FileList(); I.end() == false; ++I) + { + // Do not print 'not source' entries' + pkgCache::PkgFileIterator const File = I.File(); + if (File.Flagged(pkgCache::Flag::NotSource)) + continue; + + std::string const RS = PkgFileIteratorToRelString(File); + if (std::find(RelStrs.begin(), RelStrs.end(), RS) != RelStrs.end()) + continue; + + RelStrs.push_back(RS); + } + std::ostringstream os; + if (likely(RelStrs.empty() == false)) + { + std::copy(RelStrs.begin(), RelStrs.end()-1, std::ostream_iterator<std::string>(os, ", ")); + os << *RelStrs.rbegin(); + } + if (S->ParentPkg != 0) + os << " [" << Arch() << "]"; + return os.str(); +} + /*}}}*/ +// VerIterator::MultiArchType - string representing MultiArch flag /*{{{*/ +const char * pkgCache::VerIterator::MultiArchType() const +{ + if ((S->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same) + return "same"; + else if ((S->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign) + return "foreign"; + else if ((S->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed) + return "allowed"; + return "none"; +} + /*}}}*/ +// RlsFileIterator::RelStr - Return the release string /*{{{*/ +string pkgCache::RlsFileIterator::RelStr() +{ + string Res; + if (Version() != 0) + Res = Res + (Res.empty() == true?"v=":",v=") + Version(); + if (Origin() != 0) + Res = Res + (Res.empty() == true?"o=":",o=") + Origin(); + if (Archive() != 0) + Res = Res + (Res.empty() == true?"a=":",a=") + Archive(); + if (Codename() != 0) + Res = Res + (Res.empty() == true?"n=":",n=") + Codename(); + if (Label() != 0) + Res = Res + (Res.empty() == true?"l=":",l=") + Label(); + return Res; +} + /*}}}*/ +string pkgCache::PkgFileIterator::RelStr() /*{{{*/ +{ + std::string Res; + if (ReleaseFile() == 0) + { + if (Component() != 0) + Res = Res + (Res.empty() == true?"a=":",a=") + Component(); + } + else + { + Res = ReleaseFile().RelStr(); + if (Component() != 0) + Res = Res + (Res.empty() == true?"c=":",c=") + Component(); + } + if (Architecture() != 0) + Res = Res + (Res.empty() == true?"b=":",b=") + Architecture(); + return Res; +} + /*}}}*/ +// VerIterator::TranslatedDescriptionForLanguage - Return a DescIter for language/*{{{*/ +// --------------------------------------------------------------------- +/* return a DescIter for the specified language + */ +pkgCache::DescIterator pkgCache::VerIterator::TranslatedDescriptionForLanguage(StringView lang) const +{ + for (pkgCache::DescIterator Desc = DescriptionList(); Desc.end() == false; ++Desc) + if (lang == Desc.LanguageCode()) + return Desc; + + if (lang == "en") + return TranslatedDescriptionForLanguage(""); + + return DescIterator(); +} + + /*}}}*/ +// VerIterator::TranslatedDescription - Return the a DescIter for locale/*{{{*/ +// --------------------------------------------------------------------- +/* return a DescIter for the current locale or the default if none is + * found + */ +pkgCache::DescIterator pkgCache::VerIterator::TranslatedDescription() const +{ + std::vector<string> const lang = APT::Configuration::getLanguages(); + for (std::vector<string>::const_iterator l = lang.begin(); + l != lang.end(); ++l) + { + pkgCache::DescIterator Desc = TranslatedDescriptionForLanguage(*l); + if (Desc.IsGood()) + return Desc; + } + + pkgCache::DescIterator Desc = TranslatedDescriptionForLanguage(""); + if (Desc.IsGood()) + return Desc; + + return DescriptionList(); +} + + /*}}}*/ + +pkgCache::~pkgCache() {} diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h new file mode 100644 index 0000000..55baa3c --- /dev/null +++ b/apt-pkg/pkgcache.h @@ -0,0 +1,846 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/**\file pkgcache.h + \brief pkgCache - Structure definitions for the cache file + + The goal of the cache file is two fold: + Firstly to speed loading and processing of the package file array and + secondly to reduce memory consumption of the package file array. + + The implementation is aimed at an environment with many primary package + files, for instance someone that has a Package file for their CD-ROM, a + Package file for the latest version of the distribution on the CD-ROM and a + package file for the development version. Always present is the information + contained in the status file which might be considered a separate package + file. + + Please understand, this is designed as a <b>Cache file</b> it is not meant to be + used on any system other than the one it was created for. It is not meant to + be authoritative either, i.e. if a system crash or software failure occurs it + must be perfectly acceptable for the cache file to be in an inconsistent + state. Furthermore at any time the cache file may be erased without losing + any information. + + Also the structures and storage layout is optimized for use by the APT + and may not be suitable for all purposes. However it should be possible + to extend it with associate cache files that contain other information. + + To keep memory use down the cache file only contains often used fields and + fields that are inexpensive to store, the Package file has a full list of + fields. Also the client may assume that all items are perfectly valid and + need not perform checks against their correctness. Removal of information + from the cache is possible, but blanks will be left in the file, and + unused strings will also be present. The recommended implementation is to + simply rebuild the cache each time any of the data files change. It is + possible to add a new package file to the cache without any negative side + effects. + + <b>Note on Pointer access</b> + Clients should always use the CacheIterators classes for access to the + cache and the data in it. They also provide a simple STL-like method for + traversing the links of the datastructure. + + Every item in every structure is stored as the index to that structure. + What this means is that once the files is mmaped every data access has to + go through a fix up stage to get a real memory pointer. This is done + by taking the index, multiplying it by the type size and then adding + it to the start address of the memory block. This sounds complex, but + in C it is a single array dereference. Because all items are aligned to + their size and indexes are stored as multiples of the size of the structure + the format is immediately portable to all possible architectures - BUT the + generated files are -NOT-. + + This scheme allows code like this to be written: + <example> + void *Map = mmap(...); + Package *PkgList = (Package *)Map; + Header *Head = (Header *)Map; + char *Strings = (char *)Map; + cout << (Strings + PkgList[Head->HashTable[0]]->Name) << endl; + </example> + Notice the lack of casting or multiplication. The net result is to return + the name of the first package in the first hash bucket, without error + checks. + + The generator uses allocation pools to group similarly sized structures in + large blocks to eliminate any alignment overhead. The generator also + assures that no structures overlap and all indexes are unique. Although + at first glance it may seem like there is the potential for two structures + to exist at the same point the generator never allows this to happen. + (See the discussion of free space pools) + + See \ref pkgcachegen.h for more information about generating cache structures. */ + /*}}}*/ +#ifndef PKGLIB_PKGCACHE_H +#define PKGLIB_PKGCACHE_H +#define __PKGLIB_IN_PKGCACHE_H +#include <apt-pkg/macros.h> +#include <apt-pkg/mmap.h> + +#include <cstddef> // required for nullptr_t +#include <string> +#include <stdint.h> +#include <time.h> + +#include <apt-pkg/string_view.h> + + +// size of (potentially big) files like debs or the install size of them +typedef uint64_t map_filesize_t; +// storing file sizes of indexes, which are way below 4 GB for now +typedef uint32_t map_filesize_small_t; +// each package/group/dependency gets an id +typedef uint32_t map_id_t; +// some files get an id, too, but in far less absolute numbers +typedef uint16_t map_fileid_t; + +// relative pointer from cache start +template <typename T> class map_pointer { + uint32_t val; +public: + map_pointer() noexcept : val(0) {} + map_pointer(std::nullptr_t) noexcept : val(0) {} + explicit map_pointer(uint32_t n) noexcept : val(n) {} + explicit operator uint32_t() noexcept { return val; } + explicit operator bool() noexcept { return val != 0; } +}; + +template<typename T> inline T *operator +(T *p, map_pointer<T> m) { return p + uint32_t(m); } +template<typename T> inline bool operator ==(map_pointer<T> u, map_pointer<T> m) { return uint32_t(u) == uint32_t(m); } +template<typename T> inline bool operator !=(map_pointer<T> u, map_pointer<T> m) { return uint32_t(u) != uint32_t(m); } +template<typename T> inline bool operator <(map_pointer<T> u, map_pointer<T> m) { return uint32_t(u) < uint32_t(m); } +template<typename T> inline bool operator >(map_pointer<T> u, map_pointer<T> m) { return uint32_t(u) > uint32_t(m); } +template<typename T> inline uint32_t operator -(map_pointer<T> u, map_pointer<T> m) { return uint32_t(u) - uint32_t(m); } +template<typename T> bool operator ==(map_pointer<T> m, std::nullptr_t) { return uint32_t(m) == 0; } +template<typename T> bool operator !=(map_pointer<T> m, std::nullptr_t) { return uint32_t(m) != 0; } + +// same as the previous, but documented to be to a string item +typedef map_pointer<char> map_stringitem_t; + +// we have only a small amount of flags for each item +typedef uint8_t map_flags_t; +typedef uint8_t map_number_t; + +class pkgVersioningSystem; +class APT_PUBLIC pkgCache /*{{{*/ +{ + public: + // Cache element predeclarations + struct Header; + struct Group; + struct Package; + struct ReleaseFile; + struct PackageFile; + struct Version; + struct Description; + struct Provides; + struct Dependency; + struct DependencyData; + struct StringItem; + struct VerFile; + struct DescFile; + + // Iterators + template<typename Str, typename Itr> class Iterator; + class GrpIterator; + class PkgIterator; + class VerIterator; + class DescIterator; + class DepIterator; + class PrvIterator; + class RlsFileIterator; + class PkgFileIterator; + class VerFileIterator; + class DescFileIterator; + + class Namespace; + + // These are all the constants used in the cache structures + + // WARNING - if you change these lists you must also edit + // the stringification in pkgcache.cc and also consider whether + // the cache file will become incompatible. + struct Dep + { + enum DepType {Depends=1,PreDepends=2,Suggests=3,Recommends=4, + Conflicts=5,Replaces=6,Obsoletes=7,DpkgBreaks=8,Enhances=9}; + /** \brief available compare operators + + The lower 4 bits are used to indicate what operator is being specified and + the upper 4 bits are flags. OR indicates that the next package is + or'd with the current package. */ + enum DepCompareOp {NoOp=0,LessEq=0x1,GreaterEq=0x2,Less=0x3, + Greater=0x4,Equals=0x5,NotEquals=0x6, + Or=0x10, /*!< or'ed with the next dependency */ + MultiArchImplicit=0x20, /*!< generated internally, not spelled out in the index */ + ArchSpecific=0x40 /*!< was decorated with an explicit architecture in index */ + }; + }; + + struct State + { + /** \brief priority of a package version + + Zero is used for unparsable or absent Priority fields. */ + enum VerPriority {Required=1,Important=2,Standard=3,Optional=4,Extra=5}; + enum PkgSelectedState {Unknown=0,Install=1,Hold=2,DeInstall=3,Purge=4}; + enum PkgInstState {Ok=0,ReInstReq=1,HoldInst=2,HoldReInstReq=3}; + enum PkgCurrentState {NotInstalled=0,UnPacked=1,HalfConfigured=2, + HalfInstalled=4,ConfigFiles=5,Installed=6, + TriggersAwaited=7,TriggersPending=8}; + }; + + struct Flag + { + enum PkgFlags {Auto=(1<<0),Essential=(1<<3),Important=(1<<4)}; + enum PkgFFlags { + NotSource=(1<<0), /*!< packages can't be fetched from here, e.g. dpkg/status file */ + LocalSource=(1<<1), /*!< local sources can't and will not be verified by hashes */ + NoPackages=(1<<2), /*!< the file includes no package records itself, but additions like Translations */ + }; + enum ReleaseFileFlags { + NotAutomatic=(1<<0), /*!< archive has a default pin of 1 */ + ButAutomaticUpgrades=(1<<1), /*!< (together with the previous) archive has a default pin of 100 */ + }; + enum ProvidesFlags { + MultiArchImplicit=pkgCache::Dep::MultiArchImplicit, /*!< generated internally, not spelled out in the index */ + ArchSpecific=pkgCache::Dep::ArchSpecific /*!< was decorated with an explicit architecture in index */ + }; + }; + + protected: + + // Memory mapped cache file + std::string CacheFile; + MMap ⤅ + map_id_t sHash(APT::StringView S) const APT_PURE; + + public: + + // Pointers to the arrays of items + Header *HeaderP; + Group *GrpP; + Package *PkgP; + VerFile *VerFileP; + DescFile *DescFileP; + ReleaseFile *RlsFileP; + PackageFile *PkgFileP; + Version *VerP; + Description *DescP; + Provides *ProvideP; + Dependency *DepP; + DependencyData *DepDataP; + char *StrP; + void *reserved[12]; + + virtual bool ReMap(bool const &Errorchecks = true); + inline bool Sync() {return Map.Sync();} + inline MMap &GetMap() {return Map;} + inline void *DataEnd() {return ((unsigned char *)Map.Data()) + Map.Size();} + + // String hashing function (512 range) + inline map_id_t Hash(APT::StringView S) const {return sHash(S);} + + APT_HIDDEN uint32_t CacheHash(); + + // Useful transformation things + static const char *Priority(unsigned char Priority); + + // Accessors + GrpIterator FindGrp(APT::StringView Name); + PkgIterator FindPkg(APT::StringView Name); + PkgIterator FindPkg(APT::StringView Name, APT::StringView Arch); + + APT::StringView ViewString(map_stringitem_t idx) const + { + char *name = StrP + idx; + uint16_t len = *reinterpret_cast<const uint16_t*>(name - sizeof(uint16_t)); + return APT::StringView(name, len); + } + + Header &Head() {return *HeaderP;} + inline GrpIterator GrpBegin(); + inline GrpIterator GrpEnd(); + inline PkgIterator PkgBegin(); + inline PkgIterator PkgEnd(); + inline PkgFileIterator FileBegin(); + inline PkgFileIterator FileEnd(); + inline RlsFileIterator RlsFileBegin(); + inline RlsFileIterator RlsFileEnd(); + + inline bool MultiArchCache() const { return MultiArchEnabled; } + inline char const * NativeArch(); + + // Make me a function + pkgVersioningSystem *VS; + + // Converters + static const char *CompTypeDeb(unsigned char Comp) APT_PURE; + static const char *CompType(unsigned char Comp) APT_PURE; + static const char *DepType(unsigned char Dep); + + pkgCache(MMap *Map,bool DoMap = true); + virtual ~pkgCache(); + +private: + void * const d; + bool MultiArchEnabled; +}; + /*}}}*/ +// Header structure /*{{{*/ +struct pkgCache::Header +{ + /** \brief Signature information + + This must contain the hex value 0x98FE76DC which is designed to + verify that the system loading the image has the same byte order + and byte size as the system saving the image */ + uint32_t Signature; + /** These contain the version of the cache file */ + map_number_t MajorVersion; + map_number_t MinorVersion; + /** \brief indicates if the cache should be erased + + Dirty is true if the cache file was opened for reading, the client + expects to have written things to it and have not fully synced it. + The file should be erased and rebuilt if it is true. */ + bool Dirty; + + /** \brief Size of structure values + + All *Sz variables contains the sizeof() that particular structure. + It is used as an extra consistency check on the structure of the file. + + If any of the size values do not exactly match what the client expects + then the client should refuse the load the file. */ + uint16_t HeaderSz; + map_number_t GroupSz; + map_number_t PackageSz; + map_number_t ReleaseFileSz; + map_number_t PackageFileSz; + map_number_t VersionSz; + map_number_t DescriptionSz; + map_number_t DependencySz; + map_number_t DependencyDataSz; + map_number_t ProvidesSz; + map_number_t VerFileSz; + map_number_t DescFileSz; + + /** \brief Structure counts + + These indicate the number of each structure contained in the cache. + PackageCount is especially useful for generating user state structures. + See Package::Id for more info. */ + map_id_t GroupCount; + map_id_t PackageCount; + map_id_t VersionCount; + map_id_t DescriptionCount; + map_id_t DependsCount; + map_id_t DependsDataCount; + map_fileid_t ReleaseFileCount; + map_fileid_t PackageFileCount; + map_fileid_t VerFileCount; + map_fileid_t DescFileCount; + map_id_t ProvidesCount; + + /** \brief index of the first PackageFile structure + + The PackageFile structures are singly linked lists that represent + all package files that have been merged into the cache. */ + map_pointer<PackageFile> FileList; + /** \brief index of the first ReleaseFile structure */ + map_pointer<ReleaseFile> RlsFileList; + + /** \brief String representing the version system used */ + map_stringitem_t VerSysName; + /** \brief native architecture the cache was built against */ + map_stringitem_t Architecture; + /** \brief all architectures the cache was built against */ + map_stringitem_t Architectures; + /** \brief The maximum size of a raw entry from the original Package file */ + map_filesize_t MaxVerFileSize; + /** \brief The maximum size of a raw entry from the original Translation file */ + map_filesize_t MaxDescFileSize; + + /** \brief The Pool structures manage the allocation pools that the generator uses + + Start indicates the first byte of the pool, Count is the number of objects + remaining in the pool and ItemSize is the structure size (alignment factor) + of the pool. An ItemSize of 0 indicates the pool is empty. There should be + twice the number of pools as there are non-private structure types. The generator + stores this information so future additions can make use of any unused pool + blocks. */ + DynamicMMap::Pool Pools[2 * 12]; + + /** \brief hash tables providing rapid group/package name lookup + + Each group/package name is inserted into a hash table using pkgCache::Hash(const &string) + By iterating over each entry in the hash table it is possible to iterate over + the entire list of packages. Hash Collisions are handled with a singly linked + list of packages based at the hash item. The linked list contains only + packages that match the hashing function. + In the PkgHashTable is it possible that multiple packages have the same name - + these packages are stored as a sequence in the list. + The size of both tables is the same. */ + uint32_t HashTableSize; + uint32_t GetHashTableSize() const { return HashTableSize; } + void SetHashTableSize(unsigned int const sz) { HashTableSize = sz; } + map_stringitem_t GetArchitectures() const { return Architectures; } + void SetArchitectures(map_stringitem_t const idx) { Architectures = idx; } + +#ifdef APT_COMPILING_APT + map_pointer<Group> * GrpHashTableP() const { return (map_pointer<Group>*) (this + 1); } + map_pointer<Package> * PkgHashTableP() const { return reinterpret_cast<map_pointer<Package> *>(GrpHashTableP() + GetHashTableSize()); } +#endif + + /** \brief Hash of the file (TODO: Rename) */ + map_filesize_small_t CacheFileSize; + + bool CheckSizes(Header &Against) const APT_PURE; + Header(); +}; + /*}}}*/ +// Group structure /*{{{*/ +/** \brief groups architecture depending packages together + + On or more packages with the same name form a group, so we have + a simple way to access a package built for different architectures + Group exists in a singly linked list of group records starting at + the hash index of the name in the pkgCache::Header::GrpHashTable + + They also act as a representation of source packages, allowing you to + iterate over all binaries produced by a source package. + */ +struct pkgCache::Group +{ + /** \brief Name of the group */ + map_stringitem_t Name; + + // Linked List + /** \brief Link to the first package which belongs to the group */ + map_pointer<Package> FirstPackage; + /** \brief Link to the last package which belongs to the group */ + map_pointer<Package> LastPackage; + + /** \brief Link to the next Group */ + map_pointer<Group> Next; + /** \brief unique sequel ID */ + map_id_t ID; + + /** \brief List of binary produces by source package with this name. */ + map_pointer<Version> VersionsInSource; + + /** \brief Private pointer */ + map_pointer<void> d; +}; + /*}}}*/ +// Package structure /*{{{*/ +/** \brief contains information for a single unique package + + There can be any number of versions of a given package. + Package exists in a singly linked list of package records starting at + the hash index of the name in the pkgCache::Header::PkgHashTable + + A package can be created for every architecture so package names are + not unique, but it is guaranteed that packages with the same name + are sequencel ordered in the list. Packages with the same name can be + accessed with the Group. +*/ +struct pkgCache::Package +{ + /** \brief Architecture of the package */ + map_stringitem_t Arch; + /** \brief Base of a singly linked list of versions + + Each structure represents a unique version of the package. + The version structures contain links into PackageFile and the + original text file as well as detailed information about the size + and dependencies of the specific package. In this way multiple + versions of a package can be cleanly handled by the system. + Furthermore, this linked list is guaranteed to be sorted + from Highest version to lowest version with no duplicate entries. */ + map_pointer<Version> VersionList; + /** \brief index to the installed version */ + map_pointer<Version> CurrentVer; + /** \brief index of the group this package belongs to */ + map_pointer<pkgCache::Group> Group; + + // Linked list + /** \brief Link to the next package in the same bucket */ + map_pointer<Package> NextPackage; + /** \brief List of all dependencies on this package */ + map_pointer<Dependency> RevDepends; + /** \brief List of all "packages" this package provide */ + map_pointer<Provides> ProvidesList; + + // Install/Remove/Purge etc + /** \brief state that the user wishes the package to be in */ + map_number_t SelectedState; // What + /** \brief installation state of the package + + This should be "ok" but in case the installation failed + it will be different. + */ + map_number_t InstState; // Flags + /** \brief indicates if the package is installed */ + map_number_t CurrentState; // State + + /** \brief unique sequel ID + + ID is a unique value from 0 to Header->PackageCount assigned by the generator. + This allows clients to create an array of size PackageCount and use it to store + state information for the package map. For instance the status file emitter uses + this to track which packages have been emitted already. */ + map_id_t ID; + /** \brief some useful indicators of the package's state */ + map_flags_t Flags; + + /** \brief Private pointer */ + map_pointer<void> d; +}; + /*}}}*/ +// Release File structure /*{{{*/ +/** \brief stores information about the release files used to generate the cache + + PackageFiles reference ReleaseFiles as we need to keep record of which + version belongs to which release e.g. for pinning. */ +struct pkgCache::ReleaseFile +{ + /** \brief physical disk file that this ReleaseFile represents */ + map_stringitem_t FileName; + /** \brief the release information + + Please see the files document for a description of what the + release information means. */ + map_stringitem_t Archive; + map_stringitem_t Codename; + map_stringitem_t Version; + map_stringitem_t Origin; + map_stringitem_t Label; + /** \brief The site the index file was fetched from */ + map_stringitem_t Site; + + /** \brief Size of the file + + Used together with the modification time as a + simple check to ensure that the Packages + file has not been altered since Cache generation. */ + map_filesize_t Size; + /** \brief Modification time for the file */ + time_t mtime; + + /** @TODO document PackageFile::Flags */ + map_flags_t Flags; + + // Linked list + /** \brief Link to the next ReleaseFile in the Cache */ + map_pointer<ReleaseFile> NextFile; + /** \brief unique sequel ID */ + map_fileid_t ID; + + /** \brief Private pointer */ + map_pointer<void> d; +}; + /*}}}*/ +// Package File structure /*{{{*/ +/** \brief stores information about the files used to generate the cache + + Package files are referenced by Version structures to be able to know + after the generation still from which Packages file includes this Version + as we need this information later on e.g. for pinning. */ +struct pkgCache::PackageFile +{ + /** \brief physical disk file that this PackageFile represents */ + map_stringitem_t FileName; + /** \brief the release information */ + map_pointer<ReleaseFile> Release; + + map_stringitem_t Component; + map_stringitem_t Architecture; + + /** \brief indicates what sort of index file this is + + @TODO enumerate at least the possible indexes */ + map_stringitem_t IndexType; + /** \brief Size of the file + + Used together with the modification time as a + simple check to ensure that the Packages + file has not been altered since Cache generation. */ + map_filesize_t Size; + /** \brief Modification time for the file */ + time_t mtime; + + /** @TODO document PackageFile::Flags */ + map_flags_t Flags; + + // Linked list + /** \brief Link to the next PackageFile in the Cache */ + map_pointer<PackageFile> NextFile; + /** \brief unique sequel ID */ + map_fileid_t ID; + + /** \brief Private pointer */ + map_pointer<void> d; +}; + /*}}}*/ +// VerFile structure /*{{{*/ +/** \brief associates a version with a PackageFile + + This allows a full description of all Versions in all files + (and hence all sources) under consideration. */ +struct pkgCache::VerFile +{ + /** \brief index of the package file that this version was found in */ + map_pointer<PackageFile> File; + /** \brief next step in the linked list */ + map_pointer<VerFile> NextFile; + /** \brief position in the package file */ + map_filesize_t Offset; // File offset + /** @TODO document pkgCache::VerFile::Size */ + map_filesize_t Size; +}; + /*}}}*/ +// DescFile structure /*{{{*/ +/** \brief associates a description with a Translation file */ +struct pkgCache::DescFile +{ + /** \brief index of the file that this description was found in */ + map_pointer<PackageFile> File; + /** \brief next step in the linked list */ + map_pointer<DescFile> NextFile; + /** \brief position in the file */ + map_filesize_t Offset; // File offset + /** @TODO document pkgCache::DescFile::Size */ + map_filesize_t Size; +}; + /*}}}*/ +// Version structure /*{{{*/ +/** \brief information for a single version of a package + + The version list is always sorted from highest version to lowest + version by the generator. Equal version numbers are either merged + or handled as separate versions based on the Hash value. */ +struct pkgCache::Version +{ + struct Extra; + + /** \brief complete version string */ + map_stringitem_t VerStr; + /** \brief section this version is filled in */ + map_stringitem_t Section; + /** \brief source package name this version comes from + Always contains the name, even if it is the same as the binary name */ + map_stringitem_t SourcePkgName; + /** \brief source version this version comes from + Always contains the version string, even if it is the same as the binary version */ + map_stringitem_t SourceVerStr; + + /** \brief Multi-Arch capabilities of a package version */ + enum VerMultiArch { No = 0, /*!< is the default and doesn't trigger special behaviour */ + All = (1<<0), /*!< will cause that Ver.Arch() will report "all" */ + Foreign = (1<<1), /*!< can satisfy dependencies in another architecture */ + Same = (1<<2), /*!< can be co-installed with itself from other architectures */ + Allowed = (1<<3), /*!< other packages are allowed to depend on thispkg:any */ + AllForeign = All | Foreign, + AllAllowed = All | Allowed }; + + /** \brief stores the MultiArch capabilities of this version + + Flags used are defined in pkgCache::Version::VerMultiArch + */ + map_number_t MultiArch; + + /** \brief references all the PackageFile's that this version came from + + FileList can be used to determine what distribution(s) the Version + applies to. If FileList is 0 then this is a blank version. + The structure should also have a 0 in all other fields excluding + pkgCache::Version::VerStr and Possibly pkgCache::Version::NextVer. */ + map_pointer<VerFile> FileList; + /** \brief next (lower or equal) version in the linked list */ + map_pointer<Version> NextVer; + /** \brief next description in the linked list */ + map_pointer<Description> DescriptionList; + /** \brief base of the dependency list */ + map_pointer<Dependency> DependsList; + /** \brief links to the owning package + + This allows reverse dependencies to determine the package */ + map_pointer<Package> ParentPkg; + /** \brief list of pkgCache::Provides */ + map_pointer<Provides> ProvidesList; + + /** \brief archive size for this version + + For Debian this is the size of the .deb file. */ + map_filesize_t Size; // These are the .deb size + /** \brief uncompressed size for this version */ + map_filesize_t InstalledSize; + /** \brief characteristic value representing this version + + No two packages in existence should have the same VerStr + and Hash with different contents. */ + uint32_t Hash; + /** \brief unique sequel ID */ + map_id_t ID; + /** \brief parsed priority value */ + map_number_t Priority; + /** \brief next version in the source package (might be different binary) */ + map_pointer<Version> NextInSource; + + /** \brief Private pointer */ + map_pointer<Extra> d; +}; + +#ifdef APT_COMPILING_APT +/// \brief Extra information for packages. APT-internal use only. +struct pkgCache::Version::Extra +{ + uint8_t PhasedUpdatePercentage; +}; +#endif + /*}}}*/ +// Description structure /*{{{*/ +/** \brief datamember of a linked list of available description for a version */ +struct pkgCache::Description +{ + /** \brief Language code of this description (translation) + + If the value has a 0 length then this is read using the Package + file else the Translation-CODE file is used. */ + map_stringitem_t language_code; + /** \brief MD5sum of the original description + + Used to map Translations of a description to a version + and to check that the Translation is up-to-date. */ + map_stringitem_t md5sum; + + /** @TODO document pkgCache::Description::FileList */ + map_pointer<DescFile> FileList; + /** \brief next translation for this description */ + map_pointer<Description> NextDesc; + /** \brief the text is a description of this package */ + map_pointer<Package> ParentPkg; + + /** \brief unique sequel ID */ + map_id_t ID; +}; + /*}}}*/ +// Dependency structure /*{{{*/ +/** \brief information for a single dependency record + + The records are split up like this to ease processing by the client. + The base of the linked list is pkgCache::Version::DependsList. + All forms of dependencies are recorded here including Depends, + Recommends, Suggests, Enhances, Conflicts, Replaces and Breaks. */ +struct pkgCache::DependencyData +{ + /** \brief string of the version the dependency is applied against */ + map_stringitem_t Version; + /** \brief index of the package this depends applies to + + The generator will - if the package does not already exist - + create a blank (no version records) package. */ + map_pointer<pkgCache::Package> Package; + + /** \brief Dependency type - Depends, Recommends, Conflicts, etc */ + map_number_t Type; + /** \brief comparison operator specified on the depends line + + If the high bit is set then it is a logical OR with the previous record. */ + map_flags_t CompareOp; + + map_pointer<DependencyData> NextData; +}; +struct pkgCache::Dependency +{ + map_pointer<pkgCache::DependencyData> DependencyData; + /** \brief version of the package which has the depends */ + map_pointer<Version> ParentVer; + /** \brief next reverse dependency of this package */ + map_pointer<Dependency> NextRevDepends; + /** \brief next dependency of this version */ + map_pointer<Dependency> NextDepends; + + /** \brief unique sequel ID */ + map_id_t ID; +}; + /*}}}*/ +// Provides structure /*{{{*/ +/** \brief handles virtual packages + + When a Provides: line is encountered a new provides record is added + associating the package with a virtual package name. + The provides structures are linked off the package structures. + This simplifies the analysis of dependencies and other aspects A provides + refers to a specific version of a specific package, not all versions need to + provide that provides.*/ +struct pkgCache::Provides +{ + /** \brief index of the package providing this */ + map_pointer<Package> ParentPkg; + /** \brief index of the version this provide line applies to */ + map_pointer<pkgCache::Version> Version; + /** \brief version in the provides line (if any) + + This version allows dependencies to depend on specific versions of a + Provides, as well as allowing Provides to override existing packages. */ + map_stringitem_t ProvideVersion; + map_flags_t Flags; + /** \brief next provides (based of package) */ + map_pointer<Provides> NextProvides; + /** \brief next provides (based of version) */ + map_pointer<Provides> NextPkgProv; +}; + /*}}}*/ + +inline char const * pkgCache::NativeArch() + { return StrP + HeaderP->Architecture; } + +#include <apt-pkg/cacheiterators.h> + + inline pkgCache::GrpIterator pkgCache::GrpBegin() + { + return GrpIterator(*this); + } + inline pkgCache::GrpIterator pkgCache::GrpEnd() + { + return GrpIterator(*this, GrpP);} +inline pkgCache::PkgIterator pkgCache::PkgBegin() + {return PkgIterator(*this);} +inline pkgCache::PkgIterator pkgCache::PkgEnd() + {return PkgIterator(*this,PkgP);} +inline pkgCache::PkgFileIterator pkgCache::FileBegin() + {return PkgFileIterator(*this,PkgFileP + HeaderP->FileList);} +inline pkgCache::PkgFileIterator pkgCache::FileEnd() + {return PkgFileIterator(*this,PkgFileP);} +inline pkgCache::RlsFileIterator pkgCache::RlsFileBegin() + {return RlsFileIterator(*this,RlsFileP + HeaderP->RlsFileList);} +inline pkgCache::RlsFileIterator pkgCache::RlsFileEnd() + {return RlsFileIterator(*this,RlsFileP);} + + +// Oh I wish for Real Name Space Support +class pkgCache::Namespace /*{{{*/ +{ + public: + typedef pkgCache::GrpIterator GrpIterator; + typedef pkgCache::PkgIterator PkgIterator; + typedef pkgCache::VerIterator VerIterator; + typedef pkgCache::DescIterator DescIterator; + typedef pkgCache::DepIterator DepIterator; + typedef pkgCache::PrvIterator PrvIterator; + typedef pkgCache::RlsFileIterator RlsFileIterator; + typedef pkgCache::PkgFileIterator PkgFileIterator; + typedef pkgCache::VerFileIterator VerFileIterator; + typedef pkgCache::Version Version; + typedef pkgCache::Description Description; + typedef pkgCache::Package Package; + typedef pkgCache::Header Header; + typedef pkgCache::Dep Dep; + typedef pkgCache::Flag Flag; +}; + /*}}}*/ +#undef __PKGLIB_IN_PKGCACHE_H +#endif diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc new file mode 100644 index 0000000..807f3bf --- /dev/null +++ b/apt-pkg/pkgcachegen.cc @@ -0,0 +1,1863 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Package Cache Generator - Generator for the cache structure. + + This builds the cache structure from the abstract package list parser. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/indexfile.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/metaindex.h> +#include <apt-pkg/mmap.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgcachegen.h> +#include <apt-pkg/pkgsystem.h> +#include <apt-pkg/progress.h> +#include <apt-pkg/sourcelist.h> +#include <apt-pkg/version.h> + +#include <algorithm> +#include <iostream> +#include <memory> +#include <string> +#include <vector> +#include <stddef.h> +#include <string.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ +constexpr auto APT_CACHE_START_DEFAULT = 24 * 1024 * 1024; + +template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>; +typedef std::vector<pkgIndexFile *>::iterator FileIterator; +template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap; + +static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc, + APT::StringView CurMd5, std::string const &CurLang); + +using std::string; +using APT::StringView; + +// Convert an offset returned from e.g. DynamicMMap or ptr difference to +// an uint32_t location without data loss. +template <typename T> +static inline uint32_t NarrowOffset(T ptr) +{ + uint32_t res = static_cast<uint32_t>(ptr); + if (unlikely(ptr < 0)) + abort(); + if (unlikely(static_cast<T>(res) != ptr)) + abort(); + return res; +} + +// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* We set the dirty flag and make sure that is written to the disk */ +pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : + Map(*pMap), Cache(pMap,false), Progress(Prog), + CurrentRlsFile(nullptr), CurrentFile(nullptr), d(nullptr) +{ +} +bool pkgCacheGenerator::Start() +{ + if (Map.Size() == 0) + { + // Setup the map interface.. + Cache.HeaderP = (pkgCache::Header *)Map.Data(); + _error->PushToStack(); + Map.RawAllocate(sizeof(pkgCache::Header)); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + if (newError) + return false; + if (Map.Size() <= 0) + return false; + + Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0])); + + // Starting header + *Cache.HeaderP = pkgCache::Header(); + + // make room for the hashtables for packages and groups + if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer<void>))) == 0) + return false; + + map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label); + if (unlikely(idxVerSysName == 0)) + return false; + map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture")); + if (unlikely(idxArchitecture == 0)) + return false; + map_stringitem_t idxArchitectures; + + std::vector<std::string> archs = APT::Configuration::getArchitectures(); + if (archs.size() > 1) + { + std::vector<std::string>::const_iterator a = archs.begin(); + std::string list = *a; + for (++a; a != archs.end(); ++a) + list.append(",").append(*a); + idxArchitectures = WriteStringInMap(list); + if (unlikely(idxArchitectures == 0)) + return false; + } + else + idxArchitectures = idxArchitecture; + + Cache.HeaderP = (pkgCache::Header *)Map.Data(); + Cache.HeaderP->VerSysName = idxVerSysName; + Cache.HeaderP->Architecture = idxArchitecture; + Cache.HeaderP->SetArchitectures(idxArchitectures); + + // Calculate the hash for the empty map, so ReMap does not fail + Cache.HeaderP->CacheFileSize = Cache.CacheHash(); + Cache.ReMap(); + } + else + { + // Map directly from the existing file + Cache.ReMap(); + Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0])); + if (Cache.VS != _system->VS) + return _error->Error(_("Cache has an incompatible versioning system")); + } + + Cache.HeaderP->Dirty = true; + Map.Sync(0,sizeof(pkgCache::Header)); + return true; +} + /*}}}*/ +// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* We sync the data then unset the dirty flag in two steps so as to + advoid a problem during a crash */ +pkgCacheGenerator::~pkgCacheGenerator() +{ + if (_error->PendingError() == true || Map.validData() == false) + return; + if (Map.Sync() == false) + return; + + Cache.HeaderP->Dirty = false; + Cache.HeaderP->CacheFileSize = Cache.CacheHash(); + + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl; + Map.Sync(0,sizeof(pkgCache::Header)); +} + /*}}}*/ +void pkgCacheGenerator::ReMap(void const * const oldMap, void * const newMap, size_t oldSize) {/*{{{*/ + if (oldMap == newMap) + return; + + // Prevent multiple remaps of the same iterator. If seen.insert(iterator) + // returns (something, true) the iterator was not yet seen and we can + // remap it. + std::unordered_set<void *> seen; + + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "Remapping from " << oldMap << " to " << newMap << std::endl; + + Cache.ReMap(false); + + if (CurrentFile != nullptr) + CurrentFile = static_cast<pkgCache::PackageFile *>(newMap) + (CurrentFile - static_cast<pkgCache::PackageFile const *>(oldMap)); + if (CurrentRlsFile != nullptr) + CurrentRlsFile = static_cast<pkgCache::ReleaseFile *>(newMap) + (CurrentRlsFile - static_cast<pkgCache::ReleaseFile const *>(oldMap)); + +#define APT_REMAP(TYPE) \ + for (auto * const i : Dynamic<TYPE>::toReMap) \ + if (seen.insert(i).second) \ + i->ReMap(oldMap, newMap) + APT_REMAP(pkgCache::GrpIterator); + APT_REMAP(pkgCache::PkgIterator); + APT_REMAP(pkgCache::VerIterator); + APT_REMAP(pkgCache::DepIterator); + APT_REMAP(pkgCache::DescIterator); + APT_REMAP(pkgCache::PrvIterator); + APT_REMAP(pkgCache::PkgFileIterator); + APT_REMAP(pkgCache::RlsFileIterator); +#undef APT_REMAP + + for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) { + if (std::get<1>(seen.insert(ViewP)) == false) + continue; + // Ignore views outside of the cache. + if (ViewP->data() < static_cast<const char*>(oldMap) + || ViewP->data() > static_cast<const char*>(oldMap) + oldSize) + continue; + const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap)); + *ViewP = StringView(data , ViewP->size()); + } +} /*}}}*/ +// CacheGenerator::WriteStringInMap /*{{{*/ +map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String, + const unsigned long &Len) { + size_t oldSize = Map.Size(); + void const * const oldMap = Map.Data(); + map_stringitem_t const index{NarrowOffset(Map.WriteString(String, Len))}; + if (index != 0) + ReMap(oldMap, Map.Data(), oldSize); + return index; +} + /*}}}*/ +// CacheGenerator::WriteStringInMap /*{{{*/ +map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) { + size_t oldSize = Map.Size(); + void const * const oldMap = Map.Data(); + map_stringitem_t const index{NarrowOffset(Map.WriteString(String))}; + if (index != 0) + ReMap(oldMap, Map.Data(), oldSize); + return index; +} + /*}}}*/ +uint32_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/ + size_t oldSize = Map.Size(); + void const * const oldMap = Map.Data(); + auto index = Map.Allocate(size); + if (index != 0) + ReMap(oldMap, Map.Data(), oldSize); + if (index != static_cast<uint32_t>(index)) + abort(); // Internal error + return static_cast<uint32_t>(index); +} + /*}}}*/ +// CacheGenerator::MergeList - Merge the package list /*{{{*/ +// --------------------------------------------------------------------- +/* This provides the generation of the entries in the cache. Each loop + goes through a single package record from the underlying parse engine. */ +bool pkgCacheGenerator::MergeList(ListParser &List, + pkgCache::VerIterator *OutVer) +{ + List.Owner = this; + + unsigned int Counter = 0; + while (List.Step() == true) + { + string const PackageName = List.Package(); + if (PackageName.empty() == true) + return false; + + Counter++; + if (Counter % 100 == 0 && Progress != 0) + Progress->Progress(List.Offset()); + + APT::StringView Arch = List.Architecture(); + Dynamic<APT::StringView> DynArch(Arch); + APT::StringView Version = List.Version(); + Dynamic<APT::StringView> DynVersion(Version); + if (Version.empty() == true && Arch.empty() == true) + { + // package descriptions + if (MergeListGroup(List, PackageName) == false) + return false; + continue; + } + + // Get a pointer to the package structure + pkgCache::PkgIterator Pkg; + Dynamic<pkgCache::PkgIterator> DynPkg(Pkg); + if (NewPackage(Pkg, PackageName, Arch) == false) + // TRANSLATOR: The first placeholder is a package name, + // the other two should be copied verbatim as they include debug info + return _error->Error(_("Error occurred while processing %s (%s%d)"), + PackageName.c_str(), "NewPackage", 1); + + + if (Version.empty() == true) + { + if (MergeListPackage(List, Pkg) == false) + return false; + } + else + { + if (MergeListVersion(List, Pkg, Version, OutVer) == false) + return false; + } + + if (OutVer != 0) + return true; + } + + if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max()) + return _error->Error(_("Wow, you exceeded the number of package " + "names this APT is capable of.")); + if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max()) + return _error->Error(_("Wow, you exceeded the number of versions " + "this APT is capable of.")); + if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max()) + return _error->Error(_("Wow, you exceeded the number of descriptions " + "this APT is capable of.")); + if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max()) + return _error->Error(_("Wow, you exceeded the number of dependencies " + "this APT is capable of.")); + + return true; +} +// CacheGenerator::MergeListGroup /*{{{*/ +bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName) +{ + pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName); + // a group has no data on it's own, only packages have it but these + // stanzas like this come from Translation- files to add descriptions, + // but without a version we don't need a description for it… + if (Grp.end() == true) + return true; + Dynamic<pkgCache::GrpIterator> DynGrp(Grp); + + pkgCache::PkgIterator Pkg; + Dynamic<pkgCache::PkgIterator> DynPkg(Pkg); + for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg)) + if (MergeListPackage(List, Pkg) == false) + return false; + + return true; +} + /*}}}*/ +// CacheGenerator::MergeListPackage /*{{{*/ +bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg) +{ + // we first process the package, then the descriptions + // (for deb this package processing is in fact a no-op) + pkgCache::VerIterator Ver(Cache); + Dynamic<pkgCache::VerIterator> DynVer(Ver); + if (List.UsePackage(Pkg, Ver) == false) + return _error->Error(_("Error occurred while processing %s (%s%d)"), + Pkg.Name(), "UsePackage", 1); + + // Find the right version to write the description + StringView CurMd5 = List.Description_md5(); + std::vector<std::string> availDesc = List.AvailableDescriptionLanguages(); + for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver) + { + pkgCache::DescIterator VerDesc = Ver.DescriptionList(); + + // a version can only have one md5 describing it + if (VerDesc.end() == true || Cache.ViewString(VerDesc->md5sum) != CurMd5) + continue; + + map_stringitem_t md5idx = VerDesc->md5sum; + for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang) + { + // don't add a new description if we have one for the given + // md5 && language + if (IsDuplicateDescription(Cache, VerDesc, CurMd5, *CurLang) == true) + continue; + + AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx); + } + + // we can stop here as all "same" versions will share the description + break; + } + + return true; +} + /*}}}*/ +// CacheGenerator::MergeListVersion /*{{{*/ +bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg, + APT::StringView const &Version, pkgCache::VerIterator* &OutVer) +{ + pkgCache::VerIterator Ver = Pkg.VersionList(); + Dynamic<pkgCache::VerIterator> DynVer(Ver); + map_pointer<pkgCache::Version> *LastVer = &Pkg->VersionList; + void const * oldMap = Map.Data(); + + auto Hash = List.VersionHash(); + if (Ver.end() == false) + { + /* We know the list is sorted so we use that fact in the search. + Insertion of new versions is done with correct sorting */ + int Res = 1; + for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver) + { + char const * const VerStr = Ver.VerStr(); + Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(), + VerStr, VerStr + strlen(VerStr)); + // Version is higher as current version - insert here + if (Res > 0) + break; + // Versionstrings are equal - is hash also equal? + if (Res == 0) + { + if (List.SameVersion(Hash, Ver) == true) + break; + // sort (volatile) sources above not-sources like the status file + if (CurrentFile == nullptr || (CurrentFile->Flags & pkgCache::Flag::NotSource) == 0) + { + auto VF = Ver.FileList(); + for (; VF.end() == false; ++VF) + if (VF.File().Flagged(pkgCache::Flag::NotSource) == false) + break; + if (VF.end() == true) + break; + } + } + // proceed with the next till we have either the right + // or we found another version (which will be lower) + } + + /* We already have a version for this item, record that we saw it */ + if (Res == 0 && Ver.end() == false && Ver->Hash == Hash) + { + if (List.UsePackage(Pkg,Ver) == false) + return _error->Error(_("Error occurred while processing %s (%s%d)"), + Pkg.Name(), "UsePackage", 2); + + if (NewFileVer(Ver,List) == false) + return _error->Error(_("Error occurred while processing %s (%s%d)"), + Pkg.Name(), "NewFileVer", 1); + + // Read only a single record and return + if (OutVer != 0) + { + *OutVer = Ver; + return true; + } + + return true; + } + } + + // Add a new version + map_pointer<pkgCache::Version> const verindex = NewVersion(Ver, Version, Pkg.MapPointer(), Hash, *LastVer); + if (unlikely(verindex == 0)) + return _error->Error(_("Error occurred while processing %s (%s%d)"), + Pkg.Name(), "NewVersion", 1); + + if (oldMap != Map.Data()) + LastVer = static_cast<map_pointer<pkgCache::Version> *>(Map.Data()) + (LastVer - static_cast<map_pointer<pkgCache::Version> const *>(oldMap)); + *LastVer = verindex; + + if (unlikely(List.NewVersion(Ver) == false)) + return _error->Error(_("Error occurred while processing %s (%s%d)"), + Pkg.Name(), "NewVersion", 2); + + if (unlikely(List.UsePackage(Pkg,Ver) == false)) + return _error->Error(_("Error occurred while processing %s (%s%d)"), + Pkg.Name(), "UsePackage", 3); + + if (unlikely(NewFileVer(Ver,List) == false)) + return _error->Error(_("Error occurred while processing %s (%s%d)"), + Pkg.Name(), "NewFileVer", 2); + + pkgCache::GrpIterator Grp = Pkg.Group(); + Dynamic<pkgCache::GrpIterator> DynGrp(Grp); + + /* If it is the first version of this package we need to add implicit + Multi-Arch dependencies to all other package versions in the group now - + otherwise we just add them for this new version */ + if (Pkg.VersionList()->NextVer == 0) + { + pkgCache::PkgIterator P = Grp.PackageList(); + Dynamic<pkgCache::PkgIterator> DynP(P); + for (; P.end() != true; P = Grp.NextPkg(P)) + { + if (P->ID == Pkg->ID) + continue; + pkgCache::VerIterator V = P.VersionList(); + Dynamic<pkgCache::VerIterator> DynV(V); + for (; V.end() != true; ++V) + if (unlikely(AddImplicitDepends(V, Pkg) == false)) + return _error->Error(_("Error occurred while processing %s (%s%d)"), + Pkg.Name(), "AddImplicitDepends", 1); + } + } + if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false)) + return _error->Error(_("Error occurred while processing %s (%s%d)"), + Pkg.Name(), "AddImplicitDepends", 2); + + // Read only a single record and return + if (OutVer != 0) + { + *OutVer = Ver; + return true; + } + + /* Record the Description(s) based on their master md5sum */ + StringView CurMd5 = List.Description_md5(); + + /* Before we add a new description we first search in the group for + a version with a description of the same MD5 - if so we reuse this + description group instead of creating our own for this version */ + for (pkgCache::PkgIterator P = Grp.PackageList(); + P.end() == false; P = Grp.NextPkg(P)) + { + for (pkgCache::VerIterator V = P.VersionList(); + V.end() == false; ++V) + { + if (V->DescriptionList == 0 || Cache.ViewString(V.DescriptionList()->md5sum) != CurMd5) + continue; + Ver->DescriptionList = V->DescriptionList; + } + } + + // We haven't found reusable descriptions, so add the first description(s) + map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum; + std::vector<std::string> availDesc = List.AvailableDescriptionLanguages(); + for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang) + if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false) + return false; + return true; +} + /*}}}*/ +bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, APT::StringView CurMd5, map_stringitem_t &md5idx) /*{{{*/ +{ + pkgCache::DescIterator Desc; + Dynamic<pkgCache::DescIterator> DynDesc(Desc); + + map_pointer<pkgCache::Description> const descindex = NewDescription(Desc, lang, CurMd5, md5idx); + if (unlikely(descindex == 0)) + return _error->Error(_("Error occurred while processing %s (%s%d)"), + Ver.ParentPkg().Name(), "NewDescription", 1); + + md5idx = Desc->md5sum; + Desc->ParentPkg = Ver.ParentPkg().MapPointer(); + + // we add at the end, so that the start is constant as we need + // that to be able to efficiently share these lists + pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap + for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc); + map_pointer<pkgCache::Description> * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc; + *LastNextDesc = descindex; + + if (NewFileDesc(Desc,List) == false) + return _error->Error(_("Error occurred while processing %s (%s%d)"), + Ver.ParentPkg().Name(), "NewFileDesc", 1); + + return true; +} + /*}}}*/ + /*}}}*/ +// CacheGenerator::NewGroup - Add a new group /*{{{*/ +// --------------------------------------------------------------------- +/* This creates a new group structure and adds it to the hash table */ +bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name) +{ + Dynamic<StringView> DName(Name); + Grp = Cache.FindGrp(Name); + if (Grp.end() == false) + return true; + + // Get a structure + auto const Group = AllocateInMap<pkgCache::Group>(); + if (unlikely(Group == 0)) + return false; + + map_stringitem_t const idxName = WriteStringInMap(Name); + if (unlikely(idxName == 0)) + return false; + + Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group); + Grp->Name = idxName; + + // Insert it into the hash table + unsigned long const Hash = Cache.Hash(Name); + map_pointer<pkgCache::Group> *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash]; + + while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0) + insertAt = &(Cache.GrpP + *insertAt)->Next; + Grp->Next = *insertAt; + *insertAt = Group; + + Grp->ID = Cache.HeaderP->GroupCount++; + return true; +} + /*}}}*/ +// CacheGenerator::NewPackage - Add a new package /*{{{*/ +// --------------------------------------------------------------------- +/* This creates a new package structure and adds it to the hash table */ +bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name, + StringView Arch) { + pkgCache::GrpIterator Grp; + Dynamic<StringView> DName(Name); + Dynamic<StringView> DArch(Arch); + Dynamic<pkgCache::GrpIterator> DynGrp(Grp); + if (unlikely(NewGroup(Grp, Name) == false)) + return false; + + Pkg = Grp.FindPkg(Arch); + if (Pkg.end() == false) + return true; + + // Get a structure + auto const Package = AllocateInMap<pkgCache::Package>(); + if (unlikely(Package == 0)) + return false; + Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package); + + // Set the name, arch and the ID + Pkg->Group = Grp.MapPointer(); + // all is mapped to the native architecture + map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch); + if (unlikely(idxArch == 0)) + return false; + Pkg->Arch = idxArch; + Pkg->ID = Cache.HeaderP->PackageCount++; + + // Insert the package into our package list + if (Grp->FirstPackage == 0) // the group is new + { + Grp->FirstPackage = Package; + // Insert it into the hash table + map_id_t const Hash = Cache.Hash(Name); + map_pointer<pkgCache::Package> *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash]; + while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name)) > 0) + insertAt = &(Cache.PkgP + *insertAt)->NextPackage; + Pkg->NextPackage = *insertAt; + *insertAt = Package; + } + else // Group the Packages together + { + // if sibling is provided by another package, this one is too + { + pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do + if (M.end() == false) { + pkgCache::PrvIterator Prv; + pkgCache::VerIterator Ver; + Dynamic<pkgCache::PrvIterator> DynPrv(Prv); + Dynamic<pkgCache::VerIterator> DynVer(Ver); + for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv) + { + if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0) + continue; + Ver = Prv.OwnerVer(); + if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign && + (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0) + { + if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false) + continue; + if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false) + return false; + } + } + } + } + // let M-A:foreign package siblings provide this package + { + pkgCache::PkgIterator P; + pkgCache::VerIterator Ver; + Dynamic<pkgCache::PkgIterator> DynP(P); + Dynamic<pkgCache::VerIterator> DynVer(Ver); + + for (P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P)) + { + if (APT::Configuration::checkArchitecture(P.Arch()) == false) + continue; + for (Ver = P.VersionList(); Ver.end() == false; ++Ver) + if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign) + if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false) + return false; + } + } + // and negative dependencies, don't forget negative dependencies + { + pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); + if (M.end() == false) { + pkgCache::DepIterator Dep; + Dynamic<pkgCache::DepIterator> DynDep(Dep); + for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep) + { + if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0) + continue; + if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts && + Dep->Type != pkgCache::Dep::Replaces) + continue; + pkgCache::VerIterator Ver = Dep.ParentVer(); + Dynamic<pkgCache::VerIterator> DynVer(Ver); + map_pointer<pkgCache::Dependency> * unused = NULL; + if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false) + return false; + } + } + } + + // this package is the new last package + pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage); + Pkg->NextPackage = LastPkg->NextPackage; + LastPkg->NextPackage = Package; + } + Grp->LastPackage = Package; + + // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it + if (Arch == "any") + { + size_t const found = Name.rfind(':'); + StringView ArchA = Name.substr(found + 1); + if (ArchA != "any") + { + // ArchA is used inside the loop which might remap (NameA is not used) + Dynamic<StringView> DynArchA(ArchA); + StringView NameA = Name.substr(0, found); + pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA); + Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA); + if (PkgA.end()) + { + Dynamic<StringView> DynNameA(NameA); + if (NewPackage(PkgA, NameA, ArchA) == false) + return false; + } + if (unlikely(PkgA.end())) + return _error->Fatal("NewPackage was successful for %s:%s," + "but the package doesn't exist anyhow!", + NameA.to_string().c_str(), ArchA.to_string().c_str()); + else + { + pkgCache::PrvIterator Prv = PkgA.ProvidesList(); + for (; Prv.end() == false; ++Prv) + { + if (Prv.IsMultiArchImplicit()) + continue; + pkgCache::VerIterator V = Prv.OwnerVer(); + if (ArchA != V.ParentPkg().Arch()) + continue; + if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false) + return false; + } + pkgCache::VerIterator V = PkgA.VersionList(); + Dynamic<pkgCache::VerIterator> DynV(V); + for (; V.end() == false; ++V) + { + if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false) + return false; + } + } + } + } + return true; +} + /*}}}*/ +// CacheGenerator::AddImplicitDepends /*{{{*/ +bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G, + pkgCache::PkgIterator &P, + pkgCache::VerIterator &V) +{ + APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch(); + Dynamic<APT::StringView> DynArch(Arch); + map_pointer<pkgCache::Dependency> *OldDepLast = NULL; + /* MultiArch handling introduces a lot of implicit Dependencies: + - MultiArch: same → Co-Installable if they have the same version + - All others conflict with all other group members */ + bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same); + pkgCache::PkgIterator D = G.PackageList(); + Dynamic<pkgCache::PkgIterator> DynD(D); + map_stringitem_t const VerStrIdx = V->VerStr; + for (; D.end() != true; D = G.NextPkg(D)) + { + if (Arch == D.Arch() || D->VersionList == 0) + continue; + /* We allow only one installed arch at the time + per group, therefore each group member conflicts + with all other group members */ + if (coInstall == true) + { + // Replaces: ${self}:other ( << ${binary:Version}) + NewDepends(D, V, VerStrIdx, + pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces, + OldDepLast); + // Breaks: ${self}:other (!= ${binary:Version}) + NewDepends(D, V, VerStrIdx, + pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks, + OldDepLast); + } else { + // Conflicts: ${self}:other + NewDepends(D, V, 0, + pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts, + OldDepLast); + } + } + return true; +} +bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V, + pkgCache::PkgIterator &D) +{ + /* MultiArch handling introduces a lot of implicit Dependencies: + - MultiArch: same → Co-Installable if they have the same version + - All others conflict with all other group members */ + map_pointer<pkgCache::Dependency> *OldDepLast = NULL; + bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same); + if (coInstall == true) + { + map_stringitem_t const VerStrIdx = V->VerStr; + // Replaces: ${self}:other ( << ${binary:Version}) + NewDepends(D, V, VerStrIdx, + pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces, + OldDepLast); + // Breaks: ${self}:other (!= ${binary:Version}) + NewDepends(D, V, VerStrIdx, + pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks, + OldDepLast); + } else { + // Conflicts: ${self}:other + NewDepends(D, V, 0, + pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts, + OldDepLast); + } + return true; +} + + /*}}}*/ +// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver, + ListParser &List) +{ + if (CurrentFile == nullptr) + return true; + + // Get a structure + auto const VerFile = AllocateInMap<pkgCache::VerFile>(); + if (VerFile == 0) + return false; + + pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile); + VF->File = map_pointer<pkgCache::PackageFile>{NarrowOffset(CurrentFile - Cache.PkgFileP)}; + + // Link it to the end of the list + map_pointer<pkgCache::VerFile> *Last = &Ver->FileList; + for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V) + Last = &V->NextFile; + VF->NextFile = *Last; + *Last = VF.MapPointer(); + + VF->Offset = List.Offset(); + VF->Size = List.Size(); + if (Cache.HeaderP->MaxVerFileSize < VF->Size) + Cache.HeaderP->MaxVerFileSize = VF->Size; + Cache.HeaderP->VerFileCount++; + + return true; +} + /*}}}*/ +// CacheGenerator::NewVersion - Create a new Version /*{{{*/ +// --------------------------------------------------------------------- +/* This puts a version structure in the linked list */ +map_pointer<pkgCache::Version> pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, + APT::StringView const &VerStr, + map_pointer<pkgCache::Package> const ParentPkg, + uint32_t Hash, + map_pointer<pkgCache::Version> const Next) +{ + // Get a structure + auto const Version = AllocateInMap<pkgCache::Version>(); + if (Version == 0) + return 0; + + // Fill it in + Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version); + auto d = AllocateInMap<pkgCache::Version::Extra>(); // sequence point so Ver can be moved if needed + Ver->d = d; + if (not Ver.PhasedUpdatePercentage(100)) + abort(); + + //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it + Ver->NextVer = Next; + Ver->ParentPkg = ParentPkg; + Ver->Hash = Hash; + Ver->ID = Cache.HeaderP->VersionCount++; + + // try to find the version string in the group for reuse + pkgCache::PkgIterator Pkg = Ver.ParentPkg(); + pkgCache::GrpIterator Grp = Pkg.Group(); + if (Pkg.end() == false && Grp.end() == false) + { + for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P)) + { + if (Pkg == P) + continue; + for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V) + { + int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length()); + if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0') + { + Ver->VerStr = V->VerStr; + return Version; + } + else if (cmp < 0) + break; + } + } + } + // haven't found the version string, so create + map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr); + if (unlikely(idxVerStr == 0)) + return 0; + Ver->VerStr = idxVerStr; + return Version; +} + /*}}}*/ +// CacheGenerator::NewFileDesc - Create a new File<->Desc association /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc, + ListParser &List) +{ + if (CurrentFile == nullptr) + return true; + + // Get a structure + auto const DescFile = AllocateInMap<pkgCache::DescFile>(); + if (DescFile == 0) + return false; + + pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile); + DF->File = map_pointer<pkgCache::PackageFile>{NarrowOffset(CurrentFile - Cache.PkgFileP)}; + + // Link it to the end of the list + map_pointer<pkgCache::DescFile> *Last = &Desc->FileList; + for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D) + Last = &D->NextFile; + + DF->NextFile = *Last; + *Last = DF.MapPointer(); + + DF->Offset = List.Offset(); + DF->Size = List.Size(); + if (Cache.HeaderP->MaxDescFileSize < DF->Size) + Cache.HeaderP->MaxDescFileSize = DF->Size; + Cache.HeaderP->DescFileCount++; + + return true; +} + /*}}}*/ +// CacheGenerator::NewDescription - Create a new Description /*{{{*/ +// --------------------------------------------------------------------- +/* This puts a description structure in the linked list */ +map_pointer<pkgCache::Description> pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, + const string &Lang, + APT::StringView md5sum, + map_stringitem_t const idxmd5str) +{ + // Get a structure + auto const Description = AllocateInMap<pkgCache::Description>(); + if (Description == 0) + return 0; + + // Fill it in + Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description); + Desc->ID = Cache.HeaderP->DescriptionCount++; + map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang); + if (unlikely(idxlanguage_code == 0)) + return 0; + Desc->language_code = idxlanguage_code; + + if (idxmd5str != 0) + Desc->md5sum = idxmd5str; + else + { + map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum); + if (unlikely(idxmd5sum == 0)) + return 0; + Desc->md5sum = idxmd5sum; + } + + return Description; +} + /*}}}*/ +// CacheGenerator::NewDepends - Create a dependency element /*{{{*/ +// --------------------------------------------------------------------- +/* This creates a dependency element in the tree. It is linked to the + version and to the package that it is pointing to. */ +bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver, + map_stringitem_t const Version, + uint8_t const Op, + uint8_t const Type, + map_pointer<pkgCache::Dependency> * &OldDepLast) +{ + void const * const oldMap = Map.Data(); + // Get a structure + auto const Dependency = AllocateInMap<pkgCache::Dependency>(); + if (unlikely(Dependency == 0)) + return false; + + bool isDuplicate = false; + map_pointer<pkgCache::DependencyData> DependencyData = 0; + map_pointer<pkgCache::DependencyData> PreviousData = 0; + if (Pkg->RevDepends != 0) + { + pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends; + DependencyData = L->DependencyData; + do { + pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData; + if (Version > D->Version) + break; + if (D->Version == Version && D->Type == Type && D->CompareOp == Op) + { + isDuplicate = true; + break; + } + PreviousData = DependencyData; + DependencyData = D->NextData; + } while (DependencyData != 0); + } + + if (isDuplicate == false) + { + DependencyData = AllocateInMap<pkgCache::DependencyData>(); + if (unlikely(DependencyData == 0)) + return false; + } + + pkgCache::Dependency * Link = Cache.DepP + Dependency; + Link->ParentVer = Ver.MapPointer(); + Link->DependencyData = DependencyData; + Link->ID = Cache.HeaderP->DependsCount++; + + pkgCache::DepIterator Dep(Cache, Link); + if (isDuplicate == false) + { + Dep->Type = Type; + Dep->CompareOp = Op; + Dep->Version = Version; + Dep->Package = Pkg.MapPointer(); + ++Cache.HeaderP->DependsDataCount; + if (PreviousData != 0) + { + pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData; + Dep->NextData = D->NextData; + D->NextData = DependencyData; + } + else if (Pkg->RevDepends != 0) + { + pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends; + Dep->NextData = D->DependencyData; + } + } + + if (isDuplicate == true || PreviousData != 0) + { + pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends; + Link->NextRevDepends = L->NextRevDepends; + L->NextRevDepends = Dependency; + } + else + { + Link->NextRevDepends = Pkg->RevDepends; + Pkg->RevDepends = Dependency; + } + + + // Do we know where to link the Dependency to? + if (OldDepLast == NULL) + { + OldDepLast = &Ver->DependsList; + for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D) + OldDepLast = &D->NextDepends; + } else if (oldMap != Map.Data()) + OldDepLast = static_cast<map_pointer<pkgCache::Dependency> *>(Map.Data()) + (OldDepLast - static_cast<map_pointer<pkgCache::Dependency> const *>(oldMap)); + + Dep->NextDepends = *OldDepLast; + *OldDepLast = Dependency; + OldDepLast = &Dep->NextDepends; + return true; +} + /*}}}*/ +// ListParser::NewDepends - Create the environment for a new dependency /*{{{*/ +// --------------------------------------------------------------------- +/* This creates a Group and the Package to link this dependency to if + needed and handles also the caching of the old endpoint */ +bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver, + StringView PackageName, + StringView Arch, + StringView Version, + uint8_t const Op, + uint8_t const Type) +{ + pkgCache::GrpIterator Grp; + Dynamic<pkgCache::GrpIterator> DynGrp(Grp); + Dynamic<StringView> DynPackageName(PackageName); + Dynamic<StringView> DynArch(Arch); + Dynamic<StringView> DynVersion(Version); + if (unlikely(Owner->NewGroup(Grp, PackageName) == false)) + return false; + + map_stringitem_t idxVersion = 0; + if (Version.empty() == false) + { + int const CmpOp = Op & 0x0F; + // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages) + if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr()) + idxVersion = Ver->VerStr; + + if (idxVersion == 0) + { + idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version); + if (unlikely(idxVersion == 0)) + return false; + } + } + + bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks || + Type == pkgCache::Dep::Conflicts || + Type == pkgCache::Dep::Replaces); + + pkgCache::PkgIterator Pkg; + Dynamic<pkgCache::PkgIterator> DynPkg(Pkg); + if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0) + { + // Locate the target package + Pkg = Grp.FindPkg(Arch); + if (Pkg.end() == true) { + if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false)) + return false; + } + + /* Caching the old end point speeds up generation substantially */ + if (OldDepVer != Ver) { + OldDepLast = NULL; + OldDepVer = Ver; + } + + return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast); + } + else + { + /* Caching the old end point speeds up generation substantially */ + if (OldDepVer != Ver) { + OldDepLast = NULL; + OldDepVer = Ver; + } + + for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg)) + { + if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false) + return false; + } + } + return true; +} + /*}}}*/ +// ListParser::NewProvides - Create a Provides element /*{{{*/ +bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver, + StringView PkgName, + StringView PkgArch, + StringView Version, + uint8_t const Flags) +{ + pkgCache const &Cache = Owner->Cache; + Dynamic<StringView> DynPkgName(PkgName); + Dynamic<StringView> DynArch(PkgArch); + Dynamic<StringView> DynVersion(Version); + + // We do not add self referencing provides + if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() || + (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) && + (Version.empty() || Version == Ver.VerStr())) + return true; + + // Locate the target package + pkgCache::PkgIterator Pkg; + Dynamic<pkgCache::PkgIterator> DynPkg(Pkg); + if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false)) + return false; + + map_stringitem_t idxProvideVersion = 0; + if (Version.empty() == false) { + idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version); + if (unlikely(idxProvideVersion == 0)) + return false; + } + return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags); +} +bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver, + pkgCache::PkgIterator &Pkg, + map_stringitem_t const ProvideVersion, + uint8_t const Flags) +{ + // Get a structure + auto const Provides = AllocateInMap<pkgCache::Provides>(); + if (unlikely(Provides == 0)) + return false; + ++Cache.HeaderP->ProvidesCount; + + // Fill it in + pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP); + Prv->Version = Ver.MapPointer(); + Prv->ProvideVersion = ProvideVersion; + Prv->Flags = Flags; + Prv->NextPkgProv = Ver->ProvidesList; + Ver->ProvidesList = Prv.MapPointer(); + + // Link it to the package + Prv->ParentPkg = Pkg.MapPointer(); + Prv->NextProvides = Pkg->ProvidesList; + Pkg->ProvidesList = Prv.MapPointer(); + return true; +} + /*}}}*/ +// ListParser::NewProvidesAllArch - add provides for all architectures /*{{{*/ +bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package, + StringView Version, uint8_t const Flags) { + pkgCache &Cache = Owner->Cache; + pkgCache::GrpIterator Grp = Cache.FindGrp(Package); + Dynamic<pkgCache::GrpIterator> DynGrp(Grp); + Dynamic<StringView> DynPackage(Package); + Dynamic<StringView> DynVersion(Version); + + if (Grp.end() == true || Grp->FirstPackage == 0) + return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags); + else + { + map_stringitem_t idxProvideVersion = 0; + if (Version.empty() == false) { + idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version); + if (unlikely(idxProvideVersion == 0)) + return false; + } + + bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit; + bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific; + pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg(); + Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg); + pkgCache::PkgIterator Pkg; + Dynamic<pkgCache::PkgIterator> DynPkg(Pkg); + for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg)) + { + if (isImplicit && OwnerPkg == Pkg) + continue; + if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false) + continue; + if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false) + return false; + } + } + return true; +} + /*}}}*/ +bool pkgCacheListParser::SameVersion(uint32_t Hash, /*{{{*/ + pkgCache::VerIterator const &Ver) +{ + return Hash == Ver->Hash; +} + /*}}}*/ +// CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to /*{{{*/ +bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site, + unsigned long Flags) +{ + CurrentRlsFile = nullptr; + if (File.empty() && Site.empty()) + return true; + + // Get some space for the structure + auto const idxFile = AllocateInMap<pkgCache::ReleaseFile>(); + if (unlikely(idxFile == 0)) + return false; + CurrentRlsFile = Cache.RlsFileP + idxFile; + + // Fill it in + map_stringitem_t const idxFileName = WriteStringInMap(File); + map_stringitem_t const idxSite = StoreString(MIXED, Site); + if (unlikely(idxFileName == 0 || idxSite == 0)) + return false; + CurrentRlsFile->FileName = idxFileName; + CurrentRlsFile->Site = idxSite; + CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList; + CurrentRlsFile->Flags = Flags; + CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount; + RlsFileName = File; + Cache.HeaderP->RlsFileList = map_pointer<pkgCache::ReleaseFile>{NarrowOffset(CurrentRlsFile - Cache.RlsFileP)}; + Cache.HeaderP->ReleaseFileCount++; + + return true; +} + /*}}}*/ +// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/ +// --------------------------------------------------------------------- +/* This is used to select which file is to be associated with all newly + added versions. The caller is responsible for setting the IMS fields. */ +bool pkgCacheGenerator::SelectFile(std::string const &File, + pkgIndexFile const &Index, + std::string const &Architecture, + std::string const &Component, + unsigned long const Flags) +{ + CurrentFile = nullptr; + // Get some space for the structure + auto const idxFile = AllocateInMap<pkgCache::PackageFile>(); + if (unlikely(idxFile == 0)) + return false; + CurrentFile = Cache.PkgFileP + idxFile; + + // Fill it in + map_stringitem_t const idxFileName = WriteStringInMap(File); + if (unlikely(idxFileName == 0)) + return false; + CurrentFile->FileName = idxFileName; + CurrentFile->NextFile = Cache.HeaderP->FileList; + CurrentFile->ID = Cache.HeaderP->PackageFileCount; + map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label); + if (unlikely(idxIndexType == 0)) + return false; + CurrentFile->IndexType = idxIndexType; + if (Architecture.empty()) + CurrentFile->Architecture = 0; + else + { + map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture); + if (unlikely(arch == 0)) + return false; + CurrentFile->Architecture = arch; + } + map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component); + if (unlikely(component == 0)) + return false; + CurrentFile->Component = component; + CurrentFile->Flags = Flags; + if (CurrentRlsFile != nullptr) + CurrentFile->Release = map_pointer<pkgCache::ReleaseFile>{NarrowOffset(CurrentRlsFile - Cache.RlsFileP)}; + else + CurrentFile->Release = 0; + PkgFileName = File; + Cache.HeaderP->FileList = map_pointer<pkgCache::PackageFile>{NarrowOffset(CurrentFile - Cache.PkgFileP)}; + Cache.HeaderP->PackageFileCount++; + + if (Progress != 0) + Progress->SubProgress(Index.Size()); + return true; +} + /*}}}*/ +// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/ +// --------------------------------------------------------------------- +/* This is used to create handles to strings. Given the same text it + always returns the same number */ +map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S, + unsigned int Size) +{ + auto strings = &strMixed; + switch(type) { + case MIXED: strings = &strMixed; break; + case VERSIONNUMBER: strings = &strVersions; break; + case SECTION: strings = &strSections; break; + default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0; + } + + auto const item = strings->find({S, Size, nullptr, 0}); + if (item != strings->end()) + return item->item; + + map_stringitem_t const idxString = WriteStringInMap(S,Size); + strings->insert({nullptr, Size, this, idxString}); + return idxString; +} + /*}}}*/ +// CheckValidity - Check that a cache is up-to-date /*{{{*/ +// --------------------------------------------------------------------- +/* This just verifies that each file in the list of index files exists, + has matching attributes with the cache and the cache does not have + any extra files. */ +class APT_HIDDEN ScopedErrorRevert { +public: + ScopedErrorRevert() { _error->PushToStack(); } + ~ScopedErrorRevert() { _error->RevertToStack(); } +}; + +static bool CheckValidity(FileFd &CacheFile, std::string const &CacheFileName, + pkgSourceList &List, + FileIterator const Start, + FileIterator const End, + MMap **OutMap = 0, + pkgCache **OutCache = 0) +{ + if (CacheFileName.empty()) + return false; + ScopedErrorRevert ser; + + bool const Debug = _config->FindB("Debug::pkgCacheGen", false); + // No file, certainly invalid + if (CacheFile.Open(CacheFileName, FileFd::ReadOnly, FileFd::None) == false) + { + if (Debug == true) + std::clog << "CacheFile " << CacheFileName << " doesn't exist" << std::endl; + return false; + } + + if (_config->FindI("APT::Cache-Start", 0) == 0) + { + auto const size = CacheFile.FileSize(); + if (std::numeric_limits<int>::max() >= size && size > APT_CACHE_START_DEFAULT) + _config->Set("APT::Cache-Start", size); + } + + if (List.GetLastModifiedTime() > CacheFile.ModificationTime()) + { + if (Debug == true) + std::clog << "sources.list is newer than the cache" << std::endl; + return false; + } + + // Map it + std::unique_ptr<MMap> Map(new MMap(CacheFile,0)); + if (unlikely(Map->validData()) == false) + return false; + std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get())); + pkgCache &Cache = *CacheP.get(); + if (_error->PendingError() || Map->Size() == 0) + { + if (Debug == true) + std::clog << "Errors are pending or Map is empty() for " << CacheFileName << std::endl; + return false; + } + + std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]); + memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount); + std::vector<pkgIndexFile *> Files; + for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i) + { + if (Debug == true) + std::clog << "Checking RlsFile " << (*i)->Describe() << ": "; + pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true); + if (RlsFile.end() == true) + { + if (Debug == true) + std::clog << "FindInCache returned end-Pointer" << std::endl; + return false; + } + + RlsVisited[RlsFile->ID] = true; + if (Debug == true) + std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl; + + std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles(); + std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files), + [](pkgIndexFile const * const I) { return I->HasPackages(); }); + } + for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I) + if (RlsVisited[I] == false) + { + if (Debug == true) + std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl; + return false; + } + + std::copy(Start, End, std::back_inserter(Files)); + + /* Now we check every index file, see if it is in the cache, + verify the IMS data and check that it is on the disk too.. */ + std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]); + memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount); + for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile) + { + if (Debug == true) + std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": "; + if ((*PkgFile)->Exists() == false) + { + if (Debug == true) + std::clog << "file doesn't exist" << std::endl; + continue; + } + + // FindInCache is also expected to do an IMS check. + pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache); + if (File.end() == true) + { + if (Debug == true) + std::clog << "FindInCache returned end-Pointer" << std::endl; + return false; + } + + Visited[File->ID] = true; + if (Debug == true) + std::clog << "with ID " << File->ID << " is valid" << std::endl; + } + + for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++) + if (Visited[I] == false) + { + if (Debug == true) + std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl; + return false; + } + + if (_error->PendingError() == true) + { + if (Debug == true) + { + std::clog << "Validity failed because of pending errors:" << std::endl; + _error->DumpErrors(std::clog, GlobalError::DEBUG, false); + } + return false; + } + + if (OutMap != 0) + *OutMap = Map.release(); + if (OutCache != 0) + *OutCache = CacheP.release(); + return true; +} + /*}}}*/ +// ComputeSize - Compute the total size of a bunch of files /*{{{*/ +// --------------------------------------------------------------------- +/* Size is kind of an abstract notion that is only used for the progress + meter */ +static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End) +{ + map_filesize_t TotalSize = 0; + if (List != NULL) + { + for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i) + { + std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles(); + for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j) + if ((*j)->HasPackages() == true) + TotalSize += (*j)->Size(); + } + } + + for (; Start < End; ++Start) + { + if ((*Start)->HasPackages() == false) + continue; + TotalSize += (*Start)->Size(); + } + return TotalSize; +} + /*}}}*/ +// BuildCache - Merge the list of index files into the cache /*{{{*/ +static bool BuildCache(pkgCacheGenerator &Gen, + OpProgress * const Progress, + map_filesize_t &CurrentSize,map_filesize_t TotalSize, + pkgSourceList const * const List, + FileIterator const Start, FileIterator const End) +{ + bool mergeFailure = false; + + auto const indexFileMerge = [&](pkgIndexFile * const I) { + if (I->HasPackages() == false || mergeFailure) + return; + + if (I->Exists() == false) + return; + + if (I->FindInCache(Gen.GetCache()).end() == false) + { + _error->Warning("Duplicate sources.list entry %s", + I->Describe().c_str()); + return; + } + + map_filesize_t const Size = I->Size(); + if (Progress != NULL) + Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists")); + CurrentSize += Size; + + if (I->Merge(Gen,Progress) == false) + mergeFailure = true; + }; + + if (List != NULL) + { + for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i) + { + if ((*i)->FindInCache(Gen.GetCache(), false).end() == false) + { + _error->Warning("Duplicate sources.list entry %s", + (*i)->Describe().c_str()); + continue; + } + + if ((*i)->Merge(Gen, Progress) == false) + return false; + + std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles(); + if (Indexes != NULL) + std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge); + if (mergeFailure) + return false; + } + } + + if (Start != End) + { + Gen.SelectReleaseFile("", ""); + std::for_each(Start, End, indexFileMerge); + if (mergeFailure) + return false; + } + return true; +} + /*}}}*/ +// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/ +// --------------------------------------------------------------------- +/* This makes sure that the status cache (the cache that has all + index files from the sources list and all local ones) is ready + to be mmaped. If OutMap is not zero then a MMap object representing + the cache will be stored there. This is pretty much mandatory if you + are using AllowMem. AllowMem lets the function be run as non-root + where it builds the cache 'fast' into a memory buffer. */ +static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags) +{ + map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", APT_CACHE_START_DEFAULT); + map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024); + map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0); + Flags |= MMap::Moveable; + if (_config->FindB("APT::Cache-Fallback", false) == true) + Flags |= MMap::Fallback; + if (CacheF != NULL) + return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit); + else + return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit); +} +static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map, + std::string const &FileName) +{ + FileFd SCacheF(FileName, FileFd::WriteAtomic); + if (SCacheF.IsOpen() == false || SCacheF.Failed()) + return false; + + fchmod(SCacheF.Fd(),0644); + + // Write out the main data + if (SCacheF.Write(Map->Data(),Map->Size()) == false) + return _error->Error(_("IO Error saving source cache")); + + // Write out the proper header + Gen->GetCache().HeaderP->Dirty = false; + Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash(); + if (SCacheF.Seek(0) == false || + SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false) + return _error->Error(_("IO Error saving source cache")); + Gen->GetCache().HeaderP->Dirty = true; + return true; +} +static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen, + std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, FileFd &CacheF) +{ + Map.reset(CreateDynamicMMap(NULL, 0)); + if (unlikely(Map->validData()) == false) + return false; + if (CacheF.IsOpen() == false || CacheF.Seek(0) == false || CacheF.Failed()) + return false; + _error->PushToStack(); + uint32_t const alloc = Map->RawAllocate(CacheF.Size()); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + if (alloc == 0 && newError) + return false; + if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false) + return false; + Gen.reset(new pkgCacheGenerator(Map.get(),Progress)); + return Gen->Start(); +} +bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress, + MMap **OutMap,bool) +{ + return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true); +} +bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress, + MMap **OutMap,pkgCache **OutCache, bool) +{ + // FIXME: deprecate the ignored AllowMem parameter + bool const Debug = _config->FindB("Debug::pkgCacheGen", false); + + std::vector<pkgIndexFile *> Files; + if (_system->AddStatusFiles(Files) == false) + return false; + + // Decide if we can write to the files.. + string const CacheFileName = _config->FindFile("Dir::Cache::pkgcache"); + string const SrcCacheFileName = _config->FindFile("Dir::Cache::srcpkgcache"); + + // ensure the cache directory exists + if (CacheFileName.empty() == false || SrcCacheFileName.empty() == false) + { + string dir = _config->FindDir("Dir::Cache"); + size_t const len = dir.size(); + if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5) + dir = dir.substr(0, len - 5); + if (CacheFileName.empty() == false) + CreateDirectory(dir, flNotFile(CacheFileName)); + if (SrcCacheFileName.empty() == false) + CreateDirectory(dir, flNotFile(SrcCacheFileName)); + } + + if (Progress != NULL) + Progress->OverallProgress(0,1,1,_("Reading package lists")); + + bool pkgcache_fine = false; + bool srcpkgcache_fine = false; + bool volatile_fine = List.GetVolatileFiles().empty(); + FileFd CacheFile; + if (CheckValidity(CacheFile, CacheFileName, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL, + volatile_fine ? OutCache : NULL) == true) + { + if (Debug == true) + std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl; + pkgcache_fine = true; + srcpkgcache_fine = true; + } + + FileFd SrcCacheFile; + if (pkgcache_fine == false) + { + if (CheckValidity(SrcCacheFile, SrcCacheFileName, List, Files.end(), Files.end()) == true) + { + if (Debug == true) + std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl; + srcpkgcache_fine = true; + } + } + + if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true) + { + if (Progress != NULL) + Progress->OverallProgress(1,1,1,_("Reading package lists")); + return true; + } + + bool Writeable = false; + if (srcpkgcache_fine == false || pkgcache_fine == false) + { + if (CacheFileName.empty() == false) + Writeable = access(flNotFile(CacheFileName).c_str(),W_OK) == 0; + else if (SrcCacheFileName.empty() == false) + Writeable = access(flNotFile(SrcCacheFileName).c_str(),W_OK) == 0; + + if (Debug == true) + std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl; + } + + // At this point we know we need to construct something, so get storage ready + std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0)); + if (unlikely(Map->validData()) == false) + return false; + if (Debug == true) + std::clog << "Open memory Map (not filebased)" << std::endl; + + std::unique_ptr<pkgCacheGenerator> Gen{nullptr}; + map_filesize_t CurrentSize = 0; + std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles(); + map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end()); + if (srcpkgcache_fine == true && pkgcache_fine == false) + { + if (Debug == true) + std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl; + if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false) + return false; + srcpkgcache_fine = true; + TotalSize += ComputeSize(NULL, Files.begin(), Files.end()); + } + else if (srcpkgcache_fine == false) + { + if (Debug == true) + std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl; + Gen.reset(new pkgCacheGenerator(Map.get(),Progress)); + if (Gen->Start() == false) + return false; + + TotalSize += ComputeSize(&List, Files.begin(),Files.end()); + if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List, + Files.end(),Files.end()) == false) + return false; + + if (Writeable == true && SrcCacheFileName.empty() == false) + if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFileName) == false) + return false; + } + + if (pkgcache_fine == false) + { + if (Debug == true) + std::clog << "Building status cache in pkgcache.bin now" << std::endl; + if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL, + Files.begin(), Files.end()) == false) + return false; + + if (Writeable == true && CacheFileName.empty() == false) + if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFileName) == false) + return false; + } + + if (Debug == true) + std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl; + + if (volatile_fine == false) + { + if (Gen == nullptr) + { + if (Debug == true) + std::clog << "Populate new MMap with cachefile contents" << std::endl; + if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false) + return false; + } + + Files = List.GetVolatileFiles(); + if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL, + Files.begin(), Files.end()) == false) + return false; + } + + if (OutMap != nullptr) + *OutMap = Map.release(); + + if (Debug == true) + std::clog << "Everything is ready for shipping" << std::endl; + return true; +} + /*}}}*/ +// CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/ +class APT_HIDDEN ScopedErrorMerge { +public: + ScopedErrorMerge() { _error->PushToStack(); } + ~ScopedErrorMerge() { _error->MergeWithStack(); } +}; +bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap) +{ + std::vector<pkgIndexFile *> Files; + if (_system->AddStatusFiles(Files) == false) + return false; + + ScopedErrorMerge sem; + std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0)); + if (unlikely(Map->validData()) == false) + return false; + map_filesize_t CurrentSize = 0; + map_filesize_t TotalSize = 0; + TotalSize = ComputeSize(NULL, Files.begin(), Files.end()); + + // Build the status cache + if (Progress != NULL) + Progress->OverallProgress(0,1,1,_("Reading package lists")); + pkgCacheGenerator Gen(Map.get(),Progress); + if (Gen.Start() == false || _error->PendingError() == true) + return false; + if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL, + Files.begin(), Files.end()) == false) + return false; + + if (_error->PendingError() == true) + return false; + *OutMap = Map.release(); + + return true; +} + /*}}}*/ +// IsDuplicateDescription /*{{{*/ +static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc, + APT::StringView CurMd5, std::string const &CurLang) +{ + // Descriptions in the same link-list have all the same md5 + if (Desc.end() == true || Cache.ViewString(Desc->md5sum) != CurMd5) + return false; + for (; Desc.end() == false; ++Desc) + if (Desc.LanguageCode() == CurLang) + return true; + return false; +} + /*}}}*/ + +pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {} +pkgCacheListParser::~pkgCacheListParser() {} diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h new file mode 100644 index 0000000..f4781d7 --- /dev/null +++ b/apt-pkg/pkgcachegen.h @@ -0,0 +1,229 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Package Cache Generator - Generator for the cache structure. + + This builds the cache structure from the abstract package list parser. + Each archive source has it's own list parser that is instantiated by + the caller to provide data for the generator. + + Parts of the cache are created by this generator class while other + parts are created by the list parser. The list parser is responsible + for creating version, depends and provides structures, and some of + their contents + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_PKGCACHEGEN_H +#define PKGLIB_PKGCACHEGEN_H + +#include <apt-pkg/macros.h> +#include <apt-pkg/mmap.h> +#include <apt-pkg/pkgcache.h> + +#include <string> +#include <vector> +#if __cplusplus >= 201103L +#include <unordered_set> +#endif +#include <apt-pkg/string_view.h> + +#ifdef APT_COMPILING_APT +#include <xxhash.h> +#endif + +class FileFd; +class pkgSourceList; +class OpProgress; +class pkgIndexFile; +class pkgCacheListParser; + +class APT_HIDDEN pkgCacheGenerator /*{{{*/ +{ + APT_HIDDEN map_stringitem_t WriteStringInMap(APT::StringView String) { return WriteStringInMap(String.data(), String.size()); }; + APT_HIDDEN map_stringitem_t WriteStringInMap(const char *String); + APT_HIDDEN map_stringitem_t WriteStringInMap(const char *String, const unsigned long &Len); + APT_HIDDEN uint32_t AllocateInMap(const unsigned long &size); + template<typename T> map_pointer<T> AllocateInMap() { + return map_pointer<T>{AllocateInMap(sizeof(T))}; + } + + // Dirty hack for public users that do not use C++11 yet +#if __cplusplus >= 201103L && defined(APT_COMPILING_APT) + struct string_pointer { + const char *data_; + size_t size; + pkgCacheGenerator *generator; + map_stringitem_t item; + + const char *data() const { + return data_ != nullptr ? data_ : static_cast<char*>(generator->Map.Data()) + item; + } + + bool operator ==(string_pointer const &other) const { + return size == other.size && memcmp(data(), other.data(), size) == 0; + } + }; + struct hash { + uint32_t operator()(string_pointer const &that) const { + return XXH3_64bits(that.data(), that.size) & 0xFFFFFFFF; + } + }; + + std::unordered_set<string_pointer, hash> strMixed; + std::unordered_set<string_pointer, hash> strVersions; + std::unordered_set<string_pointer, hash> strSections; +#endif + + friend class pkgCacheListParser; + typedef pkgCacheListParser ListParser; + + public: + + template<typename Iter> class Dynamic { + public: + static std::vector<Iter*> toReMap; + explicit Dynamic(Iter &I) { + toReMap.push_back(&I); + } + + ~Dynamic() { + toReMap.pop_back(); + } + +#if __cplusplus >= 201103L + Dynamic(const Dynamic&) = delete; + void operator=(const Dynamic&) = delete; +#endif + }; + + protected: + + DynamicMMap ⤅ + pkgCache Cache; + OpProgress *Progress; + + std::string RlsFileName; + pkgCache::ReleaseFile *CurrentRlsFile; + std::string PkgFileName; + pkgCache::PackageFile *CurrentFile; + + bool NewGroup(pkgCache::GrpIterator &Grp, APT::StringView Name); + bool NewPackage(pkgCache::PkgIterator &Pkg, APT::StringView Name, APT::StringView Arch); + map_pointer<pkgCache::Version> NewVersion(pkgCache::VerIterator &Ver, APT::StringView const &VerStr, + map_pointer<pkgCache::Package> const ParentPkg, uint32_t Hash, + map_pointer<pkgCache::Version> const Next); + map_pointer<pkgCache::Description> NewDescription(pkgCache::DescIterator &Desc,const std::string &Lang, APT::StringView md5sum,map_stringitem_t const idxmd5str); + bool NewFileVer(pkgCache::VerIterator &Ver,ListParser &List); + bool NewFileDesc(pkgCache::DescIterator &Desc,ListParser &List); + bool NewDepends(pkgCache::PkgIterator &Pkg, pkgCache::VerIterator &Ver, + map_stringitem_t const Version, uint8_t const Op, + uint8_t const Type, map_pointer<pkgCache::Dependency>* &OldDepLast); + bool NewProvides(pkgCache::VerIterator &Ver, pkgCache::PkgIterator &Pkg, + map_stringitem_t const ProvidesVersion, uint8_t const Flags); + + public: + + enum StringType { MIXED, VERSIONNUMBER, SECTION }; + map_stringitem_t StoreString(StringType const type, const char * S, unsigned int const Size); + + inline map_stringitem_t StoreString(enum StringType const type, APT::StringView S) {return StoreString(type, S.data(),S.length());}; + + void DropProgress() {Progress = 0;}; + bool SelectFile(const std::string &File,pkgIndexFile const &Index, std::string const &Architecture, std::string const &Component, unsigned long Flags = 0); + bool SelectReleaseFile(const std::string &File, const std::string &Site, unsigned long Flags = 0); + bool MergeList(ListParser &List,pkgCache::VerIterator *Ver = 0); + inline pkgCache &GetCache() {return Cache;}; + inline pkgCache::PkgFileIterator GetCurFile() + {return pkgCache::PkgFileIterator(Cache,CurrentFile);}; + inline pkgCache::RlsFileIterator GetCurRlsFile() + {return pkgCache::RlsFileIterator(Cache,CurrentRlsFile);}; + + APT_PUBLIC static bool MakeStatusCache(pkgSourceList &List,OpProgress *Progress, + MMap **OutMap = 0,bool AllowMem = false); + APT_HIDDEN static bool MakeStatusCache(pkgSourceList &List,OpProgress *Progress, + MMap **OutMap,pkgCache **OutCache, bool AllowMem = false); + APT_PUBLIC static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap); + + void ReMap(void const * const oldMap, void * const newMap, size_t oldSize); + bool Start(); + + pkgCacheGenerator(DynamicMMap *Map,OpProgress *Progress); + virtual ~pkgCacheGenerator(); + + private: + void * const d; + APT_HIDDEN bool MergeListGroup(ListParser &List, std::string const &GrpName); + APT_HIDDEN bool MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg); + APT_HIDDEN bool MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg, + APT::StringView const &Version, pkgCache::VerIterator* &OutVer); + + APT_HIDDEN bool AddImplicitDepends(pkgCache::GrpIterator &G, pkgCache::PkgIterator &P, + pkgCache::VerIterator &V); + APT_HIDDEN bool AddImplicitDepends(pkgCache::VerIterator &V, pkgCache::PkgIterator &D); + + APT_HIDDEN bool AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, + std::string const &lang, APT::StringView CurMd5, map_stringitem_t &md5idx); +}; + /*}}}*/ +// This is the abstract package list parser class. /*{{{*/ +class APT_HIDDEN pkgCacheListParser +{ + pkgCacheGenerator *Owner; + friend class pkgCacheGenerator; + + // Some cache items + pkgCache::VerIterator OldDepVer; + map_pointer<pkgCache::Dependency> *OldDepLast; + + void * const d; + + protected: + inline bool NewGroup(pkgCache::GrpIterator &Grp, APT::StringView Name) { return Owner->NewGroup(Grp, Name); } + inline map_stringitem_t StoreString(pkgCacheGenerator::StringType const type, const char *S,unsigned int Size) {return Owner->StoreString(type, S, Size);}; + inline map_stringitem_t StoreString(pkgCacheGenerator::StringType const type, APT::StringView S) {return Owner->StoreString(type, S);}; + inline map_stringitem_t WriteString(APT::StringView S) {return Owner->WriteStringInMap(S.data(), S.size());}; + + inline map_stringitem_t WriteString(const char *S,unsigned int Size) {return Owner->WriteStringInMap(S,Size);}; + bool NewDepends(pkgCache::VerIterator &Ver,APT::StringView Package, APT::StringView Arch, + APT::StringView Version,uint8_t const Op, + uint8_t const Type); + bool NewProvides(pkgCache::VerIterator &Ver,APT::StringView PkgName, + APT::StringView PkgArch, APT::StringView Version, + uint8_t const Flags); + bool NewProvidesAllArch(pkgCache::VerIterator &Ver, APT::StringView Package, + APT::StringView Version, uint8_t const Flags); + public: + + // These all operate against the current section + virtual std::string Package() = 0; + virtual bool ArchitectureAll() = 0; + virtual APT::StringView Architecture() = 0; + virtual APT::StringView Version() = 0; + virtual bool NewVersion(pkgCache::VerIterator &Ver) = 0; + virtual std::vector<std::string> AvailableDescriptionLanguages() = 0; + virtual APT::StringView Description_md5() = 0; + virtual uint32_t VersionHash() = 0; + /** compare currently parsed version with given version + * + * \param Hash of the currently parsed version + * \param Ver to compare with + */ + virtual bool SameVersion(uint32_t Hash, pkgCache::VerIterator const &Ver); + virtual bool UsePackage(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver) = 0; + virtual map_filesize_t Offset() = 0; + virtual map_filesize_t Size() = 0; + + virtual bool Step() = 0; + + virtual bool CollectFileProvides(pkgCache &/*Cache*/, + pkgCache::VerIterator &/*Ver*/) {return true;}; + + pkgCacheListParser(); + virtual ~pkgCacheListParser(); +}; + /*}}}*/ + +#endif diff --git a/apt-pkg/pkgrecords.cc b/apt-pkg/pkgrecords.cc new file mode 100644 index 0000000..77aa8ce --- /dev/null +++ b/apt-pkg/pkgrecords.cc @@ -0,0 +1,79 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Package Records - Allows access to complete package description records + directly from the file. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/error.h> +#include <apt-pkg/indexfile.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgrecords.h> + +#include <vector> +#include <stddef.h> + +#include <apti18n.h> + /*}}}*/ + +// Records::pkgRecords - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* This will create the necessary structures to access the status files */ +pkgRecords::pkgRecords(pkgCache &aCache) : d(NULL), Cache(aCache), + Files(Cache.HeaderP->PackageFileCount) +{ + for (pkgCache::PkgFileIterator I = Cache.FileBegin(); + I.end() == false; ++I) + { + const pkgIndexFile::Type *Type = pkgIndexFile::Type::GetType(I.IndexType()); + if (Type == 0) + { + _error->Error(_("Index file type '%s' is not supported"),I.IndexType()); + return; + } + + Files[I->ID] = Type->CreatePkgParser(I); + if (Files[I->ID] == 0) + return; + } +} + /*}}}*/ +// Records::~pkgRecords - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgRecords::~pkgRecords() +{ + for ( std::vector<Parser*>::iterator it = Files.begin(); + it != Files.end(); + ++it) + { + delete *it; + } +} + /*}}}*/ +// Records::Lookup - Get a parser for the package version file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgRecords::Parser &pkgRecords::Lookup(pkgCache::VerFileIterator const &Ver) +{ + Files[Ver.File()->ID]->Jump(Ver); + return *Files[Ver.File()->ID]; +} + /*}}}*/ +// Records::Lookup - Get a parser for the package description file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgRecords::Parser &pkgRecords::Lookup(pkgCache::DescFileIterator const &Desc) +{ + Files[Desc.File()->ID]->Jump(Desc); + return *Files[Desc.File()->ID]; +} + /*}}}*/ + +pkgRecords::Parser::Parser() : d(NULL) {} +pkgRecords::Parser::~Parser() {} diff --git a/apt-pkg/pkgrecords.h b/apt-pkg/pkgrecords.h new file mode 100644 index 0000000..09c0b5f --- /dev/null +++ b/apt-pkg/pkgrecords.h @@ -0,0 +1,111 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Package Records - Allows access to complete package description records + directly from the file. + + The package record system abstracts the actual parsing of the + package files. This is different than the generators parser in that + it is used to access information not generate information. No + information touched by the generator should be parable from here as + it can always be retrieved directly from the cache. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_PKGRECORDS_H +#define PKGLIB_PKGRECORDS_H + +#include <apt-pkg/hashes.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> + +#include <string> +#include <vector> + +class APT_PUBLIC pkgRecords /*{{{*/ +{ + public: + class Parser; + + private: + /** \brief dpointer placeholder (for later in case we need it) */ + void * const d; + + pkgCache &Cache; + std::vector<Parser *>Files; + + public: + // Lookup function + Parser &Lookup(pkgCache::VerFileIterator const &Ver); + Parser &Lookup(pkgCache::DescFileIterator const &Desc); + + // Construct destruct + explicit pkgRecords(pkgCache &Cache); + virtual ~pkgRecords(); +}; + /*}}}*/ +class APT_PUBLIC pkgRecords::Parser /*{{{*/ +{ + protected: + + virtual bool Jump(pkgCache::VerFileIterator const &Ver) = 0; + virtual bool Jump(pkgCache::DescFileIterator const &Desc) = 0; + + public: + friend class pkgRecords; + + // These refer to the archive file for the Version + virtual std::string FileName() {return std::string();}; + virtual std::string SourcePkg() {return std::string();}; + virtual std::string SourceVer() {return std::string();}; + + /** return all known hashes in this record. + * + * For authentication proposes packages come with hashsums which + * this method is supposed to parse and return so that clients can + * choose the hash to be used. + */ + virtual HashStringList Hashes() const { return HashStringList(); }; + + // These are some general stats about the package + virtual std::string Maintainer() {return std::string();}; + /** return short description in language from record. + * + * @see #LongDesc + */ + virtual std::string ShortDesc(std::string const &/*lang*/) {return std::string();}; + /** return long description in language from record. + * + * If \b lang is empty the "best" available language will be + * returned as determined by the APT::Languages configuration. + * If a (requested) language can't be found in this record an empty + * string will be returned. + */ + virtual std::string LongDesc(std::string const &/*lang*/) {return std::string();}; + std::string ShortDesc() {return ShortDesc("");}; + std::string LongDesc() {return LongDesc("");}; + + virtual std::string Name() {return std::string();}; + virtual std::string Homepage() {return std::string();} + + // An arbitrary custom field + virtual std::string RecordField(const char * /*fieldName*/) { return std::string();}; + + // The record in binary form + virtual void GetRec(const char *&Start,const char *&Stop) {Start = Stop = 0;}; + + Parser(); + virtual ~Parser(); + + private: + void * const d; + APT_HIDDEN std::string GetHashFromHashes(char const * const type) const + { + HashStringList const hashes = Hashes(); + HashString const * const hs = hashes.find(type); + return hs != NULL ? hs->HashValue() : ""; + }; +}; + /*}}}*/ +#endif diff --git a/apt-pkg/pkgsystem.cc b/apt-pkg/pkgsystem.cc new file mode 100644 index 0000000..7e48a68 --- /dev/null +++ b/apt-pkg/pkgsystem.cc @@ -0,0 +1,72 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + System - Abstraction for running on different systems. + + Basic general structure.. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/debsystem.h> +#include <apt-pkg/error.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgsystem.h> + +#include <cassert> +#include <cstring> +#include <map> + /*}}}*/ + +pkgSystem *_system = 0; +static pkgSystem *SysList[10]; +pkgSystem **pkgSystem::GlobalList = SysList; +unsigned long pkgSystem::GlobalListLen = 0; + +class APT_HIDDEN pkgSystemPrivate /*{{{*/ +{ +public: + typedef decltype(pkgCache::Version::ID) idtype; + std::map<idtype,idtype> idmap; + pkgSystemPrivate() {} +}; + /*}}}*/ +// System::pkgSystem - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Add it to the global list.. */ +pkgSystem::pkgSystem(char const * const label, pkgVersioningSystem * const vs) : + Label(label), VS(vs), d(new pkgSystemPrivate()) +{ + assert(GlobalListLen < sizeof(SysList)/sizeof(*SysList)); + SysList[GlobalListLen] = this; + ++GlobalListLen; +} + /*}}}*/ +// System::GetSystem - Get the named system /*{{{*/ +// --------------------------------------------------------------------- +/* */ +APT_PURE pkgSystem *pkgSystem::GetSystem(const char *Label) +{ + for (unsigned I = 0; I != GlobalListLen; I++) + if (strcmp(SysList[I]->Label,Label) == 0) + return SysList[I]; + return 0; +} + /*}}}*/ +// pkgSystem::Set/GetVersionMapping - for internal/external communication/*{{{*/ +void pkgSystem::SetVersionMapping(map_id_t const in, map_id_t const out) +{ + if (in == out) + return; + d->idmap.emplace(in, out); +} +map_id_t pkgSystem::GetVersionMapping(map_id_t const in) const +{ + auto const o = d->idmap.find(in); + return (o == d->idmap.end()) ? in : o->second; +} + /*}}}*/ +pkgSystem::~pkgSystem() {} diff --git a/apt-pkg/pkgsystem.h b/apt-pkg/pkgsystem.h new file mode 100644 index 0000000..10065d0 --- /dev/null +++ b/apt-pkg/pkgsystem.h @@ -0,0 +1,139 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + System - Abstraction for running on different systems. + + Instances of this class can be thought of as factories or meta-classes + for a variety of more specialized classes. Together this class and + it's specialized offspring completely define the environment and how + to access resources for a specific system. There are several sub + areas that are all orthogonal - each system has a unique combination of + these sub areas: + - Versioning. Different systems have different ideas on versions. + Within a system all sub classes must follow the same versioning + rules. + - Local tool locking to prevent multiple tools from accessing the + same database. + - Candidate Version selection policy - this is probably almost always + managed using a standard APT class + - Actual Package installation + * Indication of what kind of binary formats are supported + - Selection of local 'status' indexes that make up the pkgCache. + + It is important to note that the handling of index files is not a + function of the system. Index files are handled through a separate + abstraction - the only requirement is that the index files have the + same idea of versioning as the target system. + + Upon startup each supported system instantiates an instance of the + pkgSystem class (using a global constructor) which will make itself + available to the main APT init routine. That routine will select the + proper system and make it the global default. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_PKGSYSTEM_H +#define PKGLIB_PKGSYSTEM_H + +#include <apt-pkg/pkgcache.h> + +#include <vector> + + +class pkgDepCache; +class pkgPackageManager; +class pkgVersioningSystem; +class Configuration; +class pkgIndexFile; +class OpProgress; + +class pkgSystemPrivate; +class APT_PUBLIC pkgSystem +{ + public: + + // Global list of supported systems + static pkgSystem **GlobalList; + static unsigned long GlobalListLen; + static pkgSystem *GetSystem(const char *Label); + + const char * const Label; + pkgVersioningSystem * const VS; + + /* Prevent other programs from touching shared data not covered by + other locks (cache or state locks) */ + virtual bool Lock(OpProgress *const Progress = nullptr) = 0; + virtual bool UnLock(bool NoErrors = false) = 0; + + /* Various helper classes to interface with specific bits of this + environment */ + virtual pkgPackageManager *CreatePM(pkgDepCache *Cache) const = 0; + + /* Load environment specific configuration and perform any other setup + necessary */ + virtual bool Initialize(Configuration &/*Cnf*/) {return true;}; + + /* Type is some kind of Globally Unique way of differentiating + archive file types.. */ + virtual bool ArchiveSupported(const char *Type) = 0; + + // Return a list of system index files.. + virtual bool AddStatusFiles(std::vector<pkgIndexFile *> &List) = 0; + + virtual bool FindIndex(pkgCache::PkgFileIterator File, + pkgIndexFile *&Found) const = 0; + + /* Evaluate how 'right' we are for this system based on the filesystem + etc.. */ + virtual signed Score(Configuration const &/*Cnf*/) { + return 0; + }; + + //FIXME: these methods should be virtual + /** does this system has support for MultiArch? + * + * Systems supporting only single arch (not systems which are single arch) + * are considered legacy systems and support for it will likely degrade over + * time. + * + * The default implementation returns always \b true. + * + * @return \b true if the system supports MultiArch, \b false if not. + */ + virtual bool MultiArchSupported() const = 0; + /** architectures supported by this system + * + * A MultiArch capable system might be configured to use + * this capability. + * + * @return a list of all architectures (native + foreign) configured + * for on this system (aka: which can be installed without force) + */ + virtual std::vector<std::string> ArchitecturesSupported() const = 0; + + APT_HIDDEN void SetVersionMapping(map_id_t const in, map_id_t const out); + APT_HIDDEN map_id_t GetVersionMapping(map_id_t const in) const; + + pkgSystem(char const * const Label, pkgVersioningSystem * const VS); + virtual ~pkgSystem(); + + + /* companions to Lock()/UnLock + * + * These functions can be called prior to calling dpkg to release an inner + * lock without releasing the overall outer lock, so that dpkg can run + * correctly but no other APT instance can acquire the system lock. + */ + virtual bool LockInner(OpProgress *const Progress = 0, int timeOutSec = 0) = 0; + virtual bool UnLockInner(bool NoErrors = false) = 0; + /// checks if the system is currently locked + virtual bool IsLocked() = 0; + private: + pkgSystemPrivate * const d; +}; + +// The environment we are operating in. +APT_PUBLIC extern pkgSystem *_system; + +#endif diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc new file mode 100644 index 0000000..68b3f5e --- /dev/null +++ b/apt-pkg/policy.cc @@ -0,0 +1,510 @@ + // -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Package Version Policy implementation + + This is just a really simple wrapper around pkgVersionMatch with + some added goodies to manage the list of things.. + + See man apt_preferences for what value means what. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/cachefilter.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/policy.h> +#include <apt-pkg/string_view.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile-keys.h> +#include <apt-pkg/tagfile.h> +#include <apt-pkg/version.h> +#include <apt-pkg/versionmatch.h> + +#include <iostream> +#include <random> +#include <sstream> +#include <string> +#include <vector> +#include <ctype.h> +#include <stddef.h> +#include <string.h> + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +constexpr short NEVER_PIN = std::numeric_limits<short>::min(); + +struct pkgPolicy::Private +{ + std::string machineID; +}; + +// Policy::Init - Startup and bind to a cache /*{{{*/ +// --------------------------------------------------------------------- +/* Set the defaults for operation. The default mode with no loaded policy + file matches the V0 policy engine. */ +pkgPolicy::pkgPolicy(pkgCache *Owner) : VerPins(nullptr), + PFPriority(nullptr), Cache(Owner), d(new Private) +{ + if (Owner == 0) + return; + PFPriority = new signed short[Owner->Head().PackageFileCount]; + VerPins = new Pin[Owner->Head().VersionCount]; + + auto VersionCount = Owner->Head().VersionCount; + for (decltype(VersionCount) I = 0; I != VersionCount; ++I) + VerPins[I].Type = pkgVersionMatch::None; + + // The config file has a master override. + string DefRel = _config->Find("APT::Default-Release"); + if (DefRel.empty() == false) + { + bool found = false; + for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F) + { + if (pkgVersionMatch::ExpressionMatches(DefRel, F.Archive()) || + pkgVersionMatch::ExpressionMatches(DefRel, F.Codename()) || + pkgVersionMatch::ExpressionMatches(DefRel, F.Version()) || + (DefRel.length() > 2 && DefRel[1] == '=')) + found = true; + } + // "now" is our internal archive name for the status file, + // which we should accept even if we have no status file at the moment + if (not found && pkgVersionMatch::ExpressionMatches(DefRel, "now")) + found = true; + if (not found) + _error->Error(_("The value '%s' is invalid for APT::Default-Release as such a release is not available in the sources"), DefRel.c_str()); + else + CreatePin(pkgVersionMatch::Release,"",DefRel,990); + } + InitDefaults(); + + d->machineID = APT::Configuration::getMachineID(); +} + /*}}}*/ +// Policy::InitDefaults - Compute the default selections /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgPolicy::InitDefaults() +{ + // Initialize the priorities based on the status of the package file + for (pkgCache::PkgFileIterator I = Cache->FileBegin(); I != Cache->FileEnd(); ++I) + { + PFPriority[I->ID] = 500; + if (I.Flagged(pkgCache::Flag::NotSource)) + PFPriority[I->ID] = 100; + else if (I.Flagged(pkgCache::Flag::ButAutomaticUpgrades)) + PFPriority[I->ID] = 100; + else if (I.Flagged(pkgCache::Flag::NotAutomatic)) + PFPriority[I->ID] = 1; + } + + // Apply the defaults.. + std::unique_ptr<bool[]> Fixed(new bool[Cache->HeaderP->PackageFileCount]); + memset(Fixed.get(),0,sizeof(Fixed[0])*Cache->HeaderP->PackageFileCount); + StatusOverride = false; + for (vector<Pin>::const_iterator I = Defaults.begin(); I != Defaults.end(); ++I) + { + pkgVersionMatch Match(I->Data,I->Type); + for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F) + { + if ((Fixed[F->ID] == false || I->Priority == NEVER_PIN) && PFPriority[F->ID] != NEVER_PIN && Match.FileMatch(F) == true) + { + PFPriority[F->ID] = I->Priority; + + if (PFPriority[F->ID] >= 1000) + StatusOverride = true; + + Fixed[F->ID] = true; + } + } + } + + if (_config->FindB("Debug::pkgPolicy",false) == true) + for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F) + std::clog << "Prio of " << F.FileName() << ' ' << PFPriority[F->ID] << std::endl; + + return true; +} + /*}}}*/ +// Policy::GetCandidateVer - Get the candidate install version /*{{{*/ +// --------------------------------------------------------------------- +/* Evaluate the package pins and the default list to determine what the + best package is. */ +pkgCache::VerIterator pkgPolicy::GetCandidateVer(pkgCache::PkgIterator const &Pkg) +{ + pkgCache::VerIterator cand; + pkgCache::VerIterator cur = Pkg.CurrentVer(); + int candPriority = -1; + pkgVersioningSystem *vs = Cache->VS; + + for (pkgCache::VerIterator ver = Pkg.VersionList(); ver.end() == false; ++ver) { + int priority = GetPriority(ver, true); + + if (priority == 0 || priority <= candPriority) + continue; + + // TODO: Maybe optimize to not compare versions + if (!cur.end() && priority < 1000 + && (vs->CmpVersion(ver.VerStr(), cur.VerStr()) < 0)) + continue; + + candPriority = priority; + cand = ver; + } + + return cand; +} + /*}}}*/ +// Policy::CreatePin - Create an entry in the pin table.. /*{{{*/ +// --------------------------------------------------------------------- +/* For performance we have 3 tables, the default table, the main cache + table (hashed to the cache). A blank package name indicates the pin + belongs to the default table. Order of insertion matters here, the + earlier defaults override later ones. */ +void pkgPolicy::CreatePin(pkgVersionMatch::MatchType Type,string Name, + string Data,signed short Priority) +{ + if (Name.empty() == true) + { + Pin *P = &*Defaults.insert(Defaults.end(),Pin()); + P->Type = Type; + P->Priority = Priority; + P->Data = Data; + return; + } + + bool IsSourcePin = APT::String::Startswith(Name, "src:"); + if (IsSourcePin) { + Name = Name.substr(sizeof("src:") - 1); + } + + size_t found = Name.rfind(':'); + string Arch; + if (found != string::npos) { + Arch = Name.substr(found+1); + Name.erase(found); + } + + // Allow pinning by wildcards - beware of package names looking like wildcards! + // TODO: Maybe we should always prefer specific pins over non-specific ones. + if ((Name[0] == '/' && Name[Name.length() - 1] == '/') || Name.find_first_of("*[?") != string::npos) + { + pkgVersionMatch match(Data, Type); + for (pkgCache::GrpIterator G = Cache->GrpBegin(); G.end() != true; ++G) + if (Name != G.Name() && match.ExpressionMatches(Name, G.Name())) + { + auto NameToPinFor = IsSourcePin ? string("src:").append(G.Name()) : string(G.Name()); + if (Arch.empty() == false) + CreatePin(Type, NameToPinFor.append(":").append(Arch), Data, Priority); + else + CreatePin(Type, NameToPinFor, Data, Priority); + } + return; + } + + // find the package (group) this pin applies to + pkgCache::GrpIterator Grp = Cache->FindGrp(Name); + bool matched = false; + if (Grp.end() == false) + { + std::string MatchingArch; + if (Arch.empty() == true) + MatchingArch = Cache->NativeArch(); + else + MatchingArch = Arch; + APT::CacheFilter::PackageArchitectureMatchesSpecification pams(MatchingArch); + + if (IsSourcePin) { + for (pkgCache::VerIterator Ver = Grp.VersionsInSource(); not Ver.end(); Ver = Ver.NextInSource()) + { + if (pams(Ver.ParentPkg().Arch()) == false) + continue; + + PkgPin P(Ver.ParentPkg().FullName()); + P.Type = Type; + P.Priority = Priority; + P.Data = Data; + // Find matching version(s) and copy the pin into it + pkgVersionMatch Match(P.Data,P.Type); + if (Match.VersionMatches(Ver)) { + Pin *VP = VerPins + Ver->ID; + if (VP->Type == pkgVersionMatch::None) { + *VP = P; + matched = true; + } + } + } + } else { + for (pkgCache::PkgIterator Pkg = Grp.PackageList(); Pkg.end() != true; Pkg = Grp.NextPkg(Pkg)) + { + if (pams(Pkg.Arch()) == false) + continue; + + PkgPin P(Pkg.FullName()); + P.Type = Type; + P.Priority = Priority; + P.Data = Data; + + // Find matching version(s) and copy the pin into it + pkgVersionMatch Match(P.Data,P.Type); + for (pkgCache::VerIterator Ver = Pkg.VersionList(); Ver.end() != true; ++Ver) + { + if (Match.VersionMatches(Ver)) { + Pin *VP = VerPins + Ver->ID; + if (VP->Type == pkgVersionMatch::None) { + *VP = P; + matched = true; + } + } + } + } + } + } + + if (matched == false) + { + PkgPin *P = &*Unmatched.insert(Unmatched.end(),PkgPin(Name)); + if (Arch.empty() == false) + P->Pkg.append(":").append(Arch); + P->Type = Type; + P->Priority = Priority; + P->Data = Data; + return; + } +} + /*}}}*/ +// Policy::GetPriority - Get the priority of the package pin /*{{{*/ +// --------------------------------------------------------------------- +/* */ +// Returns true if this update is excluded by phasing. +static inline bool ExcludePhased(std::string machineID, pkgCache::VerIterator const &Ver) +{ + if (Ver.PhasedUpdatePercentage() == 100) + return false; + + // FIXME: We have migrated to a legacy implementation until LP: #1929082 is fixed + if (not _config->FindB("APT::Get::Phase-Policy", false)) + return false; + + // The order and fallbacks for the always/never checks come from update-manager and exist + // to preserve compatibility. + if (_config->FindB("APT::Get::Always-Include-Phased-Updates", + _config->FindB("Update-Manager::Always-Include-Phased-Updates", false))) + return false; + + if (_config->FindB("APT::Get::Never-Include-Phased-Updates", + _config->FindB("Update-Manager::Never-Include-Phased-Updates", false))) + return true; + + if (machineID.empty() // no machine-id + || getenv("SOURCE_DATE_EPOCH") != nullptr // reproducible build - always include + || APT::Configuration::isChroot()) + return false; + + std::string seedStr = std::string(Ver.SourcePkgName()) + "-" + Ver.SourceVerStr() + "-" + machineID; + std::seed_seq seed(seedStr.begin(), seedStr.end()); + std::minstd_rand rand(seed); + std::uniform_int_distribution<unsigned int> dist(0, 100); + + return dist(rand) > Ver.PhasedUpdatePercentage(); +} +APT_PURE signed short pkgPolicy::GetPriority(pkgCache::VerIterator const &Ver, bool ConsiderFiles) +{ + auto ceiling = std::numeric_limits<signed int>::max(); + if (ExcludePhased(d->machineID, Ver)) + ceiling = 1; + if (VerPins[Ver->ID].Type != pkgVersionMatch::None) + { + // If all sources are never pins, the never pin wins. + if (VerPins[Ver->ID].Priority == NEVER_PIN) + return NEVER_PIN; + for (pkgCache::VerFileIterator file = Ver.FileList(); file.end() == false; file++) + if (GetPriority(file.File()) != NEVER_PIN) + return std::min((int)VerPins[Ver->ID].Priority, ceiling); + } + if (!ConsiderFiles) + return std::min(0, ceiling); + + // priorities are short ints, but we want to pick a value outside the valid range here + auto priority = std::numeric_limits<signed int>::min(); + for (pkgCache::VerFileIterator file = Ver.FileList(); file.end() == false; file++) + { + /* If this is the status file, and the current version is not the + version in the status file (ie it is not installed, or somesuch) + then it is not a candidate for installation, ever. This weeds + out bogus entries that may be due to config-file states, or + other. */ + if (file.File().Flagged(pkgCache::Flag::NotSource) && Ver.ParentPkg().CurrentVer() != Ver) + priority = std::max<decltype(priority)>(priority, -1); + else + priority = std::max<decltype(priority)>(priority, GetPriority(file.File())); + } + + return std::min(priority == std::numeric_limits<decltype(priority)>::min() ? 0 : priority, ceiling); +} +APT_PURE signed short pkgPolicy::GetPriority(pkgCache::PkgFileIterator const &File) +{ + return PFPriority[File->ID]; +} + /*}}}*/ +// SetPriority - Directly set priority /*{{{*/ +// --------------------------------------------------------------------- +void pkgPolicy::SetPriority(pkgCache::VerIterator const &Ver, signed short Priority) +{ + Pin pin; + pin.Data = "pkgPolicy::SetPriority"; + pin.Priority = Priority; + VerPins[Ver->ID] = pin; +} +void pkgPolicy::SetPriority(pkgCache::PkgFileIterator const &File, signed short Priority) +{ + PFPriority[File->ID] = Priority; +} + + /*}}}*/ +// ReadPinDir - Load the pin files from this dir into a Policy /*{{{*/ +// --------------------------------------------------------------------- +/* This will load each pin file in the given dir into a Policy. If the + given dir is empty the dir set in Dir::Etc::PreferencesParts is used. + Note also that this method will issue a warning if the dir does not + exists but it will return true in this case! */ +bool ReadPinDir(pkgPolicy &Plcy,string Dir) +{ + if (Dir.empty() == true) + Dir = _config->FindDir("Dir::Etc::PreferencesParts", "/dev/null"); + + if (DirectoryExists(Dir) == false) + { + if (APT::String::Endswith(Dir, "/dev/null") == false) + _error->WarningE("DirectoryExists",_("Unable to read %s"),Dir.c_str()); + return true; + } + + _error->PushToStack(); + vector<string> const List = GetListOfFilesInDir(Dir, "pref", true, true); + bool const PendingErrors = _error->PendingError(); + _error->MergeWithStack(); + if (PendingErrors) + return false; + + // Read the files + bool good = true; + for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I) + good = ReadPinFile(Plcy, *I) && good; + return good; +} + /*}}}*/ +// ReadPinFile - Load the pin file into a Policy /*{{{*/ +// --------------------------------------------------------------------- +/* I'd like to see the preferences file store more than just pin information + but right now that is the only stuff I have to store. Later there will + have to be some kind of combined super parser to get the data into all + the right classes.. */ +bool ReadPinFile(pkgPolicy &Plcy,string File) +{ + if (File.empty() == true) + File = _config->FindFile("Dir::Etc::Preferences"); + + if (RealFileExists(File) == false) + return true; + + FileFd Fd; + if (OpenConfigurationFileFd(File, Fd) == false) + return false; + + pkgTagFile TF(&Fd, pkgTagFile::SUPPORT_COMMENTS); + if (Fd.IsOpen() == false || Fd.Failed()) + return false; + + pkgTagSection Tags; + while (TF.Step(Tags) == true) + { + // can happen when there are only comments in a record + if (Tags.Count() == 0) + continue; + + auto Name = Tags.Find(pkgTagSection::Key::Package); + if (Name.empty()) + return _error->Error(_("Invalid record in the preferences file %s, no Package header"), File.c_str()); + if (Name == "*") + Name = APT::StringView{}; + + const char *Start; + const char *End; + if (Tags.Find("Pin",Start,End) == false) + continue; + + const char *Word = Start; + for (; Word != End && isspace(*Word) == 0; Word++); + + // Parse the type.. + pkgVersionMatch::MatchType Type; + if (stringcasecmp(Start,Word,"version") == 0 && Name.empty() == false) + Type = pkgVersionMatch::Version; + else if (stringcasecmp(Start,Word,"release") == 0) + Type = pkgVersionMatch::Release; + else if (stringcasecmp(Start,Word,"origin") == 0) + Type = pkgVersionMatch::Origin; + else + { + _error->Warning(_("Did not understand pin type %s"),string(Start,Word).c_str()); + continue; + } + for (; Word != End && isspace(*Word) != 0; Word++); + + _error->PushToStack(); + std::string sPriority = Tags.FindS("Pin-Priority"); + int priority = sPriority == "never" ? NEVER_PIN : Tags.FindI("Pin-Priority", 0); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + + if (sPriority == "never" && not Name.empty()) + return _error->Error(_("%s: The special 'Pin-Priority: %s' can only be used for 'Package: *' records"), File.c_str(), "never"); + + // Silently clamp the never pin to never pin + 1 + if (priority == NEVER_PIN && sPriority != "never") + priority = NEVER_PIN + 1; + if (priority < std::numeric_limits<short>::min() || + priority > std::numeric_limits<short>::max() || + newError) { + return _error->Error(_("%s: Value %s is outside the range of valid pin priorities (%d to %d)"), + File.c_str(), Tags.FindS("Pin-Priority").c_str(), + std::numeric_limits<short>::min(), + std::numeric_limits<short>::max()); + } + if (priority == 0) + { + return _error->Error(_("No priority (or zero) specified for pin")); + } + + std::istringstream s(Name.to_string()); + string pkg; + while(!s.eof()) + { + s >> pkg; + Plcy.CreatePin(Type, pkg, string(Word,End),priority); + }; + } + + Plcy.InitDefaults(); + return true; +} + /*}}}*/ + +pkgPolicy::~pkgPolicy() +{ + delete[] PFPriority; + delete[] VerPins; + delete d; +} diff --git a/apt-pkg/policy.h b/apt-pkg/policy.h new file mode 100644 index 0000000..589cebc --- /dev/null +++ b/apt-pkg/policy.h @@ -0,0 +1,93 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Package Version Policy implementation + + This implements the more advanced 'Version 4' APT policy engine. The + standard 'Version 0' engine is included inside the DepCache which is + it's historical location. + + The V4 engine allows the user to completely control all aspects of + version selection. There are three primary means to choose a version + * Selection by version match + * Selection by Release file match + * Selection by origin server + + Each package may be 'pinned' with a single criteria, which will ultimately + result in the selection of a single version, or no version, for each + package. + + Furthermore, the default selection can be influenced by specifying + the ordering of package files. The order is derived by reading the + package file preferences and assigning a priority to each package + file. + + A special flag may be set to indicate if no version should be returned + if no matching versions are found, otherwise the default matching + rules are used to locate a hit. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_POLICY_H +#define PKGLIB_POLICY_H + +#include <apt-pkg/depcache.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/versionmatch.h> + +#include <string> +#include <vector> + + +class APT_PUBLIC pkgPolicy : public pkgDepCache::Policy +{ + protected: + + struct Pin + { + pkgVersionMatch::MatchType Type; + std::string Data; + signed short Priority; + Pin() : Type(pkgVersionMatch::None), Priority(0) {}; + }; + + struct PkgPin : Pin + { + std::string Pkg; + explicit PkgPin(std::string const &Pkg) : Pin(), Pkg(Pkg) {}; + }; + + Pin *VerPins; + signed short *PFPriority; + std::vector<Pin> Defaults; + std::vector<PkgPin> Unmatched; + pkgCache *Cache; + bool StatusOverride; + + public: + + // Things for manipulating pins + void CreatePin(pkgVersionMatch::MatchType Type,std::string Pkg, + std::string Data,signed short Priority); + + // Things for the cache interface. + virtual pkgCache::VerIterator GetCandidateVer(pkgCache::PkgIterator const &Pkg) APT_OVERRIDE; + virtual signed short GetPriority(pkgCache::VerIterator const &Ver, bool ConsiderFiles = true) APT_OVERRIDE; + virtual signed short GetPriority(pkgCache::PkgFileIterator const &File) APT_OVERRIDE; + + void SetPriority(pkgCache::VerIterator const &Ver, signed short Priority); + void SetPriority(pkgCache::PkgFileIterator const &File, signed short Priority); + bool InitDefaults(); + + explicit pkgPolicy(pkgCache *Owner); + virtual ~pkgPolicy(); + private: + struct Private; + Private *const d; +}; + +APT_PUBLIC bool ReadPinFile(pkgPolicy &Plcy, std::string File = ""); +APT_PUBLIC bool ReadPinDir(pkgPolicy &Plcy, std::string Dir = ""); + +#endif diff --git a/apt-pkg/prettyprinters.cc b/apt-pkg/prettyprinters.cc new file mode 100644 index 0000000..87e78bb --- /dev/null +++ b/apt-pkg/prettyprinters.cc @@ -0,0 +1,123 @@ +// Description /*{{{*/ +/* ###################################################################### + + Provide pretty printers for pkgCache structs like PkgIterator + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/depcache.h> +#include <apt-pkg/prettyprinters.h> + +#include <ostream> +#include <string> + + /*}}}*/ + +std::ostream& operator<<(std::ostream& os, const APT::PrettyPkg& pp) /*{{{*/ +{ + if (pp.Pkg.end() == true) + return os << "invalid package"; + + auto state = (*pp.DepCache)[pp.Pkg]; + std::string const current = (pp.Pkg.CurVersion() == 0 ? "none" : pp.Pkg.CurVersion()); + std::string candidate = state.CandVersion; + if (candidate.empty()) + candidate = "none"; + std::string install = "none"; + if (state.InstallVer != nullptr) + install = state.InstVerIter(*pp.DepCache).VerStr(); + + os << pp.Pkg.FullName(false) << " < " << current; + if (current != install && install != "none") + os << " -> " << install; + if (install != candidate && current != candidate) + os << " | " << candidate; + os << " @"; + switch (pp.Pkg->SelectedState) + { + case pkgCache::State::Unknown: os << 'u'; break; + case pkgCache::State::Install: os << 'i'; break; + case pkgCache::State::Hold: os << 'h'; break; + case pkgCache::State::DeInstall: os << 'r'; break; + case pkgCache::State::Purge: os << 'p'; break; + default: os << 'X'; + } + switch (pp.Pkg->InstState) + { + case pkgCache::State::Ok: break; + case pkgCache::State::ReInstReq: os << 'R'; break; + case pkgCache::State::HoldInst: os << 'H'; break; + case pkgCache::State::HoldReInstReq: os << "HR"; break; + default: os << 'X'; + } + switch (pp.Pkg->CurrentState) + { + case pkgCache::State::NotInstalled: os << 'n'; break; + case pkgCache::State::ConfigFiles: os << 'c'; break; + case pkgCache::State::HalfInstalled: os << 'H'; break; + case pkgCache::State::UnPacked: os << 'U'; break; + case pkgCache::State::HalfConfigured: os << 'F'; break; + case pkgCache::State::TriggersAwaited: os << 'W'; break; + case pkgCache::State::TriggersPending: os << 'T'; break; + case pkgCache::State::Installed: os << 'i'; break; + default: os << 'X'; + } + os << ' '; + if (state.Protect()) + os << "p"; + if (state.ReInstall()) + os << "r"; + if (state.Upgradable()) + os << "u"; + if (state.Marked) + os << "m"; + if (state.Garbage) + os << "g"; + if (state.NewInstall()) + os << "N"; + else if (state.Upgrade()) + os << "U"; + else if (state.Downgrade()) + os << "D"; + else if (state.Install()) + os << "I"; + else if (state.Purge()) + os << "P"; + else if (state.Delete()) + os << "R"; + else if (state.Held()) + os << "H"; + else if (state.Keep()) + os << "K"; + if (state.NowBroken()) + os << " Nb"; + else if (state.NowPolicyBroken()) + os << " NPb"; + if (state.InstBroken()) + os << " Ib"; + else if (state.InstPolicyBroken()) + os << " IPb"; + os << " >"; + return os; +} + /*}}}*/ +std::ostream& operator<<(std::ostream& os, const APT::PrettyDep& pd) /*{{{*/ +{ + if (unlikely(pd.Dep.end() == true)) + return os << "invalid dependency"; + + pkgCache::PkgIterator P = pd.Dep.ParentPkg(); + pkgCache::PkgIterator T = pd.Dep.TargetPkg(); + + os << (P.end() ? "invalid pkg" : P.FullName(false)) << " " << pd.Dep.DepType() + << " on " << APT::PrettyPkg(pd.DepCache, T); + + if (pd.Dep->Version != 0) + os << " (" << pd.Dep.CompType() << " " << pd.Dep.TargetVer() << ")"; + + return os; +} + /*}}}*/ diff --git a/apt-pkg/prettyprinters.h b/apt-pkg/prettyprinters.h new file mode 100644 index 0000000..80b7729 --- /dev/null +++ b/apt-pkg/prettyprinters.h @@ -0,0 +1,37 @@ +#ifndef APT_PRETTYPRINTERS_H +#define APT_PRETTYPRINTERS_H +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> + +class pkgDepCache; + +namespace APT { + +/** helper to format PkgIterator for easier printing in debug messages. + * + * The actual text generated is subject to change without prior notice + * and should NOT be used as part of a general user interface. + */ +struct PrettyPkg +{ + pkgDepCache * const DepCache; + pkgCache::PkgIterator const Pkg; + PrettyPkg(pkgDepCache * const depcache, pkgCache::PkgIterator const &pkg) APT_NONNULL(2) : DepCache(depcache), Pkg(pkg) {} +}; +/** helper to format DepIterator for easier printing in debug messages. + * + * The actual text generated is subject to change without prior notice + * and should NOT be used as part of a general user interface. + */ +struct PrettyDep +{ + pkgDepCache * const DepCache; + pkgCache::DepIterator const Dep; + PrettyDep(pkgDepCache * const depcache, pkgCache::DepIterator const &dep) APT_NONNULL(2) : DepCache(depcache), Dep(dep) {} +}; + +} +APT_PUBLIC std::ostream& operator<<(std::ostream& os, const APT::PrettyPkg& pp); +APT_PUBLIC std::ostream& operator<<(std::ostream& os, const APT::PrettyDep& pd); + +#endif diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc new file mode 100644 index 0000000..0ac59fc --- /dev/null +++ b/apt-pkg/sourcelist.cc @@ -0,0 +1,626 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + List of Sources + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/cmndline.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/debindexfile.h> +#include <apt-pkg/debsrcrecords.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/indexfile.h> +#include <apt-pkg/metaindex.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/sourcelist.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile.h> + +#include <algorithm> +#include <cstring> +#include <fstream> +#include <map> +#include <string> +#include <vector> +#include <ctype.h> +#include <stddef.h> +#include <time.h> + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +// Global list of Items supported +static pkgSourceList::Type *ItmList[10]; +pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList; +unsigned long pkgSourceList::Type::GlobalListLen = 0; + +static std::vector<std::string> FindMultiValue(pkgTagSection &Tags, char const *const Field) /*{{{*/ +{ + auto values = Tags.FindS(Field); + // we ignore duplicate spaces by removing empty values + std::replace_if(values.begin(), values.end(), isspace_ascii, ' '); + auto vect = VectorizeString(values, ' '); + vect.erase(std::remove_if(vect.begin(), vect.end(), [](std::string const &s) { return s.empty(); }), vect.end()); + return vect; +} + /*}}}*/ + +// Type::Type - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Link this to the global list of items*/ +pkgSourceList::Type::Type(char const * const pName, char const * const pLabel) : Name(pName), Label(pLabel) +{ + ItmList[GlobalListLen] = this; + ++GlobalListLen; +} +pkgSourceList::Type::~Type() {} + /*}}}*/ +// Type::GetType - Get a specific meta for a given type /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgSourceList::Type *pkgSourceList::Type::GetType(const char *Type) +{ + for (unsigned I = 0; I != GlobalListLen; ++I) + if (strcmp(GlobalList[I]->Name,Type) == 0) + return GlobalList[I]; + return 0; +} + /*}}}*/ +// Type::FixupURI - Normalize the URI and check it.. /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::Type::FixupURI(string &URI) const +{ + if (URI.empty() == true) + return false; + + if (URI.find(':') == string::npos) + return false; + + URI = ::URI{SubstVar(URI, "$(ARCH)", _config->Find("APT::Architecture"))}; + + // Make sure that the URI is / postfixed + if (URI.back() != '/') + URI.push_back('/'); + + return true; +} + /*}}}*/ +bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List, /*{{{*/ + pkgTagSection &Tags, + unsigned int const i, + FileFd &Fd) +{ + map<string, string> Options; + + string Enabled = Tags.FindS("Enabled"); + if (Enabled.empty() == false && StringToBool(Enabled) == false) + return true; + + std::map<char const * const, std::pair<char const * const, bool> > mapping; +#define APT_PLUSMINUS(X, Y) \ + mapping.insert(std::make_pair(X, std::make_pair(Y, true))); \ + mapping.insert(std::make_pair(X "-Add", std::make_pair(Y "+", true))); \ + mapping.insert(std::make_pair(X "-Remove", std::make_pair(Y "-", true))) + APT_PLUSMINUS("Architectures", "arch"); + APT_PLUSMINUS("Languages", "lang"); + APT_PLUSMINUS("Targets", "target"); +#undef APT_PLUSMINUS + mapping.insert(std::make_pair("Trusted", std::make_pair("trusted", false))); + mapping.insert(std::make_pair("Check-Valid-Until", std::make_pair("check-valid-until", false))); + mapping.insert(std::make_pair("Valid-Until-Min", std::make_pair("valid-until-min", false))); + mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false))); + mapping.insert(std::make_pair("Check-Date", std::make_pair("check-date", false))); + mapping.insert(std::make_pair("Date-Max-Future", std::make_pair("date-max-future", false))); + mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false))); + mapping.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false))); + mapping.insert(std::make_pair("By-Hash", std::make_pair("by-hash", false))); + + for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m) + if (Tags.Exists(m->first)) + { + if (m->second.second) + { + auto const values = FindMultiValue(Tags, m->first); + Options[m->second.first] = APT::String::Join(values, ","); + } + else + Options[m->second.first] = Tags.FindS(m->first); + } + + { + std::string entry; + strprintf(entry, "%s:%i", Fd.Name().c_str(), i); + Options["sourceslist-entry"] = entry; + } + + // now create one item per suite/section + auto const list_uris = FindMultiValue(Tags, "URIs"); + auto const list_comp = FindMultiValue(Tags, "Components"); + auto list_suite = FindMultiValue(Tags, "Suites"); + { + auto const nativeArch = _config->Find("APT::Architecture"); + std::transform(list_suite.begin(), list_suite.end(), list_suite.begin(), + [&](std::string const &suite) { return SubstVar(suite, "$(ARCH)", nativeArch); }); + } + + if (list_uris.empty()) + // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI"); + + if (list_suite.empty()) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite"); + + for (auto URI : list_uris) + { + if (FixupURI(URI) == false) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse"); + + for (auto const &S : list_suite) + { + if (likely(S.empty() == false) && S[S.size() - 1] == '/') + { + if (list_comp.empty() == false) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component"); + if (CreateItem(List, URI, S, "", Options) == false) + return false; + } + else + { + if (list_comp.empty()) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component"); + + for (auto const &C : list_comp) + if (CreateItem(List, URI, S, C, Options) == false) + return false; + } + } + } + return true; +} + /*}}}*/ +// Type::ParseLine - Parse a single line /*{{{*/ +// --------------------------------------------------------------------- +/* This is a generic one that is the 'usual' format for sources.list + Weird types may override this. */ +bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List, + const char *Buffer, + unsigned int const CurLine, + string const &File) const +{ + for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces + + // Parse option field if it exists + // e.g.: [ option1=value1 option2=value2 ] + map<string, string> Options; + { + std::string entry; + strprintf(entry, "%s:%i", File.c_str(), CurLine); + Options["sourceslist-entry"] = entry; + } + if (Buffer != 0 && Buffer[0] == '[') + { + ++Buffer; // ignore the [ + for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces + while (*Buffer != ']') + { + // get one option, e.g. option1=value1 + string option; + if (ParseQuoteWord(Buffer,option) == false) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] unparsable"); + + if (option.length() < 3) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] too short"); + + // accept options even if the last has no space before the ]-end marker + if (option.at(option.length()-1) == ']') + { + for (; *Buffer != ']'; --Buffer); + option.resize(option.length()-1); + } + + size_t const needle = option.find('='); + if (needle == string::npos) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] not assignment"); + + string const key = string(option, 0, needle); + string const value = string(option, needle + 1, option.length()); + + if (key.empty() == true) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no key"); + + if (value.empty() == true) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no value"); + + Options[key] = value; + } + ++Buffer; // ignore the ] + for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces + } + + string URI; + string Dist; + string Section; + + if (ParseQuoteWord(Buffer,URI) == false) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI"); + if (ParseQuoteWord(Buffer,Dist) == false) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Suite"); + + if (FixupURI(URI) == false) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI parse"); + + // Check for an absolute dists specification. + if (Dist.empty() == false && Dist[Dist.size() - 1] == '/') + { + if (ParseQuoteWord(Buffer,Section) == true) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "absolute Suite Component"); + Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture")); + return CreateItem(List, URI, Dist, Section, Options); + } + + // Grab the rest of the dists + if (ParseQuoteWord(Buffer,Section) == false) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Component"); + + do + { + if (CreateItem(List, URI, Dist, Section, Options) == false) + return false; + } + while (ParseQuoteWord(Buffer,Section) == true); + + return true; +} + /*}}}*/ +// SourceList::pkgSourceList - Constructors /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgSourceList::pkgSourceList() : d(NULL) +{ +} + /*}}}*/ +// SourceList::~pkgSourceList - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgSourceList::~pkgSourceList() +{ + for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I) + delete *I; + SrcList.clear(); + for (auto F = VolatileFiles.begin(); F != VolatileFiles.end(); ++F) + delete (*F); + VolatileFiles.clear(); +} + /*}}}*/ +// SourceList::ReadMainList - Read the main source list from etc /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ReadMainList() +{ + Reset(); + // CNC:2003-11-28 - Entries in sources.list have priority over + // entries in sources.list.d. + string Main = _config->FindFile("Dir::Etc::sourcelist", "/dev/null"); + string Parts = _config->FindDir("Dir::Etc::sourceparts", "/dev/null"); + + _error->PushToStack(); + if (RealFileExists(Main) == true) + ReadAppend(Main); + else if (DirectoryExists(Parts) == false && APT::String::Endswith(Parts, "/dev/null") == false) + // Only warn if there are no sources.list.d. + _error->WarningE("DirectoryExists", _("Unable to read %s"), Parts.c_str()); + + if (DirectoryExists(Parts) == true) + ReadSourceDir(Parts); + else if (Main.empty() == false && RealFileExists(Main) == false && + APT::String::Endswith(Parts, "/dev/null") == false) + // Only warn if there is no sources.list file. + _error->WarningE("RealFileExists", _("Unable to read %s"), Main.c_str()); + + for (auto && file: _config->FindVector("APT::Sources::With")) + AddVolatileFile(file, nullptr); + + auto good = _error->PendingError() == false; + _error->MergeWithStack(); + return good; +} + /*}}}*/ +// SourceList::Reset - Clear the sourcelist contents /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgSourceList::Reset() +{ + for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I) + delete *I; + SrcList.clear(); +} + /*}}}*/ +// SourceList::Read - Parse the sourcelist file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::Read(string const &File) +{ + Reset(); + return ReadAppend(File); +} + /*}}}*/ +// SourceList::ReadAppend - Parse a sourcelist file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ReadAppend(string const &File) +{ + if (flExtension(File) == "sources") + return ParseFileDeb822(File); + else + return ParseFileOldStyle(File); +} + /*}}}*/ +// SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ParseFileOldStyle(std::string const &File) +{ + FileFd Fd; + if (OpenConfigurationFileFd(File, Fd) == false) + return false; + + std::string Buffer; + for (unsigned int CurLine = 1; Fd.ReadLine(Buffer); ++CurLine) + { + // remove comments + size_t curpos = 0; + while ((curpos = Buffer.find('#', curpos)) != std::string::npos) + { + size_t const openbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, '['); + size_t const closedbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, ']'); + if (openbrackets > closedbrackets) + { + // a # in an option, unlikely, but oh well, it was supported so stick to it + ++curpos; + continue; + } + Buffer.erase(curpos); + break; + } + // remove spaces before/after + curpos = Buffer.find_first_not_of(" \t\r"); + if (curpos != 0) + Buffer.erase(0, curpos); + curpos = Buffer.find_last_not_of(" \t\r"); + if (curpos != std::string::npos) + Buffer.erase(curpos + 1); + + if (Buffer.empty()) + continue; + + // Grok it + std::string const LineType = Buffer.substr(0, Buffer.find_first_of(" \t\v")); + if (LineType.empty() || LineType == Buffer) + return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str()); + + Type *Parse = Type::GetType(LineType.c_str()); + if (Parse == 0) + return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str()); + + if (Parse->ParseLine(SrcList, Buffer.c_str() + LineType.length(), CurLine, File) == false) + return false; + } + return true; +} + /*}}}*/ +// SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/ +// --------------------------------------------------------------------- +/* Returns: the number of stanzas parsed*/ +bool pkgSourceList::ParseFileDeb822(string const &File) +{ + // see if we can read the file + FileFd Fd; + if (OpenConfigurationFileFd(File, Fd) == false) + return false; + pkgTagFile Sources(&Fd, pkgTagFile::SUPPORT_COMMENTS); + if (Fd.IsOpen() == false || Fd.Failed()) + return _error->Error(_("Malformed stanza %u in source list %s (type)"),0,File.c_str()); + + // read step by step + pkgTagSection Tags; + unsigned int i = 0; + while (Sources.Step(Tags) == true) + { + ++i; + if(Tags.Exists("Types") == false) + return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str()); + + for (auto const &type : FindMultiValue(Tags, "Types")) + { + Type *Parse = Type::GetType(type.c_str()); + if (Parse == 0) + { + _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), type.c_str(), i, Fd.Name().c_str()); + return false; + } + + if (!Parse->ParseStanza(SrcList, Tags, i, Fd)) + return false; + } + } + return true; +} + /*}}}*/ +// SourceList::FindIndex - Get the index associated with a file /*{{{*/ +static bool FindInIndexFileContainer(std::vector<pkgIndexFile *> const &Cont, pkgCache::PkgFileIterator const &File, pkgIndexFile *&Found) +{ + auto const J = std::find_if(Cont.begin(), Cont.end(), [&File](pkgIndexFile const * const J) { + return J->FindInCache(*File.Cache()) == File; + }); + if (J != Cont.end()) + { + Found = (*J); + return true; + } + return false; +} +bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File, + pkgIndexFile *&Found) const +{ + for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I) + if (FindInIndexFileContainer(*(*I)->GetIndexFiles(), File, Found)) + return true; + + return FindInIndexFileContainer(VolatileFiles, File, Found); +} + /*}}}*/ +// SourceList::GetIndexes - Load the index files into the downloader /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const +{ + for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I) + if ((*I)->GetIndexes(Owner,GetAll) == false) + return false; + return true; +} + /*}}}*/ +// CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>. +// SourceList::ReadSourceDir - Read a directory with sources files +// Based on ReadConfigDir() /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgSourceList::ReadSourceDir(string const &Dir) +{ + std::vector<std::string> const ext = {"list", "sources"}; + // Read the files + bool good = true; + for (auto const &I : GetListOfFilesInDir(Dir, ext, true)) + good = ReadAppend(I) && good; + return good; +} + /*}}}*/ +// GetLastModified() /*{{{*/ +// --------------------------------------------------------------------- +/* */ +time_t pkgSourceList::GetLastModifiedTime() +{ + vector<string> List; + + string Main = _config->FindFile("Dir::Etc::sourcelist"); + string Parts = _config->FindDir("Dir::Etc::sourceparts"); + + // go over the parts + if (DirectoryExists(Parts) == true) + List = GetListOfFilesInDir(Parts, "list", true); + + // calculate the time + std::vector<time_t> modtimes; + modtimes.reserve(1 + List.size()); + modtimes.push_back(GetModificationTime(Main)); + std::transform(List.begin(), List.end(), std::back_inserter(modtimes), GetModificationTime); + auto const maxmtime = std::max_element(modtimes.begin(), modtimes.end()); + return *maxmtime; +} + /*}}}*/ +std::vector<pkgIndexFile*> pkgSourceList::GetVolatileFiles() const /*{{{*/ +{ + return VolatileFiles; +} + /*}}}*/ +void pkgSourceList::AddVolatileFile(pkgIndexFile * const File) /*{{{*/ +{ + if (File != nullptr) + VolatileFiles.push_back(File); +} + /*}}}*/ +static bool fileNameMatches(std::string const &filename, std::string const &idxtype)/*{{{*/ +{ + for (auto && type: APT::Configuration::getCompressionTypes()) + { + if (type == "uncompressed") + { + if (filename == idxtype || APT::String::Endswith(filename, '_' + idxtype)) + return true; + } + else if (filename == idxtype + '.' + type || + APT::String::Endswith(filename, '_' + idxtype + '.' + type)) + return true; + } + return false; +} + /*}}}*/ +bool pkgSourceList::AddVolatileFile(std::string const &File, std::vector<std::string> * const VolatileCmdL)/*{{{*/ +{ + // Note: FileExists matches directories and links, too! + if (File.empty() || FileExists(File) == false) + return false; + + std::string const ext = flExtension(File); + // udeb is not included as installing it is usually a mistake rather than intended + if (ext == "deb" || ext == "ddeb") + AddVolatileFile(new debDebPkgFileIndex(File)); + else if (ext == "dsc") + AddVolatileFile(new debDscFileIndex(File)); + else if (FileExists(flCombine(File, "debian/control"))) + AddVolatileFile(new debDscFileIndex(flCombine(File, "debian/control"))); + else if (ext == "changes") + { + debDscRecordParser changes(File, nullptr); + std::vector<pkgSrcRecords::File> fileslst; + if (changes.Files(fileslst) == false || fileslst.empty()) + return false; + auto const basedir = flNotFile(File); + for (auto && file: fileslst) + { + auto const name = flCombine(basedir, file.Path); + AddVolatileFile(name, VolatileCmdL); + if (file.Hashes.VerifyFile(name) == false) + return _error->Error("The file %s does not match with the hashes in the %s file!", name.c_str(), File.c_str()); + } + return true; + } + else + { + auto const filename = flNotDir(File); + auto const Target = IndexTarget(File, filename, File, "file:" + File, false, true, { + { "FILENAME", File }, + { "REPO_URI", "file:" + flAbsPath(flNotFile(File)) + '/' }, + { "COMPONENT", "volatile-packages-file" }, + }); + if (fileNameMatches(filename, "Packages")) + AddVolatileFile(new debPackagesIndex(Target, true)); + else if (fileNameMatches(filename, "Sources")) + AddVolatileFile(new debSourcesIndex(Target, true)); + else + return false; + } + + if (VolatileCmdL != nullptr) + VolatileCmdL->push_back(File); + return true; +} +bool pkgSourceList::AddVolatileFile(std::string const &File) +{ + return AddVolatileFile(File, nullptr); +} + /*}}}*/ +void pkgSourceList::AddVolatileFiles(CommandLine &CmdL, std::vector<std::string> * const VolatileCmdL)/*{{{*/ +{ + std::remove_if(CmdL.FileList + 1, CmdL.FileList + 1 + CmdL.FileSize(), [&](char const * const I) { + if (I != nullptr && (I[0] == '/' || (I[0] == '.' && (I[1] == '\0' || (I[1] == '.' && (I[2] == '\0' || I[2] == '/')) || I[1] == '/')))) + { + if (AddVolatileFile(I, VolatileCmdL)) + ; + else + _error->Error(_("Unsupported file %s given on commandline"), I); + return true; + } + return false; + }); +} + /*}}}*/ diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h new file mode 100644 index 0000000..4541803 --- /dev/null +++ b/apt-pkg/sourcelist.h @@ -0,0 +1,135 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + SourceList - Manage a list of sources + + The Source List class provides access to a list of sources. It + can read them from a file and generate a list of all the distinct + sources. + + All sources have a type associated with them that defines the layout + of the archive. The exact format of the file is documented in + files.sgml. + + The types are mapped through a list of type definitions which handle + the actual construction of the back end type. After loading a source + list all you have is a list of package index files that have the ability + to be Acquired. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_SOURCELIST_H +#define PKGLIB_SOURCELIST_H + +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> + +#include <time.h> + +#include <map> +#include <string> +#include <vector> + + +class FileFd; +class pkgTagSection; +class pkgAcquire; +class pkgIndexFile; +class metaIndex; +class CommandLine; + +class APT_PUBLIC pkgSourceList +{ + void * const d; + std::vector<pkgIndexFile*> VolatileFiles; + public: + + // List of supported source list types + class Type + { + public: + + // Global list of Items supported + static Type **GlobalList; + static unsigned long GlobalListLen; + static Type *GetType(const char *Type) APT_PURE; + + char const * const Name; + char const * const Label; + + bool FixupURI(std::string &URI) const; + virtual bool ParseStanza(std::vector<metaIndex *> &List, + pkgTagSection &Tags, + unsigned int const stanza_n, + FileFd &Fd); + virtual bool ParseLine(std::vector<metaIndex *> &List, + const char *Buffer, + unsigned int const CurLine,std::string const &File) const; + virtual bool CreateItem(std::vector<metaIndex *> &List,std::string const &URI, + std::string const &Dist,std::string const &Section, + std::map<std::string, std::string> const &Options) const = 0; + Type(char const * const Name, char const * const Label); + virtual ~Type(); + }; + + typedef std::vector<metaIndex *>::const_iterator const_iterator; + + protected: + + std::vector<metaIndex *> SrcList; + + private: + APT_HIDDEN bool ParseFileDeb822(std::string const &File); + APT_HIDDEN bool ParseFileOldStyle(std::string const &File); + + public: + + bool ReadMainList(); + bool Read(std::string const &File); + + // CNC:2003-03-03 + void Reset(); + bool ReadAppend(std::string const &File); + bool ReadSourceDir(std::string const &Dir); + + // List accessors + inline const_iterator begin() const {return SrcList.begin();}; + inline const_iterator end() const {return SrcList.end();}; + inline unsigned int size() const {return SrcList.size();}; + inline bool empty() const {return SrcList.empty();}; + + bool FindIndex(pkgCache::PkgFileIterator File, + pkgIndexFile *&Found) const; + bool GetIndexes(pkgAcquire *Owner, bool GetAll=false) const; + + // query last-modified time + time_t GetLastModifiedTime(); + + /** \brief add file for parsing, but not to the cache + * + * pkgIndexFiles originating from pkgSourcesList are included in + * srcpkgcache, the status files added via #AddStatusFiles are + * included in pkgcache, but these files here are not included in + * any cache to have the possibility of having a file included just + * for a single run like a local .deb/.dsc file. + * + * The volatile files do not count as "normal" sourceslist entries, + * can't be iterated over with #begin and #end and can't be + * downloaded, but they can be found via #FindIndex. + * + * @param File is an index file; pointer-ownership is transferred + */ + void AddVolatileFile(pkgIndexFile * const File); + bool AddVolatileFile(std::string const &File); + bool AddVolatileFile(std::string const &File, std::vector<std::string> * const VolatileCmdL); + void AddVolatileFiles(CommandLine &CmdL, std::vector<std::string> * const VolatileCmdL); + + /** @return list of files registered with #AddVolatileFile */ + std::vector<pkgIndexFile*> GetVolatileFiles() const; + + pkgSourceList(); + virtual ~pkgSourceList(); +}; + +#endif diff --git a/apt-pkg/srcrecords.cc b/apt-pkg/srcrecords.cc new file mode 100644 index 0000000..03bda75 --- /dev/null +++ b/apt-pkg/srcrecords.cc @@ -0,0 +1,152 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Source Package Records - Allows access to source package records + + Parses and allows access to the list of source records and searching by + source name on that list. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/debsrcrecords.h> +#include <apt-pkg/error.h> +#include <apt-pkg/indexfile.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/metaindex.h> +#include <apt-pkg/sourcelist.h> +#include <apt-pkg/srcrecords.h> + +#include <string> +#include <vector> +#include <string.h> + +#include <apti18n.h> + /*}}}*/ + +// SrcRecords::pkgSrcRecords - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Open all the source index files */ +pkgSrcRecords::pkgSrcRecords(pkgSourceList &List) : d(NULL), Files(0) +{ + for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); ++I) + { + std::vector<pkgIndexFile *> *Indexes = (*I)->GetIndexFiles(); + for (std::vector<pkgIndexFile *>::const_iterator J = Indexes->begin(); + J != Indexes->end(); ++J) + { + _error->PushToStack(); + Parser* P = (*J)->CreateSrcParser(); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + if (newError) + return; + if (P != 0) + Files.push_back(P); + } + } + + // Doesn't work without any source index files + if (Files.empty() == true) + { + _error->Error(_("You must put some 'deb-src' URIs" + " in your sources.list")); + return; + } + + Restart(); +} + /*}}}*/ +// SrcRecords::~pkgSrcRecords - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgSrcRecords::~pkgSrcRecords() +{ + // Blow away all the parser objects + for(std::vector<Parser*>::iterator I = Files.begin(); I != Files.end(); ++I) + delete *I; +} + /*}}}*/ +// SrcRecords::Restart - Restart the search /*{{{*/ +// --------------------------------------------------------------------- +/* Return all of the parsers to their starting position */ +bool pkgSrcRecords::Restart() +{ + Current = Files.begin(); + for (std::vector<Parser*>::iterator I = Files.begin(); + I != Files.end(); ++I) + if ((*I)->Offset() != 0) + (*I)->Restart(); + + return true; +} + /*}}}*/ +// SrcRecords::Step - Step to the next Source Record /*{{{*/ +// --------------------------------------------------------------------- +/* Step to the next source package record */ +const pkgSrcRecords::Parser* pkgSrcRecords::Step() +{ + if (Current == Files.end()) + return 0; + + // Step to the next record, possibly switching files + while ((*Current)->Step() == false) + { + ++Current; + if (Current == Files.end()) + return 0; + } + + return *Current; +} + /*}}}*/ +// SrcRecords::Find - Find the first source package with the given name /*{{{*/ +// --------------------------------------------------------------------- +/* This searches on both source package names and output binary names and + returns the first found. A 'cursor' like system is used to allow this + function to be called multiple times to get successive entries */ +pkgSrcRecords::Parser *pkgSrcRecords::Find(const char *Package,bool const &SrcOnly) +{ + while (true) + { + if(Step() == 0) + return 0; + + // Source name hit + if ((*Current)->Package() == Package) + return *Current; + + if (SrcOnly == true) + continue; + + // Check for a binary hit + const char **I = (*Current)->Binaries(); + for (; I != 0 && *I != 0; ++I) + if (strcmp(Package,*I) == 0) + return *Current; + } +} + /*}}}*/ +// Parser::BuildDepType - Convert a build dep to a string /*{{{*/ +// --------------------------------------------------------------------- +/* */ +const char *pkgSrcRecords::Parser::BuildDepType(unsigned char const &Type) +{ + const char *fields[] = {"Build-Depends", + "Build-Depends-Indep", + "Build-Conflicts", + "Build-Conflicts-Indep", + "Build-Depends-Arch", + "Build-Conflicts-Arch"}; + if (unlikely(Type >= sizeof(fields)/sizeof(fields[0]))) + return ""; + return fields[Type]; +} + /*}}}*/ + + +pkgSrcRecords::Parser::Parser(const pkgIndexFile *Index) : d(NULL), iIndex(Index) {} +pkgSrcRecords::Parser::~Parser() {} diff --git a/apt-pkg/srcrecords.h b/apt-pkg/srcrecords.h new file mode 100644 index 0000000..b9014a5 --- /dev/null +++ b/apt-pkg/srcrecords.h @@ -0,0 +1,108 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Source Package Records - Allows access to source package records + + Parses and allows access to the list of source records and searching by + source name on that list. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_SRCRECORDS_H +#define PKGLIB_SRCRECORDS_H + +#include <apt-pkg/hashes.h> +#include <apt-pkg/macros.h> + +#include <string> +#include <vector> + + +class pkgSourceList; +class pkgIndexFile; +class APT_PUBLIC pkgSrcRecords +{ + public: + + // Describes a single file + struct File + { + std::string Path; + std::string Type; + unsigned long long FileSize; + HashStringList Hashes; + }; + + // Abstract parser for each source record + class Parser + { + void * const d; + protected: + + const pkgIndexFile *iIndex; + + public: + + enum BuildDep {BuildDepend=0x0,BuildDependIndep=0x1, + BuildConflict=0x2,BuildConflictIndep=0x3, + BuildDependArch=0x4,BuildConflictArch=0x5}; + + struct BuildDepRec + { + std::string Package; + std::string Version; + unsigned int Op; + unsigned char Type; + }; + + inline const pkgIndexFile &Index() const {return *iIndex;}; + + virtual bool Restart() = 0; + virtual bool Step() = 0; + virtual bool Jump(unsigned long const &Off) = 0; + virtual unsigned long Offset() = 0; + virtual std::string AsStr() = 0; + + virtual std::string Package() const = 0; + virtual std::string Version() const = 0; + virtual std::string Maintainer() const = 0; + virtual std::string Section() const = 0; + virtual const char **Binaries() = 0; // Ownership does not transfer + + //FIXME: Add a parameter to specify which architecture to use for [wildcard] matching + virtual bool BuildDepends(std::vector<BuildDepRec> &BuildDeps, bool const &ArchOnly, bool const &StripMultiArch = true) = 0; + static const char *BuildDepType(unsigned char const &Type) APT_PURE; + + virtual bool Files(std::vector<pkgSrcRecords::File> &F) = 0; + + explicit Parser(const pkgIndexFile *Index); + virtual ~Parser(); + }; + + private: + /** \brief dpointer placeholder (for later in case we need it) */ + void * const d; + + // The list of files and the current parser pointer + std::vector<Parser*> Files; + std::vector<Parser *>::iterator Current; + + public: + + // Reset the search + bool Restart(); + + // Step to the next SourcePackage and return pointer to the + // next SourceRecord. The pointer is owned by libapt. + const Parser* Step(); + + // Locate a package by name and return pointer to the Parser. + // The pointer is owned by libapt. + Parser* Find(const char *Package,bool const &SrcOnly = false); + + explicit pkgSrcRecords(pkgSourceList &List); + virtual ~pkgSrcRecords(); +}; + +#endif diff --git a/apt-pkg/statechanges.cc b/apt-pkg/statechanges.cc new file mode 100644 index 0000000..bbcde71 --- /dev/null +++ b/apt-pkg/statechanges.cc @@ -0,0 +1,231 @@ +#include <config.h> + +#include <apt-pkg/cacheset.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/debsystem.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/prettyprinters.h> +#include <apt-pkg/statechanges.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <memory> + +namespace APT +{ + +class StateChanges::Private +{ +public: + APT::VersionVector hold; + APT::VersionVector unhold; + APT::VersionVector install; + APT::VersionVector deinstall; + APT::VersionVector purge; + APT::VersionVector error; +}; + +#define APT_GETTERSETTER(Name, Container) \ +void StateChanges::Name(pkgCache::VerIterator const &Ver) \ +{ \ + if (Ver.end() == false) \ + Container.push_back(Ver); \ +}\ +APT::VersionVector& StateChanges::Name() \ +{ \ + return Container; \ +} +APT_GETTERSETTER(Hold, d->hold) +APT_GETTERSETTER(Unhold, d->unhold) +APT_GETTERSETTER(Install, d->install) +APT_GETTERSETTER(Remove, d->deinstall) +APT_GETTERSETTER(Purge, d->purge) +#undef APT_GETTERSETTER +APT::VersionVector& StateChanges::Error() +{ + return d->error; +} + +void StateChanges::clear() +{ + d->hold.clear(); + d->unhold.clear(); + d->install.clear(); + d->deinstall.clear(); + d->purge.clear(); + d->error.clear(); +} + +bool StateChanges::empty() const +{ + return d->hold.empty() && + d->unhold.empty() && + d->install.empty() && + d->deinstall.empty() && + d->purge.empty() && + d->error.empty(); +} + +bool StateChanges::Save(bool const DiscardOutput) +{ + bool const Debug = _config->FindB("Debug::pkgDpkgPm", false); + d->error.clear(); + if (d->hold.empty() && d->unhold.empty() && d->install.empty() && d->deinstall.empty() && d->purge.empty()) + return true; + + std::vector<std::string> Args = debSystem::GetDpkgBaseCommand(); + // ensure dpkg knows about the package so that it keeps the status we set + if (d->hold.empty() == false || d->install.empty() == false) + { + APT::VersionVector makeDpkgAvailable; + auto const notInstalled = [](pkgCache::VerIterator const &V) { return V.ParentPkg()->CurrentVer == 0; }; + std::copy_if(d->hold.begin(), d->hold.end(), std::back_inserter(makeDpkgAvailable), notInstalled); + std::copy_if(d->install.begin(), d->install.end(), std::back_inserter(makeDpkgAvailable), notInstalled); + + if (makeDpkgAvailable.empty() == false) + { + auto const BaseArgs = Args.size(); + Args.push_back("--merge-avail"); + // FIXME: supported only since 1.17.7 in dpkg + Args.push_back("-"); + int dummyAvail = -1; + if (Debug) + { + for (auto const &V: makeDpkgAvailable) + { + std::clog << "echo 'Dummy record for " << V.ParentPkg().FullName(false) << "' | "; + std::copy(Args.begin(), Args.end(), std::ostream_iterator<std::string>(std::clog, " ")); + std::clog << std::endl; + } + } + else + { + pid_t const dpkgMergeAvail = debSystem::ExecDpkg(Args, &dummyAvail, nullptr, true); + + FILE* dpkg = fdopen(dummyAvail, "w"); + for (auto const &V: makeDpkgAvailable) + fprintf(dpkg, "Package: %s\nVersion: 0~\nArchitecture: %s\nMaintainer: Dummy Example <dummy@example.org>\n" + "Description: dummy package record\n A record is needed to put a package on hold, so here it is.\n\n", V.ParentPkg().Name(), V.Arch()); + fclose(dpkg); + + ExecWait(dpkgMergeAvail, "dpkg --merge-avail", true); + } + Args.erase(Args.begin() + BaseArgs, Args.end()); + } + } + bool const dpkgMultiArch = _system->MultiArchSupported(); + + Args.push_back("--set-selections"); + if (Debug) + { + std::string state; + auto const dpkgName = [&](pkgCache::VerIterator const &V) { + pkgCache::PkgIterator P = V.ParentPkg(); + if (strcmp(V.Arch(), "none") == 0) + ioprintf(std::clog, "echo '%s %s' | ", P.Name(), state.c_str()); + else if (dpkgMultiArch == false) + ioprintf(std::clog, "echo '%s %s' | ", P.FullName(true).c_str(), state.c_str()); + else + ioprintf(std::clog, "echo '%s:%s %s' | ", P.Name(), V.Arch(), state.c_str()); + std::copy(Args.begin(), Args.end(), std::ostream_iterator<std::string>(std::clog, " ")); + std::clog << std::endl; + }; + for (auto const &V: d->unhold) + { + if (V.ParentPkg()->CurrentVer != 0) + state = "install"; + else + state = "deinstall"; + dpkgName(V); + } + if (d->purge.empty() == false) + { + state = "purge"; + std::for_each(d->purge.begin(), d->purge.end(), dpkgName); + } + if (d->deinstall.empty() == false) + { + state = "deinstall"; + std::for_each(d->deinstall.begin(), d->deinstall.end(), dpkgName); + } + if (d->hold.empty() == false) + { + state = "hold"; + std::for_each(d->hold.begin(), d->hold.end(), dpkgName); + } + if (d->install.empty() == false) + { + state = "install"; + std::for_each(d->install.begin(), d->install.end(), dpkgName); + } + } + else + { + int selections = -1; + pid_t const dpkgSelections = debSystem::ExecDpkg(Args, &selections, nullptr, DiscardOutput); + + FILE* dpkg = fdopen(selections, "w"); + std::string state; + auto const dpkgName = [&](pkgCache::VerIterator const &V) { + pkgCache::PkgIterator P = V.ParentPkg(); + if (strcmp(V.Arch(), "none") == 0) + fprintf(dpkg, "%s %s\n", P.Name(), state.c_str()); + else if (dpkgMultiArch == false) + fprintf(dpkg, "%s %s\n", P.FullName(true).c_str(), state.c_str()); + else + fprintf(dpkg, "%s:%s %s\n", P.Name(), V.Arch(), state.c_str()); + }; + for (auto const &V: d->unhold) + { + if (V.ParentPkg()->CurrentVer != 0) + state = "install"; + else + state = "deinstall"; + dpkgName(V); + } + if (d->purge.empty() == false) + { + state = "purge"; + std::for_each(d->purge.begin(), d->purge.end(), dpkgName); + } + if (d->deinstall.empty() == false) + { + state = "deinstall"; + std::for_each(d->deinstall.begin(), d->deinstall.end(), dpkgName); + } + if (d->hold.empty() == false) + { + state = "hold"; + std::for_each(d->hold.begin(), d->hold.end(), dpkgName); + } + if (d->install.empty() == false) + { + state = "install"; + std::for_each(d->install.begin(), d->install.end(), dpkgName); + } + fclose(dpkg); + + if (ExecWait(dpkgSelections, "dpkg --set-selections") == false) + { + std::move(d->purge.begin(), d->purge.end(), std::back_inserter(d->error)); + d->purge.clear(); + std::move(d->deinstall.begin(), d->deinstall.end(), std::back_inserter(d->error)); + d->deinstall.clear(); + std::move(d->hold.begin(), d->hold.end(), std::back_inserter(d->error)); + d->hold.clear(); + std::move(d->unhold.begin(), d->unhold.end(), std::back_inserter(d->error)); + d->unhold.clear(); + std::move(d->install.begin(), d->install.end(), std::back_inserter(d->error)); + d->install.clear(); + } + } + return d->error.empty(); +} + +StateChanges::StateChanges() : d(new StateChanges::Private()) {} +StateChanges::StateChanges(StateChanges&&) = default; +StateChanges& StateChanges::operator=(StateChanges&&) = default; +StateChanges::~StateChanges() = default; + +} diff --git a/apt-pkg/statechanges.h b/apt-pkg/statechanges.h new file mode 100644 index 0000000..2f63d51 --- /dev/null +++ b/apt-pkg/statechanges.h @@ -0,0 +1,58 @@ +#include <apt-pkg/cacheset.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/pkgcache.h> + +#include <memory> + +namespace APT +{ + +/** Simple wrapper class to abstract away the differences in storing different + * states in different places potentially in different versions. + */ +class APT_PUBLIC StateChanges +{ +public: + // getter/setter for the different states +#define APT_GETTERSETTER(Name) \ + APT::VersionVector& Name(); \ + void Name(pkgCache::VerIterator const &Ver) + APT_GETTERSETTER(Hold); + APT_GETTERSETTER(Unhold); + APT_GETTERSETTER(Install); + APT_GETTERSETTER(Remove); + APT_GETTERSETTER(Purge); + APT::VersionVector& Error(); +#undef APT_GETTERSETTER + + // operate on all containers at once + void clear(); + bool empty() const; + + /** commit the staged changes to the database(s). + * + * Makes the needed calls to store the requested states. + * After this call the state containers will hold only versions + * for which the storing operation succeeded. Versions where the + * storing operation failed are collected in #Error(). Note that + * error is an upper bound as states are changed in batches so it + * isn't always clear which version triggered the failure exactly. + * + * @param DiscardOutput controls if stdout/stderr should be used + * by subprocesses for (detailed) error reporting if needed. + * @return \b false if storing failed, true otherwise. + * Note that some states might be applied even if the whole operation failed. + */ + bool Save(bool const DiscardOutput = false); + + StateChanges(); + StateChanges(StateChanges&&); + StateChanges& operator=(StateChanges&&); + ~StateChanges(); + +private: + class APT_HIDDEN Private; + std::unique_ptr<Private> d; +}; + +} diff --git a/apt-pkg/tagfile-keys.list b/apt-pkg/tagfile-keys.list new file mode 100644 index 0000000..4b57e46 --- /dev/null +++ b/apt-pkg/tagfile-keys.list @@ -0,0 +1,82 @@ +# This file is input for triehash(1) (after stripping comments) +# +# The fields listed here are accessible via pkgTagSection::Key::FIELD +# Do *NOT* edit, remove or insert new fields in the sorted section here +# as the file forms part of the ABI of the libapt library and is used +# by our apt tools. External clients are forbidden though, so if really needed +# we can use libapt Breaks: apt, but always prefer appending only. +# +# For Fields used in Packages, Sources and status files, see also tagfile-order.c +Architecture +Auto-Built-Package +Binary +Breaks +Build-Conflicts +Build-Conflicts-Arch +Build-Conflicts-Indep +Build-Depends +Build-Depends-Arch +Build-Depends-Indep +Build-Essential +Built-For-Profiles +Built-Using +Checksums-Md5 +Checksums-Sha1 +Checksums-Sha256 +Checksums-Sha512 +Conffiles +Config-Version +Conflicts +Depends +Description +Description-md5 +Directory +Enhances +Essential +Filename +Files +Format +Homepage +Important +Installed-Size +Maintainer +MD5sum +Multi-Arch +Origin +Original-Maintainer +Package +Package-List +Package-Type +Phased-Update-Percentage +Pre-Depends +Priority +Protected +Provides +Recommends +Replaces +Section +SHA1 +SHA256 +SHA512 +Size +Source +Standards-Version +Static-Built-Using +Status +Suggests +Tag +Task +Testsuite +Testsuite-Triggers +Uploaders +Vcs-Arch +Vcs-Browser +Vcs-Bzr +Vcs-Cvs +Vcs-Darcs +Vcs-Git +Vcs-Hg +Vcs-Mtn +Vcs-Svn +Version +### APPEND BELOW, sort in with next ABI break ### diff --git a/apt-pkg/tagfile-order.c b/apt-pkg/tagfile-order.c new file mode 100644 index 0000000..7cf188c --- /dev/null +++ b/apt-pkg/tagfile-order.c @@ -0,0 +1,118 @@ +/* In this file is the order defined in which e.g. apt-ftparchive will write stanzas in. + Other commands might or might not use this. 'apt-cache show' e.g. does NOT! + + The order we chose here is inspired by both dpkg and dak. + The testcase test/integration/test-apt-tagfile-fields-order intends to ensure that + this file isn't lacking (too far) behind dpkg over time. */ + +static const char *iTFRewritePackageOrder[] = { + "Package", + "Package-Type", + "Architecture", + "Subarchitecture", // NO_KEY: Used only by d-i + "Version", +// "Revision", // Obsolete (warning in dpkg) +// "Package-Revision", // Obsolete (warning in dpkg) +// "Package_Revision", // Obsolete (warning in dpkg) + "Kernel-Version", // NO_KEY: Used only by d-i + "Built-Using", + "Static-Built-Using", + "Built-For-Profiles", + "Auto-Built-Package", + "Multi-Arch", + "Status", + "Priority", +// "Class", // Obsolete alias for Priority, warning by dpkg + "Build-Essential", + "Protected", + "Important", // old name of Protected + "Essential", + "Installer-Menu-Item", // NO_KEY: Used only by d-i + "Section", + "Source", + "Origin", + "Phased-Update-Percentage", + "Maintainer", + "Original-Maintainer", // unknown in dpkg order + "Bugs", // NO_KEY: very uncommon encounter + "Config-Version", // Internal of dpkg + "Conffiles", + "Triggers-Awaited", // NO_KEY: Internal of dpkg + "Triggers-Pending", // NO_KEY: Internal of dpkg + "Installed-Size", + "Provides", + "Pre-Depends", + "Depends", + "Recommends", +// "Recommended", // Obsolete alias for Recommends, warning by dpkg + "Suggests", +// "Optional", // Obsolete alias for Suggests, warning by dpkg + "Conflicts", + "Breaks", + "Replaces", + "Enhances", + "Filename", + "MSDOS-Filename", // NO_KEY: Obsolete (used by dselect) + "Size", + "MD5sum", + "SHA1", + "SHA256", + "SHA512", + "Homepage", + "Description", + "Description-md5", + "Tag", + "Task", + 0, +}; +static const char *iTFRewriteSourceOrder[] = { + "Package", + "Source", // dsc file, renamed to Package in Sources + "Format", + "Binary", + "Architecture", + "Version", + "Priority", +// "Class", // Obsolete alias for Priority, warning by dpkg + "Section", + "Origin", + "Maintainer", + "Original-Maintainer", // unknown in dpkg order + "Uploaders", + "Dm-Upload-Allowed", // NO_KEY: Obsolete (ignored by dak) + "Standards-Version", + "Build-Depends", + "Build-Depends-Arch", + "Build-Depends-Indep", + "Build-Conflicts", + "Build-Conflicts-Arch", + "Build-Conflicts-Indep", + "Testsuite", + "Testsuite-Triggers", + "Homepage", + "Description", + "Vcs-Browser", + "Vcs-Browse", // NO_KEY: dak only (nickname?) + "Vcs-Arch", + "Vcs-Bzr", + "Vcs-Cvs", + "Vcs-Darcs", + "Vcs-Git", + "Vcs-Hg", + "Vcs-Mtn", + "Vcs-Svn", + "Directory", + "Package-List", + "Files", + "Checksums-Md5", + "Checksums-Sha1", + "Checksums-Sha256", + "Checksums-Sha512", + 0, +}; + +/* Two levels of initialization are used because gcc will set the symbol + size of an array to the length of the array, causing dynamic relinking + errors. Doing this makes the symbol size constant */ +const char **TFRewritePackageOrder = iTFRewritePackageOrder; +const char **TFRewriteSourceOrder = iTFRewriteSourceOrder; diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc new file mode 100644 index 0000000..047f889 --- /dev/null +++ b/apt-pkg/tagfile.cc @@ -0,0 +1,1083 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Fast scanner for RFC-822 type header information + + This uses a rotating buffer to load the package information into. + The scanner runs over it and isolates and indexes a single section. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/string_view.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile-keys.h> +#include <apt-pkg/tagfile.h> + +#include <list> + +#include <string> +#include <ctype.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <apti18n.h> + /*}}}*/ + +using std::string; +using APT::StringView; + +class APT_HIDDEN pkgTagFilePrivate /*{{{*/ +{ +public: + void Reset(FileFd * const pFd, unsigned long long const pSize, pkgTagFile::Flags const pFlags) + { + if (Buffer != NULL) + free(Buffer); + Buffer = NULL; + Fd = pFd; + Flags = pFlags; + Start = NULL; + End = NULL; + Done = false; + iOffset = 0; + Size = pSize; + isCommentedLine = false; + chunks.clear(); + } + + pkgTagFilePrivate(FileFd * const pFd, unsigned long long const Size, pkgTagFile::Flags const pFlags) : Buffer(NULL) + { + Reset(pFd, Size, pFlags); + } + FileFd * Fd; + pkgTagFile::Flags Flags; + char *Buffer; + char *Start; + char *End; + bool Done; + unsigned long long iOffset; + unsigned long long Size; + bool isCommentedLine; + struct FileChunk + { + bool const good; + size_t length; + FileChunk(bool const pgood, size_t const plength) noexcept : good(pgood), length(plength) {} + }; + std::list<FileChunk> chunks; + + ~pkgTagFilePrivate() + { + if (Buffer != NULL) + free(Buffer); + } +}; + /*}}}*/ +class APT_HIDDEN pkgTagSectionPrivate /*{{{*/ +{ +public: + pkgTagSectionPrivate() + { + } + struct TagData { + unsigned int StartTag; + unsigned int EndTag; + unsigned int StartValue; + unsigned int NextInBucket; + + explicit TagData(unsigned int const StartTag) : StartTag(StartTag), EndTag(0), StartValue(0), NextInBucket(0) {} + }; + std::vector<TagData> Tags; +}; + /*}}}*/ + +static unsigned long BetaHash(const char *Text, size_t Length) /*{{{*/ +{ + /* This very simple hash function for the last 8 letters gives + very good performance on the debian package files */ + if (Length > 8) + { + Text += (Length - 8); + Length = 8; + } + unsigned long Res = 0; + for (size_t i = 0; i < Length; ++i) + Res = ((unsigned long)(Text[i]) & 0xDF) ^ (Res << 1); + return Res & 0x7F; +} + /*}}}*/ + +// TagFile::pkgTagFile - Constructor /*{{{*/ +pkgTagFile::pkgTagFile(FileFd * const pFd,pkgTagFile::Flags const pFlags, unsigned long long const Size) + : d(new pkgTagFilePrivate(pFd, Size + 4, pFlags)) +{ + Init(pFd, pFlags, Size); +} +pkgTagFile::pkgTagFile(FileFd * const pFd,unsigned long long const Size) + : pkgTagFile(pFd, pkgTagFile::STRICT, Size) +{ +} +void pkgTagFile::Init(FileFd * const pFd, pkgTagFile::Flags const pFlags, unsigned long long Size) +{ + /* The size is increased by 4 because if we start with the Size of the + filename we need to try to read 1 char more to see an EOF faster, 1 + char the end-pointer can be on and maybe 2 newlines need to be added + to the end of the file -> 4 extra chars */ + Size += 4; + d->Reset(pFd, Size, pFlags); + + if (d->Fd->IsOpen() == false) + d->Start = d->End = d->Buffer = 0; + else + d->Buffer = (char*)malloc(sizeof(char) * Size); + + if (d->Buffer == NULL) + d->Done = true; + else + d->Done = false; + + d->Start = d->End = d->Buffer; + d->iOffset = 0; + if (d->Done == false) + Fill(); +} +void pkgTagFile::Init(FileFd * const pFd,unsigned long long Size) +{ + Init(pFd, pkgTagFile::STRICT, Size); +} + /*}}}*/ +// TagFile::~pkgTagFile - Destructor /*{{{*/ +pkgTagFile::~pkgTagFile() +{ + delete d; +} + /*}}}*/ +// TagFile::Offset - Return the current offset in the buffer /*{{{*/ +APT_PURE unsigned long pkgTagFile::Offset() +{ + return d->iOffset; +} + /*}}}*/ +// TagFile::Resize - Resize the internal buffer /*{{{*/ +// --------------------------------------------------------------------- +/* Resize the internal buffer (double it in size). Fail if a maximum size + * size is reached. + */ +bool pkgTagFile::Resize() +{ + // fail is the buffer grows too big + if(d->Size > 1024*1024+1) + return false; + + return Resize(d->Size * 2); +} +bool pkgTagFile::Resize(unsigned long long const newSize) +{ + unsigned long long const EndSize = d->End - d->Start; + + // get new buffer and use it + char* const newBuffer = static_cast<char*>(realloc(d->Buffer, sizeof(char) * newSize)); + if (newBuffer == NULL) + return false; + d->Buffer = newBuffer; + d->Size = newSize; + + // update the start/end pointers to the new buffer + d->Start = d->Buffer; + d->End = d->Start + EndSize; + return true; +} + /*}}}*/ +// TagFile::Step - Advance to the next section /*{{{*/ +// --------------------------------------------------------------------- +/* If the Section Scanner fails we refill the buffer and try again. + * If that fails too, double the buffer size and try again until a + * maximum buffer is reached. + */ +bool pkgTagFile::Step(pkgTagSection &Tag) +{ + if(Tag.Scan(d->Start,d->End - d->Start) == false) + { + do + { + if (Fill() == false) + return false; + + if(Tag.Scan(d->Start,d->End - d->Start, false)) + break; + + if (Resize() == false) + return _error->Error(_("Unable to parse package file %s (%d)"), + d->Fd->Name().c_str(), 1); + + } while (Tag.Scan(d->Start,d->End - d->Start, false) == false); + } + + size_t tagSize = Tag.size(); + d->Start += tagSize; + + if ((d->Flags & pkgTagFile::SUPPORT_COMMENTS) == 0) + d->iOffset += tagSize; + else + { + auto first = d->chunks.begin(); + for (; first != d->chunks.end(); ++first) + { + if (first->good == false) + d->iOffset += first->length; + else + { + if (tagSize < first->length) + { + first->length -= tagSize; + d->iOffset += tagSize; + break; + } + else + { + tagSize -= first->length; + d->iOffset += first->length; + } + } + } + d->chunks.erase(d->chunks.begin(), first); + } + + if ((d->Flags & pkgTagFile::SUPPORT_COMMENTS) == 0 || Tag.Count() != 0) + { + Tag.Trim(); + return true; + } + return Step(Tag); +} + /*}}}*/ +// TagFile::Fill - Top up the buffer /*{{{*/ +// --------------------------------------------------------------------- +/* This takes the bit at the end of the buffer and puts it at the start + then fills the rest from the file */ +static bool FillBuffer(pkgTagFilePrivate * const d) +{ + unsigned long long Actual = 0; + // See if only a bit of the file is left + unsigned long long const dataSize = d->Size - ((d->End - d->Buffer) + 1); + if (d->Fd->Read(d->End, dataSize, &Actual) == false) + return false; + if (Actual != dataSize) + d->Done = true; + d->End += Actual; + return true; +} +static void RemoveCommentsFromBuffer(pkgTagFilePrivate * const d) +{ + // look for valid comments in the buffer + char * good_start = nullptr, * bad_start = nullptr; + char * current = d->Start; + if (d->isCommentedLine == false) + { + if (d->Start == d->Buffer) + { + // the start of the buffer is a newline as a record can't start + // in the middle of a line by definition. + if (*d->Start == '#') + { + d->isCommentedLine = true; + ++current; + if (current > d->End) + d->chunks.emplace_back(false, 1); + } + } + if (d->isCommentedLine == false) + good_start = d->Start; + else + bad_start = d->Start; + } + else + bad_start = d->Start; + + std::vector<std::pair<char*, size_t>> good_parts; + while (current <= d->End) + { + size_t const restLength = (d->End - current); + if (d->isCommentedLine == false) + { + current = static_cast<char*>(memchr(current, '#', restLength)); + if (current == nullptr) + { + size_t const goodLength = d->End - good_start; + d->chunks.emplace_back(true, goodLength); + if (good_start != d->Start) + good_parts.push_back(std::make_pair(good_start, goodLength)); + break; + } + bad_start = current; + --current; + // ensure that this is really a comment and not a '#' in the middle of a line + if (*current == '\n') + { + size_t const goodLength = (current - good_start) + 1; + d->chunks.emplace_back(true, goodLength); + good_parts.push_back(std::make_pair(good_start, goodLength)); + good_start = nullptr; + d->isCommentedLine = true; + } + current += 2; + } + else // the current line is a comment + { + current = static_cast<char*>(memchr(current, '\n', restLength)); + if (current == nullptr) + { + d->chunks.emplace_back(false, (d->End - bad_start)); + break; + } + ++current; + // is the next line a comment, too? + if (current >= d->End || *current != '#') + { + d->chunks.emplace_back(false, (current - bad_start)); + good_start = current; + bad_start = nullptr; + d->isCommentedLine = false; + } + ++current; + } + } + + if (good_parts.empty() == false) + { + // we found comments, so move later parts over them + current = d->Start; + for (auto const &good: good_parts) + { + memmove(current, good.first, good.second); + current += good.second; + } + d->End = current; + } + + if (d->isCommentedLine == true) + { + // deal with a buffer containing only comments + // or an (unfinished) comment at the end + if (good_parts.empty() == true) + d->End = d->Start; + else + d->Start = d->End; + } + else + { + // the buffer was all comment, but ended with the buffer + if (good_parts.empty() == true && good_start >= d->End) + d->End = d->Start; + else + d->Start = d->End; + } +} +bool pkgTagFile::Fill() +{ + unsigned long long const EndSize = d->End - d->Start; + if (EndSize != 0) + { + memmove(d->Buffer,d->Start,EndSize); + d->Start = d->End = d->Buffer + EndSize; + } + else + d->Start = d->End = d->Buffer; + + unsigned long long Actual = 0; + while (d->Done == false && d->Size > (Actual + 1)) + { + if (FillBuffer(d) == false) + return false; + if ((d->Flags & pkgTagFile::SUPPORT_COMMENTS) != 0) + RemoveCommentsFromBuffer(d); + Actual = d->End - d->Buffer; + } + d->Start = d->Buffer; + + if (d->Done == true) + { + if (EndSize <= 3 && Actual == 0) + return false; + if (d->Size - (d->End - d->Buffer) < 4) + return true; + + // Append a double new line if one does not exist + unsigned int LineCount = 0; + for (const char *E = d->End - 1; E - d->End < 6 && (*E == '\n' || *E == '\r'); E--) + if (*E == '\n') + ++LineCount; + if (LineCount < 2) + { + if (static_cast<unsigned long long>(d->End - d->Buffer) >= d->Size) + Resize(d->Size + 3); + for (; LineCount < 2; ++LineCount) + *d->End++ = '\n'; + } + } + return true; +} + /*}}}*/ +// TagFile::Jump - Jump to a pre-recorded location in the file /*{{{*/ +// --------------------------------------------------------------------- +/* This jumps to a pre-recorded file location and reads the record + that is there */ +bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long long Offset) +{ + if ((d->Flags & pkgTagFile::SUPPORT_COMMENTS) == 0 && + // We are within a buffer space of the next hit.. + Offset >= d->iOffset && d->iOffset + (d->End - d->Start) > Offset) + { + unsigned long long Dist = Offset - d->iOffset; + d->Start += Dist; + d->iOffset += Dist; + // if we have seen the end, don't ask for more + if (d->Done == true) + return Tag.Scan(d->Start, d->End - d->Start); + else + return Step(Tag); + } + + // Reposition and reload.. + d->iOffset = Offset; + d->Done = false; + if (d->Fd->Seek(Offset) == false) + return false; + d->End = d->Start = d->Buffer; + d->isCommentedLine = false; + d->chunks.clear(); + + if (Fill() == false) + return false; + + if (Tag.Scan(d->Start, d->End - d->Start) == true) + return true; + + // This appends a double new line (for the real eof handling) + if (Fill() == false) + return false; + + if (Tag.Scan(d->Start, d->End - d->Start, false) == false) + return _error->Error(_("Unable to parse package file %s (%d)"),d->Fd->Name().c_str(), 2); + + return true; +} + /*}}}*/ +// pkgTagSection::pkgTagSection - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgTagSection::pkgTagSection() + : Section(0), d(new pkgTagSectionPrivate()), Stop(0) +{ + memset(&AlphaIndexes, 0, sizeof(AlphaIndexes)); + memset(&BetaIndexes, 0, sizeof(BetaIndexes)); +} + /*}}}*/ +// TagSection::Scan - Scan for the end of the header information /*{{{*/ +bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const Restart) +{ + Section = Start; + const char *End = Start + MaxLength; + + if (Restart == false && d->Tags.empty() == false) + { + Stop = Section + d->Tags.back().StartTag; + if (End <= Stop) + return false; + Stop = (const char *)memchr(Stop,'\n',End - Stop); + if (Stop == NULL) + return false; + ++Stop; + } + else + { + Stop = Section; + if (d->Tags.empty() == false) + { + memset(&AlphaIndexes, 0, sizeof(AlphaIndexes)); + memset(&BetaIndexes, 0, sizeof(BetaIndexes)); + d->Tags.clear(); + } + d->Tags.reserve(0x100); + } + unsigned int TagCount = d->Tags.size(); + + if (Stop == 0) + return false; + + pkgTagSectionPrivate::TagData lastTagData(0); + Key lastTagKey = Key::Unknown; + unsigned int lastTagHash = 0; + while (Stop < End) + { + TrimRecord(true,End); + + // this can happen when TrimRecord trims away the entire Record + // (e.g. because it just contains comments) + if(Stop == End) + return true; + + // Start a new index and add it to the hash + if (isspace_ascii(Stop[0]) == 0) + { + // store the last found tag + if (lastTagData.StartValue != 0) + { + if (lastTagKey != Key::Unknown) { + AlphaIndexes[static_cast<size_t>(lastTagKey)] = TagCount; + } else { + if (BetaIndexes[lastTagHash] != 0) + lastTagData.NextInBucket = BetaIndexes[lastTagHash]; + BetaIndexes[lastTagHash] = TagCount; + } + d->Tags.push_back(lastTagData); + } + + ++TagCount; + lastTagData = pkgTagSectionPrivate::TagData(Stop - Section); + // find the colon separating tag and value + char const * Colon = (char const *) memchr(Stop, ':', End - Stop); + if (Colon == NULL) + return false; + // find the end of the tag (which might or might not be the colon) + char const * EndTag = Colon; + --EndTag; + for (; EndTag > Stop && isspace_ascii(*EndTag) != 0; --EndTag) + ; + ++EndTag; + lastTagData.EndTag = EndTag - Section; + lastTagKey = pkgTagHash(Stop, EndTag - Stop); + if (lastTagKey == Key::Unknown) + lastTagHash = BetaHash(Stop, EndTag - Stop); + // find the beginning of the value + Stop = Colon + 1; + for (; Stop < End && isspace_ascii(*Stop) != 0; ++Stop) + if (*Stop == '\n' && Stop[1] != ' ') + break; + if (Stop >= End) + return false; + lastTagData.StartValue = Stop - Section; + } + + Stop = (const char *)memchr(Stop,'\n',End - Stop); + + if (Stop == 0) + return false; + + for (; Stop+1 < End && Stop[1] == '\r'; Stop++) + /* nothing */ + ; + + // Double newline marks the end of the record + if (Stop+1 < End && Stop[1] == '\n') + { + if (lastTagData.StartValue != 0) + { + if (lastTagKey != Key::Unknown) { + AlphaIndexes[static_cast<size_t>(lastTagKey)] = TagCount; + } else { + if (BetaIndexes[lastTagHash] != 0) + lastTagData.NextInBucket = BetaIndexes[lastTagHash]; + BetaIndexes[lastTagHash] = TagCount; + } + d->Tags.push_back(lastTagData); + } + + pkgTagSectionPrivate::TagData const td(Stop - Section); + d->Tags.push_back(td); + TrimRecord(false,End); + return true; + } + + Stop++; + } + + return false; +} + /*}}}*/ +// TagSection::TrimRecord - Trim off any garbage before/after a record /*{{{*/ +// --------------------------------------------------------------------- +/* There should be exactly 2 newline at the end of the record, no more. */ +void pkgTagSection::TrimRecord(bool BeforeRecord, const char*& End) +{ + if (BeforeRecord == true) + return; + for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r'); Stop++); +} + /*}}}*/ +// TagSection::Trim - Trim off any trailing garbage /*{{{*/ +// --------------------------------------------------------------------- +/* There should be exactly 1 newline at the end of the buffer, no more. */ +void pkgTagSection::Trim() +{ + for (; Stop > Section + 2 && (Stop[-2] == '\n' || Stop[-2] == '\r'); Stop--); +} + /*}}}*/ +// TagSection::Exists - return True if a tag exists /*{{{*/ +bool pkgTagSection::Exists(StringView Tag) const +{ + unsigned int tmp; + return Find(Tag, tmp); +} +bool pkgTagSection::Exists(Key key) const +{ + unsigned int tmp; + return Find(key, tmp); +} + /*}}}*/ +// TagSection::Find - Locate a tag /*{{{*/ +// --------------------------------------------------------------------- +/* This searches the section for a tag that matches the given string. */ +bool pkgTagSection::Find(Key key,unsigned int &Pos) const +{ + auto Bucket = AlphaIndexes[static_cast<size_t>(key)]; + Pos = Bucket - 1; + return Bucket != 0; +} +bool pkgTagSection::Find(StringView TagView,unsigned int &Pos) const +{ + const char * const Tag = TagView.data(); + size_t const Length = TagView.length(); + auto key = pkgTagHash(Tag, Length); + if (key != Key::Unknown) + return Find(key, Pos); + + unsigned int Bucket = BetaIndexes[BetaHash(Tag, Length)]; + if (Bucket == 0) + return false; + + for (; Bucket != 0; Bucket = d->Tags[Bucket - 1].NextInBucket) + { + if ((d->Tags[Bucket - 1].EndTag - d->Tags[Bucket - 1].StartTag) != Length) + continue; + + char const * const St = Section + d->Tags[Bucket - 1].StartTag; + if (strncasecmp(Tag,St,Length) != 0) + continue; + + Pos = Bucket - 1; + return true; + } + + Pos = 0; + return false; +} + +bool pkgTagSection::FindInternal(unsigned int Pos, const char *&Start, + const char *&End) const +{ + if (unlikely(Pos + 1 >= d->Tags.size() || Pos >= d->Tags.size())) + return _error->Error("Internal parsing error"); + + Start = Section + d->Tags[Pos].StartValue; + // Strip off the gunk from the end + End = Section + d->Tags[Pos + 1].StartTag; + if (unlikely(Start > End)) + return _error->Error("Internal parsing error"); + + for (; isspace_ascii(End[-1]) != 0 && End > Start; --End); + + return true; +} +bool pkgTagSection::Find(StringView Tag,const char *&Start, + const char *&End) const +{ + unsigned int Pos; + return Find(Tag, Pos) && FindInternal(Pos, Start, End); +} +bool pkgTagSection::Find(Key key,const char *&Start, + const char *&End) const +{ + unsigned int Pos; + return Find(key, Pos) && FindInternal(Pos, Start, End); +} + /*}}}*/ +// TagSection::FindS - Find a string /*{{{*/ +StringView pkgTagSection::Find(StringView Tag) const +{ + const char *Start; + const char *End; + if (Find(Tag,Start,End) == false) + return StringView(); + return StringView(Start, End - Start); +} +StringView pkgTagSection::Find(Key key) const +{ + const char *Start; + const char *End; + if (Find(key,Start,End) == false) + return StringView(); + return StringView(Start, End - Start); +} + /*}}}*/ +// TagSection::FindRawS - Find a string /*{{{*/ +StringView pkgTagSection::FindRawInternal(unsigned int Pos) const +{ + if (unlikely(Pos + 1 >= d->Tags.size() || Pos >= d->Tags.size())) + return _error->Error("Internal parsing error"), ""; + + char const *Start = (char const *) memchr(Section + d->Tags[Pos].EndTag, ':', d->Tags[Pos].StartValue - d->Tags[Pos].EndTag); + char const *End = Section + d->Tags[Pos + 1].StartTag; + + if (Start == nullptr) + return ""; + + ++Start; + + if (unlikely(Start > End)) + return ""; + + for (; isspace_ascii(End[-1]) != 0 && End > Start; --End); + + return StringView(Start, End - Start); +} +StringView pkgTagSection::FindRaw(StringView Tag) const +{ + unsigned int Pos; + return Find(Tag, Pos) ? FindRawInternal(Pos) : ""; +} +StringView pkgTagSection::FindRaw(Key key) const +{ + unsigned int Pos; + return Find(key, Pos) ? FindRawInternal(Pos) : ""; +} + /*}}}*/ +// TagSection::FindI - Find an integer /*{{{*/ +// --------------------------------------------------------------------- +/* */ +signed int pkgTagSection::FindIInternal(unsigned int Pos,signed long Default) const +{ + const char *Start; + const char *Stop; + if (FindInternal(Pos,Start,Stop) == false) + return Default; + + // Copy it into a temp buffer so we can use strtol + char S[300]; + if ((unsigned)(Stop - Start) >= sizeof(S)) + return Default; + strncpy(S,Start,Stop-Start); + S[Stop - Start] = 0; + + errno = 0; + char *End; + signed long Result = strtol(S,&End,10); + if (errno == ERANGE || + Result < std::numeric_limits<int>::min() || Result > std::numeric_limits<int>::max()) { + errno = ERANGE; + _error->Error(_("Cannot convert %s to integer: out of range"), S); + } + if (S == End) + return Default; + return Result; +} +signed int pkgTagSection::FindI(Key key,signed long Default) const +{ + unsigned int Pos; + + return Find(key, Pos) ? FindIInternal(Pos) : Default; +} +signed int pkgTagSection::FindI(StringView Tag,signed long Default) const +{ + unsigned int Pos; + + return Find(Tag, Pos) ? FindIInternal(Pos, Default) : Default; +} + /*}}}*/ +// TagSection::FindULL - Find an unsigned long long integer /*{{{*/ +// --------------------------------------------------------------------- +/* */ +unsigned long long pkgTagSection::FindULLInternal(unsigned int Pos, unsigned long long const &Default) const +{ + const char *Start; + const char *Stop; + if (FindInternal(Pos,Start,Stop) == false) + return Default; + + // Copy it into a temp buffer so we can use strtoull + char S[100]; + if ((unsigned)(Stop - Start) >= sizeof(S)) + return Default; + strncpy(S,Start,Stop-Start); + S[Stop - Start] = 0; + + char *End; + unsigned long long Result = strtoull(S,&End,10); + if (S == End) + return Default; + return Result; +} +unsigned long long pkgTagSection::FindULL(Key key, unsigned long long const &Default) const +{ + unsigned int Pos; + + return Find(key, Pos) ? FindULLInternal(Pos, Default) : Default; +} +unsigned long long pkgTagSection::FindULL(StringView Tag, unsigned long long const &Default) const +{ + unsigned int Pos; + + return Find(Tag, Pos) ? FindULLInternal(Pos, Default) : Default; +} + /*}}}*/ +// TagSection::FindB - Find boolean value /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgTagSection::FindBInternal(unsigned int Pos, bool Default) const +{ + const char *Start, *Stop; + if (FindInternal(Pos, Start, Stop) == false) + return Default; + return StringToBool(string(Start, Stop)); +} +bool pkgTagSection::FindB(Key key, bool Default) const +{ + unsigned int Pos; + return Find(key, Pos) ? FindBInternal(Pos, Default): Default; +} +bool pkgTagSection::FindB(StringView Tag, bool Default) const +{ + unsigned int Pos; + return Find(Tag, Pos) ? FindBInternal(Pos, Default) : Default; +} + /*}}}*/ +// TagSection::FindFlag - Locate a yes/no type flag /*{{{*/ +// --------------------------------------------------------------------- +/* The bits marked in Flag are masked on/off in Flags */ +bool pkgTagSection::FindFlagInternal(unsigned int Pos, uint8_t &Flags, + uint8_t const Flag) const +{ + const char *Start; + const char *Stop; + if (FindInternal(Pos,Start,Stop) == false) + return true; + return FindFlag(Flags, Flag, Start, Stop); +} +bool pkgTagSection::FindFlag(Key key, uint8_t &Flags, + uint8_t const Flag) const +{ + unsigned int Pos; + if (Find(key,Pos) == false) + return true; + return FindFlagInternal(Pos, Flags, Flag); +} +bool pkgTagSection::FindFlag(StringView Tag, uint8_t &Flags, + uint8_t const Flag) const +{ + unsigned int Pos; + if (Find(Tag,Pos) == false) + return true; + return FindFlagInternal(Pos, Flags, Flag); +} +bool pkgTagSection::FindFlag(uint8_t &Flags, uint8_t const Flag, + char const* const Start, char const* const Stop) +{ + switch (StringToBool(string(Start, Stop))) + { + case 0: + Flags &= ~Flag; + return true; + + case 1: + Flags |= Flag; + return true; + + default: + _error->Warning("Unknown flag value: %s",string(Start,Stop).c_str()); + return true; + } + return true; +} +bool pkgTagSection::FindFlagInternal(unsigned int Pos,unsigned long &Flags, + unsigned long Flag) const +{ + const char *Start; + const char *Stop; + if (FindInternal(Pos,Start,Stop) == false) + return true; + return FindFlag(Flags, Flag, Start, Stop); +} +bool pkgTagSection::FindFlag(Key key,unsigned long &Flags, + unsigned long Flag) const +{ + unsigned int Pos; + return Find(key, Pos) ? FindFlagInternal(Pos, Flags, Flag) : true; +} +bool pkgTagSection::FindFlag(StringView Tag,unsigned long &Flags, + unsigned long Flag) const +{ + unsigned int Pos; + return Find(Tag, Pos) ? FindFlagInternal(Pos, Flags, Flag) : true; +} +bool pkgTagSection::FindFlag(unsigned long &Flags, unsigned long Flag, + char const* Start, char const* Stop) +{ + switch (StringToBool(string(Start, Stop))) + { + case 0: + Flags &= ~Flag; + return true; + + case 1: + Flags |= Flag; + return true; + + default: + _error->Warning("Unknown flag value: %s",string(Start,Stop).c_str()); + return true; + } + return true; +} + /*}}}*/ +void pkgTagSection::Get(const char *&Start,const char *&Stop,unsigned int I) const/*{{{*/ +{ + if (unlikely(I + 1 >= d->Tags.size() || I >= d->Tags.size())) + abort(); + Start = Section + d->Tags[I].StartTag; + Stop = Section + d->Tags[I+1].StartTag; +} + /*}}}*/ +APT_PURE unsigned int pkgTagSection::Count() const { /*{{{*/ + if (d->Tags.empty() == true) + return 0; + // the last element is just marking the end and isn't a real one + return d->Tags.size() - 1; +} + /*}}}*/ +// TagSection::Write - Ordered (re)writing of fields /*{{{*/ +pkgTagSection::Tag pkgTagSection::Tag::Remove(std::string const &Name) +{ + return Tag(REMOVE, Name, ""); +} +pkgTagSection::Tag pkgTagSection::Tag::Rename(std::string const &OldName, std::string const &NewName) +{ + return Tag(RENAME, OldName, NewName); +} +pkgTagSection::Tag pkgTagSection::Tag::Rewrite(std::string const &Name, std::string const &Data) +{ + if (Data.empty() == true) + return Tag(REMOVE, Name, ""); + else + return Tag(REWRITE, Name, Data); +} +static bool WriteTag(FileFd &File, std::string Tag, StringView Value) +{ + if (Value.empty() || isspace_ascii(Value[0]) != 0) + Tag.append(":"); + else + Tag.append(": "); + Tag.append(Value.data(), Value.length()); + Tag.append("\n"); + return File.Write(Tag.c_str(), Tag.length()); +} +static bool RewriteTags(FileFd &File, pkgTagSection const * const This, char const * const Tag, + std::vector<pkgTagSection::Tag>::const_iterator &R, + std::vector<pkgTagSection::Tag>::const_iterator const &REnd) +{ + size_t const TagLen = strlen(Tag); + for (; R != REnd; ++R) + { + std::string data; + if (R->Name.length() == TagLen && strncasecmp(R->Name.c_str(), Tag, R->Name.length()) == 0) + { + if (R->Action != pkgTagSection::Tag::REWRITE) + break; + data = R->Data; + } + else if(R->Action == pkgTagSection::Tag::RENAME && R->Data.length() == TagLen && + strncasecmp(R->Data.c_str(), Tag, R->Data.length()) == 0) + data = This->FindRaw(R->Name.c_str()).to_string(); + else + continue; + + return WriteTag(File, Tag, data); + } + return true; +} +bool pkgTagSection::Write(FileFd &File, char const * const * const Order, std::vector<Tag> const &Rewrite) const +{ + // first pass: Write everything we have an order for + if (Order != NULL) + { + for (unsigned int I = 0; Order[I] != 0; ++I) + { + std::vector<Tag>::const_iterator R = Rewrite.begin(); + if (RewriteTags(File, this, Order[I], R, Rewrite.end()) == false) + return false; + if (R != Rewrite.end()) + continue; + + if (Exists(Order[I]) == false) + continue; + + if (WriteTag(File, Order[I], FindRaw(Order[I])) == false) + return false; + } + } + // second pass: See if we have tags which aren't ordered + if (d->Tags.empty() == false) + { + for (std::vector<pkgTagSectionPrivate::TagData>::const_iterator T = d->Tags.begin(); T != d->Tags.end() - 1; ++T) + { + char const * const fieldname = Section + T->StartTag; + size_t fieldnamelen = T->EndTag - T->StartTag; + if (Order != NULL) + { + unsigned int I = 0; + for (; Order[I] != 0; ++I) + { + if (fieldnamelen == strlen(Order[I]) && strncasecmp(fieldname, Order[I], fieldnamelen) == 0) + break; + } + if (Order[I] != 0) + continue; + } + + std::string const name(fieldname, fieldnamelen); + std::vector<Tag>::const_iterator R = Rewrite.begin(); + if (RewriteTags(File, this, name.c_str(), R, Rewrite.end()) == false) + return false; + if (R != Rewrite.end()) + continue; + + if (WriteTag(File, name, FindRaw(name)) == false) + return false; + } + } + // last pass: see if there are any rewrites remaining we haven't done yet + for (std::vector<Tag>::const_iterator R = Rewrite.begin(); R != Rewrite.end(); ++R) + { + if (R->Action == Tag::REMOVE) + continue; + auto const name = ((R->Action == Tag::RENAME) ? R->Data : R->Name); + if (Exists(name)) + continue; + if (Order != NULL) + { + unsigned int I = 0; + for (; Order[I] != 0; ++I) + { + if (strncasecmp(name.c_str(), Order[I], name.length()) == 0 && name.length() == strlen(Order[I])) + break; + } + if (Order[I] != 0) + continue; + } + + if (WriteTag(File, name, ((R->Action == Tag::RENAME) ? FindRaw(R->Name) : R->Data)) == false) + return false; + } + return true; +} + /*}}}*/ + +#include "tagfile-order.c" + +pkgTagSection::~pkgTagSection() { delete d; } diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h new file mode 100644 index 0000000..a9f5814 --- /dev/null +++ b/apt-pkg/tagfile.h @@ -0,0 +1,207 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Fast scanner for RFC-822 type header information + + This parser handles Debian package files (and others). Their form is + RFC-822 type header fields in groups separated by a blank line. + + The parser reads the file and provides methods to step linearly + over it or to jump to a pre-recorded start point and read that record. + + A second class is used to perform pre-parsing of the record. It works + by indexing the start of each header field and providing lookup + functions for header fields. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_TAGFILE_H +#define PKGLIB_TAGFILE_H + +#include <apt-pkg/macros.h> + +#include <stdint.h> +#include <stdio.h> + +#include <list> +#include <string> +#include <vector> +#include <apt-pkg/string_view.h> + + +class FileFd; +class pkgTagSectionPrivate; +class pkgTagFilePrivate; + +/** \class pkgTagSection parses a single deb822 stanza and provides various Find methods + * to extract the included values. It can also be used to modify and write a + * valid deb822 stanza optionally (re)ordering the fields inside the stanza. + * + * Beware: This class does \b NOT support (#-)comments in in- or output! + * If the input contains comments they have to be stripped first like pkgTagFile + * does with SUPPORT_COMMENTS flag set. */ +class APT_PUBLIC pkgTagSection +{ + const char *Section; + unsigned int AlphaIndexes[128]; + unsigned int BetaIndexes[128]; + + pkgTagSectionPrivate * const d; + + APT_HIDDEN bool FindInternal(unsigned int Pos,const char *&Start, const char *&End) const; + APT_HIDDEN APT::StringView FindInternal(unsigned int Pos) const; + APT_HIDDEN APT::StringView FindRawInternal(unsigned int Pos) const; + APT_HIDDEN signed int FindIInternal(unsigned int Pos,signed long Default = 0) const; + APT_HIDDEN bool FindBInternal(unsigned int Pos, bool Default = false) const; + APT_HIDDEN unsigned long long FindULLInternal(unsigned int Pos, unsigned long long const &Default = 0) const; + APT_HIDDEN bool FindFlagInternal(unsigned int Pos,uint8_t &Flags, uint8_t const Flag) const; + APT_HIDDEN bool FindFlagInternal(unsigned int Pos,unsigned long &Flags, unsigned long Flag) const; + + protected: + const char *Stop; + + public: + + inline bool operator ==(const pkgTagSection &rhs) {return Section == rhs.Section;}; + inline bool operator !=(const pkgTagSection &rhs) {return Section != rhs.Section;}; + + // TODO: Remove internally + std::string FindS(APT::StringView sv) const { return Find(sv).to_string(); } + std::string FindRawS(APT::StringView sv) const { return FindRaw(sv).to_string(); }; + + // Functions for lookup with a perfect hash function + enum class Key; +#ifdef APT_COMPILING_APT + bool Find(Key key,const char *&Start, const char *&End) const; + bool Find(Key key,unsigned int &Pos) const; + signed int FindI(Key key,signed long Default = 0) const; + bool FindB(Key key, bool Default = false) const; + unsigned long long FindULL(Key key, unsigned long long const &Default = 0) const; + bool FindFlag(Key key,uint8_t &Flags, uint8_t const Flag) const; + bool FindFlag(Key key,unsigned long &Flags, unsigned long Flag) const; + bool Exists(Key key) const; + APT::StringView Find(Key key) const; + APT::StringView FindRaw(Key key) const; +#endif + + bool Find(APT::StringView Tag,const char *&Start, const char *&End) const; + bool Find(APT::StringView Tag,unsigned int &Pos) const; + APT::StringView Find(APT::StringView Tag) const; + APT::StringView FindRaw(APT::StringView Tag) const; + signed int FindI(APT::StringView Tag,signed long Default = 0) const; + bool FindB(APT::StringView, bool Default = false) const; + unsigned long long FindULL(APT::StringView Tag, unsigned long long const &Default = 0) const; + + bool FindFlag(APT::StringView Tag,uint8_t &Flags, + uint8_t const Flag) const; + bool FindFlag(APT::StringView Tag,unsigned long &Flags, + unsigned long Flag) const; + bool Exists(APT::StringView Tag) const; + + bool static FindFlag(uint8_t &Flags, uint8_t const Flag, + const char* const Start, const char* const Stop); + bool static FindFlag(unsigned long &Flags, unsigned long Flag, + const char* Start, const char* Stop); + + /** \brief searches the boundaries of the current section + * + * While parameter Start marks the beginning of the section, this method + * will search for the first double newline in the data stream which marks + * the end of the section. It also does a first pass over the content of + * the section parsing it as encountered for processing later on by Find + * + * @param Start is the beginning of the section + * @param MaxLength is the size of valid data in the stream pointed to by Start + * @param Restart if enabled internal state will be cleared, otherwise it is + * assumed that now more data is available in the stream and the parsing will + * start were it encountered insufficient data the last time. + * + * @return \b true if section end was found, \b false otherwise. + * Beware that internal state will be inconsistent if \b false is returned! + */ + APT_MUSTCHECK bool Scan(const char *Start, unsigned long MaxLength, bool const Restart = true); + + inline unsigned long size() const {return Stop - Section;}; + void Trim(); + virtual void TrimRecord(bool BeforeRecord, const char* &End); + + /** \brief amount of Tags in the current section + * + * Note: if a Tag is mentioned repeatedly it will be counted multiple + * times, but only the last occurrence is available via Find methods. + */ + unsigned int Count() const; + + void Get(const char *&Start,const char *&Stop,unsigned int I) const; + + inline void GetSection(const char *&Start,const char *&Stop) const + { + Start = Section; + Stop = this->Stop; + }; + + pkgTagSection(); + virtual ~pkgTagSection(); + + struct Tag + { + enum ActionType { REMOVE, RENAME, REWRITE } Action; + std::string Name; + std::string Data; + + static Tag Remove(std::string const &Name); + static Tag Rename(std::string const &OldName, std::string const &NewName); + static Tag Rewrite(std::string const &Name, std::string const &Data); + private: + Tag(ActionType const Action, std::string const &Name, std::string const &Data) : + Action(Action), Name(Name), Data(Data) {} + }; + + /** Write this section (with optional rewrites) to a file + * + * @param File to write the section to + * @param Order in which tags should appear in the file + * @param Rewrite is a set of tags to be renamed, rewritten and/or removed + * @return \b true if successful, otherwise \b false + */ + bool Write(FileFd &File, char const * const * const Order = NULL, std::vector<Tag> const &Rewrite = std::vector<Tag>()) const; +}; + + +/** \class pkgTagFile reads and prepares a deb822 formatted file for parsing + * via #pkgTagSection. The default mode tries to be as fast as possible and + * assumes perfectly valid (machine generated) files like Packages. Support + * for comments e.g. needs to be enabled explicitly. */ +class APT_PUBLIC pkgTagFile +{ + pkgTagFilePrivate * const d; + + APT_HIDDEN bool Fill(); + APT_HIDDEN bool Resize(); + APT_HIDDEN bool Resize(unsigned long long const newSize); + +public: + + bool Step(pkgTagSection &Section); + unsigned long Offset(); + bool Jump(pkgTagSection &Tag,unsigned long long Offset); + + enum Flags + { + STRICT = 0, + SUPPORT_COMMENTS = 1 << 0, + }; + + void Init(FileFd * const F, pkgTagFile::Flags const Flags, unsigned long long Size = 32*1024); + void Init(FileFd * const F,unsigned long long const Size = 32*1024); + + pkgTagFile(FileFd * const F, pkgTagFile::Flags const Flags, unsigned long long Size = 32*1024); + pkgTagFile(FileFd * const F,unsigned long long Size = 32*1024); + virtual ~pkgTagFile(); +}; + +APT_PUBLIC extern const char **TFRewritePackageOrder; +APT_PUBLIC extern const char **TFRewriteSourceOrder; + +#endif diff --git a/apt-pkg/update.cc b/apt-pkg/update.cc new file mode 100644 index 0000000..1bf818a --- /dev/null +++ b/apt-pkg/update.cc @@ -0,0 +1,152 @@ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/acquire-item.h> +#include <apt-pkg/acquire.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/sourcelist.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/update.h> + +#include <string> + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +// ListUpdate - construct Fetcher and update the cache files /*{{{*/ +// --------------------------------------------------------------------- +/* This is a simple wrapper to update the cache. it will fetch stuff + * from the network (or any other sources defined in sources.list) + */ +bool ListUpdate(pkgAcquireStatus &Stat, + pkgSourceList &List, + int PulseInterval) +{ + pkgAcquire Fetcher(&Stat); + if (Fetcher.GetLock(_config->FindDir("Dir::State::Lists")) == false) + return false; + + // Populate it with the source selection + if (List.GetIndexes(&Fetcher) == false) + return false; + + return AcquireUpdate(Fetcher, PulseInterval, true); +} + /*}}}*/ +// AcquireUpdate - take Fetcher and update the cache files /*{{{*/ +// --------------------------------------------------------------------- +/* This is a simple wrapper to update the cache with a provided acquire + * If you only need control over Status and the used SourcesList use + * ListUpdate method instead. + */ +bool AcquireUpdate(pkgAcquire &Fetcher, int const PulseInterval, + bool const RunUpdateScripts, bool const ListCleanup) +{ + enum class ErrorMode + { + Persistent, + Any + }; + std::string errorModeS = _config->Find("APT::Update::Error-Mode", "persistent"); + ErrorMode errorMode = ErrorMode::Persistent; + if (errorModeS == "persistent") + errorMode = ErrorMode::Persistent; + else if (errorModeS == "any") + errorMode = ErrorMode::Any; + else + return _error->Error("Unknown update error mode %s", errorModeS.c_str()); + + // Run scripts + if (RunUpdateScripts == true) + RunScripts("APT::Update::Pre-Invoke"); + + pkgAcquire::RunResult res; + if(PulseInterval > 0) + res = Fetcher.Run(PulseInterval); + else + res = Fetcher.Run(); + + bool const errorsWereReported = (res == pkgAcquire::Failed); + bool Failed = errorsWereReported; + bool TransientNetworkFailure = false; + bool AllFailed = true; + for (pkgAcquire::ItemIterator I = Fetcher.ItemsBegin(); + I != Fetcher.ItemsEnd(); ++I) + { + switch ((*I)->Status) + { + case pkgAcquire::Item::StatDone: + AllFailed = false; + continue; + case pkgAcquire::Item::StatTransientNetworkError: + if (errorMode == ErrorMode::Any) + Failed = true; + else + TransientNetworkFailure = true; + break; + case pkgAcquire::Item::StatIdle: + case pkgAcquire::Item::StatFetching: + case pkgAcquire::Item::StatError: + case pkgAcquire::Item::StatAuthError: + Failed = true; + break; + } + + (*I)->Finished(); + + if (errorsWereReported) + continue; + + ::URI uri((*I)->DescURI()); + uri.User.clear(); + uri.Password.clear(); + if ((*I)->Local) + uri.Path = DeQuoteString(uri.Path); + std::string const descUri = std::string(uri); + // Show an error for non-transient failures, otherwise only warn + if ((*I)->Status == pkgAcquire::Item::StatTransientNetworkError && errorMode != ErrorMode::Any) + _error->Warning(_("Failed to fetch %s %s"), descUri.c_str(), + (*I)->ErrorText.c_str()); + else + _error->Error(_("Failed to fetch %s %s"), descUri.c_str(), + (*I)->ErrorText.c_str()); + } + + // Clean out any old list files + // Keep "APT::Get::List-Cleanup" name for compatibility, but + // this is really a global option for the APT library now + if (!TransientNetworkFailure && !Failed && ListCleanup == true && + (_config->FindB("APT::Get::List-Cleanup",true) == true && + _config->FindB("APT::List-Cleanup",true) == true)) + { + if (Fetcher.Clean(_config->FindDir("Dir::State::lists")) == false || + Fetcher.Clean(_config->FindDir("Dir::State::lists") + "partial/") == false) + // something went wrong with the clean + return false; + } + + bool Res = true; + + if (errorsWereReported == true) + Res = false; + else if (TransientNetworkFailure == true) + Res = _error->Warning(_("Some index files failed to download. They have been ignored, or old ones used instead.")); + else if (Failed == true) + Res = _error->Error(_("Some index files failed to download. They have been ignored, or old ones used instead.")); + + // Run the success scripts if all was fine + if (RunUpdateScripts == true) + { + if(AllFailed == false) + RunScripts("APT::Update::Post-Invoke-Success"); + + // Run the other scripts + RunScripts("APT::Update::Post-Invoke"); + } + return Res; +} + /*}}}*/ diff --git a/apt-pkg/update.h b/apt-pkg/update.h new file mode 100644 index 0000000..dce5cbc --- /dev/null +++ b/apt-pkg/update.h @@ -0,0 +1,22 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Update - ListUpdate related code + + ##################################################################### */ + /*}}}*/ + +#ifndef PKGLIB_UPDATE_H +#define PKGLIB_UPDATE_H + +class pkgAcquireStatus; +class pkgSourceList; +class pkgAcquire; + +APT_PUBLIC bool ListUpdate(pkgAcquireStatus &progress, pkgSourceList &List, int PulseInterval=0); +APT_PUBLIC bool AcquireUpdate(pkgAcquire &Fetcher, int const PulseInterval = 0, + bool const RunUpdateScripts = true, bool const ListCleanup = true); + + +#endif diff --git a/apt-pkg/upgrade.cc b/apt-pkg/upgrade.cc new file mode 100644 index 0000000..e3e98e5 --- /dev/null +++ b/apt-pkg/upgrade.cc @@ -0,0 +1,393 @@ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/algorithms.h> +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/depcache.h> +#include <apt-pkg/edsp.h> +#include <apt-pkg/error.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/progress.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/upgrade.h> + +#include <random> +#include <string> + +#include <apti18n.h> + /*}}}*/ + +struct PhasedUpgrader +{ + std::string machineID; + bool isChroot; + + PhasedUpgrader() + { + machineID = APT::Configuration::getMachineID(); + } + + // See if this version is a security update. This also checks, for installed packages, + // if any of the previous versions is a security update + bool IsSecurityUpdate(pkgCache::VerIterator const &Ver) + { + auto Pkg = Ver.ParentPkg(); + auto Installed = Pkg.CurrentVer(); + + auto OtherVer = Pkg.VersionList(); + + // Advance to first version < our version + while (OtherVer->ID != Ver->ID) + ++OtherVer; + ++OtherVer; + + // Iterate over all versions < our version + for (; !OtherVer.end() && (Installed.end() || OtherVer->ID != Installed->ID); OtherVer++) + { + for (auto PF = OtherVer.FileList(); !PF.end(); PF++) + if (PF.File() && PF.File().Archive() != nullptr && APT::String::Endswith(PF.File().Archive(), "-security")) + return true; + } + return false; + } + + // Check if this version is a phased update that should be ignored + bool IsIgnoredPhasedUpdate(pkgCache::VerIterator const &Ver) + { + if (_config->FindB("APT::Get::Phase-Policy", false)) + return false; + + // The order and fallbacks for the always/never checks come from update-manager and exist + // to preserve compatibility. + if (_config->FindB("APT::Get::Always-Include-Phased-Updates", + _config->FindB("Update-Manager::Always-Include-Phased-Updates", false))) + return false; + + if (_config->FindB("APT::Get::Never-Include-Phased-Updates", + _config->FindB("Update-Manager::Never-Include-Phased-Updates", false))) + return true; + + if (machineID.empty() // no machine-id + || getenv("SOURCE_DATE_EPOCH") != nullptr // reproducible build - always include + || APT::Configuration::isChroot()) + return false; + + std::string seedStr = std::string(Ver.SourcePkgName()) + "-" + Ver.SourceVerStr() + "-" + machineID; + std::seed_seq seed(seedStr.begin(), seedStr.end()); + std::minstd_rand rand(seed); + std::uniform_int_distribution<unsigned int> dist(0, 100); + + return dist(rand) > Ver.PhasedUpdatePercentage(); + } + + bool ShouldKeep(pkgDepCache &Cache, pkgCache::PkgIterator Pkg) + { + if (Pkg->CurrentVer == 0) + return false; + if (Cache[Pkg].InstallVer == 0) + return false; + if (Cache[Pkg].InstVerIter(Cache).PhasedUpdatePercentage() == 100) + return false; + if (IsSecurityUpdate(Cache[Pkg].InstVerIter(Cache))) + return false; + if (!IsIgnoredPhasedUpdate(Cache[Pkg].InstVerIter(Cache))) + return false; + + return true; + } + + // Hold back upgrades to phased versions of already installed packages, unless + // they are security updates + void HoldBackIgnoredPhasedUpdates(pkgDepCache &Cache, pkgProblemResolver *Fix) + { + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + if (not ShouldKeep(Cache, I)) + continue; + + Cache.MarkKeep(I, false, false); + Cache.MarkProtected(I); + if (Fix != nullptr) + Fix->Protect(I); + } + } +}; + +// DistUpgrade - Distribution upgrade /*{{{*/ +// --------------------------------------------------------------------- +/* This autoinstalls every package and then force installs every + pre-existing package. This creates the initial set of conditions which + most likely contain problems because too many things were installed. + + The problem resolver is used to resolve the problems. + */ +static bool pkgDistUpgrade(pkgDepCache &Cache, OpProgress * const Progress) +{ + std::string const solver = _config->Find("APT::Solver", "internal"); + auto const ret = EDSP::ResolveExternal(solver.c_str(), Cache, EDSP::Request::UPGRADE_ALL, Progress); + if (solver != "internal") + return ret; + + if (Progress != NULL) + Progress->OverallProgress(0, 100, 1, _("Calculating upgrade")); + + pkgDepCache::ActionGroup group(Cache); + + PhasedUpgrader().HoldBackIgnoredPhasedUpdates(Cache, nullptr); + + /* Upgrade all installed packages first without autoinst to help the resolver + in versioned or-groups to upgrade the old solver instead of installing + a new one (if the old solver is not the first one [anymore]) */ + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + if (I->CurrentVer != 0) + Cache.MarkInstall(I, false, 0, false); + + if (Progress != NULL) + Progress->Progress(10); + + /* Auto upgrade all installed packages, this provides the basis + for the installation */ + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + if (I->CurrentVer != 0) + Cache.MarkInstall(I, true, 0, false); + + if (Progress != NULL) + Progress->Progress(50); + + /* Now, install each essential package which is not installed + (and not provided by another package in the same name group) */ + std::string essential = _config->Find("pkgCacheGen::Essential", "all"); + if (essential == "all") + { + for (pkgCache::GrpIterator G = Cache.GrpBegin(); G.end() == false; ++G) + { + bool isEssential = false; + bool instEssential = false; + for (pkgCache::PkgIterator P = G.PackageList(); P.end() == false; P = G.NextPkg(P)) + { + if ((P->Flags & pkgCache::Flag::Essential) != pkgCache::Flag::Essential) + continue; + isEssential = true; + if (Cache[P].Install() == true) + { + instEssential = true; + break; + } + } + if (isEssential == false || instEssential == true) + continue; + pkgCache::PkgIterator P = G.FindPreferredPkg(); + Cache.MarkInstall(P, true, 0, false); + } + } + else if (essential != "none") + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + if ((I->Flags & pkgCache::Flag::Essential) == pkgCache::Flag::Essential) + Cache.MarkInstall(I, true, 0, false); + + if (Progress != NULL) + Progress->Progress(55); + + /* We do it again over all previously installed packages to force + conflict resolution on them all. */ + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + if (I->CurrentVer != 0) + Cache.MarkInstall(I, false, 0, false); + + if (Progress != NULL) + Progress->Progress(65); + + pkgProblemResolver Fix(&Cache); + + if (Progress != NULL) + Progress->Progress(95); + + // Hold back held packages. + if (_config->FindB("APT::Ignore-Hold",false) == false) + { + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + if (I->SelectedState == pkgCache::State::Hold) + { + Fix.Protect(I); + Cache.MarkKeep(I, false, false); + } + } + } + + PhasedUpgrader().HoldBackIgnoredPhasedUpdates(Cache, &Fix); + + bool const success = Fix.ResolveInternal(false); + if (Progress != NULL) + Progress->Done(); + return success; +} /*}}}*/ +// AllUpgradeNoNewPackages - Upgrade but no removals or new pkgs /*{{{*/ +static bool pkgAllUpgradeNoNewPackages(pkgDepCache &Cache, OpProgress * const Progress) +{ + std::string const solver = _config->Find("APT::Solver", "internal"); + constexpr auto flags = EDSP::Request::UPGRADE_ALL | EDSP::Request::FORBID_NEW_INSTALL | EDSP::Request::FORBID_REMOVE; + auto const ret = EDSP::ResolveExternal(solver.c_str(), Cache, flags, Progress); + if (solver != "internal") + return ret; + + if (Progress != NULL) + Progress->OverallProgress(0, 100, 1, _("Calculating upgrade")); + + pkgDepCache::ActionGroup group(Cache); + pkgProblemResolver Fix(&Cache); + PhasedUpgrader phasedUpgrader; + // Upgrade all installed packages + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + if (Cache[I].Install() == true) + Fix.Protect(I); + + if (_config->FindB("APT::Ignore-Hold",false) == false) + if (I->SelectedState == pkgCache::State::Hold) + continue; + + if (phasedUpgrader.ShouldKeep(Cache, I)) + continue; + + if (I->CurrentVer != 0 && Cache[I].InstallVer != 0) + Cache.MarkInstall(I, false, 0, false); + } + + if (Progress != NULL) + Progress->Progress(50); + + phasedUpgrader.HoldBackIgnoredPhasedUpdates(Cache, &Fix); + + // resolve remaining issues via keep + bool const success = Fix.ResolveByKeepInternal(); + if (Progress != NULL) + Progress->Done(); + return success; +} + /*}}}*/ +// AllUpgradeWithNewInstalls - Upgrade + install new packages as needed /*{{{*/ +// --------------------------------------------------------------------- +/* Right now the system must be consistent before this can be called. + * Upgrade as much as possible without deleting anything (useful for + * stable systems) + */ +static bool pkgAllUpgradeWithNewPackages(pkgDepCache &Cache, OpProgress * const Progress) +{ + std::string const solver = _config->Find("APT::Solver", "internal"); + constexpr auto flags = EDSP::Request::UPGRADE_ALL | EDSP::Request::FORBID_REMOVE; + auto const ret = EDSP::ResolveExternal(solver.c_str(), Cache, flags, Progress); + if (solver != "internal") + return ret; + + if (Progress != NULL) + Progress->OverallProgress(0, 100, 1, _("Calculating upgrade")); + + pkgDepCache::ActionGroup group(Cache); + pkgProblemResolver Fix(&Cache); + PhasedUpgrader phasedUpgrader; + + // provide the initial set of stuff we want to upgrade by marking + // all upgradable packages for upgrade + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + if (I->CurrentVer != 0 && Cache[I].InstallVer != 0) + { + if (_config->FindB("APT::Ignore-Hold",false) == false) + if (I->SelectedState == pkgCache::State::Hold) + continue; + if (phasedUpgrader.ShouldKeep(Cache, I)) + continue; + + Cache.MarkInstall(I, false, 0, false); + } + } + + if (Progress != NULL) + Progress->Progress(10); + + // then let auto-install loose + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + if (Cache[I].Install()) + Cache.MarkInstall(I, true, 0, false); + + if (Progress != NULL) + Progress->Progress(50); + + // ... but it may remove stuff, we need to clean up afterwards again + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + if (Cache[I].Delete() == true) + Cache.MarkKeep(I, false, false); + + if (Progress != NULL) + Progress->Progress(60); + + phasedUpgrader.HoldBackIgnoredPhasedUpdates(Cache, &Fix); + + // resolve remaining issues via keep + bool const success = Fix.ResolveByKeepInternal(); + if (Progress != NULL) + Progress->Done(); + return success; +} + /*}}}*/ +// MinimizeUpgrade - Minimizes the set of packages to be upgraded /*{{{*/ +// --------------------------------------------------------------------- +/* This simply goes over the entire set of packages and tries to keep + each package marked for upgrade. If a conflict is generated then + the package is restored. */ +bool pkgMinimizeUpgrade(pkgDepCache &Cache) +{ + pkgDepCache::ActionGroup group(Cache); + + if (Cache.BrokenCount() != 0) + return false; + + // We loop for 10 tries to get the minimal set size. + bool Change = false; + unsigned int Count = 0; + do + { + Change = false; + for (pkgCache::PkgIterator I = Cache.PkgBegin(); I.end() == false; ++I) + { + // Not interesting + if (Cache[I].Upgrade() == false || Cache[I].NewInstall() == true) + continue; + + // Keep it and see if that is OK + Cache.MarkKeep(I, false, false); + if (Cache.BrokenCount() != 0) + Cache.MarkInstall(I, false, 0, false); + else + { + // If keep didn't actually do anything then there was no change.. + if (Cache[I].Upgrade() == false) + Change = true; + } + } + ++Count; + } + while (Change == true && Count < 10); + + if (Cache.BrokenCount() != 0) + return _error->Error("Internal Error in pkgMinimizeUpgrade"); + + return true; +} + /*}}}*/ +// APT::Upgrade::Upgrade - Upgrade using a specific strategy /*{{{*/ +bool APT::Upgrade::Upgrade(pkgDepCache &Cache, int mode, OpProgress * const Progress) +{ + if (mode == ALLOW_EVERYTHING) + return pkgDistUpgrade(Cache, Progress); + else if ((mode & ~FORBID_REMOVE_PACKAGES) == 0) + return pkgAllUpgradeWithNewPackages(Cache, Progress); + else if ((mode & ~(FORBID_REMOVE_PACKAGES|FORBID_INSTALL_NEW_PACKAGES)) == 0) + return pkgAllUpgradeNoNewPackages(Cache, Progress); + else + _error->Error("pkgAllUpgrade called with unsupported mode %i", mode); + return false; +} + /*}}}*/ diff --git a/apt-pkg/upgrade.h b/apt-pkg/upgrade.h new file mode 100644 index 0000000..8e89601 --- /dev/null +++ b/apt-pkg/upgrade.h @@ -0,0 +1,32 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Upgrade - Upgrade/DistUpgrade related code + + ##################################################################### */ + /*}}}*/ + +#ifndef PKGLIB_UPGRADE_H +#define PKGLIB_UPGRADE_H + +#include <apt-pkg/macros.h> +#include <stddef.h> + +class pkgDepCache; +class OpProgress; + +namespace APT { + namespace Upgrade { + // FIXME: make this "enum class UpgradeMode {" once we enable c++11 + enum UpgradeMode { + FORBID_REMOVE_PACKAGES = 1, + FORBID_INSTALL_NEW_PACKAGES = 2, + ALLOW_EVERYTHING = 0 + }; + APT_PUBLIC bool Upgrade(pkgDepCache &Cache, int UpgradeMode, OpProgress * const Progress = NULL); + } +} + +APT_PUBLIC bool pkgMinimizeUpgrade(pkgDepCache &Cache); +#endif diff --git a/apt-pkg/version.cc b/apt-pkg/version.cc new file mode 100644 index 0000000..e0b83e0 --- /dev/null +++ b/apt-pkg/version.cc @@ -0,0 +1,44 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Version - Versioning system.. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/version.h> + +#include <stdlib.h> +#include <string.h> + /*}}}*/ + +static pkgVersioningSystem *VSList[10]; +pkgVersioningSystem **pkgVersioningSystem::GlobalList = VSList; +unsigned long pkgVersioningSystem::GlobalListLen = 0; + +// pkgVS::pkgVersioningSystem - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Link to the global list of versioning systems supported */ +pkgVersioningSystem::pkgVersioningSystem() : Label(NULL) +{ + VSList[GlobalListLen] = this; + ++GlobalListLen; +} + /*}}}*/ +// pkgVS::GetVS - Find a VS by name /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgVersioningSystem *pkgVersioningSystem::GetVS(const char *Label) +{ + for (unsigned I = 0; I != GlobalListLen; I++) + if (strcmp(VSList[I]->Label,Label) == 0) + return VSList[I]; + return 0; +} + /*}}}*/ + + +pkgVersioningSystem::~pkgVersioningSystem() {} diff --git a/apt-pkg/version.h b/apt-pkg/version.h new file mode 100644 index 0000000..eaf667c --- /dev/null +++ b/apt-pkg/version.h @@ -0,0 +1,57 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Version - Versioning system.. + + The versioning system represents how versions are compared, represented + and how dependencies are evaluated. As a general rule versioning + systems are not compatible unless specifically allowed by the + TestCompatibility query. + + The versions are stored in a global list of versions, but that is just + so that they can be queried when someone does 'apt-get -v'. + pkgSystem provides the proper means to access the VS for the active + system. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_VERSION_H +#define PKGLIB_VERSION_H + +#include <apt-pkg/strutl.h> +#include <string> + + +class APT_PUBLIC pkgVersioningSystem +{ + public: + // Global list of VS's + static pkgVersioningSystem **GlobalList; + static unsigned long GlobalListLen; + static pkgVersioningSystem *GetVS(const char *Label) APT_PURE; + + const char *Label; + + // Compare versions.. + virtual int DoCmpVersion(const char *A,const char *Aend, + const char *B,const char *Bend) = 0; + + virtual bool CheckDep(const char *PkgVer,int Op,const char *DepVer) = 0; + virtual int DoCmpReleaseVer(const char *A,const char *Aend, + const char *B,const char *Bend) = 0; + virtual std::string UpstreamVersion(const char *A) = 0; + + // See if the given VS is compatible with this one.. + virtual bool TestCompatibility(pkgVersioningSystem const &Against) + {return this == &Against;}; + + // Shortcuts + APT_MKSTRCMP(CmpVersion,DoCmpVersion); + APT_MKSTRCMP(CmpReleaseVer,DoCmpReleaseVer); + + pkgVersioningSystem(); + virtual ~pkgVersioningSystem(); +}; + +#endif diff --git a/apt-pkg/versionmatch.cc b/apt-pkg/versionmatch.cc new file mode 100644 index 0000000..83a969c --- /dev/null +++ b/apt-pkg/versionmatch.cc @@ -0,0 +1,307 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Version Matching + + This module takes a matching string and a type and locates the version + record that satisfies the constraint described by the matching string. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/error.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/versionmatch.h> + +#include <string> +#include <ctype.h> +#include <fnmatch.h> +#include <regex.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + /*}}}*/ + +using std::string; + +// VersionMatch::pkgVersionMatch - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Break up the data string according to the selected type */ +pkgVersionMatch::pkgVersionMatch(string Data,MatchType Type) : Type(Type) +{ + MatchAll = false; + VerPrefixMatch = false; + RelVerPrefixMatch = false; + + if (Type == None || Data.length() < 1) + return; + + // Cut up the version representation + if (Type == Version) + { + if (Data.end()[-1] == '*') + { + VerPrefixMatch = true; + VerStr = string(Data,0,Data.length()-1); + } + else + VerStr = Data; + return; + } + + if (Type == Release) + { + // All empty = match all + if (Data == "*") + { + MatchAll = true; + return; + } + + // Are we a simple specification? + string::const_iterator I = Data.begin(); + for (; I != Data.end() && *I != '='; ++I); + if (I == Data.end()) + { + // Temporary + if (isdigit(Data[0])) + RelVerStr = Data; + else + RelRelease = Data; + + if (RelVerStr.length() > 0 && RelVerStr.end()[-1] == '*') + { + RelVerPrefixMatch = true; + RelVerStr = string(RelVerStr.begin(),RelVerStr.end()-1); + } + return; + } + + char Spec[300]; + char *Fragments[20]; + snprintf(Spec,sizeof(Spec),"%s",Data.c_str()); + if (TokSplitString(',',Spec,Fragments, + sizeof(Fragments)/sizeof(Fragments[0])) == false) + { + Type = None; + return; + } + + for (unsigned J = 0; Fragments[J] != 0; J++) + { + if (strlen(Fragments[J]) < 3) + continue; + + if (stringcasecmp(Fragments[J],Fragments[J]+2,"v=") == 0) + RelVerStr = Fragments[J]+2; + else if (stringcasecmp(Fragments[J],Fragments[J]+2,"o=") == 0) + RelOrigin = Fragments[J]+2; + else if (stringcasecmp(Fragments[J],Fragments[J]+2,"a=") == 0) + RelArchive = Fragments[J]+2; + else if (stringcasecmp(Fragments[J],Fragments[J]+2,"n=") == 0) + RelCodename = Fragments[J]+2; + else if (stringcasecmp(Fragments[J],Fragments[J]+2,"l=") == 0) + RelLabel = Fragments[J]+2; + else if (stringcasecmp(Fragments[J],Fragments[J]+2,"c=") == 0) + RelComponent = Fragments[J]+2; + else if (stringcasecmp(Fragments[J],Fragments[J]+2,"b=") == 0) + RelArchitecture = Fragments[J]+2; + } + + if (RelVerStr.end()[-1] == '*') + { + RelVerPrefixMatch = true; + RelVerStr = string(RelVerStr.begin(),RelVerStr.end()-1); + } + return; + } + + if (Type == Origin) + { + if (Data[0] == '"' && Data.length() >= 2 && Data.end()[-1] == '"') + OrSite = Data.substr(1, Data.length() - 2); + else + OrSite = Data; + return; + } +} + /*}}}*/ +// VersionMatch::MatchVer - Match a version string with prefixing /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgVersionMatch::MatchVer(const char *A,string B,bool Prefix) +{ + if (A == NULL) + return false; + + const char *Ab = A; + const char *Ae = Ab + strlen(A); + + // Strings are not a compatible size. + if (((unsigned)(Ae - Ab) != B.length() && Prefix == false) || + (unsigned)(Ae - Ab) < B.length()) + return false; + + // Match (leading?) + if (stringcasecmp(B,Ab,Ab + B.length()) == 0) + return true; + + return false; +} + /*}}}*/ +// VersionMatch::Find - Locate the best match for the select type /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgCache::VerIterator pkgVersionMatch::Find(pkgCache::PkgIterator Pkg) +{ + for (auto Ver = Pkg.VersionList(); not Ver.end(); ++Ver) + if (VersionMatches(Ver)) + return Ver; + // check if the package provides itself in a matching version + for (auto Prov = Pkg.ProvidesList(); not Prov.end(); ++Prov) + if (Prov->ProvideVersion != 0 && Prov.OwnerPkg() == Prov.ParentPkg()) + if (MatchVer(Prov.ProvideVersion(), VerStr, VerPrefixMatch) || + ExpressionMatches(VerStr, Prov.ProvideVersion())) + return Prov.OwnerVer(); + return pkgCache::VerIterator{}; +} + /*}}}*/ + +// VersionMatch::Find - Locate the best match for the select type /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgVersionMatch::VersionMatches(pkgCache::VerIterator Ver) +{ + if (Type == Version) + { + if (MatchVer(Ver.VerStr(),VerStr,VerPrefixMatch) == true) + return true; + if (ExpressionMatches(VerStr, Ver.VerStr()) == true) + return true; + return false; + } + + for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF) + if (FileMatch(VF.File()) == true) + return true; + + return false; +} + /*}}}*/ + +#ifndef FNM_CASEFOLD +#define FNM_CASEFOLD 0 +#endif + +bool pkgVersionMatch::ExpressionMatches(const char *pattern, const char *string)/*{{{*/ +{ + if (pattern == NULL || string == NULL) + return false; + if (pattern[0] == '/') { + size_t length = strlen(pattern); + if (pattern[length - 1] == '/') { + bool res = false; + regex_t preg; + char *regex = strdup(pattern + 1); + regex[length - 2] = '\0'; + if (regcomp(&preg, regex, REG_EXTENDED | REG_ICASE) != 0) { + _error->Warning("Invalid regular expression: %s", regex); + } else if (regexec(&preg, string, 0, NULL, 0) == 0) { + res = true; + } + free(regex); + regfree(&preg); + return res; + } + } + return fnmatch(pattern, string, FNM_CASEFOLD) == 0; +} +bool pkgVersionMatch::ExpressionMatches(const std::string& pattern, const char *string) +{ + return ExpressionMatches(pattern.c_str(), string); +} + /*}}}*/ +// VersionMatch::FileMatch - Match against an index file /*{{{*/ +// --------------------------------------------------------------------- +/* This matcher checks against the release file and the origin location + to see if the constraints are met. */ +bool pkgVersionMatch::FileMatch(pkgCache::RlsFileIterator const &File) +{ + if (Type == Release) + { + if (MatchAll) + return true; + + if (RelVerStr.empty() && RelOrigin.empty() && + RelArchive.empty() && RelLabel.empty() && + RelRelease.empty() && RelCodename.empty() && + RelComponent.empty() && RelArchitecture.empty()) + return false; + + if (not RelVerStr.empty() && not MatchVer(File.Version(), RelVerStr, RelVerPrefixMatch) && + not ExpressionMatches(RelVerStr, File.Version())) + return false; + if (not RelOrigin.empty() && not ExpressionMatches(RelOrigin, File.Origin())) + return false; + if (not RelArchive.empty() && not ExpressionMatches(RelArchive, File.Archive())) + return false; + if (not RelCodename.empty() && not ExpressionMatches(RelCodename, File.Codename())) + return false; + if (not RelRelease.empty() && not ExpressionMatches(RelRelease, File.Archive()) && + not ExpressionMatches(RelRelease, File.Codename())) + return false; + if (not RelLabel.empty() && not ExpressionMatches(RelLabel, File.Label())) + return false; + return true; + } + + if (Type == Origin) + { + if (not OrSite.empty() && File.Site() == nullptr) + return false; + return ExpressionMatches(OrSite, File.Site()); /* both strings match */ + } + + return false; +} +bool pkgVersionMatch::FileMatch(pkgCache::PkgFileIterator File) +{ + if (auto const RlsFile = File.ReleaseFile(); not RlsFile.end()) + { + if (not FileMatch(RlsFile)) + return false; + } + else if (Type == Release) + { + // only 'bad' files like dpkg.status file have no release file + // those reuse the Component of te PkgFile to store the Archive "now". + if (not RelArchive.empty() && not ExpressionMatches(RelArchive, File.Component())) + return false; + if (not RelRelease.empty() && not ExpressionMatches(RelRelease, File.Component())) + return false; + if (not RelOrigin.empty() || not RelLabel.empty() || + not RelVerStr.empty() || not RelCodename.empty()) + return false; + } + else + return false; + + if (Type == Release) + { + if (MatchAll) + return true; + + if (not RelComponent.empty() && not ExpressionMatches(RelComponent, File.Component())) + return false; + if (not RelArchitecture.empty() && not ExpressionMatches(RelArchitecture, File.Architecture())) + return false; + } + + return true; +} + /*}}}*/ diff --git a/apt-pkg/versionmatch.h b/apt-pkg/versionmatch.h new file mode 100644 index 0000000..faf1fd4 --- /dev/null +++ b/apt-pkg/versionmatch.h @@ -0,0 +1,78 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Version Matching + + This module takes a matching string and a type and locates the version + record that satisfies the constraint described by the matching string. + + Version: 1.2* + Release: o=Debian,v=2.1*,c=main + Release: v=2.1* + Release: a=testing + Release: n=squeeze + Release: * + Origin: ftp.debian.org + + Release may be a complex type that can specify matches for any of: + Version (v= with prefix) + Origin (o=) + Archive (a=) eg, unstable, testing, stable + Codename (n=) e.g. etch, lenny, squeeze, sid + Label (l=) + Component (c=) + Binary Architecture (b=) + If there are no equals signs in the string then it is scanned in short + form - if it starts with a number it is Version otherwise it is an + Archive or a Codename. + + Release may be a '*' to match all releases. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_VERSIONMATCH_H +#define PKGLIB_VERSIONMATCH_H + +#include <apt-pkg/pkgcache.h> + +#include <string> + + +class APT_PUBLIC pkgVersionMatch +{ + // Version Matching + std::string VerStr; + bool VerPrefixMatch; + + // Release Matching + std::string RelVerStr; + bool RelVerPrefixMatch; + std::string RelOrigin; + std::string RelRelease; + std::string RelCodename; + std::string RelArchive; + std::string RelLabel; + std::string RelComponent; + std::string RelArchitecture; + bool MatchAll; + + // Origin Matching + std::string OrSite; + + public: + + enum MatchType {None = 0,Version,Release,Origin} Type; + + bool MatchVer(const char *A,std::string B,bool Prefix) APT_PURE; + static bool ExpressionMatches(const char *pattern, const char *string); + static bool ExpressionMatches(const std::string& pattern, const char *string); + bool FileMatch(pkgCache::PkgFileIterator File); + bool FileMatch(pkgCache::RlsFileIterator const &File); + pkgCache::VerIterator Find(pkgCache::PkgIterator Pkg); + bool VersionMatches(pkgCache::VerIterator Ver); + + pkgVersionMatch(std::string Data,MatchType Type); +}; + +#endif |