diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 18:07:13 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 18:07:13 +0000 |
| commit | 636c7dc17286d93d788c741d15fd756aeda066d5 (patch) | |
| tree | e7ae158cc54f591041a061b9865bcae51854f15c | |
| parent | Initial commit. (diff) | |
| download | apt-636c7dc17286d93d788c741d15fd756aeda066d5.tar.xz apt-636c7dc17286d93d788c741d15fd756aeda066d5.zip | |
Adding upstream version 1.8.2.3.upstream/1.8.2.3upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
782 files changed, 470759 insertions, 0 deletions
diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..c2989f8 --- /dev/null +++ b/.clang-format @@ -0,0 +1,25 @@ +Language: Cpp +# BasedOnStyle + +TabWidth: 8 +UseTab: Always +IndentWidth: 3 +ContinuationIndentWidth: 3 +ColumnLimit: 0 +BreakBeforeBraces: Allman +AccessModifierOffset: 0 +IncludeCategories: + - Regex: 'apti18n.h' + Priority: 9999 + - Regex: 'apt-[^/]*/' + Priority: 20 + - Regex: '^"' + Priority: 10 + - Regex: 'config.h' + Priority: 0 + - Regex: '(zlib|bzlib|lzma|lz4frame|gtest/gtest|db|gnutls/.*)\.h' + Priority: 30 + - Regex: '\.h' + Priority: 100 + - Regex: '.*' + Priority: 99 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..2126c3d --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,36 @@ +image: debian:buster +variables: + DEBIAN_FRONTEND: noninteractive + +test as root: + stage: test + script: + - adduser --home /home/travis travis --quiet --disabled-login --gecos "" --uid 1000 + - rm -f /etc/dpkg/dpkg.cfg.d/excludes + - apt-get update + - apt-get install -qq build-essential expect gcovr sudo + - chmod -R o+rwX $PWD + - ./prepare-release travis-ci + - sudo -u travis mkdir build + - sudo -u travis env -C build cmake -DCMAKE_BUILD_TYPE=Coverage -G Ninja .. + - sudo -u travis ninja -C build + - CTEST_OUTPUT_ON_FAILURE=1 ninja -C build test + - unbuffer ./test/integration/run-tests -q -j 4 + - gcovr + +test as user: + stage: test + script: + - adduser --home /home/travis travis --quiet --disabled-login --gecos "" --uid 1000 + - rm -f /etc/dpkg/dpkg.cfg.d/excludes + - apt-get update + - apt-get install -qq build-essential expect gcovr sudo + - chmod 755 /root + - chmod -R o+rwX $PWD + - ./prepare-release travis-ci + - sudo -u travis mkdir build + - sudo -u travis env -C build cmake -DCMAKE_BUILD_TYPE=Coverage -G Ninja .. + - sudo -u travis ninja -C build + - sudo -u travis CTEST_OUTPUT_ON_FAILURE=1 ninja -C build test + - sudo -u travis unbuffer ./test/integration/run-tests -q -j 4 + - sudo -u travis gcovr diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c010e13 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,23 @@ +language: cpp +cache: ccache +sudo: required +services: + - docker +env: + global: + - DEBIAN_FRONTEND=noninteractive + matrix: + - USER=travis CMAKE_FLAGS= + - USER=root CMAKE_FLAGS=-DWITH_DOC=OFF +install: + - sed -i -e "s#1000#$(id -u)#g" Dockerfile + - docker build --tag=apt-ci . +before_script: + - docker run --rm -w $PWD -v $HOME/.ccache:$HOME/.ccache -v $PWD:$PWD --user=travis apt-ci sh -e -c "mkdir build && cd build && env PATH=/usr/lib/ccache:\$PATH cmake -DCMAKE_BUILD_TYPE=Coverage -G Ninja $CMAKE_FLAGS .." + - docker run --rm -w $PWD -v $HOME/.ccache:$HOME/.ccache -v $PWD:$PWD --user=travis apt-ci ninja -C build +script: + - docker run --rm -w $PWD -v $PWD:$PWD --user=travis apt-ci env CTEST_OUTPUT_ON_FAILURE=1 ninja -C build test + - docker run --rm -w $PWD -v $PWD:$PWD --user=travis apt-ci env DESTDIR=$PWD/rootdir chronic ninja -C build install + - docker run --rm -w $PWD -v $PWD:$PWD --user=$USER --tmpfs /tmp:suid,exec apt-ci unbuffer ./test/integration/run-tests -qq -j 4 +after_script: + - docker run --rm -w $PWD/build -v $PWD:$PWD --user=$USER `bash <(curl -s https://codecov.io/env)` apt-ci bash -c 'bash <(curl -s https://codecov.io/bash)' @@ -0,0 +1,63 @@ +The project contributors: + +Michael Vogt <mvo@debian.org> +- Development, bug fixes + +David Kalnischkies <kalnischkies+debian@gmail.com> +- Development, bug fixes + +Julian Andres Klode +- Development, bug fixes + +Past Contributors: + +Robert Collins <robert.collins@canonical.com> +- Change the package index Info methods to allow apt-cache policy to be useful + when using several different archives on the same host. + +Christian Perrier <bubulle@debian.org> +- Translations hero/coordinator + +Eugene V. Lyubimkin +- Development, bug fixes + +Otavio Salvador +- Development, bug fixes + +Luca Bruno +- Development, bug fixes + +CVS:jgg Jason Gunthorpe <jgg@debian.org> +- The Mad Cow incarnate + +CVS:mdz Matt Zimmerman <mdz@debian.org> +- Ongoing maintenance and coordination of development + +CVS:piefel Michael Piefel <piefel@debian.org> +- i18n and l10n + +CVS:che Ben Gertzfield <che@debian.org> +- Packaging and Releases + +CVS:bod Brendan O'Dea <bod@debian.org> +- Perl Bindings + +CVS:tausq Randolph Chung <tausq@debian.org> +- Patches, Fixes, Debugging, GUIs and Releases + +Isaac Jones <ijones@syntaxpolice.org> and Colin Walters <walters@debian.org> + Initial implementation of authentication support (Release.gpg) + +Brian White <bcwhite@verisim.com> - Project originator +Tom Lees <tom@lpsg.demon.co.uk> - DPKG documentation and ideas +Behan Webster <behanw@verisim.com> - Original GUI design +Scott Ellis <storm@gate.net> - Original packaging and beta releases +Branden Robinson <branden@purdue.edu> - Man Page Documentation +Manoj Srivastava <srivasta@datasync.com> - 1st Generation FTP method and + dselect setup script +Adam Heath <doogie@debian.org> - 2nd Generation FTP method author +Ben Collins <bcollins@debian.org> - Initial RSH method +Many other bug reports through the Debian Bug system + +NOTE: The ChangeLog generator will parse for names and email addresses. The +'CVS:<name>' tag should indicate who this pair refers to. diff --git a/CMake/CheckCxxTarget.cmake b/CMake/CheckCxxTarget.cmake new file mode 100644 index 0000000..373c0be --- /dev/null +++ b/CMake/CheckCxxTarget.cmake @@ -0,0 +1,35 @@ +# CMake support for target-based function multiversioning +# +# Copyright (C) 2019 Canonical Ltd +# +# Author: Julian Andres Klode <jak@debian.org>. +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +function(check_cxx_target var target code) + check_cxx_source_compiles( + " + __attribute__((target(\"${target}\"))) static int foo() { ${code} return 1; } + __attribute__((target(\"default\"))) static int foo() { ${code} return 0; } + int main() { return foo(); } + " ${var}) +endfunction() diff --git a/CMake/Documentation.cmake b/CMake/Documentation.cmake new file mode 100644 index 0000000..d8a2d2c --- /dev/null +++ b/CMake/Documentation.cmake @@ -0,0 +1,328 @@ +# po4a/docbook documentation support for CMake +# - see documentation of add_docbook() +# +# Copyright (C) 2016 Julian Andres Klode <jak@debian.org>. +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +find_path(DOCBOOK_XSL manpages/docbook.xsl + # Debian + /usr/share/xml/docbook/stylesheet/docbook-xsl + /usr/share/xml/docbook/stylesheet/nwalsh + # OpenSUSE + /usr/share/xml/docbook/stylesheet/nwalsh/current + # Arch + /usr/share/xml/docbook/xsl-stylesheets + # Fedora + /usr/share/sgml/docbook/xsl-stylesheets + # Fink + ${CMAKE_INSTALL_PREFIX}/share/xml/xsl/docbook-xsl + # FreeBSD + ${CMAKE_INSTALL_PREFIX}/share/xsl/docbook/ + NO_DEFAULT_PATH) + +if(NOT DOCBOOK_XSL) + message(FATAL_ERROR "Could not find docbook xsl") +endif() + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/docbook-text-style.xsl.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/docbook-text-style.xsl) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/docbook-html-style.xsl.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/docbook-html-style.xsl) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/manpage-style.xsl.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/manpage-style.xsl) + + +# Split up a string of the form DOCUMENT[.DOCUMENT][.LANGUAGE][.SECTION].EXTENSION +# +# There might be up to two parts in the document name. The language must be +# a two char language code like de, or a 5 char code of the form de_DE. +function(po4a_components doc lang sec ext translated_full_document) + get_filename_component(name ${translated_full_document} NAME) + string(REPLACE "." ";" name "${name}") # Make it a list + + list(GET name 0 _doc) # First element is always the document + list(GET name 1 _lang) # Second *might* be a language + list(GET name -2 _sec) # Second-last *might* be a section + list(GET name -1 _ext) # Last element is always the file type + + # If the language code is neither a file type, nor a section, nor a language + # assume it is part of the file name and use the next component as the lang. + if(_lang AND NOT _lang MATCHES "^(xml|dbk|[0-9]|[a-z][a-z]|[a-z][a-z]_[A-Z][A-Z])$") + set(_doc "${_doc}.${_lang}") + list(GET name 2 _lang) + endif() + # If no language is present, we get a section; both not present => type + if(_lang MATCHES "xml|dbk|[0-9]") + set(_lang "") + endif() + if(NOT _sec MATCHES "^[0-9]$") # A (manpage) section must be a number + set(_sec "") + endif() + + set(${doc} ${_doc} PARENT_SCOPE) + set(${lang} ${_lang} PARENT_SCOPE) + set(${sec} ${_sec} PARENT_SCOPE) + set(${ext} ${_ext} PARENT_SCOPE) +endfunction() + + +# Process one document +function(po4a_one stamp_out out full_document language deps) + path_join(full_path "${CMAKE_CURRENT_SOURCE_DIR}" "${full_document}") + if (full_document MATCHES "\.ent$") + set(dest "${language}/${full_document}") + set(full_dest "${dest}") + else() + po4a_components(document _ section ext "${full_document}") + + # Calculate target file name + set(dest "${language}/${document}.${language}") + if(section) + set(dest "${dest}.${section}") + endif() + set(full_dest "${dest}.${ext}") + endif() + + # po4a might drop files not translated enough, so build a stamp file + set(stamp ${CMAKE_CURRENT_BINARY_DIR}/${dest}.po4a-stamp) + add_custom_command( + OUTPUT ${stamp} + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/${language} + COMMAND po4a --previous --no-backups + --package-name='${PROJECT_NAME}-doc' + --package-version='${PACKAGE_VERSION}' + --msgid-bugs-address='${PACKAGE_MAIL}' + --translate-only ${full_dest} + --srcdir ${CMAKE_CURRENT_SOURCE_DIR} + --destdir ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/po4a.conf + COMMAND ${CMAKE_COMMAND} -E touch ${stamp} + COMMENT "Generating ${full_dest} (or dropping it)" + DEPENDS ${full_document} ${deps} po/${language}.po + ) + # Return result + set(${stamp_out} ${stamp} PARENT_SCOPE) + set(${out} ${CMAKE_CURRENT_BINARY_DIR}/${full_dest} PARENT_SCOPE) +endfunction() + +function(xsltproc_one) + set(generated "") + set(options HTML TEXT MANPAGE) + set(oneValueArgs STAMP STAMP_OUT FULL_DOCUMENT) + set(multiValueArgs INSTALL DEPENDS) + cmake_parse_arguments(DOC "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + po4a_components(document language section ext "${DOC_FULL_DOCUMENT}") + + # Default parameters + set(params + --nonet + --xinclude + --stringparam chunk.quietly yes + --stringparam man.output.quietly yes + --path ${PROJECT_SOURCE_DIR}/vendor/${CURRENT_VENDOR}/ + --path ${CMAKE_CURRENT_SOURCE_DIR}/ + ) + + # Parameters if localized + if(language) + list(APPEND params -stringparam l10n.gentext.default.language ${language}) + endif() + + path_join(full_input_path ${CMAKE_CURRENT_SOURCE_DIR} ${DOC_FULL_DOCUMENT}) + + if (DOC_MANPAGE) + if (language) + set(manpage_output "${CMAKE_CURRENT_BINARY_DIR}/${language}/${document}.${section}") + else() + set(manpage_output "${CMAKE_CURRENT_BINARY_DIR}/${document}.${section}") + endif() + set(manpage_stylesheet "${CMAKE_CURRENT_BINARY_DIR}/manpage-style.xsl") + set(manpage_params) + + install(FILES ${manpage_output} + DESTINATION ${CMAKE_INSTALL_MANDIR}/${language}/man${section} + OPTIONAL) + endif() + if (DOC_HTML) + if (language) + set(html_output "${CMAKE_CURRENT_BINARY_DIR}/${language}/${document}.${language}.html") + else() + set(html_output "${CMAKE_CURRENT_BINARY_DIR}/${document}.html") + endif() + set(html_params --stringparam base.dir ${html_output}) + set(html_stylesheet "${CMAKE_CURRENT_BINARY_DIR}/docbook-html-style.xsl") + install(DIRECTORY ${html_output} + DESTINATION ${DOC_INSTALL} + OPTIONAL) + + endif() + if (DOC_TEXT) + if (language) + set(text_output "${CMAKE_CURRENT_BINARY_DIR}/${language}/${document}.${language}.text") + else() + set(text_output "${CMAKE_CURRENT_BINARY_DIR}/${document}.text") + endif() + set(text_params --stringparam base.dir ${text_output}) + set(text_stylesheet "${CMAKE_CURRENT_BINARY_DIR}/docbook-text-style.xsl") + + file(RELATIVE_PATH text_output_relative ${CMAKE_CURRENT_BINARY_DIR} ${text_output}) + + add_custom_command(OUTPUT ${text_output}.w3m-stamp + COMMAND ${PROJECT_SOURCE_DIR}/CMake/run_if_exists.sh + --stdout ${text_output} + ${text_output}.html + env LC_ALL=C.UTF-8 w3m -cols 78 -dump + -o display_charset=UTF-8 + -no-graph -T text/html ${text_output}.html + COMMAND ${CMAKE_COMMAND} -E touch ${text_output}.w3m-stamp + COMMENT "Generating ${text_output_relative} (if not dropped by po4a)" + DEPENDS "${text_output}.html.xsltproc-stamp" + ) + list(APPEND generated ${text_output}.w3m-stamp) + + install(FILES ${text_output} + DESTINATION ${DOC_INSTALL} + OPTIONAL) + set(text_output "${text_output}.html") + endif() + + foreach(type in manpage html text) + if (NOT ${type}_output) + continue() + endif() + + set(output ${${type}_output}) + set(stylesheet ${${type}_stylesheet}) + set(type_params ${${type}_params}) + file(RELATIVE_PATH output_relative ${CMAKE_CURRENT_BINARY_DIR} ${output}) + + add_custom_command(OUTPUT ${output}.xsltproc-stamp + COMMAND ${PROJECT_SOURCE_DIR}/CMake/run_if_exists.sh + ${full_input_path} + xsltproc ${params} ${type_params} -o ${output} + ${stylesheet} + ${full_input_path} + COMMAND ${CMAKE_COMMAND} -E touch ${output}.xsltproc-stamp + COMMENT "Generating ${output_relative} (if not dropped by po4a)" + DEPENDS ${DOC_STAMP} ${DOC_DEPENDS}) + + list(APPEND generated ${output}.xsltproc-stamp) + endforeach() + + set(${DOC_STAMP_OUT} ${generated} PARENT_SCOPE) +endfunction() + + +# add_docbook(Name [ALL] [HTML] [TEXT] [MANPAGE] +# [INSTALL install dir] +# [DEPENDS depend ...] +# [DOCUMENTS documents ...] +# [LINGUAS lingua ...]) +# +# Generate a target called name with all the documents being converted to +# the chosen output formats and translated to the chosen languages using po4a. +# +# For the translation support, the po4a.conf must be written so that +# translations for a document guide.xml are written to LANG/guide.LANG.xml, +# and for a manual page man.5.xml to a file called LANG/man.LANG.5.xml. +# +# The guide and manual page names may also contain a second component separated +# by a dot, it must however not be a valid language code. +# +# Note that po4a might chose not to generate a translated manual page for a +# given language if the translation rate is not high enough. We deal with this +# by creating stamp files. +function(add_docbook target) + set(generated "") + set(options HTML TEXT MANPAGE ALL) + set(oneValueArgs) + set(multiValueArgs INSTALL DOCUMENTS LINGUAS TRANSLATED_ENTITIES DEPENDS) + cmake_parse_arguments(DOC "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if (DOC_HTML) + list(APPEND formats HTML) + endif() + if (DOC_TEXT) + list(APPEND formats TEXT) + endif() + if (DOC_MANPAGE) + list(APPEND formats MANPAGE) + endif() + + foreach(document ${DOC_TRANSLATED_ENTITIES}) + foreach(lang ${DOC_LINGUAS}) + po4a_one(po4a_stamp po4a_out ${document} "${lang}" "") + list(APPEND DOC_DEPENDS ${po4a_stamp}) + endforeach() + endforeach() + + foreach(document ${DOC_DOCUMENTS}) + foreach(lang ${DOC_LINGUAS}) + po4a_one(po4a_stamp po4a_out ${document} "${lang}" "${DOC_DEPENDS}") + xsltproc_one(STAMP_OUT xslt_stamp + STAMP ${po4a_stamp} + FULL_DOCUMENT ${po4a_out} + INSTALL ${DOC_INSTALL} + ${formats}) + + list(APPEND stamps ${xslt_stamp}) + endforeach() + xsltproc_one(STAMP_OUT xslt_stamp + STAMP ${document} + FULL_DOCUMENT ${document} + INSTALL ${DOC_INSTALL} + ${formats}) + + list(APPEND stamps ${xslt_stamp}) + endforeach() + + if (DOC_ALL) + add_custom_target(${target} ALL DEPENDS ${stamps}) + else() + add_custom_target(${target} DEPENDS ${stamps}) + endif() +endfunction() + +# Add an update-po4a target +function(add_update_po4a target pot header) + set(WRITE_HEADER "") + + if (header) + set(WRITE_HEADER + COMMAND sed -n "/^\#$/,$p" ${pot} > ${pot}.headerfree + COMMAND cat ${header} ${pot}.headerfree > ${pot} + COMMAND rm ${pot}.headerfree + ) + endif() + add_custom_target(${target} + COMMAND po4a --previous --no-backups --force --no-translations + --msgmerge-opt --add-location=file + --porefs noline,wrap + --package-name=${PROJECT_NAME}-doc --package-version=${PACKAGE_VERSION} + --msgid-bugs-address=${PACKAGE_MAIL} po4a.conf + ${WRITE_HEADER} + VERBATIM + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) +endfunction() diff --git a/CMake/FindBerkeleyDB.cmake b/CMake/FindBerkeleyDB.cmake new file mode 100644 index 0000000..34bc3b0 --- /dev/null +++ b/CMake/FindBerkeleyDB.cmake @@ -0,0 +1,59 @@ +# - Try to find Berkeley DB +# Once done this will define +# +# BERKELEY_DB_FOUND - system has Berkeley DB +# BERKELEY_DB_INCLUDE_DIRS - the Berkeley DB include directory +# BERKELEY_DB_LIBRARIES - Link these to use Berkeley DB +# BERKELEY_DB_DEFINITIONS - Compiler switches required for using Berkeley DB + +# Copyright (c) 2006, Alexander Dymo, <adymo@kdevelop.org> +# Copyright (c) 2016, Julian Andres Klode <jak@debian.org> +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. The name of the author may not be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# We need NO_DEFAULT_PATH here, otherwise CMake helpfully picks up the wrong +# db.h on BSD systems instead of the Berkeley DB one. +find_path(BERKELEY_DB_INCLUDE_DIRS db.h + ${CMAKE_INSTALL_FULL_INCLUDEDIR}/db5 + /usr/local/include/db5 + /usr/include/db5 + + ${CMAKE_INSTALL_FULL_INCLUDEDIR}/db4 + /usr/local/include/db4 + /usr/include/db4 + + ${CMAKE_INSTALL_FULL_INCLUDEDIR} + /usr/local/include + /usr/include + + NO_DEFAULT_PATH +) + +find_library(BERKELEY_DB_LIBRARIES NAMES db db-5) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Berkeley "Could not find Berkeley DB >= 4.1" BERKELEY_DB_INCLUDE_DIRS BERKELEY_DB_LIBRARIES) +# show the BERKELEY_DB_INCLUDE_DIRS and BERKELEY_DB_LIBRARIES variables only in the advanced view +mark_as_advanced(BERKELEY_DB_INCLUDE_DIRS BERKELEY_DB_LIBRARIES) diff --git a/CMake/FindIconv.cmake b/CMake/FindIconv.cmake new file mode 100644 index 0000000..67046d9 --- /dev/null +++ b/CMake/FindIconv.cmake @@ -0,0 +1,20 @@ +find_path(ICONV_INCLUDE_DIR NAMES iconv.h) + +find_library(ICONV_LIBRARY NAMES iconv) +if (ICONV_LIBRARY) + set(ICONV_SYMBOL_FOUND "${ICONV_LIBRARY}") +else() + check_function_exists(iconv_open ICONV_SYMBOL_FOUND) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Iconv DEFAULT_MESSAGE ICONV_INCLUDE_DIR ICONV_SYMBOL_FOUND) + +if(ICONV_LIBRARY) + set(ICONV_LIBRARIES "${ICONV_LIBRARY}") +else() + set(ICONV_LIBRARIES) +endif() +set(ICONV_INCLUDE_DIRS "${ICONV_INCLUDE_DIR}") + +mark_as_advanced(ICONV_LIBRARY ICONV_INCLUDE_DIR) diff --git a/CMake/FindLFS.cmake b/CMake/FindLFS.cmake new file mode 100644 index 0000000..e1fcf96 --- /dev/null +++ b/CMake/FindLFS.cmake @@ -0,0 +1,148 @@ +# CMake support for large files +# +# Copyright (C) 2016 Julian Andres Klode <jak@debian.org>. +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# This defines the following variables +# +# LFS_DEFINITIONS - List of definitions to pass to add_definitions() +# LFS_COMPILE_OPTIONS - List of definitions to pass to add_compile_options() +# LFS_LIBRARIES - List of libraries and linker flags +# LFS_FOUND - If there is Large files support +# + +include(CheckCXXSourceCompiles) +include(FindPackageHandleStandardArgs) + +# Test program to check for LFS. Requires that off_t has at least 8 byte large +set(_lfs_test_source + " + #include <sys/types.h> + typedef char my_static_assert[sizeof(off_t) >= 8 ? 1 : -1]; + int main(void) { return 0; } + " +) + +# Check if the given options are needed +# +# This appends to the variables _lfs_cppflags, _lfs_cflags, and _lfs_ldflags, +# it also sets LFS_FOUND to 1 if it works. +function(_lfs_check_compiler_option var options definitions libraries) + set(CMAKE_REQUIRED_QUIET 1) + set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} ${options}) + set(CMAKE_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} ${definitions}) + set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_DEFINITIONS} ${libraries}) + + message(STATUS "Looking for LFS support using ${options} ${definitions} ${libraries}") + check_cxx_source_compiles("${_lfs_test_source}" ${var}) + + if(${var}) + message(STATUS "Looking for LFS support using ${options} ${definitions} ${libraries} - found") + set(_lfs_cppflags ${_lfs_cppflags} ${definitions} PARENT_SCOPE) + set(_lfs_cflags ${_lfs_cflags} ${options} PARENT_SCOPE) + set(_lfs_ldflags ${_lfs_ldflags} ${libraries} PARENT_SCOPE) + set(LFS_FOUND TRUE PARENT_SCOPE) + else() + message(STATUS "Looking for LFS support using ${options} ${definitions} ${libraries} - not found") + endif() +endfunction() + +# Check for the availability of LFS. +# The cases handled are: +# +# * Native LFS +# * Output of getconf LFS_CFLAGS; getconf LFS_LIBS; getconf LFS_LDFLAGS +# * Preprocessor flag -D_FILE_OFFSET_BITS=64 +# * Preprocessor flag -D_LARGE_FILES +# +function(_lfs_check) + set(_lfs_cflags) + set(_lfs_cppflags) + set(_lfs_ldflags) + set(_lfs_libs) + set(CMAKE_REQUIRED_QUIET 1) + message(STATUS "Looking for native LFS support") + check_cxx_source_compiles("${_lfs_test_source}" lfs_native) + if (lfs_native) + message(STATUS "Looking for native LFS support - found") + set(LFS_FOUND TRUE) + else() + message(STATUS "Looking for native LFS support - not found") + endif() + + if (NOT LFS_FOUND) + # Check using getconf. If getconf fails, don't worry, the check in + # _lfs_check_compiler_option will fail as well. + execute_process(COMMAND getconf LFS_CFLAGS + OUTPUT_VARIABLE _lfs_cflags_raw + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) + execute_process(COMMAND getconf LFS_LIBS + OUTPUT_VARIABLE _lfs_libs_tmp + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) + execute_process(COMMAND getconf LFS_LDFLAGS + OUTPUT_VARIABLE _lfs_ldflags_tmp + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) + + separate_arguments(_lfs_cflags_raw) + separate_arguments(_lfs_ldflags_tmp) + separate_arguments(_lfs_libs_tmp) + + # Move -D flags to the place they are supposed to be + foreach(flag ${_lfs_cflags_raw}) + if (flag MATCHES "-D.*") + list(APPEND _lfs_cppflags_tmp ${flag}) + else() + list(APPEND _lfs_cflags_tmp ${flag}) + endif() + endforeach() + + # Check if the flags we received (if any) produce working LFS support + _lfs_check_compiler_option(lfs_getconf_works + "${_lfs_cflags_tmp}" + "${_lfs_cppflags_tmp}" + "${_lfs_libs_tmp};${_lfs_ldflags_tmp}") + endif() + + if(NOT LFS_FOUND) # IRIX stuff + _lfs_check_compiler_option(lfs_need_n32 "-n32" "" "") + endif() + if(NOT LFS_FOUND) # Linux and friends + _lfs_check_compiler_option(lfs_need_file_offset_bits "" "-D_FILE_OFFSET_BITS=64" "") + endif() + if(NOT LFS_FOUND) # AIX + _lfs_check_compiler_option(lfs_need_large_files "" "-D_LARGE_FILES=1" "") + endif() + + set(LFS_DEFINITIONS ${_lfs_cppflags} CACHE STRING "Extra definitions for large file support") + set(LFS_COMPILE_OPTIONS ${_lfs_cflags} CACHE STRING "Extra definitions for large file support") + set(LFS_LIBRARIES ${_lfs_libs} ${_lfs_ldflags} CACHE STRING "Extra definitions for large file support") + set(LFS_FOUND ${LFS_FOUND} CACHE INTERNAL "Found LFS") +endfunction() + +if (NOT LFS_FOUND) + _lfs_check() +endif() + +find_package_handle_standard_args(LFS "Could not find LFS. Set LFS_DEFINITIONS, LFS_COMPILE_OPTIONS, LFS_LIBRARIES." LFS_FOUND) diff --git a/CMake/FindLZ4.cmake b/CMake/FindLZ4.cmake new file mode 100644 index 0000000..597f520 --- /dev/null +++ b/CMake/FindLZ4.cmake @@ -0,0 +1,25 @@ +# - Try to find LZ4 +# Once done, this will define +# +# LZ4_FOUND - system has LZ4 +# LZ4_INCLUDE_DIRS - the LZ4 include directories +# LZ4_LIBRARIES - the LZ4 library +find_package(PkgConfig) + +pkg_check_modules(LZ4_PKGCONF liblz4) + +find_path(LZ4_INCLUDE_DIRS + NAMES lz4frame.h + PATHS ${LZ4_PKGCONF_INCLUDE_DIRS} +) + + +find_library(LZ4_LIBRARIES + NAMES lz4 + PATHS ${LZ4_PKGCONF_LIBRARY_DIRS} +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(LZ4 DEFAULT_MSG LZ4_INCLUDE_DIRS LZ4_LIBRARIES) + +mark_as_advanced(LZ4_INCLUDE_DIRS LZ4_LIBRARIES) diff --git a/CMake/FindLZMA.cmake b/CMake/FindLZMA.cmake new file mode 100644 index 0000000..6abc4fa --- /dev/null +++ b/CMake/FindLZMA.cmake @@ -0,0 +1,25 @@ +# - Try to find LZMA +# Once done, this will define +# +# LZMA_FOUND - system has LZMA +# LZMA_INCLUDE_DIRS - the LZMA include directories +# LZMA_LIBRARIES - the LZMA library +find_package(PkgConfig) + +pkg_check_modules(LZMA_PKGCONF liblzma) + +find_path(LZMA_INCLUDE_DIRS + NAMES lzma.h + PATHS ${LZMA_PKGCONF_INCLUDE_DIRS} +) + + +find_library(LZMA_LIBRARIES + NAMES lzma + PATHS ${LZMA_PKGCONF_LIBRARY_DIRS} +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(LZMA DEFAULT_MSG LZMA_INCLUDE_DIRS LZMA_LIBRARIES) + +mark_as_advanced(LZMA_INCLUDE_DIRS LZMA_LIBRARIES) diff --git a/CMake/FindSeccomp.cmake b/CMake/FindSeccomp.cmake new file mode 100644 index 0000000..5cfd13a --- /dev/null +++ b/CMake/FindSeccomp.cmake @@ -0,0 +1,25 @@ +# - Try to find SECCOMP +# Once done, this will define +# +# SECCOMP_FOUND - system has SECCOMP +# SECCOMP_INCLUDE_DIRS - the SECCOMP include directories +# SECCOMP_LIBRARIES - the SECCOMP library +find_package(PkgConfig) + +pkg_check_modules(SECCOMP_PKGCONF libseccomp) + +find_path(SECCOMP_INCLUDE_DIRS + NAMES seccomp.h + PATHS ${SECCOMP_PKGCONF_INCLUDE_DIRS} +) + + +find_library(SECCOMP_LIBRARIES + NAMES seccomp + PATHS ${SECCOMP_PKGCONF_LIBRARY_DIRS} +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(SECCOMP DEFAULT_MSG SECCOMP_INCLUDE_DIRS SECCOMP_LIBRARIES) + +mark_as_advanced(SECCOMP_INCLUDE_DIRS SECCOMP_LIBRARIES) diff --git a/CMake/FindSystemd.cmake b/CMake/FindSystemd.cmake new file mode 100644 index 0000000..1c7a7de --- /dev/null +++ b/CMake/FindSystemd.cmake @@ -0,0 +1,24 @@ +# - Try to find SYSTEMD +# Once done, this will define +# +# SYSTEMD_FOUND - system has SYSTEMD +# SYSTEMD_INCLUDE_DIRS - the SYSTEMD include directories +# SYSTEMD_LIBRARIES - the SYSTEMD library +find_package(PkgConfig) + +pkg_check_modules(SYSTEMD_PKGCONF libsystemd) + +find_path(SYSTEMD_INCLUDE_DIRS + NAMES systemd/sd-bus.h + PATHS ${SYSTEMD_PKGCONF_INCLUDE_DIRS} +) + +find_library(SYSTEMD_LIBRARIES + NAMES systemd + PATHS ${SYSTEMD_PKGCONF_LIBRARY_DIRS} +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Systemd DEFAULT_MSG SYSTEMD_INCLUDE_DIRS SYSTEMD_LIBRARIES) + +mark_as_advanced(SYSTEMD_INCLUDE_DIRS SYSTEMD_LIBRARIES) diff --git a/CMake/FindUdev.cmake b/CMake/FindUdev.cmake new file mode 100644 index 0000000..e416c43 --- /dev/null +++ b/CMake/FindUdev.cmake @@ -0,0 +1,25 @@ +# - Try to find UDEV +# Once done, this will define +# +# UDEV_FOUND - system has UDEV +# UDEV_INCLUDE_DIRS - the UDEV include directories +# UDEV_LIBRARIES - the UDEV library +find_package(PkgConfig) + +pkg_check_modules(UDEV_PKGCONF libudev) + +find_path(UDEV_INCLUDE_DIRS + NAMES libudev.h + PATHS ${UDEV_PKGCONF_INCLUDE_DIRS} +) + + +find_library(UDEV_LIBRARIES + NAMES udev + PATHS ${UDEV_PKGCONF_LIBRARY_DIRS} +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Udev DEFAULT_MSG UDEV_INCLUDE_DIRS UDEV_LIBRARIES) + +mark_as_advanced(UDEV_INCLUDE_DIRS UDEV_LIBRARIES) diff --git a/CMake/FindZstd.cmake b/CMake/FindZstd.cmake new file mode 100644 index 0000000..6811804 --- /dev/null +++ b/CMake/FindZstd.cmake @@ -0,0 +1,25 @@ +# - Try to find ZSTD +# Once done, this will define +# +# ZSTD_FOUND - system has ZSTD +# ZSTD_INCLUDE_DIRS - the ZSTD include directories +# ZSTD_LIBRARIES - the ZSTD library +find_package(PkgConfig) + +pkg_check_modules(ZSTD_PKGCONF libzstd) + +find_path(ZSTD_INCLUDE_DIRS + NAMES zstd.h + PATHS ${ZSTD_PKGCONF_INCLUDE_DIRS} +) + + +find_library(ZSTD_LIBRARIES + NAMES zstd + PATHS ${ZSTD_PKGCONF_LIBRARY_DIRS} +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(ZSTD DEFAULT_MSG ZSTD_INCLUDE_DIRS ZSTD_LIBRARIES) + +mark_as_advanced(ZSTD_INCLUDE_DIRS ZSTD_LIBRARIES) diff --git a/CMake/Misc.cmake b/CMake/Misc.cmake new file mode 100644 index 0000000..6ad0b94 --- /dev/null +++ b/CMake/Misc.cmake @@ -0,0 +1,101 @@ +include(CheckCXXCompilerFlag) + +# Flatten our header structure +function(flatify target headers) + foreach(header ${headers}) + get_filename_component(tgt ${header} NAME) + configure_file(${header} ${target}/${tgt} COPYONLY) + endforeach(header ${headers}) +endfunction() + + +function(add_optional_compile_options flags) + foreach(flag ${flags}) + check_cxx_compiler_flag(-${flag} have-compiler-flag:-${flag}) + if (have-compiler-flag:-${flag}) + add_compile_options("-${flag}") + endif() + endforeach() +endfunction() + +# Substitute vendor references in a file +function(add_vendor_file) + set(options) + set(oneValueArgs OUTPUT INPUT MODE) + set(multiValueArgs VARIABLES) + cmake_parse_arguments(AVF "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + set(in ${CMAKE_CURRENT_SOURCE_DIR}/${AVF_INPUT}) + set(out ${CMAKE_CURRENT_BINARY_DIR}/${AVF_OUTPUT}) + + add_custom_command( + OUTPUT ${out} + COMMAND ${CMAKE_COMMAND} -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR} + "-DVARS=${AVF_VARIABLES}" + -DCURRENT_VENDOR=${CURRENT_VENDOR} + -DIN=${in} + -DOUT=${out} + -P ${PROJECT_SOURCE_DIR}/CMake/vendor_substitute.cmake + COMMAND chmod ${AVF_MODE} ${out} + DEPENDS ${in} + ${PROJECT_SOURCE_DIR}/doc/apt-verbatim.ent + ${PROJECT_SOURCE_DIR}/vendor/${CURRENT_VENDOR}/apt-vendor.ent + ${PROJECT_SOURCE_DIR}/vendor/getinfo + ${PROJECT_SOURCE_DIR}/CMake/vendor_substitute.cmake + VERBATIM + ) + + # Would like to use ${AVF_OUTPUT} as target name, but then ninja gets + # cycles. + add_custom_target(vendor-${AVF_OUTPUT} ALL DEPENDS ${out}) +endfunction() + +# Add symbolic links to a file +function(add_slaves destination master) + set(slaves "") + foreach(slave ${ARGN}) + add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${slave} + COMMAND ${CMAKE_COMMAND} -E create_symlink ${master} ${CMAKE_CURRENT_BINARY_DIR}/${slave}) + list(APPEND slaves ${CMAKE_CURRENT_BINARY_DIR}/${slave}) + endforeach() + + STRING(REPLACE "/" "-" master "${master}") + add_custom_target(${master}-slaves ALL DEPENDS ${slaves}) + install(FILES ${slaves} DESTINATION ${destination}) +endfunction() + +# Generates a simple version script versioning everything with current SOVERSION +function(add_version_script target) + get_target_property(soversion ${target} SOVERSION) + set(script "${CMAKE_CURRENT_BINARY_DIR}/${target}.versionscript") + string(REPLACE "-" "" name "${target}_${soversion}") + string(TOUPPER "${name}" name) + add_custom_command(OUTPUT "${script}" + COMMAND echo "${name} {global: *; };" > "${script}" + VERBATIM ) + add_custom_target(${target}-versionscript DEPENDS "${script}") + target_link_libraries(${target} PRIVATE -Wl,-version-script="${script}") + add_dependencies(${target} ${target}-versionscript) +endfunction() + +function(path_join out path1 path2) + string(SUBSTRING ${path2} 0 1 init_char) + if ("${init_char}" STREQUAL "/") + set(${out} "${path2}" PARENT_SCOPE) + else() + set(${out} "${path1}/${path2}" PARENT_SCOPE) + endif() +endfunction() + +# install_empty_directories(path ...) +# +# Creates empty directories in the install destination dir. Paths may be +# absolute or relative; in the latter case, the value of CMAKE_INSTALL_PREFIX +# is prepended. +function(install_empty_directories) + foreach(path ${ARGN}) + path_join(full_path "${CMAKE_INSTALL_PREFIX}" "${path}") + INSTALL(CODE "MESSAGE(STATUS \"Creating directory: \$ENV{DESTDIR}${full_path}\")" + CODE "FILE(MAKE_DIRECTORY \$ENV{DESTDIR}${full_path})") + endforeach() +endfunction() diff --git a/CMake/Translations.cmake b/CMake/Translations.cmake new file mode 100644 index 0000000..54a635a --- /dev/null +++ b/CMake/Translations.cmake @@ -0,0 +1,185 @@ +# translations.cmake - Translations using APT's translation system. +# Copyright (C) 2009, 2016 Julian Andres Klode <jak@debian.org> + +function(apt_add_translation_domain) + set(options) + set(oneValueArgs DOMAIN) + set(multiValueArgs TARGETS SCRIPTS EXCLUDE_LANGUAGES) + cmake_parse_arguments(NLS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + # Build the list of source files of the target + set(files "") + set(abs_files "") + set(scripts "") + set(abs_scripts "") + set(mofiles) + set(targets ${NLS_TARGETS}) + set(domain ${NLS_DOMAIN}) + set(xgettext_params + --add-comments + --foreign + --package-name=${PROJECT_NAME} + --package-version=${PACKAGE_VERSION} + --msgid-bugs-address=${PACKAGE_MAIL} + ) + foreach(source ${NLS_SCRIPTS}) + path_join(file "${CMAKE_CURRENT_SOURCE_DIR}" "${source}") + file(RELATIVE_PATH relfile ${PROJECT_SOURCE_DIR} ${file}) + list(APPEND scripts ${relfile}) + list(APPEND abs_scripts ${file}) + endforeach() + foreach(target ${targets}) + get_target_property(source_dir ${target} SOURCE_DIR) + get_target_property(sources ${target} SOURCES) + foreach(source ${sources}) + if (source MATCHES TARGET_OBJECTS) + continue() + endif() + path_join(file "${source_dir}" "${source}") + file(RELATIVE_PATH relfile ${PROJECT_SOURCE_DIR} ${file}) + set(files ${files} ${relfile}) + set(abs_files ${abs_files} ${file}) + endforeach() + + target_compile_definitions(${target} PRIVATE -DAPT_DOMAIN="${domain}") + endforeach() + + if("${scripts}" STREQUAL "") + set(sh_pot "/dev/null") + else() + set(sh_pot ${CMAKE_CURRENT_BINARY_DIR}/${domain}.sh.pot) + # Create the template for this specific sub-domain + add_custom_command (OUTPUT ${sh_pot} + COMMAND xgettext ${xgettext_params} -L Shell + -o ${sh_pot} ${scripts} + DEPENDS ${abs_scripts} + VERBATIM + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + ) + endif() + + + add_custom_command (OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${domain}.c.pot + COMMAND xgettext ${xgettext_params} -k_ -kN_ + --keyword=P_:1,2 + -o ${CMAKE_CURRENT_BINARY_DIR}/${domain}.c.pot ${files} + DEPENDS ${abs_files} + VERBATIM + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + ) + + # We are building a ${domain}.pot with a header for launchpad, but we also + # build a ${domain.pot}-tmp as a byproduct. The msgfmt command than depend + # on the byproduct while their target depends on the output, so that msgfmt + # does not have to be rerun if nothing in the template changed. + # + # Make sure the .pot-tmp has no line numbers, to avoid useless rebuilding + # of .mo files. + add_custom_command (OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${domain}.pot + BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/${domain}.pot-tmp + COMMAND msgcomm --more-than=0 --omit-header --sort-by-file --add-location=file + ${sh_pot} + ${CMAKE_CURRENT_BINARY_DIR}/${domain}.c.pot + --output=${CMAKE_CURRENT_BINARY_DIR}/${domain}.pot-tmp0 + COMMAND msgcomm --more-than=0 --sort-by-file + ${sh_pot} + ${CMAKE_CURRENT_BINARY_DIR}/${domain}.c.pot + --output=${CMAKE_CURRENT_BINARY_DIR}/${domain}.pot + COMMAND cmake -E copy_if_different + ${CMAKE_CURRENT_BINARY_DIR}/${domain}.pot-tmp0 + ${CMAKE_CURRENT_BINARY_DIR}/${domain}.pot-tmp + DEPENDS ${sh_pot} + ${CMAKE_CURRENT_BINARY_DIR}/${domain}.c.pot + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + ) + + # We need a target to depend on otherwise, the msgmerge might not get called + # with the make generator + add_custom_target(nls-${domain}-template DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${domain}.pot) + + # Build .mo files + file(GLOB translations "${PROJECT_SOURCE_DIR}/po/*.po") + list(SORT translations) + foreach(file ${translations}) + get_filename_component(langcode ${file} NAME_WE) + if ("${langcode}" IN_LIST NLS_EXCLUDE_LANGUAGES) + continue() + endif() + set(outdir ${CMAKE_CURRENT_BINARY_DIR}/locale/${langcode}/LC_MESSAGES) + file(MAKE_DIRECTORY ${outdir}) + # Command to merge and compile the messages. As explained in the custom + # command for msgcomm, this depends on byproduct to avoid reruns + add_custom_command(OUTPUT ${outdir}/${domain}.po + COMMAND msgmerge -qo ${outdir}/${domain}.po ${file} ${CMAKE_CURRENT_BINARY_DIR}/${domain}.pot-tmp + DEPENDS ${file} ${CMAKE_CURRENT_BINARY_DIR}/${domain}.pot-tmp + ) + add_custom_command(OUTPUT ${outdir}/${domain}.mo + COMMAND msgfmt --statistics -o ${outdir}/${domain}.mo ${outdir}/${domain}.po + DEPENDS ${outdir}/${domain}.po + ) + + set(mofiles ${mofiles} ${outdir}/${domain}.mo) + install(FILES ${outdir}/${domain}.mo + DESTINATION "${CMAKE_INSTALL_LOCALEDIR}/${langcode}/LC_MESSAGES") + endforeach(file ${translations}) + + add_custom_target(nls-${domain} ALL DEPENDS ${mofiles} nls-${domain}-template) +endfunction() + +# Usage: apt_add_update_po(output domain [domain ...]) +function(apt_add_update_po) + set(options) + set(oneValueArgs TEMPLATE) + set(multiValueArgs DOMAINS EXCLUDE_LANGUAGES) + cmake_parse_arguments(NLS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + set(output ${CMAKE_CURRENT_SOURCE_DIR}/${NLS_TEMPLATE}.pot) + foreach(domain ${NLS_DOMAINS}) + list(APPEND potfiles ${CMAKE_CURRENT_BINARY_DIR}/${domain}.pot) + endforeach() + + get_filename_component(master_name ${output} NAME_WE) + add_custom_target(nls-${master_name} + COMMAND msgcomm --sort-by-file --add-location=file + --more-than=0 --output=${output} + ${potfiles} + DEPENDS ${potfiles}) + + file(GLOB translations "${PROJECT_SOURCE_DIR}/po/*.po") + if (NOT TARGET update-po) + add_custom_target(update-po) + endif() + foreach(translation ${translations}) + get_filename_component(langcode ${translation} NAME_WE) + if ("${langcode}" IN_LIST NLS_EXCLUDE_LANGUAGES) + continue() + endif() + add_custom_target(update-po-${langcode} + COMMAND msgmerge -q --previous --update --backup=none ${translation} ${output} + DEPENDS nls-${master_name} + ) + add_dependencies(update-po update-po-${langcode}) + endforeach() + add_dependencies(update-po nls-${master_name}) +endfunction() + +function(apt_add_po_statistics excluded) + add_custom_target(statistics) + file(GLOB translations "${PROJECT_SOURCE_DIR}/po/*.po") + foreach(translation ${translations}) + get_filename_component(langcode ${translation} NAME_WE) + if ("${langcode}" IN_LIST excluded) + add_custom_command( + TARGET statistics PRE_BUILD + COMMAND printf "%-6s " "${langcode}:" + COMMAND echo "ignored" + VERBATIM + ) + continue() + endif() + add_custom_command( + TARGET statistics PRE_BUILD + COMMAND printf "%-6s " "${langcode}:" + COMMAND msgfmt --statistics -o /dev/null ${translation} + VERBATIM + ) + endforeach() +endfunction() diff --git a/CMake/apti18n.h.in b/CMake/apti18n.h.in new file mode 100644 index 0000000..1929795 --- /dev/null +++ b/CMake/apti18n.h.in @@ -0,0 +1,29 @@ +// -*- mode: cpp; mode: fold -*- +/* Internationalization macros for apt. This header should be included last + in each C file. */ + +// Set by autoconf +#cmakedefine USE_NLS + +#ifdef USE_NLS +// apt will use the gettext implementation of the C library +#include <libintl.h> +#include <locale.h> +# ifdef APT_DOMAIN +# define _(x) dgettext(APT_DOMAIN,x) +# define P_(msg,plural,n) dngettext(APT_DOMAIN,msg,plural,n) +# else +# define _(x) gettext(x) +# define P_(msg,plural,n) ngettext(msg,plural,n) +# endif +# define N_(x) x +#else +// apt will not use any gettext +# define setlocale(a, b) +# define textdomain(a) +# define bindtextdomain(a, b) +# define _(x) x +# define P_(msg,plural,n) (n == 1 ? msg : plural) +# define N_(x) x +# define dgettext(d, m) m +#endif diff --git a/CMake/config.h.in b/CMake/config.h.in new file mode 100644 index 0000000..a9528cc --- /dev/null +++ b/CMake/config.h.in @@ -0,0 +1,90 @@ +/* Define if your processor stores words with the most significant + byte first (like Motorola and SPARC, unlike Intel and VAX). */ +#cmakedefine WORDS_BIGENDIAN + +/* Define if we have the timegm() function */ +#cmakedefine HAVE_TIMEGM + +/* Define if we have the zlib library for gzip */ +#cmakedefine HAVE_ZLIB + +/* Define if we have the bz2 library for bzip2 */ +#cmakedefine HAVE_BZ2 + +/* Define if we have the lzma library for lzma/xz */ +#cmakedefine HAVE_LZMA + +/* Define if we have the lz4 library for lz4 */ +#cmakedefine HAVE_LZ4 + +/* Define if we have the zstd library for zst */ +#cmakedefine HAVE_ZSTD + +/* Define if we have the systemd library */ +#cmakedefine HAVE_SYSTEMD + +/* Define if we have the udev library */ +#cmakedefine HAVE_UDEV + +/* Define if we have the seccomp library */ +#cmakedefine HAVE_SECCOMP + +/* These two are used by the statvfs shim for glibc2.0 and bsd */ +/* Define if we have sys/vfs.h */ +#cmakedefine HAVE_VFS_H +#cmakedefine HAVE_STRUCT_STATFS_F_TYPE + +/* Define if we have sys/mount.h */ +#cmakedefine HAVE_MOUNT_H + +/* Define if we have sys/endian.h */ +#cmakedefine HAVE_SYS_ENDIAN_H + +/* Define if we have machine/endian.h */ +#cmakedefine HAVE_MACHINE_ENDIAN_H + +/* Check for getresuid() function and similar ones */ +#cmakedefine HAVE_GETRESUID +#cmakedefine HAVE_GETRESGID +#cmakedefine HAVE_SETRESUID +#cmakedefine HAVE_SETRESGID + +/* Check for ptsname_r() */ +#cmakedefine HAVE_PTSNAME_R + +/* Define the arch name string */ +#define COMMON_ARCH "${COMMON_ARCH}" + +/* The package name string */ +#define PACKAGE "${PACKAGE}" + +/* The version number string */ +#define PACKAGE_VERSION "${PACKAGE_VERSION}" + +/* The mail address to reach upstream */ +#define PACKAGE_MAIL "${PACKAGE_MAIL}" + +/* Various directories */ +#cmakedefine CMAKE_INSTALL_FULL_BINDIR "${CMAKE_INSTALL_FULL_BINDIR}" +#cmakedefine STATE_DIR "${STATE_DIR}" +#cmakedefine CACHE_DIR "${CACHE_DIR}" +#cmakedefine LOG_DIR "${LOG_DIR}" +#cmakedefine CONF_DIR "${CONF_DIR}" +#cmakedefine LIBEXEC_DIR "${LIBEXEC_DIR}" +#cmakedefine BIN_DIR "${BIN_DIR}" +#cmakedefine DPKG_DATADIR "${DPKG_DATADIR}" + +/* Group of the root user */ +#cmakedefine ROOT_GROUP "${ROOT_GROUP}" + +/* defined if __builtin_ia32_crc32{s,d}i() exists in an sse4.2 target */ +#cmakedefine HAVE_FMV_SSE42_AND_CRC32 +#cmakedefine HAVE_FMV_SSE42_AND_CRC32DI + +#define APT_8_CLEANER_HEADERS +#define APT_9_CLEANER_HEADERS +#define APT_10_CLEANER_HEADERS +#define APT_15_CLEANER_HEADERS + +/* unrolling is faster combined with an optimizing compiler */ +#define SHA2_UNROLL_TRANSFORM diff --git a/CMake/endian.h.in b/CMake/endian.h.in new file mode 100644 index 0000000..1d9198c --- /dev/null +++ b/CMake/endian.h.in @@ -0,0 +1,9 @@ +#include <config.h> + +#ifdef HAVE_MACHINE_ENDIAN_H +#include <machine/endian.h> +#endif +#ifdef HAVE_SYS_ENDIAN_H +#include <sys/types.h> +#include <sys/endian.h> +#endif diff --git a/CMake/run_if_exists.sh b/CMake/run_if_exists.sh new file mode 100755 index 0000000..97edd4c --- /dev/null +++ b/CMake/run_if_exists.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# Small helper for running a command +out="" +if [ "$1" = "--stdout" ]; then + out="$2" + shift 2 +fi + +if [ -e "$1" ]; then + shift + if [ "$out" ]; then + exec "$@" > $out + else + exec "$@" + fi +fi diff --git a/CMake/statvfs.h.in b/CMake/statvfs.h.in new file mode 100644 index 0000000..d0ec238 --- /dev/null +++ b/CMake/statvfs.h.in @@ -0,0 +1,13 @@ +/* Compatibility for systems with out Single Unix Spec statvfs */ +#include <config.h> + +#ifdef HAVE_VFS_H +#include <sys/vfs.h> +#endif + +#ifdef HAVE_MOUNT_H +#include <sys/param.h> +#include <sys/mount.h> +#endif + +#define statvfs statfs diff --git a/CMake/vendor_substitute.cmake b/CMake/vendor_substitute.cmake new file mode 100644 index 0000000..71449c9 --- /dev/null +++ b/CMake/vendor_substitute.cmake @@ -0,0 +1,8 @@ +file(READ ${IN} input) +foreach(variable ${VARS}) + execute_process(COMMAND ${PROJECT_SOURCE_DIR}/vendor/getinfo + --vendor ${CURRENT_VENDOR} ${variable} + OUTPUT_VARIABLE value OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REPLACE "&${variable};" "${value}" input "${input}") +endforeach() +file(WRITE ${OUT} "${input}") diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..bbc39dc --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,260 @@ +# Copyright (C) 2009, 2016 Julian Andres Klode <jak@debian.org>. +# Licensed under the same terms as APT; i.e. GPL 2 or later. + +# set minimum version +project(apt) +cmake_minimum_required(VERSION 3.4.0) +# Generic header locations +include_directories(${PROJECT_BINARY_DIR}/include) + + +enable_testing() + +option(WITH_DOC "Build documentation." ON) +option(USE_NLS "Localisation support." ON) + +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/CMake") + +# Add coverage target +set(CMAKE_CXX_FLAGS_COVERAGE "-g -fprofile-arcs -ftest-coverage") +set(CMAKE_EXE_LINKER_FLAGS_COVERAGE "-lgcov") +set(CMAKE_SHARED_LINKER_FLAGS_COVERAGE "-lgcov") + +# Work around bug in GNUInstallDirs +if (EXISTS "/etc/debian_version") + set(CMAKE_INSTALL_LIBEXECDIR "lib") +endif() + +# Include stuff +include(Misc) +include(CheckIncludeFiles) +include(CheckFunctionExists) +include(CheckStructHasMember) +include(GNUInstallDirs) +include(TestBigEndian) +find_package(Threads REQUIRED) +find_package(LFS REQUIRED) +find_package(Iconv REQUIRED) + +find_package(Perl REQUIRED) + +if(USE_NLS) + find_package(Intl REQUIRED) + link_libraries(${Intl_LIBRARIES}) + include_directories(${Intl_INCLUDE_DIRS}) +endif() + +# Add large file support +add_compile_options(${LFS_COMPILE_OPTIONS}) +add_definitions(${LFS_DEFINITIONS}) +link_libraries(${LFS_LIBRARIES}) + +# Set compiler flags +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_VISIBILITY_INLINES_HIDDEN 1) + +add_optional_compile_options(Wall) +add_optional_compile_options(Wextra) +add_optional_compile_options(Wcast-align) +add_optional_compile_options(Wlogical-op) +add_optional_compile_options(Wredundant-decls) +add_optional_compile_options(Wmissing-declarations) +add_optional_compile_options(Wunsafe-loop-optimizations) +add_optional_compile_options(Wctor-dtor-privacy) +add_optional_compile_options(Wdisabled-optimization) +add_optional_compile_options(Winit-self) +add_optional_compile_options(Wmissing-include-dirs) +add_optional_compile_options(Wnoexcept) +add_optional_compile_options(Wsign-promo) +add_optional_compile_options(Wundef) +add_optional_compile_options(Wdouble-promotion) + +# apt-ftparchive dependencies +find_package(BerkeleyDB REQUIRED) +if (BERKELEY_DB_FOUND) + set(HAVE_BDB 1) +endif() + +find_package(GnuTLS REQUIRED) +if (GNUTLS_FOUND) + set(HAVE_GNUTLS 1) +endif() + +# (De)Compressor libraries +find_package(ZLIB REQUIRED) +if (ZLIB_FOUND) + set(HAVE_ZLIB 1) +endif() + + +find_package(BZip2 REQUIRED) +if (BZIP2_FOUND) + set(HAVE_BZ2 1) +endif() + +find_package(LZMA REQUIRED) +if (LZMA_FOUND) + set(HAVE_LZMA 1) +endif() + + +find_package(LZ4 REQUIRED) +if (LZ4_FOUND) + set(HAVE_LZ4 1) +endif() + +find_package(Zstd) +if (ZSTD_FOUND) + set(HAVE_ZSTD 1) +endif() + + +find_package(Udev) +if (UDEV_FOUND) + set(HAVE_UDEV 1) +endif() + +find_package(Systemd) +if (SYSTEMD_FOUND) + set(HAVE_SYSTEMD 1) +endif() + +find_package(Seccomp) +if (SECCOMP_FOUND) + set(HAVE_SECCOMP 1) +endif() + +# Mount()ing and stat()ing and friends +check_symbol_exists(statfs sys/vfs.h HAVE_VFS_H) +check_include_files(sys/params.h HAVE_PARAMS_H) +check_symbol_exists(statfs sys/mount.h HAVE_MOUNT_H) +if (NOT HAVE_VFS_H AND NOT HAVE_MOUNT_H) + message(FATAL_ERROR "Can find neither statvfs() nor statfs()") +endif() + +check_function_exists(statvfs HAVE_STATVFS) +if (NOT HAVE_STATVFS) + configure_file(CMake/statvfs.h.in ${PROJECT_BINARY_DIR}/include/sys/statvfs.h COPYONLY) +endif() + +CHECK_STRUCT_HAS_MEMBER("struct statfs" f_type sys/vfs.h HAVE_STRUCT_STATFS_F_TYPE) + +# Other checks +check_function_exists(getresuid HAVE_GETRESUID) +check_function_exists(getresgid HAVE_GETRESGID) +check_function_exists(setresuid HAVE_SETRESUID) +check_function_exists(setresgid HAVE_SETRESGID) +check_function_exists(ptsname_r HAVE_PTSNAME_R) +check_function_exists(timegm HAVE_TIMEGM) +test_big_endian(WORDS_BIGENDIAN) + +# FreeBSD +add_definitions(-D_WITH_GETLINE=1) + +CHECK_INCLUDE_FILES(machine/endian.h HAVE_MACHINE_ENDIAN_H) +CHECK_INCLUDE_FILES(sys/endian.h HAVE_SYS_ENDIAN_H) +CHECK_INCLUDE_FILES(endian.h HAVE_ENDIAN_H) +if (NOT HAVE_ENDIAN_H) + if (HAVE_MACHINE_ENDIAN_H OR HAVE_SYS_ENDIAN_H) + configure_file(CMake/endian.h.in ${PROJECT_BINARY_DIR}/include/endian.h) + else() + message(FATAL_ERROR "Cannot find endian.h") + endif() +endif() + + +include(CheckTypeSize) +set(CMAKE_EXTRA_INCLUDE_FILES "signal.h") +check_type_size("sig_t" SIG_T LANGUAGE "CXX") +check_type_size("sighandler_t" SIGHANDLER_T LANGUAGE "CXX") +set(CMAKE_EXTRA_INCLUDE_FILES) +if (NOT HAVE_SIGHANDLER_T) + if (HAVE_SIG_T) + add_definitions(-Dsighandler_t=sig_t) + else() + message(FATAL_ERROR "Platform defines neither sig_t nor sighandler_t") + endif() +endif() + +# Handle resolving +check_function_exists(res_init HAVE_LIBC_RESOLV) +if(HAVE_LIBC_RESOLV) + set(RESOLV_LIBRARIES) +else() + set(RESOLV_LIBRARIES -lresolv) +endif() + +# Check multiversioning +include(CheckCxxTarget) +check_cxx_target(HAVE_FMV_SSE42_AND_CRC32 "sse4.2" "__builtin_ia32_crc32si(0, 1llu);") +check_cxx_target(HAVE_FMV_SSE42_AND_CRC32DI "sse4.2" "__builtin_ia32_crc32di(0, 1llu);") + +# Configure some variables like package, version and architecture. +set(PACKAGE ${PROJECT_NAME}) +set(PACKAGE_MAIL "APT Development Team <deity@lists.debian.org>") +set(PACKAGE_VERSION "1.8.2.3") + +if (NOT DEFINED DPKG_DATADIR) + execute_process(COMMAND ${PERL_EXECUTABLE} -MDpkg -e "print $Dpkg::DATADIR;" + OUTPUT_VARIABLE DPKG_DATADIR_CMD OUTPUT_STRIP_TRAILING_WHITESPACE) + message(STATUS "Found dpkg data dir: ${DPKG_DATADIR_CMD}") + set(DPKG_DATADIR "${DPKG_DATADIR_CMD}" CACHE PATH "dpkg data directory") +endif() +if (NOT DEFINED COMMON_ARCH) + execute_process(COMMAND dpkg-architecture -qDEB_HOST_ARCH + OUTPUT_VARIABLE COMMON_ARCH OUTPUT_STRIP_TRAILING_WHITESPACE) +endif() +if (NOT DEFINED ROOT_GROUP) + execute_process(COMMAND id -gn root + OUTPUT_VARIABLE ROOT_GROUP OUTPUT_STRIP_TRAILING_WHITESPACE) + message(STATUS "Found root group: ${ROOT_GROUP}") +endif() +set(ROOT_GROUP "${ROOT_GROUP}" CACHE STRING "Group of root (e.g.: wheel or root)") + +# Set various directories +set(STATE_DIR "${CMAKE_INSTALL_FULL_LOCALSTATEDIR}/lib/apt" CACHE PATH "Your /var/lib/apt") +set(CACHE_DIR "${CMAKE_INSTALL_FULL_LOCALSTATEDIR}/cache/apt" CACHE PATH "Your /var/cache/apt") +set(LOG_DIR "${CMAKE_INSTALL_FULL_LOCALSTATEDIR}/log/apt" CACHE PATH "Your /var/log/apt") +set(CONF_DIR "${CMAKE_INSTALL_FULL_SYSCONFDIR}/apt" CACHE PATH "Your /etc/apt") +set(LIBEXEC_DIR "${CMAKE_INSTALL_FULL_LIBEXECDIR}/apt" CACHE PATH "Your /usr/libexec/apt") +set(BIN_DIR "${CMAKE_INSTALL_FULL_BINDIR}") + + +# Configure our configuration headers (config.h and apti18n.h) +configure_file(CMake/config.h.in ${PROJECT_BINARY_DIR}/include/config.h) +configure_file(CMake/apti18n.h.in ${PROJECT_BINARY_DIR}/include/apti18n.h) + +# Add our subdirectories +add_subdirectory(vendor) +add_subdirectory(apt-pkg) +add_subdirectory(apt-private) +add_subdirectory(apt-inst) +add_subdirectory(cmdline) +add_subdirectory(completions) +add_subdirectory(doc) +add_subdirectory(dselect) +add_subdirectory(ftparchive) +add_subdirectory(methods) +add_subdirectory(test) + +if (USE_NLS) +add_subdirectory(po) + +# Link update-po4a into the update-po target +add_dependencies(update-po update-po4a) +endif() + +# Create our directories. +install_empty_directories( + ${CONF_DIR}/apt.conf.d + ${CONF_DIR}/auth.conf.d + ${CONF_DIR}/preferences.d + ${CONF_DIR}/sources.list.d + ${CONF_DIR}/trusted.gpg.d + ${CACHE_DIR}/archives/partial + ${STATE_DIR}/lists/partial + ${STATE_DIR}/mirrors/partial + ${STATE_DIR}/periodic + ${LOG_DIR} +) @@ -0,0 +1,22 @@ +Apt is copyright 1997, 1998, 1999 Jason Gunthorpe and others. +Apt is currently developed by APT Development Team <deity@lists.debian.org>. + +License: GPLv2+ + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. + +See /usr/share/common-licenses/GPL-2, or +<http://www.gnu.org/copyleft/gpl.txt> for the terms of the latest version +of the GNU General Public License. diff --git a/COPYING.GPL b/COPYING.GPL new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/COPYING.GPL @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..49934e4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM debian:buster +COPY . /tmp +WORKDIR /tmp +RUN sed -i s#://deb.debian.org#://cdn-fastly.deb.debian.org# /etc/apt/sources.list \ + && apt-get update \ + && adduser --home /home/travis travis --quiet --disabled-login --gecos "" --uid 1000 \ + && env DEBIAN_FRONTEND=noninteractive apt-get install build-essential ccache ninja-build expect curl git -q -y \ + && env DEBIAN_FRONTEND=noninteractive ./prepare-release travis-ci \ + && dpkg-reconfigure ccache \ + && rm -f /etc/dpkg/dpkg.cfg.d/excludes \ + && rm -r /tmp/* \ + && apt-get clean diff --git a/README.md b/README.md new file mode 100644 index 0000000..8a8b3b6 --- /dev/null +++ b/README.md @@ -0,0 +1,215 @@ +APT +=== + +apt is the main commandline package manager for Debian and its derivatives. +It provides commandline tools for searching and managing as well as querying +information about packages as well as low-level access to all features +provided by the libapt-pkg and libapt-inst libraries which higher-level +package managers can depend upon. + +Included tools are: + +* **apt-get** for retrieval of packages and information about them + from authenticated sources and for installation, upgrade and + removal of packages together with their dependencies +* **apt-cache** for querying available information about installed + as well as installable packages +* **apt-cdrom** to use removable media as a source for packages +* **apt-config** as an interface to the configuration settings +* **apt-key** as an interface to manage authentication keys +* **apt-extracttemplates** to be used by debconf to prompt for configuration + questions before installation +* **apt-ftparchive** creates Packages and other index files + needed to publish an archive of debian packages +* **apt-sortpkgs** is a Packages/Sources file normalizer +* **apt** is a high-level commandline interface for better interactive usage + +The libraries libapt-pkg and libapt-inst are also maintained as part of this project, +alongside various additional binaries like the acquire-methods used by them. +Bindings for Python ([python-apt](https://tracker.debian.org/pkg/python-apt)) and +Perl ([libapt-pkg-perl](https://tracker.debian.org/pkg/libapt-pkg-perl)) are available as separated projects. + +Discussion happens mostly on [the mailinglist](mailto:deity@lists.debian.org) ([archive](https://lists.debian.org/deity/)) and on [IRC](irc://irc.oftc.net/debian-apt). +Our bugtracker as well as a general overview can be found at the [Debian Tracker page](https://tracker.debian.org/pkg/apt). + + +Contributing +------------ +APT is maintained in git, the official repository being located at +[https://salsa.debian.org/apt-team/apt](https://salsa.debian.org/apt-team/apt), +but also available at other locations like [GitHub](https://github.com/Debian/apt). + +The default branch is `master`, other branches targeted at different +derivatives and releases being used as needed. Various topic branches in +different stages of completion might be branched of from those, which you +are encouraged to do as well. + +### Coding + +APT uses cmake. To start building, you need to run + + cmake <path to source directory> + +from a build directory. For example, if you want to build in the source tree, +run: + + cmake . + +Then you can use make as you normally would (pass -j <count> to perform <count> +jobs in parallel). + +You can also use the Ninja generator of cmake, to do that pass + -G Ninja +to the cmake invocation, and then use ninja instead of make. + +The source code uses in most parts a relatively uncommon indent convention, +namely 3 spaces with 8 space tab (see [doc/style.txt](https://anonscm.debian.org/git/apt/apt.git/tree/doc/style.txt) for more on this). +Adhering to it avoids unnecessary code-churn destroying history (aka: `git blame`) +and you are therefore encouraged to write patches in this style. +Your editor can surely help you with this, for vim the settings would be +`setlocal shiftwidth=3 noexpandtab tabstop=8` +(the later two are the default configuration and could therefore be omitted). + +### Translations + +While we welcome contributions here, we highly encourage you to contact the [Debian Internationalization (i18n) team](https://wiki.debian.org/Teams/I18n). +Various language teams have formed which can help you creating, maintaining +and improving a translation, while we could only do a basic syntax check of the +file format… + +Further more, Translating APT is split into two independent parts: +The program translation, meaning the messages printed by the tools, +as well as the manpages and other documentation shipped with APT. + +### Bug triage + +Software tools like APT which are used by thousands of users every +day have a steady flow of incoming bugreports. Not all of them are really +bugs in APT: It can be packaging bugs like failing maintainer scripts a +user reports against apt, because apt was the command he executed leading +to this failure or various wishlist items for new features. Given enough time +also the occasional duplicate enters the system. +Our bugtracker is therefore full with open bugreports which are waiting for you! ;) + +Testing +------- + +### Manual execution + +When you make changes and want to run them manually, you can just do so. CMake +automatically inserts an rpath so the binaries find the correct libraries. + +Note that you have to invoke CMake with the right install prefix set (e.g. +`-DCMAKE_INSTALL_PREFIX=/usr`) to have your build find and use the right files +by default or alternatively set the locations at runtime via an `APT_CONFIG` +configuration file. + +### Integration tests + +There is an extensive integration testsuite available which can be run via: + + $ ./test/integration/run-tests + +Each test can also be run individually as well. The tests are very noisy by +default, especially so while running all of them it might be beneficial to +enabling quiet (`-q`) or very quiet (`-qq`) mode. The tests can also be run in +parallel via `-j X` where `X` is the number of jobs to run. + +While these tests are not executed at package build-time as they require +additional dependencies, the repository contains the configuration needed to +run them on [Travis CI](https://travis-ci.org/) and +[Shippable](https://shippable.com/) as well as via autopkgtests e.g. on +[Debian Continuous Integration](https://ci.debian.net/packages/a/apt/). + +A testcase here is a shellscript embedded in a framework creating an environment in which +apt tools can be used naturally without root-rights to test every aspect of its behavior +itself as well as in conjunction with dpkg and other tools while working with packages. + + +### Unit tests + +These tests are gtest-dev based, executed by ctest, reside in `./test/libapt` +and can be run with `make test`. They are executed at package build-time, but +not by `make`. CTest by default does not show the output of tests, even if they +failed, so to see more details you can also run them with `ctest --verbose`. + +Debugging +--------- + +APT does many things, so there is no central debug mode which could be +activated. It uses instead various config-options to activate debug output +in certain areas. The following describes some common scenarios and generally +useful options, but is in no way exhaustive. + +Note that you should *NEVER* use these settings as root to avoid accidents. +Simulation mode (`-s`) is usually sufficient to help you run apt as a non-root user. + +### Using different state files + +If a dependency solver bug is reported, but can't be reproduced by the +triager easily, it is beneficial to ask the reporter for the +`/var/lib/dpkg/status` file, which includes the packages installed on the +system and in which version. Such a file can then be used via the option +`dir::state::status`. Beware of different architecture settings! +Bugreports usually include this information in the template. Assuming you +already have the `Packages` files for the architecture (see `sources.list` +manpage for the `arch=` option) you can change to a different architecture +with a config file like: + + APT::Architecture "arch1"; + #clear APT::Architectures; + APT:: Architectures { "arch1"; "arch2"; } + +If a certain mirror state is needed, see if you can reproduce it with [snapshot.debian.org](http://snapshot.debian.org/). +Your sources.list file (`dir::etc::sourcelist`) has to be correctly mention the repository, +but if it does, you can use different downloaded archive state files via `dir::state::lists`. + +In case manually vs. automatically installed matters, you can ask the reporter for +the `/var/lib/apt/extended_states` file and use it with `dir::state::extended_states`. + +### Dependency resolution + +APT works in its internal resolver in two stages: First all packages are visited +and marked for installation, keep back or removal. Option `Debug::pkgDepCache::Marker` +shows this. This also decides which packages are to be installed to satisfy dependencies, +which can be seen by `Debug::pkgDepCache::AutoInstall`. After this is done, we might +be in a situation in which two packages want to be installed, but only on of them can be. +It is the job of the pkgProblemResolver to decide which of two packages 'wins' and can +therefore decide what has to happen. You can see the contenders as well as their fight and +the resulting resolution with `Debug::pkgProblemResolver`. + +### Downloading files + +Various binaries (called 'methods') are tasked with downloading files. The Acquire system +talks to them via simple text protocol. Depending on which side you want to see, either +`Debug::pkgAcquire::Worker` or `Debug::Acquire::http` (or similar) will show the messages. + +The integration tests use a simple self-built webserver which also logs. If you find that +the http(s) methods do not behave like they should be try to implement this behavior in the +webserver for simpler and more controlled testing. + +### Installation order + +Dependencies are solved, packages downloaded: Everything read for the installation! +The last step in the chain is often forgotten, but still very important: +Packages have to be installed in a particular order so that their dependencies are +satisfied, but at the same time you don't want to install very important and optional +packages at the same time if possible, so that a broken optional package does not +block the correct installation of very important packages. Which option to use depends on +if you are interested in the topology sorting (`Debug::pkgOrderList`), the dependency-aware +cycle and unconfigured prevention (`Debug::pkgPackageManager`) or the actual calls +to dpkg (`Debug::pkgDpkgPm`). + + +Additional documentation +------------------------ + +Many more things could and should be said about APT and its usage but are more +targeted at developers of related programs or only of special interest. + +* [Protocol specification of APTs communication with external dependency solvers (EDSP)](./doc/external-dependency-solver-protocol.md) +* [Protocol specification of APTs communication with external installation planners (EIPP)](./doc/external-installation-planner-protocol.md) +* [Howto use and configure APT to acquire additional files in 'update' operations](./doc/acquire-additional-files.md) +* [Download and package installation progress reporting details](./doc/progress-reporting.md) +* [Remarks on DNS SRV record support in APT](./doc/srv-records-support.md) +* [Protocol specification of APT interfacing with external hooks via JSON](./doc/json-hooks-protocol.md) diff --git a/abicheck/apt_build.xml.in b/abicheck/apt_build.xml.in new file mode 100644 index 0000000..32886d9 --- /dev/null +++ b/abicheck/apt_build.xml.in @@ -0,0 +1,12 @@ +<version> + build-branch +</version> + + <headers> + @build_path@/include/apt-pkg + </headers> + + <libs> + @build_path@/apt-pkg/ + @build_path@/apt-inst/ + </libs> diff --git a/abicheck/apt_installed.xml.in b/abicheck/apt_installed.xml.in new file mode 100644 index 0000000..c3ddd08 --- /dev/null +++ b/abicheck/apt_installed.xml.in @@ -0,0 +1,11 @@ +<version> + installed +</version> + +<headers> + /usr/include/apt-pkg/ +</headers> + +<libs> + @installed_libapt@ +</libs> diff --git a/abicheck/run_abi_test b/abicheck/run_abi_test new file mode 100755 index 0000000..491616b --- /dev/null +++ b/abicheck/run_abi_test @@ -0,0 +1,22 @@ +#!/bin/sh + +# ensure we are in the abibreak subdirectory +cd "$(readlink -f $(dirname $0))" + +if [ ! -d ../build ]; then + echo "../build missing, did you run make?" + exit 1 +fi + +if ! command -v abi-compliance-checker 2>/dev/null >&2; then + echo "Please install the 'abi-compliance-checker' package" + exit 1 +fi + +LIBPATH=$(find /usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH) -type f -regex '.*/libapt-\(pkg\|inst\)\.so\..*' -printf %p\\\\n) +sed s#@installed_libapt@#$LIBPATH# apt_installed.xml.in > apt_installed.xml + +BUILDPATH=$(readlink -f ../build) +sed s#@build_path@#$BUILDPATH# apt_build.xml.in > apt_build.xml + +abi-compliance-checker -l apt -d1 apt_installed.xml -d2 apt_build.xml $@ diff --git a/apt-inst/CMakeLists.txt b/apt-inst/CMakeLists.txt new file mode 100644 index 0000000..31da115 --- /dev/null +++ b/apt-inst/CMakeLists.txt @@ -0,0 +1,28 @@ +# Include apt-pkg directly, as some files have #include <system.h> +include_directories(${PROJECT_BINARY_DIR}/include/apt-pkg) + +# Set the version of the library +set(MAJOR 2.0) +set(MINOR 0) +set(APT_INST_MAJOR ${MAJOR} PARENT_SCOPE) + +# Definition of the C++ files used to build the library - note that this +# is expanded at CMake time, so you have to rerun cmake if you add or remove +# a file (you can just run cmake . in the build directory) +file(GLOB_RECURSE library "*.cc") +file(GLOB_RECURSE headers "*.h") + +# Create a library using the C++ files +add_library(apt-inst SHARED ${library}) + +# Link the library and set the SONAME +target_link_libraries(apt-inst PUBLIC apt-pkg ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(apt-inst PRIVATE ${CMAKE_THREAD_LIBS_INIT}) +set_target_properties(apt-inst PROPERTIES VERSION ${MAJOR}.${MINOR}) +set_target_properties(apt-inst PROPERTIES SOVERSION ${MAJOR}) +add_version_script(apt-inst) + +# Install the library and the headers +install(TARGETS apt-inst LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) +install(FILES ${headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/apt-pkg) +flatify(${PROJECT_BINARY_DIR}/include/apt-pkg/ "${headers}") diff --git a/apt-inst/contrib/arfile.cc b/apt-inst/contrib/arfile.cc new file mode 100644 index 0000000..6d4a1f1 --- /dev/null +++ b/apt-inst/contrib/arfile.cc @@ -0,0 +1,179 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + AR File - Handle an 'AR' archive + + AR Archives have plain text headers at the start of each file + section. The headers are aligned on a 2 byte boundary. + + Information about the structure of AR files can be found in ar(5) + on a BSD system, or in the binutils source. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/arfile.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/strutl.h> + +#include <string> +#include <string.h> +#include <sys/types.h> + +#include <apti18n.h> + /*}}}*/ + +struct ARArchive::MemberHeader +{ + char Name[16]; + char MTime[12]; + char UID[6]; + char GID[6]; + char Mode[8]; + char Size[10]; + char Magic[2]; +}; + +// ARArchive::ARArchive - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +ARArchive::ARArchive(FileFd &File) : List(0), File(File) +{ + LoadHeaders(); +} + /*}}}*/ +// ARArchive::~ARArchive - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +ARArchive::~ARArchive() +{ + while (List != 0) + { + Member *Tmp = List; + List = List->Next; + delete Tmp; + } +} + /*}}}*/ +// ARArchive::LoadHeaders - Load the headers from each file /*{{{*/ +// --------------------------------------------------------------------- +/* AR files are structured with a 8 byte magic string followed by a 60 + byte plain text header then the file data, another header, data, etc */ +bool ARArchive::LoadHeaders() +{ + off_t Left = File.Size(); + + // Check the magic byte + char Magic[8]; + if (File.Read(Magic,sizeof(Magic)) == false) + return false; + if (memcmp(Magic,"!<arch>\012",sizeof(Magic)) != 0) + return _error->Error(_("Invalid archive signature")); + Left -= sizeof(Magic); + + // Read the member list + while (Left > 0) + { + MemberHeader Head; + if (File.Read(&Head,sizeof(Head)) == false) + return _error->Error(_("Error reading archive member header")); + Left -= sizeof(Head); + + // Convert all of the integer members + Member *Memb = new Member(); + if (StrToNum(Head.MTime,Memb->MTime,sizeof(Head.MTime)) == false || + StrToNum(Head.UID,Memb->UID,sizeof(Head.UID)) == false || + StrToNum(Head.GID,Memb->GID,sizeof(Head.GID)) == false || + StrToNum(Head.Mode,Memb->Mode,sizeof(Head.Mode),8) == false || + StrToNum(Head.Size,Memb->Size,sizeof(Head.Size)) == false) + { + delete Memb; + return _error->Error(_("Invalid archive member header")); + } + + if (Left < 0 || Memb->Size > static_cast<unsigned long long>(Left)) + { + delete Memb; + return _error->Error(_("Invalid archive member header")); + } + // Check for an extra long name string + if (memcmp(Head.Name,"#1/",3) == 0) + { + char S[300]; + unsigned long Len; + if (StrToNum(Head.Name+3,Len,sizeof(Head.Size)-3) == false || + Len >= sizeof(S)) + { + delete Memb; + return _error->Error(_("Invalid archive member header")); + } + + if (Len > Memb->Size) + { + delete Memb; + return _error->Error(_("Invalid archive member header")); + } + + if (File.Read(S,Len) == false) + { + delete Memb; + return false; + } + S[Len] = 0; + Memb->Name = S; + Memb->Size -= Len; + Left -= Len; + } + else + { + unsigned int I = sizeof(Head.Name) - 1; + for (; Head.Name[I] == ' ' || Head.Name[I] == '/'; I--) + { + if (I == 0) + { + delete Memb; + return _error->Error(_("Invalid archive member header")); + } + } + Memb->Name = std::string(Head.Name,I+1); + } + + // Account for the AR header alignment + off_t Skip = Memb->Size % 2; + + // Add it to the list + Memb->Next = List; + List = Memb; + Memb->Start = File.Tell(); + if (File.Skip(Memb->Size + Skip) == false) + return false; + if (Left < (off_t)(Memb->Size + Skip)) + return _error->Error(_("Archive is too short")); + Left -= Memb->Size + Skip; + } + if (Left != 0) + return _error->Error(_("Failed to read the archive headers")); + + return true; +} + /*}}}*/ +// ARArchive::FindMember - Find a name in the member list /*{{{*/ +// --------------------------------------------------------------------- +/* Find a member with the given name */ +const ARArchive::Member *ARArchive::FindMember(const char *Name) const +{ + const Member *Res = List; + while (Res != 0) + { + if (Res->Name == Name) + return Res; + Res = Res->Next; + } + + return 0; +} + /*}}}*/ diff --git a/apt-inst/contrib/arfile.h b/apt-inst/contrib/arfile.h new file mode 100644 index 0000000..8124208 --- /dev/null +++ b/apt-inst/contrib/arfile.h @@ -0,0 +1,69 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + AR File - Handle an 'AR' archive + + This is a reader for the usual 4.4 BSD AR format. It allows raw + stream access to a single member at a time. Basically all this class + provides is header parsing and verification. It is up to the client + to correctly make use of the stream start/stop points. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_ARFILE_H +#define PKGLIB_ARFILE_H + +#include <apt-pkg/macros.h> +#include <string> +#ifndef APT_8_CLEANER_HEADERS +#include <apt-pkg/fileutl.h> +#endif + +class FileFd; + +class ARArchive +{ + struct MemberHeader; + public: + struct Member; + + protected: + + // Linked list of members + Member *List; + + bool LoadHeaders(); + + public: + + // The stream file + FileFd &File; + + // Locate a member by name + const Member *FindMember(const char *Name) const; + inline Member *Members() { return List; } + + ARArchive(FileFd &File); + ~ARArchive(); +}; + +// A member of the archive +struct ARArchive::Member +{ + // Fields from the header + std::string Name; + unsigned long MTime; + unsigned long UID; + unsigned long GID; + unsigned long Mode; + unsigned long long Size; + + // Location of the data. + unsigned long long Start; + Member *Next; + + Member() : Start(0), Next(0) {}; +}; + +#endif diff --git a/apt-inst/contrib/extracttar.cc b/apt-inst/contrib/extracttar.cc new file mode 100644 index 0000000..cbee4d1 --- /dev/null +++ b/apt-inst/contrib/extracttar.cc @@ -0,0 +1,329 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Extract a Tar - Tar Extractor + + Some performance measurements showed that zlib performed quite poorly + in comparison to a forked gzip process. This tar extractor makes use + of the fact that dup'd file descriptors have the same seek pointer + and that gzip will not read past the end of a compressed stream, + even if there is more data. We use the dup property to track extraction + progress and the gzip feature to just feed gzip a fd in the middle + of an AR file. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/dirstream.h> +#include <apt-pkg/error.h> +#include <apt-pkg/extracttar.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <iostream> +#include <string> +#include <fcntl.h> +#include <signal.h> +#include <string.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +// The on disk header for a tar file. +struct ExtractTar::TarHeader +{ + char Name[100]; + char Mode[8]; + char UserID[8]; + char GroupID[8]; + char Size[12]; + char MTime[12]; + char Checksum[8]; + char LinkFlag; + char LinkName[100]; + char MagicNumber[8]; + char UserName[32]; + char GroupName[32]; + char Major[8]; + char Minor[8]; +}; + +// We need to read long names (names and link targets) into memory, so let's +// have a limit (shamelessly stolen from libarchive) to avoid people OOMing +// us with large streams. +static const unsigned long long APT_LONGNAME_LIMIT = 1048576llu; + +// A file size limit that we allow extracting. Currently, that's 128 GB. +// We also should leave some wiggle room for code adding files to it, and +// possibly conversion for signed, so this should not be larger than like 2**62. +static const unsigned long long APT_FILESIZE_LIMIT = 1llu << 37; + +// ExtractTar::ExtractTar - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +ExtractTar::ExtractTar(FileFd &Fd,unsigned long long Max,string DecompressionProgram) + : File(Fd), MaxInSize(Max), DecompressProg(DecompressionProgram) +{ + GZPid = -1; + Eof = false; +} + /*}}}*/ +// ExtractTar::ExtractTar - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +ExtractTar::~ExtractTar() +{ + // Error close + Done(); +} + /*}}}*/ +// ExtractTar::Done - Reap the gzip sub process /*{{{*/ +bool ExtractTar::Done(bool) +{ + return Done(); +} +bool ExtractTar::Done() +{ + return InFd.Close(); +} + /*}}}*/ +// ExtractTar::StartGzip - Startup gzip /*{{{*/ +// --------------------------------------------------------------------- +/* This creates a gzip sub process that has its input as the file itself. + If this tar file is embedded into something like an ar file then + gzip will efficiently ignore the extra bits. */ +bool ExtractTar::StartGzip() +{ + if (DecompressProg.empty()) + { + InFd.OpenDescriptor(File.Fd(), FileFd::ReadOnly, FileFd::None, false); + return true; + } + + std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors(); + std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin(); + for (; compressor != compressors.end(); ++compressor) { + if (compressor->Name == DecompressProg) { + return InFd.OpenDescriptor(File.Fd(), FileFd::ReadOnly, *compressor, false); + } + } + + return _error->Error(_("Cannot find a configured compressor for '%s'"), + DecompressProg.c_str()); + +} + /*}}}*/ +// ExtractTar::Go - Perform extraction /*{{{*/ +// --------------------------------------------------------------------- +/* This reads each 512 byte block from the archive and extracts the header + information into the Item structure. Then it resolves the UID/GID and + invokes the correct processing function. */ +bool ExtractTar::Go(pkgDirStream &Stream) +{ + if (StartGzip() == false) + return false; + + // Loop over all blocks + string LastLongLink, ItemLink; + string LastLongName, ItemName; + while (1) + { + bool BadRecord = false; + unsigned char Block[512]; + if (InFd.Read(Block,sizeof(Block),true) == false) + return false; + + if (InFd.Eof() == true) + break; + + // Get the checksum + TarHeader *Tar = (TarHeader *)Block; + unsigned long CheckSum; + if (StrToNum(Tar->Checksum,CheckSum,sizeof(Tar->Checksum),8) == false) + return _error->Error(_("Corrupted archive")); + + /* Compute the checksum field. The actual checksum is blanked out + with spaces so it is not included in the computation */ + unsigned long NewSum = 0; + memset(Tar->Checksum,' ',sizeof(Tar->Checksum)); + for (int I = 0; I != sizeof(Block); I++) + NewSum += Block[I]; + + /* Check for a block of nulls - in this case we kill gzip, GNU tar + does this.. */ + if (NewSum == ' '*sizeof(Tar->Checksum)) + return Done(); + + if (NewSum != CheckSum) + return _error->Error(_("Tar checksum failed, archive corrupted")); + + // Decode all of the fields + pkgDirStream::Item Itm; + if (StrToNum(Tar->Mode,Itm.Mode,sizeof(Tar->Mode),8) == false || + (Base256ToNum(Tar->UserID,Itm.UID,8) == false && + StrToNum(Tar->UserID,Itm.UID,sizeof(Tar->UserID),8) == false) || + (Base256ToNum(Tar->GroupID,Itm.GID,8) == false && + StrToNum(Tar->GroupID,Itm.GID,sizeof(Tar->GroupID),8) == false) || + (Base256ToNum(Tar->Size,Itm.Size,12) == false && + StrToNum(Tar->Size,Itm.Size,sizeof(Tar->Size),8) == false) || + (Base256ToNum(Tar->MTime,Itm.MTime,12) == false && + StrToNum(Tar->MTime,Itm.MTime,sizeof(Tar->MTime),8) == false) || + StrToNum(Tar->Major,Itm.Major,sizeof(Tar->Major),8) == false || + StrToNum(Tar->Minor,Itm.Minor,sizeof(Tar->Minor),8) == false) + return _error->Error(_("Corrupted archive")); + + // Security check. Prevents overflows below the code when rounding up in skip/copy code, + // and provides modest protection against decompression bombs. + if (Itm.Size > APT_FILESIZE_LIMIT) + return _error->Error("Tar member too large: %llu > %llu bytes", Itm.Size, APT_FILESIZE_LIMIT); + + // Grab the filename and link target: use last long name if one was + // set, otherwise use the header value as-is, but remember that it may + // fill the entire 100-byte block and needs to be zero-terminated. + // See Debian Bug #689582. + if (LastLongName.empty() == false) + Itm.Name = (char *)LastLongName.c_str(); + else + Itm.Name = (char *)ItemName.assign(Tar->Name, sizeof(Tar->Name)).c_str(); + if (Itm.Name[0] == '.' && Itm.Name[1] == '/' && Itm.Name[2] != 0) + Itm.Name += 2; + + if (LastLongLink.empty() == false) + Itm.LinkTarget = (char *)LastLongLink.c_str(); + else + Itm.LinkTarget = (char *)ItemLink.assign(Tar->LinkName, sizeof(Tar->LinkName)).c_str(); + + // Convert the type over + switch (Tar->LinkFlag) + { + case NormalFile0: + case NormalFile: + Itm.Type = pkgDirStream::Item::File; + break; + + case HardLink: + Itm.Type = pkgDirStream::Item::HardLink; + break; + + case SymbolicLink: + Itm.Type = pkgDirStream::Item::SymbolicLink; + break; + + case CharacterDevice: + Itm.Type = pkgDirStream::Item::CharDevice; + break; + + case BlockDevice: + Itm.Type = pkgDirStream::Item::BlockDevice; + break; + + case Directory: + Itm.Type = pkgDirStream::Item::Directory; + break; + + case FIFO: + Itm.Type = pkgDirStream::Item::FIFO; + break; + + case GNU_LongLink: + { + unsigned long long Length = Itm.Size; + unsigned char Block[512]; + if (Length > APT_LONGNAME_LIMIT) + return _error->Error("Long name to large: %llu bytes > %llu bytes", Length, APT_LONGNAME_LIMIT); + while (Length > 0) + { + if (InFd.Read(Block,sizeof(Block),true) == false) + return false; + if (Length <= sizeof(Block)) + { + LastLongLink.append(Block,Block+sizeof(Block)); + break; + } + LastLongLink.append(Block,Block+sizeof(Block)); + Length -= sizeof(Block); + } + continue; + } + + case GNU_LongName: + { + unsigned long long Length = Itm.Size; + unsigned char Block[512]; + if (Length > APT_LONGNAME_LIMIT) + return _error->Error("Long name to large: %llu bytes > %llu bytes", Length, APT_LONGNAME_LIMIT); + while (Length > 0) + { + if (InFd.Read(Block,sizeof(Block),true) == false) + return false; + if (Length < sizeof(Block)) + { + LastLongName.append(Block,Block+sizeof(Block)); + break; + } + LastLongName.append(Block,Block+sizeof(Block)); + Length -= sizeof(Block); + } + continue; + } + + default: + BadRecord = true; + _error->Warning(_("Unknown TAR header type %u"), (unsigned)Tar->LinkFlag); + break; + } + + int Fd = -1; + if (BadRecord == false) + if (Stream.DoItem(Itm,Fd) == false) + return false; + + // Copy the file over the FD + unsigned long long Size = Itm.Size; + while (Size != 0) + { + unsigned char Junk[32*1024]; + unsigned long Read = min(Size, (unsigned long long)sizeof(Junk)); + if (InFd.Read(Junk,((Read+511)/512)*512) == false) + return false; + + if (BadRecord == false) + { + if (Fd > 0) + { + if (write(Fd,Junk,Read) != (signed)Read) + return Stream.Fail(Itm,Fd); + } + else + { + /* An Fd of -2 means to send to a special processing + function */ + if (Fd == -2) + if (Stream.Process(Itm,Junk,Read,Itm.Size - Size) == false) + return Stream.Fail(Itm,Fd); + } + } + + Size -= Read; + } + + // And finish up + if (BadRecord == false) + if (Stream.FinishedFile(Itm,Fd) == false) + return false; + + LastLongName.erase(); + LastLongLink.erase(); + } + + return Done(); +} + /*}}}*/ diff --git a/apt-inst/contrib/extracttar.h b/apt-inst/contrib/extracttar.h new file mode 100644 index 0000000..c0b340e --- /dev/null +++ b/apt-inst/contrib/extracttar.h @@ -0,0 +1,61 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Extract a Tar - Tar Extractor + + The tar extractor takes an ordinary gzip compressed tar stream from + the given file and explodes it, passing the individual items to the + given Directory Stream for processing. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_EXTRACTTAR_H +#define PKGLIB_EXTRACTTAR_H + +#include <apt-pkg/fileutl.h> +#include <apt-pkg/macros.h> + +#include <string> + +#ifndef APT_8_CLEANER_HEADERS +#include <apt-pkg/dirstream.h> +#include <algorithm> +using std::min; +#endif + +class pkgDirStream; + +class ExtractTar +{ + protected: + + struct TarHeader; + + // The varios types items can be + enum ItemType {NormalFile0 = '\0',NormalFile = '0',HardLink = '1', + SymbolicLink = '2',CharacterDevice = '3', + BlockDevice = '4',Directory = '5',FIFO = '6', + GNU_LongLink = 'K',GNU_LongName = 'L'}; + + FileFd &File; + unsigned long long MaxInSize; + int GZPid; + FileFd InFd; + bool Eof; + std::string DecompressProg; + + // Fork and reap gzip + bool StartGzip(); + bool Done(); + APT_DEPRECATED_MSG("Parameter Force is ignored, use Done() instead.") bool Done(bool Force); + + public: + + bool Go(pkgDirStream &Stream); + + ExtractTar(FileFd &Fd,unsigned long long Max,std::string DecompressionProgram); + virtual ~ExtractTar(); +}; + +#endif diff --git a/apt-inst/deb/debfile.cc b/apt-inst/deb/debfile.cc new file mode 100644 index 0000000..bef0cd0 --- /dev/null +++ b/apt-inst/deb/debfile.cc @@ -0,0 +1,265 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Archive File (.deb) + + .DEB archives are AR files containing two tars and an empty marker + member called 'debian-binary'. The two tars contain the meta data and + the actual archive contents. Thus this class is a very simple wrapper + around ar/tar to simply extract the right tar files. + + It also uses the deb package list parser to parse the control file + into the cache. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/arfile.h> +#include <apt-pkg/debfile.h> +#include <apt-pkg/dirstream.h> +#include <apt-pkg/error.h> +#include <apt-pkg/extracttar.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/tagfile.h> + +#include <string> +#include <vector> +#include <string.h> +#include <sys/stat.h> + +#include <apti18n.h> + /*}}}*/ + +// DebFile::debDebFile - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Open the AR file and check for consistency */ +debDebFile::debDebFile(FileFd &File) : File(File), AR(File) +{ + if (_error->PendingError() == true) + return; + + if (!CheckMember("debian-binary")) { + _error->Error(_("This is not a valid DEB archive, missing '%s' member"), "debian-binary"); + return; + } + + if (!CheckMember("control.tar") && + !CheckMember("control.tar.gz") && + !CheckMember("control.tar.xz") && + !CheckMember("control.tar.zst")) + { + _error->Error(_("This is not a valid DEB archive, missing '%s' member"), "control.tar"); + return; + } + + if (!CheckMember("data.tar") && + !CheckMember("data.tar.gz") && + !CheckMember("data.tar.bz2") && + !CheckMember("data.tar.lzma") && + !CheckMember("data.tar.xz") && + !CheckMember("data.tar.zst")) + { + _error->Error(_("This is not a valid DEB archive, missing '%s' member"), "data.tar"); + return; + } +} + /*}}}*/ +// DebFile::CheckMember - Check if a named member is in the archive /*{{{*/ +// --------------------------------------------------------------------- +/* This is used to check for a correct deb and to give nicer error messages + for people playing around. */ +bool debDebFile::CheckMember(const char *Name) +{ + if (AR.FindMember(Name) == 0) + return false; + return true; +} + /*}}}*/ +// DebFile::GotoMember - Jump to a Member /*{{{*/ +// --------------------------------------------------------------------- +/* Jump in the file to the start of a named member and return the information + about that member. The caller can then read from the file up to the + returned size. Note, since this relies on the file position this is + a destructive operation, it also changes the last returned Member + structure - so don't nest them! */ +const ARArchive::Member *debDebFile::GotoMember(const char *Name) +{ + // Get the archive member and positition the file + const ARArchive::Member *Member = AR.FindMember(Name); + if (Member == 0) + { + return 0; + } + if (File.Seek(Member->Start) == false) + return 0; + + return Member; +} + /*}}}*/ +// DebFile::ExtractTarMember - Extract the contents of a tar member /*{{{*/ +// --------------------------------------------------------------------- +/* Simple wrapper around tar.. */ +bool debDebFile::ExtractTarMember(pkgDirStream &Stream,const char *Name) +{ + // Get the archive member + const ARArchive::Member *Member = NULL; + std::string Compressor; + + std::vector<APT::Configuration::Compressor> compressor = APT::Configuration::getCompressors(); + for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) + { + Member = AR.FindMember(std::string(Name).append(c->Extension).c_str()); + if (Member == NULL) + continue; + Compressor = c->Name; + break; + } + + if (Member == NULL) + Member = AR.FindMember(std::string(Name).c_str()); + + if (Member == NULL) + { + std::string ext = std::string(Name) + ".{"; + for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) { + if (!c->Extension.empty()) + ext.append(c->Extension.substr(1)); + } + ext.append("}"); + return _error->Error(_("Internal error, could not locate member %s"), ext.c_str()); + } + + if (File.Seek(Member->Start) == false) + return false; + + // Prepare Tar + ExtractTar Tar(File,Member->Size,Compressor); + if (_error->PendingError() == true) + return false; + return Tar.Go(Stream); +} + /*}}}*/ +// DebFile::ExtractArchive - Extract the archive data itself /*{{{*/ +// --------------------------------------------------------------------- +/* Simple wrapper around DebFile::ExtractTarMember. */ +bool debDebFile::ExtractArchive(pkgDirStream &Stream) +{ + return ExtractTarMember(Stream, "data.tar"); +} + /*}}}*/ + +// DebFile::ControlExtract::DoItem - Control Tar Extraction /*{{{*/ +// --------------------------------------------------------------------- +/* This directory stream handler for the control tar handles extracting + it into the temporary meta directory. It only extracts files, it does + not create directories, links or anything else. */ +bool debDebFile::ControlExtract::DoItem(Item &Itm,int &Fd) +{ + if (Itm.Type != Item::File) + return true; + + /* Cleanse the file name, prevent people from trying to unpack into + absolute paths, .., etc */ + for (char *I = Itm.Name; *I != 0; I++) + if (*I == '/') + *I = '_'; + + /* Force the ownership to be root and ensure correct permissions, + go-w, the rest are left untouched */ + Itm.UID = 0; + Itm.GID = 0; + Itm.Mode &= ~(S_IWGRP | S_IWOTH); + + return pkgDirStream::DoItem(Itm,Fd); +} + /*}}}*/ + +// MemControlExtract::DoItem - Check if it is the control file /*{{{*/ +// --------------------------------------------------------------------- +/* This sets up to extract the control block member file into a memory + block of just the right size. All other files go into the bit bucket. */ + +// Upper size limit for control files. Two reasons for having a limit here: +// +// 1. We read those files into memory and want to avoid being killed by OOM +// +// 2. We allocate (Itm.Size+2)-large arrays, so this can overflow if Itm.Size +// becomes 2**64-2 or larger. This is obviously +// +// 64 MiB seems like a terribly large size that everyone should be happy with. +static const unsigned long long DEB_CONTROL_SIZE_LIMIT = 64 * 1024 * 1024; +bool debDebFile::MemControlExtract::DoItem(Item &Itm,int &Fd) +{ + // At the control file, allocate buffer memory. + if (Member == Itm.Name) + { + if (Itm.Size > DEB_CONTROL_SIZE_LIMIT) + return _error->Error("Control file too large: %llu > %llu bytes", Itm.Size, DEB_CONTROL_SIZE_LIMIT); + delete [] Control; + Control = new char[Itm.Size+2]; + IsControl = true; + Fd = -2; // Signal to pass to Process + Length = Itm.Size; + } + else + IsControl = false; + + return true; +} + /*}}}*/ +// MemControlExtract::Process - Process extracting the control file /*{{{*/ +// --------------------------------------------------------------------- +/* Just memcopy the block from the tar extractor and put it in the right + place in the pre-allocated memory block. */ +bool debDebFile::MemControlExtract::Process(Item &/*Itm*/,const unsigned char *Data, + unsigned long long Size,unsigned long long Pos) +{ + memcpy(Control + Pos, Data,Size); + return true; +} + /*}}}*/ +// MemControlExtract::Read - Read the control information from the deb /*{{{*/ +// --------------------------------------------------------------------- +/* This uses the internal tar extractor to fetch the control file, and then + it parses it into a tag section parser. */ +bool debDebFile::MemControlExtract::Read(debDebFile &Deb) +{ + if (Deb.ExtractTarMember(*this, "control.tar") == false) + return false; + + if (Control == 0) + return true; + + Control[Length] = '\n'; + Control[Length+1] = '\n'; + if (Section.Scan(Control,Length+2) == false) + return _error->Error(_("Unparsable control file")); + return true; +} + /*}}}*/ +// MemControlExtract::TakeControl - Parse a memory block /*{{{*/ +// --------------------------------------------------------------------- +/* The given memory block is loaded into the parser and parsed as a control + record. */ +bool debDebFile::MemControlExtract::TakeControl(const void *Data,unsigned long long Size) +{ + if (Size > DEB_CONTROL_SIZE_LIMIT) + return _error->Error("Control file too large: %llu > %llu bytes", Size, DEB_CONTROL_SIZE_LIMIT); + + delete [] Control; + Control = new char[Size+2]; + Length = Size; + memcpy(Control,Data,Size); + + Control[Length] = '\n'; + Control[Length+1] = '\n'; + return Section.Scan(Control,Length+2); +} + /*}}}*/ + diff --git a/apt-inst/deb/debfile.h b/apt-inst/deb/debfile.h new file mode 100644 index 0000000..23a76bf --- /dev/null +++ b/apt-inst/deb/debfile.h @@ -0,0 +1,95 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Debian Archive File (.deb) + + This Class handles all the operations performed directly on .deb + files. It makes use of the AR and TAR classes to give the necessary + external interface. + + There are only two things that can be done with a raw package, + extract it's control information and extract the contents itself. + + This should probably subclass an as-yet unwritten super class to + produce a generic archive mechanism. + + The memory control file extractor is useful to extract a single file + into memory from the control.tar.gz + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_DEBFILE_H +#define PKGLIB_DEBFILE_H + +#include <apt-pkg/arfile.h> +#include <apt-pkg/dirstream.h> +#include <apt-pkg/macros.h> +#include <apt-pkg/tagfile.h> + +#include <string> + +#ifndef APT_8_CLEANER_HEADERS +#include <apt-pkg/md5.h> +#endif +#ifndef APT_10_CLEANER_HEADERS +#include <apt-pkg/pkgcache.h> +#endif + +class FileFd; + +class debDebFile +{ + protected: + + FileFd &File; + ARArchive AR; + + bool CheckMember(const char *Name); + + public: + class ControlExtract; + class MemControlExtract; + + bool ExtractTarMember(pkgDirStream &Stream, const char *Name); + bool ExtractArchive(pkgDirStream &Stream); + const ARArchive::Member *GotoMember(const char *Name); + inline FileFd &GetFile() {return File;}; + + debDebFile(FileFd &File); +}; + +class debDebFile::ControlExtract : public pkgDirStream +{ + public: + + virtual bool DoItem(Item &Itm,int &Fd) APT_OVERRIDE; +}; + +class debDebFile::MemControlExtract : public pkgDirStream +{ + bool IsControl; + + public: + + char *Control; + pkgTagSection Section; + unsigned long Length; + std::string Member; + + // Members from DirStream + virtual bool DoItem(Item &Itm,int &Fd) APT_OVERRIDE; + virtual bool Process(Item &Itm,const unsigned char *Data, + unsigned long long Size,unsigned long long Pos) APT_OVERRIDE; + + // Helpers + bool Read(debDebFile &Deb); + bool TakeControl(const void *Data,unsigned long long Size); + + MemControlExtract() : IsControl(false), Control(0), Length(0), Member("control") {}; + MemControlExtract(std::string Member) : IsControl(false), Control(0), Length(0), Member(Member) {}; + ~MemControlExtract() {delete [] Control;}; +}; + /*}}}*/ + +#endif diff --git a/apt-inst/dirstream.cc b/apt-inst/dirstream.cc new file mode 100644 index 0000000..d6cf0ab --- /dev/null +++ b/apt-inst/dirstream.cc @@ -0,0 +1,118 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Directory Stream + + This class provides a simple basic extractor that can be used for + a number of purposes. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/dirstream.h> +#include <apt-pkg/error.h> + +#include <errno.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <unistd.h> +#include <apti18n.h> + /*}}}*/ + +// DirStream::DoItem - Process an item /*{{{*/ +// --------------------------------------------------------------------- +/* This is a very simple extractor, it does not deal with things like + overwriting directories with files and so on. */ +bool pkgDirStream::DoItem(Item &Itm,int &Fd) +{ + switch (Itm.Type) + { + case Item::File: + { + /* Open the output file, NDELAY is used to prevent this from + blowing up on device special files.. */ + int iFd = open(Itm.Name,O_NDELAY|O_WRONLY|O_CREAT|O_TRUNC|O_APPEND, + Itm.Mode); + if (iFd < 0) + return _error->Errno("open",_("Failed to write file %s"), + Itm.Name); + + // fchmod deals with umask and fchown sets the ownership + if (fchmod(iFd,Itm.Mode) != 0) + { + close(iFd); + return _error->Errno("fchmod",_("Failed to write file %s"), Itm.Name); + } + if (fchown(iFd,Itm.UID,Itm.GID) != 0 && errno != EPERM) + { + close(iFd); + return _error->Errno("fchown",_("Failed to write file %s"), Itm.Name); + } + Fd = iFd; + return true; + } + + case Item::HardLink: + case Item::SymbolicLink: + case Item::CharDevice: + case Item::BlockDevice: + case Item::Directory: + { + struct stat Buf; + // check if the dir is already there, if so return true + if (stat(Itm.Name,&Buf) == 0) + { + if(S_ISDIR(Buf.st_mode)) + return true; + // something else is there already, return false + return false; + } + // nothing here, create the dir + if(mkdir(Itm.Name,Itm.Mode) < 0) + return false; + return true; + } + case Item::FIFO: + break; + } + + return true; +} + /*}}}*/ +// DirStream::FinishedFile - Finished processing a file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgDirStream::FinishedFile(Item &Itm,int Fd) +{ + if (Fd < 0) + return true; + + /* Set the modification times. The only way it can fail is if someone + has futzed with our file, which is intolerable :> */ + struct timeval times[2]; + times[0].tv_sec = times[1].tv_sec = Itm.MTime; + times[0].tv_usec = times[1].tv_usec = 0; + if (utimes(Itm.Name, times) != 0) + _error->Errno("utimes", "Failed to set modification time for %s",Itm.Name); + + if (close(Fd) != 0) + return _error->Errno("close",_("Failed to close file %s"),Itm.Name); + return true; +} + /*}}}*/ +// DirStream::Fail - Failed processing a file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgDirStream::Fail(Item &/*Itm*/, int Fd) +{ + if (Fd < 0) + return true; + + close(Fd); + return false; +} + /*}}}*/ diff --git a/apt-inst/dirstream.h b/apt-inst/dirstream.h new file mode 100644 index 0000000..0f0e348 --- /dev/null +++ b/apt-inst/dirstream.h @@ -0,0 +1,57 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Directory Stream + + When unpacking the contents of the archive are passed into a directory + stream class for analysis and processing. The class controls all aspects + of actually writing the directory stream from disk. The low level + archive handlers are only responsible for decoding the archive format + and sending events (via method calls) to the specified directory + stream. + + When unpacking a real file the archive handler is passed back a file + handle to write the data to, this is to support strange + archives+unpacking methods. If that fd is -1 then the file data is + simply ignored. + + The provided defaults do the 'Right Thing' for a normal unpacking + process (ie 'tar') + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_DIRSTREAM_H +#define PKGLIB_DIRSTREAM_H + +#include <apt-pkg/macros.h> + +class pkgDirStream +{ + public: + + // All possible information about a component + struct Item + { + enum Type_t {File, HardLink, SymbolicLink, CharDevice, BlockDevice, + Directory, FIFO} Type; + char *Name; + char *LinkTarget; + unsigned long Mode; + unsigned long UID; + unsigned long GID; + unsigned long long Size; + unsigned long MTime; + unsigned long Major; + unsigned long Minor; + }; + + virtual bool DoItem(Item &Itm,int &Fd); + virtual bool Fail(Item &Itm,int Fd); + virtual bool FinishedFile(Item &Itm,int Fd); + virtual bool Process(Item &/*Itm*/,const unsigned char * /*Data*/, + unsigned long long /*Size*/,unsigned long long /*Pos*/) {return true;}; + virtual ~pkgDirStream() {}; +}; + +#endif diff --git a/apt-inst/dpkg-diffs.txt b/apt-inst/dpkg-diffs.txt new file mode 100644 index 0000000..d161055 --- /dev/null +++ b/apt-inst/dpkg-diffs.txt @@ -0,0 +1,5 @@ +- Replacing directories with files + dpkg permits this with the weak condition that the directory is owned only + by the package. APT requires that the directory have no files that are not + owned by the package. Replaces are specifically not checked to prevent + file list corruption. diff --git a/apt-inst/extract.cc b/apt-inst/extract.cc new file mode 100644 index 0000000..35fa015 --- /dev/null +++ b/apt-inst/extract.cc @@ -0,0 +1,514 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Archive Extraction Directory Stream + + Extraction for each file is a bit of an involved process. Each object + undergoes an atomic backup, overwrite, erase sequence. First the + object is unpacked to '.dpkg.new' then the original is hardlinked to + '.dpkg.tmp' and finally the new object is renamed to overwrite the old + one. From an external perspective the file never ceased to exist. + After the archive has been successfully unpacked the .dpkg.tmp files + are erased. A failure causes all the .dpkg.tmp files to be restored. + + Decisions about unpacking go like this: + - Store the original filename in the file listing + - Resolve any diversions that would effect this file, all checks + below apply to the diverted name, not the real one. + - Resolve any symlinked configuration files. + - If the existing file does not exist then .dpkg-tmp is checked for. + [Note, this is reduced to only check if a file was expected to be + there] + - If the existing link/file is not a directory then it is replaced + regardless + - If the existing link/directory is being replaced by a directory then + absolutely nothing happens. + - If the existing link/directory is being replaced by a link then + absolutely nothing happens. + - If the existing link/directory is being replaced by a non-directory + then this will abort if the package is not the sole owner of the + directory. [Note, this is changed to not happen if the directory + non-empty - that is, it only includes files that are part of this + package - prevents removing user files accidentally.] + - If the non-directory exists in the listing database and it + does not belong to the current package then an overwrite condition + is invoked. + + As we unpack we record the file list differences in the FL cache. If + we need to unroll the FL cache knows which files have been unpacked + and can undo. When we need to erase then it knows which files have not + been unpacked. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/debversion.h> +#include <apt-pkg/dirstream.h> +#include <apt-pkg/error.h> +#include <apt-pkg/extract.h> +#include <apt-pkg/filelist.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/mmap.h> +#include <apt-pkg/pkgcache.h> + +#include <iostream> +#include <string> +#include <dirent.h> +#include <errno.h> +#include <stdio.h> +#include <string.h> +#include <sys/stat.h> + +#include <apti18n.h> + /*}}}*/ +using namespace std; + +static const char *TempExt = "dpkg-tmp"; +//static const char *NewExt = "dpkg-new"; + +// Extract::pkgExtract - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgExtract::pkgExtract(pkgFLCache &FLCache,pkgCache::VerIterator Ver) : + FLCache(FLCache), Ver(Ver) +{ + FLPkg = FLCache.GetPkg(Ver.ParentPkg().Name(),true); + if (FLPkg.end() == true) + return; + Debug = true; +} + /*}}}*/ +// Extract::DoItem - Handle a single item from the stream /*{{{*/ +// --------------------------------------------------------------------- +/* This performs the setup for the extraction.. */ +bool pkgExtract::DoItem(Item &Itm, int &/*Fd*/) +{ + /* Strip any leading/trailing /s from the filename, then copy it to the + temp buffer and re-apply the leading / We use a class variable + to store the new filename for use by the three extraction funcs */ + char *End = FileName+1; + const char *I = Itm.Name; + for (; *I != 0 && *I == '/'; I++); + *FileName = '/'; + for (; *I != 0 && End < FileName + sizeof(FileName); I++, End++) + *End = *I; + if (End + 20 >= FileName + sizeof(FileName)) + return _error->Error(_("The path %s is too long"),Itm.Name); + for (; End > FileName && End[-1] == '/'; End--); + *End = 0; + Itm.Name = FileName; + + /* Lookup the file. Nde is the file [group] we are going to write to and + RealNde is the actual node we are manipulating. Due to diversions + they may be entirely different. */ + pkgFLCache::NodeIterator Nde = FLCache.GetNode(Itm.Name,End,0,false,false); + pkgFLCache::NodeIterator RealNde = Nde; + + // See if the file is already in the file listing + unsigned long FileGroup = RealNde->File; + for (; RealNde.end() == false && FileGroup == RealNde->File; RealNde++) + if (RealNde.RealPackage() == FLPkg) + break; + + // Nope, create an entry + if (RealNde.end() == true) + { + RealNde = FLCache.GetNode(Itm.Name,End,FLPkg.Offset(),true,false); + if (RealNde.end() == true) + return false; + RealNde->Flags |= pkgFLCache::Node::NewFile; + } + + /* Check if this entry already was unpacked. The only time this should + ever happen is if someone has hacked tar to support capabilities, in + which case this needs to be modified anyhow.. */ + if ((RealNde->Flags & pkgFLCache::Node::Unpacked) == + pkgFLCache::Node::Unpacked) + return _error->Error(_("Unpacking %s more than once"),Itm.Name); + + if (Nde.end() == true) + Nde = RealNde; + + /* Consider a diverted file - We are not permitted to divert directories, + but everything else is fair game (including conf files!) */ + if ((Nde->Flags & pkgFLCache::Node::Diversion) != 0) + { + if (Itm.Type == Item::Directory) + return _error->Error(_("The directory %s is diverted"),Itm.Name); + + /* A package overwriting a diversion target is just the same as + overwriting a normally owned file and is checked for below in + the overwrites mechanism */ + + /* If this package is trying to overwrite the target of a diversion, + that is never, ever permitted */ + pkgFLCache::DiverIterator Div = Nde.Diversion(); + if (Div.DivertTo() == Nde) + return _error->Error(_("The package is trying to write to the " + "diversion target %s/%s"),Nde.DirN(),Nde.File()); + + // See if it is us and we are following it in the right direction + if (Div->OwnerPkg != FLPkg.Offset() && Div.DivertFrom() == Nde) + { + Nde = Div.DivertTo(); + End = FileName + snprintf(FileName,sizeof(FileName)-20,"%s/%s", + Nde.DirN(),Nde.File()); + if (End <= FileName) + return _error->Error(_("The diversion path is too long")); + } + } + + // Deal with symlinks and conf files + if ((RealNde->Flags & pkgFLCache::Node::NewConfFile) == + pkgFLCache::Node::NewConfFile) + { + string Res = flNoLink(Itm.Name); + if (Res.length() > sizeof(FileName)) + return _error->Error(_("The path %s is too long"),Res.c_str()); + if (Debug == true) + clog << "Followed conf file from " << FileName << " to " << Res << endl; + Itm.Name = strcpy(FileName,Res.c_str()); + } + + /* Get information about the existing file, and attempt to restore + a backup if it does not exist */ + struct stat LExisting; + bool EValid = false; + if (lstat(Itm.Name,&LExisting) != 0) + { + // This is bad news. + if (errno != ENOENT) + return _error->Errno("stat",_("Failed to stat %s"),Itm.Name); + + // See if we can recover the backup file + if (Nde.end() == false) + { + char Temp[sizeof(FileName)]; + snprintf(Temp,sizeof(Temp),"%s.%s",Itm.Name,TempExt); + if (rename(Temp,Itm.Name) != 0 && errno != ENOENT) + return _error->Errno("rename",_("Failed to rename %s to %s"), + Temp,Itm.Name); + if (stat(Itm.Name,&LExisting) != 0) + { + if (errno != ENOENT) + return _error->Errno("stat",_("Failed to stat %s"),Itm.Name); + } + else + EValid = true; + } + } + else + EValid = true; + + /* If the file is a link we need to stat its destination, get the + existing file modes */ + struct stat Existing = LExisting; + if (EValid == true && S_ISLNK(Existing.st_mode)) + { + if (stat(Itm.Name,&Existing) != 0) + { + if (errno != ENOENT) + return _error->Errno("stat",_("Failed to stat %s"),Itm.Name); + Existing = LExisting; + } + } + + // We pretend a non-existing file looks like it is a normal file + if (EValid == false) + Existing.st_mode = S_IFREG; + + /* Okay, at this point 'Existing' is the stat information for the + real non-link file */ + + /* The only way this can be a no-op is if a directory is being + replaced by a directory or by a link */ + if (S_ISDIR(Existing.st_mode) != 0 && + (Itm.Type == Item::Directory || Itm.Type == Item::SymbolicLink)) + return true; + + /* Non-Directory being replaced by non-directory. We check for over + writes here. */ + if (Nde.end() == false) + { + if (HandleOverwrites(Nde) == false) + return false; + } + + /* Directory being replaced by a non-directory - this needs to see if + the package is the owner and then see if the directory would be + empty after the package is removed [ie no user files will be + erased] */ + if (S_ISDIR(Existing.st_mode) != 0) + { + if (CheckDirReplace(Itm.Name) == false) + return _error->Error(_("The directory %s is being replaced by a non-directory"),Itm.Name); + } + + if (Debug == true) + clog << "Extract " << string(Itm.Name,End) << endl; +/* if (Count != 0) + return _error->Error(_("Done"));*/ + + return true; +} + /*}}}*/ +// Extract::Finished - Sequence finished, erase the temp files /*{{{*/ +// --------------------------------------------------------------------- +/* */ +APT_PURE bool pkgExtract::Finished() +{ + return true; +} + /*}}}*/ +// Extract::Aborted - Sequence aborted, undo all our unpacking /*{{{*/ +// --------------------------------------------------------------------- +/* This undoes everything that was done by all calls to the DoItem method + and restores the File Listing cache to its original form. It bases its + actions on the flags value for each node in the cache. */ +bool pkgExtract::Aborted() +{ + if (Debug == true) + clog << "Aborted, backing out" << endl; + + pkgFLCache::NodeIterator Files = FLPkg.Files(); + map_ptrloc *Last = &FLPkg->Files; + + /* Loop over all files, restore those that have been unpacked from their + dpkg-tmp entries */ + while (Files.end() == false) + { + // Locate the hash bucket for the node and locate its group head + pkgFLCache::NodeIterator Nde(FLCache,FLCache.HashNode(Files)); + for (; Nde.end() == false && Files->File != Nde->File; Nde++); + if (Nde.end() == true) + return _error->Error(_("Failed to locate node in its hash bucket")); + + if (snprintf(FileName,sizeof(FileName)-20,"%s/%s", + Nde.DirN(),Nde.File()) <= 0) + return _error->Error(_("The path is too long")); + + // Deal with diversions + if ((Nde->Flags & pkgFLCache::Node::Diversion) != 0) + { + pkgFLCache::DiverIterator Div = Nde.Diversion(); + + // See if it is us and we are following it in the right direction + if (Div->OwnerPkg != FLPkg.Offset() && Div.DivertFrom() == Nde) + { + Nde = Div.DivertTo(); + if (snprintf(FileName,sizeof(FileName)-20,"%s/%s", + Nde.DirN(),Nde.File()) <= 0) + return _error->Error(_("The diversion path is too long")); + } + } + + // Deal with overwrites+replaces + for (; Nde.end() == false && Files->File == Nde->File; Nde++) + { + if ((Nde->Flags & pkgFLCache::Node::Replaced) == + pkgFLCache::Node::Replaced) + { + if (Debug == true) + clog << "De-replaced " << FileName << " from " << Nde.RealPackage()->Name << endl; + Nde->Flags &= ~pkgFLCache::Node::Replaced; + } + } + + // Undo the change in the filesystem + if (Debug == true) + clog << "Backing out " << FileName; + + // Remove a new node + if ((Files->Flags & pkgFLCache::Node::NewFile) == + pkgFLCache::Node::NewFile) + { + if (Debug == true) + clog << " [new node]" << endl; + pkgFLCache::Node *Tmp = Files; + Files++; + *Last = Tmp->NextPkg; + Tmp->NextPkg = 0; + + FLCache.DropNode(Tmp - FLCache.NodeP); + } + else + { + if (Debug == true) + clog << endl; + + Last = &Files->NextPkg; + Files++; + } + } + + return true; +} + /*}}}*/ +// Extract::Fail - Extraction of a file Failed /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgExtract::Fail(Item &Itm,int Fd) +{ + return pkgDirStream::Fail(Itm,Fd); +} + /*}}}*/ +// Extract::FinishedFile - Finished a file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgExtract::FinishedFile(Item &Itm,int Fd) +{ + return pkgDirStream::FinishedFile(Itm,Fd); +} + /*}}}*/ +// Extract::HandleOverwrites - See if a replaces covers this overwrite /*{{{*/ +// --------------------------------------------------------------------- +/* Check if the file is in a package that is being replaced by this + package or if the file is being overwritten. Note that if the file + is really a directory but it has been erased from the filesystem + this will fail with an overwrite message. This is a limitation of the + dpkg file information format. + + XX If a new package installs and another package replaces files in this + package what should we do? */ +bool pkgExtract::HandleOverwrites(pkgFLCache::NodeIterator Nde, + bool DiverCheck) +{ + pkgFLCache::NodeIterator TmpNde = Nde; + unsigned long DiverOwner = 0; + unsigned long FileGroup = Nde->File; + for (; Nde.end() == false && FileGroup == Nde->File; Nde++) + { + if ((Nde->Flags & pkgFLCache::Node::Diversion) != 0) + { + /* Store the diversion owner if this is the forward direction + of the diversion */ + if (DiverCheck == true) + DiverOwner = Nde.Diversion()->OwnerPkg; + continue; + } + + pkgFLCache::PkgIterator FPkg(FLCache,Nde.RealPackage()); + if (FPkg.end() == true || FPkg == FLPkg) + continue; + + /* This tests trips when we are checking a diversion to see + if something has already been diverted by this diversion */ + if (FPkg.Offset() == DiverOwner) + continue; + + // Now see if this package matches one in a replace depends + pkgCache::DepIterator Dep = Ver.DependsList(); + bool Ok = false; + for (; Dep.end() == false; ++Dep) + { + if (Dep->Type != pkgCache::Dep::Replaces) + continue; + + // Does the replaces apply to this package? + if (strcmp(Dep.TargetPkg().Name(),FPkg.Name()) != 0) + continue; + + /* Check the version for match. I do not think CurrentVer can be + 0 if we are here.. */ + pkgCache::PkgIterator Pkg = Dep.TargetPkg(); + if (Pkg->CurrentVer == 0) + { + _error->Warning(_("Overwrite package match with no version for %s"),Pkg.Name()); + continue; + } + + // Replaces is met + if (debVS.CheckDep(Pkg.CurrentVer().VerStr(),Dep->CompareOp,Dep.TargetVer()) == true) + { + if (Debug == true) + clog << "Replaced file " << Nde.DirN() << '/' << Nde.File() << " from " << Pkg.Name() << endl; + Nde->Flags |= pkgFLCache::Node::Replaced; + Ok = true; + break; + } + } + + // Negative Hit + if (Ok == false) + return _error->Error(_("File %s/%s overwrites the one in the package %s"), + Nde.DirN(),Nde.File(),FPkg.Name()); + } + + /* If this is a diversion we might have to recurse to process + the other side of it */ + if ((TmpNde->Flags & pkgFLCache::Node::Diversion) != 0) + { + pkgFLCache::DiverIterator Div = TmpNde.Diversion(); + if (Div.DivertTo() == TmpNde) + return HandleOverwrites(Div.DivertFrom(),true); + } + + return true; +} + /*}}}*/ +// Extract::CheckDirReplace - See if this directory can be erased /*{{{*/ +// --------------------------------------------------------------------- +/* If this directory is owned by a single package and that package is + replacing it with something non-directoryish then dpkg allows this. + We increase the requirement to be that the directory is non-empty after + the package is removed */ +bool pkgExtract::CheckDirReplace(string Dir,unsigned int Depth) +{ + // Looping? + if (Depth > 40) + return false; + + if (Dir[Dir.size() - 1] != '/') + Dir += '/'; + + DIR *D = opendir(Dir.c_str()); + if (D == 0) + return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + + string File; + for (struct dirent *Dent = readdir(D); Dent != 0; Dent = readdir(D)) + { + // Skip some files + if (strcmp(Dent->d_name,".") == 0 || + strcmp(Dent->d_name,"..") == 0) + continue; + + // Look up the node + File = Dir + Dent->d_name; + pkgFLCache::NodeIterator Nde = FLCache.GetNode(File.c_str(), + File.c_str() + File.length(),0,false,false); + + // The file is not owned by this package + if (Nde.end() != false || Nde.RealPackage() != FLPkg) + { + closedir(D); + return false; + } + + // See if it is a directory + struct stat St; + if (lstat(File.c_str(),&St) != 0) + { + closedir(D); + return _error->Errno("lstat",_("Unable to stat %s"),File.c_str()); + } + + // Recurse down directories + if (S_ISDIR(St.st_mode) != 0) + { + if (CheckDirReplace(File,Depth + 1) == false) + { + closedir(D); + return false; + } + } + } + + // No conflicts + closedir(D); + return true; +} + /*}}}*/ diff --git a/apt-inst/extract.h b/apt-inst/extract.h new file mode 100644 index 0000000..4b4c8d7 --- /dev/null +++ b/apt-inst/extract.h @@ -0,0 +1,49 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Archive Extraction Directory Stream + + This Directory Stream implements extraction of an archive into the + filesystem. It makes the choices on what files should be unpacked and + replaces as well as guiding the actual unpacking. + + When the unpacking sequence is completed one of the two functions, + Finished or Aborted must be called. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_EXTRACT_H +#define PKGLIB_EXTRACT_H + +#include <apt-pkg/dirstream.h> +#include <apt-pkg/filelist.h> +#include <apt-pkg/pkgcache.h> + +#include <string> + +class pkgExtract : public pkgDirStream +{ + pkgFLCache &FLCache; + pkgCache::VerIterator Ver; + pkgFLCache::PkgIterator FLPkg; + char FileName[1024]; + bool Debug; + + bool HandleOverwrites(pkgFLCache::NodeIterator Nde, + bool DiverCheck = false); + bool CheckDirReplace(std::string Dir,unsigned int Depth = 0); + + public: + + virtual bool DoItem(Item &Itm,int &Fd) APT_OVERRIDE; + virtual bool Fail(Item &Itm,int Fd) APT_OVERRIDE; + virtual bool FinishedFile(Item &Itm,int Fd) APT_OVERRIDE; + + bool Finished(); + bool Aborted(); + + pkgExtract(pkgFLCache &FLCache,pkgCache::VerIterator Ver); +}; + +#endif diff --git a/apt-inst/filelist.cc b/apt-inst/filelist.cc new file mode 100644 index 0000000..44b97d0 --- /dev/null +++ b/apt-inst/filelist.cc @@ -0,0 +1,586 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + File Listing - Manages a Cache of File -> Package names. + + Diversions add some significant complexity to the system. To keep + storage space down in the very special case of a diverted file no + extra bytes are allocated in the Node structure. Instead a diversion + is inserted directly into the hash table and its flag bit set. Every + lookup for that filename will always return the diversion. + + The hash buckets are stored in sorted form, with diversions having + the highest sort order. Identical files are assigned the same file + pointer, thus after a search all of the nodes owning that file can be + found by iterating down the bucket. + + Re-updates of diversions (another extremely special case) are done by + marking all diversions as untouched, then loading the entire diversion + list again, touching each diversion and then finally going back and + releasing all untouched diversions. It is assumed that the diversion + table will always be quite small and be a very irregular case. + + Diversions that are user-installed are represented by a package with + an empty name string. + + Conf files are handled like diversions by changing the meaning of the + Pointer field to point to a conf file entry - again to reduce over + head for a special case. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/error.h> +#include <apt-pkg/filelist.h> +#include <apt-pkg/mmap.h> +#include <apt-pkg/strutl.h> + +#include <iostream> +#include <string.h> +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +// FlCache::Header::Header - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Initialize the header variables. These are the defaults used when + creating new caches */ +pkgFLCache::Header::Header() +{ + Signature = 0xEA3F1295; + + /* Whenever the structures change the major version should be bumped, + whenever the generator changes the minor version should be bumped. */ + MajorVersion = 1; + MinorVersion = 0; + Dirty = true; + + HeaderSz = sizeof(pkgFLCache::Header); + NodeSz = sizeof(pkgFLCache::Node); + DirSz = sizeof(pkgFLCache::Directory); + PackageSz = sizeof(pkgFLCache::Package); + DiversionSz = sizeof(pkgFLCache::Diversion); + ConfFileSz = sizeof(pkgFLCache::ConfFile); + + NodeCount = 0; + DirCount = 0; + PackageCount = 0; + DiversionCount = 0; + ConfFileCount = 0; + HashSize = 1 << 14; + + FileHash = 0; + DirTree = 0; + Packages = 0; + Diversions = 0; + UniqNodes = 0; + memset(Pools,0,sizeof(Pools)); +} + /*}}}*/ +// FLCache::Header::CheckSizes - Check if the two headers have same *sz /*{{{*/ +// --------------------------------------------------------------------- +/* Compare to make sure we are matching versions */ +APT_PURE bool pkgFLCache::Header::CheckSizes(Header &Against) const +{ + if (HeaderSz == Against.HeaderSz && + NodeSz == Against.NodeSz && + DirSz == Against.DirSz && + DiversionSz == Against.DiversionSz && + PackageSz == Against.PackageSz && + ConfFileSz == Against.ConfFileSz) + return true; + return false; +} + /*}}}*/ + +// FLCache::pkgFLCache - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* If this is a new cache then a new header and hash table are instantaited + otherwise the existing ones are mearly attached */ +pkgFLCache::pkgFLCache(DynamicMMap &Map) : Map(Map) +{ + if (_error->PendingError() == true) + return; + + LastTreeLookup = 0; + LastLookupSize = 0; + + // Apply the typecasts + HeaderP = (Header *)Map.Data(); + NodeP = (Node *)Map.Data(); + DirP = (Directory *)Map.Data(); + DiverP = (Diversion *)Map.Data(); + PkgP = (Package *)Map.Data(); + ConfP = (ConfFile *)Map.Data(); + StrP = (char *)Map.Data(); + AnyP = (unsigned char *)Map.Data(); + + // New mapping, create the basic cache structures + if (Map.Size() == 0) + { + Map.RawAllocate(sizeof(pkgFLCache::Header)); + *HeaderP = pkgFLCache::Header(); + HeaderP->FileHash = Map.RawAllocate(sizeof(pkgFLCache::Node)*HeaderP->HashSize, + sizeof(pkgFLCache::Node))/sizeof(pkgFLCache::Node); + } + + FileHash = NodeP + HeaderP->FileHash; + + // Setup the dynamic map manager + HeaderP->Dirty = true; + Map.Sync(0,sizeof(pkgFLCache::Header)); + Map.UsePools(*HeaderP->Pools,sizeof(HeaderP->Pools)/sizeof(HeaderP->Pools[0])); +} + /*}}}*/ +// FLCache::TreeLookup - Perform a lookup in a generic tree /*{{{*/ +// --------------------------------------------------------------------- +/* This is a simple generic tree lookup. The first three entries of + the Directory structure are used as a template, but any other similar + structure could be used in it's place. */ +map_ptrloc pkgFLCache::TreeLookup(map_ptrloc *Base,const char *Text, + const char *TextEnd,unsigned long Size, + unsigned int *Count,bool Insert) +{ + pkgFLCache::Directory *Dir; + + // Check our last entry cache + if (LastTreeLookup != 0 && LastLookupSize == Size) + { + Dir = (pkgFLCache::Directory *)(AnyP + LastTreeLookup*Size); + if (stringcmp(Text,TextEnd,StrP + Dir->Name) == 0) + return LastTreeLookup; + } + + while (1) + { + // Allocate a new one + if (*Base == 0) + { + if (Insert == false) + return 0; + + *Base = Map.Allocate(Size); + if (*Base == 0) + return 0; + + (*Count)++; + Dir = (pkgFLCache::Directory *)(AnyP + *Base*Size); + Dir->Name = Map.WriteString(Text,TextEnd - Text); + LastTreeLookup = *Base; + LastLookupSize = Size; + return *Base; + } + + // Compare this node + Dir = (pkgFLCache::Directory *)(AnyP + *Base*Size); + int Res = stringcmp(Text,TextEnd,StrP + Dir->Name); + if (Res == 0) + { + LastTreeLookup = *Base; + LastLookupSize = Size; + return *Base; + } + + if (Res > 0) + Base = &Dir->Left; + if (Res < 0) + Base = &Dir->Right; + } +} + /*}}}*/ +// FLCache::PrintTree - Print out a tree /*{{{*/ +// --------------------------------------------------------------------- +/* This is a simple generic tree dumper, meant for debugging. */ +void pkgFLCache::PrintTree(map_ptrloc Base,unsigned long Size) +{ + if (Base == 0) + return; + + pkgFLCache::Directory *Dir = (pkgFLCache::Directory *)(AnyP + Base*Size); + PrintTree(Dir->Left,Size); + cout << (StrP + Dir->Name) << endl; + PrintTree(Dir->Right,Size); +} + /*}}}*/ +// FLCache::GetPkg - Get a package pointer /*{{{*/ +// --------------------------------------------------------------------- +/* Locate a package by name in it's tree, this is just a wrapper for + TreeLookup */ +pkgFLCache::PkgIterator pkgFLCache::GetPkg(const char *Name,const char *NameEnd, + bool Insert) +{ + if (NameEnd == 0) + NameEnd = Name + strlen(Name); + + map_ptrloc Pos = TreeLookup(&HeaderP->Packages,Name,NameEnd, + sizeof(pkgFLCache::Package), + &HeaderP->PackageCount,Insert); + if (Pos == 0) + return pkgFLCache::PkgIterator(); + return pkgFLCache::PkgIterator(*this,PkgP + Pos); +} + /*}}}*/ +// FLCache::GetNode - Get the node associated with the filename /*{{{*/ +// --------------------------------------------------------------------- +/* Lookup a node in the hash table. If Insert is true then a new node is + always inserted. The hash table can have multiple instances of a + single name available. A search returns the first. It is important + that additions for the same name insert after the first entry of + the name group. */ +pkgFLCache::NodeIterator pkgFLCache::GetNode(const char *Name, + const char *NameEnd, + map_ptrloc Loc, + bool Insert,bool Divert) +{ + // Split the name into file and directory, hashing as it is copied + const char *File = Name; + unsigned long HashPos = 0; + for (const char *I = Name; I < NameEnd; I++) + { + HashPos = 1637*HashPos + *I; + if (*I == '/') + File = I; + } + + // Search for it + Node *Hash = NodeP + HeaderP->FileHash + (HashPos % HeaderP->HashSize); + int Res = 0; + map_ptrloc FilePtr = 0; + while (Hash->Pointer != 0) + { + // Compare + Res = stringcmp(File+1,NameEnd,StrP + Hash->File); + if (Res == 0) + Res = stringcmp(Name,File,StrP + DirP[Hash->Dir].Name); + + // Diversion? + if (Res == 0 && Insert == true) + { + /* Dir and File match exactly, we need to reuse the file name + when we link it in */ + FilePtr = Hash->File; + Res = Divert - ((Hash->Flags & Node::Diversion) == Node::Diversion); + } + + // Is a match + if (Res == 0) + { + if (Insert == false) + return NodeIterator(*this,Hash); + + // Only one diversion per name! + if (Divert == true) + return NodeIterator(*this,Hash); + break; + } + + // Out of sort order + if (Res > 0) + break; + + if (Hash->Next != 0) + Hash = NodeP + Hash->Next; + else + break; + } + + // Fail, not found + if (Insert == false) + return NodeIterator(*this); + + // Find a directory node + map_ptrloc Dir = TreeLookup(&HeaderP->DirTree,Name,File, + sizeof(pkgFLCache::Directory), + &HeaderP->DirCount,true); + if (Dir == 0) + return NodeIterator(*this); + + // Allocate a new node + if (Hash->Pointer != 0) + { + // Overwrite or append + if (Res > 0) + { + Node *Next = NodeP + Map.Allocate(sizeof(*Hash)); + if (Next == NodeP) + return NodeIterator(*this); + *Next = *Hash; + Hash->Next = Next - NodeP; + } + else + { + unsigned long NewNext = Map.Allocate(sizeof(*Hash)); + if (NewNext == 0) + return NodeIterator(*this); + NodeP[NewNext].Next = Hash->Next; + Hash->Next = NewNext; + Hash = NodeP + Hash->Next; + } + } + + // Insert into the new item + Hash->Dir = Dir; + Hash->Pointer = Loc; + Hash->Flags = 0; + if (Divert == true) + Hash->Flags |= Node::Diversion; + + if (FilePtr != 0) + Hash->File = FilePtr; + else + { + HeaderP->UniqNodes++; + Hash->File = Map.WriteString(File+1,NameEnd - File-1); + } + + // Link the node to the package list + if (Divert == false && Loc == 0) + { + Hash->Next = PkgP[Loc].Files; + PkgP[Loc].Files = Hash - NodeP; + } + + HeaderP->NodeCount++; + return NodeIterator(*this,Hash); +} + /*}}}*/ +// FLCache::HashNode - Return the hash bucket for the node /*{{{*/ +// --------------------------------------------------------------------- +/* This is one of two hashing functions. The other is inlined into the + GetNode routine. */ +APT_PURE pkgFLCache::Node *pkgFLCache::HashNode(NodeIterator const &Nde) +{ + // Hash the node + unsigned long HashPos = 0; + for (const char *I = Nde.DirN(); *I != 0; I++) + HashPos = 1637*HashPos + *I; + HashPos = 1637*HashPos + '/'; + for (const char *I = Nde.File(); *I != 0; I++) + HashPos = 1637*HashPos + *I; + return NodeP + HeaderP->FileHash + (HashPos % HeaderP->HashSize); +} + /*}}}*/ +// FLCache::DropNode - Drop a node from the hash table /*{{{*/ +// --------------------------------------------------------------------- +/* This erases a node from the hash table. Note that this does not unlink + the node from the package linked list. */ +void pkgFLCache::DropNode(map_ptrloc N) +{ + if (N == 0) + return; + + NodeIterator Nde(*this,NodeP + N); + + if (Nde->NextPkg != 0) + _error->Warning(_("DropNode called on still linked node")); + + // Locate it in the hash table + Node *Last = 0; + Node *Hash = HashNode(Nde); + while (Hash->Pointer != 0) + { + // Got it + if (Hash == Nde) + { + // Top of the bucket.. + if (Last == 0) + { + Hash->Pointer = 0; + if (Hash->Next == 0) + return; + *Hash = NodeP[Hash->Next]; + // Release Hash->Next + return; + } + Last->Next = Hash->Next; + // Release Hash + return; + } + + Last = Hash; + if (Hash->Next != 0) + Hash = NodeP + Hash->Next; + else + break; + } + + _error->Error(_("Failed to locate the hash element!")); +} + /*}}}*/ +// FLCache::BeginDiverLoad - Start reading new diversions /*{{{*/ +// --------------------------------------------------------------------- +/* Tag all the diversions as untouched */ +void pkgFLCache::BeginDiverLoad() +{ + for (DiverIterator I = DiverBegin(); I.end() == false; I++) + I->Flags = 0; +} + /*}}}*/ +// FLCache::FinishDiverLoad - Finish up a new diversion load /*{{{*/ +// --------------------------------------------------------------------- +/* This drops any untouched diversions. In effect removing any diversions + that where not loaded (ie missing from the diversion file) */ +void pkgFLCache::FinishDiverLoad() +{ + map_ptrloc *Cur = &HeaderP->Diversions; + while (*Cur != 0) + { + Diversion *Div = DiverP + *Cur; + if ((Div->Flags & Diversion::Touched) == Diversion::Touched) + { + Cur = &Div->Next; + continue; + } + + // Purge! + DropNode(Div->DivertTo); + DropNode(Div->DivertFrom); + *Cur = Div->Next; + } +} + /*}}}*/ +// FLCache::AddDiversion - Add a new diversion /*{{{*/ +// --------------------------------------------------------------------- +/* Add a new diversion to the diverion tables and make sure that it is + unique and non-chaining. */ +bool pkgFLCache::AddDiversion(PkgIterator const &Owner, + const char *From,const char *To) +{ + /* Locate the two hash nodes we are going to manipulate. If there + are pre-existing diversions then they will be returned */ + NodeIterator FromN = GetNode(From,From+strlen(From),0,true,true); + NodeIterator ToN = GetNode(To,To+strlen(To),0,true,true); + if (FromN.end() == true || ToN.end() == true) + return _error->Error(_("Failed to allocate diversion")); + + // Should never happen + if ((FromN->Flags & Node::Diversion) != Node::Diversion || + (ToN->Flags & Node::Diversion) != Node::Diversion) + return _error->Error(_("Internal error in AddDiversion")); + + // Now, try to reclaim an existing diversion.. + map_ptrloc Diver = 0; + if (FromN->Pointer != 0) + Diver = FromN->Pointer; + + /* Make sure from and to point to the same diversion, if they don't + then we are trying to intermix diversions - very bad */ + if (ToN->Pointer != 0 && ToN->Pointer != Diver) + { + // It could be that the other diversion is no longer in use + if ((DiverP[ToN->Pointer].Flags & Diversion::Touched) == Diversion::Touched) + return _error->Error(_("Trying to overwrite a diversion, %s -> %s and %s/%s"), + From,To,ToN.File(),ToN.Dir().Name()); + + // We can erase it. + Diversion *Div = DiverP + ToN->Pointer; + ToN->Pointer = 0; + + if (Div->DivertTo == ToN.Offset()) + Div->DivertTo = 0; + if (Div->DivertFrom == ToN.Offset()) + Div->DivertFrom = 0; + + // This diversion will be cleaned up by FinishDiverLoad + } + + // Allocate a new diversion + if (Diver == 0) + { + Diver = Map.Allocate(sizeof(Diversion)); + if (Diver == 0) + return false; + DiverP[Diver].Next = HeaderP->Diversions; + HeaderP->Diversions = Diver; + HeaderP->DiversionCount++; + } + + // Can only have one diversion of the same files + Diversion *Div = DiverP + Diver; + if ((Div->Flags & Diversion::Touched) == Diversion::Touched) + return _error->Error(_("Double add of diversion %s -> %s"),From,To); + + // Setup the From/To links + if (Div->DivertFrom != FromN.Offset() && Div->DivertFrom != ToN.Offset()) + DropNode(Div->DivertFrom); + Div->DivertFrom = FromN.Offset(); + if (Div->DivertTo != FromN.Offset() && Div->DivertTo != ToN.Offset()) + DropNode(Div->DivertTo); + Div->DivertTo = ToN.Offset(); + + // Link it to the two nodes + FromN->Pointer = Diver; + ToN->Pointer = Diver; + + // And the package + Div->OwnerPkg = Owner.Offset(); + Div->Flags |= Diversion::Touched; + + return true; +} + /*}}}*/ +// FLCache::AddConfFile - Add a new configuration file /*{{{*/ +// --------------------------------------------------------------------- +/* This simply adds a new conf file node to the hash table. This is only + used by the status file reader. It associates a hash with each conf + file entry that exists in the status file and the list file for + the proper package. Duplicate conf files (across packages) are left + up to other routines to deal with. */ +bool pkgFLCache::AddConfFile(const char *Name,const char *NameEnd, + PkgIterator const &Owner, + const unsigned char *Sum) +{ + NodeIterator Nde = GetNode(Name,NameEnd,0,false,false); + if (Nde.end() == true) + return true; + + unsigned long File = Nde->File; + for (; Nde->File == File && Nde.end() == false; Nde++) + { + if (Nde.RealPackage() != Owner) + continue; + + if ((Nde->Flags & Node::ConfFile) == Node::ConfFile) + return _error->Error(_("Duplicate conf file %s/%s"),Nde.DirN(),Nde.File()); + + // Allocate a new conf file structure + map_ptrloc Conf = Map.Allocate(sizeof(ConfFile)); + if (Conf == 0) + return false; + ConfP[Conf].OwnerPkg = Owner.Offset(); + memcpy(ConfP[Conf].MD5,Sum,sizeof(ConfP[Conf].MD5)); + + Nde->Pointer = Conf; + Nde->Flags |= Node::ConfFile; + return true; + } + + /* This means the conf file has been replaced, but the entry in the + status file was not updated */ + return true; +} + /*}}}*/ + +// NodeIterator::RealPackage - Return the package for this node /*{{{*/ +// --------------------------------------------------------------------- +/* Since the package pointer is indirected in all sorts of interesting ways + this is used to get a pointer to the owning package */ +APT_PURE pkgFLCache::Package *pkgFLCache::NodeIterator::RealPackage() const +{ + if (Nde->Pointer == 0) + return 0; + + if ((Nde->Flags & Node::ConfFile) == Node::ConfFile) + return Owner->PkgP + Owner->ConfP[Nde->Pointer].OwnerPkg; + + // Diversions are ignored + if ((Nde->Flags & Node::Diversion) == Node::Diversion) + return 0; + + return Owner->PkgP + Nde->Pointer; +} + /*}}}*/ diff --git a/apt-inst/filelist.h b/apt-inst/filelist.h new file mode 100644 index 0000000..7fe43de --- /dev/null +++ b/apt-inst/filelist.h @@ -0,0 +1,312 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + File Listing - Manages a Cache of File -> Package names. + + This is identical to the Package cache, except that the generator + (which is much simpler) is integrated directly into the main class, + and it has been designed to handle live updates. + + The storage content of the class is maintained in a memory map and is + written directly to the file system. Performance is traded against + space to give something that performs well and remains small. + The average per file usage is 32 bytes which yields about a meg every + 36k files. Directory paths are collected into a binary tree and stored + only once, this offsets the cost of the hash nodes enough to keep + memory usage slightly less than the sum of the filenames. + + The file names are stored into a fixed size chained hash table that is + linked to the package name and to the directory component. + + Each file node has a set of associated flags that indicate the current + state of the file. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_FILELIST_H +#define PKGLIB_FILELIST_H + +#include <apt-pkg/mmap.h> + +#include <cstring> +#include <string> + +class pkgFLCache +{ + public: + struct Header; + struct Node; + struct Directory; + struct Package; + struct Diversion; + struct ConfFile; + + class NodeIterator; + class DirIterator; + class PkgIterator; + class DiverIterator; + + protected: + std::string CacheFile; + DynamicMMap ⤅ + map_ptrloc LastTreeLookup; + unsigned long LastLookupSize; + + // Helpers for the addition algorithms + map_ptrloc TreeLookup(map_ptrloc *Base,const char *Text,const char *TextEnd, + unsigned long Size,unsigned int *Count = 0, + bool Insert = false); + + public: + + // Pointers to the arrays of items + Header *HeaderP; + Node *NodeP; + Directory *DirP; + Package *PkgP; + Diversion *DiverP; + ConfFile *ConfP; + char *StrP; + unsigned char *AnyP; + + // Quick accessors + Node *FileHash; + + // Accessors + Header &Head() {return *HeaderP;}; + void PrintTree(map_ptrloc Base,unsigned long Size); + + // Add/Find things + PkgIterator GetPkg(const char *Name,const char *End,bool Insert); + inline PkgIterator GetPkg(const char *Name,bool Insert); + NodeIterator GetNode(const char *Name, + const char *NameEnd, + map_ptrloc Loc, + bool Insert,bool Divert); + Node *HashNode(NodeIterator const &N); + void DropNode(map_ptrloc Node); + + inline DiverIterator DiverBegin(); + + // Diversion control + void BeginDiverLoad(); + void FinishDiverLoad(); + bool AddDiversion(PkgIterator const &Owner,const char *From, + const char *To); + bool AddConfFile(const char *Name,const char *NameEnd, + PkgIterator const &Owner,const unsigned char *Sum); + + pkgFLCache(DynamicMMap &Map); +// ~pkgFLCache(); +}; + +struct pkgFLCache::Header +{ + // Signature information + unsigned long Signature; + short MajorVersion; + short MinorVersion; + bool Dirty; + + // Size of structure values + unsigned HeaderSz; + unsigned NodeSz; + unsigned DirSz; + unsigned PackageSz; + unsigned DiversionSz; + unsigned ConfFileSz; + + // Structure Counts; + unsigned int NodeCount; + unsigned int DirCount; + unsigned int PackageCount; + unsigned int DiversionCount; + unsigned int ConfFileCount; + unsigned int HashSize; + unsigned long UniqNodes; + + // Offsets + map_ptrloc FileHash; + map_ptrloc DirTree; + map_ptrloc Packages; + map_ptrloc Diversions; + + /* Allocation pools, there should be one of these for each structure + excluding the header */ + DynamicMMap::Pool Pools[5]; + + bool CheckSizes(Header &Against) const; + Header(); +}; + +/* The bit field is used to advoid incurring an extra 4 bytes x 40000, + Pointer is the most infrequently used member of the structure */ +struct pkgFLCache::Node +{ + map_ptrloc Dir; // Dir + map_ptrloc File; // String + unsigned Pointer:24; // Package/Diversion/ConfFile + unsigned Flags:8; // Package + map_ptrloc Next; // Node + map_ptrloc NextPkg; // Node + + enum Flags {Diversion = (1<<0),ConfFile = (1<<1), + NewConfFile = (1<<2),NewFile = (1<<3), + Unpacked = (1<<4),Replaced = (1<<5)}; +}; + +struct pkgFLCache::Directory +{ + map_ptrloc Left; // Directory + map_ptrloc Right; // Directory + map_ptrloc Name; // String +}; + +struct pkgFLCache::Package +{ + map_ptrloc Left; // Package + map_ptrloc Right; // Package + map_ptrloc Name; // String + map_ptrloc Files; // Node +}; + +struct pkgFLCache::Diversion +{ + map_ptrloc OwnerPkg; // Package + map_ptrloc DivertFrom; // Node + map_ptrloc DivertTo; // String + + map_ptrloc Next; // Diversion + unsigned long Flags; + + enum Flags {Touched = (1<<0)}; +}; + +struct pkgFLCache::ConfFile +{ + map_ptrloc OwnerPkg; // Package + unsigned char MD5[16]; +}; + +class pkgFLCache::PkgIterator +{ + Package *Pkg; + pkgFLCache *Owner; + + public: + + inline bool end() const {return Owner == 0 || Pkg == Owner->PkgP?true:false;} + + // Accessors + inline Package *operator ->() {return Pkg;} + inline Package const *operator ->() const {return Pkg;} + inline Package const &operator *() const {return *Pkg;} + inline operator Package *() {return Pkg == Owner->PkgP?0:Pkg;} + inline operator Package const *() const {return Pkg == Owner->PkgP?0:Pkg;} + + inline unsigned long Offset() const {return Pkg - Owner->PkgP;} + inline const char *Name() const {return Pkg->Name == 0?0:Owner->StrP + Pkg->Name;} + inline pkgFLCache::NodeIterator Files() const; + + PkgIterator() : Pkg(0), Owner(0) {} + PkgIterator(pkgFLCache &Owner,Package *Trg) : Pkg(Trg), Owner(&Owner) {} +}; + +class pkgFLCache::DirIterator +{ + Directory *Dir; + pkgFLCache *Owner; + + public: + + // Accessors + inline Directory *operator ->() {return Dir;} + inline Directory const *operator ->() const {return Dir;} + inline Directory const &operator *() const {return *Dir;} + inline operator Directory *() {return Dir == Owner->DirP?0:Dir;} + inline operator Directory const *() const {return Dir == Owner->DirP?0:Dir;} + + inline const char *Name() const {return Dir->Name == 0?0:Owner->StrP + Dir->Name;} + + DirIterator() : Dir(0), Owner(0) {} + DirIterator(pkgFLCache &Owner,Directory *Trg) : Dir(Trg), Owner(&Owner) {} +}; + +class pkgFLCache::DiverIterator +{ + Diversion *Diver; + pkgFLCache *Owner; + + public: + + // Iteration + void operator ++(int) {if (Diver != Owner->DiverP) Diver = Owner->DiverP + Diver->Next;} + inline void operator ++() {operator ++(0);} + inline bool end() const {return Owner == 0 || Diver == Owner->DiverP;} + + // Accessors + inline Diversion *operator ->() {return Diver;} + inline Diversion const *operator ->() const {return Diver;} + inline Diversion const &operator *() const {return *Diver;} + inline operator Diversion *() {return Diver == Owner->DiverP?0:Diver;} + inline operator Diversion const *() const {return Diver == Owner->DiverP?0:Diver;} + + inline PkgIterator OwnerPkg() const {return PkgIterator(*Owner,Owner->PkgP + Diver->OwnerPkg);} + inline NodeIterator DivertFrom() const; + inline NodeIterator DivertTo() const; + + DiverIterator() : Diver(0), Owner(0) {}; + DiverIterator(pkgFLCache &Owner,Diversion *Trg) : Diver(Trg), Owner(&Owner) {} +}; + +class pkgFLCache::NodeIterator +{ + Node *Nde; + enum {NdePkg, NdeHash} Type; + pkgFLCache *Owner; + + public: + + // Iteration + void operator ++(int) {if (Nde != Owner->NodeP) Nde = Owner->NodeP + + (Type == NdePkg?Nde->NextPkg:Nde->Next);} + inline void operator ++() {operator ++(0);} + inline bool end() const {return Owner == 0 || Nde == Owner->NodeP;} + + // Accessors + inline Node *operator ->() {return Nde;} + inline Node const *operator ->() const {return Nde;} + inline Node const &operator *() const {return *Nde;} + inline operator Node *() {return Nde == Owner->NodeP?0:Nde;} + inline operator Node const *() const {return Nde == Owner->NodeP?0:Nde;} + inline unsigned long Offset() const {return Nde - Owner->NodeP;} + inline DirIterator Dir() const {return DirIterator(*Owner,Owner->DirP + Nde->Dir);} + inline DiverIterator Diversion() const {return DiverIterator(*Owner,Owner->DiverP + Nde->Pointer);} + inline const char *File() const {return Nde->File == 0?0:Owner->StrP + Nde->File;} + inline const char *DirN() const {return Owner->StrP + Owner->DirP[Nde->Dir].Name;} + Package *RealPackage() const; + + NodeIterator() : Nde(0), Type(NdeHash), Owner(0) {}; + NodeIterator(pkgFLCache &Owner) : Nde(Owner.NodeP), Type(NdeHash), Owner(&Owner) {} + NodeIterator(pkgFLCache &Owner,Node *Trg) : Nde(Trg), Type(NdeHash), Owner(&Owner) {} + NodeIterator(pkgFLCache &Owner,Node *Trg,Package *) : Nde(Trg), Type(NdePkg), Owner(&Owner) {} +}; + +/* Inlines with forward references that cannot be included directly in their + respsective classes */ +inline pkgFLCache::NodeIterator pkgFLCache::DiverIterator::DivertFrom() const + {return NodeIterator(*Owner,Owner->NodeP + Diver->DivertFrom);} +inline pkgFLCache::NodeIterator pkgFLCache::DiverIterator::DivertTo() const + {return NodeIterator(*Owner,Owner->NodeP + Diver->DivertTo);} + +inline pkgFLCache::NodeIterator pkgFLCache::PkgIterator::Files() const + {return NodeIterator(*Owner,Owner->NodeP + Pkg->Files,Pkg);} + +inline pkgFLCache::DiverIterator pkgFLCache::DiverBegin() + {return DiverIterator(*this,DiverP + HeaderP->Diversions);} + +inline pkgFLCache::PkgIterator pkgFLCache::GetPkg(const char *Name,bool Insert) + {return GetPkg(Name,Name+strlen(Name),Insert);} + +#endif diff --git a/apt-pkg/CMakeLists.txt b/apt-pkg/CMakeLists.txt new file mode 100644 index 0000000..ce73c6a --- /dev/null +++ b/apt-pkg/CMakeLists.txt @@ -0,0 +1,76 @@ +# Include apt-pkg directly, as some files have #include <system.h> +include_directories(${PROJECT_BINARY_DIR}/include/apt-pkg) + +add_definitions("-DAPT_PKG_EXPOSE_STRING_VIEW") + +file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/include/apt-pkg/) +execute_process(COMMAND ${PERL_EXECUTABLE} ${PROJECT_SOURCE_DIR}/triehash/triehash.pl + --ignore-case + --header ${PROJECT_BINARY_DIR}/include/apt-pkg/tagfile-keys.h + --code ${CMAKE_CURRENT_BINARY_DIR}/tagfile-keys.cc + --enum-class + --enum-name pkgTagSection::Key + --function-name pkgTagHash + --include "<apt-pkg/tagfile.h>" + ${CMAKE_CURRENT_SOURCE_DIR}/tagfile-keys.list) +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "tagfile-keys.list") + + +# Set the version of the library +execute_process(COMMAND awk -v ORS=. "/^\#define APT_PKG_M/ {print \$3}" + COMMAND sed "s/\\.\$//" + INPUT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/contrib/macros.h + OUTPUT_VARIABLE MAJOR OUTPUT_STRIP_TRAILING_WHITESPACE) +execute_process(COMMAND grep "^#define APT_PKG_RELEASE" + COMMAND cut -d " " -f 3 + INPUT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/contrib/macros.h + OUTPUT_VARIABLE MINOR OUTPUT_STRIP_TRAILING_WHITESPACE) + +message(STATUS "Building libapt-pkg ${MAJOR} (release ${MINOR})") +set(APT_PKG_MAJOR ${MAJOR} PARENT_SCOPE) # exporting for methods/CMakeLists.txt + +# Definition of the C++ files used to build the library - note that this +# is expanded at CMake time, so you have to rerun cmake if you add or remove +# a file (you can just run cmake . in the build directory) +file(GLOB_RECURSE library "*.cc" "${CMAKE_CURRENT_BINARY_DIR}/tagfile-keys.cc") +file(GLOB_RECURSE headers "*.h") + +# Create a library using the C++ files +add_library(apt-pkg SHARED ${library}) +add_dependencies(apt-pkg apt-pkg-versionscript) +# Link the library and set the SONAME +target_include_directories(apt-pkg + PRIVATE ${ZLIB_INCLUDE_DIRS} + ${BZIP2_INCLUDE_DIR} + ${LZMA_INCLUDE_DIRS} + ${LZ4_INCLUDE_DIRS} + $<$<BOOL:${ZSTD_FOUND}>:${ZSTD_INCLUDE_DIRS}> + $<$<BOOL:${UDEV_FOUND}>:${UDEV_INCLUDE_DIRS}> + $<$<BOOL:${SYSTEMD_FOUND}>:${SYSTEMD_INCLUDE_DIRS}> + ${ICONV_INCLUDE_DIRS} +) + +target_link_libraries(apt-pkg + PRIVATE -lutil ${CMAKE_DL_LIBS} ${RESOLV_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} + ${ZLIB_LIBRARIES} + ${BZIP2_LIBRARIES} + ${LZMA_LIBRARIES} + ${LZ4_LIBRARIES} + $<$<BOOL:${ZSTD_FOUND}>:${ZSTD_LIBRARIES}> + $<$<BOOL:${UDEV_FOUND}>:${UDEV_LIBRARIES}> + $<$<BOOL:${SYSTEMD_FOUND}>:${SYSTEMD_LIBRARIES}> + ${ICONV_LIBRARIES} +) +set_target_properties(apt-pkg PROPERTIES VERSION ${MAJOR}.${MINOR}) +set_target_properties(apt-pkg PROPERTIES SOVERSION ${MAJOR}) +add_version_script(apt-pkg) + +# Install the library and the header files +install(TARGETS apt-pkg LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) +install(FILES ${headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/apt-pkg) +flatify(${PROJECT_BINARY_DIR}/include/apt-pkg/ "${headers}") + +if(CMAKE_BUILD_TYPE STREQUAL "Coverage") + target_link_libraries(apt-pkg PUBLIC noprofile) +endif() diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc new file mode 100644 index 0000000..606c949 --- /dev/null +++ b/apt-pkg/acquire-item.cc @@ -0,0 +1,4078 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire Item - Item to acquire + + Each item can download to exactly one file at a time. This means you + cannot create an item that fetches two uri's to two files at the same + time. The pkgAcqIndex class creates a second class upon instantiation + to fetch the other index files because of this. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/acquire-item.h> +#include <apt-pkg/acquire-worker.h> +#include <apt-pkg/acquire.h> +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/gpgv.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/indexfile.h> +#include <apt-pkg/metaindex.h> +#include <apt-pkg/netrc.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/pkgrecords.h> +#include <apt-pkg/sourcelist.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile.h> + +#include <algorithm> +#include <ctime> +#include <chrono> +#include <iostream> +#include <memory> +#include <numeric> +#include <random> +#include <sstream> +#include <string> +#include <unordered_set> +#include <vector> +#include <errno.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +static std::string GetPartialFileName(std::string const &file) /*{{{*/ +{ + std::string DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += file; + return DestFile; +} + /*}}}*/ +static std::string GetPartialFileNameFromURI(std::string const &uri) /*{{{*/ +{ + return GetPartialFileName(URItoFileName(uri)); +} + /*}}}*/ +static std::string GetFinalFileNameFromURI(std::string const &uri) /*{{{*/ +{ + return _config->FindDir("Dir::State::lists") + URItoFileName(uri); +} + /*}}}*/ +static std::string GetKeepCompressedFileName(std::string file, IndexTarget const &Target)/*{{{*/ +{ + if (Target.KeepCompressed == false) + return file; + + std::string const KeepCompressedAs = Target.Option(IndexTarget::KEEPCOMPRESSEDAS); + if (KeepCompressedAs.empty() == false) + { + std::string const ext = KeepCompressedAs.substr(0, KeepCompressedAs.find(' ')); + if (ext != "uncompressed") + file.append(".").append(ext); + } + return file; +} + /*}}}*/ +static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)/*{{{*/ +{ + // rred expects the patch as $FinalFile.ed.$patchname.gz + return Final + ".ed." + Patch + ".gz"; +} + /*}}}*/ +static std::string GetDiffsPatchFileName(std::string const &Final) /*{{{*/ +{ + // rred expects the patch as $FinalFile.ed + return Final + ".ed"; +} + /*}}}*/ +static std::string GetExistingFilename(std::string const &File) /*{{{*/ +{ + if (RealFileExists(File)) + return File; + for (auto const &type : APT::Configuration::getCompressorExtensions()) + { + std::string const Final = File + type; + if (RealFileExists(Final)) + return Final; + } + return ""; +} + /*}}}*/ +static std::string GetDiffIndexFileName(std::string const &Name) /*{{{*/ +{ + return Name + ".diff/Index"; +} + /*}}}*/ +static std::string GetDiffIndexURI(IndexTarget const &Target) /*{{{*/ +{ + return Target.URI + ".diff/Index"; +} + /*}}}*/ + +static void ReportMirrorFailureToCentral(pkgAcquire::Item const &I, std::string const &FailCode, std::string const &Details)/*{{{*/ +{ + // we only act if a mirror was used at all + if(I.UsedMirror.empty()) + return; +#if 0 + std::cerr << "\nReportMirrorFailure: " + << UsedMirror + << " Uri: " << DescURI() + << " FailCode: " + << FailCode << std::endl; +#endif + string const report = _config->Find("Methods::Mirror::ProblemReporting", + LIBEXEC_DIR "/apt-report-mirror-failure"); + if(!FileExists(report)) + return; + + std::vector<char const*> const Args = { + report.c_str(), + I.UsedMirror.c_str(), + I.DescURI().c_str(), + FailCode.c_str(), + Details.c_str(), + NULL + }; + + pid_t pid = ExecFork(); + if(pid < 0) + { + _error->Error("ReportMirrorFailure Fork failed"); + return; + } + else if(pid == 0) + { + execvp(Args[0], (char**)Args.data()); + std::cerr << "Could not exec " << Args[0] << std::endl; + _exit(100); + } + if(!ExecWait(pid, "report-mirror-failure")) + _error->Warning("Couldn't report problem to '%s'", report.c_str()); +} + /*}}}*/ + +static APT_NONNULL(2) bool MessageInsecureRepository(bool const isError, char const * const msg, std::string const &repo)/*{{{*/ +{ + std::string m; + strprintf(m, msg, repo.c_str()); + if (isError) + { + _error->Error("%s", m.c_str()); + _error->Notice("%s", _("Updating from such a repository can't be done securely, and is therefore disabled by default.")); + } + else + { + _error->Warning("%s", m.c_str()); + _error->Notice("%s", _("Data from such a repository can't be authenticated and is therefore potentially dangerous to use.")); + } + _error->Notice("%s", _("See apt-secure(8) manpage for repository creation and user configuration details.")); + return false; +} + /*}}}*/ +// AllowInsecureRepositories /*{{{*/ +enum class InsecureType { UNSIGNED, WEAK, NORELEASE }; +static bool TargetIsAllowedToBe(IndexTarget const &Target, InsecureType const type) +{ + if (_config->FindB("Acquire::AllowInsecureRepositories")) + return true; + + if (Target.OptionBool(IndexTarget::ALLOW_INSECURE)) + return true; + + switch (type) + { + case InsecureType::UNSIGNED: break; + case InsecureType::NORELEASE: break; + case InsecureType::WEAK: + if (_config->FindB("Acquire::AllowWeakRepositories")) + return true; + if (Target.OptionBool(IndexTarget::ALLOW_WEAK)) + return true; + break; + } + return false; +} +static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType const msg, std::string const &repo, + metaIndex const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I) +{ + // we skip weak downgrades as its unlikely that a repository gets really weaker – + // its more realistic that apt got pickier in a newer version + if (msg != InsecureType::WEAK) + { + std::string const FinalInRelease = TransactionManager->GetFinalFilename(); + std::string const FinalReleasegpg = FinalInRelease.substr(0, FinalInRelease.length() - strlen("InRelease")) + "Release.gpg"; + if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease)) + { + char const * msgstr = nullptr; + switch (msg) + { + case InsecureType::UNSIGNED: msgstr = _("The repository '%s' is no longer signed."); break; + case InsecureType::NORELEASE: msgstr = _("The repository '%s' no longer has a Release file."); break; + case InsecureType::WEAK: /* unreachable */ break; + } + if (_config->FindB("Acquire::AllowDowngradeToInsecureRepositories") || + TransactionManager->Target.OptionBool(IndexTarget::ALLOW_DOWNGRADE_TO_INSECURE)) + { + // meh, the users wants to take risks (we still mark the packages + // from this repository as unauthenticated) + _error->Warning(msgstr, repo.c_str()); + _error->Warning(_("This is normally not allowed, but the option " + "Acquire::AllowDowngradeToInsecureRepositories was " + "given to override it.")); + } else { + MessageInsecureRepository(true, msgstr, repo); + TransactionManager->AbortTransaction(); + I->Status = pkgAcquire::Item::StatError; + return false; + } + } + } + + if(MetaIndexParser->GetTrusted() == metaIndex::TRI_YES) + return true; + + char const * msgstr = nullptr; + switch (msg) + { + case InsecureType::UNSIGNED: msgstr = _("The repository '%s' is not signed."); break; + case InsecureType::NORELEASE: msgstr = _("The repository '%s' does not have a Release file."); break; + case InsecureType::WEAK: msgstr = _("The repository '%s' provides only weak security information."); break; + } + + if (TargetIsAllowedToBe(TransactionManager->Target, msg) == true) + { + MessageInsecureRepository(false, msgstr, repo); + return true; + } + + MessageInsecureRepository(true, msgstr, repo); + TransactionManager->AbortTransaction(); + I->Status = pkgAcquire::Item::StatError; + return false; +} + /*}}}*/ +static HashStringList GetExpectedHashesFromFor(metaIndex * const Parser, std::string const &MetaKey)/*{{{*/ +{ + if (Parser == NULL) + return HashStringList(); + metaIndex::checkSum * const R = Parser->Lookup(MetaKey); + if (R == NULL) + return HashStringList(); + return R->Hashes; +} + /*}}}*/ + +class pkgAcquire::Item::Private /*{{{*/ +{ +public: + struct AlternateURI + { + std::string URI; + std::unordered_map<std::string, std::string> changefields; + AlternateURI(std::string &&u, decltype(changefields) &&cf) : URI(u), changefields(cf) {} + }; + std::list<AlternateURI> AlternativeURIs; + std::vector<std::string> BadAlternativeSites; + std::vector<std::string> PastRedirections; + std::unordered_map<std::string, std::string> CustomFields; + unsigned int Retries; + + Private() : Retries(_config->FindI("Acquire::Retries", 0)) + { + } +}; + /*}}}*/ + +// all ::HashesRequired and ::GetExpectedHashes implementations /*{{{*/ +/* ::GetExpectedHashes is abstract and has to be implemented by all subclasses. + It is best to implement it as broadly as possible, while ::HashesRequired defaults + to true and should be as restrictive as possible for false cases. Note that if + a hash is returned by ::GetExpectedHashes it must match. Only if it doesn't + ::HashesRequired is called to evaluate if its okay to have no hashes. */ +APT_PURE bool pkgAcqTransactionItem::HashesRequired() const +{ + /* signed repositories obviously have a parser and good hashes. + unsigned repositories, too, as even if we can't trust them for security, + we can at least trust them for integrity of the download itself. + Only repositories without a Release file can (obviously) not have + hashes – and they are very uncommon and strongly discouraged */ + if (TransactionManager->MetaIndexParser->GetLoadedSuccessfully() != metaIndex::TRI_YES) + return false; + if (TargetIsAllowedToBe(Target, InsecureType::WEAK)) + { + /* If we allow weak hashes, we check that we have some (weak) and then + declare hashes not needed. That will tip us in the right direction + as if hashes exist, they will be used, even if not required */ + auto const hsl = GetExpectedHashes(); + if (hsl.usable()) + return true; + if (hsl.empty() == false) + return false; + } + return true; +} +HashStringList pkgAcqTransactionItem::GetExpectedHashes() const +{ + return GetExpectedHashesFor(GetMetaKey()); +} + +APT_PURE bool pkgAcqMetaBase::HashesRequired() const +{ + // Release and co have no hashes 'by design'. + return false; +} +HashStringList pkgAcqMetaBase::GetExpectedHashes() const +{ + return HashStringList(); +} + +APT_PURE bool pkgAcqIndexDiffs::HashesRequired() const +{ + /* We can't check hashes of rred result as we don't know what the + hash of the file will be. We just know the hash of the patch(es), + the hash of the file they will apply on and the hash of the resulting + file. */ + if (State == StateFetchDiff) + return true; + return false; +} +HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const +{ + if (State == StateFetchDiff) + return available_patches[0].download_hashes; + return HashStringList(); +} + +APT_PURE bool pkgAcqIndexMergeDiffs::HashesRequired() const +{ + /* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that + we can check the rred result after all patches are applied as + we know the expected result rather than potentially apply more patches */ + if (State == StateFetchDiff) + return true; + return State == StateApplyDiff; +} +HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const +{ + if (State == StateFetchDiff) + return patch.download_hashes; + else if (State == StateApplyDiff) + return GetExpectedHashesFor(Target.MetaKey); + return HashStringList(); +} + +APT_PURE bool pkgAcqArchive::HashesRequired() const +{ + return LocalSource == false; +} +HashStringList pkgAcqArchive::GetExpectedHashes() const +{ + // figured out while parsing the records + return ExpectedHashes; +} + +APT_PURE bool pkgAcqFile::HashesRequired() const +{ + // supplied as parameter at creation time, so the caller decides + return ExpectedHashes.usable(); +} +HashStringList pkgAcqFile::GetExpectedHashes() const +{ + return ExpectedHashes; +} + /*}}}*/ +// Acquire::Item::QueueURI and specialisations from child classes /*{{{*/ +bool pkgAcquire::Item::QueueURI(pkgAcquire::ItemDesc &Item) +{ + Owner->Enqueue(Item); + return true; +} +/* The idea here is that an item isn't queued if it exists on disk and the + transition manager was a hit as this means that the files it contains + the checksums for can't be updated either (or they are and we are asking + for a hashsum mismatch to happen which helps nobody) */ +bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item) +{ + if (TransactionManager->State != TransactionStarted) + { + if (_config->FindB("Debug::Acquire::Transaction", false)) + std::clog << "Skip " << Target.URI << " as transaction was already dealt with!" << std::endl; + return false; + } + std::string const FinalFile = GetFinalFilename(); + if (TransactionManager->IMSHit == true && FileExists(FinalFile) == true) + { + PartialFile = DestFile = FinalFile; + Status = StatDone; + return false; + } + // this ensures we rewrite only once and only the first step + auto const OldBaseURI = Target.Option(IndexTarget::BASE_URI); + if (OldBaseURI.empty() || APT::String::Startswith(Item.URI, OldBaseURI) == false) + return pkgAcquire::Item::QueueURI(Item); + // the given URI is our last resort + PushAlternativeURI(std::string(Item.URI), {}, false); + // If we got the InRelease file via a mirror, pick all indexes directly from this mirror, too + std::string SameMirrorURI; + if (TransactionManager->BaseURI.empty() == false && TransactionManager->UsedMirror.empty() == false && + URI::SiteOnly(Item.URI) != URI::SiteOnly(TransactionManager->BaseURI)) + { + auto ExtraPath = Item.URI.substr(OldBaseURI.length()); + auto newURI = flCombine(TransactionManager->BaseURI, std::move(ExtraPath)); + if (IsGoodAlternativeURI(newURI)) + { + SameMirrorURI = std::move(newURI); + PushAlternativeURI(std::string(SameMirrorURI), {}, false); + } + } + // add URI and by-hash based on it + if (AcquireByHash()) + { + // if we use the mirror transport, ask it for by-hash uris + // we need to stick to the same mirror only for non-unique filenames + auto const sameMirrorException = [&]() { + if (Item.URI.find("mirror") == std::string::npos) + return false; + ::URI uri(Item.URI); + return uri.Access == "mirror" || APT::String::Startswith(uri.Access, "mirror+") || + APT::String::Endswith(uri.Access, "+mirror") || uri.Access.find("+mirror+") != std::string::npos; + }(); + if (sameMirrorException) + SameMirrorURI.clear(); + // now add the actual by-hash uris + auto const Expected = GetExpectedHashes(); + auto const TargetHash = Expected.find(nullptr); + auto const PushByHashURI = [&](std::string U) { + if (unlikely(TargetHash == nullptr)) + return false; + auto const trailing_slash = U.find_last_of("/"); + if (unlikely(trailing_slash == std::string::npos)) + return false; + auto byhashSuffix = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); + U.replace(trailing_slash, U.length() - trailing_slash, std::move(byhashSuffix)); + PushAlternativeURI(std::move(U), {}, false); + return true; + }; + PushByHashURI(Item.URI); + if (SameMirrorURI.empty() == false && PushByHashURI(SameMirrorURI) == false) + SameMirrorURI.clear(); + } + // the last URI added is the first one tried + if (unlikely(PopAlternativeURI(Item.URI) == false)) + return false; + if (SameMirrorURI.empty() == false) + { + UsedMirror = TransactionManager->UsedMirror; + if (Item.Description.find(" ") != string::npos) + Item.Description.replace(0, Item.Description.find(" "), UsedMirror); + } + return pkgAcquire::Item::QueueURI(Item); +} +/* The transition manager InRelease itself (or its older sisters-in-law + Release & Release.gpg) is always queued as this allows us to rerun gpgv + on it to verify that we aren't stalled with old files */ +bool pkgAcqMetaBase::QueueURI(pkgAcquire::ItemDesc &Item) +{ + return pkgAcquire::Item::QueueURI(Item); +} +/* the Diff/Index needs to queue also the up-to-date complete index file + to ensure that the list cleaner isn't eating it */ +bool pkgAcqDiffIndex::QueueURI(pkgAcquire::ItemDesc &Item) +{ + if (pkgAcqTransactionItem::QueueURI(Item) == true) + return true; + QueueOnIMSHit(); + return false; +} + /*}}}*/ +// Acquire::Item::GetFinalFilename and specialisations for child classes /*{{{*/ +std::string pkgAcquire::Item::GetFinalFilename() const +{ + // Beware: Desc.URI is modified by redirections + return GetFinalFileNameFromURI(Desc.URI); +} +std::string pkgAcqDiffIndex::GetFinalFilename() const +{ + std::string const FinalFile = GetFinalFileNameFromURI(GetDiffIndexURI(Target)); + // we don't want recompress, so lets keep whatever we got + if (CurrentCompressionExtension == "uncompressed") + return FinalFile; + return FinalFile + "." + CurrentCompressionExtension; +} +std::string pkgAcqIndex::GetFinalFilename() const +{ + std::string const FinalFile = GetFinalFileNameFromURI(Target.URI); + return GetKeepCompressedFileName(FinalFile, Target); +} +std::string pkgAcqMetaSig::GetFinalFilename() const +{ + return GetFinalFileNameFromURI(Target.URI); +} +std::string pkgAcqBaseIndex::GetFinalFilename() const +{ + return GetFinalFileNameFromURI(Target.URI); +} +std::string pkgAcqMetaBase::GetFinalFilename() const +{ + return GetFinalFileNameFromURI(Target.URI); +} +std::string pkgAcqArchive::GetFinalFilename() const +{ + return _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename); +} + /*}}}*/ +// pkgAcqTransactionItem::GetMetaKey and specialisations for child classes /*{{{*/ +std::string pkgAcqTransactionItem::GetMetaKey() const +{ + return Target.MetaKey; +} +std::string pkgAcqIndex::GetMetaKey() const +{ + if (Stage == STAGE_DECOMPRESS_AND_VERIFY || CurrentCompressionExtension == "uncompressed") + return Target.MetaKey; + return Target.MetaKey + "." + CurrentCompressionExtension; +} +std::string pkgAcqDiffIndex::GetMetaKey() const +{ + auto const metakey = GetDiffIndexFileName(Target.MetaKey); + if (CurrentCompressionExtension == "uncompressed") + return metakey; + return metakey + "." + CurrentCompressionExtension; +} + /*}}}*/ +//pkgAcqTransactionItem::TransactionState and specialisations for child classes /*{{{*/ +bool pkgAcqTransactionItem::TransactionState(TransactionStates const state) +{ + bool const Debug = _config->FindB("Debug::Acquire::Transaction", false); + switch(state) + { + case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break; + case TransactionAbort: + if(Debug == true) + std::clog << " Cancel: " << DestFile << std::endl; + if (Status == pkgAcquire::Item::StatIdle) + { + Status = pkgAcquire::Item::StatDone; + Dequeue(); + } + break; + case TransactionCommit: + if(PartialFile.empty() == false) + { + bool sameFile = (PartialFile == DestFile); + // we use symlinks on IMS-Hit to avoid copies + if (RealFileExists(DestFile)) + { + struct stat Buf; + if (lstat(PartialFile.c_str(), &Buf) != -1) + { + if (S_ISLNK(Buf.st_mode) && Buf.st_size > 0) + { + char partial[Buf.st_size + 1]; + ssize_t const sp = readlink(PartialFile.c_str(), partial, Buf.st_size); + if (sp == -1) + _error->Errno("pkgAcqTransactionItem::TransactionState-sp", _("Failed to readlink %s"), PartialFile.c_str()); + else + { + partial[sp] = '\0'; + sameFile = (DestFile == partial); + } + } + } + else + _error->Errno("pkgAcqTransactionItem::TransactionState-stat", _("Failed to stat %s"), PartialFile.c_str()); + } + if (sameFile == false) + { + // ensure that even without lists-cleanup all compressions are nuked + std::string FinalFile = GetFinalFileNameFromURI(Target.URI); + if (FileExists(FinalFile)) + { + if(Debug == true) + std::clog << "rm " << FinalFile << " # " << DescURI() << std::endl; + if (RemoveFile("TransactionStates-Cleanup", FinalFile) == false) + return false; + } + for (auto const &ext: APT::Configuration::getCompressorExtensions()) + { + auto const Final = FinalFile + ext; + if (FileExists(Final)) + { + if(Debug == true) + std::clog << "rm " << Final << " # " << DescURI() << std::endl; + if (RemoveFile("TransactionStates-Cleanup", Final) == false) + return false; + } + } + if(Debug == true) + std::clog << "mv " << PartialFile << " -> "<< DestFile << " # " << DescURI() << std::endl; + if (Rename(PartialFile, DestFile) == false) + return false; + } + else if(Debug == true) + std::clog << "keep " << PartialFile << " # " << DescURI() << std::endl; + + } else { + if(Debug == true) + std::clog << "rm " << DestFile << " # " << DescURI() << std::endl; + if (RemoveFile("TransItem::TransactionCommit", DestFile) == false) + return false; + } + break; + } + return true; +} +bool pkgAcqMetaBase::TransactionState(TransactionStates const state) +{ + // Do not remove InRelease on IMSHit of Release.gpg [yes, this is very edgecasey] + if (TransactionManager->IMSHit == false) + return pkgAcqTransactionItem::TransactionState(state); + return true; +} +bool pkgAcqIndex::TransactionState(TransactionStates const state) +{ + if (pkgAcqTransactionItem::TransactionState(state) == false) + return false; + + switch (state) + { + case TransactionStarted: _error->Fatal("AcqIndex %s changed to invalid transaction start state!", Target.URI.c_str()); break; + case TransactionAbort: + if (Stage == STAGE_DECOMPRESS_AND_VERIFY) + { + // keep the compressed file, but drop the decompressed + EraseFileName.clear(); + if (PartialFile.empty() == false && flExtension(PartialFile) != CurrentCompressionExtension) + RemoveFile("TransactionAbort", PartialFile); + } + break; + case TransactionCommit: + if (EraseFileName.empty() == false) + RemoveFile("AcqIndex::TransactionCommit", EraseFileName); + break; + } + return true; +} +bool pkgAcqDiffIndex::TransactionState(TransactionStates const state) +{ + if (pkgAcqTransactionItem::TransactionState(state) == false) + return false; + + switch (state) + { + case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break; + case TransactionCommit: + break; + case TransactionAbort: + std::string const Partial = GetPartialFileNameFromURI(Target.URI); + RemoveFile("TransactionAbort", Partial); + break; + } + + return true; +} + /*}}}*/ +// pkgAcqTransactionItem::AcquireByHash and specialisations for child classes /*{{{*/ +bool pkgAcqTransactionItem::AcquireByHash() const +{ + if (TransactionManager->MetaIndexParser == nullptr) + return false; + auto const useByHashConf = Target.Option(IndexTarget::BY_HASH); + if (useByHashConf == "force") + return true; + return StringToBool(useByHashConf) == true && TransactionManager->MetaIndexParser->GetSupportsAcquireByHash(); +} +// pdiff patches have a unique name already, no need for by-hash +bool pkgAcqIndexMergeDiffs::AcquireByHash() const +{ + return false; +} +bool pkgAcqIndexDiffs::AcquireByHash() const +{ + return false; +} + /*}}}*/ + +class APT_HIDDEN NoActionItem : public pkgAcquire::Item /*{{{*/ +/* The sole purpose of this class is having an item which does nothing to + reach its done state to prevent cleanup deleting the mentioned file. + Handy in cases in which we know we have the file already, like IMS-Hits. */ +{ + IndexTarget const Target; + public: + virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;}; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE {return HashStringList();}; + + NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target) : + pkgAcquire::Item(Owner), Target(Target) + { + Status = StatDone; + DestFile = GetFinalFileNameFromURI(Target.URI); + } + NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target, std::string const &FinalFile) : + pkgAcquire::Item(Owner), Target(Target) + { + Status = StatDone; + DestFile = FinalFile; + } +}; + /*}}}*/ +class APT_HIDDEN CleanupItem : public pkgAcqTransactionItem /*{{{*/ +/* This class ensures that a file which was configured but isn't downloaded + for various reasons isn't kept in an old version in the lists directory. + In a way its the reverse of NoActionItem as it helps with removing files + even if the lists-cleanup is deactivated. */ +{ + public: + virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;}; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE {return HashStringList();}; + + CleanupItem(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target) : + pkgAcqTransactionItem(Owner, TransactionManager, Target) + { + Status = StatDone; + DestFile = GetFinalFileNameFromURI(Target.URI); + } + bool TransactionState(TransactionStates const state) APT_OVERRIDE + { + switch (state) + { + case TransactionStarted: + break; + case TransactionAbort: + break; + case TransactionCommit: + if (_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "rm " << DestFile << " # " << DescURI() << std::endl; + if (RemoveFile("TransItem::TransactionCommit", DestFile) == false) + return false; + break; + } + return true; + } +}; + /*}}}*/ + +// Acquire::Item::Item - Constructor /*{{{*/ +APT_IGNORE_DEPRECATED_PUSH +pkgAcquire::Item::Item(pkgAcquire * const owner) : + FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), Local(false), + QueueCounter(0), ExpectedAdditionalItems(0), Owner(owner), d(new Private()) +{ + Owner->Add(this); + Status = StatIdle; +} +APT_IGNORE_DEPRECATED_POP + /*}}}*/ +// Acquire::Item::~Item - Destructor /*{{{*/ +pkgAcquire::Item::~Item() +{ + Owner->Remove(this); + delete d; +} + /*}}}*/ +std::string pkgAcquire::Item::Custom600Headers() const /*{{{*/ +{ + std::ostringstream header; + for (auto const &f : d->CustomFields) + if (f.second.empty() == false) + header << '\n' + << f.first << ": " << f.second; + return header.str(); +} + /*}}}*/ +std::unordered_map<std::string, std::string> &pkgAcquire::Item::ModifyCustomFields() /*{{{*/ +{ + return d->CustomFields; +} + /*}}}*/ +bool pkgAcquire::Item::PopAlternativeURI(std::string &NewURI) /*{{{*/ +{ + if (d->AlternativeURIs.empty()) + return false; + auto const AltUri = d->AlternativeURIs.front(); + d->AlternativeURIs.pop_front(); + NewURI = AltUri.URI; + auto &CustomFields = ModifyCustomFields(); + for (auto const &f : AltUri.changefields) + { + if (f.second.empty()) + CustomFields.erase(f.first); + else + CustomFields[f.first] = f.second; + } + return true; +} + /*}}}*/ +bool pkgAcquire::Item::IsGoodAlternativeURI(std::string const &AltUri) const/*{{{*/ +{ + return std::find(d->PastRedirections.cbegin(), d->PastRedirections.cend(), AltUri) == d->PastRedirections.cend() && + std::find(d->BadAlternativeSites.cbegin(), d->BadAlternativeSites.cend(), URI::SiteOnly(AltUri)) == d->BadAlternativeSites.cend(); +} + /*}}}*/ +void pkgAcquire::Item::PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back) /*{{{*/ +{ + if (IsGoodAlternativeURI(NewURI) == false) + return; + if (at_the_back) + d->AlternativeURIs.emplace_back(std::move(NewURI), std::move(fields)); + else + d->AlternativeURIs.emplace_front(std::move(NewURI), std::move(fields)); +} + /*}}}*/ +void pkgAcquire::Item::RemoveAlternativeSite(std::string &&OldSite) /*{{{*/ +{ + d->AlternativeURIs.erase(std::remove_if(d->AlternativeURIs.begin(), d->AlternativeURIs.end(), + [&](decltype(*d->AlternativeURIs.cbegin()) AltUri) { + return URI::SiteOnly(AltUri.URI) == OldSite; + }), + d->AlternativeURIs.end()); + d->BadAlternativeSites.push_back(std::move(OldSite)); +} + /*}}}*/ +unsigned int &pkgAcquire::Item::ModifyRetries() /*{{{*/ +{ + return d->Retries; +} + /*}}}*/ +std::string pkgAcquire::Item::ShortDesc() const /*{{{*/ +{ + return DescURI(); +} + /*}}}*/ +void pkgAcquire::Item::Finished() /*{{{*/ +{ +} + /*}}}*/ +APT_PURE pkgAcquire * pkgAcquire::Item::GetOwner() const /*{{{*/ +{ + return Owner; +} + /*}}}*/ +APT_PURE pkgAcquire::ItemDesc &pkgAcquire::Item::GetItemDesc() /*{{{*/ +{ + return Desc; +} + /*}}}*/ +APT_PURE bool pkgAcquire::Item::IsTrusted() const /*{{{*/ +{ + return false; +} + /*}}}*/ +// Acquire::Item::Failed - Item failed to download /*{{{*/ +// --------------------------------------------------------------------- +/* We return to an idle state if there are still other queues that could + fetch this object */ +static void formatHashsum(std::ostream &out, HashString const &hs) +{ + auto const type = hs.HashType(); + if (type == "Checksum-FileSize") + out << " - Filesize"; + else + out << " - " << type; + out << ':' << hs.HashValue(); + if (hs.usable() == false) + out << " [weak]"; + out << std::endl; +} +void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) +{ + if (QueueCounter <= 1) + { + /* This indicates that the file is not available right now but might + be sometime later. If we do a retry cycle then this should be + retried [CDROMs] */ + if (Cnf != NULL && Cnf->LocalOnly == true && + StringToBool(LookupTag(Message,"Transient-Failure"),false) == true) + { + Status = StatIdle; + Dequeue(); + return; + } + + switch (Status) + { + case StatIdle: + case StatFetching: + case StatDone: + Status = StatError; + break; + case StatAuthError: + case StatError: + case StatTransientNetworkError: + break; + } + Complete = false; + Dequeue(); + } + + FailMessage(Message); + + if (QueueCounter > 1) + Status = StatIdle; +} +void pkgAcquire::Item::FailMessage(string const &Message) +{ + string const FailReason = LookupTag(Message, "FailReason"); + enum { MAXIMUM_SIZE_EXCEEDED, HASHSUM_MISMATCH, WEAK_HASHSUMS, REDIRECTION_LOOP, OTHER } failreason = OTHER; + if ( FailReason == "MaximumSizeExceeded") + failreason = MAXIMUM_SIZE_EXCEEDED; + else if ( FailReason == "WeakHashSums") + failreason = WEAK_HASHSUMS; + else if (FailReason == "RedirectionLoop") + failreason = REDIRECTION_LOOP; + else if (Status == StatAuthError) + failreason = HASHSUM_MISMATCH; + + if(ErrorText.empty()) + { + std::ostringstream out; + switch (failreason) + { + case HASHSUM_MISMATCH: + out << _("Hash Sum mismatch") << std::endl; + break; + case WEAK_HASHSUMS: + out << _("Insufficient information available to perform this download securely") << std::endl; + break; + case REDIRECTION_LOOP: + out << "Redirection loop encountered" << std::endl; + break; + case MAXIMUM_SIZE_EXCEEDED: + out << LookupTag(Message, "Message") << std::endl; + break; + case OTHER: + out << LookupTag(Message, "Message"); + break; + } + + if (Status == StatAuthError) + { + auto const ExpectedHashes = GetExpectedHashes(); + if (ExpectedHashes.empty() == false) + { + out << "Hashes of expected file:" << std::endl; + for (auto const &hs: ExpectedHashes) + formatHashsum(out, hs); + } + if (failreason == HASHSUM_MISMATCH) + { + out << "Hashes of received file:" << std::endl; + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + { + std::string const tagname = std::string(*type) + "-Hash"; + std::string const hashsum = LookupTag(Message, tagname.c_str()); + if (hashsum.empty() == false) + formatHashsum(out, HashString(*type, hashsum)); + } + } + auto const lastmod = LookupTag(Message, "Last-Modified", ""); + if (lastmod.empty() == false) + out << "Last modification reported: " << lastmod << std::endl; + } + ErrorText = out.str(); + } + + switch (failreason) + { + case MAXIMUM_SIZE_EXCEEDED: RenameOnError(MaximumSizeExceeded); break; + case HASHSUM_MISMATCH: RenameOnError(HashSumMismatch); break; + case WEAK_HASHSUMS: break; + case REDIRECTION_LOOP: break; + case OTHER: break; + } + + if (FailReason.empty() == false) + ReportMirrorFailureToCentral(*this, FailReason, ErrorText); + else + ReportMirrorFailureToCentral(*this, ErrorText, ErrorText); +} + /*}}}*/ +// Acquire::Item::Start - Item has begun to download /*{{{*/ +// --------------------------------------------------------------------- +/* Stash status and the file size. Note that setting Complete means + sub-phases of the acquire process such as decompresion are operating */ +void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const Size) +{ + Status = StatFetching; + ErrorText.clear(); + if (FileSize == 0 && Complete == false) + FileSize = Size; +} + /*}}}*/ +// Acquire::Item::VerifyDone - check if Item was downloaded OK /*{{{*/ +/* Note that hash-verification is 'hardcoded' in acquire-worker and has + * already passed if this method is called. */ +bool pkgAcquire::Item::VerifyDone(std::string const &Message, + pkgAcquire::MethodConfig const * const /*Cnf*/) +{ + std::string const FileName = LookupTag(Message,"Filename"); + if (FileName.empty() == true) + { + Status = StatError; + ErrorText = "Method gave a blank filename"; + return false; + } + + return true; +} + /*}}}*/ +// Acquire::Item::Done - Item downloaded OK /*{{{*/ +void pkgAcquire::Item::Done(string const &/*Message*/, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const /*Cnf*/) +{ + // We just downloaded something.. + if (FileSize == 0) + { + unsigned long long const downloadedSize = Hashes.FileSize(); + if (downloadedSize != 0) + { + FileSize = downloadedSize; + } + } + Status = StatDone; + ErrorText.clear(); + Dequeue(); +} + /*}}}*/ +// Acquire::Item::Rename - Rename a file /*{{{*/ +// --------------------------------------------------------------------- +/* This helper function is used by a lot of item methods as their final + step */ +bool pkgAcquire::Item::Rename(string const &From,string const &To) +{ + if (From == To || rename(From.c_str(),To.c_str()) == 0) + return true; + + std::string S; + strprintf(S, _("rename failed, %s (%s -> %s)."), strerror(errno), + From.c_str(),To.c_str()); + Status = StatError; + if (ErrorText.empty()) + ErrorText = S; + else + ErrorText = ErrorText + ": " + S; + return false; +} + /*}}}*/ +void pkgAcquire::Item::Dequeue() /*{{{*/ +{ + d->AlternativeURIs.clear(); + Owner->Dequeue(this); +} + /*}}}*/ +bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/ +{ + if (RealFileExists(DestFile)) + Rename(DestFile, DestFile + ".FAILED"); + + std::string errtext; + switch (error) + { + case HashSumMismatch: + errtext = _("Hash Sum mismatch"); + break; + case SizeMismatch: + errtext = _("Size mismatch"); + Status = StatAuthError; + break; + case InvalidFormat: + errtext = _("Invalid file format"); + Status = StatError; + // do not report as usually its not the mirrors fault, but Portal/Proxy + break; + case SignatureError: + errtext = _("Signature error"); + Status = StatError; + break; + case NotClearsigned: + strprintf(errtext, _("Clearsigned file isn't valid, got '%s' (does the network require authentication?)"), "NOSPLIT"); + Status = StatAuthError; + break; + case MaximumSizeExceeded: + // the method is expected to report a good error for this + break; + case PDiffError: + // no handling here, done by callers + break; + } + if (ErrorText.empty()) + ErrorText = errtext; + return false; +} + /*}}}*/ +void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess)/*{{{*/ +{ + ActiveSubprocess = subprocess; + APT_IGNORE_DEPRECATED_PUSH + Mode = ActiveSubprocess.c_str(); + APT_IGNORE_DEPRECATED_POP +} + /*}}}*/ +// Acquire::Item::ReportMirrorFailure /*{{{*/ +void pkgAcquire::Item::ReportMirrorFailure(std::string const &FailCode) +{ + ReportMirrorFailureToCentral(*this, FailCode, FailCode); +} + /*}}}*/ +std::string pkgAcquire::Item::HashSum() const /*{{{*/ +{ + HashStringList const hashes = GetExpectedHashes(); + HashString const * const hs = hashes.find(NULL); + return hs != NULL ? hs->toStr() : ""; +} + /*}}}*/ +bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI) /*{{{*/ +{ + // store can fail due to permission errors and the item will "loop" then + if (APT::String::Startswith(NewURI, "store:")) + return false; + if (d->PastRedirections.empty()) + { + d->PastRedirections.push_back(NewURI); + return false; + } + auto const LastURI = std::prev(d->PastRedirections.end()); + // redirections to the same file are a way of restarting/resheduling, + // individual methods will have to make sure that they aren't looping this way + if (*LastURI == NewURI) + return false; + if (std::find(d->PastRedirections.begin(), LastURI, NewURI) != LastURI) + return true; + d->PastRedirections.push_back(NewURI); + return false; +} + /*}}}*/ +int pkgAcquire::Item::Priority() /*{{{*/ +{ + // Stage 0: Files requested by methods + // - they will usually not end up here, but if they do we make sure + // to get them as soon as possible as they are probably blocking + // the processing of files by the requesting method + if (dynamic_cast<pkgAcqAuxFile *>(this) != nullptr) + return 5000; + // Stage 1: Meta indices and diff indices + // - those need to be fetched first to have progress reporting working + // for the rest + if (dynamic_cast<pkgAcqMetaSig*>(this) != nullptr + || dynamic_cast<pkgAcqMetaBase*>(this) != nullptr + || dynamic_cast<pkgAcqDiffIndex*>(this) != nullptr) + return 1000; + // Stage 2: Diff files + // - fetch before complete indexes so we can apply the diffs while fetching + // larger files. + if (dynamic_cast<pkgAcqIndexDiffs*>(this) != nullptr || + dynamic_cast<pkgAcqIndexMergeDiffs*>(this) != nullptr) + return 800; + + // Stage 3: The rest - complete index files and other stuff + return 500; +} + /*}}}*/ + +pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner, /*{{{*/ + pkgAcqMetaClearSig * const transactionManager, IndexTarget const &target) : + pkgAcquire::Item(Owner), d(NULL), Target(target), TransactionManager(transactionManager) +{ + if (TransactionManager != this) + TransactionManager->Add(this); + ModifyCustomFields() = { + {"Target-Site", Target.Option(IndexTarget::SITE)}, + {"Target-Repo-URI", Target.Option(IndexTarget::REPO_URI)}, + {"Target-Base-URI", Target.Option(IndexTarget::BASE_URI)}, + {"Target-Component", Target.Option(IndexTarget::COMPONENT)}, + {"Target-Release", Target.Option(IndexTarget::RELEASE)}, + {"Target-Architecture", Target.Option(IndexTarget::ARCHITECTURE)}, + {"Target-Language", Target.Option(IndexTarget::LANGUAGE)}, + {"Target-Type", "index"}, + }; +} + /*}}}*/ +pkgAcqTransactionItem::~pkgAcqTransactionItem() /*{{{*/ +{ +} + /*}}}*/ +HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const &MetaKey) const /*{{{*/ +{ + return GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, MetaKey); +} + /*}}}*/ + +static void LoadLastMetaIndexParser(pkgAcqMetaClearSig * const TransactionManager, std::string const &FinalRelease, std::string const &FinalInRelease)/*{{{*/ +{ + if (TransactionManager->IMSHit == true) + return; + if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease)) + { + TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone(); + if (TransactionManager->LastMetaIndexParser != NULL) + { + _error->PushToStack(); + if (RealFileExists(FinalInRelease)) + TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL); + else + TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL); + // its unlikely to happen, but if what we have is bad ignore it + if (_error->PendingError()) + { + delete TransactionManager->LastMetaIndexParser; + TransactionManager->LastMetaIndexParser = NULL; + } + _error->RevertToStack(); + } + } +} + /*}}}*/ + +// AcqMetaBase - Constructor /*{{{*/ +pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire * const Owner, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &DataTarget) +: pkgAcqTransactionItem(Owner, TransactionManager, DataTarget), d(NULL), + AuthPass(false), IMSHit(false), State(TransactionStarted) +{ +} + /*}}}*/ +// AcqMetaBase::Add - Add a item to the current Transaction /*{{{*/ +void pkgAcqMetaBase::Add(pkgAcqTransactionItem * const I) +{ + Transaction.push_back(I); +} + /*}}}*/ +// AcqMetaBase::AbortTransaction - Abort the current Transaction /*{{{*/ +void pkgAcqMetaBase::AbortTransaction() +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "AbortTransaction: " << TransactionManager << std::endl; + + switch (TransactionManager->State) + { + case TransactionStarted: break; + case TransactionAbort: _error->Fatal("Transaction %s was already aborted and is aborted again", TransactionManager->Target.URI.c_str()); return; + case TransactionCommit: _error->Fatal("Transaction %s was already aborted and is now committed", TransactionManager->Target.URI.c_str()); return; + } + TransactionManager->State = TransactionAbort; + TransactionManager->ExpectedAdditionalItems = 0; + + // ensure the toplevel is in error state too + for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin(); + I != Transaction.end(); ++I) + { + (*I)->ExpectedAdditionalItems = 0; + if ((*I)->Status != pkgAcquire::Item::StatFetching) + (*I)->Dequeue(); + (*I)->TransactionState(TransactionAbort); + } + Transaction.clear(); +} + /*}}}*/ +// AcqMetaBase::TransactionHasError - Check for errors in Transaction /*{{{*/ +APT_PURE bool pkgAcqMetaBase::TransactionHasError() const +{ + for (std::vector<pkgAcqTransactionItem*>::const_iterator I = Transaction.begin(); + I != Transaction.end(); ++I) + { + switch((*I)->Status) { + case StatDone: break; + case StatIdle: break; + case StatAuthError: return true; + case StatError: return true; + case StatTransientNetworkError: return true; + case StatFetching: break; + } + } + return false; +} + /*}}}*/ +// AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/ +void pkgAcqMetaBase::CommitTransaction() +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "CommitTransaction: " << this << std::endl; + + switch (TransactionManager->State) + { + case TransactionStarted: break; + case TransactionAbort: _error->Fatal("Transaction %s was already committed and is now aborted", TransactionManager->Target.URI.c_str()); return; + case TransactionCommit: _error->Fatal("Transaction %s was already committed and is again committed", TransactionManager->Target.URI.c_str()); return; + } + TransactionManager->State = TransactionCommit; + + // move new files into place *and* remove files that are not + // part of the transaction but are still on disk + for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin(); + I != Transaction.end(); ++I) + { + (*I)->TransactionState(TransactionCommit); + } + Transaction.clear(); +} + /*}}}*/ +// AcqMetaBase::TransactionStageCopy - Stage a file for copying /*{{{*/ +void pkgAcqMetaBase::TransactionStageCopy(pkgAcqTransactionItem * const I, + const std::string &From, + const std::string &To) +{ + I->PartialFile = From; + I->DestFile = To; +} + /*}}}*/ +// AcqMetaBase::TransactionStageRemoval - Stage a file for removal /*{{{*/ +void pkgAcqMetaBase::TransactionStageRemoval(pkgAcqTransactionItem * const I, + const std::string &FinalFile) +{ + I->PartialFile = ""; + I->DestFile = FinalFile; +} + /*}}}*/ +// AcqMetaBase::GenerateAuthWarning - Check gpg authentication error /*{{{*/ +/* This method is called from ::Failed handlers. If it returns true, + no fallback to other files or modi is performed */ +bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message) +{ + string const Final = I->GetFinalFilename(); + std::string const GPGError = LookupTag(Message, "Message"); + if (FileExists(Final)) + { + I->Status = StatTransientNetworkError; + _error->Warning(_("An error occurred during the signature verification. " + "The repository is not updated and the previous index files will be used. " + "GPG error: %s: %s"), + Desc.Description.c_str(), + GPGError.c_str()); + RunScripts("APT::Update::Auth-Failure"); + return true; + } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) { + /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */ + _error->Error(_("GPG error: %s: %s"), + Desc.Description.c_str(), + GPGError.c_str()); + I->Status = StatAuthError; + return true; + } else { + _error->Warning(_("GPG error: %s: %s"), + Desc.Description.c_str(), + GPGError.c_str()); + } + // gpgv method failed + ReportMirrorFailureToCentral(*this, "GPGFailure", GPGError); + return false; +} + /*}}}*/ +// AcqMetaBase::Custom600Headers - Get header for AcqMetaBase /*{{{*/ +// --------------------------------------------------------------------- +string pkgAcqMetaBase::Custom600Headers() const +{ + std::string Header = pkgAcqTransactionItem::Custom600Headers(); + Header.append("\nIndex-File: true"); + std::string MaximumSize; + strprintf(MaximumSize, "\nMaximum-Size: %i", + _config->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000)); + Header += MaximumSize; + + string const FinalFile = GetFinalFilename(); + struct stat Buf; + if (stat(FinalFile.c_str(),&Buf) == 0) + Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false); + + return Header; +} + /*}}}*/ +// AcqMetaBase::QueueForSignatureVerify /*{{{*/ +void pkgAcqMetaBase::QueueForSignatureVerify(pkgAcqTransactionItem * const I, std::string const &File, std::string const &Signature) +{ + AuthPass = true; + I->Desc.URI = "gpgv:" + Signature; + I->DestFile = File; + QueueURI(I->Desc); + I->SetActiveSubprocess("gpgv"); +} + /*}}}*/ +// AcqMetaBase::CheckDownloadDone /*{{{*/ +bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const std::string &Message, HashStringList const &Hashes) const +{ + // We have just finished downloading a Release file (it is not + // verified yet) + + // Save the final base URI we got this Release file from + if (I->UsedMirror.empty() == false && _config->FindB("Acquire::SameMirrorForAllIndexes", true)) + { + auto InReleasePath = Target.Option(IndexTarget::INRELEASE_PATH); + if (InReleasePath.empty()) + InReleasePath = "InRelease"; + + if (APT::String::Endswith(I->Desc.URI, InReleasePath)) + { + TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - InReleasePath.length()); + TransactionManager->UsedMirror = I->UsedMirror; + } + else if (APT::String::Endswith(I->Desc.URI, "Release")) + { + TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - strlen("Release")); + TransactionManager->UsedMirror = I->UsedMirror; + } + } + + std::string const FileName = LookupTag(Message,"Filename"); + if (FileName != I->DestFile && RealFileExists(I->DestFile) == false) + { + I->Local = true; + I->Desc.URI = "copy:" + FileName; + I->QueueURI(I->Desc); + return false; + } + + // make sure to verify against the right file on I-M-S hit + bool IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"), false); + if (IMSHit == false && Hashes.usable()) + { + // detect IMS-Hits servers haven't detected by Hash comparison + std::string const FinalFile = I->GetFinalFilename(); + if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true) + { + IMSHit = true; + RemoveFile("CheckDownloadDone", I->DestFile); + } + } + + if(IMSHit == true) + { + // for simplicity, the transaction manager is always InRelease + // even if it doesn't exist. + TransactionManager->IMSHit = true; + I->PartialFile = I->DestFile = I->GetFinalFilename(); + } + + // set Item to complete as the remaining work is all local (verify etc) + I->Complete = true; + + return true; +} + /*}}}*/ +bool pkgAcqMetaBase::CheckAuthDone(string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/ +{ + /* If we work with a recent version of our gpgv method, we expect that it tells us + which key(s) have signed the file so stuff like CVE-2018-0501 is harder in the future */ + if (Cnf->Version != "1.0" && LookupTag(Message, "Signed-By").empty()) + { + std::string errmsg; + strprintf(errmsg, "Internal Error: Signature on %s seems good, but expected details are missing! (%s)", Target.URI.c_str(), "Signed-By"); + if (ErrorText.empty()) + ErrorText = errmsg; + Status = StatAuthError; + return _error->Error("%s", errmsg.c_str()); + } + + // At this point, the gpgv method has succeeded, so there is a + // valid signature from a key in the trusted keyring. We + // perform additional verification of its contents, and use them + // to verify the indexes we are about to download + if (_config->FindB("Debug::pkgAcquire::Auth", false)) + std::cerr << "Signature verification succeeded: " << DestFile << std::endl; + + if (TransactionManager->IMSHit == false) + { + // open the last (In)Release if we have it + std::string const FinalFile = GetFinalFilename(); + std::string FinalRelease; + std::string FinalInRelease; + if (APT::String::Endswith(FinalFile, "InRelease")) + { + FinalInRelease = FinalFile; + FinalRelease = FinalFile.substr(0, FinalFile.length() - strlen("InRelease")) + "Release"; + } + else + { + FinalInRelease = FinalFile.substr(0, FinalFile.length() - strlen("Release")) + "InRelease"; + FinalRelease = FinalFile; + } + LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease); + } + + bool const GoodAuth = TransactionManager->MetaIndexParser->Load(DestFile, &ErrorText); + if (GoodAuth == false && AllowInsecureRepositories(InsecureType::WEAK, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == false) + { + Status = StatAuthError; + return false; + } + + if (!VerifyVendor(Message)) + { + Status = StatAuthError; + return false; + } + + // Download further indexes with verification + TransactionManager->QueueIndexes(GoodAuth); + + return GoodAuth; +} + /*}}}*/ +void pkgAcqMetaClearSig::QueueIndexes(bool const verify) /*{{{*/ +{ + // at this point the real Items are loaded in the fetcher + ExpectedAdditionalItems = 0; + + std::unordered_set<std::string> targetsSeen, componentsSeen; + bool const hasReleaseFile = TransactionManager->MetaIndexParser != NULL; + bool hasHashes = true; + auto IndexTargets = TransactionManager->MetaIndexParser->GetIndexTargets(); + if (hasReleaseFile && verify == false) + hasHashes = std::any_of(IndexTargets.begin(), IndexTargets.end(), + [&](IndexTarget const &Target) { return TransactionManager->MetaIndexParser->Exists(Target.MetaKey); }); + if (_config->FindB("Acquire::IndexTargets::Randomized", true) && likely(IndexTargets.empty() == false)) + { + /* For fallback handling and to have some reasonable progress information + we can't randomize everything, but at least the order in the same type + can be as we shouldn't be telling the mirrors (and everyone else watching) + which is native/foreign arch, specific order of preference of translations, … */ + auto range_start = IndexTargets.begin(); + auto seed = (std::chrono::high_resolution_clock::now().time_since_epoch() / std::chrono::nanoseconds(1)) ^ getpid(); + std::default_random_engine g(seed); + do { + auto const type = range_start->Option(IndexTarget::CREATED_BY); + auto const range_end = std::find_if_not(range_start, IndexTargets.end(), + [&type](IndexTarget const &T) { return type == T.Option(IndexTarget::CREATED_BY); }); + std::shuffle(range_start, range_end, g); + range_start = range_end; + } while (range_start != IndexTargets.end()); + } + /* Collect all components for which files exist to prevent apt from warning users + about "hidden" components for which not all files exist like main/debian-installer + and Translation files */ + if (hasReleaseFile == true) + for (auto const &Target : IndexTargets) + if (TransactionManager->MetaIndexParser->Exists(Target.MetaKey)) + { + auto component = Target.Option(IndexTarget::COMPONENT); + if (component.empty() == false) + componentsSeen.emplace(std::move(component)); + } + + for (auto&& Target: IndexTargets) + { + // if we have seen a target which is created-by a target this one here is declared a + // fallback to, we skip acquiring the fallback (but we make sure we clean up) + if (targetsSeen.find(Target.Option(IndexTarget::FALLBACK_OF)) != targetsSeen.end()) + { + targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY)); + new CleanupItem(Owner, TransactionManager, Target); + continue; + } + // all is an implementation detail. Users shouldn't use this as arch + // We need this support trickery here as e.g. Debian has binary-all files already, + // but arch:all packages are still in the arch:any files, so we would waste precious + // download time, bandwidth and diskspace for nothing, BUT Debian doesn't feature all + // in the set of supported architectures, so we can filter based on this property rather + // than invent an entirely new flag we would need to carry for all of eternity. + if (hasReleaseFile && Target.Option(IndexTarget::ARCHITECTURE) == "all") + { + if (TransactionManager->MetaIndexParser->IsArchitectureAllSupportedFor(Target) == false) + { + new CleanupItem(Owner, TransactionManager, Target); + continue; + } + } + + bool trypdiff = Target.OptionBool(IndexTarget::PDIFFS); + if (hasReleaseFile == true) + { + if (TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false) + { + auto const component = Target.Option(IndexTarget::COMPONENT); + if (component.empty() == false && + componentsSeen.find(component) == std::end(componentsSeen) && + TransactionManager->MetaIndexParser->HasSupportForComponent(component) == false) + { + new CleanupItem(Owner, TransactionManager, Target); + _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' doesn't have the component '%s' (component misspelt in sources.list?)"), + Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str(), component.c_str()); + continue; + + } + + // optional targets that we do not have in the Release file are skipped + if (hasHashes == true && Target.IsOptional) + { + new CleanupItem(Owner, TransactionManager, Target); + continue; + } + + std::string const &arch = Target.Option(IndexTarget::ARCHITECTURE); + if (arch.empty() == false) + { + if (TransactionManager->MetaIndexParser->IsArchitectureSupported(arch) == false) + { + new CleanupItem(Owner, TransactionManager, Target); + _error->Notice(_("Skipping acquire of configured file '%s' as repository '%s' doesn't support architecture '%s'"), + Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str(), arch.c_str()); + continue; + } + // if the architecture is officially supported but currently no packages for it available, + // ignore silently as this is pretty much the same as just shipping an empty file. + // if we don't know which architectures are supported, we do NOT ignore it to notify user about this + if (hasHashes == true && TransactionManager->MetaIndexParser->IsArchitectureSupported("*undefined*") == false) + { + new CleanupItem(Owner, TransactionManager, Target); + continue; + } + } + + if (hasHashes == true) + { + new CleanupItem(Owner, TransactionManager, Target); + _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' does not seem to provide it (sources.list entry misspelt?)"), + Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str()); + continue; + } + else + { + new pkgAcqIndex(Owner, TransactionManager, Target); + continue; + } + } + else if (verify) + { + auto const hashes = GetExpectedHashesFor(Target.MetaKey); + if (hashes.empty() == false) + { + if (hashes.usable() == false && TargetIsAllowedToBe(TransactionManager->Target, InsecureType::WEAK) == false) + { + new CleanupItem(Owner, TransactionManager, Target); + _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"), + Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str()); + continue; + } + // empty files are skipped as acquiring the very small compressed files is a waste of time + else if (hashes.FileSize() == 0) + { + new CleanupItem(Owner, TransactionManager, Target); + targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY)); + continue; + } + } + } + + // autoselect the compression method + std::vector<std::string> types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' '); + types.erase(std::remove_if(types.begin(), types.end(), [&](std::string const &t) { + if (t == "uncompressed") + return TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false; + std::string const MetaKey = Target.MetaKey + "." + t; + return TransactionManager->MetaIndexParser->Exists(MetaKey) == false; + }), types.end()); + if (types.empty() == false) + { + std::ostringstream os; + std::copy(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " ")); + os << *types.rbegin(); + Target.Options["COMPRESSIONTYPES"] = os.str(); + } + else + Target.Options["COMPRESSIONTYPES"].clear(); + + std::string filename = GetExistingFilename(GetFinalFileNameFromURI(Target.URI)); + if (filename.empty() == false) + { + // if the Release file is a hit and we have an index it must be the current one + if (TransactionManager->IMSHit == true) + ; + else if (TransactionManager->LastMetaIndexParser != NULL) + { + // see if the file changed since the last Release file + // we use the uncompressed files as we might compress differently compared to the server, + // so the hashes might not match, even if they contain the same data. + HashStringList const newFile = GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, Target.MetaKey); + HashStringList const oldFile = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey); + if (newFile != oldFile) + filename.clear(); + } + else + filename.clear(); + } + else + trypdiff = false; // no file to patch + + if (filename.empty() == false) + { + new NoActionItem(Owner, Target, filename); + std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(Target)); + if (FileExists(idxfilename)) + new NoActionItem(Owner, Target, idxfilename); + targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY)); + continue; + } + + // check if we have patches available + trypdiff &= TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey)); + } + else + { + // if we have no file to patch, no point in trying + trypdiff &= (GetExistingFilename(GetFinalFileNameFromURI(Target.URI)).empty() == false); + } + + // no point in patching from local sources + if (trypdiff) + { + std::string const proto = Target.URI.substr(0, strlen("file:/")); + if (proto == "file:/" || proto == "copy:/" || proto == "cdrom:") + trypdiff = false; + } + + // Queue the Index file (Packages, Sources, Translation-$foo, …) + targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY)); + if (trypdiff) + new pkgAcqDiffIndex(Owner, TransactionManager, Target); + else + new pkgAcqIndex(Owner, TransactionManager, Target); + } +} + /*}}}*/ +bool pkgAcqMetaBase::VerifyVendor(string const &) /*{{{*/ +{ + if (TransactionManager->MetaIndexParser->GetValidUntil() > 0) + { + time_t const invalid_since = time(NULL) - TransactionManager->MetaIndexParser->GetValidUntil(); + if (invalid_since > 0) + { + std::string errmsg; + strprintf(errmsg, + // TRANSLATOR: The first %s is the URL of the bad Release file, the second is + // the time since then the file is invalid - formatted in the same way as in + // the download progress display (e.g. 7d 3h 42min 1s) + _("Release file for %s is expired (invalid since %s). " + "Updates for this repository will not be applied."), + Target.URI.c_str(), TimeToStr(invalid_since).c_str()); + if (ErrorText.empty()) + ErrorText = errmsg; + return _error->Error("%s", errmsg.c_str()); + } + } + + if (TransactionManager->MetaIndexParser->GetNotBefore() > 0) + { + time_t const invalid_for = TransactionManager->MetaIndexParser->GetNotBefore() - time(nullptr); + if (invalid_for > 0) + { + std::string errmsg; + strprintf(errmsg, + // TRANSLATOR: The first %s is the URL of the bad Release file, the second is + // the time until the file will be valid - formatted in the same way as in + // the download progress display (e.g. 7d 3h 42min 1s) + _("Release file for %s is not valid yet (invalid for another %s). " + "Updates for this repository will not be applied."), + Target.URI.c_str(), TimeToStr(invalid_for).c_str()); + if (ErrorText.empty()) + ErrorText = errmsg; + return _error->Error("%s", errmsg.c_str()); + } + } + + /* Did we get a file older than what we have? This is a last minute IMS hit and doubles + as a prevention of downgrading us to older (still valid) files */ + if (TransactionManager->IMSHit == false && TransactionManager->LastMetaIndexParser != NULL && + TransactionManager->LastMetaIndexParser->GetDate() > TransactionManager->MetaIndexParser->GetDate()) + { + TransactionManager->IMSHit = true; + RemoveFile("VerifyVendor", DestFile); + PartialFile = DestFile = GetFinalFilename(); + // load the 'old' file in the 'new' one instead of flipping pointers as + // the new one isn't owned by us, while the old one is so cleanup would be confused. + TransactionManager->MetaIndexParser->swapLoad(TransactionManager->LastMetaIndexParser); + delete TransactionManager->LastMetaIndexParser; + TransactionManager->LastMetaIndexParser = NULL; + } + + if (_config->FindB("Debug::pkgAcquire::Auth", false)) + { + std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetCodename() << std::endl; + std::cerr << "Got Suite: " << TransactionManager->MetaIndexParser->GetSuite() << std::endl; + std::cerr << "Expecting Dist: " << TransactionManager->MetaIndexParser->GetExpectedDist() << std::endl; + } + + // One day that might become fatal… + auto const ExpectedDist = TransactionManager->MetaIndexParser->GetExpectedDist(); + auto const NowCodename = TransactionManager->MetaIndexParser->GetCodename(); + if (TransactionManager->MetaIndexParser->CheckDist(ExpectedDist) == false) + _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"), + Desc.Description.c_str(), ExpectedDist.c_str(), NowCodename.c_str()); + + // changed info potentially breaks user config like pinning + if (TransactionManager->LastMetaIndexParser != nullptr) + { + std::vector<pkgAcquireStatus::ReleaseInfoChange> Changes; + auto const AllowInfoChange = _config->FindB("Acquire::AllowReleaseInfoChange", false); + auto const quietInfoChange = _config->FindB("quiet::ReleaseInfoChange", false); + struct { + char const * const Type; + bool const Allowed; + decltype(&metaIndex::GetOrigin) const Getter; + } checkers[] = { + { "Origin", AllowInfoChange, &metaIndex::GetOrigin }, + { "Label", AllowInfoChange, &metaIndex::GetLabel }, + { "Version", true, &metaIndex::GetVersion }, // numbers change all the time, that is okay + { "Suite", true, &metaIndex::GetSuite }, + { "Codename", AllowInfoChange, &metaIndex::GetCodename }, + { nullptr, false, nullptr } + }; + auto const CheckReleaseInfo = [&](char const * const Type, bool const AllowChange, decltype(checkers[0].Getter) const Getter) { + std::string const Last = (TransactionManager->LastMetaIndexParser->*Getter)(); + std::string const Now = (TransactionManager->MetaIndexParser->*Getter)(); + if (Last == Now) + return; + auto const Allow = _config->FindB(std::string("Acquire::AllowReleaseInfoChange::").append(Type), AllowChange); + if (Allow == true && _config->FindB(std::string("quiet::ReleaseInfoChange::").append(Type), quietInfoChange) == true) + return; + std::string msg; + strprintf(msg, _("Repository '%s' changed its '%s' value from '%s' to '%s'"), + Desc.Description.c_str(), Type, Last.c_str(), Now.c_str()); + Changes.push_back({Type, std::move(Last), std::move(Now), std::move(msg), Allow}); + }; + for (short i = 0; checkers[i].Type != nullptr; ++i) + CheckReleaseInfo(checkers[i].Type, checkers[i].Allowed, checkers[i].Getter); + + { + auto const Last = TransactionManager->LastMetaIndexParser->GetDefaultPin(); + auto const Now = TransactionManager->MetaIndexParser->GetDefaultPin(); + if (Last != Now) + { + auto const Allow = _config->FindB("Acquire::AllowReleaseInfoChange::DefaultPin", AllowInfoChange); + if (Allow == false || _config->FindB("quiet::ReleaseInfoChange::DefaultPin", quietInfoChange) == false) + { + std::string msg; + strprintf(msg, _("Repository '%s' changed its default priority for %s from %hi to %hi."), + Desc.Description.c_str(), "apt_preferences(5)", Last, Now); + Changes.push_back({"DefaultPin", std::to_string(Last), std::to_string(Now), std::move(msg), Allow}); + } + } + } + if (Changes.empty() == false) + { + auto const notes = TransactionManager->MetaIndexParser->GetReleaseNotes(); + if (notes.empty() == false) + { + std::string msg; + // TRANSLATOR: the "this" refers to changes in the repository like a new release or owner change + strprintf(msg, _("More information about this can be found online in the Release notes at: %s"), notes.c_str()); + Changes.push_back({"Release-Notes", "", std::move(notes), std::move(msg), true}); + } + if (std::any_of(Changes.begin(),Changes.end(),[](pkgAcquireStatus::ReleaseInfoChange const &c) { return c.DefaultAction == false; })) + { + std::string msg; + // TRANSLATOR: %s is the name of the manpage in question, e.g. apt-secure(8) + strprintf(msg, _("This must be accepted explicitly before updates for " + "this repository can be applied. See %s manpage for details."), "apt-secure(8)"); + Changes.push_back({"Confirmation", "", "", std::move(msg), true}); + } + + } + if (Owner->Log == nullptr) + return pkgAcquireStatus::ReleaseInfoChangesAsGlobalErrors(std::move(Changes)); + return Owner->Log->ReleaseInfoChanges(TransactionManager->LastMetaIndexParser, TransactionManager->MetaIndexParser, std::move(Changes)); + } + return true; +} + /*}}}*/ +pkgAcqMetaBase::~pkgAcqMetaBase() +{ +} + +pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner, /*{{{*/ + IndexTarget const &ClearsignedTarget, + IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget, + metaIndex * const MetaIndexParser) : + pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget), + d(NULL), DetachedDataTarget(DetachedDataTarget), + MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL) +{ + // index targets + (worst case:) Release/Release.gpg + ExpectedAdditionalItems = std::numeric_limits<decltype(ExpectedAdditionalItems)>::max(); + TransactionManager->Add(this); +} + /*}}}*/ +pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/ +{ + if (LastMetaIndexParser != NULL) + delete LastMetaIndexParser; +} + /*}}}*/ +// pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/ +string pkgAcqMetaClearSig::Custom600Headers() const +{ + string Header = pkgAcqMetaBase::Custom600Headers(); + Header += "\nFail-Ignore: true"; + std::string const key = TransactionManager->MetaIndexParser->GetSignedBy(); + if (key.empty() == false) + Header += "\nSigned-By: " + key; + + return Header; +} + /*}}}*/ +void pkgAcqMetaClearSig::Finished() /*{{{*/ +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "Finished: " << DestFile <<std::endl; + if(TransactionManager->State == TransactionStarted && + TransactionManager->TransactionHasError() == false) + TransactionManager->CommitTransaction(); +} + /*}}}*/ +bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message, /*{{{*/ + pkgAcquire::MethodConfig const * const Cnf) +{ + if (Item::VerifyDone(Message, Cnf) == false) + return false; + + if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile)) + return RenameOnError(NotClearsigned); + + return true; +} + /*}}}*/ +// pkgAcqMetaClearSig::Done - We got a file /*{{{*/ +void pkgAcqMetaClearSig::Done(std::string const &Message, + HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) +{ + Item::Done(Message, Hashes, Cnf); + + if(AuthPass == false) + { + if(CheckDownloadDone(this, Message, Hashes) == true) + QueueForSignatureVerify(this, DestFile, DestFile); + return; + } + else if (CheckAuthDone(Message, Cnf) == true) + { + if (TransactionManager->IMSHit == false) + TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); + else if (RealFileExists(GetFinalFilename()) == false) + { + // We got an InRelease file IMSHit, but we haven't one, which means + // we had a valid Release/Release.gpg combo stepping in, which we have + // to 'acquire' now to ensure list cleanup isn't removing them + new NoActionItem(Owner, DetachedDataTarget); + new NoActionItem(Owner, DetachedSigTarget); + } + } + else if (Status != StatAuthError) + { + string const FinalFile = GetFinalFileNameFromURI(DetachedDataTarget.URI); + string const OldFile = GetFinalFilename(); + if (TransactionManager->IMSHit == false) + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); + else if (RealFileExists(OldFile) == false) + new NoActionItem(Owner, DetachedDataTarget); + else + TransactionManager->TransactionStageCopy(this, OldFile, FinalFile); + } +} + /*}}}*/ +void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) /*{{{*/ +{ + Item::Failed(Message, Cnf); + + if (AuthPass == false) + { + if (Status == StatTransientNetworkError) + { + TransactionManager->AbortTransaction(); + return; + } + auto const failreason = LookupTag(Message, "FailReason"); + auto const httperror = "HttpError"; + if (Status == StatAuthError || + Target.Option(IndexTarget::INRELEASE_PATH).empty() == false || /* do not fallback if InRelease was requested */ + (strncmp(failreason.c_str(), httperror, strlen(httperror)) == 0 && + failreason != "HttpError404")) + { + // if we expected a ClearTextSignature (InRelease) but got a network + // error or got a file, but it wasn't valid, we end up here (see VerifyDone). + // As these is usually called by web-portals we do not try Release/Release.gpg + // as this is going to fail anyway and instead abort our try (LP#346386) + _error->PushToStack(); + _error->Error(_("Failed to fetch %s %s"), Target.URI.c_str(), ErrorText.c_str()); + if (Target.Option(IndexTarget::INRELEASE_PATH).empty() == true && AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true) + _error->RevertToStack(); + else + return; + } + + // Queue the 'old' InRelease file for removal if we try Release.gpg + // as otherwise the file will stay around and gives a false-auth + // impression (CVE-2012-0214) + TransactionManager->TransactionStageRemoval(this, GetFinalFilename()); + Status = StatDone; + + new pkgAcqMetaIndex(Owner, TransactionManager, DetachedDataTarget, DetachedSigTarget); + } + else + { + if(CheckStopAuthentication(this, Message)) + return; + + if(AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true) + { + Status = StatDone; + + /* InRelease files become Release files, otherwise + * they would be considered as trusted later on */ + string const FinalRelease = GetFinalFileNameFromURI(DetachedDataTarget.URI); + string const PartialRelease = GetPartialFileNameFromURI(DetachedDataTarget.URI); + string const FinalReleasegpg = GetFinalFileNameFromURI(DetachedSigTarget.URI); + string const FinalInRelease = GetFinalFilename(); + Rename(DestFile, PartialRelease); + TransactionManager->TransactionStageCopy(this, PartialRelease, FinalRelease); + LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease); + + // we parse the indexes here because at this point the user wanted + // a repository that may potentially harm him + if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false) + /* expired Release files are still a problem you need extra force for */; + else + TransactionManager->QueueIndexes(true); + } + } +} + /*}}}*/ + +pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire * const Owner, /*{{{*/ + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &DataTarget, + IndexTarget const &DetachedSigTarget) : + pkgAcqMetaBase(Owner, TransactionManager, DataTarget), d(NULL), + DetachedSigTarget(DetachedSigTarget) +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgAcqMetaIndex with TransactionManager " + << this->TransactionManager << std::endl; + + DestFile = GetPartialFileNameFromURI(DataTarget.URI); + + // Create the item + Desc.Description = DataTarget.Description; + Desc.Owner = this; + Desc.ShortDesc = DataTarget.ShortDesc; + + // Rewrite the description URI if INRELEASE_PATH was specified so + // we download the specified file instead. + auto InReleasePath = DataTarget.Option(IndexTarget::INRELEASE_PATH); + if (InReleasePath.empty() == false && APT::String::Endswith(DataTarget.URI, "/InRelease")) + { + Desc.URI = DataTarget.URI.substr(0, DataTarget.URI.size() - strlen("InRelease")) + InReleasePath; + } + else + { + Desc.URI = DataTarget.URI; + } + + QueueURI(Desc); +} + /*}}}*/ +void pkgAcqMetaIndex::Done(string const &Message, /*{{{*/ + HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cfg) +{ + Item::Done(Message,Hashes,Cfg); + + if(CheckDownloadDone(this, Message, Hashes)) + { + // we have a Release file, now download the Signature, all further + // verify/queue for additional downloads will be done in the + // pkgAcqMetaSig::Done() code + new pkgAcqMetaSig(Owner, TransactionManager, DetachedSigTarget, this); + } +} + /*}}}*/ +// pkgAcqMetaIndex::Failed - no Release file present /*{{{*/ +void pkgAcqMetaIndex::Failed(string const &Message, + pkgAcquire::MethodConfig const * const Cnf) +{ + pkgAcquire::Item::Failed(Message, Cnf); + Status = StatDone; + + // No Release file was present so fall + // back to queueing Packages files without verification + // only allow going further if the user explicitly wants it + if(AllowInsecureRepositories(InsecureType::NORELEASE, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true) + { + // ensure old Release files are removed + TransactionManager->TransactionStageRemoval(this, GetFinalFilename()); + + // queue without any kind of hashsum support + TransactionManager->QueueIndexes(false); + } +} + /*}}}*/ +std::string pkgAcqMetaIndex::DescURI() const /*{{{*/ +{ + return Target.URI; +} + /*}}}*/ +pkgAcqMetaIndex::~pkgAcqMetaIndex() {} + +// AcqMetaSig::AcqMetaSig - Constructor /*{{{*/ +pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, + pkgAcqMetaIndex * const MetaIndex) : + pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL), MetaIndex(MetaIndex) +{ + DestFile = GetPartialFileNameFromURI(Target.URI); + + // remove any partial downloaded sig-file in partial/. + // it may confuse proxies and is too small to warrant a + // partial download anyway + RemoveFile("pkgAcqMetaSig", DestFile); + + // set the TransactionManager + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgAcqMetaSig with TransactionManager " + << TransactionManager << std::endl; + + // Create the item + Desc.Description = Target.Description; + Desc.Owner = this; + Desc.ShortDesc = Target.ShortDesc; + Desc.URI = Target.URI; + + // If we got a hit for Release, we will get one for Release.gpg too (or obscure errors), + // so we skip the download step and go instantly to verification + if (TransactionManager->IMSHit == true && RealFileExists(GetFinalFilename())) + { + Complete = true; + Status = StatDone; + PartialFile = DestFile = GetFinalFilename(); + MetaIndexFileSignature = DestFile; + MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile); + } + else + QueueURI(Desc); +} + /*}}}*/ +pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/ +{ +} + /*}}}*/ +// pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/ +std::string pkgAcqMetaSig::Custom600Headers() const +{ + std::string Header = pkgAcqTransactionItem::Custom600Headers(); + std::string const key = TransactionManager->MetaIndexParser->GetSignedBy(); + if (key.empty() == false) + Header += "\nSigned-By: " + key; + return Header; +} + /*}}}*/ +// AcqMetaSig::Done - The signature was downloaded/verified /*{{{*/ +void pkgAcqMetaSig::Done(string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cfg) +{ + if (MetaIndexFileSignature.empty() == false) + { + DestFile = MetaIndexFileSignature; + MetaIndexFileSignature.clear(); + } + Item::Done(Message, Hashes, Cfg); + + if(MetaIndex->AuthPass == false) + { + if(MetaIndex->CheckDownloadDone(this, Message, Hashes) == true) + { + // destfile will be modified to point to MetaIndexFile for the + // gpgv method, so we need to save it here + MetaIndexFileSignature = DestFile; + MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile); + } + return; + } + else if (MetaIndex->CheckAuthDone(Message, Cfg) == true) + { + auto const Releasegpg = GetFinalFilename(); + auto const Release = MetaIndex->GetFinalFilename(); + // if this is an IMS-Hit on Release ensure we also have the Release.gpg file stored + // (previously an unknown pubkey) – but only if the Release file exists locally (unlikely + // event of InRelease removed from the mirror causing fallback but still an IMS-Hit) + if (TransactionManager->IMSHit == false || + (FileExists(Releasegpg) == false && FileExists(Release) == true)) + { + TransactionManager->TransactionStageCopy(this, DestFile, Releasegpg); + TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, Release); + } + } + else if (MetaIndex->Status != StatAuthError) + { + std::string const FinalFile = MetaIndex->GetFinalFilename(); + if (TransactionManager->IMSHit == false) + TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalFile); + else + TransactionManager->TransactionStageCopy(MetaIndex, FinalFile, FinalFile); + } +} + /*}}}*/ +void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/ +{ + Item::Failed(Message,Cnf); + + // check if we need to fail at this point + if (MetaIndex->AuthPass == true && MetaIndex->CheckStopAuthentication(this, Message)) + return; + + // ensures that a Release.gpg file in the lists/ is removed by the transaction + TransactionManager->TransactionStageRemoval(this, DestFile); + + // only allow going further if the user explicitly wants it + if (AllowInsecureRepositories(InsecureType::UNSIGNED, MetaIndex->Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true) + { + string const FinalRelease = MetaIndex->GetFinalFilename(); + string const FinalInRelease = TransactionManager->GetFinalFilename(); + LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease); + + // we parse the indexes here because at this point the user wanted + // a repository that may potentially harm him + bool const GoodLoad = TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile, &ErrorText); + if (MetaIndex->VerifyVendor(Message) == false) + /* expired Release files are still a problem you need extra force for */; + else + TransactionManager->QueueIndexes(GoodLoad); + + TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalRelease); + } + else if (TransactionManager->IMSHit == false) + Rename(MetaIndex->DestFile, MetaIndex->DestFile + ".FAILED"); + + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor + if (Cnf->LocalOnly == true || + StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) + { + // Ignore this + Status = StatDone; + } +} + /*}}}*/ + + +// AcqBaseIndex - Constructor /*{{{*/ +pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire * const Owner, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target) +: pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL) +{ +} + /*}}}*/ +void pkgAcqBaseIndex::Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/ +{ + pkgAcquire::Item::Failed(Message, Cnf); + if (Status != StatAuthError) + return; + + ErrorText.append("Release file created at: "); + auto const timespec = TransactionManager->MetaIndexParser->GetDate(); + if (timespec == 0) + ErrorText.append("<unknown>"); + else + ErrorText.append(TimeRFC1123(timespec, true)); + ErrorText.append("\n"); +} + /*}}}*/ +pkgAcqBaseIndex::~pkgAcqBaseIndex() {} + +// AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Get the DiffIndex file first and see if there are patches available + * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the + * patches. If anything goes wrong in that process, it will fall back to + * the original packages file + */ +pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target) + : pkgAcqIndex(Owner, TransactionManager, Target, true), d(NULL), diffs(NULL) +{ + // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire + ExpectedAdditionalItems = 40; + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + CompressionExtensions.clear(); + { + std::vector<std::string> types = APT::Configuration::getCompressionTypes(); + if (types.empty() == false) + { + std::ostringstream os; + std::copy_if(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "), [&](std::string const type) { + if (type == "uncompressed") + return true; + return TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey) + '.' + type); + }); + os << *types.rbegin(); + CompressionExtensions = os.str(); + } + } + Init(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Target.ShortDesc); + + if(Debug) + std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl; +} + /*}}}*/ +void pkgAcqDiffIndex::QueueOnIMSHit() const /*{{{*/ +{ + // list cleanup needs to know that this file as well as the already + // present index is ours, so we create an empty diff to save it for us + new pkgAcqIndexDiffs(Owner, TransactionManager, Target); +} + /*}}}*/ +static bool RemoveFileForBootstrapLinking(std::string &ErrorText, std::string const &For, std::string const &Boot)/*{{{*/ +{ + if (FileExists(Boot) && RemoveFile("Bootstrap-linking", Boot) == false) + { + strprintf(ErrorText, "Bootstrap for patching %s by removing stale %s failed!", For.c_str(), Boot.c_str()); + return false; + } + return true; +} + /*}}}*/ +bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ +{ + available_patches.clear(); + ExpectedAdditionalItems = 0; + // failing here is fine: our caller will take care of trying to + // get the complete file if patching fails + if(Debug) + std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile + << std::endl; + + FileFd Fd(IndexDiffFile, FileFd::ReadOnly, FileFd::Extension); + pkgTagFile TF(&Fd); + if (Fd.IsOpen() == false || Fd.Failed()) + return false; + + pkgTagSection Tags; + if(unlikely(TF.Step(Tags) == false)) + return false; + + HashStringList ServerHashes; + unsigned long long ServerSize = 0; + + auto const &posix = std::locale::classic(); + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + { + std::string tagname = *type; + tagname.append("-Current"); + std::string const tmp = Tags.FindS(tagname.c_str()); + if (tmp.empty() == true) + continue; + + string hash; + unsigned long long size; + std::stringstream ss(tmp); + ss.imbue(posix); + ss >> hash >> size; + if (unlikely(hash.empty() == true)) + continue; + if (unlikely(ServerSize != 0 && ServerSize != size)) + continue; + ServerHashes.push_back(HashString(*type, hash)); + ServerSize = size; + } + + if (ServerHashes.usable() == false) + { + ErrorText = "Did not find a good hashsum in the index"; + return false; + } + + std::string const CurrentPackagesFile = GetFinalFileNameFromURI(Target.URI); + HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey); + if (TargetFileHashes.usable() == false || ServerHashes != TargetFileHashes) + { + ErrorText = "Index has different hashes than parser (probably older)"; + return false; + } + + HashStringList LocalHashes; + // try avoiding calculating the hash here as this is costly + if (TransactionManager->LastMetaIndexParser != NULL) + LocalHashes = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey); + if (LocalHashes.usable() == false) + { + FileFd fd(CurrentPackagesFile, FileFd::ReadOnly, FileFd::Auto); + Hashes LocalHashesCalc(ServerHashes); + LocalHashesCalc.AddFD(fd); + LocalHashes = LocalHashesCalc.GetHashStringList(); + } + + if (ServerHashes == LocalHashes) + { + available_patches.clear(); + return true; + } + + if(Debug) + std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at " + << CurrentPackagesFile << " " << LocalHashes.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl; + + // historically, older hashes have more info than newer ones, so start + // collecting with older ones first to avoid implementing complicated + // information merging techniques… a failure is after all always + // recoverable with a complete file and hashes aren't changed that often. + std::vector<char const *> types; + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + types.push_back(*type); + + // parse all of (provided) history + bool firstAcceptedHashes = true; + for (auto type = types.crbegin(); type != types.crend(); ++type) + { + if (LocalHashes.find(*type) == NULL) + continue; + + std::string tagname = *type; + tagname.append("-History"); + std::string const tmp = Tags.FindS(tagname.c_str()); + if (tmp.empty() == true) + continue; + + string hash, filename; + unsigned long long size; + std::stringstream ss(tmp); + ss.imbue(posix); + + while (ss >> hash >> size >> filename) + { + if (unlikely(hash.empty() == true || filename.empty() == true)) + continue; + + // see if we have a record for this file already + std::vector<DiffInfo>::iterator cur = available_patches.begin(); + for (; cur != available_patches.end(); ++cur) + { + if (cur->file != filename) + continue; + cur->result_hashes.push_back(HashString(*type, hash)); + break; + } + if (cur != available_patches.end()) + continue; + if (firstAcceptedHashes == true) + { + DiffInfo next; + next.file = filename; + next.result_hashes.push_back(HashString(*type, hash)); + next.result_hashes.FileSize(size); + available_patches.push_back(next); + } + else + { + if (Debug == true) + std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename + << " wasn't in the list for the first parsed hash! (history)" << std::endl; + break; + } + } + firstAcceptedHashes = false; + } + + if (unlikely(available_patches.empty() == true)) + { + ErrorText = "Couldn't find any patches for the patch series"; + return false; + } + + for (auto type = types.crbegin(); type != types.crend(); ++type) + { + if (LocalHashes.find(*type) == NULL) + continue; + + std::string tagname = *type; + tagname.append("-Patches"); + std::string const tmp = Tags.FindS(tagname.c_str()); + if (tmp.empty() == true) + continue; + + string hash, filename; + unsigned long long size; + std::stringstream ss(tmp); + ss.imbue(posix); + + while (ss >> hash >> size >> filename) + { + if (unlikely(hash.empty() == true || filename.empty() == true)) + continue; + + // see if we have a record for this file already + std::vector<DiffInfo>::iterator cur = available_patches.begin(); + for (; cur != available_patches.end(); ++cur) + { + if (cur->file != filename) + continue; + if (cur->patch_hashes.empty()) + cur->patch_hashes.FileSize(size); + cur->patch_hashes.push_back(HashString(*type, hash)); + break; + } + if (cur != available_patches.end()) + continue; + if (Debug == true) + std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename + << " wasn't in the list for the first parsed hash! (patches)" << std::endl; + break; + } + } + + for (auto type = types.crbegin(); type != types.crend(); ++type) + { + std::string tagname = *type; + tagname.append("-Download"); + std::string const tmp = Tags.FindS(tagname.c_str()); + if (tmp.empty() == true) + continue; + + string hash, filename; + unsigned long long size; + std::stringstream ss(tmp); + ss.imbue(posix); + + // FIXME: all of pdiff supports only .gz compressed patches + while (ss >> hash >> size >> filename) + { + if (unlikely(hash.empty() == true || filename.empty() == true)) + continue; + if (unlikely(APT::String::Endswith(filename, ".gz") == false)) + continue; + filename.erase(filename.length() - 3); + + // see if we have a record for this file already + std::vector<DiffInfo>::iterator cur = available_patches.begin(); + for (; cur != available_patches.end(); ++cur) + { + if (cur->file != filename) + continue; + if (cur->download_hashes.empty()) + cur->download_hashes.FileSize(size); + cur->download_hashes.push_back(HashString(*type, hash)); + break; + } + if (cur != available_patches.end()) + continue; + if (Debug == true) + std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename + << " wasn't in the list for the first parsed hash! (download)" << std::endl; + break; + } + } + + + bool foundStart = false; + for (std::vector<DiffInfo>::iterator cur = available_patches.begin(); + cur != available_patches.end(); ++cur) + { + if (LocalHashes != cur->result_hashes) + continue; + + available_patches.erase(available_patches.begin(), cur); + foundStart = true; + break; + } + + if (foundStart == false || unlikely(available_patches.empty() == true)) + { + ErrorText = "Couldn't find the start of the patch series"; + return false; + } + + for (auto const &patch: available_patches) + if (patch.result_hashes.usable() == false || + patch.patch_hashes.usable() == false || + patch.download_hashes.usable() == false) + { + strprintf(ErrorText, "Provides no usable hashes for %s", patch.file.c_str()); + return false; + } + + // patching with too many files is rather slow compared to a fast download + unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0); + if (fileLimit != 0 && fileLimit < available_patches.size()) + { + strprintf(ErrorText, "Need %lu diffs, but limit is %lu", available_patches.size(), fileLimit); + return false; + } + + // calculate the size of all patches we have to get + unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100); + if (sizeLimitPercent > 0) + { + unsigned long long downloadSize = std::accumulate(available_patches.begin(), + available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) { + return T + I.download_hashes.FileSize(); + }); + if (downloadSize != 0) + { + unsigned long long downloadSizeIdx = 0; + auto const types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' '); + for (auto const &t : types) + { + std::string MetaKey = Target.MetaKey; + if (t != "uncompressed") + MetaKey += '.' + t; + HashStringList const hsl = GetExpectedHashesFor(MetaKey); + if (unlikely(hsl.usable() == false)) + continue; + downloadSizeIdx = hsl.FileSize(); + break; + } + unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent; + if ((sizeLimit/100) < downloadSize) + { + strprintf(ErrorText, "Need %llu compressed bytes, but limit is %llu and original is %llu", downloadSize, (sizeLimit/100), downloadSizeIdx); + return false; + } + } + } + + /* decide if we should download patches one by one or in one go: + The first is good if the server merges patches, but many don't so client + based merging can be attempt in which case the second is better. + "bad things" will happen if patches are merged on the server, + but client side merging is attempt as well */ + pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true); + if (pdiff_merge == true) + { + // reprepro adds this flag if it has merged patches on the server + std::string const precedence = Tags.FindS("X-Patch-Precedence"); + pdiff_merge = (precedence != "merged"); + } + + // clean the plate + { + std::string const Final = GetExistingFilename(CurrentPackagesFile); + if (unlikely(Final.empty())) // because we wouldn't be called in such a case + return false; + std::string const PartialFile = GetPartialFileNameFromURI(Target.URI); + std::string const PatchedFile = GetKeepCompressedFileName(PartialFile + "-patched", Target); + if (RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PartialFile) == false || + RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PatchedFile) == false) + return false; + for (auto const &ext : APT::Configuration::getCompressorExtensions()) + { + if (RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PartialFile + ext) == false || + RemoveFileForBootstrapLinking(ErrorText, CurrentPackagesFile, PatchedFile + ext) == false) + return false; + } + std::string const Ext = Final.substr(CurrentPackagesFile.length()); + std::string const Partial = PartialFile + Ext; + if (symlink(Final.c_str(), Partial.c_str()) != 0) + { + strprintf(ErrorText, "Bootstrap for patching by linking %s to %s failed!", Final.c_str(), Partial.c_str()); + return false; + } + } + + return true; +} + /*}}}*/ +void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/ +{ + if (CommonFailed(GetDiffIndexURI(Target), Message, Cnf)) + return; + + RenameOnError(PDiffError); + Status = StatDone; + ExpectedAdditionalItems = 0; + + if(Debug) + std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl + << "Falling back to normal index file acquire" << std::endl; + + new pkgAcqIndex(Owner, TransactionManager, Target); +} + /*}}}*/ +bool pkgAcqDiffIndex::VerifyDone(std::string const &Message, pkgAcquire::MethodConfig const * const)/*{{{*/ +{ + string const FinalFile = GetFinalFilename(); + if(StringToBool(LookupTag(Message,"IMS-Hit"),false)) + DestFile = FinalFile; + + if (ParseDiffIndex(DestFile)) + return true; + + Status = StatError; + if (ErrorText.empty()) + ErrorText = "Couldn't parse pdiff index"; + return false; +} + /*}}}*/ +void pkgAcqDiffIndex::Done(string const &Message,HashStringList const &Hashes, /*{{{*/ + pkgAcquire::MethodConfig const * const Cnf) +{ + if(Debug) + std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl; + + Item::Done(Message, Hashes, Cnf); + + if (available_patches.empty()) + { + // we have the same sha1 as the server so we are done here + if(Debug) + std::clog << "pkgAcqDiffIndex: Package file is up-to-date" << std::endl; + QueueOnIMSHit(); + } + else + { + if (pdiff_merge == false) + new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches); + else + { + diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size()); + for(size_t i = 0; i < available_patches.size(); ++i) + (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager, + Target, + available_patches[i], + diffs); + } + } + + TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); + + Complete = true; + Status = StatDone; + Dequeue(); + + return; +} + /*}}}*/ +pkgAcqDiffIndex::~pkgAcqDiffIndex() +{ + if (diffs != NULL) + delete diffs; +} + +// AcqIndexDiffs::AcqIndexDiffs - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* The package diff is added to the queue. one object is constructed + * for each diff and the index + */ +pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *const Owner, + pkgAcqMetaClearSig *const TransactionManager, + IndexTarget const &Target, + vector<DiffInfo> const &diffs) + : pkgAcqBaseIndex(Owner, TransactionManager, Target), + available_patches(diffs) +{ + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); + + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Desc.Owner = this; + Desc.ShortDesc = Target.ShortDesc; + + if(available_patches.empty() == true) + { + // we are done (yeah!), check hashes against the final file + DestFile = GetKeepCompressedFileName(GetFinalFileNameFromURI(Target.URI), Target); + Finish(true); + } + else + { + State = StateFetchDiff; + QueueNextDiff(); + } +} + /*}}}*/ +void pkgAcqIndexDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/ +{ + pkgAcqBaseIndex::Failed(Message,Cnf); + Status = StatDone; + + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); + if(Debug) + std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl + << "Falling back to normal index file acquire " << std::endl; + RenameOnError(PDiffError); + std::string const patchname = GetDiffsPatchFileName(DestFile); + if (RealFileExists(patchname)) + Rename(patchname, patchname + ".FAILED"); + std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI)); + if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile)) + Rename(UnpatchedFile, UnpatchedFile + ".FAILED"); + new pkgAcqIndex(Owner, TransactionManager, Target); + Finish(); +} + /*}}}*/ +// Finish - helper that cleans the item out of the fetcher queue /*{{{*/ +void pkgAcqIndexDiffs::Finish(bool allDone) +{ + if(Debug) + std::clog << "pkgAcqIndexDiffs::Finish(): " + << allDone << " " + << Desc.URI << std::endl; + + // we restore the original name, this is required, otherwise + // the file will be cleaned + if(allDone) + { + std::string const Final = GetKeepCompressedFileName(GetFinalFilename(), Target); + TransactionManager->TransactionStageCopy(this, DestFile, Final); + + // this is for the "real" finish + Complete = true; + Status = StatDone; + Dequeue(); + if(Debug) + std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl; + return; + } + else + DestFile.clear(); + + if(Debug) + std::clog << "Finishing: " << Desc.URI << std::endl; + Complete = false; + Status = StatDone; + Dequeue(); + return; +} + /*}}}*/ +bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ +{ + // calc sha1 of the just patched file + std::string const PartialFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI)); + if(unlikely(PartialFile.empty())) + { + Failed("Message: The file " + GetPartialFileNameFromURI(Target.URI) + " isn't available", NULL); + return false; + } + + FileFd fd(PartialFile, FileFd::ReadOnly, FileFd::Extension); + Hashes LocalHashesCalc; + LocalHashesCalc.AddFD(fd); + HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList(); + + if(Debug) + std::clog << "QueueNextDiff: " << PartialFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl; + + HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey); + if (unlikely(LocalHashes.usable() == false || TargetFileHashes.usable() == false)) + { + Failed("Local/Expected hashes are not usable for " + PartialFile, NULL); + return false; + } + + // final file reached before all patches are applied + if(LocalHashes == TargetFileHashes) + { + Finish(true); + return true; + } + + // remove all patches until the next matching patch is found + // this requires the Index file to be ordered + available_patches.erase(available_patches.begin(), + std::find_if(available_patches.begin(), available_patches.end(), [&](DiffInfo const &I) { + return I.result_hashes == LocalHashes; + })); + + // error checking and falling back if no patch was found + if(available_patches.empty() == true) + { + Failed("No patches left to reach target for " + PartialFile, NULL); + return false; + } + + // queue the right diff + Desc.URI = Target.URI + ".diff/" + available_patches[0].file + ".gz"; + Desc.Description = Target.Description + " " + available_patches[0].file + string(".pdiff"); + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI + ".diff/" + available_patches[0].file), Target); + + if(Debug) + std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl; + + QueueURI(Desc); + + return true; +} + /*}}}*/ +void pkgAcqIndexDiffs::Done(string const &Message, HashStringList const &Hashes, /*{{{*/ + pkgAcquire::MethodConfig const * const Cnf) +{ + if (Debug) + std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl; + + Item::Done(Message, Hashes, Cnf); + + std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI); + std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile); + std::string const PatchFile = GetDiffsPatchFileName(UnpatchedFile); + std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target); + + switch (State) + { + // success in downloading a diff, enter ApplyDiff state + case StateFetchDiff: + Rename(DestFile, PatchFile); + DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target); + if(Debug) + std::clog << "Sending to rred method: " << UnpatchedFile << std::endl; + State = StateApplyDiff; + Local = true; + Desc.URI = "rred:" + UnpatchedFile; + QueueURI(Desc); + SetActiveSubprocess("rred"); + return; + // success in download/apply a diff, queue next (if needed) + case StateApplyDiff: + // remove the just applied patch and base file + available_patches.erase(available_patches.begin()); + RemoveFile("pkgAcqIndexDiffs::Done", PatchFile); + RemoveFile("pkgAcqIndexDiffs::Done", UnpatchedFile); + if(Debug) + std::clog << "Moving patched file in place: " << std::endl + << DestFile << " -> " << PatchedFile << std::endl; + Rename(DestFile, PatchedFile); + + // see if there is more to download + if(available_patches.empty() == false) + { + new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches); + Finish(); + } else { + DestFile = PatchedFile; + Finish(true); + } + return; + } +} + /*}}}*/ +std::string pkgAcqIndexDiffs::Custom600Headers() const /*{{{*/ +{ + if(State != StateApplyDiff) + return pkgAcqBaseIndex::Custom600Headers(); + std::ostringstream patchhashes; + for (auto && hs : available_patches[0].result_hashes) + patchhashes << "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue(); + for (auto && hs : available_patches[0].patch_hashes) + patchhashes << "\nPatch-0-" << hs.HashType() << "-Hash: " << hs.HashValue(); + patchhashes << pkgAcqBaseIndex::Custom600Headers(); + return patchhashes.str(); +} + /*}}}*/ +pkgAcqIndexDiffs::~pkgAcqIndexDiffs() {} + +// AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ +pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *const Owner, + pkgAcqMetaClearSig *const TransactionManager, + IndexTarget const &Target, + DiffInfo const &patch, + std::vector<pkgAcqIndexMergeDiffs *> const *const allPatches) + : pkgAcqBaseIndex(Owner, TransactionManager, Target), + patch(patch), allPatches(allPatches), State(StateFetchDiff) +{ + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Desc.Owner = this; + Desc.ShortDesc = Target.ShortDesc; + Desc.URI = Target.URI + ".diff/" + patch.file + ".gz"; + Desc.Description = Target.Description + " " + patch.file + ".pdiff"; + DestFile = GetPartialFileNameFromURI(Target.URI + ".diff/" + patch.file + ".gz"); + + if(Debug) + std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl; + + QueueURI(Desc); +} + /*}}}*/ +void pkgAcqIndexMergeDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/ +{ + if(Debug) + std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl; + + pkgAcqBaseIndex::Failed(Message,Cnf); + Status = StatDone; + + // check if we are the first to fail, otherwise we are done here + State = StateDoneDiff; + for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + if ((*I)->State == StateErrorDiff) + { + State = StateErrorDiff; + return; + } + + // first failure means we should fallback + State = StateErrorDiff; + if (Debug) + std::clog << "Falling back to normal index file acquire" << std::endl; + RenameOnError(PDiffError); + std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI)); + if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile)) + Rename(UnpatchedFile, UnpatchedFile + ".FAILED"); + DestFile.clear(); + new pkgAcqIndex(Owner, TransactionManager, Target); +} + /*}}}*/ +void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Hashes, /*{{{*/ + pkgAcquire::MethodConfig const * const Cnf) +{ + if(Debug) + std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl; + + Item::Done(Message, Hashes, Cnf); + + if (std::any_of(allPatches->begin(), allPatches->end(), + [](pkgAcqIndexMergeDiffs const * const P) { return P->State == StateErrorDiff; })) + { + if(Debug) + std::clog << "Another patch failed already, no point in processing this one." << std::endl; + State = StateErrorDiff; + return; + } + + std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI); + std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile); + if (UnpatchedFile.empty()) + { + _error->Fatal("Unpatched file %s doesn't exist (anymore)!", UncompressedUnpatchedFile.c_str()); + State = StateErrorDiff; + return; + } + std::string const PatchFile = GetMergeDiffsPatchFileName(UnpatchedFile, patch.file); + std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target); + + switch (State) + { + case StateFetchDiff: + Rename(DestFile, PatchFile); + + // check if this is the last completed diff + State = StateDoneDiff; + for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + if ((*I)->State != StateDoneDiff) + { + if(Debug) + std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl; + return; + } + // this is the last completed diff, so we are ready to apply now + DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target); + if(Debug) + std::clog << "Sending to rred method: " << UnpatchedFile << std::endl; + State = StateApplyDiff; + Local = true; + Desc.URI = "rred:" + UnpatchedFile; + QueueURI(Desc); + SetActiveSubprocess("rred"); + return; + case StateApplyDiff: + // success in download & apply all diffs, finialize and clean up + if(Debug) + std::clog << "Queue patched file in place: " << std::endl + << DestFile << " -> " << PatchedFile << std::endl; + + // queue for copy by the transaction manager + TransactionManager->TransactionStageCopy(this, DestFile, GetKeepCompressedFileName(GetFinalFilename(), Target)); + + // ensure the ed's are gone regardless of list-cleanup + for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + RemoveFile("pkgAcqIndexMergeDiffs::Done", GetMergeDiffsPatchFileName(UnpatchedFile, (*I)->patch.file)); + RemoveFile("pkgAcqIndexMergeDiffs::Done", UnpatchedFile); + + // all set and done + Complete = true; + if(Debug) + std::clog << "allDone: " << DestFile << "\n" << std::endl; + return; + case StateDoneDiff: _error->Fatal("Done called for %s which is in an invalid Done state", PatchFile.c_str()); break; + case StateErrorDiff: _error->Fatal("Done called for %s which is in an invalid Error state", PatchFile.c_str()); break; + } +} + /*}}}*/ +std::string pkgAcqIndexMergeDiffs::Custom600Headers() const /*{{{*/ +{ + if(State != StateApplyDiff) + return pkgAcqBaseIndex::Custom600Headers(); + std::ostringstream patchhashes; + unsigned int seen_patches = 0; + for (auto && hs : (*allPatches)[0]->patch.result_hashes) + patchhashes << "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue(); + for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + { + HashStringList const ExpectedHashes = (*I)->patch.patch_hashes; + for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs) + patchhashes << "\nPatch-" << std::to_string(seen_patches) << "-" << hs->HashType() << "-Hash: " << hs->HashValue(); + ++seen_patches; + } + patchhashes << pkgAcqBaseIndex::Custom600Headers(); + return patchhashes.str(); +} + /*}}}*/ +pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {} + +// AcqIndex::AcqIndex - Constructor /*{{{*/ +pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, bool const Derived) + : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD), + CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES)) +{ + if (Derived) + return; + Init(Target.URI, Target.Description, Target.ShortDesc); + + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgIndex with TransactionManager " + << TransactionManager << std::endl; +} + /*}}}*/ +// AcqIndex::Init - deferred Constructor /*{{{*/ +void pkgAcqIndex::Init(string const &URI, string const &URIDesc, + string const &ShortDesc) +{ + Stage = STAGE_DOWNLOAD; + + DestFile = GetPartialFileNameFromURI(URI); + size_t const nextExt = CompressionExtensions.find(' '); + if (nextExt == std::string::npos) + { + CurrentCompressionExtension = CompressionExtensions; + CompressionExtensions.clear(); + } + else + { + CurrentCompressionExtension = CompressionExtensions.substr(0, nextExt); + CompressionExtensions = CompressionExtensions.substr(nextExt+1); + } + + if (CurrentCompressionExtension == "uncompressed") + { + Desc.URI = URI; + } + else if (unlikely(CurrentCompressionExtension.empty())) + return; + else + { + Desc.URI = URI + '.' + CurrentCompressionExtension; + DestFile = DestFile + '.' + CurrentCompressionExtension; + } + + // store file size of the download to ensure the fetcher gives + // accurate progress reporting + FileSize = GetExpectedHashes().FileSize(); + + Desc.Description = URIDesc; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + + QueueURI(Desc); +} + /*}}}*/ +// AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ +// --------------------------------------------------------------------- +/* The only header we use is the last-modified header. */ +string pkgAcqIndex::Custom600Headers() const +{ + + string msg = "\nIndex-File: true"; + + if (TransactionManager->LastMetaIndexParser == NULL) + { + std::string const Final = GetFinalFilename(); + + struct stat Buf; + if (stat(Final.c_str(),&Buf) == 0) + msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false); + } + + if(Target.IsOptional) + msg += "\nFail-Ignore: true"; + + return msg; +} + /*}}}*/ +// AcqIndex::Failed - getting the indexfile failed /*{{{*/ +bool pkgAcqIndex::CommonFailed(std::string const &TargetURI, + std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) +{ + pkgAcqBaseIndex::Failed(Message,Cnf); + // authorisation matches will not be fixed by other compression types + if (Status != StatAuthError) + { + if (CompressionExtensions.empty() == false) + { + Init(TargetURI, Desc.Description, Desc.ShortDesc); + Status = StatIdle; + return true; + } + } + return false; +} +void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) +{ + if (CommonFailed(Target.URI, Message, Cnf)) + return; + + if(Target.IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD) + Status = StatDone; + else + TransactionManager->AbortTransaction(); +} + /*}}}*/ +// AcqIndex::Done - Finished a fetch /*{{{*/ +// --------------------------------------------------------------------- +/* This goes through a number of states.. On the initial fetch the + method could possibly return an alternate filename which points + to the uncompressed version of the file. If this is so the file + is copied into the partial directory. In all other cases the file + is decompressed with a compressed uri. */ +void pkgAcqIndex::Done(string const &Message, + HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cfg) +{ + Item::Done(Message,Hashes,Cfg); + + switch(Stage) + { + case STAGE_DOWNLOAD: + StageDownloadDone(Message); + break; + case STAGE_DECOMPRESS_AND_VERIFY: + StageDecompressDone(); + break; + } +} + /*}}}*/ +// AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/ +void pkgAcqIndex::StageDownloadDone(string const &Message) +{ + Local = true; + Complete = true; + + std::string const AltFilename = LookupTag(Message,"Alt-Filename"); + std::string Filename = LookupTag(Message,"Filename"); + + // we need to verify the file against the current Release file again + // on if-modfied-since hit to avoid a stale attack against us + if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + { + // copy FinalFile into partial/ so that we check the hash again + string const FinalFile = GetExistingFilename(GetFinalFileNameFromURI(Target.URI)); + if (symlink(FinalFile.c_str(), DestFile.c_str()) != 0) + _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking final file %s back to %s failed", FinalFile.c_str(), DestFile.c_str()); + else + { + EraseFileName = DestFile; + Filename = DestFile; + } + Stage = STAGE_DECOMPRESS_AND_VERIFY; + Desc.URI = "store:" + Filename; + QueueURI(Desc); + SetActiveSubprocess(::URI(Desc.URI).Access); + return; + } + // methods like file:// give us an alternative (uncompressed) file + else if (Target.KeepCompressed == false && AltFilename.empty() == false) + { + Filename = AltFilename; + EraseFileName.clear(); + } + // Methods like e.g. "file:" will give us a (compressed) FileName that is + // not the "DestFile" we set, in this case we uncompress from the local file + else if (Filename != DestFile && RealFileExists(DestFile) == false) + { + // symlinking ensures that the filename can be used for compression detection + // that is e.g. needed for by-hash which has no extension over file + if (symlink(Filename.c_str(),DestFile.c_str()) != 0) + _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str()); + else + { + EraseFileName = DestFile; + Filename = DestFile; + } + } + + Stage = STAGE_DECOMPRESS_AND_VERIFY; + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); + if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile)) + Desc.URI = "copy:" + Filename; + else + Desc.URI = "store:" + Filename; + if (DestFile == Filename) + { + if (CurrentCompressionExtension == "uncompressed") + return StageDecompressDone(); + DestFile = "/dev/null"; + } + + if (EraseFileName.empty() && Filename != AltFilename) + EraseFileName = Filename; + + // queue uri for the next stage + QueueURI(Desc); + SetActiveSubprocess(::URI(Desc.URI).Access); +} + /*}}}*/ +// AcqIndex::StageDecompressDone - Final verification /*{{{*/ +void pkgAcqIndex::StageDecompressDone() +{ + if (DestFile == "/dev/null") + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); + + // Done, queue for rename on transaction finished + TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); +} + /*}}}*/ +pkgAcqIndex::~pkgAcqIndex() {} + +// AcqArchive::AcqArchive - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* This just sets up the initial fetch environment and queues the first + possibilitiy */ +APT_IGNORE_DEPRECATED_PUSH +pkgAcqArchive::pkgAcqArchive(pkgAcquire *const Owner, pkgSourceList *const Sources, + pkgRecords *const Recs, pkgCache::VerIterator const &Version, + string &StoreFilename) : Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs), + StoreFilename(StoreFilename), Vf(), + Trusted(false) +{ + if (Version.Arch() == 0) + { + _error->Error(_("I wasn't able to locate a file for the %s package. " + "This might mean you need to manually fix this package. " + "(due to missing arch)"), + Version.ParentPkg().FullName().c_str()); + return; + } + + // check if we have one trusted source for the package. if so, switch + // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode + bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false); + bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false); + bool seenUntrusted = false; + for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i) + { + pkgIndexFile *Index; + if (Sources->FindIndex(i.File(),Index) == false) + continue; + + if (debugAuth == true) + std::cerr << "Checking index: " << Index->Describe() + << "(Trusted=" << Index->IsTrusted() << ")" << std::endl; + + if (Index->IsTrusted() == true) + { + Trusted = true; + if (allowUnauth == false) + break; + } + else + seenUntrusted = true; + } + + // "allow-unauthenticated" restores apts old fetching behaviour + // that means that e.g. unauthenticated file:// uris are higher + // priority than authenticated http:// uris + if (allowUnauth == true && seenUntrusted == true) + Trusted = false; + + StoreFilename.clear(); + std::set<string> targetComponents, targetCodenames, targetSuites; + std::vector<std::unique_ptr<FileFd>> authconfs; + for (auto Vf = Version.FileList(); Vf.end() == false; ++Vf) + { + auto const PkgF = Vf.File(); + if (unlikely(PkgF.end())) + continue; + if (PkgF.Flagged(pkgCache::Flag::NotSource)) + continue; + if (PkgF.Flagged(pkgCache::Flag::PackagesRequireAuthorization) && !IsAuthorized(PkgF, authconfs)) + continue; + pkgIndexFile *Index; + if (Sources->FindIndex(PkgF, Index) == false) + continue; + if (Trusted && Index->IsTrusted() == false) + continue; + + pkgRecords::Parser &Parse = Recs->Lookup(Vf); + // collect the hashes from the indexes + auto hsl = Parse.Hashes(); + if (ExpectedHashes.empty()) + ExpectedHashes = hsl; + else + { + // bad things will likely happen, but the user might be "lucky" still + // if the sources provide the same hashtypes (so that they aren't mixed) + for (auto const &hs : hsl) + if (ExpectedHashes.push_back(hs) == false) + { + _error->Warning("Sources disagree on hashes for supposely identical version '%s' of '%s'.", + Version.VerStr(), Version.ParentPkg().FullName(false).c_str()); + break; + } + } + // only allow local volatile sources to have no hashes + if (PkgF.Flagged(pkgCache::Flag::LocalSource)) + LocalSource = true; + else if (hsl.empty()) + continue; + + std::string poolfilename = Parse.FileName(); + if (poolfilename.empty()) + continue; + + std::remove_reference<decltype(ModifyCustomFields())>::type fields; + { + auto const debIndex = dynamic_cast<pkgDebianIndexTargetFile const *const>(Index); + if (debIndex != nullptr) + { + auto const IT = debIndex->GetIndexTarget(); + fields.emplace("Target-Repo-URI", IT.Option(IndexTarget::REPO_URI)); + fields.emplace("Target-Release", IT.Option(IndexTarget::RELEASE)); + fields.emplace("Target-Site", IT.Option(IndexTarget::SITE)); + } + } + fields.emplace("Target-Base-URI", Index->ArchiveURI("")); + if (PkgF->Component != 0) + fields.emplace("Target-Component", PkgF.Component()); + auto const RelF = PkgF.ReleaseFile(); + if (RelF.end() == false) + { + if (RelF->Codename != 0) + fields.emplace("Target-Codename", RelF.Codename()); + if (RelF->Archive != 0) + fields.emplace("Target-Suite", RelF.Archive()); + } + fields.emplace("Target-Architecture", Version.Arch()); + fields.emplace("Target-Type", flExtension(poolfilename)); + + if (StoreFilename.empty()) + { + /* We pick a filename based on the information we have for the version, + but we don't know what extension such a file should have, so we look + at the filenames used online and assume that they are the same for + all repositories containing this file */ + StoreFilename = QuoteString(Version.ParentPkg().Name(), "_:") + '_' + + QuoteString(Version.VerStr(), "_:") + '_' + + QuoteString(Version.Arch(), "_:.") + + "." + flExtension(poolfilename); + + Desc.URI = Index->ArchiveURI(poolfilename); + Desc.Description = Index->ArchiveInfo(Version); + Desc.Owner = this; + Desc.ShortDesc = Version.ParentPkg().FullName(true); + auto &customfields = ModifyCustomFields(); + for (auto const &f : fields) + customfields[f.first] = f.second; + FileSize = Version->Size; + } + else + PushAlternativeURI(Index->ArchiveURI(poolfilename), std::move(fields), true); + } + if (StoreFilename.empty()) + { + _error->Error(_("Can't find a source to download version '%s' of '%s'"), + Version.VerStr(), Version.ParentPkg().FullName(false).c_str()); + return; + } + + // Check if we already downloaded the file + struct stat Buf; + auto FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename); + if (stat(FinalFile.c_str(), &Buf) == 0) + { + // Make sure the size matches + if ((unsigned long long)Buf.st_size == Version->Size) + { + Complete = true; + Local = true; + Status = StatDone; + StoreFilename = DestFile = FinalFile; + return; + } + + /* Hmm, we have a file and its size does not match, this shouldn't + happen.. */ + RemoveFile("pkgAcqArchive::QueueNext", FinalFile); + } + + // Check the destination file + DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename); + if (stat(DestFile.c_str(), &Buf) == 0) + { + // Hmm, the partial file is too big, erase it + if ((unsigned long long)Buf.st_size > Version->Size) + RemoveFile("pkgAcqArchive::QueueNext", DestFile); + else + PartialSize = Buf.st_size; + } + + // Disables download of archives - useful if no real installation follows, + // e.g. if we are just interested in proposed installation order + if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true) + { + Complete = true; + Local = true; + Status = StatDone; + StoreFilename = DestFile = FinalFile; + return; + } + + // Create the item + Local = false; + QueueURI(Desc); +} +APT_IGNORE_DEPRECATED_POP + /*}}}*/ +bool pkgAcqArchive::QueueNext() /*{{{*/ +{ + return false; +} + /*}}}*/ +// AcqArchive::Done - Finished fetching /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cfg) +{ + Item::Done(Message, Hashes, Cfg); + + // Grab the output filename + std::string const FileName = LookupTag(Message,"Filename"); + if (DestFile != FileName && RealFileExists(DestFile) == false) + { + StoreFilename = DestFile = FileName; + Local = true; + Complete = true; + return; + } + + // Done, move it into position + string const FinalFile = GetFinalFilename(); + Rename(DestFile,FinalFile); + StoreFilename = DestFile = FinalFile; + Complete = true; +} + /*}}}*/ +// AcqArchive::Failed - Failure handler /*{{{*/ +// --------------------------------------------------------------------- +/* Here we try other sources */ +void pkgAcqArchive::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) +{ + Item::Failed(Message,Cnf); +} + /*}}}*/ +APT_PURE bool pkgAcqArchive::IsTrusted() const /*{{{*/ +{ + return Trusted; +} + /*}}}*/ +void pkgAcqArchive::Finished() /*{{{*/ +{ + if (Status == pkgAcquire::Item::StatDone && + Complete == true) + return; + StoreFilename = string(); +} + /*}}}*/ +std::string pkgAcqArchive::DescURI() const /*{{{*/ +{ + return Desc.URI; +} + /*}}}*/ +std::string pkgAcqArchive::ShortDesc() const /*{{{*/ +{ + return Desc.ShortDesc; +} + /*}}}*/ +pkgAcqArchive::~pkgAcqArchive() {} + +// AcqChangelog::pkgAcqChangelog - Constructors /*{{{*/ +class pkgAcqChangelog::Private +{ + public: + std::string FinalFile; +}; +pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver, + std::string const &DestDir, std::string const &DestFilename) : + pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(Ver.SourcePkgName()), SrcVersion(Ver.SourceVerStr()) +{ + Desc.URI = URI(Ver); + Init(DestDir, DestFilename); +} +// some parameters are char* here as they come likely from char* interfaces – which can also return NULL +pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &RlsFile, + char const * const Component, char const * const SrcName, char const * const SrcVersion, + const string &DestDir, const string &DestFilename) : + pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion) +{ + Desc.URI = URI(RlsFile, Component, SrcName, SrcVersion); + Init(DestDir, DestFilename); +} +pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, + std::string const &URI, char const * const SrcName, char const * const SrcVersion, + const string &DestDir, const string &DestFilename) : + pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion) +{ + Desc.URI = URI; + Init(DestDir, DestFilename); +} +void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFilename) +{ + if (Desc.URI.empty()) + { + Status = StatError; + // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1 + strprintf(ErrorText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str()); + // Let the error message print something sensible rather than "Failed to fetch /" + if (DestFilename.empty()) + DestFile = SrcName + ".changelog"; + else + DestFile = DestFilename; + Desc.URI = "changelog:/" + DestFile; + return; + } + + std::string DestFileName; + if (DestFilename.empty()) + DestFileName = flCombine(DestFile, SrcName + ".changelog"); + else + DestFileName = flCombine(DestFile, DestFilename); + + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + std::string const systemTemp = GetTempDir(SandboxUser); + char tmpname[1000]; + snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", systemTemp.c_str()); + if (NULL == mkdtemp(tmpname)) + { + _error->Errno("mkdtemp", "mkdtemp failed in changelog acquire of %s %s", SrcName.c_str(), SrcVersion.c_str()); + Status = StatError; + return; + } + TemporaryDirectory = tmpname; + + ChangeOwnerAndPermissionOfFile("pkgAcqChangelog::Init", TemporaryDirectory.c_str(), + SandboxUser.c_str(), ROOT_GROUP, 0700); + + DestFile = flCombine(TemporaryDirectory, DestFileName); + if (DestDir.empty() == false) + { + d->FinalFile = flCombine(DestDir, DestFileName); + if (RealFileExists(d->FinalFile)) + { + FileFd file1, file2; + if (file1.Open(DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive) && + file2.Open(d->FinalFile, FileFd::ReadOnly) && CopyFile(file2, file1)) + { + ChangeOwnerAndPermissionOfFile("pkgAcqChangelog::Init", DestFile.c_str(), "root", ROOT_GROUP, 0644); + struct timeval times[2]; + times[0].tv_sec = times[1].tv_sec = file2.ModificationTime(); + times[0].tv_usec = times[1].tv_usec = 0; + utimes(DestFile.c_str(), times); + } + } + } + + Desc.ShortDesc = "Changelog"; + strprintf(Desc.Description, "%s %s %s Changelog", URI::SiteOnly(Desc.URI).c_str(), SrcName.c_str(), SrcVersion.c_str()); + Desc.Owner = this; + QueueURI(Desc); +} + /*}}}*/ +std::string pkgAcqChangelog::URI(pkgCache::VerIterator const &Ver) /*{{{*/ +{ + std::string const confOnline = "Acquire::Changelogs::AlwaysOnline"; + bool AlwaysOnline = _config->FindB(confOnline, false); + if (AlwaysOnline == false) + for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF) + { + pkgCache::PkgFileIterator const PF = VF.File(); + if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0) + continue; + pkgCache::RlsFileIterator const RF = PF.ReleaseFile(); + if (RF->Origin != 0 && _config->FindB(confOnline + "::Origin::" + RF.Origin(), false)) + { + AlwaysOnline = true; + break; + } + } + if (AlwaysOnline == false) + { + pkgCache::PkgIterator const Pkg = Ver.ParentPkg(); + if (Pkg->CurrentVer != 0 && Pkg.CurrentVer() == Ver) + { + std::string const root = _config->FindDir("Dir"); + std::string const basename = root + std::string("usr/share/doc/") + Pkg.Name() + "/changelog"; + std::string const debianname = basename + ".Debian"; + if (FileExists(debianname)) + return "copy://" + debianname; + else if (FileExists(debianname + ".gz")) + return "store://" + debianname + ".gz"; + else if (FileExists(basename)) + return "copy://" + basename; + else if (FileExists(basename + ".gz")) + return "store://" + basename + ".gz"; + } + } + + char const * const SrcName = Ver.SourcePkgName(); + char const * const SrcVersion = Ver.SourceVerStr(); + // find the first source for this version which promises a changelog + for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF) + { + pkgCache::PkgFileIterator const PF = VF.File(); + if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0) + continue; + pkgCache::RlsFileIterator const RF = PF.ReleaseFile(); + std::string const uri = URI(RF, PF.Component(), SrcName, SrcVersion); + if (uri.empty()) + continue; + return uri; + } + return ""; +} +std::string pkgAcqChangelog::URITemplate(pkgCache::RlsFileIterator const &Rls) +{ + if (Rls.end() == true || (Rls->Label == 0 && Rls->Origin == 0)) + return ""; + std::string const serverConfig = "Acquire::Changelogs::URI"; + std::string server; +#define APT_EMPTY_SERVER \ + if (server.empty() == false) \ + { \ + if (server != "no") \ + return server; \ + return ""; \ + } +#define APT_CHECK_SERVER(X, Y) \ + if (Rls->X != 0) \ + { \ + std::string const specialServerConfig = serverConfig + "::" + Y + #X + "::" + Rls.X(); \ + server = _config->Find(specialServerConfig); \ + APT_EMPTY_SERVER \ + } + // this way e.g. Debian-Security can fallback to Debian + APT_CHECK_SERVER(Label, "Override::") + APT_CHECK_SERVER(Origin, "Override::") + + if (RealFileExists(Rls.FileName())) + { + _error->PushToStack(); + FileFd rf; + /* This can be costly. A caller wanting to get millions of URIs might + want to do this on its own once and use Override settings. + We don't do this here as Origin/Label are not as unique as they + should be so this could produce request order-dependent anomalies */ + if (OpenMaybeClearSignedFile(Rls.FileName(), rf) == true) + { + pkgTagFile TagFile(&rf, rf.Size()); + pkgTagSection Section; + if (TagFile.Step(Section) == true) + server = Section.FindS("Changelogs"); + } + _error->RevertToStack(); + APT_EMPTY_SERVER + } + + APT_CHECK_SERVER(Label, "") + APT_CHECK_SERVER(Origin, "") +#undef APT_CHECK_SERVER +#undef APT_EMPTY_SERVER + return ""; +} +std::string pkgAcqChangelog::URI(pkgCache::RlsFileIterator const &Rls, + char const * const Component, char const * const SrcName, + char const * const SrcVersion) +{ + return URI(URITemplate(Rls), Component, SrcName, SrcVersion); +} +std::string pkgAcqChangelog::URI(std::string const &Template, + char const * const Component, char const * const SrcName, + char const * const SrcVersion) +{ + if (Template.find("@CHANGEPATH@") == std::string::npos) + return ""; + + // the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. main/a/apt/apt_1.1 or contrib/liba/libapt/libapt_2.0 + std::string Src = SrcName; + std::string path = APT::String::Startswith(SrcName, "lib") ? Src.substr(0, 4) : Src.substr(0,1); + path.append("/").append(Src).append("/"); + path.append(Src).append("_").append(StripEpoch(SrcVersion)); + // we omit component for releases without one (= flat-style repositories) + if (Component != NULL && strlen(Component) != 0) + path = std::string(Component) + "/" + path; + + return SubstVar(Template, "@CHANGEPATH@", path); +} + /*}}}*/ +// AcqChangelog::Failed - Failure handler /*{{{*/ +void pkgAcqChangelog::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf) +{ + Item::Failed(Message,Cnf); + + std::string errText; + // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1 + strprintf(errText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str()); + + // Error is probably something techy like 404 Not Found + if (ErrorText.empty()) + ErrorText = errText; + else + ErrorText = errText + " (" + ErrorText + ")"; +} + /*}}}*/ +// AcqChangelog::Done - Item downloaded OK /*{{{*/ +void pkgAcqChangelog::Done(string const &Message,HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const * const Cnf) +{ + Item::Done(Message,CalcHashes,Cnf); + if (d->FinalFile.empty() == false) + { + if (RemoveFile("pkgAcqChangelog::Done", d->FinalFile) == false || + Rename(DestFile, d->FinalFile) == false) + Status = StatError; + } + + Complete = true; +} + /*}}}*/ +pkgAcqChangelog::~pkgAcqChangelog() /*{{{*/ +{ + if (TemporaryDirectory.empty() == false) + { + RemoveFile("~pkgAcqChangelog", DestFile); + rmdir(TemporaryDirectory.c_str()); + } + delete d; +} + /*}}}*/ + +// AcqFile::pkgAcqFile - Constructor /*{{{*/ +APT_IGNORE_DEPRECATED_PUSH +pkgAcqFile::pkgAcqFile(pkgAcquire *const Owner, string const &URI, HashStringList const &Hashes, + unsigned long long const Size, string const &Dsc, string const &ShortDesc, + const string &DestDir, const string &DestFilename, + bool const IsIndexFile) : Item(Owner), d(NULL), Retries(0), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes) +{ + if(!DestFilename.empty()) + DestFile = DestFilename; + else if(!DestDir.empty()) + DestFile = DestDir + "/" + flNotDir(URI); + else + DestFile = flNotDir(URI); + + // Create the item + Desc.URI = URI; + Desc.Description = Dsc; + Desc.Owner = this; + + // Set the short description to the archive component + Desc.ShortDesc = ShortDesc; + + // Get the transfer sizes + FileSize = Size; + struct stat Buf; + if (stat(DestFile.c_str(),&Buf) == 0) + { + // Hmm, the partial file is too big, erase it + if ((Size > 0) && (unsigned long long)Buf.st_size > Size) + RemoveFile("pkgAcqFile", DestFile); + else + PartialSize = Buf.st_size; + } + + QueueURI(Desc); +} +APT_IGNORE_DEPRECATED_POP + /*}}}*/ +// AcqFile::Done - Item downloaded OK /*{{{*/ +void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const * const Cnf) +{ + Item::Done(Message,CalcHashes,Cnf); + + std::string const FileName = LookupTag(Message,"Filename"); + Complete = true; + + // The files timestamp matches + if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + return; + + // We have to copy it into place + if (RealFileExists(DestFile.c_str()) == false) + { + Local = true; + if (_config->FindB("Acquire::Source-Symlinks",true) == false || + Cnf->Removable == true) + { + Desc.URI = "copy:" + FileName; + QueueURI(Desc); + return; + } + + // Erase the file if it is a symlink so we can overwrite it + struct stat St; + if (lstat(DestFile.c_str(),&St) == 0) + { + if (S_ISLNK(St.st_mode) != 0) + RemoveFile("pkgAcqFile::Done", DestFile); + } + + // Symlink the file + if (symlink(FileName.c_str(),DestFile.c_str()) != 0) + { + _error->PushToStack(); + _error->Errno("pkgAcqFile::Done", "Symlinking file %s failed", DestFile.c_str()); + std::stringstream msg; + _error->DumpErrors(msg, GlobalError::DEBUG, false); + _error->RevertToStack(); + ErrorText = msg.str(); + Status = StatError; + Complete = false; + } + } +} + /*}}}*/ +void pkgAcqFile::Failed(string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/ +{ + // FIXME: Remove this pointless overload on next ABI break + Item::Failed(Message, Cnf); +} + /*}}}*/ +string pkgAcqFile::Custom600Headers() const /*{{{*/ +{ + if (IsIndexFile) + return "\nIndex-File: true"; + return ""; +} + /*}}}*/ +pkgAcqFile::~pkgAcqFile() {} + +void pkgAcqAuxFile::Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/ +{ + pkgAcqFile::Failed(Message, Cnf); + if (Status == StatIdle) + return; + if (RealFileExists(DestFile)) + Rename(DestFile, DestFile + ".FAILED"); + Worker->ReplyAux(Desc); +} + /*}}}*/ +void pkgAcqAuxFile::Done(std::string const &Message, HashStringList const &CalcHashes, /*{{{*/ + pkgAcquire::MethodConfig const *const Cnf) +{ + pkgAcqFile::Done(Message, CalcHashes, Cnf); + if (Status == StatDone) + Worker->ReplyAux(Desc); + else if (Status == StatAuthError || Status == StatError) + Worker->ReplyAux(Desc); +} + /*}}}*/ +std::string pkgAcqAuxFile::Custom600Headers() const /*{{{*/ +{ + if (MaximumSize == 0) + return pkgAcqFile::Custom600Headers(); + std::string maxsize; + strprintf(maxsize, "\nMaximum-Size: %llu", MaximumSize); + return pkgAcqFile::Custom600Headers().append(maxsize); +} + /*}}}*/ +void pkgAcqAuxFile::Finished() /*{{{*/ +{ + auto dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"); + if (APT::String::Startswith(DestFile, dirname)) + { + // the file is never returned by method requesting it, so fix up the permission now + if (FileExists(DestFile)) + { + ChangeOwnerAndPermissionOfFile("pkgAcqAuxFile", DestFile.c_str(), "root", ROOT_GROUP, 0644); + if (Status == StatDone) + return; + } + } + else + { + dirname = flNotFile(DestFile); + RemoveFile("pkgAcqAuxFile::Finished", DestFile); + RemoveFile("pkgAcqAuxFile::Finished", DestFile + ".FAILED"); + rmdir(dirname.c_str()); + } + DestFile.clear(); +} + /*}}}*/ +// GetAuxFileNameFromURI /*{{{*/ +static std::string GetAuxFileNameFromURIInLists(std::string const &uri) +{ + // check if we have write permission for our usual location. + auto const dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"); + char const * const filetag = ".apt-acquire-privs-test.XXXXXX"; + std::string const tmpfile_tpl = flCombine(dirname, filetag); + std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free }; + int const fd = mkstemp(tmpfile.get()); + if (fd == -1) + return ""; + RemoveFile("GetAuxFileNameFromURI", tmpfile.get()); + close(fd); + return flCombine(dirname, URItoFileName(uri)); +} +static std::string GetAuxFileNameFromURI(std::string const &uri) +{ + auto const lists = GetAuxFileNameFromURIInLists(uri); + if (lists.empty() == false) + return lists; + + std::string tmpdir_tpl; + strprintf(tmpdir_tpl, "%s/apt-auxfiles-XXXXXX", GetTempDir().c_str()); + std::unique_ptr<char, decltype(std::free) *> tmpdir { strndup(tmpdir_tpl.data(), tmpdir_tpl.length()), std::free }; + if (mkdtemp(tmpdir.get()) == nullptr) + { + _error->Errno("GetAuxFileNameFromURI", "mkdtemp of %s failed", tmpdir.get()); + return flCombine("/nonexistent/auxfiles/", URItoFileName(uri)); + } + chmod(tmpdir.get(), 0755); + auto const filename = flCombine(tmpdir.get(), URItoFileName(uri)); + _error->PushToStack(); + FileFd in(flCombine(flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"), URItoFileName(uri)), FileFd::ReadOnly); + if (in.IsOpen()) + { + FileFd out(filename, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive); + CopyFile(in, out); + ChangeOwnerAndPermissionOfFile("GetAuxFileNameFromURI", filename.c_str(), "root", ROOT_GROUP, 0644); + } + _error->RevertToStack(); + return filename; +} + /*}}}*/ +pkgAcqAuxFile::pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker, + std::string const &ShortDesc, std::string const &Desc, std::string const &URI, + HashStringList const &Hashes, unsigned long long const MaximumSize) : pkgAcqFile(Owner->GetOwner(), URI, Hashes, Hashes.FileSize(), Desc, ShortDesc, "", GetAuxFileNameFromURI(URI), false), + Owner(Owner), Worker(Worker), MaximumSize(MaximumSize) +{ + /* very bad failures can happen while constructing which causes + us to hang as the aux request is never answered (e.g. method not available) + Ideally we catch failures earlier, but a safe guard can't hurt. */ + if (Status == pkgAcquire::Item::StatIdle || Status == pkgAcquire::Item::StatFetching) + return; + Failed(std::string("400 URI Failure\n") + + "URI: " + URI + "\n" + + "Filename: " + DestFile, + nullptr); +} +pkgAcqAuxFile::~pkgAcqAuxFile() {} diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h new file mode 100644 index 0000000..70651d9 --- /dev/null +++ b/apt-pkg/acquire-item.h @@ -0,0 +1,1250 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire Item - Item to acquire + + When an item is instantiated it will add it self to the local list in + the Owner Acquire class. Derived classes will then call QueueURI to + register all the URI's they wish to fetch at the initial moment. + + Three item classes are provided to provide functionality for + downloading of Index, Translation and Packages files. + + A Archive class is provided for downloading .deb files. It does Hash + checking and source location as well as a retry algorithm. + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_ACQUIRE_ITEM_H +#define PKGLIB_ACQUIRE_ITEM_H + +#include <apt-pkg/acquire.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/indexfile.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/weakptr.h> + +#include <map> +#include <string> +#include <unordered_map> +#include <vector> + +#ifndef APT_8_CLEANER_HEADERS +#include <apt-pkg/pkgrecords.h> +#include <apt-pkg/sourcelist.h> +#endif + +/** \addtogroup acquire + * @{ + * + * \file acquire-item.h + */ + +class pkgRecords; +class pkgSourceList; +class pkgAcqMetaClearSig; +class pkgAcqIndexMergeDiffs; +class metaIndex; + +class pkgAcquire::Item : public WeakPointable /*{{{*/ +/** \brief Represents the process by which a pkgAcquire object should + * retrieve a file or a collection of files. + * + * By convention, Item subclasses should insert themselves into the + * acquire queue when they are created by calling QueueURI(), and + * remove themselves by calling Dequeue() when either Done() or + * Failed() is invoked. Item objects are also responsible for + * notifying the download progress indicator (accessible via + * #Owner->Log) of their status. + * + * \see pkgAcquire + */ +{ + public: + + /** \brief The current status of this item. */ + enum ItemState + { + /** \brief The item is waiting to be downloaded. */ + StatIdle, + + /** \brief The item is currently being downloaded. */ + StatFetching, + + /** \brief The item has been successfully downloaded. */ + StatDone, + + /** \brief An error was encountered while downloading this + * item. + */ + StatError, + + /** \brief The item was downloaded but its authenticity could + * not be verified. + */ + StatAuthError, + + /** \brief The item was could not be downloaded because of + * a transient network error (e.g. network down) + */ + StatTransientNetworkError, + } Status; + + /** \brief Contains a textual description of the error encountered + * if #ItemState is #StatError or #StatAuthError. + */ + std::string ErrorText; + + /** \brief The size of the object to fetch. */ + unsigned long long FileSize; + + /** \brief How much of the object was already fetched. */ + unsigned long long PartialSize; + + /** \brief If not \b NULL, contains the name of a subprocess that + * is operating on this object (for instance, "gzip" or "gpgv"). + */ + APT_DEPRECATED_MSG("Use the std::string member ActiveSubprocess instead") const char *Mode; + + /** \brief contains the name of the subprocess that is operating on this object + * (for instance, "gzip", "rred" or "gpgv"). This is obsoleting #Mode from above + * as it can manage the lifetime of included string properly. */ + std::string ActiveSubprocess; + + /** \brief A client-supplied unique identifier. + * + * This field is initialized to 0; it is meant to be filled in by + * clients that wish to use it to uniquely identify items. + * + * APT progress reporting will store an ID there as shown in "Get:42 …" + */ + unsigned long ID; + + /** \brief If \b true, the entire object has been successfully fetched. + * + * Subclasses should set this to \b true when appropriate. + */ + bool Complete; + + /** \brief If \b true, the URI of this object is "local". + * + * The only effect of this field is to exclude the object from the + * download progress indicator's overall statistics. + */ + bool Local; + + std::string UsedMirror; + + /** \brief The number of fetch queues into which this item has been + * inserted. + * + * There is one queue for each source from which an item could be + * downloaded. + * + * \sa pkgAcquire + */ + unsigned int QueueCounter; + + /** \brief The number of additional fetch items that are expected + * once this item is done. + * + * Some items like pkgAcqMeta{Index,Sig} will queue additional + * items. This variable can be set by the methods if it knows + * in advance how many items to expect to get a more accurate + * progress. + */ + unsigned int ExpectedAdditionalItems; + + /** \brief The name of the file into which the retrieved object + * will be written. + */ + std::string DestFile; + + /** \brief Invoked by the acquire worker when the object couldn't + * be fetched. + * + * This is a branch of the continuation of the fetch process. + * + * \param Message An RFC822-formatted message from the acquire + * method describing what went wrong. Use LookupTag() to parse + * it. + * + * \param Cnf The method via which the worker tried to fetch this object. + * + * \sa pkgAcqMethod + */ + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf); + APT_HIDDEN void FailMessage(std::string const &Message); + + /** \brief Invoked by the acquire worker to check if the successfully + * fetched object is also the objected we wanted to have. + * + * Note that the object might \e not have been written to + * DestFile; check for the presence of an Alt-Filename entry in + * Message to find the file to which it was really written. + * + * This is called before Done is called and can prevent it by returning + * \b false which will result in Failed being called instead. + * + * You should prefer to use this method over calling Failed() from Done() + * as this has e.g. the wrong progress reporting. + * + * \param Message Data from the acquire method. Use LookupTag() + * to parse it. + * \param Cnf The method via which the object was fetched. + * + * \sa pkgAcqMethod + */ + virtual bool VerifyDone(std::string const &Message, + pkgAcquire::MethodConfig const * const Cnf); + + /** \brief Invoked by the acquire worker when the object was + * fetched successfully. + * + * Note that the object might \e not have been written to + * DestFile; check for the presence of an Alt-Filename entry in + * Message to find the file to which it was really written. + * + * Done is often used to switch from one stage of the processing + * to the next (e.g. fetching, unpacking, copying). It is one + * branch of the continuation of the fetch process. + * + * \param Message Data from the acquire method. Use LookupTag() + * to parse it. + * \param Hashes The HashSums of the object that was fetched. + * \param Cnf The method via which the object was fetched. + * + * \sa pkgAcqMethod + */ + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf); + + /** \brief Invoked when the worker starts to fetch this object. + * + * \param Message RFC822-formatted data from the worker process. + * Use LookupTag() to parse it. + * + * \param Hashes The expected hashes of the object being fetched. + * + * \sa pkgAcqMethod + */ + virtual void Start(std::string const &Message, unsigned long long const Size); + + /** \brief Custom headers to be sent to the fetch process. + * + * \return a string containing RFC822-style headers that are to be + * inserted into the 600 URI Acquire message sent to the fetch + * subprocess. The headers are inserted after a newline-less + * line, so they should (if nonempty) have a leading newline and + * no trailing newline. + */ + virtual std::string Custom600Headers() const; + // Retries should really be a member of the Item, but can't be for ABI reasons + APT_HIDDEN unsigned int &ModifyRetries(); + // this is more a hack than a proper external interface, hence hidden + APT_HIDDEN std::unordered_map<std::string, std::string> &ModifyCustomFields(); + // this isn't the super nicest interface either… + APT_HIDDEN bool PopAlternativeURI(std::string &NewURI); + APT_HIDDEN bool IsGoodAlternativeURI(std::string const &AltUri) const; + APT_HIDDEN void PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back); + APT_HIDDEN void RemoveAlternativeSite(std::string &&OldSite); + + /** \brief A "descriptive" URI-like string. + * + * \return a URI that should be used to describe what is being fetched. + */ + virtual std::string DescURI() const = 0; + /** \brief Short item description. + * + * \return a brief description of the object being fetched. + */ + virtual std::string ShortDesc() const; + + /** \brief Invoked by the worker when the download is completely done. */ + virtual void Finished(); + + /** \return HashSums the DestFile is supposed to have in this stage */ + virtual HashStringList GetExpectedHashes() const = 0; + /** \return the 'best' hash for display proposes like --print-uris */ + std::string HashSum() const; + + /** \return if having no hashes is a hard failure or not + * + * Idealy this is always \b true for every subclass, but thanks to + * historical grow we don't have hashes for all files in all cases + * in all steps, so it is slightly more complicated than it should be. + */ + virtual bool HashesRequired() const { return true; } + + /** \return the acquire process with which this item is associated. */ + pkgAcquire *GetOwner() const; + pkgAcquire::ItemDesc &GetItemDesc(); + + /** \return \b true if this object is being fetched from a trusted source. */ + virtual bool IsTrusted() const; + + /** \brief Report mirror problem + * + * This allows reporting mirror failures back to a centralized + * server. The apt-report-mirror-failure script is called for this + * + * \param FailCode A short failure string that is send + */ + APT_DEPRECATED_MSG("Item::Failed does this for you") void ReportMirrorFailure(std::string const &FailCode); + + /** \brief Set the name of the current active subprocess + * + * See also #ActiveSubprocess + */ + void SetActiveSubprocess(std::string const &subprocess); + + /** \brief Initialize an item. + * + * Adds the item to the list of items known to the acquire + * process, but does not place it into any fetch queues (you must + * manually invoke QueueURI() to do so). + * + * \param Owner The new owner of this item. + */ + explicit Item(pkgAcquire * const Owner); + + /** \brief Remove this item from its owner's queue by invoking + * pkgAcquire::Remove. + */ + virtual ~Item(); + + bool APT_HIDDEN IsRedirectionLoop(std::string const &NewURI); + /** \brief The priority of the item, used for queuing */ + int APT_HIDDEN Priority(); + + protected: + /** \brief The acquire object with which this item is associated. */ + pkgAcquire * const Owner; + + /** \brief The item that is currently being downloaded. */ + pkgAcquire::ItemDesc Desc; + + enum RenameOnErrorState { + HashSumMismatch, + SizeMismatch, + InvalidFormat, + SignatureError, + NotClearsigned, + MaximumSizeExceeded, + PDiffError, + }; + + /** \brief Rename failed file and set error + * + * \param state respresenting the error we encountered + */ + bool RenameOnError(RenameOnErrorState const state); + + /** \brief Insert this item into its owner's queue. + * + * The method is designed to check if the request would end + * in an IMSHit and if it determines that it would, it isn't + * queueing the Item and instead sets it to completion instantly. + * + * \param Item Metadata about this item (its URI and + * description). + * \return true if the item was inserted, false if IMSHit was detected + */ + virtual bool QueueURI(ItemDesc &Item); + + /** \brief Remove this item from its owner's queue. */ + void Dequeue(); + + /** \brief Rename a file without modifying its timestamp. + * + * Many item methods call this as their final action. + * + * \param From The file to be renamed. + * + * \param To The new name of \a From. If \a To exists it will be + * overwritten. If \a From and \a To are equal nothing happens. + */ + bool Rename(std::string const &From, std::string const &To); + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const; + + private: + class Private; + Private * const d; + + friend class pkgAcqMetaBase; + friend class pkgAcqMetaClearSig; +}; + /*}}}*/ +class APT_HIDDEN pkgAcqTransactionItem: public pkgAcquire::Item /*{{{*/ +/** \brief baseclass for the indexes files to manage them all together */ +{ + void * const d; + protected: + HashStringList GetExpectedHashesFor(std::string const &MetaKey) const; + + bool QueueURI(pkgAcquire::ItemDesc &Item) APT_OVERRIDE; + + public: + IndexTarget const Target; + + /** \brief storge name until a transaction is finished */ + std::string PartialFile; + + /** \brief TransactionManager */ + pkgAcqMetaClearSig * const TransactionManager; + + enum TransactionStates { + TransactionStarted, + TransactionCommit, + TransactionAbort, + }; + virtual bool TransactionState(TransactionStates const state); + + virtual std::string DescURI() const APT_OVERRIDE { return Target.URI; } + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE; + virtual std::string GetMetaKey() const; + virtual bool HashesRequired() const APT_OVERRIDE; + virtual bool AcquireByHash() const; + + pkgAcqTransactionItem(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target) APT_NONNULL(2, 3); + virtual ~pkgAcqTransactionItem(); + + friend class pkgAcqMetaBase; + friend class pkgAcqMetaClearSig; +}; + /*}}}*/ +class APT_HIDDEN pkgAcqMetaBase : public pkgAcqTransactionItem /*{{{*/ +/** \brief the manager of a transaction */ +{ + void * const d; + protected: + std::vector<pkgAcqTransactionItem*> Transaction; + + /** \brief If \b true, the index's signature is currently being verified. + */ + bool AuthPass; + + /** \brief Called when a file is finished being retrieved. + * + * If the file was not downloaded to DestFile, a copy process is + * set up to copy it to DestFile; otherwise, Complete is set to \b + * true and the file is moved to its final location. + * + * \param Message The message block received from the fetch + * subprocess. + */ + bool CheckDownloadDone(pkgAcqTransactionItem * const I, const std::string &Message, HashStringList const &Hashes) const; + + /** \brief Queue the downloaded Signature for verification */ + void QueueForSignatureVerify(pkgAcqTransactionItem * const I, std::string const &File, std::string const &Signature); + + virtual std::string Custom600Headers() const APT_OVERRIDE; + + /** \brief Called when authentication succeeded. + * + * Sanity-checks the authenticated file, queues up the individual + * index files for download, and saves the signature in the lists + * directory next to the authenticated list file. + * + * \param Message The message block received from the fetch + * subprocess. + * \param Cnf The method and its configuration which handled the request + */ + bool CheckAuthDone(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf); + + /** Check if the current item should fail at this point */ + bool CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message); + + /** \brief Check that the release file is a release file for the + * correct distribution. + * + * \return \b true if no fatal errors were encountered. + */ + bool VerifyVendor(std::string const &Message); + + virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE; + + public: + // This refers more to the Transaction-Manager than the actual file + bool IMSHit; + TransactionStates State; + std::string BaseURI; + + virtual bool QueueURI(pkgAcquire::ItemDesc &Item) APT_OVERRIDE; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE; + virtual bool HashesRequired() const APT_OVERRIDE; + + // transaction code + void Add(pkgAcqTransactionItem * const I); + void AbortTransaction(); + bool TransactionHasError() const; + void CommitTransaction(); + + /** \brief Stage (queue) a copy action when the transaction is committed + */ + void TransactionStageCopy(pkgAcqTransactionItem * const I, + const std::string &From, + const std::string &To); + /** \brief Stage (queue) a removal action when the transaction is committed + */ + void TransactionStageRemoval(pkgAcqTransactionItem * const I, const std::string &FinalFile); + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + + pkgAcqMetaBase(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &DataTarget) APT_NONNULL(2, 3); + virtual ~pkgAcqMetaBase(); +}; + /*}}}*/ +/** \brief An item that is responsible for downloading the meta-index {{{ + * file (i.e., Release) itself and verifying its signature. + * + * Once the download and verification are complete, the downloads of + * the individual index files are queued up using pkgAcqDiffIndex. + * If the meta-index file had a valid signature, the expected hashsums + * of the index files will be the md5sums listed in the meta-index; + * otherwise, the expected hashsums will be "" (causing the + * authentication of the index files to be bypassed). + */ +class APT_HIDDEN pkgAcqMetaIndex : public pkgAcqMetaBase +{ + void * const d; + protected: + IndexTarget const DetachedSigTarget; + + /** \brief delayed constructor */ + void Init(std::string const &URIDesc, std::string const &ShortDesc); + + public: + virtual std::string DescURI() const APT_OVERRIDE; + + // Specialized action members + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + + /** \brief Create a new pkgAcqMetaIndex. */ + pkgAcqMetaIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &DataTarget, IndexTarget const &DetachedSigTarget) APT_NONNULL(2, 3); + virtual ~pkgAcqMetaIndex(); + + friend class pkgAcqMetaSig; +}; + /*}}}*/ +/** \brief An acquire item that downloads the detached signature {{{ + * of a meta-index (Release) file, then queues up the release + * file itself. + * + * \todo Why protected members? + * + * \sa pkgAcqMetaIndex + */ +class APT_HIDDEN pkgAcqMetaSig : public pkgAcqTransactionItem +{ + void * const d; + + pkgAcqMetaIndex * const MetaIndex; + + /** \brief The file we use to verify the MetaIndexFile with (not always set!) */ + std::string MetaIndexFileSignature; + + protected: + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + + public: + virtual bool HashesRequired() const APT_OVERRIDE { return false; } + + // Specialized action members + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + + /** \brief Create a new pkgAcqMetaSig. */ + pkgAcqMetaSig(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, pkgAcqMetaIndex * const MetaIndex) APT_NONNULL(2, 3, 5); + virtual ~pkgAcqMetaSig(); +}; + /*}}}*/ +/** \brief An item responsible for downloading clearsigned metaindexes {{{*/ +class APT_HIDDEN pkgAcqMetaClearSig : public pkgAcqMetaIndex +{ + void * const d; + IndexTarget const DetachedDataTarget; + + public: + /** \brief A package-system-specific parser for the meta-index file. */ + metaIndex *MetaIndexParser; + metaIndex *LastMetaIndexParser; + + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual bool VerifyDone(std::string const &Message, pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Finished() APT_OVERRIDE; + + /** \brief Starts downloading the individual index files. + * + * \param verify If \b true, only indices whose expected hashsum + * can be determined from the meta-index will be downloaded, and + * the hashsums of indices will be checked (reporting + * #StatAuthError if there is a mismatch). If verify is \b false, + * no hashsum checking will be performed. + */ + void QueueIndexes(bool const verify); + + /** \brief Create a new pkgAcqMetaClearSig. */ + pkgAcqMetaClearSig(pkgAcquire * const Owner, + IndexTarget const &ClearsignedTarget, + IndexTarget const &DetachedDataTarget, + IndexTarget const &DetachedSigTarget, + metaIndex * const MetaIndexParser); + virtual ~pkgAcqMetaClearSig(); +}; + /*}}}*/ +/** \brief Common base class for all classes that deal with fetching indexes {{{*/ +class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem +{ + void * const d; + + public: + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + + pkgAcqBaseIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target) APT_NONNULL(2, 3); + virtual ~pkgAcqBaseIndex(); +}; + /*}}}*/ +/** \brief An acquire item that is responsible for fetching an index {{{ + * file (e.g., Packages or Sources). + * + * \sa pkgAcqDiffIndex, pkgAcqIndexDiffs, pkgAcqIndexTrans + * + * \todo Why does pkgAcqIndex have protected members? + */ +class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex +{ + void * const d; + + protected: + + /** \brief The stages the method goes through + * + * The method first downloads the indexfile, then its decompressed (or + * copied) and verified + */ + enum AllStages { + STAGE_DOWNLOAD, + STAGE_DECOMPRESS_AND_VERIFY, + }; + AllStages Stage; + + /** \brief Handle what needs to be done when the download is done */ + void StageDownloadDone(std::string const &Message); + + /** \brief Handle what needs to be done when the decompression/copy is + * done + */ + void StageDecompressDone(); + + /** \brief If \b set, this partially downloaded file will be + * removed when the download completes. + */ + std::string EraseFileName; + + /** \brief The compression-related file extensions that are being + * added to the downloaded file one by one if first fails (e.g., "gz bz2"). + */ + std::string CompressionExtensions; + + /** \brief The actual compression extension currently used */ + std::string CurrentCompressionExtension; + + /** \brief Do the changes needed to fetch via AptByHash (if needed) */ + void InitByHashIfNeeded(); + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + + virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE; + + public: + // Specialized action members + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;}; + virtual std::string GetMetaKey() const APT_OVERRIDE; + + pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, bool const Derived = false) APT_NONNULL(2, 3); + virtual ~pkgAcqIndex(); + + protected: + APT_HIDDEN void Init(std::string const &URI, std::string const &URIDesc, + std::string const &ShortDesc); + APT_HIDDEN bool CommonFailed(std::string const &TargetURI, + std::string const &Message, pkgAcquire::MethodConfig const *const Cnf); +}; + /*}}}*/ +struct APT_HIDDEN DiffInfo { /*{{{*/ + /** The filename of the diff. */ + std::string file; + + /** The hashes of the file after the diff is applied */ + HashStringList result_hashes; + + /** The hashes of the diff */ + HashStringList patch_hashes; + + /** The hashes of the compressed diff */ + HashStringList download_hashes; +}; + /*}}}*/ +/** \brief An item that is responsible for fetching an index file of {{{ + * package list diffs and starting the package list's download. + * + * This item downloads the Index file and parses it, then enqueues + * additional downloads of either the individual patches (using + * pkgAcqIndexDiffs) or the entire Packages file (using pkgAcqIndex). + * + * \sa pkgAcqIndexDiffs, pkgAcqIndex + */ +class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqIndex +{ + void * const d; + std::vector<pkgAcqIndexMergeDiffs*> * diffs; + std::vector<DiffInfo> available_patches; + bool pdiff_merge; + + protected: + /** \brief If \b true, debugging information will be written to std::clog. */ + bool Debug; + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + + virtual bool QueueURI(pkgAcquire::ItemDesc &Item) APT_OVERRIDE; + + virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE; + public: + // Specialized action members + virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual bool VerifyDone(std::string const &Message, pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Target.URI + "Index";}; + virtual std::string GetMetaKey() const APT_OVERRIDE; + + /** \brief Parse the Index file for a set of Packages diffs. + * + * Parses the Index file and creates additional download items as + * necessary. + * + * \param IndexDiffFile The name of the Index file. + * + * \return \b true if the Index file was successfully parsed, \b + * false otherwise. + */ + bool ParseDiffIndex(std::string const &IndexDiffFile); + + /** \brief Create a new pkgAcqDiffIndex. + * + * \param Owner The Acquire object that owns this item. + * + * \param URI The URI of the list file to download. + * + * \param URIDesc A long description of the list file to download. + * + * \param ShortDesc A short description of the list file to download. + */ + pkgAcqDiffIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target) APT_NONNULL(2, 3); + virtual ~pkgAcqDiffIndex(); + private: + APT_HIDDEN void QueueOnIMSHit() const; +}; + /*}}}*/ +/** \brief An item that is responsible for fetching client-merge patches {{{ + * that need to be applied to a given package index file. + * + * Instead of downloading and applying each patch one by one like its + * sister #pkgAcqIndexDiffs this class will download all patches at once + * and call rred with all the patches downloaded once. Rred will then + * merge and apply them in one go, which should be a lot faster – but is + * incompatible with server-based merges of patches like reprepro can do. + * + * \sa pkgAcqDiffIndex, pkgAcqIndex + */ +class APT_HIDDEN pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex +{ + protected: + + /** \brief If \b true, debugging output will be written to + * std::clog. + */ + bool Debug; + + /** \brief information about the current patch */ + struct DiffInfo const patch; + + /** \brief list of all download items for the patches */ + std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches; + + /** The current status of this patch. */ + enum DiffState + { + /** \brief The diff is currently being fetched. */ + StateFetchDiff, + + /** \brief The diff is currently being applied. */ + StateApplyDiff, + + /** \brief the work with this diff is done */ + StateDoneDiff, + + /** \brief something bad happened and fallback was triggered */ + StateErrorDiff + } State; + + public: + /** \brief Called when the patch file failed to be downloaded. + * + * This method will fall back to downloading the whole index file + * outright; its arguments are ignored. + */ + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Target.URI + "Index";}; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE; + virtual bool HashesRequired() const APT_OVERRIDE; + virtual bool AcquireByHash() const APT_OVERRIDE; + + /** \brief Create an index merge-diff item. + * + * \param Owner The pkgAcquire object that owns this item. + * \param TransactionManager responsible for this item + * \param Target we intend to built via pdiff patching + * \param baseURI is the URI used for the Index, but stripped down to Target + * \param DiffInfo of the patch in question + * \param patch contains infos about the patch this item is supposed + * to download which were read from the index + * \param allPatches contains all related items so that each item can + * check if it was the last one to complete the download step + */ + pkgAcqIndexMergeDiffs(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, + IndexTarget const &Target, DiffInfo const &patch, + std::vector<pkgAcqIndexMergeDiffs *> const *const allPatches) APT_NONNULL(2, 3, 6); + virtual ~pkgAcqIndexMergeDiffs(); +}; + /*}}}*/ +/** \brief An item that is responsible for fetching server-merge patches {{{ + * that need to be applied to a given package index file. + * + * After downloading and applying a single patch, this item will + * enqueue a new pkgAcqIndexDiffs to download and apply the remaining + * patches. If no patch can be found that applies to an intermediate + * file or if one of the patches cannot be downloaded, falls back to + * downloading the entire package index file using pkgAcqIndex. + * + * \sa pkgAcqDiffIndex, pkgAcqIndex + */ +class APT_HIDDEN pkgAcqIndexDiffs : public pkgAcqBaseIndex +{ + private: + + /** \brief Queue up the next diff download. + * + * Search for the next available diff that applies to the file + * that currently exists on disk, and enqueue it by calling + * QueueURI(). + * + * \return \b true if an applicable diff was found, \b false + * otherwise. + */ + APT_HIDDEN bool QueueNextDiff(); + + /** \brief Handle tasks that must be performed after the item + * finishes downloading. + * + * Dequeues the item and checks the resulting file's hashsums + * against ExpectedHashes after the last patch was applied. + * There is no need to check the md5/sha1 after a "normal" + * patch because QueueNextDiff() will check the sha1 later. + * + * \param allDone If \b true, the file was entirely reconstructed, + * and its md5sum is verified. + */ + APT_HIDDEN void Finish(bool const allDone=false); + + protected: + + /** \brief If \b true, debugging output will be written to + * std::clog. + */ + bool Debug; + + /** The patches that remain to be downloaded, including the patch + * being downloaded right now. This list should be ordered so + * that each diff appears before any diff that depends on it. + * + * \todo These are indexed by sha1sum; why not use some sort of + * dictionary instead of relying on ordering and stripping them + * off the front? + */ + std::vector<DiffInfo> available_patches; + + /** The current status of this patch. */ + enum DiffState + { + /** \brief The diff is currently being fetched. */ + StateFetchDiff, + + /** \brief The diff is currently being applied. */ + StateApplyDiff + } State; + + public: + + /** \brief Called when the patch file failed to be downloaded. + * + * This method will fall back to downloading the whole index file + * outright; its arguments are ignored. + */ + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Target.URI + "IndexDiffs";}; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE; + virtual bool HashesRequired() const APT_OVERRIDE; + virtual bool AcquireByHash() const APT_OVERRIDE; + + /** \brief Create an index diff item. + * + * After filling in its basic fields, this invokes Finish(true) if + * \a diffs is empty, or QueueNextDiff() otherwise. + * + * \param Owner The pkgAcquire object that owns this item. + * \param TransactionManager responsible for this item + * \param Target we want to built via pdiff patching + * \param baseURI is the URI used for the Index, but stripped down to Target + * \param diffs The remaining diffs from the index of diffs. They + * should be ordered so that each diff appears before any diff + * that depends on it. + */ + pkgAcqIndexDiffs(pkgAcquire *const Owner, pkgAcqMetaClearSig *const TransactionManager, + IndexTarget const &Target, + std::vector<DiffInfo> const &diffs = std::vector<DiffInfo>()) APT_NONNULL(2, 3); + virtual ~pkgAcqIndexDiffs(); +}; + /*}}}*/ +/** \brief An item that is responsible for fetching a package file. {{{ + * + * If the package file already exists in the cache, nothing will be + * done. + */ +class pkgAcqArchive : public pkgAcquire::Item +{ + void * const d; + + bool LocalSource; + HashStringList ExpectedHashes; + + protected: + /** \brief The package version being fetched. */ + pkgCache::VerIterator Version; + + /** \brief The list of sources from which to pick archives to + * download this package from. + */ + pkgSourceList *Sources; + + /** \brief A package records object, used to look up the file + * corresponding to each version of the package. + */ + pkgRecords *Recs; + + /** \brief A location in which the actual filename of the package + * should be stored. + */ + std::string &StoreFilename; + + /** \brief The next file for this version to try to download. */ + APT_DEPRECATED_MSG("Unused member") + pkgCache::VerFileIterator Vf; + + /** \brief How many (more) times to try to find a new source from + * which to download this package version if it fails. + * + * Set from Acquire::Retries. + */ + APT_DEPRECATED_MSG("Unused member. See pkgAcqItem::Retries.") + unsigned int Retries; + + /** \brief \b true if this version file is being downloaded from a + * trusted source. + */ + bool Trusted; + + /** \brief Queue up the next available file for this version. */ + bool QueueNext(); + + /** \brief Get the full pathname of the final file for the current URI */ + virtual std::string GetFinalFilename() const APT_OVERRIDE; + + public: + + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &Hashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE; + virtual std::string ShortDesc() const APT_OVERRIDE; + virtual void Finished() APT_OVERRIDE; + virtual bool IsTrusted() const APT_OVERRIDE; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE; + virtual bool HashesRequired() const APT_OVERRIDE; + + /** \brief Create a new pkgAcqArchive. + * + * \param Owner The pkgAcquire object with which this item is + * associated. + * + * \param Sources The sources from which to download version + * files. + * + * \param Recs A package records object, used to look up the file + * corresponding to each version of the package. + * + * \param Version The package version to download. + * + * \param[out] StoreFilename A location in which the actual filename of + * the package should be stored. It will be set to a guessed + * basename in the constructor, and filled in with a fully + * qualified filename once the download finishes. + */ + pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sources, + pkgRecords * const Recs,pkgCache::VerIterator const &Version, + std::string &StoreFilename); + virtual ~pkgAcqArchive(); +}; + /*}}}*/ +/** \brief Retrieve the changelog for the given version {{{ + * + * Downloads the changelog to a temporary file it will also remove again + * while it is deconstructed or downloads it to a named location. + */ +class pkgAcqChangelog : public pkgAcquire::Item +{ + class Private; + Private * const d; + std::string TemporaryDirectory; + std::string const SrcName; + std::string const SrcVersion; + + public: + // we will never have hashes for changelogs. + // If you need verified ones, download the deb and extract the changelog. + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE { return HashStringList(); } + virtual bool HashesRequired() const APT_OVERRIDE { return false; } + + // Specialized action members + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;}; + + /** returns the URI to the changelog of this version + * + * @param Ver is the version to get the changelog for + * @return the URI which will be used to acquire the changelog + */ + static std::string URI(pkgCache::VerIterator const &Ver); + + /** returns the URI to the changelog of this version + * + * \param Rls is the Release file the package comes from + * \param Component in which the package resides, can be empty + * \param SrcName is the source package name + * \param SrcVersion is the source package version + * @return the URI which will be used to acquire the changelog + */ + static std::string URI(pkgCache::RlsFileIterator const &Rls, + char const * const Component, char const * const SrcName, + char const * const SrcVersion); + + /** returns the URI to the changelog of this version + * + * \param Template URI where @CHANGEPATH@ has to be filled in + * \param Component in which the package resides, can be empty + * \param SrcName is the source package name + * \param SrcVersion is the source package version + * @return the URI which will be used to acquire the changelog + */ + static std::string URI(std::string const &Template, + char const * const Component, char const * const SrcName, + char const * const SrcVersion); + + /** returns the URI template for this release file + * + * \param Rls is a Release file + * @return the URI template to use for this release file + */ + static std::string URITemplate(pkgCache::RlsFileIterator const &Rls); + + /** \brief Create a new pkgAcqChangelog object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * \param Ver is the version to get the changelog for + * \param DestDir The directory the file should be downloaded into. + * Will be an autocreated (and cleaned up) temporary directory if not set. + * \param DestFilename The filename the file should have in #DestDir + * Defaults to sourcepackagename.changelog if not set. + */ + pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver, + std::string const &DestDir="", std::string const &DestFilename=""); + + /** \brief Create a new pkgAcqChangelog object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * \param Rls is the Release file the package comes from + * \param Component in which the package resides, can be empty + * \param SrcName is the source package name + * \param SrcVersion is the source package version + * \param DestDir The directory the file should be downloaded into. + * Will be an autocreated (and cleaned up) temporary directory if not set. + * \param DestFilename The filename the file should have in #DestDir + * Defaults to sourcepackagename.changelog if not set. + */ + pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &Rls, + char const * const Component, char const * const SrcName, char const * const SrcVersion, + std::string const &DestDir="", std::string const &DestFilename=""); + + /** \brief Create a new pkgAcqChangelog object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * \param URI is to be used to get the changelog + * \param SrcName is the source package name + * \param SrcVersion is the source package version + * \param DestDir The directory the file should be downloaded into. + * Will be an autocreated (and cleaned up) temporary directory if not set. + * \param DestFilename The filename the file should have in #DestDir + * Defaults to sourcepackagename.changelog if not set. + */ + pkgAcqChangelog(pkgAcquire * const Owner, std::string const &URI, + char const * const SrcName, char const * const SrcVersion, + std::string const &DestDir="", std::string const &DestFilename=""); + + virtual ~pkgAcqChangelog(); + +private: + APT_HIDDEN void Init(std::string const &DestDir, std::string const &DestFilename); +}; + /*}}}*/ +/** \brief Retrieve an arbitrary file to the current directory. {{{ + * + * The file is retrieved even if it is accessed via a URL type that + * normally is a NOP, such as "file". If the download fails, the + * partial file is renamed to get a ".FAILED" extension. + */ +class pkgAcqFile : public pkgAcquire::Item +{ + void * const d; + + /** \brief How many times to retry the download, set from + * Acquire::Retries. + */ + APT_DEPRECATED_MSG("Unused member. See pkgAcqItem::Retries.") + unsigned int Retries; + + /** \brief Should this file be considered a index file */ + bool IsIndexFile; + + HashStringList const ExpectedHashes; + public: + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE; + virtual bool HashesRequired() const APT_OVERRIDE; + + // Specialized action members + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE; + virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;}; + virtual std::string Custom600Headers() const APT_OVERRIDE; + + /** \brief Create a new pkgAcqFile object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * + * \param URI The URI to download. + * + * \param Hashes The hashsums of the file to download, if they are known; + * otherwise empty list. + * + * \param Size The size of the file to download, if it is known; + * otherwise 0. + * + * \param Desc A description of the file being downloaded. + * + * \param ShortDesc A brief description of the file being + * downloaded. + * + * \param DestDir The directory the file should be downloaded into. + * + * \param DestFilename The filename+path the file is downloaded to. + * + * \param IsIndexFile The file is considered a IndexFile and cache-control + * headers like "cache-control: max-age=0" are send + * + * If DestFilename is empty, download to DestDir/\<basename\> if + * DestDir is non-empty, $CWD/\<basename\> otherwise. If + * DestFilename is NOT empty, DestDir is ignored and DestFilename + * is the absolute name to which the file should be downloaded. + */ + + pkgAcqFile(pkgAcquire * const Owner, std::string const &URI, HashStringList const &Hashes, unsigned long long const Size, + std::string const &Desc, std::string const &ShortDesc, + std::string const &DestDir="", std::string const &DestFilename="", + bool const IsIndexFile=false); + virtual ~pkgAcqFile(); +}; + /*}}}*/ +class APT_HIDDEN pkgAcqAuxFile : public pkgAcqFile /*{{{*/ +{ + pkgAcquire::Item *const Owner; + pkgAcquire::Worker *const Worker; + unsigned long long MaximumSize; + + public: + virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual void Finished() APT_OVERRIDE; + + pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker, + std::string const &ShortDesc, std::string const &Desc, std::string const &URI, + HashStringList const &Hashes, unsigned long long const MaximumSize); + virtual ~pkgAcqAuxFile(); +}; + /*}}}*/ +/** @} */ + +#endif diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc new file mode 100644 index 0000000..ae5ae4a --- /dev/null +++ b/apt-pkg/acquire-method.cc @@ -0,0 +1,565 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire Method + + This is a skeleton class that implements most of the functionality + of a method and some useful functions to make method implementation + simpler. The methods all derive this and specialize it. The most + complex implementation is the http method which needs to provide + pipelining, it runs the message engine at the same time it is + downloading files.. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/acquire-method.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/md5.h> +#include <apt-pkg/sha1.h> +#include <apt-pkg/sha2.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <iostream> +#include <iterator> +#include <sstream> +#include <string> +#include <vector> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + /*}}}*/ + +using namespace std; + +// poor mans unordered_map::try_emplace for C++11 as it is a C++17 feature /*{{{*/ +template <typename Arg> +static void try_emplace(std::unordered_map<std::string, std::string> &fields, std::string &&name, Arg &&value) +{ + if (fields.find(name) == fields.end()) + fields.emplace(std::move(name), std::forward<Arg>(value)); +} + /*}}}*/ + +// AcqMethod::pkgAcqMethod - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* This constructs the initialization text */ +pkgAcqMethod::pkgAcqMethod(const char *Ver,unsigned long Flags) +{ + std::unordered_map<std::string, std::string> fields; + try_emplace(fields, "Version", Ver); + if ((Flags & SingleInstance) == SingleInstance) + try_emplace(fields, "Single-Instance", "true"); + + if ((Flags & Pipeline) == Pipeline) + try_emplace(fields, "Pipeline", "true"); + + if ((Flags & SendConfig) == SendConfig) + try_emplace(fields, "Send-Config", "true"); + + if ((Flags & LocalOnly) == LocalOnly) + try_emplace(fields, "Local-Only", "true"); + + if ((Flags & NeedsCleanup) == NeedsCleanup) + try_emplace(fields, "Needs-Cleanup", "true"); + + if ((Flags & Removable) == Removable) + try_emplace(fields, "Removable", "true"); + + if ((Flags & AuxRequests) == AuxRequests) + try_emplace(fields, "AuxRequests", "true"); + + SendMessage("100 Capabilities", std::move(fields)); + + SetNonBlock(STDIN_FILENO,true); + + Queue = 0; + QueueBack = 0; +} + /*}}}*/ +void pkgAcqMethod::SendMessage(std::string const &header, std::unordered_map<std::string, std::string> &&fields) /*{{{*/ +{ + auto CheckKey = [](std::string const &str) { + // Space, hyphen-minus, and alphanum are allowed for keys/headers. + return str.find_first_not_of(" -0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz") == std::string::npos; + }; + + auto CheckValue = [](std::string const &str) { + return std::all_of(str.begin(), str.end(), [](unsigned char c) -> bool { + return c > 127 // unicode + || (c > 31 && c < 127) // printable chars + || c == '\n' || c == '\t'; // special whitespace + }); + }; + + auto Error = [this]() { + _error->Error("SECURITY: Message contains control characters, rejecting."); + _error->DumpErrors(); + SendMessage("400 URI Failure", {{"URI", "<UNKNOWN>"}, {"Message", "SECURITY: Message contains control characters, rejecting."}}); + abort(); + }; + + if (!CheckKey(header)) + return Error(); + + for (auto const &f : fields) + { + if (!CheckKey(f.first)) + return Error(); + if (!CheckValue(f.second)) + return Error(); + } + + std::cout << header << '\n'; + for (auto const &f : fields) + { + if (f.second.empty()) + continue; + std::cout << f.first << ": "; + auto const lines = VectorizeString(f.second, '\n'); + if (likely(lines.empty() == false)) + { + std::copy(lines.begin(), std::prev(lines.end()), std::ostream_iterator<std::string>(std::cout, "\n ")); + std::cout << *lines.rbegin(); + } + std::cout << '\n'; + } + std::cout << '\n' + << std::flush; +} + /*}}}*/ +// AcqMethod::Fail - A fetch has failed /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMethod::Fail(bool Transient) +{ + string Err = "Undetermined Error"; + if (_error->empty() == false) + { + Err.clear(); + while (_error->empty() == false) + { + std::string msg; + if (_error->PopMessage(msg)) + { + if (Err.empty() == false) + Err.append("\n"); + Err.append(msg); + } + } + } + Fail(Err, Transient); +} + /*}}}*/ +// AcqMethod::Fail - A fetch has failed /*{{{*/ +void pkgAcqMethod::Fail(string Err,bool Transient) +{ + // Strip out junk from the error messages + std::transform(Err.begin(), Err.end(), Err.begin(), [](char const c) { + if (c == '\r' || c == '\n') + return ' '; + return c; + }); + if (IP.empty() == false && _config->FindB("Acquire::Failure::ShowIP", true) == true) + Err.append(" ").append(IP); + + std::unordered_map<std::string, std::string> fields; + if (Queue != nullptr) + try_emplace(fields, "URI", Queue->Uri); + else + try_emplace(fields, "URI", "<UNKNOWN>"); + try_emplace(fields, "Message", Err); + + if(FailReason.empty() == false) + try_emplace(fields, "FailReason", FailReason); + if (UsedMirror.empty() == false) + try_emplace(fields, "UsedMirror", UsedMirror); + if (Transient == true) + try_emplace(fields, "Transient-Failure", "true"); + + SendMessage("400 URI Failure", std::move(fields)); + + if (Queue != nullptr) + Dequeue(); +} + /*}}}*/ +// AcqMethod::DropPrivsOrDie - Drop privileges or die /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMethod::DropPrivsOrDie() +{ + if (!DropPrivileges()) { + Fail(false); + exit(112); /* call the european emergency number */ + } +} + + /*}}}*/ +// AcqMethod::URIStart - Indicate a download is starting /*{{{*/ +void pkgAcqMethod::URIStart(FetchResult &Res) +{ + if (Queue == 0) + abort(); + + std::unordered_map<std::string, std::string> fields; + try_emplace(fields, "URI", Queue->Uri); + if (Res.Size != 0) + try_emplace(fields, "Size", std::to_string(Res.Size)); + if (Res.LastModified != 0) + try_emplace(fields, "Last-Modified", TimeRFC1123(Res.LastModified, true)); + if (Res.ResumePoint != 0) + try_emplace(fields, "Resume-Point", std::to_string(Res.ResumePoint)); + if (UsedMirror.empty() == false) + try_emplace(fields, "UsedMirror", UsedMirror); + + SendMessage("200 URI Start", std::move(fields)); +} + /*}}}*/ +// AcqMethod::URIDone - A URI is finished /*{{{*/ +static void printHashStringList(std::unordered_map<std::string, std::string> &fields, std::string const &Prefix, HashStringList const &list) +{ + for (auto const &hash : list) + { + // very old compatibility name for MD5Sum + if (hash.HashType() == "MD5Sum") + try_emplace(fields, Prefix + "MD5-Hash", hash.HashValue()); + try_emplace(fields, Prefix + hash.HashType() + "-Hash", hash.HashValue()); + } +} +void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt) +{ + if (Queue == 0) + abort(); + + std::unordered_map<std::string, std::string> fields; + try_emplace(fields, "URI", Queue->Uri); + if (Res.Filename.empty() == false) + try_emplace(fields, "Filename", Res.Filename); + if (Res.Size != 0) + try_emplace(fields, "Size", std::to_string(Res.Size)); + if (Res.LastModified != 0) + try_emplace(fields, "Last-Modified", TimeRFC1123(Res.LastModified, true)); + printHashStringList(fields, "", Res.Hashes); + + if (UsedMirror.empty() == false) + try_emplace(fields, "UsedMirror", UsedMirror); + if (Res.GPGVOutput.empty() == false) + { + std::ostringstream os; + std::copy(Res.GPGVOutput.begin(), Res.GPGVOutput.end() - 1, std::ostream_iterator<std::string>(os, "\n")); + os << *Res.GPGVOutput.rbegin(); + try_emplace(fields, "GPGVOutput", os.str()); + } + if (Res.ResumePoint != 0) + try_emplace(fields, "Resume-Point", std::to_string(Res.ResumePoint)); + if (Res.IMSHit == true) + try_emplace(fields, "IMS-Hit", "true"); + + if (Alt != nullptr) + { + if (Alt->Filename.empty() == false) + try_emplace(fields, "Alt-Filename", Alt->Filename); + if (Alt->Size != 0) + try_emplace(fields, "Alt-Size", std::to_string(Alt->Size)); + if (Alt->LastModified != 0) + try_emplace(fields, "Alt-Last-Modified", TimeRFC1123(Alt->LastModified, true)); + if (Alt->IMSHit == true) + try_emplace(fields, "Alt-IMS-Hit", "true"); + printHashStringList(fields, "Alt-", Alt->Hashes); + } + + SendMessage("201 URI Done", std::move(fields)); + Dequeue(); +} + /*}}}*/ +// AcqMethod::MediaFail - Synchronous request for new media /*{{{*/ +// --------------------------------------------------------------------- +/* This sends a 403 Media Failure message to the APT and waits for it + to be ackd */ +bool pkgAcqMethod::MediaFail(string Required,string Drive) +{ + fprintf(stdout, "403 Media Failure\nMedia: %s\nDrive: %s\n", + Required.c_str(),Drive.c_str()); + std::cout << "\n" << std::flush; + + vector<string> MyMessages; + + /* Here we read messages until we find a 603, each non 603 message is + appended to the main message list for later processing */ + while (1) + { + if (WaitFd(STDIN_FILENO) == false) + return false; + + if (ReadMessages(STDIN_FILENO,MyMessages) == false) + return false; + + string Message = MyMessages.front(); + MyMessages.erase(MyMessages.begin()); + + // Fetch the message number + char *End; + int Number = strtol(Message.c_str(),&End,10); + if (End == Message.c_str()) + { + cerr << "Malformed message!" << endl; + exit(100); + } + + // Change ack + if (Number == 603) + { + while (MyMessages.empty() == false) + { + Messages.push_back(MyMessages.front()); + MyMessages.erase(MyMessages.begin()); + } + + return !StringToBool(LookupTag(Message,"Failed"),false); + } + + Messages.push_back(Message); + } +} + /*}}}*/ +// AcqMethod::Configuration - Handle the configuration message /*{{{*/ +// --------------------------------------------------------------------- +/* This parses each configuration entry and puts it into the _config + Configuration class. */ +bool pkgAcqMethod::Configuration(string Message) +{ + ::Configuration &Cnf = *_config; + + const char *I = Message.c_str(); + const char *MsgEnd = I + Message.length(); + + unsigned int Length = strlen("Config-Item"); + for (; I + Length < MsgEnd; I++) + { + // Not a config item + if (I[Length] != ':' || stringcasecmp(I,I+Length,"Config-Item") != 0) + continue; + + I += Length + 1; + + for (; I < MsgEnd && *I == ' '; I++); + const char *Equals = (const char*) memchr(I, '=', MsgEnd - I); + if (Equals == NULL) + return false; + const char *End = (const char*) memchr(Equals, '\n', MsgEnd - Equals); + if (End == NULL) + End = MsgEnd; + + Cnf.Set(DeQuoteString(string(I,Equals-I)), + DeQuoteString(string(Equals+1,End-Equals-1))); + I = End; + } + + return true; +} + /*}}}*/ +// AcqMethod::Run - Run the message engine /*{{{*/ +// --------------------------------------------------------------------- +/* Fetch any messages and execute them. In single mode it returns 1 if + there are no more available messages - any other result is a + fatal failure code! */ +int pkgAcqMethod::Run(bool Single) +{ + while (1) + { + // Block if the message queue is empty + if (Messages.empty() == true) + { + if (Single == false) + if (WaitFd(STDIN_FILENO) == false) + break; + if (ReadMessages(STDIN_FILENO,Messages) == false) + break; + } + + // Single mode exits if the message queue is empty + if (Single == true && Messages.empty() == true) + return -1; + + string Message = Messages.front(); + Messages.erase(Messages.begin()); + + // Fetch the message number + char *End; + int Number = strtol(Message.c_str(),&End,10); + if (End == Message.c_str()) + { + cerr << "Malformed message!" << endl; + return 100; + } + + switch (Number) + { + case 601: + if (Configuration(Message) == false) + return 100; + break; + + case 600: + { + FetchItem *Tmp = new FetchItem; + + Tmp->Uri = LookupTag(Message,"URI"); + Tmp->Proxy(LookupTag(Message, "Proxy")); + Tmp->DestFile = LookupTag(Message,"FileName"); + if (RFC1123StrToTime(LookupTag(Message,"Last-Modified").c_str(),Tmp->LastModified) == false) + Tmp->LastModified = 0; + Tmp->IndexFile = StringToBool(LookupTag(Message,"Index-File"),false); + Tmp->FailIgnore = StringToBool(LookupTag(Message,"Fail-Ignore"),false); + Tmp->ExpectedHashes = HashStringList(); + for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t) + { + std::string tag = "Expected-"; + tag.append(*t); + std::string const hash = LookupTag(Message, tag.c_str()); + if (hash.empty() == false) + Tmp->ExpectedHashes.push_back(HashString(*t, hash)); + } + char *End; + if (Tmp->ExpectedHashes.FileSize() > 0) + Tmp->MaximumSize = Tmp->ExpectedHashes.FileSize(); + else + Tmp->MaximumSize = strtoll(LookupTag(Message, "Maximum-Size", "0").c_str(), &End, 10); + Tmp->Next = 0; + + // Append it to the list + FetchItem **I = &Queue; + for (; *I != 0; I = &(*I)->Next); + *I = Tmp; + if (QueueBack == 0) + QueueBack = Tmp; + + // Notify that this item is to be fetched. + if (URIAcquire(Message, Tmp) == false) + Fail(); + + break; + } + } + } + + Exit(); + return 0; +} + /*}}}*/ +// AcqMethod::PrintStatus - privately really send a log/status message /*{{{*/ +void pkgAcqMethod::PrintStatus(char const * const header, const char* Format, + va_list &args) const +{ + string CurrentURI = "<UNKNOWN>"; + if (Queue != 0) + CurrentURI = Queue->Uri; + if (UsedMirror.empty() == true) + fprintf(stdout, "%s\nURI: %s\nMessage: ", + header, CurrentURI.c_str()); + else + fprintf(stdout, "%s\nURI: %s\nUsedMirror: %s\nMessage: ", + header, CurrentURI.c_str(), UsedMirror.c_str()); + vfprintf(stdout,Format,args); + std::cout << "\n\n" << std::flush; +} + /*}}}*/ +// AcqMethod::Log - Send a log message /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMethod::Log(const char *Format,...) +{ + va_list args; + va_start(args,Format); + PrintStatus("101 Log", Format, args); + va_end(args); +} + /*}}}*/ +// AcqMethod::Status - Send a status message /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMethod::Status(const char *Format,...) +{ + va_list args; + va_start(args,Format); + PrintStatus("102 Status", Format, args); + va_end(args); +} + /*}}}*/ +// AcqMethod::Redirect - Send a redirect message /*{{{*/ +// --------------------------------------------------------------------- +/* This method sends the redirect message and dequeues the item as + * the worker will enqueue again later on to the right queue */ +void pkgAcqMethod::Redirect(const string &NewURI) +{ + if (NewURI.find_first_not_of(" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~") != std::string::npos) + { + _error->Error("SECURITY: URL redirect target contains control characters, rejecting."); + Fail(); + return; + } + std::unordered_map<std::string, std::string> fields; + try_emplace(fields, "URI", Queue->Uri); + try_emplace(fields, "New-URI", NewURI); + SendMessage("103 Redirect", std::move(fields)); + Dequeue(); +} + /*}}}*/ +// AcqMethod::FetchResult::FetchResult - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgAcqMethod::FetchResult::FetchResult() : LastModified(0), + IMSHit(false), Size(0), ResumePoint(0), d(NULL) +{ +} + /*}}}*/ +// AcqMethod::FetchResult::TakeHashes - Load hashes /*{{{*/ +// --------------------------------------------------------------------- +/* This hides the number of hashes we are supporting from the caller. + It just deals with the hash class. */ +void pkgAcqMethod::FetchResult::TakeHashes(class Hashes &Hash) +{ + Hashes = Hash.GetHashStringList(); +} + /*}}}*/ +void pkgAcqMethod::Dequeue() { /*{{{*/ + FetchItem const * const Tmp = Queue; + Queue = Queue->Next; + if (Tmp == QueueBack) + QueueBack = Queue; + delete Tmp; +} + /*}}}*/ +pkgAcqMethod::~pkgAcqMethod() {} + +struct pkgAcqMethod::FetchItem::Private +{ + std::string Proxy; +}; + +pkgAcqMethod::FetchItem::FetchItem() : Next(nullptr), DestFileFd(-1), LastModified(0), IndexFile(false), + FailIgnore(false), MaximumSize(0), d(new Private) +{} + +std::string pkgAcqMethod::FetchItem::Proxy() +{ + return d->Proxy; +} + +void pkgAcqMethod::FetchItem::Proxy(std::string const &Proxy) +{ + d->Proxy = Proxy; +} + +pkgAcqMethod::FetchItem::~FetchItem() { delete d; } + +pkgAcqMethod::FetchResult::~FetchResult() {} diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h new file mode 100644 index 0000000..664b95c --- /dev/null +++ b/apt-pkg/acquire-method.h @@ -0,0 +1,140 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire Method - Method helper class + functions + + These functions are designed to be used within the method task to + ease communication with APT. + + ##################################################################### */ + /*}}}*/ + +/** \addtogroup acquire + * @{ + * + * \file acquire-method.h + */ + +#ifndef PKGLIB_ACQUIRE_METHOD_H +#define PKGLIB_ACQUIRE_METHOD_H + +#include <apt-pkg/hashes.h> +#include <apt-pkg/macros.h> + +#include <stdarg.h> +#include <time.h> + +#include <string> +#include <unordered_map> +#include <vector> + +#ifndef APT_8_CLEANER_HEADERS +#include <apt-pkg/configuration.h> +#include <apt-pkg/strutl.h> +#endif + +class pkgAcqMethod +{ + protected: + + struct FetchItem + { + FetchItem *Next; + + std::string Uri; + std::string DestFile; + int DestFileFd; + time_t LastModified; + bool IndexFile; + bool FailIgnore; + HashStringList ExpectedHashes; + // a maximum size we will download, this can be the exact filesize + // for when we know it or a arbitrary limit when we don't know the + // filesize (like a InRelease file) + unsigned long long MaximumSize; + + FetchItem(); + virtual ~FetchItem(); + std::string Proxy(); // For internal use only. + void Proxy(std::string const &Proxy) APT_HIDDEN; + + private: + struct Private; + Private *const d; + }; + + struct FetchResult + { + HashStringList Hashes; + std::vector<std::string> GPGVOutput; + time_t LastModified; + bool IMSHit; + std::string Filename; + unsigned long long Size; + unsigned long long ResumePoint; + + void TakeHashes(class Hashes &Hash); + FetchResult(); + virtual ~FetchResult(); + private: + void * const d; + }; + + // State + std::vector<std::string> Messages; + FetchItem *Queue; + FetchItem *QueueBack; + std::string FailReason; + std::string UsedMirror; + std::string IP; + + // Handlers for messages + virtual bool Configuration(std::string Message); + virtual bool Fetch(FetchItem * /*Item*/) {return true;}; + virtual bool URIAcquire(std::string const &/*Message*/, FetchItem *Itm) { return Fetch(Itm); }; + + // Outgoing messages + void Fail(bool Transient = false); + inline void Fail(const char *Why, bool Transient = false) {Fail(std::string(Why),Transient);}; + virtual void Fail(std::string Why, bool Transient = false); + virtual void URIStart(FetchResult &Res); + virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + void SendMessage(std::string const &header, std::unordered_map<std::string, std::string> &&fields); + + bool MediaFail(std::string Required,std::string Drive); + virtual void Exit() {}; + + void PrintStatus(char const * const header, const char* Format, va_list &args) const; + + public: + enum CnfFlags + { + SingleInstance = (1 << 0), + Pipeline = (1 << 1), + SendConfig = (1 << 2), + LocalOnly = (1 << 3), + NeedsCleanup = (1 << 4), + Removable = (1 << 5), + AuxRequests = (1 << 6) + }; + + void Log(const char *Format,...); + void Status(const char *Format,...); + + void Redirect(const std::string &NewURI); + + int Run(bool Single = false); + inline void SetFailReason(std::string Msg) {FailReason = Msg;}; + inline void SetIP(std::string aIP) {IP = aIP;}; + + pkgAcqMethod(const char *Ver,unsigned long Flags = 0); + virtual ~pkgAcqMethod(); + void DropPrivsOrDie(); + private: + APT_HIDDEN void Dequeue(); +}; + +/** @} */ + +#endif diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc new file mode 100644 index 0000000..b361861 --- /dev/null +++ b/apt-pkg/acquire-worker.cc @@ -0,0 +1,964 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Acquire Worker + + The worker process can startup either as a Configuration prober + or as a queue runner. As a configuration prober it only reads the + configuration message and + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/acquire-item.h> +#include <apt-pkg/acquire-worker.h> +#include <apt-pkg/acquire.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/proxy.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <iostream> +#include <string> +#include <vector> + +#include <sstream> +#include <errno.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +// Worker::Worker - Constructor for Queue startup /*{{{*/ +pkgAcquire::Worker::Worker(Queue *Q, MethodConfig *Cnf, pkgAcquireStatus *log) : + d(NULL), OwnerQ(Q), Log(log), Config(Cnf), Access(Cnf->Access), + CurrentItem(nullptr), CurrentSize(0), TotalSize(0) +{ + Construct(); +} + /*}}}*/ +// Worker::Worker - Constructor for method config startup /*{{{*/ +pkgAcquire::Worker::Worker(MethodConfig *Cnf) : Worker(nullptr, Cnf, nullptr) +{ +} + /*}}}*/ +// Worker::Construct - Constructor helper /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcquire::Worker::Construct() +{ + NextQueue = 0; + NextAcquire = 0; + Process = -1; + InFd = -1; + OutFd = -1; + OutReady = false; + InReady = false; + Debug = _config->FindB("Debug::pkgAcquire::Worker",false); +} + /*}}}*/ +// Worker::~Worker - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgAcquire::Worker::~Worker() +{ + close(InFd); + close(OutFd); + + if (Process > 0) + { + /* Closing of stdin is the signal to exit and die when the process + indicates it needs cleanup */ + if (Config->NeedsCleanup == false) + kill(Process,SIGINT); + ExecWait(Process,Access.c_str(),true); + } +} + /*}}}*/ +// Worker::Start - Start the worker process /*{{{*/ +// --------------------------------------------------------------------- +/* This forks the method and inits the communication channel */ +bool pkgAcquire::Worker::Start() +{ + // Get the method path + constexpr char const * const methodsDir = "Dir::Bin::Methods"; + std::string const confItem = std::string(methodsDir) + "::" + Access; + std::string Method; + if (_config->Exists(confItem)) + Method = _config->FindFile(confItem.c_str()); + else if (Access == "ftp" || Access == "rsh" || Access == "ssh") + return _error->Error(_("The method '%s' is unsupported and disabled by default. Consider switching to http(s). Set Dir::Bin::Methods::%s to \"%s\" to enable it again."), Access.c_str(), Access.c_str(), Access.c_str()); + else + Method = _config->FindDir(methodsDir) + Access; + if (FileExists(Method) == false) + { + if (flNotDir(Method) == "false") + { + _error->Error(_("The method '%s' is explicitly disabled via configuration."), Access.c_str()); + if (Access == "http" || Access == "https") + _error->Notice(_("If you meant to use Tor remember to use %s instead of %s."), ("tor+" + Access).c_str(), Access.c_str()); + return false; + } + _error->Error(_("The method driver %s could not be found."),Method.c_str()); + std::string const A(Access.cbegin(), std::find(Access.cbegin(), Access.cend(), '+')); + std::string pkg; + strprintf(pkg, "apt-transport-%s", A.c_str()); + _error->Notice(_("Is the package %s installed?"), pkg.c_str()); + return false; + } + std::string const Calling = _config->FindDir(methodsDir) + Access; + + if (Debug == true) + { + std::clog << "Starting method '" << Calling << "'"; + if (Calling != Method) + std::clog << " ( via " << Method << " )"; + std::clog << endl; + } + + // Create the pipes + int Pipes[4] = {-1,-1,-1,-1}; + if (pipe(Pipes) != 0 || pipe(Pipes+2) != 0) + { + _error->Errno("pipe","Failed to create IPC pipe to subprocess"); + for (int I = 0; I != 4; I++) + close(Pipes[I]); + return false; + } + for (int I = 0; I != 4; I++) + SetCloseExec(Pipes[I],true); + + // Fork off the process + Process = ExecFork(); + if (Process == 0) + { + // Setup the FDs + dup2(Pipes[1],STDOUT_FILENO); + dup2(Pipes[2],STDIN_FILENO); + SetCloseExec(STDOUT_FILENO,false); + SetCloseExec(STDIN_FILENO,false); + SetCloseExec(STDERR_FILENO,false); + + const char * const Args[] = { Calling.c_str(), nullptr }; + execv(Method.c_str() ,const_cast<char **>(Args)); + std::cerr << "Failed to exec method " << Calling << " ( via " << Method << ")" << endl; + _exit(100); + } + + // Fix up our FDs + InFd = Pipes[0]; + OutFd = Pipes[3]; + SetNonBlock(Pipes[0],true); + SetNonBlock(Pipes[3],true); + close(Pipes[1]); + close(Pipes[2]); + OutReady = false; + InReady = true; + + // Read the configuration data + if (WaitFd(InFd) == false || + ReadMessages() == false) + return _error->Error(_("Method %s did not start correctly"),Method.c_str()); + + RunMessages(); + if (OwnerQ != 0) + SendConfiguration(); + + return true; +} + /*}}}*/ +// Worker::ReadMessages - Read all pending messages into the list /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgAcquire::Worker::ReadMessages() +{ + if (::ReadMessages(InFd,MessageQueue) == false) + return MethodFailure(); + return true; +} + /*}}}*/ +// Worker::RunMessage - Empty the message queue /*{{{*/ +// --------------------------------------------------------------------- +/* This takes the messages from the message queue and runs them through + the parsers in order. */ +enum class APT_HIDDEN MessageType +{ + CAPABILITIES = 100, + LOG = 101, + STATUS = 102, + REDIRECT = 103, + WARNING = 104, + URI_START = 200, + URI_DONE = 201, + AUX_REQUEST = 351, + URI_FAILURE = 400, + GENERAL_FAILURE = 401, + MEDIA_CHANGE = 403 +}; +static bool isDoomedItem(pkgAcquire::Item const * const Itm) +{ + auto const TransItm = dynamic_cast<pkgAcqTransactionItem const * const>(Itm); + if (TransItm == nullptr) + return false; + return TransItm->TransactionManager->State != pkgAcqTransactionItem::TransactionStarted; +} +static HashStringList GetHashesFromMessage(std::string const &Prefix, std::string const &Message) +{ + HashStringList hsl; + for (char const *const *type = HashString::SupportedHashes(); *type != NULL; ++type) + { + std::string const tagname = Prefix + *type + "-Hash"; + std::string const hashsum = LookupTag(Message, tagname.c_str()); + if (hashsum.empty() == false) + hsl.push_back(HashString(*type, hashsum)); + } + return hsl; +} +static void APT_NONNULL(3) ChangeSiteIsMirrorChange(std::string const |