summaryrefslogtreecommitdiffstats
path: root/methods
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-13 09:59:37 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-13 09:59:37 +0000
commit76e2632459410dec81337edb6a9fee33c9a660f3 (patch)
treea73345df208eede4a4daad340515c9328f34625c /methods
parentInitial commit. (diff)
downloadapt-76e2632459410dec81337edb6a9fee33c9a660f3.tar.xz
apt-76e2632459410dec81337edb6a9fee33c9a660f3.zip
Adding upstream version 2.7.12.upstream/2.7.12
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'methods')
-rw-r--r--methods/CMakeLists.txt36
-rw-r--r--methods/aptmethod.h595
-rw-r--r--methods/basehttp.cc957
-rw-r--r--methods/basehttp.h180
-rw-r--r--methods/cdrom.cc289
-rw-r--r--methods/connect.cc1053
-rw-r--r--methods/connect.h50
-rw-r--r--methods/copy.cc94
-rw-r--r--methods/file.cc133
-rw-r--r--methods/ftp.cc1188
-rw-r--r--methods/ftp.h91
-rw-r--r--methods/gpgv.cc597
-rw-r--r--methods/http.cc1053
-rw-r--r--methods/http.h142
-rw-r--r--methods/mirror.cc414
-rw-r--r--methods/rfc2553emu.cc244
-rw-r--r--methods/rfc2553emu.h112
-rw-r--r--methods/rred.cc885
-rw-r--r--methods/rsh.cc561
-rw-r--r--methods/rsh.h75
-rw-r--r--methods/store.cc147
21 files changed, 8896 insertions, 0 deletions
diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt
new file mode 100644
index 0000000..a5a3602
--- /dev/null
+++ b/methods/CMakeLists.txt
@@ -0,0 +1,36 @@
+# Create the executable targets
+include_directories($<$<BOOL:${SECCOMP_FOUND}>:${SECCOMP_INCLUDE_DIR}>)
+link_libraries(apt-pkg $<$<BOOL:${SECCOMP_FOUND}>:${SECCOMP_LIBRARIES}>)
+
+add_library(connectlib OBJECT connect.cc rfc2553emu.cc)
+
+add_executable(file file.cc)
+add_executable(copy copy.cc)
+add_executable(store store.cc)
+add_executable(gpgv gpgv.cc)
+add_executable(cdrom cdrom.cc)
+add_executable(http http.cc basehttp.cc $<TARGET_OBJECTS:connectlib>)
+add_executable(mirror mirror.cc)
+add_executable(ftp ftp.cc $<TARGET_OBJECTS:connectlib>)
+add_executable(rred rred.cc)
+add_executable(rsh rsh.cc)
+
+target_compile_definitions(connectlib PRIVATE ${GNUTLS_DEFINITIONS})
+target_include_directories(connectlib PRIVATE ${GNUTLS_INCLUDE_DIR})
+target_include_directories(http PRIVATE $<$<BOOL:${SYSTEMD_FOUND}>:${SYSTEMD_INCLUDE_DIRS}>)
+
+# Additional libraries to link against for networked stuff
+target_link_libraries(http ${GNUTLS_LIBRARIES} $<$<BOOL:${SYSTEMD_FOUND}>:${SYSTEMD_LIBRARIES}>)
+target_link_libraries(ftp ${GNUTLS_LIBRARIES})
+
+target_link_libraries(rred apt-private)
+
+# Install the library
+install(TARGETS file copy store gpgv cdrom http ftp rred rsh mirror
+ RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/apt/methods)
+
+add_links(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods mirror mirror+ftp mirror+http mirror+https mirror+file mirror+copy)
+add_links(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods rsh ssh)
+
+
+add_links(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods http https)
diff --git a/methods/aptmethod.h b/methods/aptmethod.h
new file mode 100644
index 0000000..26b8c0b
--- /dev/null
+++ b/methods/aptmethod.h
@@ -0,0 +1,595 @@
+#ifndef APT_APTMETHOD_H
+#define APT_APTMETHOD_H
+
+#include "config.h"
+
+#include <apt-pkg/acquire-method.h>
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/netrc.h>
+#include <apt-pkg/strutl.h>
+
+#include <algorithm>
+#include <locale>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <cstdlib>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <apti18n.h>
+
+#ifdef HAVE_SECCOMP
+#include <csignal>
+
+#include <seccomp.h>
+#endif
+
+enum class ResultState
+{
+ TRANSIENT_ERROR,
+ FATAL_ERROR,
+ SUCCESSFUL
+};
+
+static bool hasDoubleColon(std::string const &n)
+{
+ return n.find("::") != std::string::npos;
+}
+
+class aptConfigWrapperForMethods
+{
+protected:
+ std::vector<std::string> methodNames;
+public:
+ void setPostfixForMethodNames(char const * const postfix) APT_NONNULL(2)
+ {
+ methodNames.erase(std::remove_if(methodNames.begin(), methodNames.end(), hasDoubleColon), methodNames.end());
+ decltype(methodNames) toAdd;
+ for (auto && name: methodNames)
+ toAdd.emplace_back(name + "::" + postfix);
+ std::move(toAdd.begin(), toAdd.end(), std::back_inserter(methodNames));
+ }
+
+ bool DebugEnabled() const
+ {
+ if (methodNames.empty())
+ return false;
+ auto const sni = std::find_if_not(methodNames.crbegin(), methodNames.crend(), hasDoubleColon);
+ if (unlikely(sni == methodNames.crend()))
+ return false;
+ auto const ln = methodNames[methodNames.size() - 1];
+ // worst case: all three are the same
+ std::string confln, confsn, confpn;
+ strprintf(confln, "Debug::Acquire::%s", ln.c_str());
+ strprintf(confsn, "Debug::Acquire::%s", sni->c_str());
+ auto const pni = sni->substr(0, sni->find('+'));
+ strprintf(confpn, "Debug::Acquire::%s", pni.c_str());
+ return _config->FindB(confln,_config->FindB(confsn, _config->FindB(confpn, false)));
+ }
+ std::string ConfigFind(char const * const postfix, std::string const &defValue) const APT_NONNULL(2)
+ {
+ for (auto name = methodNames.rbegin(); name != methodNames.rend(); ++name)
+ {
+ std::string conf;
+ strprintf(conf, "Acquire::%s::%s", name->c_str(), postfix);
+ auto value = _config->Find(conf);
+ if (not value.empty())
+ return value;
+ }
+ return defValue;
+ }
+ std::string ConfigFind(std::string const &postfix, std::string const &defValue) const
+ {
+ return ConfigFind(postfix.c_str(), defValue);
+ }
+ bool ConfigFindB(char const * const postfix, bool const defValue) const APT_NONNULL(2)
+ {
+ return StringToBool(ConfigFind(postfix, defValue ? "yes" : "no"), defValue);
+ }
+ int ConfigFindI(char const * const postfix, int const defValue) const APT_NONNULL(2)
+ {
+ char *End;
+ std::string const value = ConfigFind(postfix, "");
+ auto const Res = strtol(value.c_str(), &End, 0);
+ if (value.c_str() == End)
+ return defValue;
+ return Res;
+ }
+
+ explicit aptConfigWrapperForMethods(std::string const &name) : methodNames{{name}} {}
+ explicit aptConfigWrapperForMethods(std::vector<std::string> &&names) : methodNames{std::move(names)} {}
+};
+
+class aptMethod : public pkgAcqMethod, public aptConfigWrapperForMethods
+{
+protected:
+ std::string const Binary;
+ unsigned long SeccompFlags;
+ enum Seccomp
+ {
+ BASE = (1 << 1),
+ NETWORK = (1 << 2),
+ DIRECTORY = (1 << 3),
+ };
+
+ public:
+ virtual bool Configuration(std::string Message) APT_OVERRIDE
+ {
+ if (pkgAcqMethod::Configuration(Message) == false)
+ return false;
+
+ std::string const conf = std::string("Binary::") + Binary;
+ _config->MoveSubTree(conf.c_str(), NULL);
+
+ DropPrivsOrDie();
+ if (LoadSeccomp() == false)
+ return false;
+
+ return true;
+ }
+
+ bool RunningInQemu(void)
+ {
+ int status;
+ pid_t pid;
+
+ pid = fork();
+ if (pid == 0)
+ {
+ close(0);
+ close(1);
+ close(2);
+ setenv("QEMU_VERSION", "meow", 1);
+ char path[] = LIBEXEC_DIR "/apt-helper";
+ char *const argv[] = {path, NULL};
+ execv(argv[0], argv);
+ _exit(255);
+ }
+
+ // apt-helper is supposed to exit with an error. If it exited with 0,
+ // qemu-user had problems with QEMU_VERSION and returned 0 => running in
+ // qemu-user.
+
+ if (waitpid(pid, &status, 0) == pid && WIFEXITED(status) && WEXITSTATUS(status) == 0)
+ return true;
+
+ return false;
+ }
+
+ bool LoadSeccomp()
+ {
+#ifdef HAVE_SECCOMP
+ int rc;
+ scmp_filter_ctx ctx = NULL;
+
+ if (SeccompFlags == 0)
+ return true;
+
+ if (_config->FindB("APT::Sandbox::Seccomp", false) == false)
+ return true;
+
+ if (RunningInQemu() == true)
+ {
+ Warning("Running in qemu-user, not using seccomp");
+ return true;
+ }
+
+ ctx = seccomp_init(SCMP_ACT_TRAP);
+ if (ctx == NULL)
+ return _error->FatalE("HttpMethod::Configuration", "Cannot init seccomp");
+
+#define ALLOW(what) \
+ if ((rc = seccomp_rule_add(ctx, SCMP_ACT_ALLOW, SCMP_SYS(what), 0))) \
+ return _error->FatalE("HttpMethod::Configuration", "Cannot allow %s: %s", #what, strerror(-rc));
+
+ for (auto &custom : _config->FindVector("APT::Sandbox::Seccomp::Trap"))
+ {
+ if ((rc = seccomp_rule_add(ctx, SCMP_ACT_TRAP, seccomp_syscall_resolve_name(custom.c_str()), 0)))
+ return _error->FatalE("HttpMethod::Configuration", "Cannot trap %s: %s", custom.c_str(), strerror(-rc));
+ }
+
+ ALLOW(access);
+ ALLOW(arch_prctl);
+ ALLOW(brk);
+ ALLOW(chmod);
+ ALLOW(chown);
+ ALLOW(chown32);
+ ALLOW(clock_getres);
+ ALLOW(clock_getres_time64);
+ ALLOW(clock_gettime);
+ ALLOW(clock_gettime64);
+ ALLOW(clock_nanosleep);
+ ALLOW(clock_nanosleep_time64);
+ ALLOW(close);
+ ALLOW(creat);
+ ALLOW(dup);
+ ALLOW(dup2);
+ ALLOW(dup3);
+ ALLOW(exit);
+ ALLOW(exit_group);
+ ALLOW(faccessat);
+ ALLOW(fchmod);
+ ALLOW(fchmodat);
+ ALLOW(fchown);
+ ALLOW(fchown32);
+ ALLOW(fchownat);
+ ALLOW(fcntl);
+ ALLOW(fcntl64);
+ ALLOW(fdatasync);
+ ALLOW(flock);
+ ALLOW(fstat);
+ ALLOW(fstat64);
+ ALLOW(fstatat64);
+ ALLOW(fstatfs);
+ ALLOW(fstatfs64);
+ ALLOW(fsync);
+ ALLOW(ftime);
+ ALLOW(ftruncate);
+ ALLOW(ftruncate64);
+ ALLOW(futex);
+ ALLOW(futex_time64);
+ ALLOW(futimesat);
+ ALLOW(getegid);
+ ALLOW(getegid32);
+ ALLOW(geteuid);
+ ALLOW(geteuid32);
+ ALLOW(getgid);
+ ALLOW(getgid32);
+ ALLOW(getgroups);
+ ALLOW(getgroups32);
+ ALLOW(getpeername);
+ ALLOW(getpgid);
+ ALLOW(getpgrp);
+ ALLOW(getpid);
+ ALLOW(getppid);
+ ALLOW(getrandom);
+ ALLOW(getresgid);
+ ALLOW(getresgid32);
+ ALLOW(getresuid);
+ ALLOW(getresuid32);
+ ALLOW(getrlimit);
+ ALLOW(get_robust_list);
+ ALLOW(getrusage);
+ ALLOW(gettid);
+ ALLOW(gettimeofday);
+ ALLOW(getuid);
+ ALLOW(getuid32);
+ ALLOW(ioctl);
+ ALLOW(lchown);
+ ALLOW(lchown32);
+ ALLOW(_llseek);
+ ALLOW(lseek);
+ ALLOW(lstat);
+ ALLOW(lstat64);
+ ALLOW(madvise);
+ ALLOW(mmap);
+ ALLOW(mmap2);
+ ALLOW(mprotect);
+ ALLOW(mremap);
+ ALLOW(msync);
+ ALLOW(munmap);
+ ALLOW(nanosleep);
+ ALLOW(newfstatat);
+ ALLOW(_newselect);
+ ALLOW(oldfstat);
+ ALLOW(oldlstat);
+ ALLOW(oldolduname);
+ ALLOW(oldstat);
+ ALLOW(olduname);
+ ALLOW(open);
+ ALLOW(openat);
+ ALLOW(pipe);
+ ALLOW(pipe2);
+ ALLOW(poll);
+ ALLOW(ppoll);
+ ALLOW(ppoll_time64);
+ ALLOW(prctl);
+ ALLOW(prlimit64);
+ ALLOW(pselect6);
+ ALLOW(pselect6_time64);
+ ALLOW(read);
+ ALLOW(readv);
+ ALLOW(rename);
+ ALLOW(renameat);
+ ALLOW(renameat2);
+ ALLOW(restart_syscall);
+ ALLOW(rt_sigaction);
+ ALLOW(rt_sigpending);
+ ALLOW(rt_sigprocmask);
+ ALLOW(rt_sigqueueinfo);
+ ALLOW(rt_sigreturn);
+ ALLOW(rt_sigsuspend);
+ ALLOW(rt_sigtimedwait);
+ ALLOW(sched_yield);
+ ALLOW(select);
+ ALLOW(set_robust_list);
+ ALLOW(sigaction);
+ ALLOW(sigpending);
+ ALLOW(sigprocmask);
+ ALLOW(sigreturn);
+ ALLOW(sigsuspend);
+ ALLOW(stat);
+ ALLOW(stat64);
+ ALLOW(statfs);
+ ALLOW(statfs64);
+#ifdef __NR_statx
+ ALLOW(statx);
+#endif
+ ALLOW(sync);
+ ALLOW(syscall);
+ ALLOW(sysinfo);
+ ALLOW(time);
+ ALLOW(truncate);
+ ALLOW(truncate64);
+ ALLOW(ugetrlimit);
+ ALLOW(umask);
+ ALLOW(uname);
+ ALLOW(unlink);
+ ALLOW(unlinkat);
+ ALLOW(utime);
+ ALLOW(utimensat);
+ ALLOW(utimensat_time64);
+ ALLOW(utimes);
+ ALLOW(write);
+ ALLOW(writev);
+
+ if ((SeccompFlags & Seccomp::NETWORK) != 0)
+ {
+ ALLOW(bind);
+ ALLOW(connect);
+ ALLOW(getsockname);
+ ALLOW(getsockopt);
+ ALLOW(recv);
+ ALLOW(recvfrom);
+ ALLOW(recvmmsg);
+ ALLOW(recvmmsg_time64);
+ ALLOW(recvmsg);
+ ALLOW(send);
+ ALLOW(sendmmsg);
+ ALLOW(sendmsg);
+ ALLOW(sendto);
+ ALLOW(setsockopt);
+ ALLOW(shutdown);
+ ALLOW(socket);
+ ALLOW(socketcall);
+ }
+
+ if ((SeccompFlags & Seccomp::DIRECTORY) != 0)
+ {
+ ALLOW(readdir);
+ ALLOW(getdents);
+ ALLOW(getdents64);
+ }
+
+ if (getenv("FAKED_MODE"))
+ {
+ ALLOW(semop);
+ ALLOW(semget);
+ ALLOW(msgsnd);
+ ALLOW(msgrcv);
+ ALLOW(msgget);
+ ALLOW(msgctl);
+ ALLOW(ipc);
+ }
+
+ for (auto &custom : _config->FindVector("APT::Sandbox::Seccomp::Allow"))
+ {
+ if ((rc = seccomp_rule_add(ctx, SCMP_ACT_ALLOW, seccomp_syscall_resolve_name(custom.c_str()), 0)))
+ return _error->FatalE("aptMethod::Configuration", "Cannot allow %s: %s", custom.c_str(), strerror(-rc));
+ }
+
+#undef ALLOW
+
+ rc = seccomp_load(ctx);
+ if (rc == -EINVAL)
+ {
+ std::string msg;
+ strprintf(msg, "aptMethod::Configuration: could not load seccomp policy: %s", strerror(-rc));
+ Warning(std::move(msg));
+ }
+ else if (rc != 0)
+ return _error->FatalE("aptMethod::Configuration", "could not load seccomp policy: %s", strerror(-rc));
+
+ if (_config->FindB("APT::Sandbox::Seccomp::Print", true))
+ {
+ struct sigaction action;
+ memset(&action, 0, sizeof(action));
+ sigemptyset(&action.sa_mask);
+ action.sa_sigaction = [](int, siginfo_t *info, void *) {
+ // Formats a number into a 10 digit ASCII string
+ char buffer[10];
+ int number = info->si_syscall;
+
+ for (int i = sizeof(buffer) - 1; i >= 0; i--)
+ {
+ buffer[i] = (number % 10) + '0';
+ number /= 10;
+ }
+
+ constexpr const char *str1 = "\n **** Seccomp prevented execution of syscall ";
+ constexpr const char *str2 = " on architecture ";
+ constexpr const char *str3 = " ****\n";
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+#endif
+ write(2, str1, strlen(str1));
+ write(2, buffer, sizeof(buffer));
+ write(2, str2, strlen(str2));
+ write(2, COMMON_ARCH, strlen(COMMON_ARCH));
+ write(2, str3, strlen(str3));
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+ _exit(31);
+ };
+ action.sa_flags = SA_SIGINFO;
+
+ sigaction(SIGSYS, &action, nullptr);
+ }
+#endif
+ return true;
+ }
+
+ bool CalculateHashes(FetchItem const * const Itm, FetchResult &Res) const APT_NONNULL(2)
+ {
+ Hashes Hash(Itm->ExpectedHashes);
+ FileFd Fd;
+ if (Fd.Open(Res.Filename, FileFd::ReadOnly) == false || Hash.AddFD(Fd) == false)
+ return false;
+ Res.TakeHashes(Hash);
+ return true;
+ }
+
+ void Warning(std::string &&msg)
+ {
+ std::unordered_map<std::string, std::string> fields;
+ if (Queue != 0)
+ fields.emplace("URI", Queue->Uri);
+ else
+ fields.emplace("URI", "<UNKNOWN>");
+ if (not UsedMirror.empty())
+ fields.emplace("UsedMirror", UsedMirror);
+ fields.emplace("Message", std::move(msg));
+ SendMessage("104 Warning", std::move(fields));
+ }
+
+ bool TransferModificationTimes(char const * const From, char const * const To, time_t &LastModified) APT_NONNULL(2, 3)
+ {
+ if (strcmp(To, "/dev/null") == 0)
+ return true;
+
+ struct stat Buf2;
+ if (lstat(To, &Buf2) != 0 || S_ISLNK(Buf2.st_mode))
+ return true;
+
+ struct stat Buf;
+ if (stat(From, &Buf) != 0)
+ return _error->Errno("stat",_("Failed to stat"));
+
+ // we don't use utimensat here for compatibility reasons: #738567
+ struct timeval times[2];
+ times[0].tv_sec = Buf.st_atime;
+ LastModified = times[1].tv_sec = Buf.st_mtime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ if (utimes(To, times) != 0)
+ return _error->Errno("utimes",_("Failed to set modification time"));
+ return true;
+ }
+
+ // This is a copy of #pkgAcqMethod::Dequeue which is private & hidden
+ void Dequeue()
+ {
+ FetchItem const *const Tmp = Queue;
+ Queue = Queue->Next;
+ if (Tmp == QueueBack)
+ QueueBack = Queue;
+ delete Tmp;
+ }
+ static std::string URIEncode(std::string const &part)
+ {
+ // The "+" is encoded as a workaround for an S3 bug (LP#1003633 and LP#1086997)
+ return QuoteString(part, _config->Find("Acquire::URIEncode", "+~ ").c_str());
+ }
+
+ static std::string DecodeSendURI(std::string const &part)
+ {
+ if (_config->FindB("Acquire::Send-URI-Encoded", false))
+ return DeQuoteString(part);
+ return part;
+ }
+
+ aptMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3)
+ : pkgAcqMethod(Ver, Flags), aptConfigWrapperForMethods(Binary), Binary(std::move(Binary)), SeccompFlags(0)
+ {
+ try {
+ std::locale::global(std::locale(""));
+ } catch (...) {
+ setlocale(LC_ALL, "");
+ }
+ }
+};
+class aptAuthConfMethod : public aptMethod
+{
+ std::vector<std::unique_ptr<FileFd>> authconfs;
+
+ public:
+ virtual bool Configuration(std::string Message) APT_OVERRIDE
+ {
+ if (pkgAcqMethod::Configuration(Message) == false)
+ return false;
+
+ std::string const conf = std::string("Binary::") + Binary;
+ _config->MoveSubTree(conf.c_str(), NULL);
+
+ // ignore errors with opening the auth file as it doesn't need to exist
+ _error->PushToStack();
+ auto const netrc = _config->FindFile("Dir::Etc::netrc");
+ if (netrc.empty() == false)
+ {
+ authconfs.emplace_back(new FileFd());
+ authconfs.back()->Open(netrc, FileFd::ReadOnly);
+ }
+
+ auto const netrcparts = _config->FindDir("Dir::Etc::netrcparts");
+ if (netrcparts.empty() == false)
+ {
+ for (auto &&netrcpart : GetListOfFilesInDir(netrcparts, "conf", true, true))
+ {
+ authconfs.emplace_back(new FileFd());
+ authconfs.back()->Open(netrcpart, FileFd::ReadOnly);
+ }
+ }
+ _error->RevertToStack();
+
+ DropPrivsOrDie();
+
+ if (LoadSeccomp() == false)
+ return false;
+
+ return true;
+ }
+
+ bool MaybeAddAuthTo(URI &uri)
+ {
+ bool result = true;
+
+ if (uri.User.empty() == false || uri.Password.empty() == false)
+ return true;
+
+ _error->PushToStack();
+ for (auto &authconf : authconfs)
+ {
+ if (authconf->IsOpen() == false)
+ continue;
+ if (authconf->Seek(0) == false)
+ {
+ result = false;
+ continue;
+ }
+
+ result &= MaybeAddAuth(*authconf, uri);
+ }
+
+ while (not _error->empty())
+ {
+ std::string message;
+ _error->PopMessage(message);
+ Warning(std::move(message));
+ }
+ _error->RevertToStack();
+
+ return result;
+ }
+
+ aptAuthConfMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3)
+ : aptMethod(std::move(Binary), Ver, Flags) {}
+};
+#endif
diff --git a/methods/basehttp.cc b/methods/basehttp.cc
new file mode 100644
index 0000000..26b633c
--- /dev/null
+++ b/methods/basehttp.cc
@@ -0,0 +1,957 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ HTTP and HTTPS share a lot of common code and these classes are
+ exactly the dumping ground for this common code
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/debversion.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/strutl.h>
+
+#include <cctype>
+#include <csignal>
+#include <cstdio>
+#include <cstdlib>
+#include <ctime>
+#include <iostream>
+#include <limits>
+#include <map>
+#include <string>
+#include <string_view>
+#include <vector>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#include "basehttp.h"
+
+#include <apti18n.h>
+ /*}}}*/
+using namespace std;
+
+string BaseHttpMethod::FailFile;
+int BaseHttpMethod::FailFd = -1;
+time_t BaseHttpMethod::FailTime = 0;
+
+// Number of successful requests in a pipeline needed to continue
+// pipelining after a connection reset.
+constexpr int PIPELINE_MIN_SUCCESSFUL_ANSWERS_TO_CONTINUE = 3;
+
+// ServerState::RunHeaders - Get the headers before the data /*{{{*/
+// ---------------------------------------------------------------------
+/* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
+ parse error occurred */
+ServerState::RunHeadersResult ServerState::RunHeaders(RequestState &Req,
+ const std::string &Uri)
+{
+ Owner->Status(_("Waiting for headers"));
+ do
+ {
+ string Data;
+ if (ReadHeaderLines(Data) == false)
+ continue;
+
+ if (Owner->Debug == true)
+ clog << "Answer for: " << Uri << endl << Data;
+
+ for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
+ {
+ string::const_iterator J = I;
+ for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
+ if (Req.HeaderLine(string(I,J)) == false)
+ return RUN_HEADERS_PARSE_ERROR;
+ I = J;
+ }
+
+ // 100 Continue is a Nop...
+ if (Req.Result == 100)
+ continue;
+
+ // Tidy up the connection persistence state.
+ if (Req.Encoding == RequestState::Closes && Req.haveContent == HaveContent::TRI_TRUE)
+ Persistent = false;
+
+ return RUN_HEADERS_OK;
+ } while (LoadNextResponse(false, Req) == ResultState::SUCCESSFUL);
+
+ return RUN_HEADERS_IO_ERROR;
+}
+ /*}}}*/
+bool RequestState::HeaderLine(string const &Line) /*{{{*/
+{
+ if (Line.empty() == true)
+ return true;
+
+ if (Result == 0 && Line.size() > 4 && stringcasecmp(Line.data(), Line.data() + 4, "HTTP") == 0)
+ {
+ // Evil servers return no version
+ if (Line[4] == '/')
+ {
+ int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
+ if (elements == 3)
+ {
+ Code[0] = '\0';
+ if (Owner != NULL && Owner->Debug == true)
+ clog << "HTTP server doesn't give Reason-Phrase for " << std::to_string(Result) << std::endl;
+ }
+ else if (elements != 4)
+ return _error->Error(_("The HTTP server sent an invalid reply header"));
+ }
+ else
+ {
+ Major = 0;
+ Minor = 9;
+ if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
+ return _error->Error(_("The HTTP server sent an invalid reply header"));
+ }
+ auto const CodeLen = strlen(Code);
+ auto const CodeEnd = std::remove_if(Code, Code + CodeLen, [](char c) { return isprint(c) == 0; });
+ *CodeEnd = '\0';
+
+ /* Check the HTTP response header to get the default persistence
+ state. */
+ if (Major < 1)
+ Server->Persistent = false;
+ else
+ {
+ if (Major == 1 && Minor == 0)
+ {
+ Server->Persistent = false;
+ }
+ else
+ {
+ Server->Persistent = true;
+ if (Server->PipelineAllowed)
+ Server->Pipeline = true;
+ }
+ }
+
+ return true;
+ }
+
+ // Blah, some servers use "connection:closes", evil.
+ // and some even send empty header fields…
+ string::size_type Pos = Line.find(':');
+ if (Pos == string::npos)
+ return _error->Error(_("Bad header line"));
+ ++Pos;
+
+ // Parse off any trailing spaces between the : and the next word.
+ string::size_type Pos2 = Pos;
+ while (Pos2 < Line.length() && isspace_ascii(Line[Pos2]) != 0)
+ Pos2++;
+
+ string const Tag(Line,0,Pos);
+ string const Val(Line,Pos2);
+
+ if (stringcasecmp(Tag,"Content-Length:") == 0)
+ {
+ auto ContentLength = strtoull(Val.c_str(), NULL, 10);
+ if (ContentLength == 0)
+ {
+ haveContent = HaveContent::TRI_FALSE;
+ return true;
+ }
+ if (Encoding == Closes)
+ Encoding = Stream;
+ haveContent = HaveContent::TRI_TRUE;
+
+ unsigned long long * DownloadSizePtr = &DownloadSize;
+ if (Result == 416 || (Result >= 300 && Result < 400))
+ DownloadSizePtr = &JunkSize;
+
+ *DownloadSizePtr = ContentLength;
+ if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max())
+ return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
+ else if (*DownloadSizePtr == 0)
+ haveContent = HaveContent::TRI_FALSE;
+
+ // On partial content (206) the Content-Length less than the real
+ // size, so do not set it here but leave that to the Content-Range
+ // header instead
+ if(Result != 206 && TotalFileSize == 0)
+ TotalFileSize = DownloadSize;
+
+ return true;
+ }
+
+ if (stringcasecmp(Tag,"Content-Type:") == 0)
+ {
+ if (haveContent == HaveContent::TRI_UNKNOWN)
+ haveContent = HaveContent::TRI_TRUE;
+ return true;
+ }
+
+ // The Content-Range field only has a meaning in HTTP/1.1 for the
+ // 206 (Partial Content) and 416 (Range Not Satisfiable) responses
+ // according to RFC7233 "Range Requests", §4.2, so only consider it
+ // for such responses.
+ if ((Result == 416 || Result == 206) && stringcasecmp(Tag,"Content-Range:") == 0)
+ {
+ if (haveContent == HaveContent::TRI_UNKNOWN)
+ haveContent = HaveContent::TRI_TRUE;
+
+ // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
+ if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
+ ; // we got the expected filesize which is all we wanted
+ else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
+ return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
+ if (StartPos > TotalFileSize)
+ return _error->Error(_("This HTTP server has broken range support"));
+
+ // figure out what we will download
+ DownloadSize = TotalFileSize - StartPos;
+ return true;
+ }
+
+ if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
+ {
+ if (haveContent == HaveContent::TRI_UNKNOWN)
+ haveContent = HaveContent::TRI_TRUE;
+ if (stringcasecmp(Val,"chunked") == 0)
+ Encoding = Chunked;
+ return true;
+ }
+
+ if (stringcasecmp(Tag,"Connection:") == 0)
+ {
+ if (stringcasecmp(Val,"close") == 0)
+ {
+ Server->Persistent = false;
+ Server->Pipeline = false;
+ /* Some servers send error pages (as they are dynamically generated)
+ for simplicity via a connection close instead of e.g. chunked,
+ so assuming an always closing server only if we get a file + close */
+ if (Result >= 200 && Result < 300 && Server->PipelineAnswersReceived < PIPELINE_MIN_SUCCESSFUL_ANSWERS_TO_CONTINUE)
+ {
+ Server->PipelineAllowed = false;
+ Server->PipelineAnswersReceived = 0;
+ }
+ }
+ else if (stringcasecmp(Val,"keep-alive") == 0)
+ Server->Persistent = true;
+ return true;
+ }
+
+ if (stringcasecmp(Tag,"Last-Modified:") == 0)
+ {
+ if (RFC1123StrToTime(Val, Date) == false)
+ return _error->Error(_("Unknown date format"));
+ return true;
+ }
+
+ if (stringcasecmp(Tag,"Location:") == 0)
+ {
+ Location = Val;
+ return true;
+ }
+
+ if (Server->RangesAllowed && stringcasecmp(Tag, "Accept-Ranges:") == 0)
+ {
+ std::string ranges = ',' + Val + ',';
+ ranges.erase(std::remove(ranges.begin(), ranges.end(), ' '), ranges.end());
+ if (ranges.find(",bytes,") == std::string::npos)
+ Server->RangesAllowed = false;
+ return true;
+ }
+
+ if (Server->RangesAllowed && stringcasecmp(Tag, "Via:") == 0)
+ {
+ auto const parts = VectorizeString(Val, ' ');
+ std::string_view const varnish{"(Varnish/"};
+ if (parts.size() != 3 || parts[1] != "varnish" || parts[2].empty() ||
+ not APT::String::Startswith(parts[2], std::string{varnish}) ||
+ parts[2].back() != ')')
+ return true;
+ auto const version = parts[2].substr(varnish.length(), parts[2].length() - (varnish.length() + 1));
+ if (version.empty())
+ return true;
+ std::string_view const varnishsupport{"6.4~"};
+ if (debVersioningSystem::CmpFragment(version.data(), version.data() + version.length(),
+ varnishsupport.begin(), varnishsupport.end()) < 0)
+ Server->RangesAllowed = false;
+ return true;
+ }
+
+ return true;
+}
+ /*}}}*/
+// ServerState::ServerState - Constructor /*{{{*/
+ServerState::ServerState(URI Srv, BaseHttpMethod *Owner) :
+ ServerName(Srv), TimeOut(30), Owner(Owner)
+{
+ Reset();
+}
+ /*}}}*/
+bool RequestState::AddPartialFileToHashes(FileFd &File) /*{{{*/
+{
+ File.Truncate(StartPos);
+ return Server->GetHashes()->AddFD(File, StartPos);
+}
+ /*}}}*/
+void ServerState::Reset() /*{{{*/
+{
+ Persistent = false;
+ Pipeline = false;
+ PipelineAllowed = true;
+ PipelineAnswersReceived = 0;
+}
+ /*}}}*/
+
+// BaseHttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
+// ---------------------------------------------------------------------
+/* We look at the header data we got back from the server and decide what
+ to do. Returns DealWithHeadersResult (see http.h for details).
+ */
+static std::string fixURIEncoding(std::string const &part)
+{
+ // if the server sends a space this is not an encoded URI
+ // so other clients seem to encode it and we do it as well
+ if (part.find_first_of(" ") != std::string::npos)
+ return aptMethod::URIEncode(part);
+ return part;
+}
+BaseHttpMethod::DealWithHeadersResult
+BaseHttpMethod::DealWithHeaders(FetchResult &Res, RequestState &Req)
+{
+ // Not Modified
+ if (Req.Result == 304)
+ {
+ RemoveFile("server", Queue->DestFile);
+ Res.IMSHit = true;
+ Res.LastModified = Queue->LastModified;
+ Res.Size = 0;
+ return IMS_HIT;
+ }
+
+ /* Note that it is only OK for us to treat all redirection the same
+ because we *always* use GET, not other HTTP methods.
+ Codes not mentioned are handled as errors later as required by the
+ HTTP spec to handle unknown codes the same as the x00 code. */
+ constexpr unsigned int RedirectCodes[] = {
+ 301, // Moved Permanently
+ 302, // Found
+ 303, // See Other
+ 307, // Temporary Redirect
+ 308, // Permanent Redirect
+ };
+ if (AllowRedirect && std::find(std::begin(RedirectCodes), std::end(RedirectCodes), Req.Result) != std::end(RedirectCodes))
+ {
+ if (Req.Location.empty() == true)
+ ;
+ else if (Req.Location[0] == '/' && Queue->Uri.empty() == false)
+ {
+ URI Uri(Queue->Uri);
+ if (Uri.Host.empty() == false)
+ NextURI = URI::SiteOnly(Uri);
+ else
+ NextURI.clear();
+ if (_config->FindB("Acquire::Send-URI-Encoded", false))
+ NextURI.append(fixURIEncoding(Req.Location));
+ else
+ NextURI.append(DeQuoteString(Req.Location));
+ if (Queue->Uri == NextURI)
+ {
+ SetFailReason("RedirectionLoop");
+ _error->Error("Redirection loop encountered");
+ if (Req.haveContent == HaveContent::TRI_TRUE)
+ return ERROR_WITH_CONTENT_PAGE;
+ return ERROR_UNRECOVERABLE;
+ }
+ return TRY_AGAIN_OR_REDIRECT;
+ }
+ else
+ {
+ bool const SendURIEncoded = _config->FindB("Acquire::Send-URI-Encoded", false);
+ if (not SendURIEncoded)
+ Req.Location = DeQuoteString(Req.Location);
+ URI tmpURI(Req.Location);
+ if (SendURIEncoded)
+ tmpURI.Path = fixURIEncoding(tmpURI.Path);
+ if (tmpURI.Access.find('+') != std::string::npos)
+ {
+ _error->Error("Server tried to trick us into using a specific implementation: %s", tmpURI.Access.c_str());
+ if (Req.haveContent == HaveContent::TRI_TRUE)
+ return ERROR_WITH_CONTENT_PAGE;
+ return ERROR_UNRECOVERABLE;
+ }
+ NextURI = tmpURI;
+ URI Uri(Queue->Uri);
+ if (Binary.find('+') != std::string::npos)
+ {
+ auto base = Binary.substr(0, Binary.find('+'));
+ if (base != tmpURI.Access)
+ {
+ tmpURI.Access = base + '+' + tmpURI.Access;
+ if (tmpURI.Access == Binary)
+ {
+ std::swap(tmpURI.Access, Uri.Access);
+ NextURI = tmpURI;
+ std::swap(tmpURI.Access, Uri.Access);
+ }
+ else
+ NextURI = tmpURI;
+ }
+ }
+ if (Queue->Uri == NextURI)
+ {
+ SetFailReason("RedirectionLoop");
+ _error->Error("Redirection loop encountered");
+ if (Req.haveContent == HaveContent::TRI_TRUE)
+ return ERROR_WITH_CONTENT_PAGE;
+ return ERROR_UNRECOVERABLE;
+ }
+ Uri.Access = Binary;
+ // same protocol redirects are okay
+ if (tmpURI.Access == Uri.Access)
+ return TRY_AGAIN_OR_REDIRECT;
+ // as well as http to https
+ else if ((Uri.Access == "http" || Uri.Access == "https+http") && tmpURI.Access == "https")
+ return TRY_AGAIN_OR_REDIRECT;
+ else
+ {
+ auto const tmpplus = tmpURI.Access.find('+');
+ if (tmpplus != std::string::npos && tmpURI.Access.substr(tmpplus + 1) == "https")
+ {
+ auto const uriplus = Uri.Access.find('+');
+ if (uriplus == std::string::npos)
+ {
+ if (Uri.Access == tmpURI.Access.substr(0, tmpplus)) // foo -> foo+https
+ return TRY_AGAIN_OR_REDIRECT;
+ }
+ else if (Uri.Access.substr(uriplus + 1) == "http" &&
+ Uri.Access.substr(0, uriplus) == tmpURI.Access.substr(0, tmpplus)) // foo+http -> foo+https
+ return TRY_AGAIN_OR_REDIRECT;
+ }
+ }
+ _error->Error("Redirection from %s to '%s' is forbidden", Uri.Access.c_str(), NextURI.c_str());
+ }
+ /* else pass through for error message */
+ }
+ // the server is not supporting ranges as much as we would like. Retry without ranges
+ else if (not Server->RangesAllowed && (Req.Result == 416 || Req.Result == 206))
+ {
+ RemoveFile("server", Queue->DestFile);
+ NextURI = Queue->Uri;
+ return TRY_AGAIN_OR_REDIRECT;
+ }
+ // retry after an invalid range response without partial data
+ else if (Req.Result == 416)
+ {
+ struct stat SBuf;
+ if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
+ {
+ bool partialHit = false;
+ if (Queue->ExpectedHashes.usable() == true)
+ {
+ Hashes resultHashes(Queue->ExpectedHashes);
+ FileFd file(Queue->DestFile, FileFd::ReadOnly);
+ Req.TotalFileSize = file.FileSize();
+ Req.Date = file.ModificationTime();
+ resultHashes.AddFD(file);
+ HashStringList const hashList = resultHashes.GetHashStringList();
+ partialHit = (Queue->ExpectedHashes == hashList);
+ }
+ else if ((unsigned long long)SBuf.st_size == Req.TotalFileSize)
+ partialHit = true;
+ if (partialHit == true)
+ {
+ // the file is completely downloaded, but was not moved
+ if (Req.haveContent == HaveContent::TRI_TRUE)
+ {
+ // nuke the sent error page
+ Server->RunDataToDevNull(Req);
+ Req.haveContent = HaveContent::TRI_FALSE;
+ }
+ Req.StartPos = Req.TotalFileSize;
+ Req.Result = 200;
+ }
+ else if (RemoveFile("server", Queue->DestFile))
+ {
+ NextURI = Queue->Uri;
+ return TRY_AGAIN_OR_REDIRECT;
+ }
+ }
+ }
+
+ /* We have a reply we don't handle. This should indicate a perm server
+ failure */
+ if (Req.Result < 200 || Req.Result >= 300)
+ {
+ if (_error->PendingError() == false)
+ {
+ std::string err;
+ strprintf(err, "HttpError%u", Req.Result);
+ SetFailReason(err);
+ _error->Error("%u %s", Req.Result, Req.Code);
+ }
+ if (Req.haveContent == HaveContent::TRI_TRUE)
+ return ERROR_WITH_CONTENT_PAGE;
+ return ERROR_UNRECOVERABLE;
+ }
+
+ // This is some sort of 2xx 'data follows' reply
+ Res.LastModified = Req.Date;
+ Res.Size = Req.TotalFileSize;
+ return FILE_IS_OPEN;
+}
+ /*}}}*/
+// BaseHttpMethod::SigTerm - Handle a fatal signal /*{{{*/
+// ---------------------------------------------------------------------
+/* This closes and timestamps the open file. This is necessary to get
+ resume behavior on user abort */
+void BaseHttpMethod::SigTerm(int)
+{
+ if (FailFd == -1)
+ _exit(100);
+
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(FailFile.c_str(), times);
+ close(FailFd);
+
+ _exit(100);
+}
+ /*}}}*/
+// BaseHttpMethod::Fetch - Fetch an item /*{{{*/
+// ---------------------------------------------------------------------
+/* This adds an item to the pipeline. We keep the pipeline at a fixed
+ depth. */
+bool BaseHttpMethod::Fetch(FetchItem *)
+{
+ if (Server == nullptr || QueueBack == nullptr)
+ return true;
+
+ // If pipelining is disabled, we only queue 1 request
+ auto const AllowedDepth = Server->Pipeline ? PipelineDepth : 0;
+ // how deep is our pipeline currently?
+ decltype(PipelineDepth) CurrentDepth = 0;
+ for (FetchItem const *I = Queue; I != QueueBack; I = I->Next)
+ ++CurrentDepth;
+ if (CurrentDepth > AllowedDepth)
+ return true;
+
+ do {
+ // Make sure we stick with the same server
+ if (Server->Comp(URI(QueueBack->Uri)) == false)
+ break;
+
+ bool const UsableHashes = QueueBack->ExpectedHashes.usable();
+ // if we have no hashes, do at most one such request
+ // as we can't fixup pipeling misbehaviors otherwise
+ if (CurrentDepth != 0 && UsableHashes == false)
+ break;
+
+ if (UsableHashes && FileExists(QueueBack->DestFile))
+ {
+ FileFd partial(QueueBack->DestFile, FileFd::ReadOnly);
+ Hashes wehave(QueueBack->ExpectedHashes);
+ if (QueueBack->ExpectedHashes.FileSize() == partial.FileSize())
+ {
+ if (wehave.AddFD(partial) &&
+ wehave.GetHashStringList() == QueueBack->ExpectedHashes)
+ {
+ FetchResult Res;
+ Res.Filename = QueueBack->DestFile;
+ Res.ResumePoint = QueueBack->ExpectedHashes.FileSize();
+ URIStart(Res);
+ // move item to the start of the queue as URIDone will
+ // always dequeued the first item in the queue
+ if (Queue != QueueBack)
+ {
+ FetchItem *Prev = Queue;
+ for (; Prev->Next != QueueBack; Prev = Prev->Next)
+ /* look for the previous queue item */;
+ Prev->Next = QueueBack->Next;
+ QueueBack->Next = Queue;
+ Queue = QueueBack;
+ QueueBack = Prev->Next;
+ }
+ Res.TakeHashes(wehave);
+ URIDone(Res);
+ continue;
+ }
+ else
+ RemoveFile("Fetch-Partial", QueueBack->DestFile);
+ }
+ }
+ auto const Tmp = QueueBack;
+ QueueBack = QueueBack->Next;
+ SendReq(Tmp);
+ ++CurrentDepth;
+ } while (CurrentDepth <= AllowedDepth && QueueBack != nullptr);
+
+ return true;
+}
+ /*}}}*/
+// BaseHttpMethod::Loop - Main loop /*{{{*/
+int BaseHttpMethod::Loop()
+{
+ signal(SIGTERM,SigTerm);
+ signal(SIGINT,SigTerm);
+
+ Server = 0;
+
+ int FailCounter = 0;
+ while (1)
+ {
+ // We have no commands, wait for some to arrive
+ if (Queue == 0)
+ {
+ if (WaitFd(STDIN_FILENO) == false)
+ return 0;
+ }
+
+ /* Run messages, we can accept 0 (no message) if we didn't
+ do a WaitFd above.. Otherwise the FD is closed. */
+ int Result = Run(true);
+ if (Result != -1 && (Result != 0 || Queue == 0))
+ {
+ if(FailReason.empty() == false ||
+ ConfigFindB("DependOnSTDIN", true) == true)
+ return 100;
+ else
+ return 0;
+ }
+
+ if (Queue == 0)
+ continue;
+
+ // Connect to the server
+ if (Server == 0 || Server->Comp(URI(Queue->Uri)) == false)
+ {
+ if (!Queue->Proxy().empty())
+ {
+ URI uri(Queue->Uri);
+ _config->Set("Acquire::" + uri.Access + "::proxy::" + uri.Host, Queue->Proxy());
+ }
+ Server = CreateServerState(URI(Queue->Uri));
+ setPostfixForMethodNames(::URI(Queue->Uri).Host.c_str());
+ AllowRedirect = ConfigFindB("AllowRedirect", true);
+ PipelineDepth = ConfigFindI("Pipeline-Depth", 10);
+ Server->RangesAllowed = ConfigFindB("AllowRanges", true);
+ Debug = DebugEnabled();
+ }
+
+ /* If the server has explicitly said this is the last connection
+ then we pre-emptively shut down the pipeline and tear down
+ the connection. This will speed up HTTP/1.0 servers a tad
+ since we don't have to wait for the close sequence to
+ complete */
+ if (Server->Persistent == false)
+ Server->Close();
+
+ // Reset the pipeline
+ if (Server->IsOpen() == false) {
+ QueueBack = Queue;
+ Server->PipelineAnswersReceived = 0;
+ }
+
+ // Connect to the host
+ switch (Server->Open())
+ {
+ case ResultState::FATAL_ERROR:
+ Fail(false);
+ Server = nullptr;
+ continue;
+ case ResultState::TRANSIENT_ERROR:
+ Fail(true);
+ Server = nullptr;
+ continue;
+ case ResultState::SUCCESSFUL:
+ break;
+ }
+
+ // Fill the pipeline.
+ Fetch(0);
+
+ RequestState Req(this, Server.get());
+ // Fetch the next URL header data from the server.
+ switch (Server->RunHeaders(Req, Queue->Uri))
+ {
+ case ServerState::RUN_HEADERS_OK:
+ break;
+
+ // The header data is bad
+ case ServerState::RUN_HEADERS_PARSE_ERROR:
+ {
+ _error->Error(_("Bad header data"));
+ Fail(true);
+ Server->Close();
+ RotateDNS();
+ continue;
+ }
+
+ // The server closed a connection during the header get..
+ default:
+ case ServerState::RUN_HEADERS_IO_ERROR:
+ {
+ FailCounter++;
+ _error->Discard();
+ Server->Close();
+ Server->Pipeline = false;
+ Server->PipelineAllowed = false;
+
+ if (FailCounter >= 2)
+ {
+ Fail(_("Connection failed"),true);
+ FailCounter = 0;
+ }
+
+ RotateDNS();
+ continue;
+ }
+ };
+
+ // Decide what to do.
+ FetchResult Res;
+ Res.Filename = Queue->DestFile;
+ switch (DealWithHeaders(Res, Req))
+ {
+ // Ok, the file is Open
+ case FILE_IS_OPEN:
+ {
+ URIStart(Res);
+
+ // Run the data
+ ResultState Result = ResultState::SUCCESSFUL;
+
+ // ensure we don't fetch too much
+ // we could do "Server->MaximumSize = Queue->MaximumSize" here
+ // but that would break the clever pipeline messup detection
+ // so instead we use the size of the biggest item in the queue
+ Req.MaximumSize = FindMaximumObjectSizeInQueue();
+
+ if (Req.haveContent == HaveContent::TRI_TRUE)
+ {
+ /* If the server provides Content-Length we can figure out with it if
+ this satisfies any request we have made so far (in the pipeline).
+ If not we can kill the connection as whatever file the server is trying
+ to send to us would be rejected with a hashsum mismatch later or triggers
+ a maximum size error. We don't run the data to /dev/null as this can be MBs
+ of junk data we would waste bandwidth on and instead just close the connection
+ to reopen a fresh one which should be more cost/time efficient */
+ if (Req.DownloadSize > 0)
+ {
+ decltype(Queue->ExpectedHashes.FileSize()) const filesize = Req.StartPos + Req.DownloadSize;
+ bool found = false;
+ for (FetchItem const *I = Queue; I != 0 && I != QueueBack; I = I->Next)
+ {
+ auto const fs = I->ExpectedHashes.FileSize();
+ if (fs == 0 || fs == filesize)
+ {
+ found = true;
+ break;
+ }
+ }
+ if (found == false)
+ {
+ SetFailReason("MaximumSizeExceeded");
+ _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
+ filesize, Queue->ExpectedHashes.FileSize());
+ Result = ResultState::FATAL_ERROR;
+ }
+ }
+ if (Result == ResultState::SUCCESSFUL)
+ Result = Server->RunData(Req);
+ }
+
+ /* If the server is sending back sizeless responses then fill in
+ the size now */
+ if (Res.Size == 0)
+ Res.Size = Req.File.Size();
+
+ // Close the file, destroy the FD object and timestamp it
+ FailFd = -1;
+ Req.File.Close();
+
+ // Timestamp
+ struct timeval times[2];
+ times[0].tv_sec = times[1].tv_sec = Req.Date;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(Queue->DestFile.c_str(), times);
+
+ // Send status to APT
+ if (Result == ResultState::SUCCESSFUL)
+ {
+ Hashes * const resultHashes = Server->GetHashes();
+ HashStringList const hashList = resultHashes->GetHashStringList();
+ if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
+ {
+ // we did not get the expected hash… mhhh:
+ // could it be that server/proxy messed up pipelining?
+ FetchItem * BeforeI = Queue;
+ for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
+ {
+ if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
+ {
+ // yes, he did! Disable pipelining and rewrite queue
+ if (Server->Pipeline == true)
+ {
+ std::string msg;
+ strprintf(msg, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::Pipeline-Depth");
+ Warning(std::move(msg));
+ Server->Pipeline = false;
+ Server->PipelineAllowed = false;
+ // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
+ }
+ Rename(Res.Filename, I->DestFile);
+ Res.Filename = I->DestFile;
+ BeforeI->Next = I->Next;
+ I->Next = Queue;
+ Queue = I;
+ break;
+ }
+ BeforeI = I;
+ }
+ }
+ if (Server->Pipeline == true)
+ {
+ Server->PipelineAnswersReceived++;
+ }
+ Res.TakeHashes(*resultHashes);
+ URIDone(Res);
+ }
+ else
+ {
+ if (not Server->IsOpen())
+ {
+ // Reset the pipeline
+ QueueBack = Queue;
+ Server->PipelineAnswersReceived = 0;
+ }
+
+ Server->Close();
+ FailCounter = 0;
+ switch (Result)
+ {
+ case ResultState::TRANSIENT_ERROR:
+ Fail(true);
+ break;
+ case ResultState::FATAL_ERROR:
+ case ResultState::SUCCESSFUL:
+ Fail(false);
+ break;
+ }
+ }
+ break;
+ }
+
+ // IMS hit
+ case IMS_HIT:
+ {
+ URIDone(Res);
+ break;
+ }
+
+ // Hard server error, not found or something
+ case ERROR_UNRECOVERABLE:
+ {
+ Fail();
+ break;
+ }
+
+ // Hard internal error, kill the connection and fail
+ case ERROR_NOT_FROM_SERVER:
+ {
+ Fail();
+ RotateDNS();
+ Server->Close();
+ break;
+ }
+
+ // We need to flush the data, the header is like a 404 w/ error text
+ case ERROR_WITH_CONTENT_PAGE:
+ {
+ Server->RunDataToDevNull(Req);
+ constexpr unsigned int TransientCodes[] = {
+ 408, // Request Timeout
+ 429, // Too Many Requests
+ 500, // Internal Server Error
+ 502, // Bad Gateway
+ 503, // Service Unavailable
+ 504, // Gateway Timeout
+ 599, // Network Connect Timeout Error
+ };
+ if (std::find(std::begin(TransientCodes), std::end(TransientCodes), Req.Result) != std::end(TransientCodes))
+ Fail(true);
+ else
+ Fail();
+ break;
+ }
+
+ // Try again with a new URL
+ case TRY_AGAIN_OR_REDIRECT:
+ {
+ // Clear rest of response if there is content
+ if (Req.haveContent == HaveContent::TRI_TRUE)
+ Server->RunDataToDevNull(Req);
+ Redirect(NextURI);
+ break;
+ }
+
+ default:
+ Fail(_("Internal error"));
+ break;
+ }
+
+ FailCounter = 0;
+ }
+
+ return 0;
+}
+ /*}}}*/
+unsigned long long BaseHttpMethod::FindMaximumObjectSizeInQueue() const /*{{{*/
+{
+ unsigned long long MaxSizeInQueue = 0;
+ for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
+ {
+ if (I->MaximumSize == 0)
+ return 0;
+ MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
+ }
+ return MaxSizeInQueue;
+}
+ /*}}}*/
+BaseHttpMethod::BaseHttpMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) /*{{{*/
+ : aptAuthConfMethod(std::move(Binary), Ver, Flags), Server(nullptr),
+ AllowRedirect(false), Debug(false), PipelineDepth(10)
+{
+}
+ /*}}}*/
+bool BaseHttpMethod::Configuration(std::string Message) /*{{{*/
+{
+ if (aptAuthConfMethod::Configuration(Message) == false)
+ return false;
+
+ _config->CndSet("Acquire::tor::Proxy",
+ "socks5h://apt-transport-tor@127.0.0.1:9050");
+ return true;
+}
+ /*}}}*/
+bool BaseHttpMethod::AddProxyAuth(URI &Proxy, URI const &Server) /*{{{*/
+{
+ MaybeAddAuthTo(Proxy);
+ if (std::find(methodNames.begin(), methodNames.end(), "tor") != methodNames.end() &&
+ Proxy.User == "apt-transport-tor" && Proxy.Password.empty())
+ {
+ std::string pass = Server.Host;
+ pass.erase(std::remove_if(pass.begin(), pass.end(), [](char const c) { return std::isalnum(c) == 0; }), pass.end());
+ if (pass.length() > 255)
+ Proxy.Password = pass.substr(0, 255);
+ else
+ Proxy.Password = std::move(pass);
+ }
+ return true;
+}
+ /*}}}*/
diff --git a/methods/basehttp.h b/methods/basehttp.h
new file mode 100644
index 0000000..080ea52
--- /dev/null
+++ b/methods/basehttp.h
@@ -0,0 +1,180 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ Classes dealing with the abstraction of talking to a end via a text
+ protocol like HTTP (which is used by the http and https methods)
+
+ ##################################################################### */
+ /*}}}*/
+
+#ifndef APT_SERVER_H
+#define APT_SERVER_H
+
+#include "aptmethod.h"
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/strutl.h>
+
+#include <ctime>
+#include <iostream>
+#include <memory>
+#include <string>
+
+using std::cout;
+using std::endl;
+
+class Hashes;
+class BaseHttpMethod;
+struct ServerState;
+
+enum class HaveContent
+{
+ TRI_UNKNOWN,
+ TRI_FALSE,
+ TRI_TRUE,
+};
+struct RequestState
+{
+ unsigned int Major = 0;
+ unsigned int Minor = 0;
+ unsigned int Result = 0;
+ char Code[360];
+
+ // total size of the usable content (aka: the file)
+ unsigned long long TotalFileSize = 0;
+ // size we actually download (can be smaller than Size if we have partial content)
+ unsigned long long DownloadSize = 0;
+ // size of junk content (aka: server error pages)
+ unsigned long long JunkSize = 0;
+ // The start of the data (for partial content)
+ unsigned long long StartPos = 0;
+
+ unsigned long long MaximumSize = 0;
+
+ time_t Date;
+ HaveContent haveContent = HaveContent::TRI_UNKNOWN;
+
+ enum {Chunked,Stream,Closes} Encoding = Closes;
+ enum {Header, Data} State = Header;
+ std::string Location;
+
+ FileFd File;
+
+ BaseHttpMethod * const Owner;
+ ServerState * const Server;
+
+ bool HeaderLine(std::string const &Line);
+ bool AddPartialFileToHashes(FileFd &File);
+
+ RequestState(BaseHttpMethod * const Owner, ServerState * const Server) :
+ Owner(Owner), Server(Server) { time(&Date); Code[0] = '\0'; }
+};
+struct ServerState
+{
+ bool Persistent;
+ bool PipelineAllowed;
+ bool RangesAllowed;
+ unsigned long PipelineAnswersReceived;
+
+ bool Pipeline;
+ URI ServerName;
+ URI Proxy;
+ unsigned long TimeOut;
+
+ protected:
+ BaseHttpMethod *Owner;
+
+ virtual bool ReadHeaderLines(std::string &Data) = 0;
+ virtual ResultState LoadNextResponse(bool const ToFile, RequestState &Req) = 0;
+
+ public:
+
+ /** \brief Result of the header acquire */
+ enum RunHeadersResult {
+ /** \brief Header ok */
+ RUN_HEADERS_OK,
+ /** \brief IO error while retrieving */
+ RUN_HEADERS_IO_ERROR,
+ /** \brief Parse error after retrieving */
+ RUN_HEADERS_PARSE_ERROR
+ };
+ /** \brief Get the headers before the data */
+ RunHeadersResult RunHeaders(RequestState &Req, const std::string &Uri);
+
+ bool Comp(URI Other) const {return Other.Access == ServerName.Access && Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
+ virtual void Reset();
+ virtual bool WriteResponse(std::string const &Data) = 0;
+
+ /** \brief Transfer the data from the socket */
+ virtual ResultState RunData(RequestState &Req) = 0;
+ virtual ResultState RunDataToDevNull(RequestState &Req) = 0;
+
+ virtual ResultState Open() = 0;
+ virtual bool IsOpen() = 0;
+ virtual bool Close() = 0;
+ virtual bool InitHashes(HashStringList const &ExpectedHashes) = 0;
+ virtual ResultState Die(RequestState &Req) = 0;
+ virtual bool Flush(FileFd *const File, bool MustComplete = false) = 0;
+ virtual ResultState Go(bool ToFile, RequestState &Req) = 0;
+ virtual Hashes * GetHashes() = 0;
+
+ ServerState(URI Srv, BaseHttpMethod *Owner);
+ virtual ~ServerState() {};
+};
+
+class BaseHttpMethod : public aptAuthConfMethod
+{
+ protected:
+ virtual bool Fetch(FetchItem *) APT_OVERRIDE;
+
+ std::unique_ptr<ServerState> Server;
+ std::string NextURI;
+
+ bool AllowRedirect;
+
+ // Find the biggest item in the fetch queue for the checking of the maximum
+ // size
+ unsigned long long FindMaximumObjectSizeInQueue() const APT_PURE;
+
+ public:
+ bool Debug;
+ unsigned long PipelineDepth;
+
+ /** \brief Result of the header parsing */
+ enum DealWithHeadersResult {
+ /** \brief The file is open and ready */
+ FILE_IS_OPEN,
+ /** \brief We got a IMS hit, the file has not changed */
+ IMS_HIT,
+ /** \brief The server reported a unrecoverable error */
+ ERROR_UNRECOVERABLE,
+ /** \brief The server reported a error with a error content page */
+ ERROR_WITH_CONTENT_PAGE,
+ /** \brief An error on the client side */
+ ERROR_NOT_FROM_SERVER,
+ /** \brief A redirect or retry request */
+ TRY_AGAIN_OR_REDIRECT
+ };
+ /** \brief Handle the retrieved header data */
+ virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res, RequestState &Req);
+
+ // In the event of a fatal signal this file will be closed and timestamped.
+ static std::string FailFile;
+ static int FailFd;
+ static time_t FailTime;
+ static APT_NORETURN void SigTerm(int);
+
+ int Loop();
+
+ virtual void SendReq(FetchItem *Itm) = 0;
+ virtual std::unique_ptr<ServerState> CreateServerState(URI const &uri) = 0;
+ virtual void RotateDNS() = 0;
+ virtual bool Configuration(std::string Message) APT_OVERRIDE;
+
+ bool AddProxyAuth(URI &Proxy, URI const &Server);
+
+ BaseHttpMethod(std::string &&Binary, char const * const Ver,unsigned long const Flags);
+ virtual ~BaseHttpMethod() {};
+};
+
+#endif
diff --git a/methods/cdrom.cc b/methods/cdrom.cc
new file mode 100644
index 0000000..f534a06
--- /dev/null
+++ b/methods/cdrom.cc
@@ -0,0 +1,289 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ CDROM URI method for APT
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include <apt-pkg/cdrom.h>
+#include <apt-pkg/cdromutl.h>
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/hashes.h>
+#include <apt-pkg/strutl.h>
+
+#include "aptmethod.h"
+
+#include <string>
+#include <vector>
+#include <sys/stat.h>
+
+#include <iostream>
+#include <apti18n.h>
+ /*}}}*/
+
+using namespace std;
+
+class CDROMMethod : public aptMethod
+{
+ bool DatabaseLoaded;
+ bool Debug;
+
+ ::Configuration Database;
+ string CurrentID;
+ string CDROM;
+ bool MountedByApt;
+ pkgUdevCdromDevices UdevCdroms;
+
+ bool IsCorrectCD(URI want, string MountPath, string& NewID);
+ bool AutoDetectAndMount(const URI, string &NewID);
+ virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE;
+ std::string GetID(std::string const &Name);
+ virtual void Exit() APT_OVERRIDE;
+ virtual bool Configuration(std::string Message) APT_OVERRIDE;
+
+ public:
+
+ CDROMMethod();
+};
+
+// CDROMMethod::CDROMethod - Constructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+CDROMMethod::CDROMMethod() : aptMethod("cdrom", "1.0",SingleInstance | LocalOnly |
+ SendConfig | NeedsCleanup |
+ Removable | SendURIEncoded),
+ DatabaseLoaded(false),
+ Debug(false),
+ MountedByApt(false)
+{
+}
+ /*}}}*/
+// CDROMMethod::Exit - Unmount the disc if necessary /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void CDROMMethod::Exit()
+{
+ if (MountedByApt == true)
+ UnmountCdrom(CDROM);
+}
+ /*}}}*/
+// CDROMMethod::GetID - Search the database for a matching string /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+std::string CDROMMethod::GetID(std::string const &Name)
+{
+ // Search for an ID
+ const Configuration::Item *Top = Database.Tree("CD");
+ if (Top != 0)
+ Top = Top->Child;
+
+ for (; Top != 0;)
+ {
+ if (Top->Value == Name)
+ return Top->Tag;
+
+ Top = Top->Next;
+ }
+ return string();
+}
+ /*}}}*/
+// CDROMMethod::AutoDetectAndMount /*{{{*/
+// ---------------------------------------------------------------------
+/* Modifies class variable CDROM to the mountpoint */
+bool CDROMMethod::AutoDetectAndMount(const URI Get, string &NewID)
+{
+ vector<struct CdromDevice> v = UdevCdroms.Scan();
+
+ // first check if its mounted somewhere already
+ for (unsigned int i=0; i < v.size(); i++)
+ {
+ if (v[i].Mounted)
+ {
+ if (Debug)
+ clog << "Checking mounted cdrom device " << v[i].DeviceName << endl;
+ if (IsCorrectCD(Get, v[i].MountPath, NewID))
+ {
+ CDROM = v[i].MountPath;
+ return true;
+ }
+ }
+ }
+
+ // we are not supposed to mount, exit
+ if (_config->FindB("APT::CDROM::NoMount",false) == true)
+ return false;
+
+ // check if we have the mount point
+ string AptMountPoint = _config->FindDir("Dir::Media::MountPath");
+ if (!FileExists(AptMountPoint))
+ mkdir(AptMountPoint.c_str(), 0750);
+
+ // now try mounting
+ for (unsigned int i=0; i < v.size(); i++)
+ {
+ if (!v[i].Mounted)
+ {
+ if(MountCdrom(AptMountPoint, v[i].DeviceName))
+ {
+ if (IsCorrectCD(Get, AptMountPoint, NewID))
+ {
+ MountedByApt = true;
+ CDROM = AptMountPoint;
+ return true;
+ } else {
+ UnmountCdrom(AptMountPoint);
+ }
+ }
+ }
+ }
+
+ return false;
+}
+ /*}}}*/
+// CDROMMethod::IsCorrectCD /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool CDROMMethod::IsCorrectCD(URI want, string MountPath, string& NewID)
+{
+ for (unsigned int Version = 2; Version != 0; Version--)
+ {
+ if (IdentCdrom(MountPath,NewID,Version) == false)
+ return false;
+
+ if (Debug)
+ clog << "ID " << Version << " " << NewID << endl;
+
+ // A hit
+ if (Database.Find("CD::" + NewID) == want.Host)
+ return true;
+ }
+
+ return false;
+}
+ /*}}}*/
+// CDROMMethod::Fetch - Fetch a file /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool CDROMMethod::Fetch(FetchItem *Itm)
+{
+ FetchResult Res;
+
+ URI Get(Itm->Uri);
+ std::string const File = DecodeSendURI(Get.Path);
+ Debug = DebugEnabled();
+
+ if (Debug)
+ clog << "CDROMMethod::Fetch " << Itm->Uri << endl;
+
+ /* All IMS queries are returned as a hit, CDROMs are readonly so
+ time stamps never change */
+ if (Itm->LastModified != 0)
+ {
+ Res.LastModified = Itm->LastModified;
+ Res.IMSHit = true;
+ Res.Filename = Itm->DestFile;
+ URIDone(Res);
+ return true;
+ }
+
+ // Load the database
+ if (DatabaseLoaded == false)
+ {
+ // Read the database
+ string DFile = _config->FindFile("Dir::State::cdroms");
+ if (FileExists(DFile) == true)
+ {
+ if (ReadConfigFile(Database,DFile) == false)
+ return _error->Error(_("Unable to read the cdrom database %s"),
+ DFile.c_str());
+ }
+ DatabaseLoaded = true;
+ }
+
+ // All non IMS queries for package files fail.
+ if (Itm->IndexFile == true || GetID(Get.Host).empty() == true)
+ {
+ Fail(_("Please use apt-cdrom to make this CD-ROM recognized by APT."
+ " apt-get update cannot be used to add new CD-ROMs"));
+ return true;
+ }
+
+ // We already have a CD inserted, but it is the wrong one
+ if (CurrentID.empty() == false &&
+ CurrentID != "FAIL" &&
+ Database.Find("CD::" + CurrentID) != Get.Host)
+ {
+ Fail(_("Wrong CD-ROM"),true);
+ return true;
+ }
+
+ bool const AutoDetect = ConfigFindB("AutoDetect", true);
+ CDROM = _config->FindDir("Acquire::cdrom::mount");
+ if (Debug)
+ clog << "Looking for CDROM at " << CDROM << endl;
+
+ if (CDROM[0] == '.')
+ CDROM= SafeGetCWD() + '/' + CDROM;
+
+ string NewID;
+ while (CurrentID.empty() == true)
+ {
+ if (AutoDetect)
+ AutoDetectAndMount(Get, NewID);
+
+ if(!IsMounted(CDROM))
+ MountedByApt = MountCdrom(CDROM);
+
+ if (IsCorrectCD(Get, CDROM, NewID))
+ break;
+
+ // I suppose this should prompt somehow?
+ if (_config->FindB("APT::CDROM::NoMount",false) == false &&
+ UnmountCdrom(CDROM) == false)
+ return _error->Error(_("Unable to unmount the CD-ROM in %s, it may still be in use."),
+ CDROM.c_str());
+ if (MediaFail(Get.Host,CDROM) == false)
+ {
+ CurrentID = "FAIL";
+ return _error->Error(_("Disk not found."));
+ }
+ }
+
+ // Found a CD
+ Res.Filename = CDROM + File;
+ struct stat Buf;
+ if (stat(Res.Filename.c_str(),&Buf) != 0)
+ return _error->Error(_("File not found"));
+
+ URIStart(Res);
+ if (NewID.empty() == false)
+ CurrentID = NewID;
+ Res.LastModified = Buf.st_mtime;
+ Res.Size = Buf.st_size;
+
+ Hashes Hash(Itm->ExpectedHashes);
+ FileFd Fd(Res.Filename, FileFd::ReadOnly);
+ Hash.AddFD(Fd);
+ Res.TakeHashes(Hash);
+
+ URIDone(Res);
+ return true;
+}
+ /*}}}*/
+bool CDROMMethod::Configuration(std::string Message) /*{{{*/
+{
+ _config->CndSet("Binary::cdrom::Debug::NoDropPrivs", true);
+ return aptMethod::Configuration(Message);
+}
+ /*}}}*/
+
+int main()
+{
+ return CDROMMethod().Run();
+}
diff --git a/methods/connect.cc b/methods/connect.cc
new file mode 100644
index 0000000..110f2fc
--- /dev/null
+++ b/methods/connect.cc
@@ -0,0 +1,1053 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ Connect - Replacement connect call
+
+ This was originally authored by Jason Gunthorpe <jgg@debian.org>
+ and is placed in the Public Domain, do with it what you will.
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include <apt-pkg/acquire-method.h>
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/srvrec.h>
+#include <apt-pkg/strutl.h>
+
+#include <gnutls/gnutls.h>
+#include <gnutls/x509.h>
+
+#include <cerrno>
+#include <cstdio>
+#include <cstring>
+#include <list>
+#include <set>
+#include <sstream>
+#include <string>
+#include <unistd.h>
+
+// Internet stuff
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <sys/select.h>
+#include <sys/socket.h>
+
+#include "aptmethod.h"
+#include "connect.h"
+#include "rfc2553emu.h"
+#include <apti18n.h>
+ /*}}}*/
+
+static std::string LastHost;
+static std::string LastService;
+static struct addrinfo *LastHostAddr = 0;
+static struct addrinfo *LastUsed = 0;
+
+static std::vector<SrvRec> SrvRecords;
+
+// Set of IP/hostnames that we timed out before or couldn't resolve
+static std::set<std::string> bad_addr;
+
+// RotateDNS - Select a new server from a DNS rotation /*{{{*/
+// ---------------------------------------------------------------------
+/* This is called during certain errors in order to recover by selecting a
+ new server */
+void RotateDNS()
+{
+ if (LastUsed != 0 && LastUsed->ai_next != 0)
+ LastUsed = LastUsed->ai_next;
+ else
+ LastUsed = LastHostAddr;
+}
+ /*}}}*/
+static bool ConnectionAllowed(char const * const Service, std::string const &Host)/*{{{*/
+{
+ if (unlikely(Host.empty())) // the only legal empty host (RFC2782 '.' target) is detected by caller
+ return false;
+ if (APT::String::Endswith(Host, ".onion") && _config->FindB("Acquire::BlockDotOnion", true))
+ {
+ // TRANSLATOR: %s is e.g. Tor's ".onion" which would likely fail or leak info (RFC7686)
+ _error->Error(_("Direct connection to %s domains is blocked by default."), ".onion");
+ if (strcmp(Service, "http") == 0)
+ _error->Error(_("If you meant to use Tor remember to use %s instead of %s."), "tor+http", "http");
+ return false;
+ }
+ return true;
+}
+ /*}}}*/
+
+// File Descriptor based Fd /*{{{*/
+struct FdFd : public MethodFd
+{
+ int fd = -1;
+ int Fd() APT_OVERRIDE { return fd; }
+ ssize_t Read(void *buf, size_t count) APT_OVERRIDE { return ::read(fd, buf, count); }
+ ssize_t Write(void *buf, size_t count) APT_OVERRIDE { return ::write(fd, buf, count); }
+ int Close() APT_OVERRIDE
+ {
+ int result = 0;
+ if (fd != -1)
+ result = ::close(fd);
+ fd = -1;
+ return result;
+ }
+};
+
+bool MethodFd::HasPending()
+{
+ return false;
+}
+std::unique_ptr<MethodFd> MethodFd::FromFd(int iFd)
+{
+ FdFd *fd = new FdFd();
+ fd->fd = iFd;
+ return std::unique_ptr<MethodFd>(fd);
+}
+ /*}}}*/
+// DoConnect - Attempt a connect operation /*{{{*/
+// ---------------------------------------------------------------------
+/* This helper function attempts a connection to a single address. */
+struct Connection
+{
+ struct addrinfo *Addr;
+ std::string Host;
+ aptMethod *Owner;
+ std::unique_ptr<FdFd> Fd;
+ char Name[NI_MAXHOST];
+ char Service[NI_MAXSERV];
+
+ Connection(struct addrinfo *Addr, std::string const &Host, aptMethod *Owner) : Addr(Addr), Host(Host), Owner(Owner), Fd(new FdFd()), Name{0}, Service{0}
+ {
+ }
+
+ // Allow moving values, but not connections.
+ Connection(Connection &&Conn) = default;
+ Connection(const Connection &Conn) = delete;
+ Connection &operator=(const Connection &) = delete;
+ Connection &operator=(Connection &&Conn) = default;
+
+ ~Connection()
+ {
+ if (Fd != nullptr)
+ {
+ Fd->Close();
+ }
+ }
+
+ std::unique_ptr<MethodFd> Take()
+ {
+ /* Store the IP we are using.. If something goes
+ wrong this will get tacked onto the end of the error message */
+ std::stringstream ss;
+ ioprintf(ss, _("[IP: %s %s]"), Name, Service);
+ Owner->SetIP(ss.str());
+ Owner->Status(_("Connected to %s (%s)"), Host.c_str(), Name);
+ _error->Discard();
+ Owner->SetFailReason("");
+ LastUsed = Addr;
+ return std::move(Fd);
+ }
+
+ ResultState DoConnect();
+
+ ResultState CheckError();
+};
+
+ResultState Connection::DoConnect()
+{
+ getnameinfo(Addr->ai_addr,Addr->ai_addrlen,
+ Name,sizeof(Name),Service,sizeof(Service),
+ NI_NUMERICHOST|NI_NUMERICSERV);
+ Owner->Status(_("Connecting to %s (%s)"),Host.c_str(),Name);
+
+ // if that addr did timeout before, we do not try it again
+ if(bad_addr.find(std::string(Name)) != bad_addr.end())
+ return ResultState::TRANSIENT_ERROR;
+
+ // Get a socket
+ if ((static_cast<FdFd *>(Fd.get())->fd = socket(Addr->ai_family, Addr->ai_socktype,
+ Addr->ai_protocol)) < 0)
+ {
+ _error->Errno("socket", _("Could not create a socket for %s (f=%u t=%u p=%u)"),
+ Name, Addr->ai_family, Addr->ai_socktype, Addr->ai_protocol);
+ return ResultState::FATAL_ERROR;
+ }
+
+ SetNonBlock(Fd->Fd(), true);
+ if (connect(Fd->Fd(), Addr->ai_addr, Addr->ai_addrlen) < 0 &&
+ errno != EINPROGRESS)
+ {
+ _error->Errno("connect", _("Cannot initiate the connection "
+ "to %s:%s (%s)."),
+ Host.c_str(), Service, Name);
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ return ResultState::SUCCESSFUL;
+}
+
+ResultState Connection::CheckError()
+{
+ // Check the socket for an error condition
+ unsigned int Err;
+ unsigned int Len = sizeof(Err);
+ if (getsockopt(Fd->Fd(), SOL_SOCKET, SO_ERROR, &Err, &Len) != 0)
+ {
+ _error->Errno("getsockopt", _("Failed"));
+ return ResultState::FATAL_ERROR;
+ }
+
+ if (Err != 0)
+ {
+ errno = Err;
+ if(errno == ECONNREFUSED)
+ Owner->SetFailReason("ConnectionRefused");
+ else if (errno == ETIMEDOUT)
+ Owner->SetFailReason("ConnectionTimedOut");
+ bad_addr.insert(bad_addr.begin(), std::string(Name));
+ _error->Errno("connect", _("Could not connect to %s:%s (%s)."), Host.c_str(),
+ Service, Name);
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ Owner->SetFailReason("");
+
+ return ResultState::SUCCESSFUL;
+}
+ /*}}}*/
+// Order the given host names returned by getaddrinfo() /*{{{*/
+static std::vector<struct addrinfo *> OrderAddresses(struct addrinfo *CurHost)
+{
+ std::vector<struct addrinfo *> preferredAddrs;
+ std::vector<struct addrinfo *> otherAddrs;
+ std::vector<struct addrinfo *> allAddrs;
+
+ // Partition addresses into preferred and other address families
+ while (CurHost != 0)
+ {
+ if (preferredAddrs.empty() || CurHost->ai_family == preferredAddrs[0]->ai_family)
+ preferredAddrs.push_back(CurHost);
+ else
+ otherAddrs.push_back(CurHost);
+
+ // Ignore UNIX domain sockets
+ do
+ {
+ CurHost = CurHost->ai_next;
+ } while (CurHost != 0 && CurHost->ai_family == AF_UNIX);
+
+ /* If we reached the end of the search list then wrap around to the
+ start */
+ if (CurHost == 0 && LastUsed != 0)
+ CurHost = LastHostAddr;
+
+ // Reached the end of the search cycle
+ if (CurHost == LastUsed)
+ break;
+ }
+
+ // Build a new address vector alternating between preferred and other
+ for (auto prefIter = preferredAddrs.cbegin(), otherIter = otherAddrs.cbegin();
+ prefIter != preferredAddrs.end() || otherIter != otherAddrs.end();)
+ {
+ if (prefIter != preferredAddrs.end())
+ allAddrs.push_back(*prefIter++);
+ if (otherIter != otherAddrs.end())
+ allAddrs.push_back(*otherIter++);
+ }
+
+ return allAddrs;
+}
+ /*}}}*/
+// Check for errors and report them /*{{{*/
+static ResultState WaitAndCheckErrors(std::list<Connection> &Conns, std::unique_ptr<MethodFd> &Fd, long TimeoutMsec, bool ReportTimeout)
+{
+ // The last error detected
+ ResultState Result = ResultState::TRANSIENT_ERROR;
+
+ struct timeval tv = {
+ // Split our millisecond timeout into seconds and microseconds
+ .tv_sec = TimeoutMsec / 1000,
+ .tv_usec = (TimeoutMsec % 1000) * 1000,
+ };
+
+ // We will return once we have no more connections, a time out, or
+ // a success.
+ while (!Conns.empty())
+ {
+ fd_set Set;
+ int nfds = -1;
+
+ FD_ZERO(&Set);
+
+ for (auto &Conn : Conns)
+ {
+ int fd = Conn.Fd->Fd();
+ FD_SET(fd, &Set);
+ nfds = std::max(nfds, fd);
+ }
+
+ {
+ int Res;
+ do
+ {
+ Res = select(nfds + 1, 0, &Set, 0, (TimeoutMsec != 0 ? &tv : 0));
+ } while (Res < 0 && errno == EINTR);
+
+ if (Res == 0)
+ {
+ if (ReportTimeout)
+ {
+ for (auto &Conn : Conns)
+ {
+ Conn.Owner->SetFailReason("Timeout");
+ bad_addr.insert(bad_addr.begin(), Conn.Name);
+ _error->Error(_("Could not connect to %s:%s (%s), "
+ "connection timed out"),
+ Conn.Host.c_str(), Conn.Service, Conn.Name);
+ }
+ }
+ return ResultState::TRANSIENT_ERROR;
+ }
+ }
+
+ // iterate over connections, remove failed ones, and return if
+ // there was a successful one.
+ for (auto ConnI = Conns.begin(); ConnI != Conns.end();)
+ {
+ if (!FD_ISSET(ConnI->Fd->Fd(), &Set))
+ {
+ ConnI++;
+ continue;
+ }
+
+ Result = ConnI->CheckError();
+ if (Result == ResultState::SUCCESSFUL)
+ {
+ Fd = ConnI->Take();
+ return Result;
+ }
+
+ // Connection failed. Erase it and continue to next position
+ ConnI = Conns.erase(ConnI);
+ }
+ }
+
+ return Result;
+}
+ /*}}}*/
+// Connect to a given Hostname /*{{{*/
+static ResultState ConnectToHostname(std::string const &Host, int const Port,
+ const char *const Service, int DefPort, std::unique_ptr<MethodFd> &Fd,
+ unsigned long const TimeOut, aptMethod *const Owner)
+{
+ if (ConnectionAllowed(Service, Host) == false)
+ return ResultState::FATAL_ERROR;
+
+ // Used by getaddrinfo(); prefer port if given, else fallback to service
+ std::string ServiceNameOrPort = Port != 0 ? std::to_string(Port) : Service;
+
+ /* We used a cached address record.. Yes this is against the spec but
+ the way we have setup our rotating dns suggests that this is more
+ sensible */
+ if (LastHost != Host || LastService != ServiceNameOrPort)
+ {
+ Owner->Status(_("Connecting to %s"),Host.c_str());
+
+ // Free the old address structure
+ if (LastHostAddr != 0)
+ {
+ freeaddrinfo(LastHostAddr);
+ LastHostAddr = 0;
+ LastUsed = 0;
+ }
+
+ // We only understand SOCK_STREAM sockets.
+ struct addrinfo Hints;
+ memset(&Hints,0,sizeof(Hints));
+ Hints.ai_socktype = SOCK_STREAM;
+ Hints.ai_flags = 0;
+#ifdef AI_IDN
+ if (_config->FindB("Acquire::Connect::IDN", true) == true)
+ Hints.ai_flags |= AI_IDN;
+#endif
+ // see getaddrinfo(3): only return address if system has such a address configured
+ // useful if system is ipv4 only, to not get ipv6, but that fails if the system has
+ // no address configured: e.g. offline and trying to connect to localhost.
+ if (_config->FindB("Acquire::Connect::AddrConfig", true) == true)
+ Hints.ai_flags |= AI_ADDRCONFIG;
+ Hints.ai_protocol = 0;
+
+ if(_config->FindB("Acquire::ForceIPv4", false) == true)
+ Hints.ai_family = AF_INET;
+ else if(_config->FindB("Acquire::ForceIPv6", false) == true)
+ Hints.ai_family = AF_INET6;
+ else
+ Hints.ai_family = AF_UNSPEC;
+
+ // if we couldn't resolve the host before, we don't try now
+ if (bad_addr.find(Host) != bad_addr.end())
+ {
+ _error->Error(_("Could not resolve '%s'"), Host.c_str());
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ // Resolve both the host and service simultaneously
+ while (1)
+ {
+ int Res;
+ if ((Res = getaddrinfo(Host.c_str(), ServiceNameOrPort.c_str(), &Hints, &LastHostAddr)) != 0 ||
+ LastHostAddr == 0)
+ {
+ if (Res == EAI_NONAME || Res == EAI_SERVICE)
+ {
+ if (DefPort != 0)
+ {
+ ServiceNameOrPort = std::to_string(DefPort);
+ DefPort = 0;
+ continue;
+ }
+ bad_addr.insert(bad_addr.begin(), Host);
+ Owner->SetFailReason("ResolveFailure");
+ _error->Error(_("Could not resolve '%s'"), Host.c_str());
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ if (Res == EAI_AGAIN)
+ {
+ Owner->SetFailReason("TmpResolveFailure");
+ _error->Error(_("Temporary failure resolving '%s'"),
+ Host.c_str());
+ return ResultState::TRANSIENT_ERROR;
+ }
+ if (Res == EAI_SYSTEM)
+ _error->Errno("getaddrinfo", _("System error resolving '%s:%s'"),
+ Host.c_str(), ServiceNameOrPort.c_str());
+ else
+ _error->Error(_("Something wicked happened resolving '%s:%s' (%i - %s)"),
+ Host.c_str(), ServiceNameOrPort.c_str(), Res, gai_strerror(Res));
+ return ResultState::TRANSIENT_ERROR;
+ }
+ break;
+ }
+
+ LastHost = Host;
+ LastService = ServiceNameOrPort;
+ }
+
+ // When we have an IP rotation stay with the last IP.
+ auto Addresses = OrderAddresses(LastUsed != nullptr ? LastUsed : LastHostAddr);
+ std::list<Connection> Conns;
+ ResultState Result = ResultState::SUCCESSFUL;
+
+ for (auto Addr : Addresses)
+ {
+ Connection Conn(Addr, Host, Owner);
+ if (Conn.DoConnect() != ResultState::SUCCESSFUL)
+ continue;
+
+ Conns.push_back(std::move(Conn));
+
+ Result = WaitAndCheckErrors(Conns, Fd, Owner->ConfigFindI("ConnectionAttemptDelayMsec", 250), false);
+
+ if (Result == ResultState::SUCCESSFUL)
+ return ResultState::SUCCESSFUL;
+ }
+
+ if (!Conns.empty())
+ return WaitAndCheckErrors(Conns, Fd, TimeOut * 1000, true);
+ if (Result != ResultState::SUCCESSFUL)
+ return Result;
+ if (_error->PendingError() == true)
+ return ResultState::FATAL_ERROR;
+ _error->Error(_("Unable to connect to %s:%s:"), Host.c_str(), ServiceNameOrPort.c_str());
+ return ResultState::TRANSIENT_ERROR;
+}
+ /*}}}*/
+// Connect - Connect to a server /*{{{*/
+// ---------------------------------------------------------------------
+/* Performs a connection to the server (including SRV record lookup) */
+ResultState Connect(std::string Host, int Port, const char *Service,
+ int DefPort, std::unique_ptr<MethodFd> &Fd,
+ unsigned long TimeOut, aptMethod *Owner)
+{
+ if (_error->PendingError() == true)
+ return ResultState::FATAL_ERROR;
+
+ if (ConnectionAllowed(Service, Host) == false)
+ return ResultState::FATAL_ERROR;
+
+ // Used by getaddrinfo(); prefer port if given, else fallback to service
+ std::string ServiceNameOrPort = Port != 0 ? std::to_string(Port) : Service;
+
+ if(LastHost != Host || LastService != ServiceNameOrPort)
+ {
+ SrvRecords.clear();
+ if (_config->FindB("Acquire::EnableSrvRecords", true) == true)
+ {
+ GetSrvRecords(Host, DefPort, SrvRecords);
+ // RFC2782 defines that a lonely '.' target is an abort reason
+ if (SrvRecords.size() == 1 && SrvRecords[0].target.empty())
+ {
+ _error->Error("SRV records for %s indicate that "
+ "%s service is not available at this domain",
+ Host.c_str(), Service);
+ return ResultState::FATAL_ERROR;
+ }
+ }
+ }
+
+ size_t stackSize = 0;
+ // try to connect in the priority order of the srv records
+ std::string initialHost{std::move(Host)};
+ auto const initialPort = Port;
+ while(SrvRecords.empty() == false)
+ {
+ _error->PushToStack();
+ ++stackSize;
+ // PopFromSrvRecs will also remove the server
+ auto Srv = PopFromSrvRecs(SrvRecords);
+ Host = Srv.target;
+ Port = Srv.port;
+ auto const ret = ConnectToHostname(Host, Port, Service, DefPort, Fd, TimeOut, Owner);
+ if (ret == ResultState::SUCCESSFUL)
+ {
+ while(stackSize--)
+ _error->RevertToStack();
+ return ret;
+ }
+ }
+ Host = std::move(initialHost);
+ Port = initialPort;
+
+ // we have no (good) SrvRecords for this host, connect right away
+ _error->PushToStack();
+ ++stackSize;
+ auto const ret = ConnectToHostname(Host, Port, Service, DefPort, Fd,
+ TimeOut, Owner);
+ while(stackSize--)
+ if (ret == ResultState::SUCCESSFUL)
+ _error->RevertToStack();
+ else
+ _error->MergeWithStack();
+ return ret;
+}
+ /*}}}*/
+// UnwrapSocks - Handle SOCKS setup /*{{{*/
+// ---------------------------------------------------------------------
+/* This does socks magic */
+static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy,
+ char const *const type, bool const ReadWrite, uint8_t *const ToFrom,
+ unsigned int const Size, unsigned int const Timeout)
+{
+ if (WaitFd(ServerFd, ReadWrite, Timeout) == false)
+ {
+ if (ReadWrite)
+ return _error->Error("Timed out while waiting to write '%s' to proxy %s", type, URI::SiteOnly(Proxy).c_str());
+ else
+ return _error->Error("Timed out while waiting to read '%s' from proxy %s", type, URI::SiteOnly(Proxy).c_str());
+ }
+ if (ReadWrite == false)
+ {
+ if (FileFd::Read(ServerFd, ToFrom, Size) == false)
+ return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
+ }
+ else
+ {
+ if (FileFd::Write(ServerFd, ToFrom, Size) == false)
+ return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
+ }
+ return true;
+}
+
+ResultState UnwrapSocks(std::string Host, int Port, URI Proxy, std::unique_ptr<MethodFd> &Fd,
+ unsigned long Timeout, aptMethod *Owner)
+{
+ /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect
+ * for not offering GSSAPI auth which is a must (we only do no or user/pass auth).
+ * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */
+ std::string const ProxyInfo = URI::SiteOnly(Proxy);
+ Owner->Status(_("Connecting to %s (%s)"), "SOCKS5h proxy", ProxyInfo.c_str());
+#define APT_WriteOrFail(TYPE, DATA, LENGTH) \
+ if (TalkToSocksProxy(Fd->Fd(), ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) \
+ return ResultState::TRANSIENT_ERROR
+#define APT_ReadOrFail(TYPE, DATA, LENGTH) \
+ if (TalkToSocksProxy(Fd->Fd(), ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) \
+ return ResultState::TRANSIENT_ERROR
+ if (Host.length() > 255)
+ {
+ _error->Error("Can't use SOCKS5h as hostname %s is too long!", Host.c_str());
+ return ResultState::FATAL_ERROR;
+ }
+ if (Proxy.User.length() > 255 || Proxy.Password.length() > 255)
+ {
+ _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length());
+ return ResultState::FATAL_ERROR;
+ }
+ if (Proxy.User.empty())
+ {
+ uint8_t greeting[] = {0x05, 0x01, 0x00};
+ APT_WriteOrFail("greet-1", greeting, sizeof(greeting));
+ }
+ else
+ {
+ uint8_t greeting[] = {0x05, 0x02, 0x00, 0x02};
+ APT_WriteOrFail("greet-2", greeting, sizeof(greeting));
+ }
+ uint8_t greeting[2];
+ APT_ReadOrFail("greet back", greeting, sizeof(greeting));
+ if (greeting[0] != 0x05)
+ {
+ _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]);
+ return ResultState::FATAL_ERROR;
+ }
+ if (greeting[1] == 0x00)
+ ; // no auth has no method-dependent sub-negotiations
+ else if (greeting[1] == 0x02)
+ {
+ if (Proxy.User.empty())
+ {
+ _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str());
+ return ResultState::FATAL_ERROR;
+ }
+ // user&pass auth sub-negotiations are defined by RFC1929
+ std::vector<uint8_t> auth = {{0x01, static_cast<uint8_t>(Proxy.User.length())}};
+ std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth));
+ auth.push_back(static_cast<uint8_t>(Proxy.Password.length()));
+ std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth));
+ APT_WriteOrFail("user&pass auth", auth.data(), auth.size());
+ uint8_t authstatus[2];
+ APT_ReadOrFail("auth report", authstatus, sizeof(authstatus));
+ if (authstatus[0] != 0x01)
+ {
+ _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]);
+ return ResultState::FATAL_ERROR;
+ }
+ if (authstatus[1] != 0x00)
+ {
+ _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]);
+ return ResultState::FATAL_ERROR;
+ }
+ }
+ else
+ {
+ _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]);
+ return ResultState::FATAL_ERROR;
+ }
+ union {
+ uint16_t *i;
+ uint8_t *b;
+ } portu;
+ uint16_t port = htons(static_cast<uint16_t>(Port));
+ portu.i = &port;
+ std::vector<uint8_t> request = {{0x05, 0x01, 0x00, 0x03, static_cast<uint8_t>(Host.length())}};
+ std::copy(Host.begin(), Host.end(), std::back_inserter(request));
+ request.push_back(portu.b[0]);
+ request.push_back(portu.b[1]);
+ APT_WriteOrFail("request", request.data(), request.size());
+ uint8_t response[4];
+ APT_ReadOrFail("first part of response", response, sizeof(response));
+ if (response[0] != 0x05)
+ {
+ _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]);
+ return ResultState::FATAL_ERROR;
+ }
+ if (response[2] != 0x00)
+ {
+ _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]);
+ return ResultState::FATAL_ERROR;
+ }
+ std::string bindaddr;
+ if (response[3] == 0x01) // IPv4 address
+ {
+ uint8_t ip4port[6];
+ APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port));
+ portu.b[0] = ip4port[4];
+ portu.b[1] = ip4port[5];
+ port = ntohs(*portu.i);
+ strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port);
+ }
+ else if (response[3] == 0x03) // hostname
+ {
+ uint8_t namelength;
+ APT_ReadOrFail("hostname length of response", &namelength, 1);
+ uint8_t hostname[namelength + 2];
+ APT_ReadOrFail("hostname of response", hostname, sizeof(hostname));
+ portu.b[0] = hostname[namelength];
+ portu.b[1] = hostname[namelength + 1];
+ port = ntohs(*portu.i);
+ hostname[namelength] = '\0';
+ strprintf(bindaddr, "%s:%d", hostname, port);
+ }
+ else if (response[3] == 0x04) // IPv6 address
+ {
+ uint8_t ip6port[18];
+ APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port));
+ portu.b[0] = ip6port[16];
+ portu.b[1] = ip6port[17];
+ port = ntohs(*portu.i);
+ strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d",
+ ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7],
+ ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15],
+ port);
+ }
+ else
+ {
+ _error->Error("SOCKS proxy %s destination address is of unknown type: %d",
+ ProxyInfo.c_str(), response[3]);
+ return ResultState::FATAL_ERROR;
+ }
+ if (response[1] != 0x00)
+ {
+ char const *errstr = nullptr;
+ auto errcode = response[1];
+ bool Transient = false;
+ // Tor error reporting can be a bit arcane, lets try to detect & fix it up
+ if (bindaddr == "0.0.0.0:0")
+ {
+ auto const lastdot = Host.rfind('.');
+ if (lastdot == std::string::npos || Host.substr(lastdot) != ".onion")
+ ;
+ else if (errcode == 0x01)
+ {
+ auto const prevdot = Host.rfind('.', lastdot - 1);
+ if (prevdot == std::string::npos && (lastdot == 16 || lastdot == 56))
+ ; // valid .onion address
+ else if (prevdot != std::string::npos && ((lastdot - prevdot) == 17 || (lastdot - prevdot) == 57))
+ ; // valid .onion address with subdomain(s)
+ else
+ {
+ errstr = "Invalid hostname: onion service name must be either 16 or 56 characters long";
+ Owner->SetFailReason("SOCKS");
+ }
+ }
+ // in all likelihood the service is either down or the address has
+ // a typo and so "Host unreachable" is the better understood error
+ // compared to the technically correct "TLL expired".
+ else if (errcode == 0x06)
+ errcode = 0x04;
+ }
+ if (errstr == nullptr)
+ {
+ switch (errcode)
+ {
+ case 0x01:
+ errstr = "general SOCKS server failure";
+ Owner->SetFailReason("SOCKS");
+ break;
+ case 0x02:
+ errstr = "connection not allowed by ruleset";
+ Owner->SetFailReason("SOCKS");
+ break;
+ case 0x03:
+ errstr = "Network unreachable";
+ Owner->SetFailReason("ConnectionTimedOut");
+ Transient = true;
+ break;
+ case 0x04:
+ errstr = "Host unreachable";
+ Owner->SetFailReason("ConnectionTimedOut");
+ Transient = true;
+ break;
+ case 0x05:
+ errstr = "Connection refused";
+ Owner->SetFailReason("ConnectionRefused");
+ Transient = true;
+ break;
+ case 0x06:
+ errstr = "TTL expired";
+ Owner->SetFailReason("Timeout");
+ Transient = true;
+ break;
+ case 0x07:
+ errstr = "Command not supported";
+ Owner->SetFailReason("SOCKS");
+ break;
+ case 0x08:
+ errstr = "Address type not supported";
+ Owner->SetFailReason("SOCKS");
+ break;
+ default:
+ errstr = "Unknown error";
+ Owner->SetFailReason("SOCKS");
+ break;
+ }
+ }
+ _error->Error("SOCKS proxy %s could not connect to %s (%s) due to: %s (%d)",
+ ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str(), errstr, response[1]);
+ return Transient ? ResultState::TRANSIENT_ERROR : ResultState::FATAL_ERROR;
+ }
+ else if (Owner->DebugEnabled())
+ ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s (%s)\n",
+ ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str());
+
+ if (WaitFd(Fd->Fd(), true, Timeout) == false)
+ {
+ _error->Error("SOCKS proxy %s reported connection to %s (%s), but timed out",
+ ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str());
+ return ResultState::TRANSIENT_ERROR;
+ }
+#undef APT_ReadOrFail
+#undef APT_WriteOrFail
+
+ return ResultState::SUCCESSFUL;
+}
+ /*}}}*/
+// UnwrapTLS - Handle TLS connections /*{{{*/
+// ---------------------------------------------------------------------
+/* Performs a TLS handshake on the socket */
+struct TlsFd : public MethodFd
+{
+ std::unique_ptr<MethodFd> UnderlyingFd;
+ gnutls_session_t session;
+ gnutls_certificate_credentials_t credentials;
+ std::string hostname;
+ unsigned long Timeout;
+
+ int Fd() APT_OVERRIDE { return UnderlyingFd->Fd(); }
+
+ ssize_t Read(void *buf, size_t count) APT_OVERRIDE
+ {
+ return HandleError(gnutls_record_recv(session, buf, count));
+ }
+ ssize_t Write(void *buf, size_t count) APT_OVERRIDE
+ {
+ return HandleError(gnutls_record_send(session, buf, count));
+ }
+
+ ssize_t DoTLSHandshake()
+ {
+ int err;
+ // Do the handshake. Our socket is non-blocking, so we need to call WaitFd()
+ // accordingly.
+ do
+ {
+ err = gnutls_handshake(session);
+ if ((err == GNUTLS_E_INTERRUPTED || err == GNUTLS_E_AGAIN) &&
+ WaitFd(this->Fd(), gnutls_record_get_direction(session) == 1, Timeout) == false)
+ {
+ _error->Errno("select", "Could not wait for server fd");
+ return err;
+ }
+ } while (err < 0 && gnutls_error_is_fatal(err) == 0);
+
+ if (err < 0)
+ {
+ // Print reason why validation failed.
+ if (err == GNUTLS_E_CERTIFICATE_VERIFICATION_ERROR)
+ {
+ gnutls_datum_t txt;
+ auto type = gnutls_certificate_type_get(session);
+ auto status = gnutls_session_get_verify_cert_status(session);
+ if (gnutls_certificate_verification_status_print(status, type, &txt, 0) == 0)
+ {
+ _error->Error("Certificate verification failed: %s", txt.data);
+ }
+ gnutls_free(txt.data);
+ }
+ _error->Error("Could not handshake: %s", gnutls_strerror(err));
+ }
+ return err;
+ }
+
+ template <typename T>
+ T HandleError(T err)
+ {
+ // Server may request re-handshake if client certificates need to be provided
+ // based on resource requested
+ if (err == GNUTLS_E_REHANDSHAKE)
+ {
+ int rc = DoTLSHandshake();
+ // Only reset err if DoTLSHandshake() fails.
+ // Otherwise, we want to follow the original error path and set errno to EAGAIN
+ // so that the request is retried.
+ if (rc < 0)
+ err = rc;
+ }
+
+ if (err < 0 && gnutls_error_is_fatal(err))
+ errno = EIO;
+ else if (err < 0)
+ errno = EAGAIN;
+ else
+ errno = 0;
+ return err;
+ }
+
+ int Close() APT_OVERRIDE
+ {
+ auto err = HandleError(gnutls_bye(session, GNUTLS_SHUT_RDWR));
+ auto lower = UnderlyingFd->Close();
+ return err < 0 ? HandleError(err) : lower;
+ }
+
+ bool HasPending() APT_OVERRIDE
+ {
+ return gnutls_record_check_pending(session) > 0;
+ }
+};
+
+ResultState UnwrapTLS(std::string const &Host, std::unique_ptr<MethodFd> &Fd,
+ unsigned long const Timeout, aptMethod * const Owner,
+ aptConfigWrapperForMethods const * const OwnerConf)
+{
+ if (_config->FindB("Acquire::AllowTLS", true) == false)
+ {
+ _error->Error("TLS support has been disabled: Acquire::AllowTLS is false.");
+ return ResultState::FATAL_ERROR;
+ }
+
+ int err;
+ TlsFd *tlsFd = new TlsFd();
+
+ tlsFd->hostname = Host;
+ tlsFd->UnderlyingFd = MethodFd::FromFd(-1); // For now
+ tlsFd->Timeout = Timeout;
+
+ if ((err = gnutls_init(&tlsFd->session, GNUTLS_CLIENT | GNUTLS_NONBLOCK)) < 0)
+ {
+ _error->Error("Internal error: could not allocate credentials: %s", gnutls_strerror(err));
+ return ResultState::FATAL_ERROR;
+ }
+
+ FdFd *fdfd = dynamic_cast<FdFd *>(Fd.get());
+ if (fdfd != nullptr)
+ {
+ gnutls_transport_set_int(tlsFd->session, fdfd->fd);
+ }
+ else
+ {
+ gnutls_transport_set_ptr(tlsFd->session, Fd.get());
+ gnutls_transport_set_pull_function(tlsFd->session,
+ [](gnutls_transport_ptr_t p, void *buf, size_t size) -> ssize_t {
+ return reinterpret_cast<MethodFd *>(p)->Read(buf, size);
+ });
+ gnutls_transport_set_push_function(tlsFd->session,
+ [](gnutls_transport_ptr_t p, const void *buf, size_t size) -> ssize_t {
+ return reinterpret_cast<MethodFd *>(p)->Write((void *)buf, size);
+ });
+ }
+
+ if ((err = gnutls_certificate_allocate_credentials(&tlsFd->credentials)) < 0)
+ {
+ _error->Error("Internal error: could not allocate credentials: %s", gnutls_strerror(err));
+ return ResultState::FATAL_ERROR;
+ }
+
+ // Credential setup
+ std::string fileinfo = OwnerConf->ConfigFind("CaInfo", "");
+ if (fileinfo.empty())
+ {
+ // No CaInfo specified, use system trust store.
+ err = gnutls_certificate_set_x509_system_trust(tlsFd->credentials);
+ if (err == 0)
+ Owner->Warning("No system certificates available. Try installing ca-certificates.");
+ else if (err < 0)
+ {
+ _error->Error("Could not load system TLS certificates: %s", gnutls_strerror(err));
+ return ResultState::FATAL_ERROR;
+ }
+ }
+ else
+ {
+ // CA location has been set, use the specified one instead
+ gnutls_certificate_set_verify_flags(tlsFd->credentials, GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT);
+ err = gnutls_certificate_set_x509_trust_file(tlsFd->credentials, fileinfo.c_str(), GNUTLS_X509_FMT_PEM);
+ if (err < 0)
+ {
+ _error->Error("Could not load certificates from %s (CaInfo option): %s", fileinfo.c_str(), gnutls_strerror(err));
+ return ResultState::FATAL_ERROR;
+ }
+ }
+
+ if (not OwnerConf->ConfigFind("IssuerCert", "").empty())
+ {
+ _error->Error("The option '%s' is not supported anymore", "IssuerCert");
+ return ResultState::FATAL_ERROR;
+ }
+ if (not OwnerConf->ConfigFind("SslForceVersion", "").empty())
+ {
+ _error->Error("The option '%s' is not supported anymore", "SslForceVersion");
+ return ResultState::FATAL_ERROR;
+ }
+
+ // For client authentication, certificate file ...
+ std::string const cert = OwnerConf->ConfigFind("SslCert", "");
+ std::string const key = OwnerConf->ConfigFind("SslKey", "");
+ if (cert.empty() == false)
+ {
+ if ((err = gnutls_certificate_set_x509_key_file(
+ tlsFd->credentials,
+ cert.c_str(),
+ key.empty() ? cert.c_str() : key.c_str(),
+ GNUTLS_X509_FMT_PEM)) < 0)
+ {
+ _error->Error("Could not load client certificate (%s, SslCert option) or key (%s, SslKey option): %s", cert.c_str(), key.c_str(), gnutls_strerror(err));
+ return ResultState::FATAL_ERROR;
+ }
+ }
+
+ // CRL file
+ std::string const crlfile = OwnerConf->ConfigFind("CrlFile", "");
+ if (crlfile.empty() == false)
+ {
+ if ((err = gnutls_certificate_set_x509_crl_file(tlsFd->credentials,
+ crlfile.c_str(),
+ GNUTLS_X509_FMT_PEM)) < 0)
+ {
+ _error->Error("Could not load custom certificate revocation list %s (CrlFile option): %s", crlfile.c_str(), gnutls_strerror(err));
+ return ResultState::FATAL_ERROR;
+ }
+ }
+
+ if ((err = gnutls_credentials_set(tlsFd->session, GNUTLS_CRD_CERTIFICATE, tlsFd->credentials)) < 0)
+ {
+ _error->Error("Internal error: Could not add certificates to session: %s", gnutls_strerror(err));
+ return ResultState::FATAL_ERROR;
+ }
+
+ if ((err = gnutls_set_default_priority(tlsFd->session)) < 0)
+ {
+ _error->Error("Internal error: Could not set algorithm preferences: %s", gnutls_strerror(err));
+ return ResultState::FATAL_ERROR;
+ }
+
+ if (OwnerConf->ConfigFindB("Verify-Peer", true))
+ {
+ gnutls_session_set_verify_cert(tlsFd->session, OwnerConf->ConfigFindB("Verify-Host", true) ? tlsFd->hostname.c_str() : nullptr, 0);
+ }
+
+ // set SNI only if the hostname is really a name and not an address
+ {
+ struct in_addr addr4;
+ struct in6_addr addr6;
+
+ if (inet_pton(AF_INET, tlsFd->hostname.c_str(), &addr4) == 1 ||
+ inet_pton(AF_INET6, tlsFd->hostname.c_str(), &addr6) == 1)
+ /* not a host name */;
+ else if ((err = gnutls_server_name_set(tlsFd->session, GNUTLS_NAME_DNS, tlsFd->hostname.c_str(), tlsFd->hostname.length())) < 0)
+ {
+ _error->Error("Could not set host name %s to indicate to server: %s", tlsFd->hostname.c_str(), gnutls_strerror(err));
+ return ResultState::FATAL_ERROR;
+ }
+ }
+
+ // Set the FD now, so closing it works reliably.
+ tlsFd->UnderlyingFd = std::move(Fd);
+ Fd.reset(tlsFd);
+
+ // Do the handshake.
+ err = tlsFd->DoTLSHandshake();
+
+ if (err < 0)
+ return ResultState::TRANSIENT_ERROR;
+
+ return ResultState::SUCCESSFUL;
+}
+ /*}}}*/
diff --git a/methods/connect.h b/methods/connect.h
new file mode 100644
index 0000000..5c79049
--- /dev/null
+++ b/methods/connect.h
@@ -0,0 +1,50 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ Connect - Replacement connect call
+
+ ##################################################################### */
+ /*}}}*/
+#ifndef CONNECT_H
+#define CONNECT_H
+
+#include <cstddef>
+#include <memory>
+#include <string>
+
+#include "aptmethod.h"
+
+/**
+ * \brief Small representation of a file descriptor for network traffic.
+ *
+ * This provides support for TLS, SOCKS, and HTTP CONNECT proxies.
+ */
+struct MethodFd
+{
+ /// \brief Returns -1 for unusable, or an fd to select() on otherwise
+ virtual int Fd() = 0;
+ /// \brief Should behave like read(2)
+ virtual ssize_t Read(void *buf, size_t count) = 0;
+ /// \brief Should behave like write(2)
+ virtual ssize_t Write(void *buf, size_t count) = 0;
+ /// \brief Closes the file descriptor. Can be called multiple times.
+ virtual int Close() = 0;
+ /// \brief Destructor
+ virtual ~MethodFd(){};
+ /// \brief Construct a MethodFd from a UNIX file descriptor
+ static std::unique_ptr<MethodFd> FromFd(int iFd);
+ /// \brief If there is pending data.
+ virtual bool HasPending();
+};
+
+ResultState Connect(std::string To, int Port, const char *Service, int DefPort,
+ std::unique_ptr<MethodFd> &Fd, unsigned long TimeOut, aptMethod *Owner);
+
+ResultState UnwrapSocks(std::string To, int Port, URI Proxy, std::unique_ptr<MethodFd> &Fd, unsigned long Timeout, aptMethod *Owner);
+ResultState UnwrapTLS(std::string const &To, std::unique_ptr<MethodFd> &Fd, unsigned long Timeout, aptMethod *Owner,
+ aptConfigWrapperForMethods const * OwnerConf);
+
+void RotateDNS();
+
+#endif
diff --git a/methods/copy.cc b/methods/copy.cc
new file mode 100644
index 0000000..82eed15
--- /dev/null
+++ b/methods/copy.cc
@@ -0,0 +1,94 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ Copy URI - This method takes a uri like a file: uri and copies it
+ to the destination file.
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include "aptmethod.h"
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/hashes.h>
+#include <apt-pkg/strutl.h>
+
+#include <string>
+#include <sys/stat.h>
+#include <sys/time.h>
+
+#include <apti18n.h>
+ /*}}}*/
+
+class CopyMethod : public aptMethod
+{
+ virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE;
+
+ public:
+ CopyMethod() : aptMethod("copy", "1.0", SingleInstance | SendConfig | SendURIEncoded)
+ {
+ SeccompFlags = aptMethod::BASE;
+ }
+};
+
+// CopyMethod::Fetch - Fetch a file /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool CopyMethod::Fetch(FetchItem *Itm)
+{
+ // this ensures that relative paths work in copy
+ std::string const File = DecodeSendURI(Itm->Uri.substr(Itm->Uri.find(':')+1));
+
+ // Stat the file and send a start message
+ struct stat Buf;
+ if (stat(File.c_str(),&Buf) != 0)
+ return _error->Errno("stat",_("Failed to stat"));
+
+ // Forumulate a result and send a start message
+ FetchResult Res;
+ Res.Size = Buf.st_size;
+ Res.Filename = Itm->DestFile;
+ Res.LastModified = Buf.st_mtime;
+ Res.IMSHit = false;
+ URIStart(Res);
+
+ // just calc the hashes if the source and destination are identical
+ if (File == Itm->DestFile || Itm->DestFile == "/dev/null")
+ {
+ CalculateHashes(Itm, Res);
+ URIDone(Res);
+ return true;
+ }
+
+ // See if the file exists
+ FileFd From(File,FileFd::ReadOnly);
+ FileFd To(Itm->DestFile,FileFd::WriteAtomic);
+ To.EraseOnFailure();
+
+ // Copy the file
+ if (CopyFile(From,To) == false)
+ {
+ To.OpFail();
+ return false;
+ }
+
+ From.Close();
+ To.Close();
+
+ if (TransferModificationTimes(File.c_str(), Res.Filename.c_str(), Res.LastModified) == false)
+ return false;
+
+ CalculateHashes(Itm, Res);
+ URIDone(Res);
+ return true;
+}
+ /*}}}*/
+
+int main()
+{
+ return CopyMethod().Run();
+}
diff --git a/methods/file.cc b/methods/file.cc
new file mode 100644
index 0000000..b2fe133
--- /dev/null
+++ b/methods/file.cc
@@ -0,0 +1,133 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ File URI method for APT
+
+ This simply checks that the file specified exists, if so the relevant
+ information is returned. If a .gz filename is specified then the file
+ name with .gz removed will also be checked and information about it
+ will be returned in Alt-*
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include "aptmethod.h"
+#include <apt-pkg/aptconfiguration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/hashes.h>
+#include <apt-pkg/strutl.h>
+
+#include <string>
+#include <sys/stat.h>
+
+#include <apti18n.h>
+ /*}}}*/
+
+class FileMethod : public aptMethod
+{
+ virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE;
+
+ public:
+ FileMethod() : aptMethod("file", "1.0", SingleInstance | SendConfig | LocalOnly | SendURIEncoded)
+ {
+ SeccompFlags = aptMethod::BASE;
+ }
+};
+
+// FileMethod::Fetch - Fetch a file /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool FileMethod::Fetch(FetchItem *Itm)
+{
+ URI Get(Itm->Uri);
+ std::string const File = DecodeSendURI(Get.Path);
+ FetchResult Res;
+ if (Get.Host.empty() == false)
+ return _error->Error(_("Invalid URI, local URIS must not start with //"));
+
+ struct stat Buf;
+ // deal with destination files which might linger around
+ if (lstat(Itm->DestFile.c_str(), &Buf) == 0)
+ {
+ if ((Buf.st_mode & S_IFREG) != 0)
+ {
+ if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
+ {
+ if (Itm->ExpectedHashes.VerifyFile(File))
+ {
+ Res.Filename = Itm->DestFile;
+ Res.IMSHit = true;
+ }
+ }
+ }
+ }
+ if (Res.IMSHit != true)
+ RemoveFile("file", Itm->DestFile);
+
+ int olderrno = 0;
+ // See if the file exists
+ if (stat(File.c_str(),&Buf) == 0)
+ {
+ Res.Size = Buf.st_size;
+ Res.Filename = File;
+ Res.LastModified = Buf.st_mtime;
+ Res.IMSHit = false;
+ if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
+ {
+ unsigned long long const filesize = Itm->ExpectedHashes.FileSize();
+ if (filesize != 0 && filesize == Res.Size)
+ Res.IMSHit = true;
+ }
+
+ CalculateHashes(Itm, Res);
+ }
+ else
+ olderrno = errno;
+ if (Res.IMSHit == false)
+ URIStart(Res);
+
+ // See if the uncompressed file exists and reuse it
+ FetchResult AltRes;
+ AltRes.Filename.clear();
+ std::vector<std::string> extensions = APT::Configuration::getCompressorExtensions();
+ for (std::vector<std::string>::const_iterator ext = extensions.begin(); ext != extensions.end(); ++ext)
+ {
+ if (APT::String::Endswith(File, *ext) == true)
+ {
+ std::string const unfile = File.substr(0, File.length() - ext->length());
+ if (stat(unfile.c_str(),&Buf) == 0)
+ {
+ AltRes.Size = Buf.st_size;
+ AltRes.Filename = unfile;
+ AltRes.LastModified = Buf.st_mtime;
+ AltRes.IMSHit = false;
+ if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
+ AltRes.IMSHit = true;
+ break;
+ }
+ // no break here as we could have situations similar to '.gz' vs '.tar.gz' here
+ }
+ }
+
+ if (AltRes.Filename.empty() == false)
+ URIDone(Res,&AltRes);
+ else if (Res.Filename.empty() == false)
+ URIDone(Res);
+ else
+ {
+ errno = olderrno;
+ return _error->Errno(File.c_str(), _("File not found"));
+ }
+
+ return true;
+}
+ /*}}}*/
+
+int main()
+{
+ return FileMethod().Run();
+}
diff --git a/methods/ftp.cc b/methods/ftp.cc
new file mode 100644
index 0000000..379bf32
--- /dev/null
+++ b/methods/ftp.cc
@@ -0,0 +1,1188 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ FTP Acquire Method - This is the FTP acquire method for APT.
+
+ This is a very simple implementation that does not try to optimize
+ at all. Commands are sent synchronously with the FTP server (as the
+ rfc recommends, but it is not really necessary..) and no tricks are
+ done to speed things along.
+
+ RFC 2428 describes the IPv6 FTP behavior
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/hashes.h>
+#include <apt-pkg/strutl.h>
+
+#include <cctype>
+#include <cerrno>
+#include <csignal>
+#include <cstdarg>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+// Internet stuff
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+
+#include "connect.h"
+#include "ftp.h"
+#include "rfc2553emu.h"
+
+#include <apti18n.h>
+ /*}}}*/
+
+using namespace std;
+
+/* This table is for the EPRT and EPSV commands, it maps the OS address
+ family to the IETF address families */
+struct AFMap
+{
+ unsigned long Family;
+ unsigned long IETFFamily;
+};
+
+#ifndef AF_INET6
+struct AFMap AFMap[] = {{AF_INET,1},{0, 0}};
+#else
+struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{0, 0}};
+#endif
+
+unsigned long TimeOut = 30;
+URI Proxy;
+string FtpMethod::FailFile;
+int FtpMethod::FailFd = -1;
+time_t FtpMethod::FailTime = 0;
+
+// FTPConn::FTPConn - Constructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(MethodFd::FromFd(-1)), DataFd(-1),
+ DataListenFd(-1), ServerName(Srv),
+ ForceExtended(false), TryPassive(true),
+ PeerAddrLen(0), ServerAddrLen(0)
+{
+ Debug = _config->FindB("Debug::Acquire::Ftp",false);
+ PasvAddr = 0;
+ Buffer[0] = '\0';
+}
+ /*}}}*/
+// FTPConn::~FTPConn - Destructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+FTPConn::~FTPConn()
+{
+ Close();
+}
+ /*}}}*/
+// FTPConn::Close - Close down the connection /*{{{*/
+// ---------------------------------------------------------------------
+/* Just tear down the socket and data socket */
+void FTPConn::Close()
+{
+ ServerFd->Close();
+ close(DataFd);
+ DataFd = -1;
+ close(DataListenFd);
+ DataListenFd = -1;
+
+ if (PasvAddr != 0)
+ freeaddrinfo(PasvAddr);
+ PasvAddr = 0;
+}
+ /*}}}*/
+// FTPConn::Open - Open a new connection /*{{{*/
+// ---------------------------------------------------------------------
+/* Connect to the server using a non-blocking connection and perform a
+ login. */
+ResultState FTPConn::Open(aptMethod *Owner)
+{
+ // Use the already open connection if possible.
+ if (ServerFd->Fd() != -1)
+ return ResultState::SUCCESSFUL;
+
+ Close();
+
+ // Determine the proxy setting
+ string SpecificProxy = _config->Find("Acquire::ftp::Proxy::" + ServerName.Host);
+ if (!SpecificProxy.empty())
+ {
+ if (SpecificProxy == "DIRECT")
+ Proxy = "";
+ else
+ Proxy = SpecificProxy;
+ }
+ else
+ {
+ string DefProxy = _config->Find("Acquire::ftp::Proxy");
+ if (!DefProxy.empty())
+ {
+ Proxy = DefProxy;
+ }
+ else
+ {
+ char* result = getenv("ftp_proxy");
+ Proxy = result ? result : "";
+ }
+ }
+
+ // Parse no_proxy, a , separated list of domains
+ if (getenv("no_proxy") != 0)
+ {
+ if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
+ Proxy = "";
+ }
+
+ // Determine what host and port to use based on the proxy settings
+ int Port = 0;
+ string Host;
+ if (Proxy.empty() == true)
+ {
+ if (ServerName.Port != 0)
+ Port = ServerName.Port;
+ Host = ServerName.Host;
+ }
+ else
+ {
+ if (Proxy.Port != 0)
+ Port = Proxy.Port;
+ Host = Proxy.Host;
+ }
+
+ /* Connect to the remote server. Since FTP is connection oriented we
+ want to make sure we get a new server every time we reconnect */
+ RotateDNS();
+ auto result = Connect(Host, Port, "ftp", 21, ServerFd, TimeOut, Owner);
+ if (result != ResultState::SUCCESSFUL)
+ return result;
+
+ // Login must be before getpeername otherwise dante won't work.
+ Owner->Status(_("Logging in"));
+ result = Login();
+ if (result != ResultState::SUCCESSFUL)
+ return result;
+
+ // Get the remote server's address
+ PeerAddrLen = sizeof(PeerAddr);
+ if (getpeername(ServerFd->Fd(), (sockaddr *)&PeerAddr, &PeerAddrLen) != 0)
+ {
+ _error->Errno("getpeername", _("Unable to determine the peer name"));
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ // Get the local machine's address
+ ServerAddrLen = sizeof(ServerAddr);
+ if (getsockname(ServerFd->Fd(), (sockaddr *)&ServerAddr, &ServerAddrLen) != 0)
+ {
+ _error->Errno("getsockname", _("Unable to determine the local name"));
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ return ResultState::SUCCESSFUL;
+}
+ /*}}}*/
+// FTPConn::Login - Login to the remote server /*{{{*/
+// ---------------------------------------------------------------------
+/* This performs both normal login and proxy login using a simples script
+ stored in the config file. */
+ResultState FTPConn::Login()
+{
+ unsigned int Tag;
+ string Msg;
+
+ // Setup the variables needed for authentication
+ string User = "anonymous";
+ string Pass = "apt_get_ftp_2.1@debian.linux.user";
+
+ // Fill in the user/pass
+ if (ServerName.User.empty() == false)
+ User = ServerName.User;
+ if (ServerName.Password.empty() == false)
+ Pass = ServerName.Password;
+
+ // Perform simple login
+ if (Proxy.empty() == true)
+ {
+ // Read the initial response
+ if (ReadResp(Tag,Msg) == false)
+ return ResultState::TRANSIENT_ERROR;
+ if (Tag >= 400)
+ {
+ _error->Error(_("The server refused the connection and said: %s"), Msg.c_str());
+ return ResultState::FATAL_ERROR;
+ }
+
+ // Send the user
+ if (WriteMsg(Tag,Msg,"USER %s",User.c_str()) == false)
+ return ResultState::TRANSIENT_ERROR;
+ if (Tag >= 400)
+ {
+ _error->Error(_("USER failed, server said: %s"), Msg.c_str());
+ return ResultState::FATAL_ERROR;
+ }
+
+ if (Tag == 331) { // 331 User name okay, need password.
+ // Send the Password
+ if (WriteMsg(Tag,Msg,"PASS %s",Pass.c_str()) == false)
+ return ResultState::TRANSIENT_ERROR;
+ if (Tag >= 400)
+ {
+ _error->Error(_("PASS failed, server said: %s"), Msg.c_str());
+ return ResultState::FATAL_ERROR;
+ }
+ }
+
+ // Enter passive mode
+ if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
+ TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
+ else
+ TryPassive = _config->FindB("Acquire::FTP::Passive",true);
+ }
+ else
+ {
+ // Read the initial response
+ if (ReadResp(Tag,Msg) == false)
+ return ResultState::TRANSIENT_ERROR;
+ if (Tag >= 400)
+ {
+ _error->Error(_("The server refused the connection and said: %s"), Msg.c_str());
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ // Perform proxy script execution
+ Configuration::Item const *Opts = _config->Tree("Acquire::ftp::ProxyLogin");
+ if (Opts == 0 || Opts->Child == 0)
+ {
+ _error->Error(_("A proxy server was specified but no login "
+ "script, Acquire::ftp::ProxyLogin is empty."));
+ return ResultState::FATAL_ERROR;
+ }
+ Opts = Opts->Child;
+
+ // Iterate over the entire login script
+ for (; Opts != 0; Opts = Opts->Next)
+ {
+ if (Opts->Value.empty() == true)
+ continue;
+
+ // Substitute the variables into the command
+ string Tmp = Opts->Value;
+ Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
+ Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
+ Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
+ Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
+ if (ServerName.Port != 0)
+ {
+ std::string SitePort;
+ strprintf(SitePort, "%u", ServerName.Port);
+ Tmp = SubstVar(Tmp,"$(SITE_PORT)", SitePort);
+ }
+ else
+ Tmp = SubstVar(Tmp,"$(SITE_PORT)", "21");
+ Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
+
+ // Send the command
+ if (WriteMsg(Tag,Msg,"%s",Tmp.c_str()) == false)
+ return ResultState::TRANSIENT_ERROR;
+ if (Tag >= 400)
+ {
+ _error->Error(_("Login script command '%s' failed, server said: %s"), Tmp.c_str(), Msg.c_str());
+ return ResultState::FATAL_ERROR;
+ }
+ }
+
+ // Enter passive mode
+ TryPassive = false;
+ if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
+ TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
+ else
+ {
+ if (_config->Exists("Acquire::FTP::Proxy::Passive") == true)
+ TryPassive = _config->FindB("Acquire::FTP::Proxy::Passive",true);
+ else
+ TryPassive = _config->FindB("Acquire::FTP::Passive",true);
+ }
+ }
+
+ // Force the use of extended commands
+ if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true)
+ ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true);
+ else
+ ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false);
+
+ // Binary mode
+ if (WriteMsg(Tag,Msg,"TYPE I") == false)
+ return ResultState::TRANSIENT_ERROR;
+ if (Tag >= 400)
+ {
+ _error->Error(_("TYPE failed, server said: %s"), Msg.c_str());
+ return ResultState::FATAL_ERROR;
+ }
+ return ResultState::SUCCESSFUL;
+}
+ /*}}}*/
+// FTPConn::ReadLine - Read a line from the server /*{{{*/
+// ---------------------------------------------------------------------
+/* This performs a very simple buffered read. */
+bool FTPConn::ReadLine(string &Text)
+{
+ if (ServerFd->Fd() == -1)
+ return false;
+
+ // Suck in a line
+ while (Len < sizeof(Buffer))
+ {
+ // Scan the buffer for a new line
+ for (unsigned int I = 0; I != Len; I++)
+ {
+ // Escape some special chars
+ if (Buffer[I] == 0)
+ Buffer[I] = '?';
+
+ // End of line?
+ if (Buffer[I] != '\n')
+ continue;
+
+ I++;
+ Text = string(Buffer,I);
+ memmove(Buffer,Buffer+I,Len - I);
+ Len -= I;
+ return true;
+ }
+
+ // Wait for some data..
+ if (WaitFd(ServerFd->Fd(), false, TimeOut) == false)
+ {
+ Close();
+ return _error->Error(_("Connection timeout"));
+ }
+
+ // Suck it back
+ int Res = ServerFd->Read(Buffer + Len, sizeof(Buffer) - Len);
+ if (Res == 0)
+ _error->Error(_("Server closed the connection"));
+ if (Res <= 0)
+ {
+ _error->Errno("read",_("Read error"));
+ Close();
+ return false;
+ }
+ Len += Res;
+ }
+
+ return _error->Error(_("A response overflowed the buffer."));
+}
+ /*}}}*/
+// FTPConn::ReadResp - Read a full response from the server /*{{{*/
+// ---------------------------------------------------------------------
+/* This reads a reply code from the server, it handles both p */
+bool FTPConn::ReadResp(unsigned int &Ret,string &Text)
+{
+ // Grab the first line of the response
+ string Msg;
+ if (ReadLine(Msg) == false)
+ return false;
+
+ // Get the ID code
+ char *End;
+ Ret = strtol(Msg.c_str(),&End,10);
+ if (End - Msg.c_str() != 3)
+ return _error->Error(_("Protocol corruption"));
+
+ // All done ?
+ Text = Msg.c_str()+4;
+ if (*End == ' ')
+ {
+ if (Debug == true)
+ cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
+ return true;
+ }
+
+ if (*End != '-')
+ return _error->Error(_("Protocol corruption"));
+
+ /* Okay, here we do the continued message trick. This is foolish, but
+ proftpd follows the protocol as specified and wu-ftpd doesn't, so
+ we filter. I wonder how many clients break if you use proftpd and
+ put a '- in the 3rd spot in the message? */
+ char Leader[4];
+ strncpy(Leader,Msg.c_str(),3);
+ Leader[3] = 0;
+ while (ReadLine(Msg) == true)
+ {
+ // Short, it must be using RFC continuation..
+ if (Msg.length() < 4)
+ {
+ Text += Msg;
+ continue;
+ }
+
+ // Oops, finished
+ if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == ' ')
+ {
+ Text += Msg.c_str()+4;
+ break;
+ }
+
+ // This message has the wu-ftpd style reply code prefixed
+ if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == '-')
+ {
+ Text += Msg.c_str()+4;
+ continue;
+ }
+
+ // Must be RFC style prefixing
+ Text += Msg;
+ }
+
+ if (Debug == true && _error->PendingError() == false)
+ cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
+
+ return !_error->PendingError();
+}
+ /*}}}*/
+// FTPConn::WriteMsg - Send a message to the server /*{{{*/
+// ---------------------------------------------------------------------
+/* Simple printf like function.. */
+bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...)
+{
+ va_list args;
+ va_start(args,Fmt);
+
+ // sprintf the description
+ char S[400];
+ vsnprintf(S,sizeof(S) - 4,Fmt,args);
+ strcat(S,"\r\n");
+ va_end(args);
+
+ if (Debug == true)
+ cerr << "-> '" << QuoteString(S,"") << "'" << endl;
+
+ // Send it off
+ unsigned long Len = strlen(S);
+ unsigned long Start = 0;
+ while (Len != 0)
+ {
+ if (WaitFd(ServerFd->Fd(), true, TimeOut) == false)
+ {
+ Close();
+ return _error->Error(_("Connection timeout"));
+ }
+
+ int Res = ServerFd->Write(S + Start, Len);
+ if (Res <= 0)
+ {
+ _error->Errno("write",_("Write error"));
+ Close();
+ return false;
+ }
+
+ Len -= Res;
+ Start += Res;
+ }
+
+ return ReadResp(Ret,Text);
+}
+ /*}}}*/
+// FTPConn::GoPasv - Enter Passive mode /*{{{*/
+// ---------------------------------------------------------------------
+/* Try to enter passive mode, the return code does not indicate if passive
+ mode could or could not be established, only if there was a fatal error.
+ We have to enter passive mode every time we make a data connection :| */
+bool FTPConn::GoPasv()
+{
+ /* The PASV command only works on IPv4 sockets, even though it could
+ in theory suppory IPv6 via an all zeros reply */
+ if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET ||
+ ForceExtended == true)
+ return ExtGoPasv();
+
+ if (PasvAddr != 0)
+ freeaddrinfo(PasvAddr);
+ PasvAddr = 0;
+
+ // Try to enable pasv mode
+ unsigned int Tag;
+ string Msg;
+ if (WriteMsg(Tag,Msg,"PASV") == false)
+ return false;
+
+ // Unsupported function
+ string::size_type Pos = Msg.find('(');
+ if (Tag >= 400)
+ return true;
+
+ //wu-2.6.2(1) ftp server, returns
+ //227 Entering Passive Mode 193,219,28,140,150,111
+ //without parentheses, let's try to cope with it.
+ //wget(1) and ftp(1) can.
+ if (Pos == string::npos)
+ Pos = Msg.rfind(' ');
+ else
+ ++Pos;
+
+ // Still unsupported function
+ if (Pos == string::npos)
+ return true;
+
+ // Scan it
+ unsigned a0,a1,a2,a3,p0,p1;
+ if (sscanf(Msg.c_str() + Pos,"%u,%u,%u,%u,%u,%u",&a0,&a1,&a2,&a3,&p0,&p1) != 6)
+ return true;
+
+ /* Some evil servers return 0 to mean their addr. We can actually speak
+ to these servers natively using IPv6 */
+ if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0)
+ {
+ // Get the IP in text form
+ char Name[NI_MAXHOST];
+ char Service[NI_MAXSERV];
+ getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
+ Name,sizeof(Name),Service,sizeof(Service),
+ NI_NUMERICHOST|NI_NUMERICSERV);
+
+ struct addrinfo Hints;
+ memset(&Hints,0,sizeof(Hints));
+ Hints.ai_socktype = SOCK_STREAM;
+ Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
+ Hints.ai_flags |= AI_NUMERICHOST;
+
+ // Get a new passive address.
+ char Port[100];
+ snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
+ if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
+ return true;
+ return true;
+ }
+
+ struct addrinfo Hints;
+ memset(&Hints,0,sizeof(Hints));
+ Hints.ai_socktype = SOCK_STREAM;
+ Hints.ai_family = AF_INET;
+ Hints.ai_flags |= AI_NUMERICHOST;
+
+ // Get a new passive address.
+ char Port[100];
+ snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
+ char Name[100];
+ snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3);
+ if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
+ return true;
+ return true;
+}
+ /*}}}*/
+// FTPConn::ExtGoPasv - Enter Extended Passive mode /*{{{*/
+// ---------------------------------------------------------------------
+/* Try to enter extended passive mode. See GoPasv above and RFC 2428 */
+bool FTPConn::ExtGoPasv()
+{
+ if (PasvAddr != 0)
+ freeaddrinfo(PasvAddr);
+ PasvAddr = 0;
+
+ // Try to enable pasv mode
+ unsigned int Tag;
+ string Msg;
+ if (WriteMsg(Tag,Msg,"EPSV") == false)
+ return false;
+
+ // Unsupported function
+ string::size_type Pos = Msg.find('(');
+ if (Tag >= 400 || Pos == string::npos)
+ return true;
+
+ // Scan it
+ string::const_iterator List[4];
+ unsigned Count = 0;
+ Pos++;
+ for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); ++I)
+ {
+ if (*I != Msg[Pos])
+ continue;
+ if (Count >= 4)
+ return true;
+ List[Count++] = I;
+ }
+ if (Count != 4)
+ return true;
+
+ // Break it up ..
+ unsigned long Proto = 0;
+ unsigned long Port = 0;
+ string IP;
+ IP = string(List[1]+1,List[2]);
+ Port = atoi(string(List[2]+1,List[3]).c_str());
+ if (IP.empty() == false)
+ Proto = atoi(string(List[0]+1,List[1]).c_str());
+
+ if (Port == 0)
+ return false;
+
+ // String version of the port
+ char PStr[100];
+ snprintf(PStr,sizeof(PStr),"%lu",Port);
+
+ // Get the IP in text form
+ struct addrinfo Hints;
+ memset(&Hints,0,sizeof(Hints));
+ Hints.ai_socktype = SOCK_STREAM;
+ Hints.ai_flags |= AI_NUMERICHOST;
+
+ /* The RFC defined case, connect to the old IP/protocol using the
+ new port. */
+ if (IP.empty() == true)
+ {
+ // Get the IP in text form
+ char Name[NI_MAXHOST];
+ char Service[NI_MAXSERV];
+ getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
+ Name,sizeof(Name),Service,sizeof(Service),
+ NI_NUMERICHOST|NI_NUMERICSERV);
+ IP = Name;
+ Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
+ }
+ else
+ {
+ // Get the family..
+ Hints.ai_family = 0;
+ for (unsigned J = 0; AFMap[J].Family != 0; J++)
+ if (AFMap[J].IETFFamily == Proto)
+ Hints.ai_family = AFMap[J].Family;
+ if (Hints.ai_family == 0)
+ return true;
+ }
+
+ // Get a new passive address.
+ if (getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr) != 0)
+ return true;
+
+ return true;
+}
+ /*}}}*/
+// FTPConn::Size - Return the size of a file /*{{{*/
+// ---------------------------------------------------------------------
+/* Grab the file size from the server, 0 means no size or empty file */
+bool FTPConn::Size(const char *Path,unsigned long long &Size)
+{
+ // Query the size
+ unsigned int Tag;
+ string Msg;
+ Size = 0;
+ if (WriteMsg(Tag,Msg,"SIZE %s",Path) == false)
+ return false;
+
+ char *End;
+ Size = strtoull(Msg.c_str(),&End,10);
+ if (Tag >= 400 || End == Msg.c_str())
+ Size = 0;
+ return true;
+}
+ /*}}}*/
+// FTPConn::ModTime - Return the modification time of the file /*{{{*/
+// ---------------------------------------------------------------------
+/* Like Size no error is returned if the command is not supported. If the
+ command fails then time is set to the current time of day to fool
+ date checks. */
+bool FTPConn::ModTime(const char *Path, time_t &Time)
+{
+ Time = time(&Time);
+
+ // Query the mod time
+ unsigned int Tag;
+ string Msg;
+ if (WriteMsg(Tag,Msg,"MDTM %s",Path) == false)
+ return false;
+ if (Tag >= 400 || Msg.empty() == true || isdigit(Msg[0]) == 0)
+ return true;
+
+ // Parse it
+ return FTPMDTMStrToTime(Msg.c_str(), Time);
+}
+ /*}}}*/
+// FTPConn::CreateDataFd - Get a data connection /*{{{*/
+// ---------------------------------------------------------------------
+/* Create the data connection. Call FinalizeDataFd after this though.. */
+bool FTPConn::CreateDataFd()
+{
+ close(DataFd);
+ DataFd = -1;
+
+ // Attempt to enter passive mode.
+ if (TryPassive == true)
+ {
+ if (GoPasv() == false)
+ return false;
+
+ // Oops, didn't work out, don't bother trying again.
+ if (PasvAddr == 0)
+ TryPassive = false;
+ }
+
+ // Passive mode?
+ if (PasvAddr != 0)
+ {
+ // Get a socket
+ if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype,
+ PasvAddr->ai_protocol)) < 0)
+ return _error->Errno("socket",_("Could not create a socket"));
+
+ // Connect to the server
+ SetNonBlock(DataFd,true);
+ if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 &&
+ errno != EINPROGRESS)
+ return _error->Errno("socket",_("Could not create a socket"));
+
+ /* This implements a timeout for connect by opening the connection
+ nonblocking */
+ if (WaitFd(DataFd,true,TimeOut) == false)
+ return _error->Error(_("Could not connect data socket, connection timed out"));
+ unsigned int Err;
+ unsigned int Len = sizeof(Err);
+ if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
+ return _error->Errno("getsockopt",_("Failed"));
+ if (Err != 0)
+ return _error->Error(_("Could not connect passive socket."));
+
+ return true;
+ }
+
+ // Port mode :<
+ close(DataListenFd);
+ DataListenFd = -1;
+
+ // Get the information for a listening socket.
+ struct addrinfo *BindAddr = NULL;
+ struct addrinfo Hints;
+ memset(&Hints,0,sizeof(Hints));
+ Hints.ai_socktype = SOCK_STREAM;
+ Hints.ai_flags |= AI_PASSIVE;
+ Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family;
+ if (getaddrinfo(0,"0",&Hints,&BindAddr) != 0 || BindAddr == NULL)
+ return _error->Error(_("getaddrinfo was unable to get a listening socket"));
+
+ // Construct the socket
+ if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype,
+ BindAddr->ai_protocol)) < 0)
+ {
+ freeaddrinfo(BindAddr);
+ return _error->Errno("socket",_("Could not create a socket"));
+ }
+
+ // Bind and listen
+ if (::bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0)
+ {
+ freeaddrinfo(BindAddr);
+ return _error->Errno("bind",_("Could not bind a socket"));
+ }
+ freeaddrinfo(BindAddr);
+ if (listen(DataListenFd,1) < 0)
+ return _error->Errno("listen",_("Could not listen on the socket"));
+ SetNonBlock(DataListenFd,true);
+
+ // Determine the name to send to the remote
+ struct sockaddr_storage Addr;
+ socklen_t AddrLen = sizeof(Addr);
+ if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0)
+ return _error->Errno("getsockname",_("Could not determine the socket's name"));
+
+
+ // Reverse the address. We need the server address and the data port.
+ char Name[NI_MAXHOST];
+ char Service[NI_MAXSERV];
+ char Service2[NI_MAXSERV];
+ getnameinfo((struct sockaddr *)&Addr,AddrLen,
+ Name,sizeof(Name),Service,sizeof(Service),
+ NI_NUMERICHOST|NI_NUMERICSERV);
+ getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen,
+ Name,sizeof(Name),Service2,sizeof(Service2),
+ NI_NUMERICHOST|NI_NUMERICSERV);
+
+ // Send off an IPv4 address in the old port format
+ if (((struct sockaddr *)&Addr)->sa_family == AF_INET &&
+ ForceExtended == false)
+ {
+ // Convert the dots in the quad into commas
+ for (char *I = Name; *I != 0; I++)
+ if (*I == '.')
+ *I = ',';
+ unsigned long Port = atoi(Service);
+
+ // Send the port command
+ unsigned int Tag;
+ string Msg;
+ if (WriteMsg(Tag,Msg,"PORT %s,%d,%d",
+ Name,
+ (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false)
+ return false;
+ if (Tag >= 400)
+ return _error->Error(_("Unable to send PORT command"));
+ return true;
+ }
+
+ // Construct an EPRT command
+ unsigned Proto = 0;
+ for (unsigned J = 0; AFMap[J].Family != 0; J++)
+ if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family)
+ Proto = AFMap[J].IETFFamily;
+ if (Proto == 0)
+ return _error->Error(_("Unknown address family %u (AF_*)"),
+ ((struct sockaddr *)&Addr)->sa_family);
+
+ // Send the EPRT command
+ unsigned int Tag;
+ string Msg;
+ if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false)
+ return false;
+ if (Tag >= 400)
+ return _error->Error(_("EPRT failed, server said: %s"),Msg.c_str());
+ return true;
+}
+ /*}}}*/
+// FTPConn::Finalize - Complete the Data connection /*{{{*/
+// ---------------------------------------------------------------------
+/* If the connection is in port mode this waits for the other end to hook
+ up to us. */
+bool FTPConn::Finalize()
+{
+ // Passive mode? Do nothing
+ if (PasvAddr != 0)
+ return true;
+
+ // Close any old socket..
+ close(DataFd);
+ DataFd = -1;
+
+ // Wait for someone to connect..
+ if (WaitFd(DataListenFd,false,TimeOut) == false)
+ return _error->Error(_("Data socket connect timed out"));
+
+ // Accept the connection
+ struct sockaddr_in Addr;
+ socklen_t Len = sizeof(Addr);
+ DataFd = accept(DataListenFd,(struct sockaddr *)&Addr,&Len);
+ if (DataFd < 0)
+ return _error->Errno("accept",_("Unable to accept connection"));
+
+ close(DataListenFd);
+ DataListenFd = -1;
+
+ return true;
+}
+ /*}}}*/
+// FTPConn::Get - Get a file /*{{{*/
+// ---------------------------------------------------------------------
+/* This opens a data connection, sends REST and RETR and then
+ transfers the file over. */
+bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
+ Hashes &Hash,bool &Missing, unsigned long long MaximumSize,
+ pkgAcqMethod *Owner)
+{
+ Missing = false;
+ if (CreateDataFd() == false)
+ return false;
+
+ unsigned int Tag;
+ string Msg;
+ if (Resume != 0)
+ {
+ if (WriteMsg(Tag,Msg,"REST %u",Resume) == false)
+ return false;
+ if (Tag >= 400)
+ Resume = 0;
+ }
+
+ if (To.Truncate(Resume) == false)
+ return false;
+
+ if (To.Seek(0) == false)
+ return false;
+
+ if (Resume != 0)
+ {
+ if (Hash.AddFD(To,Resume) == false)
+ {
+ _error->Errno("read",_("Problem hashing file"));
+ return false;
+ }
+ }
+
+ // Send the get command
+ if (WriteMsg(Tag,Msg,"RETR %s",Path) == false)
+ return false;
+
+ if (Tag >= 400)
+ {
+ if (Tag == 550)
+ Missing = true;
+ return _error->Error(_("Unable to fetch file, server said '%s'"),Msg.c_str());
+ }
+
+ // Finish off the data connection
+ if (Finalize() == false)
+ return false;
+
+ // Copy loop
+ unsigned char Buffer[4096];
+ while (1)
+ {
+ // Wait for some data..
+ if (WaitFd(DataFd,false,TimeOut) == false)
+ {
+ Close();
+ return _error->Error(_("Data socket timed out"));
+ }
+
+ // Read the data..
+ int Res = read(DataFd,Buffer,sizeof(Buffer));
+ if (Res == 0)
+ break;
+ if (Res < 0)
+ {
+ if (errno == EAGAIN)
+ continue;
+ break;
+ }
+
+ Hash.Add(Buffer,Res);
+ if (To.Write(Buffer,Res) == false)
+ {
+ Close();
+ return false;
+ }
+
+ if (MaximumSize > 0 && To.Tell() > MaximumSize)
+ {
+ Owner->SetFailReason("MaximumSizeExceeded");
+ return _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
+ To.Tell(), MaximumSize);
+ }
+ }
+
+ // All done
+ close(DataFd);
+ DataFd = -1;
+
+ // Read the closing message from the server
+ if (ReadResp(Tag,Msg) == false)
+ return false;
+ if (Tag >= 400)
+ return _error->Error(_("Data transfer failed, server said '%s'"),Msg.c_str());
+ return true;
+}
+ /*}}}*/
+
+// FtpMethod::FtpMethod - Constructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+FtpMethod::FtpMethod() : aptAuthConfMethod("ftp", "1.0", SendConfig | SendURIEncoded)
+{
+ SeccompFlags = aptMethod::BASE | aptMethod::NETWORK;
+ signal(SIGTERM,SigTerm);
+ signal(SIGINT,SigTerm);
+
+ Server = 0;
+ FailFd = -1;
+}
+ /*}}}*/
+// FtpMethod::SigTerm - Handle a fatal signal /*{{{*/
+// ---------------------------------------------------------------------
+/* This closes and timestamps the open file. This is necessary to get
+ resume behavior on user abort */
+void FtpMethod::SigTerm(int)
+{
+ if (FailFd == -1)
+ _exit(100);
+
+ // Timestamp
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(FailFile.c_str(), times);
+
+ close(FailFd);
+
+ _exit(100);
+}
+ /*}}}*/
+// FtpMethod::Configuration - Handle a configuration message /*{{{*/
+// ---------------------------------------------------------------------
+/* We stash the desired pipeline depth */
+bool FtpMethod::Configuration(string Message)
+{
+ if (aptAuthConfMethod::Configuration(Message) == false)
+ return false;
+
+ TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut);
+
+ return true;
+}
+ /*}}}*/
+// FtpMethod::Fetch - Fetch a file /*{{{*/
+// ---------------------------------------------------------------------
+/* Fetch a single file, called by the base class.. */
+bool FtpMethod::Fetch(FetchItem *Itm)
+{
+ URI Get(Itm->Uri);
+ auto const File = DecodeSendURI(Get.Path);
+ FetchResult Res;
+ Res.Filename = Itm->DestFile;
+ Res.IMSHit = false;
+
+ MaybeAddAuthTo(Get);
+
+ // Connect to the server
+ if (Server == 0 || Server->Comp(Get) == false)
+ {
+ delete Server;
+ Server = new FTPConn(Get);
+ }
+
+ // Could not connect is a transient error..
+ switch (Server->Open(this))
+ {
+ case ResultState::TRANSIENT_ERROR:
+ Server->Close();
+ Fail(true);
+ return true;
+ case ResultState::FATAL_ERROR:
+ Server->Close();
+ Fail(false);
+ return true;
+ case ResultState::SUCCESSFUL:
+ break;
+ }
+
+ // Get the files information
+ Status(_("Query"));
+ unsigned long long Size;
+ if (not Server->Size(File.c_str(), Size) ||
+ not Server->ModTime(File.c_str(), FailTime))
+ {
+ Fail(true);
+ return true;
+ }
+ Res.Size = Size;
+
+ // See if it is an IMS hit
+ if (Itm->LastModified == FailTime)
+ {
+ Res.Size = 0;
+ Res.IMSHit = true;
+ URIDone(Res);
+ return true;
+ }
+
+ // See if the file exists
+ struct stat Buf;
+ if (stat(Itm->DestFile.c_str(),&Buf) == 0)
+ {
+ if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime)
+ {
+ Res.Size = Buf.st_size;
+ Res.LastModified = Buf.st_mtime;
+ Res.ResumePoint = Buf.st_size;
+ URIDone(Res);
+ return true;
+ }
+
+ // Resume?
+ if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size)
+ Res.ResumePoint = Buf.st_size;
+ }
+
+ // Open the file
+ Hashes Hash(Itm->ExpectedHashes);
+ {
+ FileFd Fd(Itm->DestFile,FileFd::WriteAny);
+ if (_error->PendingError() == true)
+ return false;
+
+ URIStart(Res);
+
+ FailFile = Itm->DestFile;
+ (void)(FailFile.c_str()); // Make sure we don't do a malloc in the signal handler
+ FailFd = Fd.Fd();
+
+ bool Missing;
+ if (not Server->Get(File.c_str(), Fd, Res.ResumePoint, Hash, Missing, Itm->MaximumSize, this))
+ {
+ Fd.Close();
+
+ // Timestamp
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(FailFile.c_str(), times);
+
+ // If the file is missing we hard fail and delete the destfile
+ // otherwise transient fail
+ if (Missing == true) {
+ RemoveFile("ftp", FailFile);
+ return false;
+ }
+ Fail(true);
+ return true;
+ }
+
+ Res.Size = Fd.Size();
+
+ // Timestamp
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(Fd.Name().c_str(), times);
+ FailFd = -1;
+ }
+
+ Res.LastModified = FailTime;
+ Res.TakeHashes(Hash);
+
+ URIDone(Res);
+
+ return true;
+}
+ /*}}}*/
+
+int main(int, const char *argv[])
+{
+ /* See if we should be come the http client - we do this for http
+ proxy urls */
+ if (getenv("ftp_proxy") != 0)
+ {
+ URI Proxy(string(getenv("ftp_proxy")));
+
+ // Run the HTTP method
+ if (Proxy.Access == "http")
+ {
+ // Copy over the environment setting
+ char S[300];
+ snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy"));
+ putenv(S);
+ putenv((char *)"no_proxy=");
+
+ // Run the http method
+ string Path = flNotFile(argv[0]) + "http";
+ execl(Path.c_str(),Path.c_str(),(char *)NULL);
+ cerr << _("Unable to invoke ") << Path << endl;
+ exit(100);
+ }
+ }
+ return FtpMethod().Run();
+}
diff --git a/methods/ftp.h b/methods/ftp.h
new file mode 100644
index 0000000..953a1da
--- /dev/null
+++ b/methods/ftp.h
@@ -0,0 +1,91 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ FTP Acquire Method - This is the FTP acquire method for APT.
+
+ ##################################################################### */
+ /*}}}*/
+#ifndef APT_FTP_H
+#define APT_FTP_H
+
+#include "aptmethod.h"
+#include "connect.h"
+#include <apt-pkg/strutl.h>
+
+#include <ctime>
+#include <string>
+#include <sys/socket.h>
+#include <sys/types.h>
+
+class FTPConn
+{
+ char Buffer[1024*10];
+ unsigned long Len;
+ std::unique_ptr<MethodFd> ServerFd;
+ int DataFd;
+ int DataListenFd;
+ URI ServerName;
+ bool ForceExtended;
+ bool TryPassive;
+ bool Debug;
+
+ struct addrinfo *PasvAddr;
+
+ // Generic Peer Address
+ struct sockaddr_storage PeerAddr;
+ socklen_t PeerAddrLen;
+
+ // Generic Server Address (us)
+ struct sockaddr_storage ServerAddr;
+ socklen_t ServerAddrLen;
+
+ // Private helper functions
+ bool ReadLine(std::string &Text);
+ ResultState Login();
+ bool CreateDataFd();
+ bool Finalize();
+
+ public:
+
+ bool Comp(URI Other) {return Other.Host == ServerName.Host && Other.Port == ServerName.Port && Other.User == ServerName.User && Other.Password == ServerName.Password; };
+
+ // Raw connection IO
+ bool ReadResp(unsigned int &Ret,std::string &Text);
+ bool WriteMsg(unsigned int &Ret,std::string &Text,const char *Fmt,...);
+
+ // Connection control
+ ResultState Open(aptMethod *Owner);
+ void Close();
+ bool GoPasv();
+ bool ExtGoPasv();
+
+ // Query
+ bool Size(const char *Path,unsigned long long &Size);
+ bool ModTime(const char *Path, time_t &Time);
+ bool Get(const char *Path,FileFd &To,unsigned long long Resume,
+ Hashes &MD5,bool &Missing, unsigned long long MaximumSize,
+ pkgAcqMethod *Owner);
+
+ explicit FTPConn(URI Srv);
+ ~FTPConn();
+};
+
+class FtpMethod : public aptAuthConfMethod
+{
+ virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE;
+ virtual bool Configuration(std::string Message) APT_OVERRIDE;
+
+ FTPConn *Server;
+
+ static std::string FailFile;
+ static int FailFd;
+ static time_t FailTime;
+ static APT_NORETURN void SigTerm(int);
+
+ public:
+
+ FtpMethod();
+};
+
+#endif
diff --git a/methods/gpgv.cc b/methods/gpgv.cc
new file mode 100644
index 0000000..f89aa8d
--- /dev/null
+++ b/methods/gpgv.cc
@@ -0,0 +1,597 @@
+#include <config.h>
+
+#include "aptmethod.h"
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/gpgv.h>
+#include <apt-pkg/strutl.h>
+
+#include <cctype>
+#include <cerrno>
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <array>
+#include <iostream>
+#include <iterator>
+#include <map>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include <apti18n.h>
+
+using std::string;
+using std::vector;
+
+#define GNUPGPREFIX "[GNUPG:]"
+#define GNUPGBADSIG "[GNUPG:] BADSIG"
+#define GNUPGERRSIG "[GNUPG:] ERRSIG"
+#define GNUPGNOPUBKEY "[GNUPG:] NO_PUBKEY"
+#define GNUPGVALIDSIG "[GNUPG:] VALIDSIG"
+#define GNUPGGOODSIG "[GNUPG:] GOODSIG"
+#define GNUPGEXPKEYSIG "[GNUPG:] EXPKEYSIG"
+#define GNUPGEXPSIG "[GNUPG:] EXPSIG"
+#define GNUPGREVKEYSIG "[GNUPG:] REVKEYSIG"
+#define GNUPGNODATA "[GNUPG:] NODATA"
+#define APTKEYWARNING "[APTKEY:] WARNING"
+#define APTKEYERROR "[APTKEY:] ERROR"
+
+struct Digest {
+ enum class State {
+ Untrusted,
+ Weak,
+ Trusted,
+ } state;
+ char name[32];
+
+ State getState() const {
+ std::string optionUntrusted;
+ std::string optionWeak;
+ strprintf(optionUntrusted, "APT::Hashes::%s::Untrusted", name);
+ strprintf(optionWeak, "APT::Hashes::%s::Weak", name);
+ if (_config->FindB(optionUntrusted, false) == true)
+ return State::Untrusted;
+ if (_config->FindB(optionWeak, false) == true)
+ return State::Weak;
+
+ return state;
+ }
+};
+
+static constexpr Digest Digests[] = {
+ {Digest::State::Untrusted, "Invalid digest"},
+ {Digest::State::Untrusted, "MD5"},
+ {Digest::State::Untrusted, "SHA1"},
+ {Digest::State::Untrusted, "RIPE-MD/160"},
+ {Digest::State::Trusted, "Reserved digest"},
+ {Digest::State::Trusted, "Reserved digest"},
+ {Digest::State::Trusted, "Reserved digest"},
+ {Digest::State::Trusted, "Reserved digest"},
+ {Digest::State::Trusted, "SHA256"},
+ {Digest::State::Trusted, "SHA384"},
+ {Digest::State::Trusted, "SHA512"},
+ {Digest::State::Trusted, "SHA224"},
+};
+
+static Digest FindDigest(std::string const & Digest)
+{
+ int id = atoi(Digest.c_str());
+ if (id >= 0 && static_cast<unsigned>(id) < APT_ARRAY_SIZE(Digests))
+ {
+ return Digests[id];
+ } else {
+ return Digests[0];
+ }
+}
+
+struct Signer {
+ std::string key;
+ std::string note;
+};
+static bool IsTheSameKey(std::string const &validsig, std::string const &goodsig) {
+ // VALIDSIG reports a fingerprint (40 = 24 + 16), GOODSIG can be longid (16) or
+ // fingerprint according to documentation in DETAILS.gz
+ if (goodsig.length() == 40 + strlen("GOODSIG "))
+ return validsig.compare(0, 40, goodsig, strlen("GOODSIG "), 40) == 0;
+ return validsig.compare(24, 16, goodsig, strlen("GOODSIG "), 16) == 0;
+}
+
+struct APT_HIDDEN SignersStorage {
+ std::vector<std::string> Good;
+ std::vector<std::string> Bad;
+ std::vector<std::string> Worthless;
+ // a worthless signature is a expired or revoked one
+ std::vector<Signer> SoonWorthless;
+ std::vector<std::string> NoPubKey;
+ std::vector<std::string> Valid;
+ std::vector<std::string> SignedBy;
+};
+class GPGVMethod : public aptMethod
+{
+ private:
+ string VerifyGetSigners(const char *file, const char *outfile,
+ vector<string> const &keyFpts,
+ vector<string> const &keyFiles,
+ SignersStorage &Signers);
+ string VerifyGetSignersWithLegacy(const char *file, const char *outfile,
+ vector<string> const &keyFpts,
+ vector<string> const &keyFiles,
+ SignersStorage &Signers);
+
+ protected:
+ virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE;
+ public:
+ GPGVMethod() : aptMethod("gpgv", "1.1", SingleInstance | SendConfig | SendURIEncoded){};
+};
+static void PushEntryWithKeyID(std::vector<std::string> &Signers, char * const buffer, bool const Debug)
+{
+ char * const msg = buffer + sizeof(GNUPGPREFIX);
+ char *p = msg;
+ // skip the message
+ while (*p && !isspace(*p))
+ ++p;
+ // skip the separator whitespace
+ ++p;
+ // skip the hexdigit fingerprint
+ while (*p && isxdigit(*p))
+ ++p;
+ // cut the rest from the message
+ *p = '\0';
+ if (Debug == true)
+ std::clog << "Got " << msg << " !" << std::endl;
+ Signers.push_back(msg);
+}
+static void PushEntryWithUID(std::vector<std::string> &Signers, char * const buffer, bool const Debug)
+{
+ std::string msg = buffer + sizeof(GNUPGPREFIX);
+ auto const nuke = msg.find_last_not_of("\n\t\r");
+ if (nuke != std::string::npos)
+ msg.erase(nuke + 1);
+ if (Debug == true)
+ std::clog << "Got " << msg << " !" << std::endl;
+ Signers.push_back(msg);
+}
+static void implodeVector(std::vector<std::string> const &vec, std::ostream &out, char const * const sep)
+{
+ if (vec.empty())
+ return;
+ std::copy(vec.begin(), std::prev(vec.end()), std::ostream_iterator<std::string>(out, sep));
+ out << *vec.rbegin();
+ return;
+}
+string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile,
+ vector<string> const &keyFpts,
+ vector<string> const &keyFiles,
+ SignersStorage &Signers)
+{
+ bool const Debug = DebugEnabled();
+
+ if (Debug == true)
+ std::clog << "inside VerifyGetSigners" << std::endl;
+
+ int fd[2];
+
+ if (pipe(fd) < 0)
+ return "Couldn't create pipe";
+
+ pid_t pid = fork();
+ if (pid < 0)
+ return string("Couldn't spawn new process") + strerror(errno);
+ else if (pid == 0)
+ {
+ std::ostringstream keys;
+ implodeVector(keyFiles, keys, ",");
+ setenv("APT_KEY_NO_LEGACY_KEYRING", "1", true);
+ ExecGPGV(outfile, file, 3, fd, keys.str());
+ }
+ close(fd[1]);
+
+ FILE *pipein = fdopen(fd[0], "r");
+
+ // Loop over the output of apt-key (which really is gnupg), and check the signatures.
+ std::vector<std::string> ErrSigners;
+ std::map<std::string, std::vector<std::string>> SubKeyMapping;
+ size_t buffersize = 0;
+ char *buffer = NULL;
+ bool gotNODATA = false;
+ while (1)
+ {
+ if (getline(&buffer, &buffersize, pipein) == -1)
+ break;
+ if (Debug == true)
+ std::clog << "Read: " << buffer << std::endl;
+
+ // Push the data into three separate vectors, which
+ // we later concatenate. They're kept separate so
+ // if we improve the apt method communication stuff later
+ // it will be better.
+ if (strncmp(buffer, GNUPGBADSIG, sizeof(GNUPGBADSIG)-1) == 0)
+ PushEntryWithUID(Signers.Bad, buffer, Debug);
+ else if (strncmp(buffer, GNUPGERRSIG, sizeof(GNUPGERRSIG)-1) == 0)
+ PushEntryWithKeyID(ErrSigners, buffer, Debug);
+ else if (strncmp(buffer, GNUPGNOPUBKEY, sizeof(GNUPGNOPUBKEY)-1) == 0)
+ {
+ PushEntryWithKeyID(Signers.NoPubKey, buffer, Debug);
+ ErrSigners.erase(std::remove_if(ErrSigners.begin(), ErrSigners.end(), [&](std::string const &errsig) {
+ return errsig.compare(strlen("ERRSIG "), 16, buffer, sizeof(GNUPGNOPUBKEY), 16) == 0; }), ErrSigners.end());
+ }
+ else if (strncmp(buffer, GNUPGNODATA, sizeof(GNUPGNODATA)-1) == 0)
+ gotNODATA = true;
+ else if (strncmp(buffer, GNUPGEXPKEYSIG, sizeof(GNUPGEXPKEYSIG)-1) == 0)
+ PushEntryWithUID(Signers.Worthless, buffer, Debug);
+ else if (strncmp(buffer, GNUPGEXPSIG, sizeof(GNUPGEXPSIG)-1) == 0)
+ PushEntryWithUID(Signers.Worthless, buffer, Debug);
+ else if (strncmp(buffer, GNUPGREVKEYSIG, sizeof(GNUPGREVKEYSIG)-1) == 0)
+ PushEntryWithUID(Signers.Worthless, buffer, Debug);
+ else if (strncmp(buffer, GNUPGGOODSIG, sizeof(GNUPGGOODSIG)-1) == 0)
+ PushEntryWithKeyID(Signers.Good, buffer, Debug);
+ else if (strncmp(buffer, GNUPGVALIDSIG, sizeof(GNUPGVALIDSIG)-1) == 0)
+ {
+ std::istringstream iss(buffer + sizeof(GNUPGVALIDSIG));
+ vector<string> tokens{std::istream_iterator<string>{iss},
+ std::istream_iterator<string>{}};
+ auto const sig = tokens[0];
+ // Reject weak digest algorithms
+ Digest digest = FindDigest(tokens[7]);
+ switch (digest.getState()) {
+ case Digest::State::Weak:
+ // Treat them like an expired key: For that a message about expiry
+ // is emitted, a VALIDSIG, but no GOODSIG.
+ Signers.SoonWorthless.push_back({sig, digest.name});
+ if (Debug == true)
+ std::clog << "Got weak VALIDSIG, key ID: " << sig << std::endl;
+ break;
+ case Digest::State::Untrusted:
+ // Treat them like an expired key: For that a message about expiry
+ // is emitted, a VALIDSIG, but no GOODSIG.
+ Signers.Worthless.push_back(sig);
+ Signers.Good.erase(std::remove_if(Signers.Good.begin(), Signers.Good.end(), [&](std::string const &goodsig) {
+ return IsTheSameKey(sig, goodsig); }), Signers.Good.end());
+ if (Debug == true)
+ std::clog << "Got untrusted VALIDSIG, key ID: " << sig << std::endl;
+ break;
+
+ case Digest::State::Trusted:
+ if (Debug == true)
+ std::clog << "Got trusted VALIDSIG, key ID: " << sig << std::endl;
+ break;
+ }
+
+ Signers.Valid.push_back(sig);
+
+ if (tokens.size() > 9 && sig != tokens[9])
+ SubKeyMapping[tokens[9]].emplace_back(sig);
+ }
+ else if (strncmp(buffer, APTKEYWARNING, sizeof(APTKEYWARNING)-1) == 0)
+ Warning(buffer + sizeof(APTKEYWARNING));
+ else if (strncmp(buffer, APTKEYERROR, sizeof(APTKEYERROR)-1) == 0)
+ _error->Error("%s", buffer + sizeof(APTKEYERROR));
+ }
+ fclose(pipein);
+ free(buffer);
+ std::move(ErrSigners.begin(), ErrSigners.end(), std::back_inserter(Signers.Worthless));
+
+ // apt-key has a --keyid parameter, but this requires gpg, so we call it without it
+ // and instead check after the fact which keyids where used for verification
+ if (keyFpts.empty() == false)
+ {
+ if (Debug == true)
+ {
+ std::clog << "GoodSigs needs to be limited to keyid(s): ";
+ implodeVector(keyFpts, std::clog, ", ");
+ std::clog << "\n";
+ }
+ std::vector<std::string> filteredGood;
+ for (auto &&good: Signers.Good)
+ {
+ if (Debug == true)
+ std::clog << "Key " << good << " is good sig, is it also a valid and allowed one? ";
+ bool found = false;
+ for (auto l : keyFpts)
+ {
+ bool exactKey = false;
+ if (APT::String::Endswith(l, "!"))
+ {
+ exactKey = true;
+ l.erase(l.length() - 1);
+ }
+ if (IsTheSameKey(l, good))
+ {
+ // GOODSIG might be "just" a longid, so we check VALIDSIG which is always a fingerprint
+ if (std::find(Signers.Valid.cbegin(), Signers.Valid.cend(), l) == Signers.Valid.cend())
+ continue;
+ found = true;
+ Signers.SignedBy.push_back(l + "!");
+ break;
+ }
+ else if (exactKey == false)
+ {
+ auto const primary = SubKeyMapping.find(l);
+ if (primary == SubKeyMapping.end())
+ continue;
+ auto const validsubkeysig = std::find_if(primary->second.cbegin(), primary->second.cend(), [&](auto const subkey) {
+ return IsTheSameKey(subkey, good) && std::find(Signers.Valid.cbegin(), Signers.Valid.cend(), subkey) != Signers.Valid.cend();
+ });
+ if (validsubkeysig != primary->second.cend())
+ {
+ found = true;
+ Signers.SignedBy.push_back(l);
+ Signers.SignedBy.push_back(*validsubkeysig + "!");
+ break;
+ }
+ }
+ }
+ if (Debug)
+ std::clog << (found ? "yes" : "no") << "\n";
+ if (found)
+ filteredGood.emplace_back(std::move(good));
+ else
+ Signers.NoPubKey.emplace_back(std::move(good));
+ }
+ Signers.Good= std::move(filteredGood);
+ }
+ else
+ {
+ // for gpg an expired key is valid, too, but we want only the valid & good ones
+ for (auto const &v : Signers.Valid)
+ if (std::any_of(Signers.Good.begin(), Signers.Good.end(),
+ [&v](std::string const &g) { return IsTheSameKey(v, g); }))
+ Signers.SignedBy.push_back(v + "!");
+ for (auto sub : SubKeyMapping)
+ if (std::any_of(sub.second.begin(), sub.second.end(),
+ [&](std::string const &s) {
+ if (std::find(Signers.Valid.begin(), Signers.Valid.end(), s) == Signers.Valid.end())
+ return false;
+ return std::any_of(Signers.Good.begin(), Signers.Good.end(),
+ [&s](std::string const &g) { return IsTheSameKey(s, g); });
+ }))
+ Signers.SignedBy.push_back(sub.first);
+ }
+ std::sort(Signers.SignedBy.begin(), Signers.SignedBy.end());
+
+ int status;
+ waitpid(pid, &status, 0);
+ if (Debug == true)
+ {
+ ioprintf(std::clog, "gpgv exited with status %i\n", WEXITSTATUS(status));
+ }
+
+ if (Debug)
+ {
+ std::cerr << "Summary:\n Good: ";
+ implodeVector(Signers.Good, std::cerr, ", ");
+ std::cerr << "\n Valid: ";
+ implodeVector(Signers.Valid, std::cerr, ", ");
+ std::cerr << "\n Bad: ";
+ implodeVector(Signers.Bad, std::cerr, ", ");
+ std::cerr << "\n Worthless: ";
+ implodeVector(Signers.Worthless, std::cerr, ", ");
+ std::cerr << "\n SoonWorthless: ";
+ std::for_each(Signers.SoonWorthless.begin(), Signers.SoonWorthless.end(), [](Signer const &sig) { std::cerr << sig.key << ", "; });
+ std::cerr << "\n NoPubKey: ";
+ implodeVector(Signers.NoPubKey, std::cerr, ", ");
+ std::cerr << "\n Signed-By: ";
+ implodeVector(Signers.SignedBy, std::cerr, ", ");
+ std::cerr << std::endl << " NODATA: " << (gotNODATA ? "yes" : "no") << std::endl;
+ }
+
+ if (WEXITSTATUS(status) == 112)
+ {
+ // acquire system checks for "NODATA" to generate GPG errors (the others are only warnings)
+ std::string errmsg;
+ //TRANSLATORS: %s is a single techy word like 'NODATA'
+ strprintf(errmsg, _("Clearsigned file isn't valid, got '%s' (does the network require authentication?)"), "NODATA");
+ return errmsg;
+ }
+ else if (gotNODATA)
+ {
+ // acquire system checks for "NODATA" to generate GPG errors (the others are only warnings)
+ std::string errmsg;
+ //TRANSLATORS: %s is a single techy word like 'NODATA'
+ strprintf(errmsg, _("Signed file isn't valid, got '%s' (does the network require authentication?)"), "NODATA");
+ return errmsg;
+ }
+ else if (WEXITSTATUS(status) == 0)
+ {
+ if (keyFpts.empty() == false)
+ {
+ // gpgv will report success, but we want to enforce a certain keyring
+ // so if we haven't found the key the valid we found is in fact invalid
+ if (Signers.Good.empty())
+ return _("At least one invalid signature was encountered.");
+ }
+ else
+ {
+ if (Signers.Good.empty())
+ return _("Internal error: Good signature, but could not determine key fingerprint?!");
+ }
+ return "";
+ }
+ else if (WEXITSTATUS(status) == 1)
+ return _("At least one invalid signature was encountered.");
+ else if (WEXITSTATUS(status) == 111)
+ return _("Could not execute 'apt-key' to verify signature (is gnupg installed?)");
+ else
+ return _("Unknown error executing apt-key");
+}
+string GPGVMethod::VerifyGetSignersWithLegacy(const char *file, const char *outfile,
+ vector<string> const &keyFpts,
+ vector<string> const &keyFiles,
+ SignersStorage &Signers)
+{
+ string const msg = VerifyGetSigners(file, outfile, keyFpts, keyFiles, Signers);
+ if (_error->PendingError())
+ return msg;
+
+ // Bad signature always remains bad, no need to retry against trusted.gpg
+ if (!Signers.Bad.empty())
+ return msg;
+
+ // We do not have a key file pinned, did not find a good signature, but found
+ // missing keys - let's retry with trusted.gpg
+ if (keyFiles.empty() && Signers.Valid.empty() && !Signers.NoPubKey.empty())
+ {
+ std::vector<std::string> legacyKeyFiles{_config->FindFile("Dir::Etc::trusted")};
+ if (legacyKeyFiles[0].empty())
+ return msg;
+ if (DebugEnabled())
+ std::clog << "Retrying against " << legacyKeyFiles[0] << "\n";
+
+ SignersStorage legacySigners;
+
+ string const legacyMsg = VerifyGetSigners(file, outfile, keyFpts, legacyKeyFiles, legacySigners);
+ if (_error->PendingError())
+ return legacyMsg;
+ // Hooray, we found a key apparently, something verified as good or bad
+ if (!legacySigners.Valid.empty() || !legacySigners.Bad.empty())
+ {
+ std::string warning;
+ strprintf(warning,
+ _("Key is stored in legacy trusted.gpg keyring (%s), see the DEPRECATION section in apt-key(8) for details."),
+ legacyKeyFiles[0].c_str());
+ Warning(std::move(warning));
+ Signers = std::move(legacySigners);
+ return legacyMsg;
+ }
+
+ }
+ return msg;
+}
+static std::string GenerateKeyFile(std::string const key)
+{
+ FileFd fd;
+ GetTempFile("apt-key.XXXXXX.asc", false, &fd);
+ fd.Write(key.data(), key.size());
+ return fd.Name();
+}
+
+bool GPGVMethod::URIAcquire(std::string const &Message, FetchItem *Itm)
+{
+ URI const Get(Itm->Uri);
+ std::string const Path = DecodeSendURI(Get.Host + Get.Path); // To account for relative paths
+ SignersStorage Signers;
+
+ std::vector<std::string> keyFpts, keyFiles;
+ struct TemporaryFile
+ {
+ std::string name = "";
+ ~TemporaryFile() { RemoveFile("~TemporaryFile", name); }
+ } tmpKey;
+
+ std::string SignedBy = DeQuoteString(LookupTag(Message, "Signed-By"));
+
+ if (SignedBy.find("-----BEGIN PGP PUBLIC KEY BLOCK-----") != std::string::npos)
+ {
+ tmpKey.name = GenerateKeyFile(SignedBy);
+ keyFiles.emplace_back(tmpKey.name);
+ }
+ else
+ {
+ for (auto &&key : VectorizeString(SignedBy, ','))
+ if (key.empty() == false && key[0] == '/')
+ keyFiles.emplace_back(std::move(key));
+ else
+ keyFpts.emplace_back(std::move(key));
+ }
+
+ // Run apt-key on file, extract contents and get the key ID of the signer
+ string const msg = VerifyGetSignersWithLegacy(Path.c_str(), Itm->DestFile.c_str(), keyFpts, keyFiles, Signers);
+ if (_error->PendingError())
+ return false;
+
+ // Check if all good signers are soon worthless and warn in that case
+ if (std::all_of(Signers.Good.begin(), Signers.Good.end(), [&](std::string const &good) {
+ return std::any_of(Signers.SoonWorthless.begin(), Signers.SoonWorthless.end(), [&](Signer const &weak) {
+ return IsTheSameKey(weak.key, good);
+ });
+ }))
+ {
+ for (auto const & Signer : Signers.SoonWorthless)
+ {
+ std::string msg;
+ // TRANSLATORS: The second %s is the reason and is untranslated for repository owners.
+ strprintf(msg, _("Signature by key %s uses weak digest algorithm (%s)"), Signer.key.c_str(), Signer.note.c_str());
+ Warning(std::move(msg));
+ }
+ }
+
+ if (Signers.Good.empty() || !Signers.Bad.empty() || !Signers.NoPubKey.empty())
+ {
+ string errmsg;
+ // In this case, something bad probably happened, so we just go
+ // with what the other method gave us for an error message.
+ if (Signers.Bad.empty() && Signers.Worthless.empty() && Signers.NoPubKey.empty())
+ errmsg = msg;
+ else
+ {
+ if (!Signers.Bad.empty())
+ {
+ errmsg += _("The following signatures were invalid:\n");
+ for (auto const &I : Signers.Bad)
+ errmsg.append(I).append("\n");
+ }
+ if (!Signers.Worthless.empty())
+ {
+ errmsg += _("The following signatures were invalid:\n");
+ for (auto const &I : Signers.Worthless)
+ errmsg.append(I).append("\n");
+ }
+ if (!Signers.NoPubKey.empty())
+ {
+ errmsg += _("The following signatures couldn't be verified because the public key is not available:\n");
+ for (auto const &I : Signers.NoPubKey)
+ errmsg.append(I).append("\n");
+ }
+ }
+ // this is only fatal if we have no good sigs or if we have at
+ // least one bad signature. good signatures and NoPubKey signatures
+ // happen easily when a file is signed with multiple signatures
+ if (Signers.Good.empty() or !Signers.Bad.empty())
+ return _error->Error("%s", errmsg.c_str());
+ }
+
+ std::unordered_map<std::string, std::string> fields;
+ fields.emplace("URI", Itm->Uri);
+ fields.emplace("Filename", Itm->DestFile);
+ if (Signers.SignedBy.empty() == false)
+ {
+ std::ostringstream out;
+ implodeVector(Signers.SignedBy, out, "\n");
+ fields.emplace("Signed-By", out.str());
+ }
+ {
+ // Just pass the raw output up, because passing it as a real data
+ // structure is too difficult with the method stuff. We keep it
+ // as three separate vectors for future extensibility.
+ std::vector<std::string> gpgvoutput;
+ std::move(Signers.Good.begin(), Signers.Good.end(), std::back_inserter(gpgvoutput));
+ std::move(Signers.Bad.begin(), Signers.Bad.end(), std::back_inserter(gpgvoutput));
+ std::move(Signers.NoPubKey.begin(), Signers.NoPubKey.end(), std::back_inserter(gpgvoutput));
+ if (gpgvoutput.empty() == false)
+ {
+ std::ostringstream out;
+ implodeVector(gpgvoutput, out, "\n");
+ fields.emplace("GPGVOutput", out.str());
+ }
+ }
+ SendMessage("201 URI Done", std::move(fields));
+ Dequeue();
+
+ if (DebugEnabled())
+ std::clog << "apt-key succeeded\n";
+
+ return true;
+}
+
+
+int main()
+{
+ return GPGVMethod().Run();
+}
diff --git a/methods/http.cc b/methods/http.cc
new file mode 100644
index 0000000..9b45506
--- /dev/null
+++ b/methods/http.cc
@@ -0,0 +1,1053 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ HTTP Acquire Method - This is the HTTP acquire method for APT.
+
+ It uses HTTP/1.1 and many of the fancy options there-in, such as
+ pipelining, range, if-range and so on.
+
+ It is based on a doubly buffered select loop. A groupe of requests are
+ fed into a single output buffer that is constantly fed out the
+ socket. This provides ideal pipelining as in many cases all of the
+ requests will fit into a single packet. The input socket is buffered
+ the same way and fed into the fd for the file (may be a pipe in future).
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/hashes.h>
+#include <apt-pkg/proxy.h>
+#include <apt-pkg/string_view.h>
+#include <apt-pkg/strutl.h>
+
+#include <cerrno>
+#include <chrono>
+#include <csignal>
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <sstream>
+#include <arpa/inet.h>
+#include <sys/select.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#include "config.h"
+#include "connect.h"
+#include "http.h"
+
+#include <apti18n.h>
+
+#ifdef HAVE_SYSTEMD
+#include <systemd/sd-login.h>
+#endif
+ /*}}}*/
+using namespace std;
+
+unsigned long long CircleBuf::BwReadLimit=0;
+unsigned long long CircleBuf::BwTickReadData=0;
+std::chrono::steady_clock::duration CircleBuf::BwReadTick{0};
+const unsigned int CircleBuf::BW_HZ=10;
+
+// CircleBuf::CircleBuf - Circular input buffer /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+CircleBuf::CircleBuf(HttpMethod const * const Owner, unsigned long long Size)
+ : Size(Size), Hash(NULL), TotalWriten(0)
+{
+ Buf = new unsigned char[Size];
+ Reset();
+
+ CircleBuf::BwReadLimit = Owner->ConfigFindI("Dl-Limit", 0) * 1024;
+}
+ /*}}}*/
+// CircleBuf::Reset - Reset to the default state /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void CircleBuf::Reset()
+{
+ InP = 0;
+ OutP = 0;
+ StrPos = 0;
+ TotalWriten = 0;
+ MaxGet = (unsigned long long)-1;
+ OutQueue = string();
+ if (Hash != NULL)
+ {
+ delete Hash;
+ Hash = NULL;
+ }
+}
+ /*}}}*/
+// CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
+// ---------------------------------------------------------------------
+/* This fills up the buffer with as much data as is in the FD, assuming it
+ is non-blocking.. */
+bool CircleBuf::Read(std::unique_ptr<MethodFd> const &Fd)
+{
+ size_t ReadThisCycle = 0;
+ while (1)
+ {
+ // Woops, buffer is full
+ if (InP - OutP == Size)
+ return true;
+
+ // what's left to read in this tick
+ unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
+
+ if(CircleBuf::BwReadLimit) {
+ auto const now = std::chrono::steady_clock::now().time_since_epoch();
+ auto const d = now - CircleBuf::BwReadTick;
+
+ auto const tickLen = std::chrono::microseconds(std::chrono::seconds(1)) / BW_HZ;
+ if(d > tickLen) {
+ CircleBuf::BwReadTick = now;
+ CircleBuf::BwTickReadData = 0;
+ }
+
+ if(CircleBuf::BwTickReadData >= BwReadMax) {
+ usleep(tickLen.count());
+ return true;
+ }
+ }
+
+ // Write the buffer segment
+ ssize_t Res;
+ if(CircleBuf::BwReadLimit) {
+ Res = Fd->Read(Buf + (InP % Size),
+ BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
+ } else
+ Res = Fd->Read(Buf + (InP % Size), LeftRead());
+
+ if(Res > 0 && BwReadLimit > 0)
+ CircleBuf::BwTickReadData += Res;
+
+ if (Res == 0)
+ return ReadThisCycle != 0;
+ if (Res < 0)
+ {
+ if (errno == EAGAIN)
+ return true;
+ return false;
+ }
+
+ InP += Res;
+ ReadThisCycle += Res;
+ }
+}
+ /*}}}*/
+// CircleBuf::Read - Put the string into the buffer /*{{{*/
+// ---------------------------------------------------------------------
+/* This will hold the string in and fill the buffer with it as it empties */
+bool CircleBuf::Read(string const &Data)
+{
+ OutQueue.append(Data);
+ FillOut();
+ return true;
+}
+ /*}}}*/
+// CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void CircleBuf::FillOut()
+{
+ if (OutQueue.empty() == true)
+ return;
+ while (1)
+ {
+ // Woops, buffer is full
+ if (InP - OutP == Size)
+ return;
+
+ // Write the buffer segment
+ unsigned long long Sz = LeftRead();
+ if (OutQueue.length() - StrPos < Sz)
+ Sz = OutQueue.length() - StrPos;
+ memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
+
+ // Advance
+ StrPos += Sz;
+ InP += Sz;
+ if (OutQueue.length() == StrPos)
+ {
+ StrPos = 0;
+ OutQueue = "";
+ return;
+ }
+ }
+}
+ /*}}}*/
+// CircleBuf::Write - Write from the buffer into a FD /*{{{*/
+// ---------------------------------------------------------------------
+/* This empties the buffer into the FD. */
+bool CircleBuf::Write(std::unique_ptr<MethodFd> const &Fd)
+{
+ while (1)
+ {
+ FillOut();
+
+ // Woops, buffer is empty
+ if (OutP == InP)
+ return true;
+
+ if (OutP == MaxGet)
+ return true;
+
+ // Write the buffer segment
+ ssize_t Res;
+ Res = Fd->Write(Buf + (OutP % Size), LeftWrite());
+
+ if (Res < 0)
+ {
+ if (errno == EAGAIN)
+ return true;
+
+ return false;
+ }
+
+ TotalWriten += Res;
+
+ if (Hash != NULL)
+ Hash->Add(Buf + (OutP%Size),Res);
+
+ OutP += Res;
+ }
+}
+ /*}}}*/
+// CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
+// ---------------------------------------------------------------------
+/* This copies till the first empty line */
+bool CircleBuf::WriteTillEl(string &Data,bool Single)
+{
+ // We cheat and assume it is unneeded to have more than one buffer load
+ for (unsigned long long I = OutP; I < InP; I++)
+ {
+ if (Buf[I%Size] != '\n')
+ continue;
+ ++I;
+
+ if (Single == false)
+ {
+ if (I < InP && Buf[I%Size] == '\r')
+ ++I;
+ if (I >= InP || Buf[I%Size] != '\n')
+ continue;
+ ++I;
+ }
+
+ Data = "";
+ while (OutP < I)
+ {
+ unsigned long long Sz = LeftWrite();
+ if (Sz == 0)
+ return false;
+ if (I - OutP < Sz)
+ Sz = I - OutP;
+ Data += string((char *)(Buf + (OutP%Size)),Sz);
+ OutP += Sz;
+ }
+ return true;
+ }
+ return false;
+}
+ /*}}}*/
+// CircleBuf::Write - Write from the buffer to a string /*{{{*/
+// ---------------------------------------------------------------------
+/* This copies everything */
+bool CircleBuf::Write(string &Data)
+{
+ Data = std::string((char *)Buf + (OutP % Size), LeftWrite());
+ OutP += LeftWrite();
+ return true;
+}
+ /*}}}*/
+CircleBuf::~CircleBuf() /*{{{*/
+{
+ delete [] Buf;
+ delete Hash;
+}
+ /*}}}*/
+
+// UnwrapHTTPConnect - Does the HTTP CONNECT handshake /*{{{*/
+// ---------------------------------------------------------------------
+/* Performs a TLS handshake on the socket */
+struct HttpConnectFd : public MethodFd
+{
+ std::unique_ptr<MethodFd> UnderlyingFd;
+ std::string Buffer;
+
+ int Fd() APT_OVERRIDE { return UnderlyingFd->Fd(); }
+
+ ssize_t Read(void *buf, size_t count) APT_OVERRIDE
+ {
+ if (!Buffer.empty())
+ {
+ auto read = count < Buffer.size() ? count : Buffer.size();
+
+ memcpy(buf, Buffer.data(), read);
+ Buffer.erase(Buffer.begin(), Buffer.begin() + read);
+ return read;
+ }
+
+ return UnderlyingFd->Read(buf, count);
+ }
+ ssize_t Write(void *buf, size_t count) APT_OVERRIDE
+ {
+ return UnderlyingFd->Write(buf, count);
+ }
+
+ int Close() APT_OVERRIDE
+ {
+ return UnderlyingFd->Close();
+ }
+
+ bool HasPending() APT_OVERRIDE
+ {
+ return !Buffer.empty();
+ }
+};
+
+static ResultState UnwrapHTTPConnect(std::string Host, int Port, URI Proxy, std::unique_ptr<MethodFd> &Fd,
+ unsigned long Timeout, aptAuthConfMethod *Owner)
+{
+ Owner->Status(_("Connecting to %s (%s)"), "HTTP proxy", URI::SiteOnly(Proxy).c_str());
+ // The HTTP server expects a hostname with a trailing :port
+ std::stringstream Req;
+ std::string ProperHost;
+
+ if (Host.find(':') != std::string::npos)
+ ProperHost = '[' + Host + ']';
+ else
+ ProperHost = Host;
+
+ // Build the connect
+ Req << "CONNECT " << Host << ":" << std::to_string(Port) << " HTTP/1.1\r\n";
+ if (Proxy.Port != 0)
+ Req << "Host: " << ProperHost << ":" << std::to_string(Port) << "\r\n";
+ else
+ Req << "Host: " << ProperHost << "\r\n";
+
+ Owner->MaybeAddAuthTo(Proxy);
+ if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
+ Req << "Proxy-Authorization: Basic "
+ << Base64Encode(Proxy.User + ":" + Proxy.Password) << "\r\n";
+
+ Req << "User-Agent: " << Owner->ConfigFind("User-Agent", "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n";
+
+ Req << "\r\n";
+
+ CircleBuf In(dynamic_cast<HttpMethod *>(Owner), 4096);
+ CircleBuf Out(dynamic_cast<HttpMethod *>(Owner), 4096);
+ std::string Headers;
+
+ if (Owner->DebugEnabled() == true)
+ cerr << Req.str() << endl;
+ Out.Read(Req.str());
+
+ // Writing from proxy
+ while (Out.WriteSpace())
+ {
+ if (WaitFd(Fd->Fd(), true, Timeout) == false)
+ {
+ _error->Errno("select", "Writing to proxy failed");
+ return ResultState::TRANSIENT_ERROR;
+ }
+ if (Out.Write(Fd) == false)
+ {
+ _error->Errno("write", "Writing to proxy failed");
+ return ResultState::TRANSIENT_ERROR;
+ }
+ }
+
+ while (In.ReadSpace())
+ {
+ if (WaitFd(Fd->Fd(), false, Timeout) == false)
+ {
+ _error->Errno("select", "Reading from proxy failed");
+ return ResultState::TRANSIENT_ERROR;
+ }
+ if (In.Read(Fd) == false)
+ {
+ _error->Errno("read", "Reading from proxy failed");
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ if (In.WriteTillEl(Headers))
+ break;
+ }
+
+ if (Owner->DebugEnabled() == true)
+ cerr << Headers << endl;
+
+ if (!(APT::String::Startswith(Headers, "HTTP/1.0 200") || APT::String::Startswith(Headers, "HTTP/1.1 200")))
+ {
+ _error->Error("Invalid response from proxy: %s", Headers.c_str());
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ if (In.WriteSpace())
+ {
+ // Maybe there is actual data already read, if so we need to buffer it
+ std::unique_ptr<HttpConnectFd> NewFd(new HttpConnectFd());
+ In.Write(NewFd->Buffer);
+ NewFd->UnderlyingFd = std::move(Fd);
+ Fd = std::move(NewFd);
+ }
+
+ return ResultState::SUCCESSFUL;
+}
+ /*}}}*/
+
+// HttpServerState::HttpServerState - Constructor /*{{{*/
+HttpServerState::HttpServerState(URI Srv, HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, APT_BUFFER_SIZE), Out(Owner, 4 * 1024)
+{
+ TimeOut = Owner->ConfigFindI("Timeout", TimeOut);
+ ServerFd = MethodFd::FromFd(-1);
+ Reset();
+}
+ /*}}}*/
+// HttpServerState::Open - Open a connection to the server /*{{{*/
+// ---------------------------------------------------------------------
+/* This opens a connection to the server. */
+ResultState HttpServerState::Open()
+{
+ // Use the already open connection if possible.
+ if (ServerFd->Fd() != -1)
+ return ResultState::SUCCESSFUL;
+
+ Close();
+ In.Reset();
+ Out.Reset();
+ Persistent = true;
+
+ bool tls = (ServerName.Access == "https" || APT::String::Endswith(ServerName.Access, "+https"));
+
+ // Determine the proxy setting
+ // Used to run AutoDetectProxy(ServerName) here, but we now send a Proxy
+ // header in the URI Acquire request and set "Acquire::"+uri.Access+"::proxy::"+uri.Host
+ // to it in BaseHttpMethod::Loop()
+ string SpecificProxy = Owner->ConfigFind("Proxy::" + ServerName.Host, "");
+ if (!SpecificProxy.empty())
+ {
+ if (SpecificProxy == "DIRECT")
+ Proxy = "";
+ else
+ Proxy = SpecificProxy;
+ }
+ else
+ {
+ string DefProxy = Owner->ConfigFind("Proxy", "");
+ if (!DefProxy.empty())
+ {
+ Proxy = DefProxy;
+ }
+ else
+ {
+ char *result = getenv("http_proxy");
+ Proxy = result ? result : "";
+ if (tls == true)
+ {
+ char *result = getenv("https_proxy");
+ if (result != nullptr)
+ {
+ Proxy = result;
+ }
+ }
+ }
+ }
+
+ // Parse no_proxy, a , separated list of domains
+ if (getenv("no_proxy") != 0)
+ {
+ if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
+ Proxy = "";
+ }
+
+ if (Proxy.empty() == false)
+ Owner->AddProxyAuth(Proxy, ServerName);
+
+ auto const DefaultService = tls ? "https" : "http";
+ auto const DefaultPort = tls ? 443 : 80;
+ if (Proxy.Access == "socks5h")
+ {
+ auto result = Connect(Proxy.Host, Proxy.Port, "socks", 1080, ServerFd, TimeOut, Owner);
+ if (result != ResultState::SUCCESSFUL)
+ return result;
+
+ result = UnwrapSocks(ServerName.Host, ServerName.Port == 0 ? DefaultPort : ServerName.Port,
+ Proxy, ServerFd, Owner->ConfigFindI("TimeOut", 30), Owner);
+ if (result != ResultState::SUCCESSFUL)
+ return result;
+ }
+ else
+ {
+ // Determine what host and port to use based on the proxy settings
+ int Port = 0;
+ string Host;
+ if (Proxy.empty() == true || Proxy.Host.empty() == true)
+ {
+ if (ServerName.Port != 0)
+ Port = ServerName.Port;
+ Host = ServerName.Host;
+ }
+ else if (Proxy.Access != "http" && Proxy.Access != "https")
+ {
+ _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str());
+ return ResultState::FATAL_ERROR;
+ }
+ else
+ {
+ if (Proxy.Port != 0)
+ Port = Proxy.Port;
+ Host = Proxy.Host;
+
+ if (Proxy.Access == "https" && Port == 0)
+ Port = 443;
+ }
+ auto result = Connect(Host, Port, DefaultService, DefaultPort, ServerFd, TimeOut, Owner);
+ if (result != ResultState::SUCCESSFUL)
+ return result;
+ if (Host == Proxy.Host && Proxy.Access == "https")
+ {
+ aptConfigWrapperForMethods ProxyConf{std::vector<std::string>{"http", "https"}};
+ ProxyConf.setPostfixForMethodNames(Proxy.Host.c_str());
+ result = UnwrapTLS(Proxy.Host, ServerFd, TimeOut, Owner, &ProxyConf);
+ if (result != ResultState::SUCCESSFUL)
+ return result;
+ }
+ if (Host == Proxy.Host && tls)
+ {
+ result = UnwrapHTTPConnect(ServerName.Host, ServerName.Port == 0 ? DefaultPort : ServerName.Port, Proxy, ServerFd, Owner->ConfigFindI("TimeOut", 30), Owner);
+ if (result != ResultState::SUCCESSFUL)
+ return result;
+ }
+ }
+
+ if (tls)
+ return UnwrapTLS(ServerName.Host, ServerFd, TimeOut, Owner, Owner);
+
+ return ResultState::SUCCESSFUL;
+}
+ /*}}}*/
+// HttpServerState::Close - Close a connection to the server /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool HttpServerState::Close()
+{
+ ServerFd->Close();
+ return true;
+}
+ /*}}}*/
+// HttpServerState::RunData - Transfer the data from the socket /*{{{*/
+ResultState HttpServerState::RunData(RequestState &Req)
+{
+ Req.State = RequestState::Data;
+
+ // Chunked transfer encoding is fun..
+ if (Req.Encoding == RequestState::Chunked)
+ {
+ while (1)
+ {
+ // Grab the block size
+ ResultState Last = ResultState::SUCCESSFUL;
+ string Data;
+ In.Limit(-1);
+ do
+ {
+ if (In.WriteTillEl(Data,true) == true)
+ break;
+ } while ((Last = Go(false, Req)) == ResultState::SUCCESSFUL);
+
+ if (Last != ResultState::SUCCESSFUL)
+ return Last;
+
+ // See if we are done
+ unsigned long long Len = strtoull(Data.c_str(),0,16);
+ if (Len == 0)
+ {
+ In.Limit(-1);
+
+ // We have to remove the entity trailer
+ Last = ResultState::SUCCESSFUL;
+ do
+ {
+ if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
+ break;
+ } while ((Last = Go(false, Req)) == ResultState::SUCCESSFUL);
+ return Last;
+ }
+
+ // Transfer the block
+ In.Limit(Len);
+ while (Go(true, Req) == ResultState::SUCCESSFUL)
+ if (In.IsLimit() == true)
+ break;
+
+ // Error
+ if (In.IsLimit() == false)
+ return ResultState::TRANSIENT_ERROR;
+
+ // The server sends an extra new line before the next block specifier..
+ In.Limit(-1);
+ Last = ResultState::SUCCESSFUL;
+ do
+ {
+ if (In.WriteTillEl(Data,true) == true)
+ break;
+ } while ((Last = Go(false, Req)) == ResultState::SUCCESSFUL);
+ if (Last != ResultState::SUCCESSFUL)
+ return Last;
+ }
+ }
+ else
+ {
+ /* Closes encoding is used when the server did not specify a size, the
+ loss of the connection means we are done */
+ if (Req.JunkSize != 0)
+ In.Limit(Req.JunkSize);
+ else if (Req.DownloadSize != 0)
+ {
+ if (Req.MaximumSize != 0 && Req.DownloadSize > Req.MaximumSize)
+ {
+ Owner->SetFailReason("MaximumSizeExceeded");
+ _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
+ Req.DownloadSize, Req.MaximumSize);
+ return ResultState::FATAL_ERROR;
+ }
+ In.Limit(Req.DownloadSize);
+ }
+ else if (Persistent == false)
+ In.Limit(-1);
+
+ // Just transfer the whole block.
+ while (true)
+ {
+ if (In.IsLimit() == false)
+ {
+ auto const result = Go(true, Req);
+ if (result == ResultState::SUCCESSFUL)
+ continue;
+ return result;
+ }
+
+ In.Limit(-1);
+ return _error->PendingError() ? ResultState::FATAL_ERROR : ResultState::SUCCESSFUL;
+ }
+ }
+
+ if (Flush(&Req.File) == false)
+ return ResultState::TRANSIENT_ERROR;
+ return ResultState::SUCCESSFUL;
+}
+ /*}}}*/
+ResultState HttpServerState::RunDataToDevNull(RequestState &Req) /*{{{*/
+{
+ // no need to clean up if we discard the connection anyhow
+ if (Persistent == false)
+ return ResultState::SUCCESSFUL;
+ Req.File.Open("/dev/null", FileFd::WriteOnly);
+ return RunData(Req);
+}
+ /*}}}*/
+bool HttpServerState::ReadHeaderLines(std::string &Data) /*{{{*/
+{
+ return In.WriteTillEl(Data);
+}
+ /*}}}*/
+ResultState HttpServerState::LoadNextResponse(bool const ToFile, RequestState &Req) /*{{{*/
+{
+ return Go(ToFile, Req);
+}
+ /*}}}*/
+bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/
+{
+ return Out.Read(Data);
+}
+ /*}}}*/
+APT_PURE bool HttpServerState::IsOpen() /*{{{*/
+{
+ return (ServerFd->Fd() != -1);
+}
+ /*}}}*/
+bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/
+{
+ delete In.Hash;
+ In.Hash = new Hashes(ExpectedHashes);
+ return true;
+}
+ /*}}}*/
+void HttpServerState::Reset() /*{{{*/
+{
+ ServerState::Reset();
+ ServerFd->Close();
+}
+ /*}}}*/
+
+APT_PURE Hashes * HttpServerState::GetHashes() /*{{{*/
+{
+ return In.Hash;
+}
+ /*}}}*/
+// HttpServerState::Die - The server has closed the connection. /*{{{*/
+ResultState HttpServerState::Die(RequestState &Req)
+{
+ unsigned int LErrno = errno;
+
+ Close();
+
+ switch (Req.State)
+ {
+ case RequestState::Data:
+ // We have read all data we could, or the connection is not persistent
+ if (In.IsLimit() == true || Persistent == false)
+ return ResultState::SUCCESSFUL;
+ break;
+ case RequestState::Header:
+ In.Limit(-1);
+ // We have read some headers, but we might also have read the content
+ // and an EOF and hence reached this point. This is fine.
+ if (In.WriteSpace())
+ return ResultState::SUCCESSFUL;
+ break;
+ }
+
+ // We have reached an actual error, tell the user about it.
+ if (LErrno == 0)
+ {
+ _error->Error(_("Error reading from server. Remote end closed connection"));
+ return ResultState::TRANSIENT_ERROR;
+ }
+ errno = LErrno;
+ _error->Errno("read", _("Error reading from server"));
+
+ return ResultState::TRANSIENT_ERROR;
+}
+ /*}}}*/
+// HttpServerState::Flush - Dump the buffer into the file /*{{{*/
+// ---------------------------------------------------------------------
+/* This takes the current input buffer from the Server FD and writes it
+ into the file */
+bool HttpServerState::Flush(FileFd *const File, bool MustComplete)
+{
+ if (File != nullptr)
+ {
+ if (In.WriteSpace() == false)
+ return true;
+
+ while (In.WriteSpace() == true)
+ {
+ if (In.Write(MethodFd::FromFd(File->Fd())) == false)
+ return _error->Errno("write",_("Error writing to file"));
+ if (In.IsLimit() == true)
+ return true;
+ }
+
+ if (In.IsLimit() == true || Persistent == false || not MustComplete)
+ return true;
+ }
+ return false;
+}
+ /*}}}*/
+// HttpServerState::Go - Run a single loop /*{{{*/
+// ---------------------------------------------------------------------
+/* This runs the select loop over the server FDs, Output file FDs and
+ stdin. */
+ResultState HttpServerState::Go(bool ToFile, RequestState &Req)
+{
+ // Server has closed the connection
+ if (ServerFd->Fd() == -1 && (In.WriteSpace() == false ||
+ ToFile == false))
+ return ResultState::TRANSIENT_ERROR;
+
+ // Record if we have data pending to read in the server, so that we can
+ // skip the wait in select(). This can happen if data has already been
+ // read into a methodfd's buffer - the TCP queue might be empty at that
+ // point.
+ bool ServerPending = ServerFd->HasPending();
+
+ fd_set rfds,wfds;
+ FD_ZERO(&rfds);
+ FD_ZERO(&wfds);
+
+ /* Add the server. We only send more requests if the connection will
+ be persisting */
+ if (Out.WriteSpace() == true && ServerFd->Fd() != -1 && Persistent == true)
+ FD_SET(ServerFd->Fd(), &wfds);
+ if (In.ReadSpace() == true && ServerFd->Fd() != -1)
+ FD_SET(ServerFd->Fd(), &rfds);
+
+ // Add the file. Note that we need to add the file to the select and
+ // then write before we read from the server so we do not have content
+ // left to write if the server closes the connection when we read from it.
+ //
+ // An alternative would be to just flush the file in those circumstances
+ // and then return. Because otherwise we might end up blocking indefinitely
+ // in the select() call if we were to continue but all that was left to do
+ // was write to the local file.
+ if (In.WriteSpace() == true && ToFile == true && Req.File.IsOpen())
+ FD_SET(Req.File.Fd(), &wfds);
+
+ // Add stdin
+ if (Owner->ConfigFindB("DependOnSTDIN", true) == true)
+ FD_SET(STDIN_FILENO,&rfds);
+
+ // Figure out the max fd
+ int MaxFd = Req.File.Fd();
+ if (MaxFd < ServerFd->Fd())
+ MaxFd = ServerFd->Fd();
+
+ // Select
+ struct timeval tv;
+ tv.tv_sec = ServerPending ? 0 : TimeOut;
+ tv.tv_usec = 0;
+ int Res = 0;
+ if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
+ {
+ if (errno == EINTR)
+ return ResultState::SUCCESSFUL;
+ _error->Errno("select", _("Select failed"));
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ if (Res == 0 && not ServerPending)
+ {
+ _error->Error(_("Connection timed out"));
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ // Flush any data before talking to the server, in case the server
+ // closed the connection, we want to be done writing.
+ if (Req.File.IsOpen() && FD_ISSET(Req.File.Fd(), &wfds))
+ {
+ if (not Flush(&Req.File, false))
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ // Handle server IO
+ if (ServerPending || (ServerFd->Fd() != -1 && FD_ISSET(ServerFd->Fd(), &rfds)))
+ {
+ errno = 0;
+ if (In.Read(ServerFd) == false)
+ return Die(Req);
+ }
+
+ // Send data to the file
+ if (In.WriteSpace() == true && ToFile == true && Req.File.IsOpen())
+ {
+ if (not Flush(&Req.File, false))
+ return ResultState::TRANSIENT_ERROR;
+ }
+
+ if (ServerFd->Fd() != -1 && FD_ISSET(ServerFd->Fd(), &wfds))
+ {
+ errno = 0;
+ if (Out.Write(ServerFd) == false)
+ return Die(Req);
+ }
+
+ if (Req.MaximumSize > 0 && Req.File.IsOpen() && Req.File.Failed() == false && Req.File.Tell() > Req.MaximumSize)
+ {
+ Owner->SetFailReason("MaximumSizeExceeded");
+ _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
+ Req.File.Tell(), Req.MaximumSize);
+ return ResultState::FATAL_ERROR;
+ }
+
+ // Handle commands from APT
+ if (FD_ISSET(STDIN_FILENO,&rfds))
+ {
+ if (Owner->Run(true) != -1)
+ exit(100);
+ }
+
+ return ResultState::SUCCESSFUL;
+}
+ /*}}}*/
+
+// HttpMethod::SendReq - Send the HTTP request /*{{{*/
+// ---------------------------------------------------------------------
+/* This places the http request in the outbound buffer */
+void HttpMethod::SendReq(FetchItem *Itm)
+{
+ URI Uri(Itm->Uri);
+ {
+ auto const plus = Binary.find('+');
+ if (plus != std::string::npos)
+ Uri.Access = Binary.substr(plus + 1);
+ }
+
+ // The HTTP server expects a hostname with a trailing :port
+ std::stringstream Req;
+ string ProperHost;
+
+ if (Uri.Host.find(':') != string::npos)
+ ProperHost = '[' + Uri.Host + ']';
+ else
+ ProperHost = Uri.Host;
+
+ /* RFC 2616 §5.1.2 requires absolute URIs for requests to proxies,
+ but while its a must for all servers to accept absolute URIs,
+ it is assumed clients will sent an absolute path for non-proxies */
+ std::string requesturi;
+ if ((Server->Proxy.Access != "http" && Server->Proxy.Access != "https") || APT::String::Endswith(Uri.Access, "https") || Server->Proxy.empty() == true || Server->Proxy.Host.empty())
+ requesturi = Uri.Path;
+ else
+ requesturi = Uri;
+
+ if (not _config->FindB("Acquire::Send-URI-Encoded", false))
+ requesturi = URIEncode(requesturi);
+
+ /* Build the request. No keep-alive is included as it is the default
+ in 1.1, can cause problems with proxies, and we are an HTTP/1.1
+ client anyway.
+ C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
+ Req << "GET " << requesturi << " HTTP/1.1\r\n";
+ if (Uri.Port != 0)
+ Req << "Host: " << ProperHost << ":" << std::to_string(Uri.Port) << "\r\n";
+ else
+ Req << "Host: " << ProperHost << "\r\n";
+
+ // generate a cache control header (if needed)
+ if (ConfigFindB("No-Cache",false) == true)
+ Req << "Cache-Control: no-cache\r\n"
+ << "Pragma: no-cache\r\n";
+ else if (Itm->IndexFile == true)
+ Req << "Cache-Control: max-age=" << std::to_string(ConfigFindI("Max-Age", 0)) << "\r\n";
+ else if (ConfigFindB("No-Store", false) == true)
+ Req << "Cache-Control: no-store\r\n";
+
+ // If we ask for uncompressed files servers might respond with content-
+ // negotiation which lets us end up with compressed files we do not support,
+ // see 657029, 657560 and co, so if we have no extension on the request
+ // ask for text only. As a sidenote: If there is nothing to negotate servers
+ // seem to be nice and ignore it.
+ if (ConfigFindB("SendAccept", true) == true)
+ {
+ size_t const filepos = Itm->Uri.find_last_of('/');
+ string const file = Itm->Uri.substr(filepos + 1);
+ if (flExtension(file) == file)
+ Req << "Accept: text/*\r\n";
+ }
+
+ // Check for a partial file and send if-queries accordingly
+ struct stat SBuf;
+ if (Server->RangesAllowed && stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
+ Req << "Range: bytes=" << std::to_string(SBuf.st_size) << "-\r\n"
+ << "If-Range: " << TimeRFC1123(SBuf.st_mtime, false) << "\r\n";
+ else if (Itm->LastModified != 0)
+ Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified, false).c_str() << "\r\n";
+
+ if ((Server->Proxy.Access == "http" || Server->Proxy.Access == "https") &&
+ (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false))
+ Req << "Proxy-Authorization: Basic "
+ << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n";
+
+ MaybeAddAuthTo(Uri);
+ if (Uri.User.empty() == false || Uri.Password.empty() == false)
+ Req << "Authorization: Basic "
+ << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n";
+
+ Req << "User-Agent: " << ConfigFind("User-Agent",
+ "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")");
+
+#ifdef HAVE_SYSTEMD
+ if (ConfigFindB("User-Agent-Non-Interactive", false))
+ {
+ using APT::operator""_sv;
+ char *unit = nullptr;
+ sd_pid_get_unit(getpid(), &unit);
+ if (unit != nullptr && *unit != '\0' && not APT::String::Startswith(unit, "user@") // user@ _is_ interactive
+ && "packagekit.service"_sv != unit // packagekit likely is interactive
+ && "dbus.service"_sv != unit) // aptdaemon and qapt don't have systemd services
+ Req << " non-interactive";
+
+ free(unit);
+ }
+#endif
+
+ Req << "\r\n";
+
+ // the famously typoed HTTP header field
+ auto const referrer = ConfigFind("Referer", "");
+ if (referrer.empty() == false)
+ Req << "Referer: " << referrer << "\r\n";
+
+ Req << "\r\n";
+
+ if (Debug == true)
+ cerr << Req.str() << endl;
+
+ Server->WriteResponse(Req.str());
+}
+ /*}}}*/
+std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/
+{
+ return std::unique_ptr<ServerState>(new HttpServerState(uri, this));
+}
+ /*}}}*/
+void HttpMethod::RotateDNS() /*{{{*/
+{
+ ::RotateDNS();
+}
+ /*}}}*/
+BaseHttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res, RequestState &Req)/*{{{*/
+{
+ auto ret = BaseHttpMethod::DealWithHeaders(Res, Req);
+ if (ret != BaseHttpMethod::FILE_IS_OPEN)
+ return ret;
+ if (Req.File.Open(Queue->DestFile, FileFd::WriteAny) == false)
+ return ERROR_NOT_FROM_SERVER;
+
+ FailFile = Queue->DestFile;
+ (void)(FailFile.c_str()); // Make sure we don't do a malloc in the signal handler
+ FailFd = Req.File.Fd();
+ FailTime = Req.Date;
+
+ if (Server->InitHashes(Queue->ExpectedHashes) == false || Req.AddPartialFileToHashes(Req.File) == false)
+ {
+ _error->Errno("read",_("Problem hashing file"));
+ return ERROR_NOT_FROM_SERVER;
+ }
+ if (Req.StartPos > 0)
+ Res.ResumePoint = Req.StartPos;
+
+ return FILE_IS_OPEN;
+}
+ /*}}}*/
+HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(std::move(pProg), "1.2", Pipeline | SendConfig | SendURIEncoded) /*{{{*/
+{
+ SeccompFlags = aptMethod::BASE | aptMethod::NETWORK;
+
+ auto addName = std::inserter(methodNames, methodNames.begin());
+ if (Binary != "http")
+ addName = "http";
+ auto const plus = Binary.find('+');
+ if (plus != std::string::npos)
+ {
+ auto name2 = Binary.substr(plus + 1);
+ if (std::find(methodNames.begin(), methodNames.end(), name2) == methodNames.end())
+ addName = std::move(name2);
+ addName = Binary.substr(0, plus);
+ }
+}
+ /*}}}*/
+
+int main(int, const char *argv[])
+{
+ // ignore SIGPIPE, this can happen on write() if the socket
+ // closes the connection (this is dealt with via ServerDie())
+ signal(SIGPIPE, SIG_IGN);
+ std::string Binary = flNotDir(argv[0]);
+ if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http")
+ Binary.append("+http");
+ return HttpMethod(std::move(Binary)).Loop();
+}
diff --git a/methods/http.h b/methods/http.h
new file mode 100644
index 0000000..cae579a
--- /dev/null
+++ b/methods/http.h
@@ -0,0 +1,142 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ HTTP Acquire Method - This is the HTTP acquire method for APT.
+
+ ##################################################################### */
+ /*}}}*/
+
+#ifndef APT_HTTP_H
+#define APT_HTTP_H
+
+#include <apt-pkg/strutl.h>
+
+#include <chrono>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <sys/time.h>
+
+#include "basehttp.h"
+#include "connect.h"
+
+using std::cout;
+using std::endl;
+
+class FileFd;
+class HttpMethod;
+class Hashes;
+
+class CircleBuf
+{
+ unsigned char *Buf;
+ unsigned long long Size;
+ unsigned long long InP;
+ unsigned long long OutP;
+ std::string OutQueue;
+ unsigned long long StrPos;
+ unsigned long long MaxGet;
+
+ static unsigned long long BwReadLimit;
+ static unsigned long long BwTickReadData;
+ static std::chrono::steady_clock::duration BwReadTick;
+ static const unsigned int BW_HZ;
+
+ unsigned long long LeftRead() const
+ {
+ unsigned long long Sz = Size - (InP - OutP);
+ if (Sz > Size - (InP%Size))
+ Sz = Size - (InP%Size);
+ return Sz;
+ }
+ unsigned long long LeftWrite() const
+ {
+ unsigned long long Sz = InP - OutP;
+ if (InP > MaxGet)
+ Sz = MaxGet - OutP;
+ if (Sz > Size - (OutP%Size))
+ Sz = Size - (OutP%Size);
+ return Sz;
+ }
+ void FillOut();
+
+ public:
+ Hashes *Hash;
+ // total amount of data that got written so far
+ unsigned long long TotalWriten;
+
+ // Read data in
+ bool Read(std::unique_ptr<MethodFd> const &Fd);
+ bool Read(std::string const &Data);
+
+ // Write data out
+ bool Write(std::unique_ptr<MethodFd> const &Fd);
+ bool Write(std::string &Data);
+ bool WriteTillEl(std::string &Data,bool Single = false);
+
+ // Control the write limit
+ void Limit(long long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;}
+ bool IsLimit() const {return MaxGet == OutP;};
+ void Print() const {cout << MaxGet << ',' << OutP << endl;};
+
+ // Test for free space in the buffer
+ bool ReadSpace() const {return Size - (InP - OutP) > 0;};
+ bool WriteSpace() const {return InP - OutP > 0;};
+
+ void Reset();
+
+ CircleBuf(HttpMethod const * const Owner, unsigned long long Size);
+ ~CircleBuf();
+};
+
+struct HttpServerState: public ServerState
+{
+ // This is the connection itself. Output is data FROM the server
+ CircleBuf In;
+ CircleBuf Out;
+ std::unique_ptr<MethodFd> ServerFd;
+
+ protected:
+ virtual bool ReadHeaderLines(std::string &Data) APT_OVERRIDE;
+ virtual ResultState LoadNextResponse(bool const ToFile, RequestState &Req) APT_OVERRIDE;
+ virtual bool WriteResponse(std::string const &Data) APT_OVERRIDE;
+
+ public:
+ virtual void Reset() APT_OVERRIDE;
+
+ virtual ResultState RunData(RequestState &Req) APT_OVERRIDE;
+ virtual ResultState RunDataToDevNull(RequestState &Req) APT_OVERRIDE;
+
+ virtual ResultState Open() APT_OVERRIDE;
+ virtual bool IsOpen() APT_OVERRIDE;
+ virtual bool Close() APT_OVERRIDE;
+ virtual bool InitHashes(HashStringList const &ExpectedHashes) APT_OVERRIDE;
+ virtual Hashes * GetHashes() APT_OVERRIDE;
+ virtual ResultState Die(RequestState &Req) APT_OVERRIDE;
+ virtual bool Flush(FileFd *const File, bool MustComplete = true) APT_OVERRIDE;
+ virtual ResultState Go(bool ToFile, RequestState &Req) APT_OVERRIDE;
+
+ HttpServerState(URI Srv, HttpMethod *Owner);
+ virtual ~HttpServerState() {Close();};
+};
+
+class HttpMethod : public BaseHttpMethod
+{
+ public:
+ virtual void SendReq(FetchItem *Itm) APT_OVERRIDE;
+
+ virtual std::unique_ptr<ServerState> CreateServerState(URI const &uri) APT_OVERRIDE;
+ virtual void RotateDNS() APT_OVERRIDE;
+ virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res, RequestState &Req) APT_OVERRIDE;
+
+ protected:
+ std::string AutoDetectProxyCmd;
+
+ public:
+ friend struct HttpServerState;
+
+ explicit HttpMethod(std::string &&pProg);
+};
+
+#endif
diff --git a/methods/mirror.cc b/methods/mirror.cc
new file mode 100644
index 0000000..787e4c7
--- /dev/null
+++ b/methods/mirror.cc
@@ -0,0 +1,414 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ Mirror URI – This method helps avoiding hardcoding of mirrors in the
+ sources.lists by looking up a list of mirrors first to which the
+ following requests are redirected.
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include "aptmethod.h"
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/metaindex.h>
+#include <apt-pkg/sourcelist.h>
+#include <apt-pkg/strutl.h>
+
+#include <functional>
+#include <random>
+#include <string>
+#include <unordered_map>
+
+#include <sys/utsname.h>
+
+#include <apti18n.h>
+ /*}}}*/
+constexpr char const *const disallowLocal[] = {"ftp", "http", "https"};
+
+static void sortByLength(std::vector<std::string> &vec) /*{{{*/
+{
+ // this ensures having mirror://foo/ and mirror://foo/bar/ works as expected
+ // by checking for the longest matches first
+ std::sort(vec.begin(), vec.end(), [](std::string const &a, std::string const &b) {
+ return a.length() > b.length();
+ });
+}
+ /*}}}*/
+class MirrorMethod : public aptMethod /*{{{*/
+{
+ std::mt19937 genrng;
+ std::vector<std::string> sourceslist;
+ std::unordered_map<std::string, std::string> msgCache;
+ enum MirrorFileState
+ {
+ REQUESTED,
+ FAILED,
+ AVAILABLE
+ };
+ struct MirrorInfo
+ {
+ std::string uri;
+ unsigned long priority = std::numeric_limits<decltype(priority)>::max();
+ decltype(genrng)::result_type seed = 0;
+ std::unordered_map<std::string, std::vector<std::string>> tags;
+ explicit MirrorInfo(std::string const &u, std::vector<std::string> &&ptags = {}) : uri(u)
+ {
+ for (auto &&tag : ptags)
+ {
+ auto const colonfound = tag.find(':');
+ if (unlikely(colonfound == std::string::npos))
+ continue;
+ auto name = tag.substr(0, colonfound);
+ auto value = tag.substr(colonfound + 1);
+ if (name == "arch")
+ tags["Architecture"].emplace_back(std::move(value));
+ else if (name == "lang")
+ tags["Language"].emplace_back(std::move(value));
+ else if (name == "priority")
+ priority = std::strtoul(value.c_str(), nullptr, 10);
+ else if (likely(name.empty() == false))
+ {
+ if (name == "codename" || name == "suite")
+ tags["Release"].push_back(value);
+ name[0] = std::toupper(name[0]);
+ tags[std::move(name)].emplace_back(std::move(value));
+ }
+ }
+ }
+ };
+ struct MirrorListInfo
+ {
+ MirrorFileState state;
+ std::string baseuri;
+ std::vector<MirrorInfo> list;
+ };
+ std::unordered_map<std::string, MirrorListInfo> mirrorfilestate;
+
+ virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE;
+
+ void RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message);
+ bool MirrorListFileReceived(MirrorListInfo &info, FetchItem *const Itm);
+ std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm);
+ void DealWithPendingItems(std::vector<std::string> const &baseuris, MirrorListInfo const &info, FetchItem *const Itm, std::function<void()> handler);
+
+ public:
+ explicit MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests | SendURIEncoded), genrng(clock())
+ {
+ SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY;
+ }
+};
+ /*}}}*/
+void MirrorMethod::RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message) /*{{{*/
+{
+ std::unordered_map<std::string, std::string> matchers;
+ matchers.emplace("Architecture", LookupTag(Message, "Target-Architecture"));
+ matchers.emplace("Codename", LookupTag(Message, "Target-Codename"));
+ matchers.emplace("Component", LookupTag(Message, "Target-Component"));
+ matchers.emplace("Language", LookupTag(Message, "Target-Language"));
+ matchers.emplace("Release", LookupTag(Message, "Target-Release"));
+ matchers.emplace("Suite", LookupTag(Message, "Target-Suite"));
+ matchers.emplace("Type", LookupTag(Message, "Target-Type"));
+ decltype(info.list) possMirrors;
+ for (auto const &mirror : info.list)
+ {
+ bool failedMatch = false;
+ for (auto const &m : matchers)
+ {
+ if (m.second.empty())
+ continue;
+ auto const tagsetiter = mirror.tags.find(m.first);
+ if (tagsetiter == mirror.tags.end())
+ continue;
+ auto const tagset = tagsetiter->second;
+ if (tagset.empty() == false && std::find(tagset.begin(), tagset.end(), m.second) == tagset.end())
+ {
+ failedMatch = true;
+ break;
+ }
+ }
+ if (failedMatch)
+ continue;
+ possMirrors.push_back(mirror);
+ }
+ for (auto &&mirror : possMirrors)
+ mirror.seed = genrng();
+ std::sort(possMirrors.begin(), possMirrors.end(), [](MirrorInfo const &a, MirrorInfo const &b) {
+ if (a.priority != b.priority)
+ return a.priority < b.priority;
+ return a.seed < b.seed;
+ });
+ std::string const path = Itm->Uri.substr(info.baseuri.length());
+ std::string altMirrors;
+ std::unordered_map<std::string, std::string> fields;
+ fields.emplace("URI", Itm->Uri);
+ for (auto curMirror = possMirrors.cbegin(); curMirror != possMirrors.cend(); ++curMirror)
+ {
+ std::string mirror = curMirror->uri;
+ if (APT::String::Endswith(mirror, "/") == false)
+ mirror.append("/");
+ mirror.append(path);
+ if (curMirror == possMirrors.cbegin())
+ fields.emplace("New-URI", mirror);
+ else if (altMirrors.empty())
+ altMirrors.append(mirror);
+ else
+ altMirrors.append("\n").append(mirror);
+ }
+ fields.emplace("Alternate-URIs", altMirrors);
+ SendMessage("103 Redirect", std::move(fields));
+
+ // Remove Itm from the queue, then delete
+ if (Queue == Itm)
+ Queue = Itm->Next;
+ else
+ {
+ FetchItem *previous = Queue;
+ while (previous->Next != Itm)
+ previous = previous->Next;
+
+ previous->Next = Itm->Next;
+ }
+ delete Itm;
+}
+ /*}}}*/
+void MirrorMethod::DealWithPendingItems(std::vector<std::string> const &baseuris, /*{{{*/
+ MirrorListInfo const &info, FetchItem *const Itm,
+ std::function<void()> handler)
+{
+ FetchItem **LastItm = &Itm->Next;
+ while (*LastItm != nullptr)
+ LastItm = &((*LastItm)->Next);
+ while (Queue != Itm)
+ {
+ if (APT::String::Startswith(Queue->Uri, info.baseuri) == false ||
+ std::any_of(baseuris.cbegin(), baseuris.cend(), [&](std::string const &b) { return APT::String::Startswith(Queue->Uri, b); }))
+ {
+ // move the item behind the aux file not related to it
+ *LastItm = Queue;
+ Queue = QueueBack = Queue->Next;
+ (*LastItm)->Next = nullptr;
+ LastItm = &((*LastItm)->Next);
+ }
+ else
+ {
+ handler();
+ }
+ }
+ // now remove out trigger
+ QueueBack = Queue = Queue->Next;
+ delete Itm;
+}
+ /*}}}*/
+bool MirrorMethod::MirrorListFileReceived(MirrorListInfo &info, FetchItem *const Itm) /*{{{*/
+{
+ std::vector<std::string> baseuris;
+ for (auto const &i : mirrorfilestate)
+ if (info.baseuri.length() < i.second.baseuri.length() &&
+ i.second.state == REQUESTED &&
+ APT::String::Startswith(i.second.baseuri, info.baseuri))
+ baseuris.push_back(i.second.baseuri);
+ sortByLength(baseuris);
+
+ FileFd mirrorlist;
+ if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension))
+ {
+ auto const accessColon = info.baseuri.find(':');
+ auto access = info.baseuri.substr(0, accessColon);
+ std::string prefixAccess;
+ if (APT::String::Startswith(access, "mirror") == false)
+ {
+ auto const plus = info.baseuri.find('+');
+ prefixAccess = info.baseuri.substr(0, plus);
+ access.erase(0, plus + 1);
+ }
+ std::vector<std::string> limitAccess;
+ // If the mirror file comes from an online source, allow only other online
+ // sources, not e.g. file:///. If the mirrorlist comes from there we can assume
+ // the admin knows what (s)he is doing through and not limit the options.
+ if (std::any_of(std::begin(disallowLocal), std::end(disallowLocal),
+ [&access](char const *const a) { return APT::String::Endswith(access, std::string("+") + a); }) ||
+ access == "mirror")
+ {
+ std::copy(std::begin(disallowLocal), std::end(disallowLocal), std::back_inserter(limitAccess));
+ }
+ std::string line;
+ while (mirrorlist.ReadLine(line))
+ {
+ if (line.empty() || line[0] == '#')
+ continue;
+ auto const access = line.substr(0, line.find(':'));
+ if (limitAccess.empty() == false && std::find(limitAccess.begin(), limitAccess.end(), access) == limitAccess.end())
+ continue;
+ auto const tab = line.find('\t');
+ if (tab == std::string::npos)
+ {
+ if (prefixAccess.empty())
+ info.list.emplace_back(std::move(line));
+ else
+ info.list.emplace_back(prefixAccess + '+' + line);
+ }
+ else
+ {
+ auto uri = line.substr(0, tab);
+ if (prefixAccess.empty() == false)
+ uri = prefixAccess + '+' + uri;
+ auto tagline = line.substr(tab + 1);
+ std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' ');
+ auto tags = VectorizeString(tagline, ' ');
+ tags.erase(std::remove_if(tags.begin(), tags.end(), [](std::string const &a) { return a.empty(); }), tags.end());
+ info.list.emplace_back(std::move(uri), std::move(tags));
+ }
+ }
+ mirrorlist.Close();
+
+ if (info.list.empty())
+ {
+ info.state = FAILED;
+ DealWithPendingItems(baseuris, info, Itm, [&]() {
+ std::string msg;
+ strprintf(msg, "Mirror list %s is empty for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
+ Fail(msg, false);
+ });
+ }
+ else
+ {
+ info.state = AVAILABLE;
+ DealWithPendingItems(baseuris, info, Itm, [&]() {
+ RedirectItem(info, Queue, msgCache[Queue->Uri]);
+ });
+ msgCache.clear();
+ }
+ }
+ else
+ {
+ info.state = FAILED;
+ DealWithPendingItems(baseuris, info, Itm, [&]() {
+ std::string msg;
+ strprintf(msg, "Downloading mirror file %s failed for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
+ Fail(msg, false);
+ });
+ }
+ return true;
+}
+ /*}}}*/
+std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem *const Itm) /*{{{*/
+{
+ if (APT::String::Startswith(Itm->Uri, Binary))
+ {
+ std::string const repouri = LookupTag(Message, "Target-Repo-Uri");
+ if (repouri.empty() == false && std::find(sourceslist.cbegin(), sourceslist.cend(), repouri) == sourceslist.cend())
+ sourceslist.push_back(repouri);
+ }
+ if (sourceslist.empty())
+ {
+ // read sources.list and find the matching base uri
+ pkgSourceList sl;
+ if (sl.ReadMainList() == false)
+ {
+ _error->Error(_("The list of sources could not be read."));
+ return "";
+ }
+ std::string const needle = Binary + ":";
+ for (auto const &SL : sl)
+ {
+ std::string uristr = SL->GetURI();
+ if (APT::String::Startswith(uristr, needle))
+ sourceslist.push_back(uristr);
+ }
+ sortByLength(sourceslist);
+ }
+ for (auto uristr : sourceslist)
+ {
+ if (APT::String::Startswith(Itm->Uri, uristr))
+ {
+ uristr.erase(uristr.length() - 1); // remove the ending '/'
+ auto const colon = uristr.find(':');
+ if (unlikely(colon == std::string::npos))
+ continue;
+ auto const plus = uristr.find("+");
+ if (plus < colon)
+ {
+ // started as tor+mirror+http we want to get the file via tor+http
+ auto const access = uristr.substr(0, colon);
+ if (APT::String::Startswith(access, "mirror") == false)
+ {
+ uristr.erase(plus, strlen("mirror") + 1);
+ return uristr;
+ }
+ else
+ return uristr.substr(plus + 1);
+ }
+ else
+ {
+ uristr.replace(0, strlen("mirror"), "http");
+ return uristr;
+ }
+ }
+ }
+ return "";
+}
+ /*}}}*/
+bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*/
+{
+ auto mirrorinfo = mirrorfilestate.find(Itm->Uri);
+ if (mirrorinfo != mirrorfilestate.end())
+ return MirrorListFileReceived(mirrorinfo->second, Itm);
+
+ std::string const mirrorfileuri = GetMirrorFileURI(Message, Itm);
+ if (mirrorfileuri.empty())
+ {
+ _error->Error("Couldn't determine mirror list to query for %s", Itm->Uri.c_str());
+ return false;
+ }
+ if (DebugEnabled())
+ std::clog << "Mirror-URI: " << mirrorfileuri << " for " << Itm->Uri << std::endl;
+
+ // have we requested this mirror file already?
+ auto const state = mirrorfilestate.find(mirrorfileuri);
+ if (state == mirrorfilestate.end())
+ {
+ msgCache[Itm->Uri] = Message;
+ MirrorListInfo info;
+ info.state = REQUESTED;
+ info.baseuri = mirrorfileuri + '/';
+ auto const colon = info.baseuri.find(':');
+ if (unlikely(colon == std::string::npos))
+ return false;
+ info.baseuri.replace(0, colon, Binary);
+ mirrorfilestate[mirrorfileuri] = info;
+ std::unordered_map<std::string, std::string> fields;
+ fields.emplace("URI", Itm->Uri);
+ fields.emplace("MaximumSize", std::to_string(1 * 1024 * 1024)); //FIXME: 1 MB is enough for everyone
+ fields.emplace("Aux-ShortDesc", "Mirrorlist");
+ fields.emplace("Aux-Description", mirrorfileuri + " Mirrorlist");
+ fields.emplace("Aux-Uri", mirrorfileuri);
+ SendMessage("351 Aux Request", std::move(fields));
+ return true;
+ }
+
+ switch (state->second.state)
+ {
+ case REQUESTED:
+ // lets wait for the requested mirror file
+ msgCache[Itm->Uri] = Message;
+ return true;
+ case FAILED:
+ Fail("Downloading mirror file failed", false);
+ return true;
+ case AVAILABLE:
+ RedirectItem(state->second, Itm, Message);
+ return true;
+ }
+ return false;
+}
+ /*}}}*/
+
+int main(int, const char *argv[])
+{
+ return MirrorMethod(flNotDir(argv[0])).Run();
+}
diff --git a/methods/rfc2553emu.cc b/methods/rfc2553emu.cc
new file mode 100644
index 0000000..f5a3fc9
--- /dev/null
+++ b/methods/rfc2553emu.cc
@@ -0,0 +1,244 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ RFC 2553 Emulation - Provides emulation for RFC 2553 getaddrinfo,
+ freeaddrinfo and getnameinfo
+
+ This is really C code, it just has a .cc extensions to play nicer with
+ the rest of APT.
+
+ Originally written by Jason Gunthorpe <jgg@debian.org> and placed into
+ the Public Domain, do with it what you will.
+
+ ##################################################################### */
+ /*}}}*/
+#include <config.h>
+
+#include "rfc2553emu.h"
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+
+#ifndef HAVE_GETADDRINFO
+// getaddrinfo - Resolve a hostname /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+int getaddrinfo(const char *nodename, const char *servname,
+ const struct addrinfo *hints,
+ struct addrinfo **res)
+{
+ struct addrinfo **Result = res;
+ hostent *Addr;
+ unsigned int Port;
+ int Proto;
+ const char *End;
+ char **CurAddr;
+
+ // Try to convert the service as a number
+ Port = htons(strtol(servname,(char **)&End,0));
+ Proto = SOCK_STREAM;
+
+ if (hints != 0 && hints->ai_socktype != 0)
+ Proto = hints->ai_socktype;
+
+ // Not a number, must be a name.
+ if (End != servname + strlen(servname))
+ {
+ struct servent *Srv = 0;
+
+ // Do a lookup in the service database
+ if (hints == 0 || hints->ai_socktype == SOCK_STREAM)
+ Srv = getservbyname(servname,"tcp");
+ if (hints != 0 && hints->ai_socktype == SOCK_DGRAM)
+ Srv = getservbyname(servname,"udp");
+ if (Srv == 0)
+ return EAI_NONAME;
+
+ // Get the right protocol
+ Port = Srv->s_port;
+ if (strcmp(Srv->s_proto,"tcp") == 0)
+ Proto = SOCK_STREAM;
+ else
+ {
+ if (strcmp(Srv->s_proto,"udp") == 0)
+ Proto = SOCK_DGRAM;
+ else
+ return EAI_NONAME;
+ }
+
+ if (hints != 0 && hints->ai_socktype != Proto &&
+ hints->ai_socktype != 0)
+ return EAI_SERVICE;
+ }
+
+ // Hostname lookup, only if this is not a listening socket
+ if (hints != 0 && (hints->ai_flags & AI_PASSIVE) != AI_PASSIVE)
+ {
+ Addr = gethostbyname(nodename);
+ if (Addr == 0)
+ {
+ if (h_errno == TRY_AGAIN)
+ return EAI_AGAIN;
+ if (h_errno == NO_RECOVERY)
+ return EAI_FAIL;
+ return EAI_NONAME;
+ }
+
+ // No A records
+ if (Addr->h_addr_list[0] == 0)
+ return EAI_NONAME;
+
+ CurAddr = Addr->h_addr_list;
+ }
+ else
+ CurAddr = (char **)&End; // Fake!
+
+ // Start constructing the linked list
+ *res = 0;
+ for (; *CurAddr != 0; CurAddr++)
+ {
+ // New result structure
+ *Result = (struct addrinfo *)calloc(sizeof(**Result),1);
+ if (*Result == 0)
+ {
+ freeaddrinfo(*res);
+ return EAI_MEMORY;
+ }
+ if (*res == 0)
+ *res = *Result;
+
+ (*Result)->ai_family = AF_INET;
+ (*Result)->ai_socktype = Proto;
+
+ // If we have the IPPROTO defines we can set the protocol field
+ #ifdef IPPROTO_TCP
+ if (Proto == SOCK_STREAM)
+ (*Result)->ai_protocol = IPPROTO_TCP;
+ if (Proto == SOCK_DGRAM)
+ (*Result)->ai_protocol = IPPROTO_UDP;
+ #endif
+
+ // Allocate space for the address
+ (*Result)->ai_addrlen = sizeof(struct sockaddr_in);
+ (*Result)->ai_addr = (struct sockaddr *)calloc(sizeof(sockaddr_in),1);
+ if ((*Result)->ai_addr == 0)
+ {
+ freeaddrinfo(*res);
+ return EAI_MEMORY;
+ }
+
+ // Set the address
+ ((struct sockaddr_in *)(*Result)->ai_addr)->sin_family = AF_INET;
+ ((struct sockaddr_in *)(*Result)->ai_addr)->sin_port = Port;
+
+ if (hints != 0 && (hints->ai_flags & AI_PASSIVE) != AI_PASSIVE)
+ ((struct sockaddr_in *)(*Result)->ai_addr)->sin_addr = *(in_addr *)(*CurAddr);
+ else
+ {
+ // Already zerod by calloc.
+ break;
+ }
+
+ Result = &(*Result)->ai_next;
+ }
+
+ return 0;
+}
+ /*}}}*/
+// freeaddrinfo - Free the result of getaddrinfo /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void freeaddrinfo(struct addrinfo *ai)
+{
+ while (ai != 0)
+ {
+ free(ai->ai_addr);
+ ai = ai->ai_next;
+ free(ai);
+ }
+}
+ /*}}}*/
+#endif // HAVE_GETADDRINFO
+
+#ifndef HAVE_GETNAMEINFO
+// getnameinfo - Convert a sockaddr to a string /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+int getnameinfo(const struct sockaddr *sa, socklen_t salen,
+ char *host, size_t hostlen,
+ char *serv, size_t servlen,
+ int flags)
+{
+ struct sockaddr_in *sin = (struct sockaddr_in *)sa;
+
+ // This routine only supports internet addresses
+ if (sa->sa_family != AF_INET)
+ return EAI_ADDRFAMILY;
+
+ if (host != 0)
+ {
+ // Try to resolve the hostname
+ if ((flags & NI_NUMERICHOST) != NI_NUMERICHOST)
+ {
+ struct hostent *Ent = gethostbyaddr((char *)&sin->sin_addr,sizeof(sin->sin_addr),
+ AF_INET);
+ if (Ent != 0)
+ strncpy(host,Ent->h_name,hostlen);
+ else
+ {
+ if ((flags & NI_NAMEREQD) == NI_NAMEREQD)
+ {
+ if (h_errno == TRY_AGAIN)
+ return EAI_AGAIN;
+ if (h_errno == NO_RECOVERY)
+ return EAI_FAIL;
+ return EAI_NONAME;
+ }
+
+ flags |= NI_NUMERICHOST;
+ }
+ }
+
+ // Resolve as a plain numeric
+ if ((flags & NI_NUMERICHOST) == NI_NUMERICHOST)
+ {
+ strncpy(host,inet_ntoa(sin->sin_addr),hostlen);
+ }
+ }
+
+ if (serv != 0)
+ {
+ // Try to resolve the hostname
+ if ((flags & NI_NUMERICSERV) != NI_NUMERICSERV)
+ {
+ struct servent *Ent;
+ if ((flags & NI_DATAGRAM) == NI_DATAGRAM)
+ Ent = getservbyport(ntohs(sin->sin_port),"udp");
+ else
+ Ent = getservbyport(ntohs(sin->sin_port),"tcp");
+
+ if (Ent != 0)
+ strncpy(serv,Ent->s_name,servlen);
+ else
+ {
+ if ((flags & NI_NAMEREQD) == NI_NAMEREQD)
+ return EAI_NONAME;
+
+ flags |= NI_NUMERICSERV;
+ }
+ }
+
+ // Resolve as a plain numeric
+ if ((flags & NI_NUMERICSERV) == NI_NUMERICSERV)
+ {
+ snprintf(serv,servlen,"%u",ntohs(sin->sin_port));
+ }
+ }
+
+ return 0;
+}
+ /*}}}*/
+#endif // HAVE_GETNAMEINFO
diff --git a/methods/rfc2553emu.h b/methods/rfc2553emu.h
new file mode 100644
index 0000000..65d744d
--- /dev/null
+++ b/methods/rfc2553emu.h
@@ -0,0 +1,112 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ RFC 2553 Emulation - Provides emulation for RFC 2553 getaddrinfo,
+ freeaddrinfo and getnameinfo
+
+ These functions are necessary to write portable protocol independent
+ networking. They transparently support IPv4, IPv6 and probably many
+ other protocols too. This implementation is needed when the host does
+ not support these standards. It implements a simple wrapper that
+ basically supports only IPv4.
+
+ Perfect emulation is not provided, but it is passable..
+
+ Originally written by Jason Gunthorpe <jgg@debian.org> and placed into
+ the Public Domain, do with it what you will.
+
+ ##################################################################### */
+ /*}}}*/
+#ifndef RFC2553EMU_H
+#define RFC2553EMU_H
+
+#include <netdb.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+
+// Autosense getaddrinfo
+#if defined(AI_PASSIVE) && defined(EAI_NONAME)
+#define HAVE_GETADDRINFO
+#endif
+
+// Autosense getnameinfo
+#if defined(NI_NUMERICHOST)
+#define HAVE_GETNAMEINFO
+#endif
+
+// getaddrinfo support?
+#ifndef HAVE_GETADDRINFO
+ // Renamed to advoid type clashing.. (for debugging)
+ struct addrinfo_emu
+ {
+ int ai_flags; /* AI_PASSIVE, AI_CANONNAME, AI_NUMERICHOST */
+ int ai_family; /* PF_xxx */
+ int ai_socktype; /* SOCK_xxx */
+ int ai_protocol; /* 0 or IPPROTO_xxx for IPv4 and IPv6 */
+ size_t ai_addrlen; /* length of ai_addr */
+ char *ai_canonname; /* canonical name for nodename */
+ struct sockaddr *ai_addr; /* binary address */
+ struct addrinfo_emu *ai_next; /* next structure in linked list */
+ };
+ #define addrinfo addrinfo_emu
+
+ int getaddrinfo(const char *nodename, const char *servname,
+ const struct addrinfo *hints,
+ struct addrinfo **res);
+ void freeaddrinfo(struct addrinfo *ai);
+
+ #ifndef AI_PASSIVE
+ #define AI_PASSIVE (1<<1)
+ #endif
+
+ #ifndef EAI_NONAME
+ #define EAI_NONAME -1
+ #define EAI_AGAIN -2
+ #define EAI_FAIL -3
+ #define EAI_NODATA -4
+ #define EAI_FAMILY -5
+ #define EAI_SOCKTYPE -6
+ #define EAI_SERVICE -7
+ #define EAI_ADDRFAMILY -8
+ #define EAI_SYSTEM -10
+ #define EAI_MEMORY -11
+ #endif
+
+ /* If we don't have getaddrinfo then we probably don't have
+ sockaddr_storage either (same RFC) so we definitely will not be
+ doing any IPv6 stuff. Do not use the members of this structure to
+ retain portability, cast to a sockaddr. */
+ #define sockaddr_storage sockaddr_in
+#endif
+
+// getnameinfo support (glibc2.0 has getaddrinfo only)
+#ifndef HAVE_GETNAMEINFO
+
+ int getnameinfo(const struct sockaddr *sa, socklen_t salen,
+ char *host, size_t hostlen,
+ char *serv, size_t servlen,
+ int flags);
+
+ #ifndef NI_MAXHOST
+ #define NI_MAXHOST 1025
+ #define NI_MAXSERV 32
+ #endif
+
+ #ifndef NI_NUMERICHOST
+ #define NI_NUMERICHOST (1<<0)
+ #define NI_NUMERICSERV (1<<1)
+// #define NI_NOFQDN (1<<2)
+ #define NI_NAMEREQD (1<<3)
+ #define NI_DATAGRAM (1<<4)
+ #endif
+
+ #define sockaddr_storage sockaddr_in
+#endif
+
+// Glibc 2.0.7 misses this one
+#ifndef AI_NUMERICHOST
+#define AI_NUMERICHOST 0
+#endif
+
+#endif
diff --git a/methods/rred.cc b/methods/rred.cc
new file mode 100644
index 0000000..aeefea5
--- /dev/null
+++ b/methods/rred.cc
@@ -0,0 +1,885 @@
+// Copyright (c) 2014 Anthony Towns
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+
+#include <config.h>
+
+#ifndef APT_EXCLUDE_RRED_METHOD_CODE
+#include "aptmethod.h"
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/init.h>
+#endif
+
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/hashes.h>
+#include <apt-pkg/strutl.h>
+
+#include <apt-private/private-cmndline.h>
+
+#include <cstddef>
+#include <iostream>
+#include <list>
+#include <string>
+#include <vector>
+
+#include <cassert>
+#include <cerrno>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <sys/stat.h>
+#include <sys/time.h>
+
+#include <apti18n.h>
+
+#ifndef APT_MEMBLOCK_SIZE
+#define APT_MEMBLOCK_SIZE (512*1024)
+#endif
+
+static bool ShowHelp(CommandLine &)
+{
+ std::cout <<
+ "Usage: rred [options] -t input output patch-1 … patch-N\n"
+ " rred [options] -f patch-1 … patch-N < input > output\n"
+ " rred [options] patch-1 … patch-N > merged-patch\n"
+ "\n"
+ "The main use of this binary is by APTs acquire system, a mode reached\n"
+ "by calling it without any arguments and driven via messages on stdin.\n"
+ "\n"
+ "For the propose of testing as well as simpler direct usage the above\n"
+ "mentioned modes to work with \"reversed restricted ed\" patches as well.\n"
+ "\n"
+ "The arguments used above are:\n"
+ "* input: denotes a file you want to patch.\n"
+ "* output: a file you want to store the patched content in.\n"
+ "* patch-1 … patch-N: One or more files containing a patch.\n"
+ "* merged-patch: All changes by patch-1 … patch-N in one patch.\n"
+ "\n"
+ "This rred supports the commands 'a', 'c' and 'd', both single as well\n"
+ "as multi line. Other commands are not supported (hence 'restricted').\n"
+ "The command to patch the last line must appear first in the patch\n"
+ "(hence 'reversed'). Such a patch can e.g. be produced with 'diff --ed'.\n"
+ ;
+ return true;
+}
+
+class MemBlock {
+ char *start;
+ size_t size;
+ char *free;
+ MemBlock *next = nullptr;
+
+ explicit MemBlock(size_t size) : size(size)
+ {
+ free = start = new char[size];
+ }
+
+ size_t avail(void) { return size - (free - start); }
+
+ public:
+
+ MemBlock() : MemBlock(APT_MEMBLOCK_SIZE) {}
+
+ ~MemBlock() {
+ delete [] start;
+ delete next;
+ }
+
+ void clear(void) {
+ free = start;
+ if (next)
+ next->clear();
+ }
+
+ char *add_easy(char *src, size_t len, char *last)
+ {
+ if (last) {
+ for (MemBlock *k = this; k; k = k->next) {
+ if (k->free == last) {
+ if (len <= k->avail()) {
+ char * const n = k->add(src, len);
+ assert(last == n); // we checked already that the block is big enough, so a new one shouldn't be used
+ return (last == n) ? nullptr : n;
+ } else {
+ break;
+ }
+ } else if (last >= start && last < free) {
+ break;
+ }
+ }
+ }
+ return add(src, len);
+ }
+
+ char *add(char *src, size_t len) {
+ if (len > avail()) {
+ if (!next) {
+ if (len > APT_MEMBLOCK_SIZE) {
+ next = new MemBlock(len);
+ } else {
+ next = new MemBlock();
+ }
+ }
+ return next->add(src, len);
+ }
+ char *dst = free;
+ free += len;
+ memcpy(dst, src, len);
+ return dst;
+ }
+};
+
+struct Change {
+ /* Ordering:
+ *
+ * 1. write out <offset> lines unchanged
+ * 2. skip <del_cnt> lines from source
+ * 3. write out <add_cnt> lines (<add>/<add_len>)
+ */
+ size_t offset;
+ size_t del_cnt;
+ size_t add_cnt; /* lines */
+ size_t add_len; /* bytes */
+ char *add;
+
+ explicit Change(size_t off)
+ {
+ offset = off;
+ del_cnt = add_cnt = add_len = 0;
+ add = NULL;
+ }
+
+ /* actually, don't write <lines> lines from <add> */
+ bool skip_lines(size_t lines)
+ {
+ while (lines > 0) {
+ char *s = (char*) memchr(add, '\n', add_len);
+ if (s == nullptr)
+ return _error->Error("No line left in add_len data to skip (1)");
+ s++;
+ add_len -= (s - add);
+ add_cnt--;
+ lines--;
+ if (add_len == 0) {
+ add = nullptr;
+ if (add_cnt != 0 || lines != 0)
+ return _error->Error("No line left in add_len data to skip (2)");
+ } else {
+ add = s;
+ if (add_cnt == 0)
+ return _error->Error("No line left in add_len data to skip (3)");
+ }
+ }
+ return true;
+ }
+};
+
+class FileChanges {
+ std::list<struct Change> changes;
+ std::list<struct Change>::iterator where;
+ size_t pos; // line number is as far left of iterator as possible
+
+ bool pos_is_okay(void) const
+ {
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+ // this isn't unsafe, it is just a moderately expensive check we want to avoid normally
+ size_t cpos = 0;
+ std::list<struct Change>::const_iterator x;
+ for (x = changes.begin(); x != where; ++x) {
+ assert(x != changes.end());
+ cpos += x->offset + x->add_cnt;
+ }
+ return cpos == pos;
+#else
+ return true;
+#endif
+ }
+
+ public:
+ FileChanges() {
+ where = changes.end();
+ pos = 0;
+ }
+
+ std::list<struct Change>::iterator begin(void) { return changes.begin(); }
+ std::list<struct Change>::iterator end(void) { return changes.end(); }
+
+ std::list<struct Change>::reverse_iterator rbegin(void) { return changes.rbegin(); }
+ std::list<struct Change>::reverse_iterator rend(void) { return changes.rend(); }
+
+ bool add_change(Change c) {
+ assert(pos_is_okay());
+ if (not go_to_change_for(c.offset) ||
+ pos + where->offset != c.offset)
+ return false;
+ if (c.del_cnt > 0)
+ if (not delete_lines(c.del_cnt))
+ return false;
+ if (pos + where->offset != c.offset)
+ return false;
+ if (c.add_len > 0) {
+ assert(pos_is_okay());
+ if (where->add_len > 0)
+ if (not new_change())
+ return false;
+ if (where->add_len != 0 || where->add_cnt != 0)
+ return false;
+
+ where->add_len = c.add_len;
+ where->add_cnt = c.add_cnt;
+ where->add = c.add;
+ }
+ assert(pos_is_okay());
+ if (not merge())
+ return false;
+ return pos_is_okay();
+ }
+
+ private:
+ bool merge(void)
+ {
+ while (where->offset == 0 && where != changes.begin()) {
+ if (not left())
+ return false;
+ }
+ std::list<struct Change>::iterator next = where;
+ ++next;
+
+ while (next != changes.end() && next->offset == 0) {
+ where->del_cnt += next->del_cnt;
+ next->del_cnt = 0;
+ if (next->add == NULL) {
+ next = changes.erase(next);
+ } else if (where->add == NULL) {
+ where->add = next->add;
+ where->add_len = next->add_len;
+ where->add_cnt = next->add_cnt;
+ next = changes.erase(next);
+ } else {
+ ++next;
+ }
+ }
+ return true;
+ }
+
+ bool go_to_change_for(size_t line)
+ {
+ while(where != changes.end()) {
+ if (line < pos) {
+ if (not left())
+ return false;
+ continue;
+ }
+ if (pos + where->offset + where->add_cnt <= line) {
+ if (not right())
+ return false;
+ continue;
+ }
+ // line is somewhere in this slot
+ if (line < pos + where->offset) {
+ break;
+ } else if (line == pos + where->offset) {
+ return true;
+ } else {
+ if (not split(line - pos))
+ return false;
+ return right();
+ }
+ }
+ /* it goes before this patch */
+ return insert(line-pos);
+ }
+
+ bool new_change(void) { return insert(where->offset); }
+
+ bool insert(size_t offset)
+ {
+ assert(pos_is_okay());
+ if (where != changes.end() && offset > where->offset)
+ return false;
+ if (where != changes.end())
+ where->offset -= offset;
+ changes.insert(where, Change(offset));
+ --where;
+ return pos_is_okay();
+ }
+
+ bool split(size_t offset)
+ {
+ assert(pos_is_okay());
+ if (where->offset >= offset || offset >= where->offset + where->add_cnt)
+ return false;
+
+ size_t keep_lines = offset - where->offset;
+
+ Change before(*where);
+
+ where->del_cnt = 0;
+ where->offset = 0;
+ if (not where->skip_lines(keep_lines))
+ return false;
+
+ before.add_cnt = keep_lines;
+ before.add_len -= where->add_len;
+
+ changes.insert(where, before);
+ --where;
+ return pos_is_okay();
+ }
+
+ bool delete_lines(size_t cnt)
+ {
+ assert(pos_is_okay());
+ std::list<struct Change>::iterator x = where;
+ while (cnt > 0)
+ {
+ size_t del;
+ del = x->add_cnt;
+ if (del > cnt)
+ del = cnt;
+ if (not x->skip_lines(del))
+ return false;
+ cnt -= del;
+
+ ++x;
+ if (x == changes.end()) {
+ del = cnt;
+ } else {
+ del = x->offset;
+ if (del > cnt)
+ del = cnt;
+ x->offset -= del;
+ }
+ where->del_cnt += del;
+ cnt -= del;
+ }
+ return pos_is_okay();
+ }
+
+ bool left(void) {
+ assert(pos_is_okay());
+ --where;
+ pos -= where->offset + where->add_cnt;
+ return pos_is_okay();
+ }
+
+ bool right(void) {
+ assert(pos_is_okay());
+ pos += where->offset + where->add_cnt;
+ ++where;
+ return pos_is_okay();
+ }
+};
+
+class Patch {
+ FileChanges filechanges;
+ MemBlock add_text;
+
+ static bool retry_fwrite(char *b, size_t l, FileFd &f, Hashes * const start_hash, Hashes * const end_hash = nullptr) APT_NONNULL(1)
+ {
+ if (f.Write(b, l) == false)
+ return false;
+ if (start_hash)
+ start_hash->Add((unsigned char*)b, l);
+ if (end_hash)
+ end_hash->Add((unsigned char*)b, l);
+ return true;
+ }
+
+ static void dump_rest(FileFd &o, FileFd &i,
+ Hashes * const start_hash, Hashes * const end_hash)
+ {
+ char buffer[APT_MEMBLOCK_SIZE];
+ unsigned long long l = 0;
+ while (i.Read(buffer, sizeof(buffer), &l)) {
+ if (l ==0 || !retry_fwrite(buffer, l, o, start_hash, end_hash))
+ break;
+ }
+ }
+
+ static void dump_lines(FileFd &o, FileFd &i, size_t n,
+ Hashes * const start_hash, Hashes * const end_hash)
+ {
+ char buffer[APT_MEMBLOCK_SIZE];
+ while (n > 0) {
+ if (i.ReadLine(buffer, sizeof(buffer)) == NULL)
+ buffer[0] = '\0';
+ size_t const l = strlen(buffer);
+ if (l == 0 || buffer[l-1] == '\n')
+ n--;
+ retry_fwrite(buffer, l, o, start_hash, end_hash);
+ }
+ }
+
+ static void skip_lines(FileFd &i, int n, Hashes * const start_hash)
+ {
+ char buffer[APT_MEMBLOCK_SIZE];
+ while (n > 0) {
+ if (i.ReadLine(buffer, sizeof(buffer)) == NULL)
+ buffer[0] = '\0';
+ size_t const l = strlen(buffer);
+ if (l == 0 || buffer[l-1] == '\n')
+ n--;
+ if (start_hash)
+ start_hash->Add((unsigned char*)buffer, l);
+ }
+ }
+
+ static void dump_mem(FileFd &o, char *p, size_t s, Hashes *hash) APT_NONNULL(2) {
+ retry_fwrite(p, s, o, nullptr, hash);
+ }
+
+ public:
+
+ bool read_diff(FileFd &f, Hashes * const h)
+ {
+ char buffer[APT_MEMBLOCK_SIZE];
+ bool cmdwanted = true;
+
+ Change ch(std::numeric_limits<size_t>::max());
+ if (f.ReadLine(buffer, sizeof(buffer)) == nullptr)
+ {
+ if (f.Eof())
+ return true;
+ return _error->Error("Reading first line of patchfile %s failed", f.Name().c_str());
+ }
+ do {
+ if (h != NULL)
+ h->Add(buffer);
+ if (cmdwanted) {
+ char *m, *c;
+ size_t s, e;
+ errno = 0;
+ s = strtoul(buffer, &m, 10);
+ if (unlikely(m == buffer || s == std::numeric_limits<unsigned long>::max() || errno != 0))
+ return _error->Error("Parsing patchfile %s failed: Expected an effected line start", f.Name().c_str());
+ else if (*m == ',') {
+ ++m;
+ e = strtol(m, &c, 10);
+ if (unlikely(m == c || e == std::numeric_limits<unsigned long>::max() || errno != 0))
+ return _error->Error("Parsing patchfile %s failed: Expected an effected line end", f.Name().c_str());
+ if (unlikely(e < s))
+ return _error->Error("Parsing patchfile %s failed: Effected lines end %lu is before start %lu", f.Name().c_str(), e, s);
+ } else {
+ e = s;
+ c = m;
+ }
+ if (s > ch.offset)
+ return _error->Error("Parsing patchfile %s failed: Effected line is after previous effected line", f.Name().c_str());
+ switch(*c) {
+ case 'a':
+ cmdwanted = false;
+ ch.add = NULL;
+ ch.add_cnt = 0;
+ ch.add_len = 0;
+ ch.offset = s;
+ ch.del_cnt = 0;
+ break;
+ case 'c':
+ if (unlikely(s == 0))
+ return _error->Error("Parsing patchfile %s failed: Change command can't effect line zero", f.Name().c_str());
+ cmdwanted = false;
+ ch.add = NULL;
+ ch.add_cnt = 0;
+ ch.add_len = 0;
+ ch.offset = s - 1;
+ ch.del_cnt = e - s + 1;
+ break;
+ case 'd':
+ if (unlikely(s == 0))
+ return _error->Error("Parsing patchfile %s failed: Delete command can't effect line zero", f.Name().c_str());
+ ch.offset = s - 1;
+ ch.del_cnt = e - s + 1;
+ ch.add = NULL;
+ ch.add_cnt = 0;
+ ch.add_len = 0;
+ if (not filechanges.add_change(ch))
+ return _error->Error("Parsing patchfile %s failed: Delete command could not be added to changes", f.Name().c_str());
+ break;
+ default:
+ return _error->Error("Parsing patchfile %s failed: Unknown command", f.Name().c_str());
+ }
+ } else { /* !cmdwanted */
+ if (strcmp(buffer, ".\n") == 0) {
+ cmdwanted = true;
+ if (not filechanges.add_change(ch))
+ return _error->Error("Parsing patchfile %s failed: Data couldn't be added for command (1)", f.Name().c_str());
+ } else {
+ char *last = NULL;
+ char *add;
+ size_t l;
+ if (ch.add)
+ last = ch.add + ch.add_len;
+ l = strlen(buffer);
+ add = add_text.add_easy(buffer, l, last);
+ if (!add) {
+ ch.add_len += l;
+ ch.add_cnt++;
+ } else {
+ if (ch.add) {
+ if (not filechanges.add_change(ch))
+ return _error->Error("Parsing patchfile %s failed: Data couldn't be added for command (2)", f.Name().c_str());
+ ch.del_cnt = 0;
+ }
+ ch.offset += ch.add_cnt;
+ ch.add = add;
+ ch.add_len = l;
+ ch.add_cnt = 1;
+ }
+ }
+ }
+ } while(f.ReadLine(buffer, sizeof(buffer)));
+ return true;
+ }
+
+ void write_diff(FileFd &f)
+ {
+ unsigned long long line = 0;
+ std::list<struct Change>::reverse_iterator ch;
+ for (ch = filechanges.rbegin(); ch != filechanges.rend(); ++ch) {
+ line += ch->offset + ch->del_cnt;
+ }
+
+ for (ch = filechanges.rbegin(); ch != filechanges.rend(); ++ch) {
+ std::list<struct Change>::reverse_iterator mg_i, mg_e = ch;
+ while (ch->del_cnt == 0 && ch->offset == 0)
+ {
+ ++ch;
+ if (unlikely(ch == filechanges.rend()))
+ return;
+ }
+ line -= ch->del_cnt;
+ std::string buf;
+ if (ch->add_cnt > 0) {
+ if (ch->del_cnt == 0) {
+ strprintf(buf, "%llua\n", line);
+ } else if (ch->del_cnt == 1) {
+ strprintf(buf, "%lluc\n", line+1);
+ } else {
+ strprintf(buf, "%llu,%lluc\n", line+1, line+ch->del_cnt);
+ }
+ f.Write(buf.c_str(), buf.length());
+
+ mg_i = ch;
+ do {
+ dump_mem(f, mg_i->add, mg_i->add_len, NULL);
+ } while (mg_i-- != mg_e);
+
+ buf = ".\n";
+ f.Write(buf.c_str(), buf.length());
+ } else if (ch->del_cnt == 1) {
+ strprintf(buf, "%llud\n", line+1);
+ f.Write(buf.c_str(), buf.length());
+ } else if (ch->del_cnt > 1) {
+ strprintf(buf, "%llu,%llud\n", line+1, line+ch->del_cnt);
+ f.Write(buf.c_str(), buf.length());
+ }
+ line -= ch->offset;
+ }
+ }
+
+ void apply_against_file(FileFd &out, FileFd &in,
+ Hashes * const start_hash = nullptr, Hashes * const end_hash = nullptr)
+ {
+ std::list<struct Change>::iterator ch;
+ for (ch = filechanges.begin(); ch != filechanges.end(); ++ch) {
+ dump_lines(out, in, ch->offset, start_hash, end_hash);
+ skip_lines(in, ch->del_cnt, start_hash);
+ if (ch->add_len != 0)
+ dump_mem(out, ch->add, ch->add_len, end_hash);
+ }
+ dump_rest(out, in, start_hash, end_hash);
+ out.Flush();
+ }
+};
+
+#ifndef APT_EXCLUDE_RRED_METHOD_CODE
+class RredMethod : public aptMethod {
+ private:
+ bool Debug;
+
+ struct PDiffFile {
+ std::string FileName;
+ HashStringList ExpectedHashes;
+ PDiffFile(std::string const &FileName, HashStringList const &ExpectedHashes) :
+ FileName(FileName), ExpectedHashes(ExpectedHashes) {}
+ };
+
+ HashStringList ReadExpectedHashesForPatch(unsigned int const patch, std::string const &Message)
+ {
+ HashStringList ExpectedHashes;
+ for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+ {
+ std::string tagname;
+ strprintf(tagname, "Patch-%d-%s-Hash", patch, *type);
+ std::string const hashsum = LookupTag(Message, tagname.c_str());
+ if (hashsum.empty() == false)
+ ExpectedHashes.push_back(HashString(*type, hashsum));
+ }
+ return ExpectedHashes;
+ }
+
+ protected:
+ virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE {
+ Debug = DebugEnabled();
+ URI Get(Itm->Uri);
+ std::string Path = DecodeSendURI(Get.Host + Get.Path); // rred:/path - no host
+
+ FetchResult Res;
+ Res.Filename = Itm->DestFile;
+ if (Itm->Uri.empty())
+ {
+ Path = Itm->DestFile;
+ Itm->DestFile.append(".result");
+ } else
+ URIStart(Res);
+
+ std::vector<PDiffFile> patchfiles;
+ Patch patch;
+
+ HashStringList StartHashes;
+ for (char const * const * type = HashString::SupportedHashes(); *type != nullptr; ++type)
+ {
+ std::string tagname;
+ strprintf(tagname, "Start-%s-Hash", *type);
+ std::string const hashsum = LookupTag(Message, tagname.c_str());
+ if (hashsum.empty() == false)
+ StartHashes.push_back(HashString(*type, hashsum));
+ }
+
+ if (FileExists(Path + ".ed") == true)
+ {
+ HashStringList const ExpectedHashes = ReadExpectedHashesForPatch(0, Message);
+ std::string const FileName = Path + ".ed";
+ if (ExpectedHashes.usable() == false)
+ return _error->Error("No hashes found for uncompressed patch: %s", FileName.c_str());
+ patchfiles.push_back(PDiffFile(FileName, ExpectedHashes));
+ }
+ else
+ {
+ _error->PushToStack();
+ std::vector<std::string> patches = GetListOfFilesInDir(flNotFile(Path), "gz", true, false);
+ _error->RevertToStack();
+
+ std::string const baseName = Path + ".ed.";
+ unsigned int seen_patches = 0;
+ for (std::vector<std::string>::const_iterator p = patches.begin();
+ p != patches.end(); ++p)
+ {
+ if (p->compare(0, baseName.length(), baseName) == 0)
+ {
+ HashStringList const ExpectedHashes = ReadExpectedHashesForPatch(seen_patches, Message);
+ if (ExpectedHashes.usable() == false)
+ return _error->Error("No hashes found for uncompressed patch %d: %s", seen_patches, p->c_str());
+ patchfiles.push_back(PDiffFile(*p, ExpectedHashes));
+ ++seen_patches;
+ }
+ }
+ }
+
+ std::string patch_name;
+ for (std::vector<PDiffFile>::iterator I = patchfiles.begin();
+ I != patchfiles.end();
+ ++I)
+ {
+ patch_name = I->FileName;
+ if (Debug == true)
+ std::clog << "Patching " << Path << " with " << patch_name
+ << std::endl;
+
+ FileFd p;
+ Hashes patch_hash(I->ExpectedHashes);
+ // all patches are compressed, even if the name doesn't reflect it
+ if (p.Open(patch_name, FileFd::ReadOnly, FileFd::Gzip) == false ||
+ patch.read_diff(p, &patch_hash) == false)
+ {
+ _error->DumpErrors(std::cerr, GlobalError::DEBUG, false);
+ return false;
+ }
+ p.Close();
+ HashStringList const hsl = patch_hash.GetHashStringList();
+ if (hsl != I->ExpectedHashes)
+ return _error->Error("Hash Sum mismatch for uncompressed patch %s", patch_name.c_str());
+ }
+
+ if (Debug == true)
+ std::clog << "Applying patches against " << Path
+ << " and writing results to " << Itm->DestFile
+ << std::endl;
+
+ FileFd inp, out;
+ if (inp.Open(Path, FileFd::ReadOnly, FileFd::Extension) == false)
+ {
+ if (Debug == true)
+ std::clog << "FAILED to open inp " << Path << std::endl;
+ return _error->Error("Failed to open inp %s", Path.c_str());
+ }
+ if (out.Open(Itm->DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Empty | FileFd::BufferedWrite, FileFd::Extension) == false)
+ {
+ if (Debug == true)
+ std::clog << "FAILED to open out " << Itm->DestFile << std::endl;
+ return _error->Error("Failed to open out %s", Itm->DestFile.c_str());
+ }
+
+ Hashes end_hash(Itm->ExpectedHashes);
+ if (StartHashes.usable())
+ {
+ Hashes start_hash(StartHashes);
+ patch.apply_against_file(out, inp, &start_hash, &end_hash);
+ if (start_hash.GetHashStringList() != StartHashes)
+ _error->Error("The input file hadn't the expected hash!");
+ }
+ else
+ patch.apply_against_file(out, inp, nullptr, &end_hash);
+
+ out.Close();
+ inp.Close();
+
+ if (_error->PendingError() == true) {
+ if (Debug == true)
+ std::clog << "FAILED to read or write files" << std::endl;
+ return false;
+ }
+
+ if (Debug == true) {
+ std::clog << "rred: finished file patching of " << Path << "." << std::endl;
+ }
+
+ struct stat bufbase, bufpatch;
+ if (stat(Path.c_str(), &bufbase) != 0 ||
+ stat(patch_name.c_str(), &bufpatch) != 0)
+ return _error->Errno("stat", _("Failed to stat %s"), Path.c_str());
+
+ struct timeval times[2];
+ times[0].tv_sec = bufbase.st_atime;
+ times[1].tv_sec = bufpatch.st_mtime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ if (utimes(Itm->DestFile.c_str(), times) != 0)
+ return _error->Errno("utimes",_("Failed to set modification time"));
+
+ if (stat(Itm->DestFile.c_str(), &bufbase) != 0)
+ return _error->Errno("stat", _("Failed to stat %s"), Itm->DestFile.c_str());
+
+ Res.LastModified = bufbase.st_mtime;
+ Res.Size = bufbase.st_size;
+ Res.TakeHashes(end_hash);
+ URIDone(Res);
+
+ return true;
+ }
+
+ public:
+ RredMethod() : aptMethod("rred", "2.0", SendConfig | SendURIEncoded), Debug(false)
+ {
+ SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY;
+ }
+};
+
+static const APT::Configuration::Compressor *FindCompressor(std::vector<APT::Configuration::Compressor> const &compressors, std::string const &name) /*{{{*/
+{
+ APT::Configuration::Compressor const * compressor = nullptr;
+ for (auto const & c : compressors)
+ {
+ if (compressor != nullptr && c.Cost >= compressor->Cost)
+ continue;
+ if (c.Name == name || c.Extension == name || (!c.Extension.empty() && c.Extension.substr(1) == name))
+ compressor = &c;
+ }
+ return compressor;
+}
+ /*}}}*/
+static std::vector<aptDispatchWithHelp> GetCommands()
+{
+ return {{nullptr, nullptr, nullptr}};
+}
+int main(int argc, const char *argv[])
+{
+ if (argc <= 1)
+ return RredMethod().Run();
+
+ CommandLine CmdL;
+ auto const Cmds = ParseCommandLine(CmdL, APT_CMD::RRED, &_config, nullptr, argc, argv, &ShowHelp, &GetCommands);
+
+ FileFd input, output;
+ unsigned int argi = 0;
+ auto const argmax = CmdL.FileSize();
+ bool const quiet = _config->FindI("quiet", 0) >= 2;
+
+ std::string const compressorName = _config->Find("Rred::Compress", "");
+ auto const compressors = APT::Configuration::getCompressors();
+ APT::Configuration::Compressor const * compressor = nullptr;
+ if (not compressorName.empty())
+ {
+ compressor = FindCompressor(compressors, compressorName);
+ if (compressor == nullptr)
+ {
+ std::cerr << "E: Could not find compressor: " << compressorName << '\n';
+ return 101;
+ }
+ }
+
+ bool just_diff = false;
+ if (_config->FindB("Rred::T", false))
+ {
+ if (argmax < 3)
+ {
+ std::cerr << "E: Not enough filenames given on the command line for mode 't'\n";
+ return 101;
+ }
+ if (not quiet)
+ std::clog << "Patching " << CmdL.FileList[0] << " into " << CmdL.FileList[1] << "\n";
+ input.Open(CmdL.FileList[0], FileFd::ReadOnly,FileFd::Extension);
+ if (compressor == nullptr)
+ output.Open(CmdL.FileList[1], FileFd::WriteOnly | FileFd::Create | FileFd::Empty | FileFd::BufferedWrite, FileFd::Extension);
+ else
+ output.Open(CmdL.FileList[1], FileFd::WriteOnly | FileFd::Create | FileFd::Empty | FileFd::BufferedWrite, *compressor);
+ argi = 2;
+ }
+ else
+ {
+ if (compressor == nullptr)
+ output.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly | FileFd::Create | FileFd::BufferedWrite);
+ else
+ output.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly | FileFd::Create | FileFd::BufferedWrite, *compressor);
+ if (_config->FindB("Rred::F", false))
+ input.OpenDescriptor(STDIN_FILENO, FileFd::ReadOnly);
+ else
+ just_diff = true;
+ }
+
+ if (argi + 1 > argmax)
+ {
+ std::cerr << "E: At least one patch needs to be given on the command line\n";
+ return 101;
+ }
+
+ Patch merged_patch;
+ for (; argi < argmax; ++argi)
+ {
+ FileFd patch;
+ if (not patch.Open(CmdL.FileList[argi], FileFd::ReadOnly, FileFd::Extension))
+ {
+ _error->DumpErrors(std::cerr);
+ return 1;
+ }
+ if (not merged_patch.read_diff(patch, nullptr))
+ {
+ _error->DumpErrors(std::cerr);
+ return 2;
+ }
+ }
+
+ if (just_diff)
+ merged_patch.write_diff(output);
+ else
+ merged_patch.apply_against_file(output, input);
+
+ output.Close();
+ input.Close();
+ return DispatchCommandLine(CmdL, {});
+}
+#endif
diff --git a/methods/rsh.cc b/methods/rsh.cc
new file mode 100644
index 0000000..4432d5a
--- /dev/null
+++ b/methods/rsh.cc
@@ -0,0 +1,561 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ RSH method - Transfer files via rsh compatible program
+
+ Written by Ben Collins <bcollins@debian.org>, Copyright (c) 2000
+
+ This file stated:
+ Licensed under the GNU General Public License v2 [no exception clauses]
+
+ We believe that this was intended to be not a statement against future
+ versions of the GPL, but meant to exclude the Qt license exception in
+ place in APT until that time.
+
+ We received permission from Ben in 2021 to relicense under GPL-2+,
+ contributions from Adam Heath and Daniel Hartwig may still have to
+ be considered GPL-2 for the time being.
+
+ Other contributions are GPL-2+
+
+ See https://lists.debian.org/deity/2021/04/msg00013.html for details
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/hashes.h>
+#include <apt-pkg/strutl.h>
+
+#include "rsh.h"
+#include <cerrno>
+#include <csignal>
+#include <cstdarg>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#include <apti18n.h>
+ /*}}}*/
+
+unsigned long TimeOut = 30;
+Configuration::Item const *RshOptions = 0;
+time_t RSHMethod::FailTime = 0;
+std::string RSHMethod::FailFile;
+int RSHMethod::FailFd = -1;
+
+// RSHConn::RSHConn - Constructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+RSHConn::RSHConn(std::string const &pProg, URI Srv) : Len(0), WriteFd(-1), ReadFd(-1),
+ ServerName(Srv), Prog(pProg), Process(-1) {
+ Buffer[0] = '\0';
+}
+ /*}}}*/
+// RSHConn::RSHConn - Destructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+RSHConn::~RSHConn()
+{
+ Close();
+}
+ /*}}}*/
+// RSHConn::Close - Forcibly terminate the connection /*{{{*/
+// ---------------------------------------------------------------------
+/* Often this is called when things have gone wrong to indicate that the
+ connection is no longer usable. */
+void RSHConn::Close()
+{
+ if (Process == -1)
+ return;
+
+ close(WriteFd);
+ close(ReadFd);
+ kill(Process,SIGINT);
+ ExecWait(Process,"",true);
+ WriteFd = -1;
+ ReadFd = -1;
+ Process = -1;
+}
+ /*}}}*/
+// RSHConn::Open - Connect to a host /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool RSHConn::Open()
+{
+ // Use the already open connection if possible.
+ if (Process != -1)
+ return true;
+
+ if (Connect(ServerName.Host,ServerName.Port,ServerName.User) == false)
+ return false;
+
+ return true;
+}
+ /*}}}*/
+// RSHConn::Connect - Fire up rsh and connect /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool RSHConn::Connect(std::string const &Host, unsigned int Port, std::string const &User)
+{
+ char *PortStr = NULL;
+ if (Port != 0)
+ {
+ if (asprintf (&PortStr, "%d", Port) == -1 || PortStr == NULL)
+ return _error->Errno("asprintf", _("Failed"));
+ }
+
+ // Create the pipes
+ int Pipes[4] = {-1,-1,-1,-1};
+ if (pipe(Pipes) != 0 || pipe(Pipes+2) != 0)
+ {
+ _error->Errno("pipe",_("Failed to create IPC pipe to subprocess"));
+ for (int I = 0; I != 4; I++)
+ close(Pipes[I]);
+ return false;
+ }
+ for (int I = 0; I != 4; I++)
+ SetCloseExec(Pipes[I],true);
+
+ Process = ExecFork();
+
+ // The child
+ if (Process == 0)
+ {
+ const char *Args[400];
+ unsigned int i = 0;
+
+ dup2(Pipes[1],STDOUT_FILENO);
+ dup2(Pipes[2],STDIN_FILENO);
+
+ // Probably should do
+ // dup2(open("/dev/null",O_RDONLY),STDERR_FILENO);
+
+ Args[i++] = Prog.c_str();
+
+ // Insert user-supplied command line options
+ Configuration::Item const *Opts = RshOptions;
+ if (Opts != 0)
+ {
+ Opts = Opts->Child;
+ for (; Opts != 0; Opts = Opts->Next)
+ {
+ if (Opts->Value.empty() == true)
+ continue;
+ Args[i++] = Opts->Value.c_str();
+ }
+ }
+
+ if (User.empty() == false) {
+ Args[i++] = "-l";
+ Args[i++] = User.c_str();
+ }
+ if (PortStr != NULL) {
+ Args[i++] = "-p";
+ Args[i++] = PortStr;
+ }
+ if (Host.empty() == false) {
+ Args[i++] = Host.c_str();
+ }
+ Args[i++] = "/bin/sh";
+ Args[i] = 0;
+ execvp(Args[0],(char **)Args);
+ exit(100);
+ }
+
+ if (PortStr != NULL)
+ free(PortStr);
+
+ ReadFd = Pipes[0];
+ WriteFd = Pipes[3];
+ SetNonBlock(Pipes[0],true);
+ SetNonBlock(Pipes[3],true);
+ close(Pipes[1]);
+ close(Pipes[2]);
+
+ return true;
+}
+bool RSHConn::Connect(std::string const &Host, std::string const &User)
+{
+ return Connect(Host, 0, User);
+}
+ /*}}}*/
+// RSHConn::ReadLine - Very simple buffered read with timeout /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool RSHConn::ReadLine(std::string &Text)
+{
+ if (Process == -1 || ReadFd == -1)
+ return false;
+
+ // Suck in a line
+ while (Len < sizeof(Buffer))
+ {
+ // Scan the buffer for a new line
+ for (unsigned int I = 0; I != Len; I++)
+ {
+ // Escape some special chars
+ if (Buffer[I] == 0)
+ Buffer[I] = '?';
+
+ // End of line?
+ if (Buffer[I] != '\n')
+ continue;
+
+ I++;
+ Text = std::string(Buffer,I);
+ memmove(Buffer,Buffer+I,Len - I);
+ Len -= I;
+ return true;
+ }
+
+ // Wait for some data..
+ if (WaitFd(ReadFd,false,TimeOut) == false)
+ {
+ Close();
+ return _error->Error(_("Connection timeout"));
+ }
+
+ // Suck it back
+ int Res = read(ReadFd,Buffer + Len,sizeof(Buffer) - Len);
+ if (Res <= 0)
+ {
+ _error->Errno("read",_("Read error"));
+ Close();
+ return false;
+ }
+ Len += Res;
+ }
+
+ return _error->Error(_("A response overflowed the buffer."));
+}
+ /*}}}*/
+// RSHConn::WriteMsg - Send a message with optional remote sync. /*{{{*/
+// ---------------------------------------------------------------------
+/* The remote sync flag appends a || echo which will insert blank line
+ once the command completes. */
+bool RSHConn::WriteMsg(std::string &Text,bool Sync,const char *Fmt,...)
+{
+ va_list args;
+ va_start(args,Fmt);
+
+ // sprintf into a buffer
+ char Tmp[1024];
+ vsnprintf(Tmp,sizeof(Tmp),Fmt,args);
+ va_end(args);
+
+ // concat to create the real msg
+ std::string Msg;
+ if (Sync == true)
+ Msg = std::string(Tmp) + " 2> /dev/null || echo\n";
+ else
+ Msg = std::string(Tmp) + " 2> /dev/null\n";
+
+ // Send it off
+ const char *S = Msg.c_str();
+ unsigned long Len = strlen(S);
+ unsigned long Start = 0;
+ while (Len != 0)
+ {
+ if (WaitFd(WriteFd,true,TimeOut) == false)
+ {
+
+ Close();
+ return _error->Error(_("Connection timeout"));
+ }
+
+ int Res = write(WriteFd,S + Start,Len);
+ if (Res <= 0)
+ {
+ _error->Errno("write",_("Write error"));
+ Close();
+ return false;
+ }
+
+ Len -= Res;
+ Start += Res;
+ }
+
+ if (Sync == true)
+ return ReadLine(Text);
+ return true;
+}
+ /*}}}*/
+// RSHConn::Size - Return the size of the file /*{{{*/
+// ---------------------------------------------------------------------
+/* Right now for successful transfer the file size must be known in
+ advance. */
+bool RSHConn::Size(const char *Path,unsigned long long &Size)
+{
+ // Query the size
+ std::string Msg;
+ Size = 0;
+
+ if (WriteMsg(Msg,true,"find %s -follow -printf '%%s\\n'",Path) == false)
+ return false;
+
+ // FIXME: Sense if the bad reply is due to a File Not Found.
+
+ char *End;
+ Size = strtoull(Msg.c_str(),&End,10);
+ if (End == Msg.c_str())
+ return _error->Error(_("File not found"));
+ return true;
+}
+ /*}}}*/
+// RSHConn::ModTime - Get the modification time in UTC /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool RSHConn::ModTime(const char *Path, time_t &Time)
+{
+ Time = time(&Time);
+ // Query the mod time
+ std::string Msg;
+
+ if (WriteMsg(Msg,true,"TZ=UTC find %s -follow -printf '%%TY%%Tm%%Td%%TH%%TM%%TS\\n'",Path) == false)
+ return false;
+
+ // Parse it
+ return FTPMDTMStrToTime(Msg.c_str(), Time);
+}
+ /*}}}*/
+// RSHConn::Get - Get a file /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool RSHConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
+ Hashes &Hash,bool &Missing, unsigned long long Size)
+{
+ Missing = false;
+
+ // Round to a 2048 byte block
+ Resume = Resume - (Resume % 2048);
+
+ if (To.Truncate(Resume) == false)
+ return false;
+ if (To.Seek(0) == false)
+ return false;
+
+ if (Resume != 0) {
+ if (Hash.AddFD(To,Resume) == false) {
+ _error->Errno("read",_("Problem hashing file"));
+ return false;
+ }
+ }
+
+ // FIXME: Detect file-not openable type errors.
+ std::string Jnk;
+ if (WriteMsg(Jnk,false,"dd if=%s bs=2048 skip=%u", Path, Resume / 2048) == false)
+ return false;
+
+ // Copy loop
+ unsigned long long MyLen = Resume;
+ unsigned char Buffer[4096];
+ while (MyLen < Size)
+ {
+ // Wait for some data..
+ if (WaitFd(ReadFd,false,TimeOut) == false)
+ {
+ Close();
+ return _error->Error(_("Data socket timed out"));
+ }
+
+ // Read the data..
+ int Res = read(ReadFd,Buffer,sizeof(Buffer));
+ if (Res == 0)
+ {
+ Close();
+ return _error->Error(_("Connection closed prematurely"));
+ }
+
+ if (Res < 0)
+ {
+ if (errno == EAGAIN)
+ continue;
+ break;
+ }
+ MyLen += Res;
+
+ Hash.Add(Buffer,Res);
+ if (To.Write(Buffer,Res) == false)
+ {
+ Close();
+ return false;
+ }
+ }
+
+ return true;
+}
+ /*}}}*/
+
+// RSHMethod::RSHMethod - Constructor /*{{{*/
+RSHMethod::RSHMethod(std::string &&pProg) : aptMethod(std::move(pProg),"1.0",SendConfig | SendURIEncoded)
+{
+ signal(SIGTERM,SigTerm);
+ signal(SIGINT,SigTerm);
+ Server = 0;
+ FailFd = -1;
+}
+ /*}}}*/
+// RSHMethod::Configuration - Handle a configuration message /*{{{*/
+// ---------------------------------------------------------------------
+bool RSHMethod::Configuration(std::string Message)
+{
+ // enabling privilege dropping for this method requires configuration…
+ // … which is otherwise lifted straight from root, so use it by default.
+ _config->Set(std::string("Binary::") + Binary + "::APT::Sandbox::User", "");
+
+ if (aptMethod::Configuration(Message) == false)
+ return false;
+
+ std::string const timeconf = std::string("Acquire::") + Binary + "::Timeout";
+ TimeOut = _config->FindI(timeconf, TimeOut);
+ std::string const optsconf = std::string("Acquire::") + Binary + "::Options";
+ RshOptions = _config->Tree(optsconf.c_str());
+
+ return true;
+}
+ /*}}}*/
+// RSHMethod::SigTerm - Clean up and timestamp the files on exit /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void RSHMethod::SigTerm(int)
+{
+ if (FailFd == -1)
+ _exit(100);
+
+ // Transfer the modification times
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(FailFile.c_str(), times);
+ close(FailFd);
+
+ _exit(100);
+}
+ /*}}}*/
+// RSHMethod::Fetch - Fetch a URI /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool RSHMethod::Fetch(FetchItem *Itm)
+{
+ URI Get(Itm->Uri);
+ auto const File = DecodeSendURI(Get.Path);
+ FetchResult Res;
+ Res.Filename = Itm->DestFile;
+ Res.IMSHit = false;
+
+ // Connect to the server
+ if (Server == 0 || Server->Comp(Get) == false) {
+ delete Server;
+ Server = new RSHConn(Binary, Get);
+ }
+
+ // Could not connect is a transient error..
+ if (Server->Open() == false) {
+ Server->Close();
+ Fail(true);
+ return true;
+ }
+
+ // We say this mainly because the pause here is for the
+ // ssh connection that is still going
+ Status(_("Connecting to %s"), Get.Host.c_str());
+
+ // Get the files information
+ unsigned long long Size;
+ if (not Server->Size(File.c_str(), Size) ||
+ not Server->ModTime(File.c_str(), FailTime))
+ {
+ //Fail(true);
+ //_error->Error(_("File not found")); // Will be handled by Size
+ return false;
+ }
+ Res.Size = Size;
+
+ // See if it is an IMS hit
+ if (Itm->LastModified == FailTime) {
+ Res.Size = 0;
+ Res.IMSHit = true;
+ URIDone(Res);
+ return true;
+ }
+
+ // See if the file exists
+ struct stat Buf;
+ if (stat(Itm->DestFile.c_str(),&Buf) == 0) {
+ if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime) {
+ Res.Size = Buf.st_size;
+ Res.LastModified = Buf.st_mtime;
+ Res.ResumePoint = Buf.st_size;
+ URIDone(Res);
+ return true;
+ }
+
+ // Resume?
+ if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size)
+ Res.ResumePoint = Buf.st_size;
+ }
+
+ // Open the file
+ Hashes Hash(Itm->ExpectedHashes);
+ {
+ FileFd Fd(Itm->DestFile,FileFd::WriteAny);
+ if (_error->PendingError() == true)
+ return false;
+
+ URIStart(Res);
+
+ FailFile = Itm->DestFile;
+ (void)(FailFile.c_str()); // Make sure we don't do a malloc in the signal handler
+ FailFd = Fd.Fd();
+
+ bool Missing;
+ if (not Server->Get(File.c_str(), Fd, Res.ResumePoint, Hash, Missing, Res.Size))
+ {
+ Fd.Close();
+
+ // Timestamp
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(FailFile.c_str(), times);
+
+ // If the file is missing we hard fail otherwise transient fail
+ if (Missing == true)
+ return false;
+ Fail(true);
+ return true;
+ }
+
+ Res.Size = Fd.Size();
+ struct timeval times[2];
+ times[0].tv_sec = FailTime;
+ times[1].tv_sec = FailTime;
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(Fd.Name().c_str(), times);
+ FailFd = -1;
+ }
+
+ Res.LastModified = FailTime;
+ Res.TakeHashes(Hash);
+
+ URIDone(Res);
+
+ return true;
+}
+ /*}}}*/
+
+int main(int, const char *argv[])
+{
+ return RSHMethod(flNotDir(argv[0])).Run();
+}
diff --git a/methods/rsh.h b/methods/rsh.h
new file mode 100644
index 0000000..82ac543
--- /dev/null
+++ b/methods/rsh.h
@@ -0,0 +1,75 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ RSH method - Transfer files via rsh compatible program
+
+ ##################################################################### */
+ /*}}}*/
+#ifndef APT_RSH_H
+#define APT_RSH_H
+
+#include <ctime>
+#include <string>
+
+#include <apt-pkg/strutl.h>
+
+class Hashes;
+class FileFd;
+
+class RSHConn
+{
+ char Buffer[1024*10];
+ unsigned long Len;
+ int WriteFd;
+ int ReadFd;
+ URI ServerName;
+ std::string const Prog;
+
+ // Private helper functions
+ bool ReadLine(std::string &Text);
+
+ public:
+
+ pid_t Process;
+
+ // Raw connection IO
+ bool WriteMsg(std::string &Text,bool Sync,const char *Fmt,...);
+ bool Connect(std::string const &Host, std::string const &User);
+ bool Connect(std::string const &Host, unsigned int Port, std::string const &User);
+ bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
+
+ // Connection control
+ bool Open();
+ void Close();
+
+ // Query
+ bool Size(const char *Path,unsigned long long &Size);
+ bool ModTime(const char *Path, time_t &Time);
+ bool Get(const char *Path,FileFd &To,unsigned long long Resume,
+ Hashes &Hash,bool &Missing, unsigned long long Size);
+
+ RSHConn(std::string const &Prog, URI Srv);
+ ~RSHConn();
+};
+
+#include "aptmethod.h"
+
+class RSHMethod : public aptMethod
+{
+ virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE;
+ virtual bool Configuration(std::string Message) APT_OVERRIDE;
+
+ RSHConn *Server;
+
+ static std::string FailFile;
+ static int FailFd;
+ static time_t FailTime;
+ static APT_NORETURN void SigTerm(int);
+
+ public:
+
+ explicit RSHMethod(std::string &&Prog);
+};
+
+#endif
diff --git a/methods/store.cc b/methods/store.cc
new file mode 100644
index 0000000..8b30efa
--- /dev/null
+++ b/methods/store.cc
@@ -0,0 +1,147 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ Store method - Takes a file URI and stores its content (for which it will
+ calculate the hashes) in the given destination. The input file will be
+ extracted based on its file extension (or with the given compressor if
+ called with one of the compatible symlinks) and potentially recompressed
+ based on the file extension of the destination filename.
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include "aptmethod.h"
+#include <apt-pkg/aptconfiguration.h>
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/hashes.h>
+#include <apt-pkg/strutl.h>
+
+#include <cstring>
+#include <string>
+#include <vector>
+#include <sys/stat.h>
+#include <sys/time.h>
+
+#include <apti18n.h>
+ /*}}}*/
+
+class StoreMethod : public aptMethod
+{
+ virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE;
+
+ public:
+
+ explicit StoreMethod(std::string &&pProg) : aptMethod(std::move(pProg),"1.2",SingleInstance | SendConfig | SendURIEncoded)
+ {
+ SeccompFlags = aptMethod::BASE;
+ if (Binary != "store")
+ methodNames.insert(methodNames.begin(), "store");
+ }
+};
+
+static bool OpenFileWithCompressorByName(FileFd &fileFd, std::string const &Filename, unsigned int const Mode, std::string const &Name)
+{
+ if (Name == "store")
+ return fileFd.Open(Filename, Mode, FileFd::Extension);
+
+ std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors();
+ std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin();
+ for (; compressor != compressors.end(); ++compressor)
+ if (compressor->Name == Name)
+ break;
+ if (compressor == compressors.end())
+ return _error->Error("Extraction of file %s requires unknown compressor %s", Filename.c_str(), Name.c_str());
+ return fileFd.Open(Filename, Mode, *compressor);
+}
+
+
+ /*}}}*/
+bool StoreMethod::Fetch(FetchItem *Itm) /*{{{*/
+{
+ URI Get(Itm->Uri);
+ std::string Path = DecodeSendURI(Get.Host + Get.Path); // To account for relative paths
+
+ FetchResult Res;
+ Res.Filename = Itm->DestFile;
+ URIStart(Res);
+
+ // Open the source and destination files
+ FileFd From;
+ if (_config->FindB("Method::Compress", false) == false)
+ {
+ if (OpenFileWithCompressorByName(From, Path, FileFd::ReadOnly, Binary) == false)
+ return false;
+ if(From.IsCompressed() && From.FileSize() == 0)
+ return _error->Error(_("Empty files can't be valid archives"));
+ }
+ else
+ From.Open(Path, FileFd::ReadOnly, FileFd::Extension);
+ if (From.IsOpen() == false || From.Failed() == true)
+ return false;
+
+ FileFd To;
+ if (Itm->DestFile != "/dev/null" && Itm->DestFile != Path)
+ {
+ if (_config->FindB("Method::Compress", false) == false)
+ To.Open(Itm->DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Atomic, FileFd::Extension);
+ else if (OpenFileWithCompressorByName(To, Itm->DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Empty, Binary) == false)
+ return false;
+
+ if (To.IsOpen() == false || To.Failed() == true)
+ return false;
+ To.EraseOnFailure();
+ }
+
+ // Read data from source, generate checksums and write
+ Hashes Hash(Itm->ExpectedHashes);
+ bool Failed = false;
+ Res.Size = 0;
+ while (1)
+ {
+ unsigned char Buffer[APT_BUFFER_SIZE];
+ unsigned long long Count = 0;
+
+ if (!From.Read(Buffer,sizeof(Buffer),&Count))
+ {
+ if (To.IsOpen())
+ To.OpFail();
+ return false;
+ }
+ if (Count == 0)
+ break;
+ Res.Size += Count;
+
+ Hash.Add(Buffer,Count);
+ if (To.IsOpen() && To.Write(Buffer,Count) == false)
+ {
+ Failed = true;
+ break;
+ }
+ }
+
+ From.Close();
+ To.Close();
+
+ if (Failed == true)
+ return false;
+
+ if (TransferModificationTimes(Path.c_str(), Itm->DestFile.c_str(), Res.LastModified) == false)
+ return false;
+
+ // Return a Done response
+ Res.TakeHashes(Hash);
+
+ URIDone(Res);
+ return true;
+}
+ /*}}}*/
+
+int main(int, char *argv[])
+{
+ return StoreMethod(flNotDir(argv[0])).Run();
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-05 11:05:03 +0000