diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:00:48 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:00:48 +0000 |
commit | 851b6a097165af4d51c0db01b5e05256e5006896 (patch) | |
tree | 5f7c388ec894a7806c49a99f3bdb605d0b299a7c /methods | |
parent | Initial commit. (diff) | |
download | apt-upstream.tar.xz apt-upstream.zip |
Adding upstream version 2.6.1.upstream/2.6.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r-- | methods/CMakeLists.txt | 36 | ||||
-rw-r--r-- | methods/aptmethod.h | 595 | ||||
-rw-r--r-- | methods/basehttp.cc | 957 | ||||
-rw-r--r-- | methods/basehttp.h | 180 | ||||
-rw-r--r-- | methods/cdrom.cc | 289 | ||||
-rw-r--r-- | methods/connect.cc | 1053 | ||||
-rw-r--r-- | methods/connect.h | 50 | ||||
-rw-r--r-- | methods/copy.cc | 94 | ||||
-rw-r--r-- | methods/file.cc | 133 | ||||
-rw-r--r-- | methods/ftp.cc | 1188 | ||||
-rw-r--r-- | methods/ftp.h | 91 | ||||
-rw-r--r-- | methods/gpgv.cc | 597 | ||||
-rw-r--r-- | methods/http.cc | 1053 | ||||
-rw-r--r-- | methods/http.h | 142 | ||||
-rw-r--r-- | methods/mirror.cc | 414 | ||||
-rw-r--r-- | methods/rfc2553emu.cc | 244 | ||||
-rw-r--r-- | methods/rfc2553emu.h | 112 | ||||
-rw-r--r-- | methods/rred.cc | 885 | ||||
-rw-r--r-- | methods/rsh.cc | 561 | ||||
-rw-r--r-- | methods/rsh.h | 75 | ||||
-rw-r--r-- | methods/store.cc | 147 |
21 files changed, 8896 insertions, 0 deletions
diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt new file mode 100644 index 0000000..a5a3602 --- /dev/null +++ b/methods/CMakeLists.txt @@ -0,0 +1,36 @@ +# Create the executable targets +include_directories($<$<BOOL:${SECCOMP_FOUND}>:${SECCOMP_INCLUDE_DIR}>) +link_libraries(apt-pkg $<$<BOOL:${SECCOMP_FOUND}>:${SECCOMP_LIBRARIES}>) + +add_library(connectlib OBJECT connect.cc rfc2553emu.cc) + +add_executable(file file.cc) +add_executable(copy copy.cc) +add_executable(store store.cc) +add_executable(gpgv gpgv.cc) +add_executable(cdrom cdrom.cc) +add_executable(http http.cc basehttp.cc $<TARGET_OBJECTS:connectlib>) +add_executable(mirror mirror.cc) +add_executable(ftp ftp.cc $<TARGET_OBJECTS:connectlib>) +add_executable(rred rred.cc) +add_executable(rsh rsh.cc) + +target_compile_definitions(connectlib PRIVATE ${GNUTLS_DEFINITIONS}) +target_include_directories(connectlib PRIVATE ${GNUTLS_INCLUDE_DIR}) +target_include_directories(http PRIVATE $<$<BOOL:${SYSTEMD_FOUND}>:${SYSTEMD_INCLUDE_DIRS}>) + +# Additional libraries to link against for networked stuff +target_link_libraries(http ${GNUTLS_LIBRARIES} $<$<BOOL:${SYSTEMD_FOUND}>:${SYSTEMD_LIBRARIES}>) +target_link_libraries(ftp ${GNUTLS_LIBRARIES}) + +target_link_libraries(rred apt-private) + +# Install the library +install(TARGETS file copy store gpgv cdrom http ftp rred rsh mirror + RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/apt/methods) + +add_links(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods mirror mirror+ftp mirror+http mirror+https mirror+file mirror+copy) +add_links(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods rsh ssh) + + +add_links(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods http https) diff --git a/methods/aptmethod.h b/methods/aptmethod.h new file mode 100644 index 0000000..afc761c --- /dev/null +++ b/methods/aptmethod.h @@ -0,0 +1,595 @@ +#ifndef APT_APTMETHOD_H +#define APT_APTMETHOD_H + +#include "config.h" + +#include <apt-pkg/acquire-method.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/netrc.h> +#include <apt-pkg/strutl.h> + +#include <algorithm> +#include <locale> +#include <memory> +#include <string> +#include <vector> + +#include <stdlib.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> + +#include <apti18n.h> + +#ifdef HAVE_SECCOMP +#include <signal.h> + +#include <seccomp.h> +#endif + +enum class ResultState +{ + TRANSIENT_ERROR, + FATAL_ERROR, + SUCCESSFUL +}; + +static bool hasDoubleColon(std::string const &n) +{ + return n.find("::") != std::string::npos; +} + +class aptConfigWrapperForMethods +{ +protected: + std::vector<std::string> methodNames; +public: + void setPostfixForMethodNames(char const * const postfix) APT_NONNULL(2) + { + methodNames.erase(std::remove_if(methodNames.begin(), methodNames.end(), hasDoubleColon), methodNames.end()); + decltype(methodNames) toAdd; + for (auto && name: methodNames) + toAdd.emplace_back(name + "::" + postfix); + std::move(toAdd.begin(), toAdd.end(), std::back_inserter(methodNames)); + } + + bool DebugEnabled() const + { + if (methodNames.empty()) + return false; + auto const sni = std::find_if_not(methodNames.crbegin(), methodNames.crend(), hasDoubleColon); + if (unlikely(sni == methodNames.crend())) + return false; + auto const ln = methodNames[methodNames.size() - 1]; + // worst case: all three are the same + std::string confln, confsn, confpn; + strprintf(confln, "Debug::Acquire::%s", ln.c_str()); + strprintf(confsn, "Debug::Acquire::%s", sni->c_str()); + auto const pni = sni->substr(0, sni->find('+')); + strprintf(confpn, "Debug::Acquire::%s", pni.c_str()); + return _config->FindB(confln,_config->FindB(confsn, _config->FindB(confpn, false))); + } + std::string ConfigFind(char const * const postfix, std::string const &defValue) const APT_NONNULL(2) + { + for (auto name = methodNames.rbegin(); name != methodNames.rend(); ++name) + { + std::string conf; + strprintf(conf, "Acquire::%s::%s", name->c_str(), postfix); + auto value = _config->Find(conf); + if (not value.empty()) + return value; + } + return defValue; + } + std::string ConfigFind(std::string const &postfix, std::string const &defValue) const + { + return ConfigFind(postfix.c_str(), defValue); + } + bool ConfigFindB(char const * const postfix, bool const defValue) const APT_NONNULL(2) + { + return StringToBool(ConfigFind(postfix, defValue ? "yes" : "no"), defValue); + } + int ConfigFindI(char const * const postfix, int const defValue) const APT_NONNULL(2) + { + char *End; + std::string const value = ConfigFind(postfix, ""); + auto const Res = strtol(value.c_str(), &End, 0); + if (value.c_str() == End) + return defValue; + return Res; + } + + explicit aptConfigWrapperForMethods(std::string const &name) : methodNames{{name}} {} + explicit aptConfigWrapperForMethods(std::vector<std::string> &&names) : methodNames{std::move(names)} {} +}; + +class aptMethod : public pkgAcqMethod, public aptConfigWrapperForMethods +{ +protected: + std::string const Binary; + unsigned long SeccompFlags; + enum Seccomp + { + BASE = (1 << 1), + NETWORK = (1 << 2), + DIRECTORY = (1 << 3), + }; + + public: + virtual bool Configuration(std::string Message) APT_OVERRIDE + { + if (pkgAcqMethod::Configuration(Message) == false) + return false; + + std::string const conf = std::string("Binary::") + Binary; + _config->MoveSubTree(conf.c_str(), NULL); + + DropPrivsOrDie(); + if (LoadSeccomp() == false) + return false; + + return true; + } + + bool RunningInQemu(void) + { + int status; + pid_t pid; + + pid = fork(); + if (pid == 0) + { + close(0); + close(1); + close(2); + setenv("QEMU_VERSION", "meow", 1); + char path[] = LIBEXEC_DIR "/apt-helper"; + char *const argv[] = {path, NULL}; + execv(argv[0], argv); + _exit(255); + } + + // apt-helper is supposed to exit with an error. If it exited with 0, + // qemu-user had problems with QEMU_VERSION and returned 0 => running in + // qemu-user. + + if (waitpid(pid, &status, 0) == pid && WIFEXITED(status) && WEXITSTATUS(status) == 0) + return true; + + return false; + } + + bool LoadSeccomp() + { +#ifdef HAVE_SECCOMP + int rc; + scmp_filter_ctx ctx = NULL; + + if (SeccompFlags == 0) + return true; + + if (_config->FindB("APT::Sandbox::Seccomp", false) == false) + return true; + + if (RunningInQemu() == true) + { + Warning("Running in qemu-user, not using seccomp"); + return true; + } + + ctx = seccomp_init(SCMP_ACT_TRAP); + if (ctx == NULL) + return _error->FatalE("HttpMethod::Configuration", "Cannot init seccomp"); + +#define ALLOW(what) \ + if ((rc = seccomp_rule_add(ctx, SCMP_ACT_ALLOW, SCMP_SYS(what), 0))) \ + return _error->FatalE("HttpMethod::Configuration", "Cannot allow %s: %s", #what, strerror(-rc)); + + for (auto &custom : _config->FindVector("APT::Sandbox::Seccomp::Trap")) + { + if ((rc = seccomp_rule_add(ctx, SCMP_ACT_TRAP, seccomp_syscall_resolve_name(custom.c_str()), 0))) + return _error->FatalE("HttpMethod::Configuration", "Cannot trap %s: %s", custom.c_str(), strerror(-rc)); + } + + ALLOW(access); + ALLOW(arch_prctl); + ALLOW(brk); + ALLOW(chmod); + ALLOW(chown); + ALLOW(chown32); + ALLOW(clock_getres); + ALLOW(clock_getres_time64); + ALLOW(clock_gettime); + ALLOW(clock_gettime64); + ALLOW(clock_nanosleep); + ALLOW(clock_nanosleep_time64); + ALLOW(close); + ALLOW(creat); + ALLOW(dup); + ALLOW(dup2); + ALLOW(dup3); + ALLOW(exit); + ALLOW(exit_group); + ALLOW(faccessat); + ALLOW(fchmod); + ALLOW(fchmodat); + ALLOW(fchown); + ALLOW(fchown32); + ALLOW(fchownat); + ALLOW(fcntl); + ALLOW(fcntl64); + ALLOW(fdatasync); + ALLOW(flock); + ALLOW(fstat); + ALLOW(fstat64); + ALLOW(fstatat64); + ALLOW(fstatfs); + ALLOW(fstatfs64); + ALLOW(fsync); + ALLOW(ftime); + ALLOW(ftruncate); + ALLOW(ftruncate64); + ALLOW(futex); + ALLOW(futex_time64); + ALLOW(futimesat); + ALLOW(getegid); + ALLOW(getegid32); + ALLOW(geteuid); + ALLOW(geteuid32); + ALLOW(getgid); + ALLOW(getgid32); + ALLOW(getgroups); + ALLOW(getgroups32); + ALLOW(getpeername); + ALLOW(getpgid); + ALLOW(getpgrp); + ALLOW(getpid); + ALLOW(getppid); + ALLOW(getrandom); + ALLOW(getresgid); + ALLOW(getresgid32); + ALLOW(getresuid); + ALLOW(getresuid32); + ALLOW(getrlimit); + ALLOW(get_robust_list); + ALLOW(getrusage); + ALLOW(gettid); + ALLOW(gettimeofday); + ALLOW(getuid); + ALLOW(getuid32); + ALLOW(ioctl); + ALLOW(lchown); + ALLOW(lchown32); + ALLOW(_llseek); + ALLOW(lseek); + ALLOW(lstat); + ALLOW(lstat64); + ALLOW(madvise); + ALLOW(mmap); + ALLOW(mmap2); + ALLOW(mprotect); + ALLOW(mremap); + ALLOW(msync); + ALLOW(munmap); + ALLOW(nanosleep); + ALLOW(newfstatat); + ALLOW(_newselect); + ALLOW(oldfstat); + ALLOW(oldlstat); + ALLOW(oldolduname); + ALLOW(oldstat); + ALLOW(olduname); + ALLOW(open); + ALLOW(openat); + ALLOW(pipe); + ALLOW(pipe2); + ALLOW(poll); + ALLOW(ppoll); + ALLOW(ppoll_time64); + ALLOW(prctl); + ALLOW(prlimit64); + ALLOW(pselect6); + ALLOW(pselect6_time64); + ALLOW(read); + ALLOW(readv); + ALLOW(rename); + ALLOW(renameat); + ALLOW(renameat2); + ALLOW(restart_syscall); + ALLOW(rt_sigaction); + ALLOW(rt_sigpending); + ALLOW(rt_sigprocmask); + ALLOW(rt_sigqueueinfo); + ALLOW(rt_sigreturn); + ALLOW(rt_sigsuspend); + ALLOW(rt_sigtimedwait); + ALLOW(sched_yield); + ALLOW(select); + ALLOW(set_robust_list); + ALLOW(sigaction); + ALLOW(sigpending); + ALLOW(sigprocmask); + ALLOW(sigreturn); + ALLOW(sigsuspend); + ALLOW(stat); + ALLOW(stat64); + ALLOW(statfs); + ALLOW(statfs64); +#ifdef __NR_statx + ALLOW(statx); +#endif + ALLOW(sync); + ALLOW(syscall); + ALLOW(sysinfo); + ALLOW(time); + ALLOW(truncate); + ALLOW(truncate64); + ALLOW(ugetrlimit); + ALLOW(umask); + ALLOW(uname); + ALLOW(unlink); + ALLOW(unlinkat); + ALLOW(utime); + ALLOW(utimensat); + ALLOW(utimensat_time64); + ALLOW(utimes); + ALLOW(write); + ALLOW(writev); + + if ((SeccompFlags & Seccomp::NETWORK) != 0) + { + ALLOW(bind); + ALLOW(connect); + ALLOW(getsockname); + ALLOW(getsockopt); + ALLOW(recv); + ALLOW(recvfrom); + ALLOW(recvmmsg); + ALLOW(recvmmsg_time64); + ALLOW(recvmsg); + ALLOW(send); + ALLOW(sendmmsg); + ALLOW(sendmsg); + ALLOW(sendto); + ALLOW(setsockopt); + ALLOW(shutdown); + ALLOW(socket); + ALLOW(socketcall); + } + + if ((SeccompFlags & Seccomp::DIRECTORY) != 0) + { + ALLOW(readdir); + ALLOW(getdents); + ALLOW(getdents64); + } + + if (getenv("FAKED_MODE")) + { + ALLOW(semop); + ALLOW(semget); + ALLOW(msgsnd); + ALLOW(msgrcv); + ALLOW(msgget); + ALLOW(msgctl); + ALLOW(ipc); + } + + for (auto &custom : _config->FindVector("APT::Sandbox::Seccomp::Allow")) + { + if ((rc = seccomp_rule_add(ctx, SCMP_ACT_ALLOW, seccomp_syscall_resolve_name(custom.c_str()), 0))) + return _error->FatalE("aptMethod::Configuration", "Cannot allow %s: %s", custom.c_str(), strerror(-rc)); + } + +#undef ALLOW + + rc = seccomp_load(ctx); + if (rc == -EINVAL) + { + std::string msg; + strprintf(msg, "aptMethod::Configuration: could not load seccomp policy: %s", strerror(-rc)); + Warning(std::move(msg)); + } + else if (rc != 0) + return _error->FatalE("aptMethod::Configuration", "could not load seccomp policy: %s", strerror(-rc)); + + if (_config->FindB("APT::Sandbox::Seccomp::Print", true)) + { + struct sigaction action; + memset(&action, 0, sizeof(action)); + sigemptyset(&action.sa_mask); + action.sa_sigaction = [](int, siginfo_t *info, void *) { + // Formats a number into a 10 digit ASCII string + char buffer[10]; + int number = info->si_syscall; + + for (int i = sizeof(buffer) - 1; i >= 0; i--) + { + buffer[i] = (number % 10) + '0'; + number /= 10; + } + + constexpr const char *str1 = "\n **** Seccomp prevented execution of syscall "; + constexpr const char *str2 = " on architecture "; + constexpr const char *str3 = " ****\n"; +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-result" +#endif + write(2, str1, strlen(str1)); + write(2, buffer, sizeof(buffer)); + write(2, str2, strlen(str2)); + write(2, COMMON_ARCH, strlen(COMMON_ARCH)); + write(2, str3, strlen(str3)); +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + _exit(31); + }; + action.sa_flags = SA_SIGINFO; + + sigaction(SIGSYS, &action, nullptr); + } +#endif + return true; + } + + bool CalculateHashes(FetchItem const * const Itm, FetchResult &Res) const APT_NONNULL(2) + { + Hashes Hash(Itm->ExpectedHashes); + FileFd Fd; + if (Fd.Open(Res.Filename, FileFd::ReadOnly) == false || Hash.AddFD(Fd) == false) + return false; + Res.TakeHashes(Hash); + return true; + } + + void Warning(std::string &&msg) + { + std::unordered_map<std::string, std::string> fields; + if (Queue != 0) + fields.emplace("URI", Queue->Uri); + else + fields.emplace("URI", "<UNKNOWN>"); + if (not UsedMirror.empty()) + fields.emplace("UsedMirror", UsedMirror); + fields.emplace("Message", std::move(msg)); + SendMessage("104 Warning", std::move(fields)); + } + + bool TransferModificationTimes(char const * const From, char const * const To, time_t &LastModified) APT_NONNULL(2, 3) + { + if (strcmp(To, "/dev/null") == 0) + return true; + + struct stat Buf2; + if (lstat(To, &Buf2) != 0 || S_ISLNK(Buf2.st_mode)) + return true; + + struct stat Buf; + if (stat(From, &Buf) != 0) + return _error->Errno("stat",_("Failed to stat")); + + // we don't use utimensat here for compatibility reasons: #738567 + struct timeval times[2]; + times[0].tv_sec = Buf.st_atime; + LastModified = times[1].tv_sec = Buf.st_mtime; + times[0].tv_usec = times[1].tv_usec = 0; + if (utimes(To, times) != 0) + return _error->Errno("utimes",_("Failed to set modification time")); + return true; + } + + // This is a copy of #pkgAcqMethod::Dequeue which is private & hidden + void Dequeue() + { + FetchItem const *const Tmp = Queue; + Queue = Queue->Next; + if (Tmp == QueueBack) + QueueBack = Queue; + delete Tmp; + } + static std::string URIEncode(std::string const &part) + { + // The "+" is encoded as a workaround for an S3 bug (LP#1003633 and LP#1086997) + return QuoteString(part, _config->Find("Acquire::URIEncode", "+~ ").c_str()); + } + + static std::string DecodeSendURI(std::string const &part) + { + if (_config->FindB("Acquire::Send-URI-Encoded", false)) + return DeQuoteString(part); + return part; + } + + aptMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3) + : pkgAcqMethod(Ver, Flags), aptConfigWrapperForMethods(Binary), Binary(std::move(Binary)), SeccompFlags(0) + { + try { + std::locale::global(std::locale("")); + } catch (...) { + setlocale(LC_ALL, ""); + } + } +}; +class aptAuthConfMethod : public aptMethod +{ + std::vector<std::unique_ptr<FileFd>> authconfs; + + public: + virtual bool Configuration(std::string Message) APT_OVERRIDE + { + if (pkgAcqMethod::Configuration(Message) == false) + return false; + + std::string const conf = std::string("Binary::") + Binary; + _config->MoveSubTree(conf.c_str(), NULL); + + // ignore errors with opening the auth file as it doesn't need to exist + _error->PushToStack(); + auto const netrc = _config->FindFile("Dir::Etc::netrc"); + if (netrc.empty() == false) + { + authconfs.emplace_back(new FileFd()); + authconfs.back()->Open(netrc, FileFd::ReadOnly); + } + + auto const netrcparts = _config->FindDir("Dir::Etc::netrcparts"); + if (netrcparts.empty() == false) + { + for (auto &&netrcpart : GetListOfFilesInDir(netrcparts, "conf", true, true)) + { + authconfs.emplace_back(new FileFd()); + authconfs.back()->Open(netrcpart, FileFd::ReadOnly); + } + } + _error->RevertToStack(); + + DropPrivsOrDie(); + + if (LoadSeccomp() == false) + return false; + + return true; + } + + bool MaybeAddAuthTo(URI &uri) + { + bool result = true; + + if (uri.User.empty() == false || uri.Password.empty() == false) + return true; + + _error->PushToStack(); + for (auto &authconf : authconfs) + { + if (authconf->IsOpen() == false) + continue; + if (authconf->Seek(0) == false) + { + result = false; + continue; + } + + result &= MaybeAddAuth(*authconf, uri); + } + + while (not _error->empty()) + { + std::string message; + _error->PopMessage(message); + Warning(std::move(message)); + } + _error->RevertToStack(); + + return result; + } + + aptAuthConfMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3) + : aptMethod(std::move(Binary), Ver, Flags) {} +}; +#endif diff --git a/methods/basehttp.cc b/methods/basehttp.cc new file mode 100644 index 0000000..9ed3081 --- /dev/null +++ b/methods/basehttp.cc @@ -0,0 +1,957 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + HTTP and HTTPS share a lot of common code and these classes are + exactly the dumping ground for this common code + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/debversion.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/strutl.h> + +#include <iostream> +#include <limits> +#include <map> +#include <string> +#include <string_view> +#include <vector> +#include <ctype.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <time.h> +#include <unistd.h> + +#include "basehttp.h" + +#include <apti18n.h> + /*}}}*/ +using namespace std; + +string BaseHttpMethod::FailFile; +int BaseHttpMethod::FailFd = -1; +time_t BaseHttpMethod::FailTime = 0; + +// Number of successful requests in a pipeline needed to continue +// pipelining after a connection reset. +constexpr int PIPELINE_MIN_SUCCESSFUL_ANSWERS_TO_CONTINUE = 3; + +// ServerState::RunHeaders - Get the headers before the data /*{{{*/ +// --------------------------------------------------------------------- +/* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header + parse error occurred */ +ServerState::RunHeadersResult ServerState::RunHeaders(RequestState &Req, + const std::string &Uri) +{ + Owner->Status(_("Waiting for headers")); + do + { + string Data; + if (ReadHeaderLines(Data) == false) + continue; + + if (Owner->Debug == true) + clog << "Answer for: " << Uri << endl << Data; + + for (string::const_iterator I = Data.begin(); I < Data.end(); ++I) + { + string::const_iterator J = I; + for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J); + if (Req.HeaderLine(string(I,J)) == false) + return RUN_HEADERS_PARSE_ERROR; + I = J; + } + + // 100 Continue is a Nop... + if (Req.Result == 100) + continue; + + // Tidy up the connection persistence state. + if (Req.Encoding == RequestState::Closes && Req.haveContent == HaveContent::TRI_TRUE) + Persistent = false; + + return RUN_HEADERS_OK; + } while (LoadNextResponse(false, Req) == ResultState::SUCCESSFUL); + + return RUN_HEADERS_IO_ERROR; +} + /*}}}*/ +bool RequestState::HeaderLine(string const &Line) /*{{{*/ +{ + if (Line.empty() == true) + return true; + + if (Result == 0 && Line.size() > 4 && stringcasecmp(Line.data(), Line.data() + 4, "HTTP") == 0) + { + // Evil servers return no version + if (Line[4] == '/') + { + int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code); + if (elements == 3) + { + Code[0] = '\0'; + if (Owner != NULL && Owner->Debug == true) + clog << "HTTP server doesn't give Reason-Phrase for " << std::to_string(Result) << std::endl; + } + else if (elements != 4) + return _error->Error(_("The HTTP server sent an invalid reply header")); + } + else + { + Major = 0; + Minor = 9; + if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2) + return _error->Error(_("The HTTP server sent an invalid reply header")); + } + auto const CodeLen = strlen(Code); + auto const CodeEnd = std::remove_if(Code, Code + CodeLen, [](char c) { return isprint(c) == 0; }); + *CodeEnd = '\0'; + + /* Check the HTTP response header to get the default persistence + state. */ + if (Major < 1) + Server->Persistent = false; + else + { + if (Major == 1 && Minor == 0) + { + Server->Persistent = false; + } + else + { + Server->Persistent = true; + if (Server->PipelineAllowed) + Server->Pipeline = true; + } + } + + return true; + } + + // Blah, some servers use "connection:closes", evil. + // and some even send empty header fields… + string::size_type Pos = Line.find(':'); + if (Pos == string::npos) + return _error->Error(_("Bad header line")); + ++Pos; + + // Parse off any trailing spaces between the : and the next word. + string::size_type Pos2 = Pos; + while (Pos2 < Line.length() && isspace_ascii(Line[Pos2]) != 0) + Pos2++; + + string const Tag(Line,0,Pos); + string const Val(Line,Pos2); + + if (stringcasecmp(Tag,"Content-Length:") == 0) + { + auto ContentLength = strtoull(Val.c_str(), NULL, 10); + if (ContentLength == 0) + { + haveContent = HaveContent::TRI_FALSE; + return true; + } + if (Encoding == Closes) + Encoding = Stream; + haveContent = HaveContent::TRI_TRUE; + + unsigned long long * DownloadSizePtr = &DownloadSize; + if (Result == 416 || (Result >= 300 && Result < 400)) + DownloadSizePtr = &JunkSize; + + *DownloadSizePtr = ContentLength; + if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max()) + return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header")); + else if (*DownloadSizePtr == 0) + haveContent = HaveContent::TRI_FALSE; + + // On partial content (206) the Content-Length less than the real + // size, so do not set it here but leave that to the Content-Range + // header instead + if(Result != 206 && TotalFileSize == 0) + TotalFileSize = DownloadSize; + + return true; + } + + if (stringcasecmp(Tag,"Content-Type:") == 0) + { + if (haveContent == HaveContent::TRI_UNKNOWN) + haveContent = HaveContent::TRI_TRUE; + return true; + } + + // The Content-Range field only has a meaning in HTTP/1.1 for the + // 206 (Partial Content) and 416 (Range Not Satisfiable) responses + // according to RFC7233 "Range Requests", §4.2, so only consider it + // for such responses. + if ((Result == 416 || Result == 206) && stringcasecmp(Tag,"Content-Range:") == 0) + { + if (haveContent == HaveContent::TRI_UNKNOWN) + haveContent = HaveContent::TRI_TRUE; + + // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416 + if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1) + ; // we got the expected filesize which is all we wanted + else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2) + return _error->Error(_("The HTTP server sent an invalid Content-Range header")); + if (StartPos > TotalFileSize) + return _error->Error(_("This HTTP server has broken range support")); + + // figure out what we will download + DownloadSize = TotalFileSize - StartPos; + return true; + } + + if (stringcasecmp(Tag,"Transfer-Encoding:") == 0) + { + if (haveContent == HaveContent::TRI_UNKNOWN) + haveContent = HaveContent::TRI_TRUE; + if (stringcasecmp(Val,"chunked") == 0) + Encoding = Chunked; + return true; + } + + if (stringcasecmp(Tag,"Connection:") == 0) + { + if (stringcasecmp(Val,"close") == 0) + { + Server->Persistent = false; + Server->Pipeline = false; + /* Some servers send error pages (as they are dynamically generated) + for simplicity via a connection close instead of e.g. chunked, + so assuming an always closing server only if we get a file + close */ + if (Result >= 200 && Result < 300 && Server->PipelineAnswersReceived < PIPELINE_MIN_SUCCESSFUL_ANSWERS_TO_CONTINUE) + { + Server->PipelineAllowed = false; + Server->PipelineAnswersReceived = 0; + } + } + else if (stringcasecmp(Val,"keep-alive") == 0) + Server->Persistent = true; + return true; + } + + if (stringcasecmp(Tag,"Last-Modified:") == 0) + { + if (RFC1123StrToTime(Val, Date) == false) + return _error->Error(_("Unknown date format")); + return true; + } + + if (stringcasecmp(Tag,"Location:") == 0) + { + Location = Val; + return true; + } + + if (Server->RangesAllowed && stringcasecmp(Tag, "Accept-Ranges:") == 0) + { + std::string ranges = ',' + Val + ','; + ranges.erase(std::remove(ranges.begin(), ranges.end(), ' '), ranges.end()); + if (ranges.find(",bytes,") == std::string::npos) + Server->RangesAllowed = false; + return true; + } + + if (Server->RangesAllowed && stringcasecmp(Tag, "Via:") == 0) + { + auto const parts = VectorizeString(Val, ' '); + std::string_view const varnish{"(Varnish/"}; + if (parts.size() != 3 || parts[1] != "varnish" || parts[2].empty() || + not APT::String::Startswith(parts[2], std::string{varnish}) || + parts[2].back() != ')') + return true; + auto const version = parts[2].substr(varnish.length(), parts[2].length() - (varnish.length() + 1)); + if (version.empty()) + return true; + std::string_view const varnishsupport{"6.4~"}; + if (debVersioningSystem::CmpFragment(version.data(), version.data() + version.length(), + varnishsupport.begin(), varnishsupport.end()) < 0) + Server->RangesAllowed = false; + return true; + } + + return true; +} + /*}}}*/ +// ServerState::ServerState - Constructor /*{{{*/ +ServerState::ServerState(URI Srv, BaseHttpMethod *Owner) : + ServerName(Srv), TimeOut(30), Owner(Owner) +{ + Reset(); +} + /*}}}*/ +bool RequestState::AddPartialFileToHashes(FileFd &File) /*{{{*/ +{ + File.Truncate(StartPos); + return Server->GetHashes()->AddFD(File, StartPos); +} + /*}}}*/ +void ServerState::Reset() /*{{{*/ +{ + Persistent = false; + Pipeline = false; + PipelineAllowed = true; + PipelineAnswersReceived = 0; +} + /*}}}*/ + +// BaseHttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/ +// --------------------------------------------------------------------- +/* We look at the header data we got back from the server and decide what + to do. Returns DealWithHeadersResult (see http.h for details). + */ +static std::string fixURIEncoding(std::string const &part) +{ + // if the server sends a space this is not an encoded URI + // so other clients seem to encode it and we do it as well + if (part.find_first_of(" ") != std::string::npos) + return aptMethod::URIEncode(part); + return part; +} +BaseHttpMethod::DealWithHeadersResult +BaseHttpMethod::DealWithHeaders(FetchResult &Res, RequestState &Req) +{ + // Not Modified + if (Req.Result == 304) + { + RemoveFile("server", Queue->DestFile); + Res.IMSHit = true; + Res.LastModified = Queue->LastModified; + Res.Size = 0; + return IMS_HIT; + } + + /* Note that it is only OK for us to treat all redirection the same + because we *always* use GET, not other HTTP methods. + Codes not mentioned are handled as errors later as required by the + HTTP spec to handle unknown codes the same as the x00 code. */ + constexpr unsigned int RedirectCodes[] = { + 301, // Moved Permanently + 302, // Found + 303, // See Other + 307, // Temporary Redirect + 308, // Permanent Redirect + }; + if (AllowRedirect && std::find(std::begin(RedirectCodes), std::end(RedirectCodes), Req.Result) != std::end(RedirectCodes)) + { + if (Req.Location.empty() == true) + ; + else if (Req.Location[0] == '/' && Queue->Uri.empty() == false) + { + URI Uri(Queue->Uri); + if (Uri.Host.empty() == false) + NextURI = URI::SiteOnly(Uri); + else + NextURI.clear(); + if (_config->FindB("Acquire::Send-URI-Encoded", false)) + NextURI.append(fixURIEncoding(Req.Location)); + else + NextURI.append(DeQuoteString(Req.Location)); + if (Queue->Uri == NextURI) + { + SetFailReason("RedirectionLoop"); + _error->Error("Redirection loop encountered"); + if (Req.haveContent == HaveContent::TRI_TRUE) + return ERROR_WITH_CONTENT_PAGE; + return ERROR_UNRECOVERABLE; + } + return TRY_AGAIN_OR_REDIRECT; + } + else + { + bool const SendURIEncoded = _config->FindB("Acquire::Send-URI-Encoded", false); + if (not SendURIEncoded) + Req.Location = DeQuoteString(Req.Location); + URI tmpURI(Req.Location); + if (SendURIEncoded) + tmpURI.Path = fixURIEncoding(tmpURI.Path); + if (tmpURI.Access.find('+') != std::string::npos) + { + _error->Error("Server tried to trick us into using a specific implementation: %s", tmpURI.Access.c_str()); + if (Req.haveContent == HaveContent::TRI_TRUE) + return ERROR_WITH_CONTENT_PAGE; + return ERROR_UNRECOVERABLE; + } + NextURI = tmpURI; + URI Uri(Queue->Uri); + if (Binary.find('+') != std::string::npos) + { + auto base = Binary.substr(0, Binary.find('+')); + if (base != tmpURI.Access) + { + tmpURI.Access = base + '+' + tmpURI.Access; + if (tmpURI.Access == Binary) + { + std::swap(tmpURI.Access, Uri.Access); + NextURI = tmpURI; + std::swap(tmpURI.Access, Uri.Access); + } + else + NextURI = tmpURI; + } + } + if (Queue->Uri == NextURI) + { + SetFailReason("RedirectionLoop"); + _error->Error("Redirection loop encountered"); + if (Req.haveContent == HaveContent::TRI_TRUE) + return ERROR_WITH_CONTENT_PAGE; + return ERROR_UNRECOVERABLE; + } + Uri.Access = Binary; + // same protocol redirects are okay + if (tmpURI.Access == Uri.Access) + return TRY_AGAIN_OR_REDIRECT; + // as well as http to https + else if ((Uri.Access == "http" || Uri.Access == "https+http") && tmpURI.Access == "https") + return TRY_AGAIN_OR_REDIRECT; + else + { + auto const tmpplus = tmpURI.Access.find('+'); + if (tmpplus != std::string::npos && tmpURI.Access.substr(tmpplus + 1) == "https") + { + auto const uriplus = Uri.Access.find('+'); + if (uriplus == std::string::npos) + { + if (Uri.Access == tmpURI.Access.substr(0, tmpplus)) // foo -> foo+https + return TRY_AGAIN_OR_REDIRECT; + } + else if (Uri.Access.substr(uriplus + 1) == "http" && + Uri.Access.substr(0, uriplus) == tmpURI.Access.substr(0, tmpplus)) // foo+http -> foo+https + return TRY_AGAIN_OR_REDIRECT; + } + } + _error->Error("Redirection from %s to '%s' is forbidden", Uri.Access.c_str(), NextURI.c_str()); + } + /* else pass through for error message */ + } + // the server is not supporting ranges as much as we would like. Retry without ranges + else if (not Server->RangesAllowed && (Req.Result == 416 || Req.Result == 206)) + { + RemoveFile("server", Queue->DestFile); + NextURI = Queue->Uri; + return TRY_AGAIN_OR_REDIRECT; + } + // retry after an invalid range response without partial data + else if (Req.Result == 416) + { + struct stat SBuf; + if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) + { + bool partialHit = false; + if (Queue->ExpectedHashes.usable() == true) + { + Hashes resultHashes(Queue->ExpectedHashes); + FileFd file(Queue->DestFile, FileFd::ReadOnly); + Req.TotalFileSize = file.FileSize(); + Req.Date = file.ModificationTime(); + resultHashes.AddFD(file); + HashStringList const hashList = resultHashes.GetHashStringList(); + partialHit = (Queue->ExpectedHashes == hashList); + } + else if ((unsigned long long)SBuf.st_size == Req.TotalFileSize) + partialHit = true; + if (partialHit == true) + { + // the file is completely downloaded, but was not moved + if (Req.haveContent == HaveContent::TRI_TRUE) + { + // nuke the sent error page + Server->RunDataToDevNull(Req); + Req.haveContent = HaveContent::TRI_FALSE; + } + Req.StartPos = Req.TotalFileSize; + Req.Result = 200; + } + else if (RemoveFile("server", Queue->DestFile)) + { + NextURI = Queue->Uri; + return TRY_AGAIN_OR_REDIRECT; + } + } + } + + /* We have a reply we don't handle. This should indicate a perm server + failure */ + if (Req.Result < 200 || Req.Result >= 300) + { + if (_error->PendingError() == false) + { + std::string err; + strprintf(err, "HttpError%u", Req.Result); + SetFailReason(err); + _error->Error("%u %s", Req.Result, Req.Code); + } + if (Req.haveContent == HaveContent::TRI_TRUE) + return ERROR_WITH_CONTENT_PAGE; + return ERROR_UNRECOVERABLE; + } + + // This is some sort of 2xx 'data follows' reply + Res.LastModified = Req.Date; + Res.Size = Req.TotalFileSize; + return FILE_IS_OPEN; +} + /*}}}*/ +// BaseHttpMethod::SigTerm - Handle a fatal signal /*{{{*/ +// --------------------------------------------------------------------- +/* This closes and timestamps the open file. This is necessary to get + resume behavior on user abort */ +void BaseHttpMethod::SigTerm(int) +{ + if (FailFd == -1) + _exit(100); + + struct timeval times[2]; + times[0].tv_sec = FailTime; + times[1].tv_sec = FailTime; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(FailFile.c_str(), times); + close(FailFd); + + _exit(100); +} + /*}}}*/ +// BaseHttpMethod::Fetch - Fetch an item /*{{{*/ +// --------------------------------------------------------------------- +/* This adds an item to the pipeline. We keep the pipeline at a fixed + depth. */ +bool BaseHttpMethod::Fetch(FetchItem *) +{ + if (Server == nullptr || QueueBack == nullptr) + return true; + + // If pipelining is disabled, we only queue 1 request + auto const AllowedDepth = Server->Pipeline ? PipelineDepth : 0; + // how deep is our pipeline currently? + decltype(PipelineDepth) CurrentDepth = 0; + for (FetchItem const *I = Queue; I != QueueBack; I = I->Next) + ++CurrentDepth; + if (CurrentDepth > AllowedDepth) + return true; + + do { + // Make sure we stick with the same server + if (Server->Comp(URI(QueueBack->Uri)) == false) + break; + + bool const UsableHashes = QueueBack->ExpectedHashes.usable(); + // if we have no hashes, do at most one such request + // as we can't fixup pipeling misbehaviors otherwise + if (CurrentDepth != 0 && UsableHashes == false) + break; + + if (UsableHashes && FileExists(QueueBack->DestFile)) + { + FileFd partial(QueueBack->DestFile, FileFd::ReadOnly); + Hashes wehave(QueueBack->ExpectedHashes); + if (QueueBack->ExpectedHashes.FileSize() == partial.FileSize()) + { + if (wehave.AddFD(partial) && + wehave.GetHashStringList() == QueueBack->ExpectedHashes) + { + FetchResult Res; + Res.Filename = QueueBack->DestFile; + Res.ResumePoint = QueueBack->ExpectedHashes.FileSize(); + URIStart(Res); + // move item to the start of the queue as URIDone will + // always dequeued the first item in the queue + if (Queue != QueueBack) + { + FetchItem *Prev = Queue; + for (; Prev->Next != QueueBack; Prev = Prev->Next) + /* look for the previous queue item */; + Prev->Next = QueueBack->Next; + QueueBack->Next = Queue; + Queue = QueueBack; + QueueBack = Prev->Next; + } + Res.TakeHashes(wehave); + URIDone(Res); + continue; + } + else + RemoveFile("Fetch-Partial", QueueBack->DestFile); + } + } + auto const Tmp = QueueBack; + QueueBack = QueueBack->Next; + SendReq(Tmp); + ++CurrentDepth; + } while (CurrentDepth <= AllowedDepth && QueueBack != nullptr); + + return true; +} + /*}}}*/ +// BaseHttpMethod::Loop - Main loop /*{{{*/ +int BaseHttpMethod::Loop() +{ + signal(SIGTERM,SigTerm); + signal(SIGINT,SigTerm); + + Server = 0; + + int FailCounter = 0; + while (1) + { + // We have no commands, wait for some to arrive + if (Queue == 0) + { + if (WaitFd(STDIN_FILENO) == false) + return 0; + } + + /* Run messages, we can accept 0 (no message) if we didn't + do a WaitFd above.. Otherwise the FD is closed. */ + int Result = Run(true); + if (Result != -1 && (Result != 0 || Queue == 0)) + { + if(FailReason.empty() == false || + ConfigFindB("DependOnSTDIN", true) == true) + return 100; + else + return 0; + } + + if (Queue == 0) + continue; + + // Connect to the server + if (Server == 0 || Server->Comp(URI(Queue->Uri)) == false) + { + if (!Queue->Proxy().empty()) + { + URI uri(Queue->Uri); + _config->Set("Acquire::" + uri.Access + "::proxy::" + uri.Host, Queue->Proxy()); + } + Server = CreateServerState(URI(Queue->Uri)); + setPostfixForMethodNames(::URI(Queue->Uri).Host.c_str()); + AllowRedirect = ConfigFindB("AllowRedirect", true); + PipelineDepth = ConfigFindI("Pipeline-Depth", 10); + Server->RangesAllowed = ConfigFindB("AllowRanges", true); + Debug = DebugEnabled(); + } + + /* If the server has explicitly said this is the last connection + then we pre-emptively shut down the pipeline and tear down + the connection. This will speed up HTTP/1.0 servers a tad + since we don't have to wait for the close sequence to + complete */ + if (Server->Persistent == false) + Server->Close(); + + // Reset the pipeline + if (Server->IsOpen() == false) { + QueueBack = Queue; + Server->PipelineAnswersReceived = 0; + } + + // Connect to the host + switch (Server->Open()) + { + case ResultState::FATAL_ERROR: + Fail(false); + Server = nullptr; + continue; + case ResultState::TRANSIENT_ERROR: + Fail(true); + Server = nullptr; + continue; + case ResultState::SUCCESSFUL: + break; + } + + // Fill the pipeline. + Fetch(0); + + RequestState Req(this, Server.get()); + // Fetch the next URL header data from the server. + switch (Server->RunHeaders(Req, Queue->Uri)) + { + case ServerState::RUN_HEADERS_OK: + break; + + // The header data is bad + case ServerState::RUN_HEADERS_PARSE_ERROR: + { + _error->Error(_("Bad header data")); + Fail(true); + Server->Close(); + RotateDNS(); + continue; + } + + // The server closed a connection during the header get.. + default: + case ServerState::RUN_HEADERS_IO_ERROR: + { + FailCounter++; + _error->Discard(); + Server->Close(); + Server->Pipeline = false; + Server->PipelineAllowed = false; + + if (FailCounter >= 2) + { + Fail(_("Connection failed"),true); + FailCounter = 0; + } + + RotateDNS(); + continue; + } + }; + + // Decide what to do. + FetchResult Res; + Res.Filename = Queue->DestFile; + switch (DealWithHeaders(Res, Req)) + { + // Ok, the file is Open + case FILE_IS_OPEN: + { + URIStart(Res); + + // Run the data + ResultState Result = ResultState::SUCCESSFUL; + + // ensure we don't fetch too much + // we could do "Server->MaximumSize = Queue->MaximumSize" here + // but that would break the clever pipeline messup detection + // so instead we use the size of the biggest item in the queue + Req.MaximumSize = FindMaximumObjectSizeInQueue(); + + if (Req.haveContent == HaveContent::TRI_TRUE) + { + /* If the server provides Content-Length we can figure out with it if + this satisfies any request we have made so far (in the pipeline). + If not we can kill the connection as whatever file the server is trying + to send to us would be rejected with a hashsum mismatch later or triggers + a maximum size error. We don't run the data to /dev/null as this can be MBs + of junk data we would waste bandwidth on and instead just close the connection + to reopen a fresh one which should be more cost/time efficient */ + if (Req.DownloadSize > 0) + { + decltype(Queue->ExpectedHashes.FileSize()) const filesize = Req.StartPos + Req.DownloadSize; + bool found = false; + for (FetchItem const *I = Queue; I != 0 && I != QueueBack; I = I->Next) + { + auto const fs = I->ExpectedHashes.FileSize(); + if (fs == 0 || fs == filesize) + { + found = true; + break; + } + } + if (found == false) + { + SetFailReason("MaximumSizeExceeded"); + _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"), + filesize, Queue->ExpectedHashes.FileSize()); + Result = ResultState::FATAL_ERROR; + } + } + if (Result == ResultState::SUCCESSFUL) + Result = Server->RunData(Req); + } + + /* If the server is sending back sizeless responses then fill in + the size now */ + if (Res.Size == 0) + Res.Size = Req.File.Size(); + + // Close the file, destroy the FD object and timestamp it + FailFd = -1; + Req.File.Close(); + + // Timestamp + struct timeval times[2]; + times[0].tv_sec = times[1].tv_sec = Req.Date; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(Queue->DestFile.c_str(), times); + + // Send status to APT + if (Result == ResultState::SUCCESSFUL) + { + Hashes * const resultHashes = Server->GetHashes(); + HashStringList const hashList = resultHashes->GetHashStringList(); + if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList) + { + // we did not get the expected hash… mhhh: + // could it be that server/proxy messed up pipelining? + FetchItem * BeforeI = Queue; + for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next) + { + if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList) + { + // yes, he did! Disable pipelining and rewrite queue + if (Server->Pipeline == true) + { + std::string msg; + strprintf(msg, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::Pipeline-Depth"); + Warning(std::move(msg)); + Server->Pipeline = false; + Server->PipelineAllowed = false; + // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well + } + Rename(Res.Filename, I->DestFile); + Res.Filename = I->DestFile; + BeforeI->Next = I->Next; + I->Next = Queue; + Queue = I; + break; + } + BeforeI = I; + } + } + if (Server->Pipeline == true) + { + Server->PipelineAnswersReceived++; + } + Res.TakeHashes(*resultHashes); + URIDone(Res); + } + else + { + if (not Server->IsOpen()) + { + // Reset the pipeline + QueueBack = Queue; + Server->PipelineAnswersReceived = 0; + } + + Server->Close(); + FailCounter = 0; + switch (Result) + { + case ResultState::TRANSIENT_ERROR: + Fail(true); + break; + case ResultState::FATAL_ERROR: + case ResultState::SUCCESSFUL: + Fail(false); + break; + } + } + break; + } + + // IMS hit + case IMS_HIT: + { + URIDone(Res); + break; + } + + // Hard server error, not found or something + case ERROR_UNRECOVERABLE: + { + Fail(); + break; + } + + // Hard internal error, kill the connection and fail + case ERROR_NOT_FROM_SERVER: + { + Fail(); + RotateDNS(); + Server->Close(); + break; + } + + // We need to flush the data, the header is like a 404 w/ error text + case ERROR_WITH_CONTENT_PAGE: + { + Server->RunDataToDevNull(Req); + constexpr unsigned int TransientCodes[] = { + 408, // Request Timeout + 429, // Too Many Requests + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 599, // Network Connect Timeout Error + }; + if (std::find(std::begin(TransientCodes), std::end(TransientCodes), Req.Result) != std::end(TransientCodes)) + Fail(true); + else + Fail(); + break; + } + + // Try again with a new URL + case TRY_AGAIN_OR_REDIRECT: + { + // Clear rest of response if there is content + if (Req.haveContent == HaveContent::TRI_TRUE) + Server->RunDataToDevNull(Req); + Redirect(NextURI); + break; + } + + default: + Fail(_("Internal error")); + break; + } + + FailCounter = 0; + } + + return 0; +} + /*}}}*/ +unsigned long long BaseHttpMethod::FindMaximumObjectSizeInQueue() const /*{{{*/ +{ + unsigned long long MaxSizeInQueue = 0; + for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next) + { + if (I->MaximumSize == 0) + return 0; + MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize); + } + return MaxSizeInQueue; +} + /*}}}*/ +BaseHttpMethod::BaseHttpMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) /*{{{*/ + : aptAuthConfMethod(std::move(Binary), Ver, Flags), Server(nullptr), + AllowRedirect(false), Debug(false), PipelineDepth(10) +{ +} + /*}}}*/ +bool BaseHttpMethod::Configuration(std::string Message) /*{{{*/ +{ + if (aptAuthConfMethod::Configuration(Message) == false) + return false; + + _config->CndSet("Acquire::tor::Proxy", + "socks5h://apt-transport-tor@127.0.0.1:9050"); + return true; +} + /*}}}*/ +bool BaseHttpMethod::AddProxyAuth(URI &Proxy, URI const &Server) /*{{{*/ +{ + MaybeAddAuthTo(Proxy); + if (std::find(methodNames.begin(), methodNames.end(), "tor") != methodNames.end() && + Proxy.User == "apt-transport-tor" && Proxy.Password.empty()) + { + std::string pass = Server.Host; + pass.erase(std::remove_if(pass.begin(), pass.end(), [](char const c) { return std::isalnum(c) == 0; }), pass.end()); + if (pass.length() > 255) + Proxy.Password = pass.substr(0, 255); + else + Proxy.Password = std::move(pass); + } + return true; +} + /*}}}*/ diff --git a/methods/basehttp.h b/methods/basehttp.h new file mode 100644 index 0000000..0f776b0 --- /dev/null +++ b/methods/basehttp.h @@ -0,0 +1,180 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Classes dealing with the abstraction of talking to a end via a text + protocol like HTTP (which is used by the http and https methods) + + ##################################################################### */ + /*}}}*/ + +#ifndef APT_SERVER_H +#define APT_SERVER_H + +#include "aptmethod.h" +#include <apt-pkg/fileutl.h> +#include <apt-pkg/strutl.h> + +#include <iostream> +#include <memory> +#include <string> +#include <time.h> + +using std::cout; +using std::endl; + +class Hashes; +class BaseHttpMethod; +struct ServerState; + +enum class HaveContent +{ + TRI_UNKNOWN, + TRI_FALSE, + TRI_TRUE, +}; +struct RequestState +{ + unsigned int Major = 0; + unsigned int Minor = 0; + unsigned int Result = 0; + char Code[360]; + + // total size of the usable content (aka: the file) + unsigned long long TotalFileSize = 0; + // size we actually download (can be smaller than Size if we have partial content) + unsigned long long DownloadSize = 0; + // size of junk content (aka: server error pages) + unsigned long long JunkSize = 0; + // The start of the data (for partial content) + unsigned long long StartPos = 0; + + unsigned long long MaximumSize = 0; + + time_t Date; + HaveContent haveContent = HaveContent::TRI_UNKNOWN; + + enum {Chunked,Stream,Closes} Encoding = Closes; + enum {Header, Data} State = Header; + std::string Location; + + FileFd File; + + BaseHttpMethod * const Owner; + ServerState * const Server; + + bool HeaderLine(std::string const &Line); + bool AddPartialFileToHashes(FileFd &File); + + RequestState(BaseHttpMethod * const Owner, ServerState * const Server) : + Owner(Owner), Server(Server) { time(&Date); Code[0] = '\0'; } +}; +struct ServerState +{ + bool Persistent; + bool PipelineAllowed; + bool RangesAllowed; + unsigned long PipelineAnswersReceived; + + bool Pipeline; + URI ServerName; + URI Proxy; + unsigned long TimeOut; + + protected: + BaseHttpMethod *Owner; + + virtual bool ReadHeaderLines(std::string &Data) = 0; + virtual ResultState LoadNextResponse(bool const ToFile, RequestState &Req) = 0; + + public: + + /** \brief Result of the header acquire */ + enum RunHeadersResult { + /** \brief Header ok */ + RUN_HEADERS_OK, + /** \brief IO error while retrieving */ + RUN_HEADERS_IO_ERROR, + /** \brief Parse error after retrieving */ + RUN_HEADERS_PARSE_ERROR + }; + /** \brief Get the headers before the data */ + RunHeadersResult RunHeaders(RequestState &Req, const std::string &Uri); + + bool Comp(URI Other) const {return Other.Access == ServerName.Access && Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; + virtual void Reset(); + virtual bool WriteResponse(std::string const &Data) = 0; + + /** \brief Transfer the data from the socket */ + virtual ResultState RunData(RequestState &Req) = 0; + virtual ResultState RunDataToDevNull(RequestState &Req) = 0; + + virtual ResultState Open() = 0; + virtual bool IsOpen() = 0; + virtual bool Close() = 0; + virtual bool InitHashes(HashStringList const &ExpectedHashes) = 0; + virtual ResultState Die(RequestState &Req) = 0; + virtual bool Flush(FileFd *const File, bool MustComplete = false) = 0; + virtual ResultState Go(bool ToFile, RequestState &Req) = 0; + virtual Hashes * GetHashes() = 0; + + ServerState(URI Srv, BaseHttpMethod *Owner); + virtual ~ServerState() {}; +}; + +class BaseHttpMethod : public aptAuthConfMethod +{ + protected: + virtual bool Fetch(FetchItem *) APT_OVERRIDE; + + std::unique_ptr<ServerState> Server; + std::string NextURI; + + bool AllowRedirect; + + // Find the biggest item in the fetch queue for the checking of the maximum + // size + unsigned long long FindMaximumObjectSizeInQueue() const APT_PURE; + + public: + bool Debug; + unsigned long PipelineDepth; + + /** \brief Result of the header parsing */ + enum DealWithHeadersResult { + /** \brief The file is open and ready */ + FILE_IS_OPEN, + /** \brief We got a IMS hit, the file has not changed */ + IMS_HIT, + /** \brief The server reported a unrecoverable error */ + ERROR_UNRECOVERABLE, + /** \brief The server reported a error with a error content page */ + ERROR_WITH_CONTENT_PAGE, + /** \brief An error on the client side */ + ERROR_NOT_FROM_SERVER, + /** \brief A redirect or retry request */ + TRY_AGAIN_OR_REDIRECT + }; + /** \brief Handle the retrieved header data */ + virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res, RequestState &Req); + + // In the event of a fatal signal this file will be closed and timestamped. + static std::string FailFile; + static int FailFd; + static time_t FailTime; + static APT_NORETURN void SigTerm(int); + + int Loop(); + + virtual void SendReq(FetchItem *Itm) = 0; + virtual std::unique_ptr<ServerState> CreateServerState(URI const &uri) = 0; + virtual void RotateDNS() = 0; + virtual bool Configuration(std::string Message) APT_OVERRIDE; + + bool AddProxyAuth(URI &Proxy, URI const &Server); + + BaseHttpMethod(std::string &&Binary, char const * const Ver,unsigned long const Flags); + virtual ~BaseHttpMethod() {}; +}; + +#endif diff --git a/methods/cdrom.cc b/methods/cdrom.cc new file mode 100644 index 0000000..f534a06 --- /dev/null +++ b/methods/cdrom.cc @@ -0,0 +1,289 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + CDROM URI method for APT + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/cdrom.h> +#include <apt-pkg/cdromutl.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/strutl.h> + +#include "aptmethod.h" + +#include <string> +#include <vector> +#include <sys/stat.h> + +#include <iostream> +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +class CDROMMethod : public aptMethod +{ + bool DatabaseLoaded; + bool Debug; + + ::Configuration Database; + string CurrentID; + string CDROM; + bool MountedByApt; + pkgUdevCdromDevices UdevCdroms; + + bool IsCorrectCD(URI want, string MountPath, string& NewID); + bool AutoDetectAndMount(const URI, string &NewID); + virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; + std::string GetID(std::string const &Name); + virtual void Exit() APT_OVERRIDE; + virtual bool Configuration(std::string Message) APT_OVERRIDE; + + public: + + CDROMMethod(); +}; + +// CDROMMethod::CDROMethod - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +CDROMMethod::CDROMMethod() : aptMethod("cdrom", "1.0",SingleInstance | LocalOnly | + SendConfig | NeedsCleanup | + Removable | SendURIEncoded), + DatabaseLoaded(false), + Debug(false), + MountedByApt(false) +{ +} + /*}}}*/ +// CDROMMethod::Exit - Unmount the disc if necessary /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void CDROMMethod::Exit() +{ + if (MountedByApt == true) + UnmountCdrom(CDROM); +} + /*}}}*/ +// CDROMMethod::GetID - Search the database for a matching string /*{{{*/ +// --------------------------------------------------------------------- +/* */ +std::string CDROMMethod::GetID(std::string const &Name) +{ + // Search for an ID + const Configuration::Item *Top = Database.Tree("CD"); + if (Top != 0) + Top = Top->Child; + + for (; Top != 0;) + { + if (Top->Value == Name) + return Top->Tag; + + Top = Top->Next; + } + return string(); +} + /*}}}*/ +// CDROMMethod::AutoDetectAndMount /*{{{*/ +// --------------------------------------------------------------------- +/* Modifies class variable CDROM to the mountpoint */ +bool CDROMMethod::AutoDetectAndMount(const URI Get, string &NewID) +{ + vector<struct CdromDevice> v = UdevCdroms.Scan(); + + // first check if its mounted somewhere already + for (unsigned int i=0; i < v.size(); i++) + { + if (v[i].Mounted) + { + if (Debug) + clog << "Checking mounted cdrom device " << v[i].DeviceName << endl; + if (IsCorrectCD(Get, v[i].MountPath, NewID)) + { + CDROM = v[i].MountPath; + return true; + } + } + } + + // we are not supposed to mount, exit + if (_config->FindB("APT::CDROM::NoMount",false) == true) + return false; + + // check if we have the mount point + string AptMountPoint = _config->FindDir("Dir::Media::MountPath"); + if (!FileExists(AptMountPoint)) + mkdir(AptMountPoint.c_str(), 0750); + + // now try mounting + for (unsigned int i=0; i < v.size(); i++) + { + if (!v[i].Mounted) + { + if(MountCdrom(AptMountPoint, v[i].DeviceName)) + { + if (IsCorrectCD(Get, AptMountPoint, NewID)) + { + MountedByApt = true; + CDROM = AptMountPoint; + return true; + } else { + UnmountCdrom(AptMountPoint); + } + } + } + } + + return false; +} + /*}}}*/ +// CDROMMethod::IsCorrectCD /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool CDROMMethod::IsCorrectCD(URI want, string MountPath, string& NewID) +{ + for (unsigned int Version = 2; Version != 0; Version--) + { + if (IdentCdrom(MountPath,NewID,Version) == false) + return false; + + if (Debug) + clog << "ID " << Version << " " << NewID << endl; + + // A hit + if (Database.Find("CD::" + NewID) == want.Host) + return true; + } + + return false; +} + /*}}}*/ +// CDROMMethod::Fetch - Fetch a file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool CDROMMethod::Fetch(FetchItem *Itm) +{ + FetchResult Res; + + URI Get(Itm->Uri); + std::string const File = DecodeSendURI(Get.Path); + Debug = DebugEnabled(); + + if (Debug) + clog << "CDROMMethod::Fetch " << Itm->Uri << endl; + + /* All IMS queries are returned as a hit, CDROMs are readonly so + time stamps never change */ + if (Itm->LastModified != 0) + { + Res.LastModified = Itm->LastModified; + Res.IMSHit = true; + Res.Filename = Itm->DestFile; + URIDone(Res); + return true; + } + + // Load the database + if (DatabaseLoaded == false) + { + // Read the database + string DFile = _config->FindFile("Dir::State::cdroms"); + if (FileExists(DFile) == true) + { + if (ReadConfigFile(Database,DFile) == false) + return _error->Error(_("Unable to read the cdrom database %s"), + DFile.c_str()); + } + DatabaseLoaded = true; + } + + // All non IMS queries for package files fail. + if (Itm->IndexFile == true || GetID(Get.Host).empty() == true) + { + Fail(_("Please use apt-cdrom to make this CD-ROM recognized by APT." + " apt-get update cannot be used to add new CD-ROMs")); + return true; + } + + // We already have a CD inserted, but it is the wrong one + if (CurrentID.empty() == false && + CurrentID != "FAIL" && + Database.Find("CD::" + CurrentID) != Get.Host) + { + Fail(_("Wrong CD-ROM"),true); + return true; + } + + bool const AutoDetect = ConfigFindB("AutoDetect", true); + CDROM = _config->FindDir("Acquire::cdrom::mount"); + if (Debug) + clog << "Looking for CDROM at " << CDROM << endl; + + if (CDROM[0] == '.') + CDROM= SafeGetCWD() + '/' + CDROM; + + string NewID; + while (CurrentID.empty() == true) + { + if (AutoDetect) + AutoDetectAndMount(Get, NewID); + + if(!IsMounted(CDROM)) + MountedByApt = MountCdrom(CDROM); + + if (IsCorrectCD(Get, CDROM, NewID)) + break; + + // I suppose this should prompt somehow? + if (_config->FindB("APT::CDROM::NoMount",false) == false && + UnmountCdrom(CDROM) == false) + return _error->Error(_("Unable to unmount the CD-ROM in %s, it may still be in use."), + CDROM.c_str()); + if (MediaFail(Get.Host,CDROM) == false) + { + CurrentID = "FAIL"; + return _error->Error(_("Disk not found.")); + } + } + + // Found a CD + Res.Filename = CDROM + File; + struct stat Buf; + if (stat(Res.Filename.c_str(),&Buf) != 0) + return _error->Error(_("File not found")); + + URIStart(Res); + if (NewID.empty() == false) + CurrentID = NewID; + Res.LastModified = Buf.st_mtime; + Res.Size = Buf.st_size; + + Hashes Hash(Itm->ExpectedHashes); + FileFd Fd(Res.Filename, FileFd::ReadOnly); + Hash.AddFD(Fd); + Res.TakeHashes(Hash); + + URIDone(Res); + return true; +} + /*}}}*/ +bool CDROMMethod::Configuration(std::string Message) /*{{{*/ +{ + _config->CndSet("Binary::cdrom::Debug::NoDropPrivs", true); + return aptMethod::Configuration(Message); +} + /*}}}*/ + +int main() +{ + return CDROMMethod().Run(); +} diff --git a/methods/connect.cc b/methods/connect.cc new file mode 100644 index 0000000..bc2fe1d --- /dev/null +++ b/methods/connect.cc @@ -0,0 +1,1053 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Connect - Replacement connect call + + This was originally authored by Jason Gunthorpe <jgg@debian.org> + and is placed in the Public Domain, do with it what you will. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/acquire-method.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/srvrec.h> +#include <apt-pkg/strutl.h> + +#include <gnutls/gnutls.h> +#include <gnutls/x509.h> + +#include <list> +#include <set> +#include <sstream> +#include <string> +#include <errno.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> + +// Internet stuff +#include <netdb.h> +#include <arpa/inet.h> +#include <netinet/in.h> +#include <sys/select.h> +#include <sys/socket.h> + +#include "aptmethod.h" +#include "connect.h" +#include "rfc2553emu.h" +#include <apti18n.h> + /*}}}*/ + +static std::string LastHost; +static std::string LastService; +static struct addrinfo *LastHostAddr = 0; +static struct addrinfo *LastUsed = 0; + +static std::vector<SrvRec> SrvRecords; + +// Set of IP/hostnames that we timed out before or couldn't resolve +static std::set<std::string> bad_addr; + +// RotateDNS - Select a new server from a DNS rotation /*{{{*/ +// --------------------------------------------------------------------- +/* This is called during certain errors in order to recover by selecting a + new server */ +void RotateDNS() +{ + if (LastUsed != 0 && LastUsed->ai_next != 0) + LastUsed = LastUsed->ai_next; + else + LastUsed = LastHostAddr; +} + /*}}}*/ +static bool ConnectionAllowed(char const * const Service, std::string const &Host)/*{{{*/ +{ + if (unlikely(Host.empty())) // the only legal empty host (RFC2782 '.' target) is detected by caller + return false; + if (APT::String::Endswith(Host, ".onion") && _config->FindB("Acquire::BlockDotOnion", true)) + { + // TRANSLATOR: %s is e.g. Tor's ".onion" which would likely fail or leak info (RFC7686) + _error->Error(_("Direct connection to %s domains is blocked by default."), ".onion"); + if (strcmp(Service, "http") == 0) + _error->Error(_("If you meant to use Tor remember to use %s instead of %s."), "tor+http", "http"); + return false; + } + return true; +} + /*}}}*/ + +// File Descriptor based Fd /*{{{*/ +struct FdFd : public MethodFd +{ + int fd = -1; + int Fd() APT_OVERRIDE { return fd; } + ssize_t Read(void *buf, size_t count) APT_OVERRIDE { return ::read(fd, buf, count); } + ssize_t Write(void *buf, size_t count) APT_OVERRIDE { return ::write(fd, buf, count); } + int Close() APT_OVERRIDE + { + int result = 0; + if (fd != -1) + result = ::close(fd); + fd = -1; + return result; + } +}; + +bool MethodFd::HasPending() +{ + return false; +} +std::unique_ptr<MethodFd> MethodFd::FromFd(int iFd) +{ + FdFd *fd = new FdFd(); + fd->fd = iFd; + return std::unique_ptr<MethodFd>(fd); +} + /*}}}*/ +// DoConnect - Attempt a connect operation /*{{{*/ +// --------------------------------------------------------------------- +/* This helper function attempts a connection to a single address. */ +struct Connection +{ + struct addrinfo *Addr; + std::string Host; + aptMethod *Owner; + std::unique_ptr<FdFd> Fd; + char Name[NI_MAXHOST]; + char Service[NI_MAXSERV]; + + Connection(struct addrinfo *Addr, std::string const &Host, aptMethod *Owner) : Addr(Addr), Host(Host), Owner(Owner), Fd(new FdFd()), Name{0}, Service{0} + { + } + + // Allow moving values, but not connections. + Connection(Connection &&Conn) = default; + Connection(const Connection &Conn) = delete; + Connection &operator=(const Connection &) = delete; + Connection &operator=(Connection &&Conn) = default; + + ~Connection() + { + if (Fd != nullptr) + { + Fd->Close(); + } + } + + std::unique_ptr<MethodFd> Take() + { + /* Store the IP we are using.. If something goes + wrong this will get tacked onto the end of the error message */ + std::stringstream ss; + ioprintf(ss, _("[IP: %s %s]"), Name, Service); + Owner->SetIP(ss.str()); + Owner->Status(_("Connected to %s (%s)"), Host.c_str(), Name); + _error->Discard(); + Owner->SetFailReason(""); + LastUsed = Addr; + return std::move(Fd); + } + + ResultState DoConnect(); + + ResultState CheckError(); +}; + +ResultState Connection::DoConnect() +{ + getnameinfo(Addr->ai_addr,Addr->ai_addrlen, + Name,sizeof(Name),Service,sizeof(Service), + NI_NUMERICHOST|NI_NUMERICSERV); + Owner->Status(_("Connecting to %s (%s)"),Host.c_str(),Name); + + // if that addr did timeout before, we do not try it again + if(bad_addr.find(std::string(Name)) != bad_addr.end()) + return ResultState::TRANSIENT_ERROR; + + // Get a socket + if ((static_cast<FdFd *>(Fd.get())->fd = socket(Addr->ai_family, Addr->ai_socktype, + Addr->ai_protocol)) < 0) + { + _error->Errno("socket", _("Could not create a socket for %s (f=%u t=%u p=%u)"), + Name, Addr->ai_family, Addr->ai_socktype, Addr->ai_protocol); + return ResultState::FATAL_ERROR; + } + + SetNonBlock(Fd->Fd(), true); + if (connect(Fd->Fd(), Addr->ai_addr, Addr->ai_addrlen) < 0 && + errno != EINPROGRESS) + { + _error->Errno("connect", _("Cannot initiate the connection " + "to %s:%s (%s)."), + Host.c_str(), Service, Name); + return ResultState::TRANSIENT_ERROR; + } + + return ResultState::SUCCESSFUL; +} + +ResultState Connection::CheckError() +{ + // Check the socket for an error condition + unsigned int Err; + unsigned int Len = sizeof(Err); + if (getsockopt(Fd->Fd(), SOL_SOCKET, SO_ERROR, &Err, &Len) != 0) + { + _error->Errno("getsockopt", _("Failed")); + return ResultState::FATAL_ERROR; + } + + if (Err != 0) + { + errno = Err; + if(errno == ECONNREFUSED) + Owner->SetFailReason("ConnectionRefused"); + else if (errno == ETIMEDOUT) + Owner->SetFailReason("ConnectionTimedOut"); + bad_addr.insert(bad_addr.begin(), std::string(Name)); + _error->Errno("connect", _("Could not connect to %s:%s (%s)."), Host.c_str(), + Service, Name); + return ResultState::TRANSIENT_ERROR; + } + + Owner->SetFailReason(""); + + return ResultState::SUCCESSFUL; +} + /*}}}*/ +// Order the given host names returned by getaddrinfo() /*{{{*/ +static std::vector<struct addrinfo *> OrderAddresses(struct addrinfo *CurHost) +{ + std::vector<struct addrinfo *> preferredAddrs; + std::vector<struct addrinfo *> otherAddrs; + std::vector<struct addrinfo *> allAddrs; + + // Partition addresses into preferred and other address families + while (CurHost != 0) + { + if (preferredAddrs.empty() || CurHost->ai_family == preferredAddrs[0]->ai_family) + preferredAddrs.push_back(CurHost); + else + otherAddrs.push_back(CurHost); + + // Ignore UNIX domain sockets + do + { + CurHost = CurHost->ai_next; + } while (CurHost != 0 && CurHost->ai_family == AF_UNIX); + + /* If we reached the end of the search list then wrap around to the + start */ + if (CurHost == 0 && LastUsed != 0) + CurHost = LastHostAddr; + + // Reached the end of the search cycle + if (CurHost == LastUsed) + break; + } + + // Build a new address vector alternating between preferred and other + for (auto prefIter = preferredAddrs.cbegin(), otherIter = otherAddrs.cbegin(); + prefIter != preferredAddrs.end() || otherIter != otherAddrs.end();) + { + if (prefIter != preferredAddrs.end()) + allAddrs.push_back(*prefIter++); + if (otherIter != otherAddrs.end()) + allAddrs.push_back(*otherIter++); + } + + return allAddrs; +} + /*}}}*/ +// Check for errors and report them /*{{{*/ +static ResultState WaitAndCheckErrors(std::list<Connection> &Conns, std::unique_ptr<MethodFd> &Fd, long TimeoutMsec, bool ReportTimeout) +{ + // The last error detected + ResultState Result = ResultState::TRANSIENT_ERROR; + + struct timeval tv = { + // Split our millisecond timeout into seconds and microseconds + .tv_sec = TimeoutMsec / 1000, + .tv_usec = (TimeoutMsec % 1000) * 1000, + }; + + // We will return once we have no more connections, a time out, or + // a success. + while (!Conns.empty()) + { + fd_set Set; + int nfds = -1; + + FD_ZERO(&Set); + + for (auto &Conn : Conns) + { + int fd = Conn.Fd->Fd(); + FD_SET(fd, &Set); + nfds = std::max(nfds, fd); + } + + { + int Res; + do + { + Res = select(nfds + 1, 0, &Set, 0, (TimeoutMsec != 0 ? &tv : 0)); + } while (Res < 0 && errno == EINTR); + + if (Res == 0) + { + if (ReportTimeout) + { + for (auto &Conn : Conns) + { + Conn.Owner->SetFailReason("Timeout"); + bad_addr.insert(bad_addr.begin(), Conn.Name); + _error->Error(_("Could not connect to %s:%s (%s), " + "connection timed out"), + Conn.Host.c_str(), Conn.Service, Conn.Name); + } + } + return ResultState::TRANSIENT_ERROR; + } + } + + // iterate over connections, remove failed ones, and return if + // there was a successful one. + for (auto ConnI = Conns.begin(); ConnI != Conns.end();) + { + if (!FD_ISSET(ConnI->Fd->Fd(), &Set)) + { + ConnI++; + continue; + } + + Result = ConnI->CheckError(); + if (Result == ResultState::SUCCESSFUL) + { + Fd = ConnI->Take(); + return Result; + } + + // Connection failed. Erase it and continue to next position + ConnI = Conns.erase(ConnI); + } + } + + return Result; +} + /*}}}*/ +// Connect to a given Hostname /*{{{*/ +static ResultState ConnectToHostname(std::string const &Host, int const Port, + const char *const Service, int DefPort, std::unique_ptr<MethodFd> &Fd, + unsigned long const TimeOut, aptMethod *const Owner) +{ + if (ConnectionAllowed(Service, Host) == false) + return ResultState::FATAL_ERROR; + + // Used by getaddrinfo(); prefer port if given, else fallback to service + std::string ServiceNameOrPort = Port != 0 ? std::to_string(Port) : Service; + + /* We used a cached address record.. Yes this is against the spec but + the way we have setup our rotating dns suggests that this is more + sensible */ + if (LastHost != Host || LastService != ServiceNameOrPort) + { + Owner->Status(_("Connecting to %s"),Host.c_str()); + + // Free the old address structure + if (LastHostAddr != 0) + { + freeaddrinfo(LastHostAddr); + LastHostAddr = 0; + LastUsed = 0; + } + + // We only understand SOCK_STREAM sockets. + struct addrinfo Hints; + memset(&Hints,0,sizeof(Hints)); + Hints.ai_socktype = SOCK_STREAM; + Hints.ai_flags = 0; +#ifdef AI_IDN + if (_config->FindB("Acquire::Connect::IDN", true) == true) + Hints.ai_flags |= AI_IDN; +#endif + // see getaddrinfo(3): only return address if system has such a address configured + // useful if system is ipv4 only, to not get ipv6, but that fails if the system has + // no address configured: e.g. offline and trying to connect to localhost. + if (_config->FindB("Acquire::Connect::AddrConfig", true) == true) + Hints.ai_flags |= AI_ADDRCONFIG; + Hints.ai_protocol = 0; + + if(_config->FindB("Acquire::ForceIPv4", false) == true) + Hints.ai_family = AF_INET; + else if(_config->FindB("Acquire::ForceIPv6", false) == true) + Hints.ai_family = AF_INET6; + else + Hints.ai_family = AF_UNSPEC; + + // if we couldn't resolve the host before, we don't try now + if (bad_addr.find(Host) != bad_addr.end()) + { + _error->Error(_("Could not resolve '%s'"), Host.c_str()); + return ResultState::TRANSIENT_ERROR; + } + + // Resolve both the host and service simultaneously + while (1) + { + int Res; + if ((Res = getaddrinfo(Host.c_str(), ServiceNameOrPort.c_str(), &Hints, &LastHostAddr)) != 0 || + LastHostAddr == 0) + { + if (Res == EAI_NONAME || Res == EAI_SERVICE) + { + if (DefPort != 0) + { + ServiceNameOrPort = std::to_string(DefPort); + DefPort = 0; + continue; + } + bad_addr.insert(bad_addr.begin(), Host); + Owner->SetFailReason("ResolveFailure"); + _error->Error(_("Could not resolve '%s'"), Host.c_str()); + return ResultState::TRANSIENT_ERROR; + } + + if (Res == EAI_AGAIN) + { + Owner->SetFailReason("TmpResolveFailure"); + _error->Error(_("Temporary failure resolving '%s'"), + Host.c_str()); + return ResultState::TRANSIENT_ERROR; + } + if (Res == EAI_SYSTEM) + _error->Errno("getaddrinfo", _("System error resolving '%s:%s'"), + Host.c_str(), ServiceNameOrPort.c_str()); + else + _error->Error(_("Something wicked happened resolving '%s:%s' (%i - %s)"), + Host.c_str(), ServiceNameOrPort.c_str(), Res, gai_strerror(Res)); + return ResultState::TRANSIENT_ERROR; + } + break; + } + + LastHost = Host; + LastService = ServiceNameOrPort; + } + + // When we have an IP rotation stay with the last IP. + auto Addresses = OrderAddresses(LastUsed != nullptr ? LastUsed : LastHostAddr); + std::list<Connection> Conns; + ResultState Result = ResultState::SUCCESSFUL; + + for (auto Addr : Addresses) + { + Connection Conn(Addr, Host, Owner); + if (Conn.DoConnect() != ResultState::SUCCESSFUL) + continue; + + Conns.push_back(std::move(Conn)); + + Result = WaitAndCheckErrors(Conns, Fd, Owner->ConfigFindI("ConnectionAttemptDelayMsec", 250), false); + + if (Result == ResultState::SUCCESSFUL) + return ResultState::SUCCESSFUL; + } + + if (!Conns.empty()) + return WaitAndCheckErrors(Conns, Fd, TimeOut * 1000, true); + if (Result != ResultState::SUCCESSFUL) + return Result; + if (_error->PendingError() == true) + return ResultState::FATAL_ERROR; + _error->Error(_("Unable to connect to %s:%s:"), Host.c_str(), ServiceNameOrPort.c_str()); + return ResultState::TRANSIENT_ERROR; +} + /*}}}*/ +// Connect - Connect to a server /*{{{*/ +// --------------------------------------------------------------------- +/* Performs a connection to the server (including SRV record lookup) */ +ResultState Connect(std::string Host, int Port, const char *Service, + int DefPort, std::unique_ptr<MethodFd> &Fd, + unsigned long TimeOut, aptMethod *Owner) +{ + if (_error->PendingError() == true) + return ResultState::FATAL_ERROR; + + if (ConnectionAllowed(Service, Host) == false) + return ResultState::FATAL_ERROR; + + // Used by getaddrinfo(); prefer port if given, else fallback to service + std::string ServiceNameOrPort = Port != 0 ? std::to_string(Port) : Service; + + if(LastHost != Host || LastService != ServiceNameOrPort) + { + SrvRecords.clear(); + if (_config->FindB("Acquire::EnableSrvRecords", true) == true) + { + GetSrvRecords(Host, DefPort, SrvRecords); + // RFC2782 defines that a lonely '.' target is an abort reason + if (SrvRecords.size() == 1 && SrvRecords[0].target.empty()) + { + _error->Error("SRV records for %s indicate that " + "%s service is not available at this domain", + Host.c_str(), Service); + return ResultState::FATAL_ERROR; + } + } + } + + size_t stackSize = 0; + // try to connect in the priority order of the srv records + std::string initialHost{std::move(Host)}; + auto const initialPort = Port; + while(SrvRecords.empty() == false) + { + _error->PushToStack(); + ++stackSize; + // PopFromSrvRecs will also remove the server + auto Srv = PopFromSrvRecs(SrvRecords); + Host = Srv.target; + Port = Srv.port; + auto const ret = ConnectToHostname(Host, Port, Service, DefPort, Fd, TimeOut, Owner); + if (ret == ResultState::SUCCESSFUL) + { + while(stackSize--) + _error->RevertToStack(); + return ret; + } + } + Host = std::move(initialHost); + Port = initialPort; + + // we have no (good) SrvRecords for this host, connect right away + _error->PushToStack(); + ++stackSize; + auto const ret = ConnectToHostname(Host, Port, Service, DefPort, Fd, + TimeOut, Owner); + while(stackSize--) + if (ret == ResultState::SUCCESSFUL) + _error->RevertToStack(); + else + _error->MergeWithStack(); + return ret; +} + /*}}}*/ +// UnwrapSocks - Handle SOCKS setup /*{{{*/ +// --------------------------------------------------------------------- +/* This does socks magic */ +static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy, + char const *const type, bool const ReadWrite, uint8_t *const ToFrom, + unsigned int const Size, unsigned int const Timeout) +{ + if (WaitFd(ServerFd, ReadWrite, Timeout) == false) + { + if (ReadWrite) + return _error->Error("Timed out while waiting to write '%s' to proxy %s", type, URI::SiteOnly(Proxy).c_str()); + else + return _error->Error("Timed out while waiting to read '%s' from proxy %s", type, URI::SiteOnly(Proxy).c_str()); + } + if (ReadWrite == false) + { + if (FileFd::Read(ServerFd, ToFrom, Size) == false) + return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); + } + else + { + if (FileFd::Write(ServerFd, ToFrom, Size) == false) + return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); + } + return true; +} + +ResultState UnwrapSocks(std::string Host, int Port, URI Proxy, std::unique_ptr<MethodFd> &Fd, + unsigned long Timeout, aptMethod *Owner) +{ + /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect + * for not offering GSSAPI auth which is a must (we only do no or user/pass auth). + * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */ + std::string const ProxyInfo = URI::SiteOnly(Proxy); + Owner->Status(_("Connecting to %s (%s)"), "SOCKS5h proxy", ProxyInfo.c_str()); +#define APT_WriteOrFail(TYPE, DATA, LENGTH) \ + if (TalkToSocksProxy(Fd->Fd(), ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) \ + return ResultState::TRANSIENT_ERROR +#define APT_ReadOrFail(TYPE, DATA, LENGTH) \ + if (TalkToSocksProxy(Fd->Fd(), ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) \ + return ResultState::TRANSIENT_ERROR + if (Host.length() > 255) + { + _error->Error("Can't use SOCKS5h as hostname %s is too long!", Host.c_str()); + return ResultState::FATAL_ERROR; + } + if (Proxy.User.length() > 255 || Proxy.Password.length() > 255) + { + _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length()); + return ResultState::FATAL_ERROR; + } + if (Proxy.User.empty()) + { + uint8_t greeting[] = {0x05, 0x01, 0x00}; + APT_WriteOrFail("greet-1", greeting, sizeof(greeting)); + } + else + { + uint8_t greeting[] = {0x05, 0x02, 0x00, 0x02}; + APT_WriteOrFail("greet-2", greeting, sizeof(greeting)); + } + uint8_t greeting[2]; + APT_ReadOrFail("greet back", greeting, sizeof(greeting)); + if (greeting[0] != 0x05) + { + _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]); + return ResultState::FATAL_ERROR; + } + if (greeting[1] == 0x00) + ; // no auth has no method-dependent sub-negotiations + else if (greeting[1] == 0x02) + { + if (Proxy.User.empty()) + { + _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str()); + return ResultState::FATAL_ERROR; + } + // user&pass auth sub-negotiations are defined by RFC1929 + std::vector<uint8_t> auth = {{0x01, static_cast<uint8_t>(Proxy.User.length())}}; + std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth)); + auth.push_back(static_cast<uint8_t>(Proxy.Password.length())); + std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth)); + APT_WriteOrFail("user&pass auth", auth.data(), auth.size()); + uint8_t authstatus[2]; + APT_ReadOrFail("auth report", authstatus, sizeof(authstatus)); + if (authstatus[0] != 0x01) + { + _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]); + return ResultState::FATAL_ERROR; + } + if (authstatus[1] != 0x00) + { + _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]); + return ResultState::FATAL_ERROR; + } + } + else + { + _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]); + return ResultState::FATAL_ERROR; + } + union { + uint16_t *i; + uint8_t *b; + } portu; + uint16_t port = htons(static_cast<uint16_t>(Port)); + portu.i = &port; + std::vector<uint8_t> request = {{0x05, 0x01, 0x00, 0x03, static_cast<uint8_t>(Host.length())}}; + std::copy(Host.begin(), Host.end(), std::back_inserter(request)); + request.push_back(portu.b[0]); + request.push_back(portu.b[1]); + APT_WriteOrFail("request", request.data(), request.size()); + uint8_t response[4]; + APT_ReadOrFail("first part of response", response, sizeof(response)); + if (response[0] != 0x05) + { + _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]); + return ResultState::FATAL_ERROR; + } + if (response[2] != 0x00) + { + _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]); + return ResultState::FATAL_ERROR; + } + std::string bindaddr; + if (response[3] == 0x01) // IPv4 address + { + uint8_t ip4port[6]; + APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port)); + portu.b[0] = ip4port[4]; + portu.b[1] = ip4port[5]; + port = ntohs(*portu.i); + strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port); + } + else if (response[3] == 0x03) // hostname + { + uint8_t namelength; + APT_ReadOrFail("hostname length of response", &namelength, 1); + uint8_t hostname[namelength + 2]; + APT_ReadOrFail("hostname of response", hostname, sizeof(hostname)); + portu.b[0] = hostname[namelength]; + portu.b[1] = hostname[namelength + 1]; + port = ntohs(*portu.i); + hostname[namelength] = '\0'; + strprintf(bindaddr, "%s:%d", hostname, port); + } + else if (response[3] == 0x04) // IPv6 address + { + uint8_t ip6port[18]; + APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port)); + portu.b[0] = ip6port[16]; + portu.b[1] = ip6port[17]; + port = ntohs(*portu.i); + strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d", + ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7], + ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15], + port); + } + else + { + _error->Error("SOCKS proxy %s destination address is of unknown type: %d", + ProxyInfo.c_str(), response[3]); + return ResultState::FATAL_ERROR; + } + if (response[1] != 0x00) + { + char const *errstr = nullptr; + auto errcode = response[1]; + bool Transient = false; + // Tor error reporting can be a bit arcane, lets try to detect & fix it up + if (bindaddr == "0.0.0.0:0") + { + auto const lastdot = Host.rfind('.'); + if (lastdot == std::string::npos || Host.substr(lastdot) != ".onion") + ; + else if (errcode == 0x01) + { + auto const prevdot = Host.rfind('.', lastdot - 1); + if (prevdot == std::string::npos && (lastdot == 16 || lastdot == 56)) + ; // valid .onion address + else if (prevdot != std::string::npos && ((lastdot - prevdot) == 17 || (lastdot - prevdot) == 57)) + ; // valid .onion address with subdomain(s) + else + { + errstr = "Invalid hostname: onion service name must be either 16 or 56 characters long"; + Owner->SetFailReason("SOCKS"); + } + } + // in all likelihood the service is either down or the address has + // a typo and so "Host unreachable" is the better understood error + // compared to the technically correct "TLL expired". + else if (errcode == 0x06) + errcode = 0x04; + } + if (errstr == nullptr) + { + switch (errcode) + { + case 0x01: + errstr = "general SOCKS server failure"; + Owner->SetFailReason("SOCKS"); + break; + case 0x02: + errstr = "connection not allowed by ruleset"; + Owner->SetFailReason("SOCKS"); + break; + case 0x03: + errstr = "Network unreachable"; + Owner->SetFailReason("ConnectionTimedOut"); + Transient = true; + break; + case 0x04: + errstr = "Host unreachable"; + Owner->SetFailReason("ConnectionTimedOut"); + Transient = true; + break; + case 0x05: + errstr = "Connection refused"; + Owner->SetFailReason("ConnectionRefused"); + Transient = true; + break; + case 0x06: + errstr = "TTL expired"; + Owner->SetFailReason("Timeout"); + Transient = true; + break; + case 0x07: + errstr = "Command not supported"; + Owner->SetFailReason("SOCKS"); + break; + case 0x08: + errstr = "Address type not supported"; + Owner->SetFailReason("SOCKS"); + break; + default: + errstr = "Unknown error"; + Owner->SetFailReason("SOCKS"); + break; + } + } + _error->Error("SOCKS proxy %s could not connect to %s (%s) due to: %s (%d)", + ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str(), errstr, response[1]); + return Transient ? ResultState::TRANSIENT_ERROR : ResultState::FATAL_ERROR; + } + else if (Owner->DebugEnabled()) + ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s (%s)\n", + ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str()); + + if (WaitFd(Fd->Fd(), true, Timeout) == false) + { + _error->Error("SOCKS proxy %s reported connection to %s (%s), but timed out", + ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str()); + return ResultState::TRANSIENT_ERROR; + } +#undef APT_ReadOrFail +#undef APT_WriteOrFail + + return ResultState::SUCCESSFUL; +} + /*}}}*/ +// UnwrapTLS - Handle TLS connections /*{{{*/ +// --------------------------------------------------------------------- +/* Performs a TLS handshake on the socket */ +struct TlsFd : public MethodFd +{ + std::unique_ptr<MethodFd> UnderlyingFd; + gnutls_session_t session; + gnutls_certificate_credentials_t credentials; + std::string hostname; + unsigned long Timeout; + + int Fd() APT_OVERRIDE { return UnderlyingFd->Fd(); } + + ssize_t Read(void *buf, size_t count) APT_OVERRIDE + { + return HandleError(gnutls_record_recv(session, buf, count)); + } + ssize_t Write(void *buf, size_t count) APT_OVERRIDE + { + return HandleError(gnutls_record_send(session, buf, count)); + } + + ssize_t DoTLSHandshake() + { + int err; + // Do the handshake. Our socket is non-blocking, so we need to call WaitFd() + // accordingly. + do + { + err = gnutls_handshake(session); + if ((err == GNUTLS_E_INTERRUPTED || err == GNUTLS_E_AGAIN) && + WaitFd(this->Fd(), gnutls_record_get_direction(session) == 1, Timeout) == false) + { + _error->Errno("select", "Could not wait for server fd"); + return err; + } + } while (err < 0 && gnutls_error_is_fatal(err) == 0); + + if (err < 0) + { + // Print reason why validation failed. + if (err == GNUTLS_E_CERTIFICATE_VERIFICATION_ERROR) + { + gnutls_datum_t txt; + auto type = gnutls_certificate_type_get(session); + auto status = gnutls_session_get_verify_cert_status(session); + if (gnutls_certificate_verification_status_print(status, type, &txt, 0) == 0) + { + _error->Error("Certificate verification failed: %s", txt.data); + } + gnutls_free(txt.data); + } + _error->Error("Could not handshake: %s", gnutls_strerror(err)); + } + return err; + } + + template <typename T> + T HandleError(T err) + { + // Server may request re-handshake if client certificates need to be provided + // based on resource requested + if (err == GNUTLS_E_REHANDSHAKE) + { + int rc = DoTLSHandshake(); + // Only reset err if DoTLSHandshake() fails. + // Otherwise, we want to follow the original error path and set errno to EAGAIN + // so that the request is retried. + if (rc < 0) + err = rc; + } + + if (err < 0 && gnutls_error_is_fatal(err)) + errno = EIO; + else if (err < 0) + errno = EAGAIN; + else + errno = 0; + return err; + } + + int Close() APT_OVERRIDE + { + auto err = HandleError(gnutls_bye(session, GNUTLS_SHUT_RDWR)); + auto lower = UnderlyingFd->Close(); + return err < 0 ? HandleError(err) : lower; + } + + bool HasPending() APT_OVERRIDE + { + return gnutls_record_check_pending(session) > 0; + } +}; + +ResultState UnwrapTLS(std::string const &Host, std::unique_ptr<MethodFd> &Fd, + unsigned long const Timeout, aptMethod * const Owner, + aptConfigWrapperForMethods const * const OwnerConf) +{ + if (_config->FindB("Acquire::AllowTLS", true) == false) + { + _error->Error("TLS support has been disabled: Acquire::AllowTLS is false."); + return ResultState::FATAL_ERROR; + } + + int err; + TlsFd *tlsFd = new TlsFd(); + + tlsFd->hostname = Host; + tlsFd->UnderlyingFd = MethodFd::FromFd(-1); // For now + tlsFd->Timeout = Timeout; + + if ((err = gnutls_init(&tlsFd->session, GNUTLS_CLIENT | GNUTLS_NONBLOCK)) < 0) + { + _error->Error("Internal error: could not allocate credentials: %s", gnutls_strerror(err)); + return ResultState::FATAL_ERROR; + } + + FdFd *fdfd = dynamic_cast<FdFd *>(Fd.get()); + if (fdfd != nullptr) + { + gnutls_transport_set_int(tlsFd->session, fdfd->fd); + } + else + { + gnutls_transport_set_ptr(tlsFd->session, Fd.get()); + gnutls_transport_set_pull_function(tlsFd->session, + [](gnutls_transport_ptr_t p, void *buf, size_t size) -> ssize_t { + return reinterpret_cast<MethodFd *>(p)->Read(buf, size); + }); + gnutls_transport_set_push_function(tlsFd->session, + [](gnutls_transport_ptr_t p, const void *buf, size_t size) -> ssize_t { + return reinterpret_cast<MethodFd *>(p)->Write((void *)buf, size); + }); + } + + if ((err = gnutls_certificate_allocate_credentials(&tlsFd->credentials)) < 0) + { + _error->Error("Internal error: could not allocate credentials: %s", gnutls_strerror(err)); + return ResultState::FATAL_ERROR; + } + + // Credential setup + std::string fileinfo = OwnerConf->ConfigFind("CaInfo", ""); + if (fileinfo.empty()) + { + // No CaInfo specified, use system trust store. + err = gnutls_certificate_set_x509_system_trust(tlsFd->credentials); + if (err == 0) + Owner->Warning("No system certificates available. Try installing ca-certificates."); + else if (err < 0) + { + _error->Error("Could not load system TLS certificates: %s", gnutls_strerror(err)); + return ResultState::FATAL_ERROR; + } + } + else + { + // CA location has been set, use the specified one instead + gnutls_certificate_set_verify_flags(tlsFd->credentials, GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT); + err = gnutls_certificate_set_x509_trust_file(tlsFd->credentials, fileinfo.c_str(), GNUTLS_X509_FMT_PEM); + if (err < 0) + { + _error->Error("Could not load certificates from %s (CaInfo option): %s", fileinfo.c_str(), gnutls_strerror(err)); + return ResultState::FATAL_ERROR; + } + } + + if (not OwnerConf->ConfigFind("IssuerCert", "").empty()) + { + _error->Error("The option '%s' is not supported anymore", "IssuerCert"); + return ResultState::FATAL_ERROR; + } + if (not OwnerConf->ConfigFind("SslForceVersion", "").empty()) + { + _error->Error("The option '%s' is not supported anymore", "SslForceVersion"); + return ResultState::FATAL_ERROR; + } + + // For client authentication, certificate file ... + std::string const cert = OwnerConf->ConfigFind("SslCert", ""); + std::string const key = OwnerConf->ConfigFind("SslKey", ""); + if (cert.empty() == false) + { + if ((err = gnutls_certificate_set_x509_key_file( + tlsFd->credentials, + cert.c_str(), + key.empty() ? cert.c_str() : key.c_str(), + GNUTLS_X509_FMT_PEM)) < 0) + { + _error->Error("Could not load client certificate (%s, SslCert option) or key (%s, SslKey option): %s", cert.c_str(), key.c_str(), gnutls_strerror(err)); + return ResultState::FATAL_ERROR; + } + } + + // CRL file + std::string const crlfile = OwnerConf->ConfigFind("CrlFile", ""); + if (crlfile.empty() == false) + { + if ((err = gnutls_certificate_set_x509_crl_file(tlsFd->credentials, + crlfile.c_str(), + GNUTLS_X509_FMT_PEM)) < 0) + { + _error->Error("Could not load custom certificate revocation list %s (CrlFile option): %s", crlfile.c_str(), gnutls_strerror(err)); + return ResultState::FATAL_ERROR; + } + } + + if ((err = gnutls_credentials_set(tlsFd->session, GNUTLS_CRD_CERTIFICATE, tlsFd->credentials)) < 0) + { + _error->Error("Internal error: Could not add certificates to session: %s", gnutls_strerror(err)); + return ResultState::FATAL_ERROR; + } + + if ((err = gnutls_set_default_priority(tlsFd->session)) < 0) + { + _error->Error("Internal error: Could not set algorithm preferences: %s", gnutls_strerror(err)); + return ResultState::FATAL_ERROR; + } + + if (OwnerConf->ConfigFindB("Verify-Peer", true)) + { + gnutls_session_set_verify_cert(tlsFd->session, OwnerConf->ConfigFindB("Verify-Host", true) ? tlsFd->hostname.c_str() : nullptr, 0); + } + + // set SNI only if the hostname is really a name and not an address + { + struct in_addr addr4; + struct in6_addr addr6; + + if (inet_pton(AF_INET, tlsFd->hostname.c_str(), &addr4) == 1 || + inet_pton(AF_INET6, tlsFd->hostname.c_str(), &addr6) == 1) + /* not a host name */; + else if ((err = gnutls_server_name_set(tlsFd->session, GNUTLS_NAME_DNS, tlsFd->hostname.c_str(), tlsFd->hostname.length())) < 0) + { + _error->Error("Could not set host name %s to indicate to server: %s", tlsFd->hostname.c_str(), gnutls_strerror(err)); + return ResultState::FATAL_ERROR; + } + } + + // Set the FD now, so closing it works reliably. + tlsFd->UnderlyingFd = std::move(Fd); + Fd.reset(tlsFd); + + // Do the handshake. + err = tlsFd->DoTLSHandshake(); + + if (err < 0) + return ResultState::TRANSIENT_ERROR; + + return ResultState::SUCCESSFUL; +} + /*}}}*/ diff --git a/methods/connect.h b/methods/connect.h new file mode 100644 index 0000000..413484a --- /dev/null +++ b/methods/connect.h @@ -0,0 +1,50 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Connect - Replacement connect call + + ##################################################################### */ + /*}}}*/ +#ifndef CONNECT_H +#define CONNECT_H + +#include <memory> +#include <string> +#include <stddef.h> + +#include "aptmethod.h" + +/** + * \brief Small representation of a file descriptor for network traffic. + * + * This provides support for TLS, SOCKS, and HTTP CONNECT proxies. + */ +struct MethodFd +{ + /// \brief Returns -1 for unusable, or an fd to select() on otherwise + virtual int Fd() = 0; + /// \brief Should behave like read(2) + virtual ssize_t Read(void *buf, size_t count) = 0; + /// \brief Should behave like write(2) + virtual ssize_t Write(void *buf, size_t count) = 0; + /// \brief Closes the file descriptor. Can be called multiple times. + virtual int Close() = 0; + /// \brief Destructor + virtual ~MethodFd(){}; + /// \brief Construct a MethodFd from a UNIX file descriptor + static std::unique_ptr<MethodFd> FromFd(int iFd); + /// \brief If there is pending data. + virtual bool HasPending(); +}; + +ResultState Connect(std::string To, int Port, const char *Service, int DefPort, + std::unique_ptr<MethodFd> &Fd, unsigned long TimeOut, aptMethod *Owner); + +ResultState UnwrapSocks(std::string To, int Port, URI Proxy, std::unique_ptr<MethodFd> &Fd, unsigned long Timeout, aptMethod *Owner); +ResultState UnwrapTLS(std::string const &To, std::unique_ptr<MethodFd> &Fd, unsigned long Timeout, aptMethod *Owner, + aptConfigWrapperForMethods const * OwnerConf); + +void RotateDNS(); + +#endif diff --git a/methods/copy.cc b/methods/copy.cc new file mode 100644 index 0000000..82eed15 --- /dev/null +++ b/methods/copy.cc @@ -0,0 +1,94 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Copy URI - This method takes a uri like a file: uri and copies it + to the destination file. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include "aptmethod.h" +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/strutl.h> + +#include <string> +#include <sys/stat.h> +#include <sys/time.h> + +#include <apti18n.h> + /*}}}*/ + +class CopyMethod : public aptMethod +{ + virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; + + public: + CopyMethod() : aptMethod("copy", "1.0", SingleInstance | SendConfig | SendURIEncoded) + { + SeccompFlags = aptMethod::BASE; + } +}; + +// CopyMethod::Fetch - Fetch a file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool CopyMethod::Fetch(FetchItem *Itm) +{ + // this ensures that relative paths work in copy + std::string const File = DecodeSendURI(Itm->Uri.substr(Itm->Uri.find(':')+1)); + + // Stat the file and send a start message + struct stat Buf; + if (stat(File.c_str(),&Buf) != 0) + return _error->Errno("stat",_("Failed to stat")); + + // Forumulate a result and send a start message + FetchResult Res; + Res.Size = Buf.st_size; + Res.Filename = Itm->DestFile; + Res.LastModified = Buf.st_mtime; + Res.IMSHit = false; + URIStart(Res); + + // just calc the hashes if the source and destination are identical + if (File == Itm->DestFile || Itm->DestFile == "/dev/null") + { + CalculateHashes(Itm, Res); + URIDone(Res); + return true; + } + + // See if the file exists + FileFd From(File,FileFd::ReadOnly); + FileFd To(Itm->DestFile,FileFd::WriteAtomic); + To.EraseOnFailure(); + + // Copy the file + if (CopyFile(From,To) == false) + { + To.OpFail(); + return false; + } + + From.Close(); + To.Close(); + + if (TransferModificationTimes(File.c_str(), Res.Filename.c_str(), Res.LastModified) == false) + return false; + + CalculateHashes(Itm, Res); + URIDone(Res); + return true; +} + /*}}}*/ + +int main() +{ + return CopyMethod().Run(); +} diff --git a/methods/file.cc b/methods/file.cc new file mode 100644 index 0000000..b2fe133 --- /dev/null +++ b/methods/file.cc @@ -0,0 +1,133 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + File URI method for APT + + This simply checks that the file specified exists, if so the relevant + information is returned. If a .gz filename is specified then the file + name with .gz removed will also be checked and information about it + will be returned in Alt-* + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include "aptmethod.h" +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/strutl.h> + +#include <string> +#include <sys/stat.h> + +#include <apti18n.h> + /*}}}*/ + +class FileMethod : public aptMethod +{ + virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; + + public: + FileMethod() : aptMethod("file", "1.0", SingleInstance | SendConfig | LocalOnly | SendURIEncoded) + { + SeccompFlags = aptMethod::BASE; + } +}; + +// FileMethod::Fetch - Fetch a file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool FileMethod::Fetch(FetchItem *Itm) +{ + URI Get(Itm->Uri); + std::string const File = DecodeSendURI(Get.Path); + FetchResult Res; + if (Get.Host.empty() == false) + return _error->Error(_("Invalid URI, local URIS must not start with //")); + + struct stat Buf; + // deal with destination files which might linger around + if (lstat(Itm->DestFile.c_str(), &Buf) == 0) + { + if ((Buf.st_mode & S_IFREG) != 0) + { + if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) + { + if (Itm->ExpectedHashes.VerifyFile(File)) + { + Res.Filename = Itm->DestFile; + Res.IMSHit = true; + } + } + } + } + if (Res.IMSHit != true) + RemoveFile("file", Itm->DestFile); + + int olderrno = 0; + // See if the file exists + if (stat(File.c_str(),&Buf) == 0) + { + Res.Size = Buf.st_size; + Res.Filename = File; + Res.LastModified = Buf.st_mtime; + Res.IMSHit = false; + if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) + { + unsigned long long const filesize = Itm->ExpectedHashes.FileSize(); + if (filesize != 0 && filesize == Res.Size) + Res.IMSHit = true; + } + + CalculateHashes(Itm, Res); + } + else + olderrno = errno; + if (Res.IMSHit == false) + URIStart(Res); + + // See if the uncompressed file exists and reuse it + FetchResult AltRes; + AltRes.Filename.clear(); + std::vector<std::string> extensions = APT::Configuration::getCompressorExtensions(); + for (std::vector<std::string>::const_iterator ext = extensions.begin(); ext != extensions.end(); ++ext) + { + if (APT::String::Endswith(File, *ext) == true) + { + std::string const unfile = File.substr(0, File.length() - ext->length()); + if (stat(unfile.c_str(),&Buf) == 0) + { + AltRes.Size = Buf.st_size; + AltRes.Filename = unfile; + AltRes.LastModified = Buf.st_mtime; + AltRes.IMSHit = false; + if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) + AltRes.IMSHit = true; + break; + } + // no break here as we could have situations similar to '.gz' vs '.tar.gz' here + } + } + + if (AltRes.Filename.empty() == false) + URIDone(Res,&AltRes); + else if (Res.Filename.empty() == false) + URIDone(Res); + else + { + errno = olderrno; + return _error->Errno(File.c_str(), _("File not found")); + } + + return true; +} + /*}}}*/ + +int main() +{ + return FileMethod().Run(); +} diff --git a/methods/ftp.cc b/methods/ftp.cc new file mode 100644 index 0000000..aa0c069 --- /dev/null +++ b/methods/ftp.cc @@ -0,0 +1,1188 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + FTP Acquire Method - This is the FTP acquire method for APT. + + This is a very simple implementation that does not try to optimize + at all. Commands are sent synchronously with the FTP server (as the + rfc recommends, but it is not really necessary..) and no tricks are + done to speed things along. + + RFC 2428 describes the IPv6 FTP behavior + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/strutl.h> + +#include <iostream> +#include <ctype.h> +#include <errno.h> +#include <signal.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <unistd.h> + +// Internet stuff +#include <netdb.h> +#include <arpa/inet.h> +#include <netinet/in.h> + +#include "connect.h" +#include "ftp.h" +#include "rfc2553emu.h" + +#include <apti18n.h> + /*}}}*/ + +using namespace std; + +/* This table is for the EPRT and EPSV commands, it maps the OS address + family to the IETF address families */ +struct AFMap +{ + unsigned long Family; + unsigned long IETFFamily; +}; + +#ifndef AF_INET6 +struct AFMap AFMap[] = {{AF_INET,1},{0, 0}}; +#else +struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{0, 0}}; +#endif + +unsigned long TimeOut = 30; +URI Proxy; +string FtpMethod::FailFile; +int FtpMethod::FailFd = -1; +time_t FtpMethod::FailTime = 0; + +// FTPConn::FTPConn - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(MethodFd::FromFd(-1)), DataFd(-1), + DataListenFd(-1), ServerName(Srv), + ForceExtended(false), TryPassive(true), + PeerAddrLen(0), ServerAddrLen(0) +{ + Debug = _config->FindB("Debug::Acquire::Ftp",false); + PasvAddr = 0; + Buffer[0] = '\0'; +} + /*}}}*/ +// FTPConn::~FTPConn - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +FTPConn::~FTPConn() +{ + Close(); +} + /*}}}*/ +// FTPConn::Close - Close down the connection /*{{{*/ +// --------------------------------------------------------------------- +/* Just tear down the socket and data socket */ +void FTPConn::Close() +{ + ServerFd->Close(); + close(DataFd); + DataFd = -1; + close(DataListenFd); + DataListenFd = -1; + + if (PasvAddr != 0) + freeaddrinfo(PasvAddr); + PasvAddr = 0; +} + /*}}}*/ +// FTPConn::Open - Open a new connection /*{{{*/ +// --------------------------------------------------------------------- +/* Connect to the server using a non-blocking connection and perform a + login. */ +ResultState FTPConn::Open(aptMethod *Owner) +{ + // Use the already open connection if possible. + if (ServerFd->Fd() != -1) + return ResultState::SUCCESSFUL; + + Close(); + + // Determine the proxy setting + string SpecificProxy = _config->Find("Acquire::ftp::Proxy::" + ServerName.Host); + if (!SpecificProxy.empty()) + { + if (SpecificProxy == "DIRECT") + Proxy = ""; + else + Proxy = SpecificProxy; + } + else + { + string DefProxy = _config->Find("Acquire::ftp::Proxy"); + if (!DefProxy.empty()) + { + Proxy = DefProxy; + } + else + { + char* result = getenv("ftp_proxy"); + Proxy = result ? result : ""; + } + } + + // Parse no_proxy, a , separated list of domains + if (getenv("no_proxy") != 0) + { + if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true) + Proxy = ""; + } + + // Determine what host and port to use based on the proxy settings + int Port = 0; + string Host; + if (Proxy.empty() == true) + { + if (ServerName.Port != 0) + Port = ServerName.Port; + Host = ServerName.Host; + } + else + { + if (Proxy.Port != 0) + Port = Proxy.Port; + Host = Proxy.Host; + } + + /* Connect to the remote server. Since FTP is connection oriented we + want to make sure we get a new server every time we reconnect */ + RotateDNS(); + auto result = Connect(Host, Port, "ftp", 21, ServerFd, TimeOut, Owner); + if (result != ResultState::SUCCESSFUL) + return result; + + // Login must be before getpeername otherwise dante won't work. + Owner->Status(_("Logging in")); + result = Login(); + if (result != ResultState::SUCCESSFUL) + return result; + + // Get the remote server's address + PeerAddrLen = sizeof(PeerAddr); + if (getpeername(ServerFd->Fd(), (sockaddr *)&PeerAddr, &PeerAddrLen) != 0) + { + _error->Errno("getpeername", _("Unable to determine the peer name")); + return ResultState::TRANSIENT_ERROR; + } + + // Get the local machine's address + ServerAddrLen = sizeof(ServerAddr); + if (getsockname(ServerFd->Fd(), (sockaddr *)&ServerAddr, &ServerAddrLen) != 0) + { + _error->Errno("getsockname", _("Unable to determine the local name")); + return ResultState::TRANSIENT_ERROR; + } + + return ResultState::SUCCESSFUL; +} + /*}}}*/ +// FTPConn::Login - Login to the remote server /*{{{*/ +// --------------------------------------------------------------------- +/* This performs both normal login and proxy login using a simples script + stored in the config file. */ +ResultState FTPConn::Login() +{ + unsigned int Tag; + string Msg; + + // Setup the variables needed for authentication + string User = "anonymous"; + string Pass = "apt_get_ftp_2.1@debian.linux.user"; + + // Fill in the user/pass + if (ServerName.User.empty() == false) + User = ServerName.User; + if (ServerName.Password.empty() == false) + Pass = ServerName.Password; + + // Perform simple login + if (Proxy.empty() == true) + { + // Read the initial response + if (ReadResp(Tag,Msg) == false) + return ResultState::TRANSIENT_ERROR; + if (Tag >= 400) + { + _error->Error(_("The server refused the connection and said: %s"), Msg.c_str()); + return ResultState::FATAL_ERROR; + } + + // Send the user + if (WriteMsg(Tag,Msg,"USER %s",User.c_str()) == false) + return ResultState::TRANSIENT_ERROR; + if (Tag >= 400) + { + _error->Error(_("USER failed, server said: %s"), Msg.c_str()); + return ResultState::FATAL_ERROR; + } + + if (Tag == 331) { // 331 User name okay, need password. + // Send the Password + if (WriteMsg(Tag,Msg,"PASS %s",Pass.c_str()) == false) + return ResultState::TRANSIENT_ERROR; + if (Tag >= 400) + { + _error->Error(_("PASS failed, server said: %s"), Msg.c_str()); + return ResultState::FATAL_ERROR; + } + } + + // Enter passive mode + if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true) + TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true); + else + TryPassive = _config->FindB("Acquire::FTP::Passive",true); + } + else + { + // Read the initial response + if (ReadResp(Tag,Msg) == false) + return ResultState::TRANSIENT_ERROR; + if (Tag >= 400) + { + _error->Error(_("The server refused the connection and said: %s"), Msg.c_str()); + return ResultState::TRANSIENT_ERROR; + } + + // Perform proxy script execution + Configuration::Item const *Opts = _config->Tree("Acquire::ftp::ProxyLogin"); + if (Opts == 0 || Opts->Child == 0) + { + _error->Error(_("A proxy server was specified but no login " + "script, Acquire::ftp::ProxyLogin is empty.")); + return ResultState::FATAL_ERROR; + } + Opts = Opts->Child; + + // Iterate over the entire login script + for (; Opts != 0; Opts = Opts->Next) + { + if (Opts->Value.empty() == true) + continue; + + // Substitute the variables into the command + string Tmp = Opts->Value; + Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User); + Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password); + Tmp = SubstVar(Tmp,"$(SITE_USER)",User); + Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass); + if (ServerName.Port != 0) + { + std::string SitePort; + strprintf(SitePort, "%u", ServerName.Port); + Tmp = SubstVar(Tmp,"$(SITE_PORT)", SitePort); + } + else + Tmp = SubstVar(Tmp,"$(SITE_PORT)", "21"); + Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host); + + // Send the command + if (WriteMsg(Tag,Msg,"%s",Tmp.c_str()) == false) + return ResultState::TRANSIENT_ERROR; + if (Tag >= 400) + { + _error->Error(_("Login script command '%s' failed, server said: %s"), Tmp.c_str(), Msg.c_str()); + return ResultState::FATAL_ERROR; + } + } + + // Enter passive mode + TryPassive = false; + if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true) + TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true); + else + { + if (_config->Exists("Acquire::FTP::Proxy::Passive") == true) + TryPassive = _config->FindB("Acquire::FTP::Proxy::Passive",true); + else + TryPassive = _config->FindB("Acquire::FTP::Passive",true); + } + } + + // Force the use of extended commands + if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true) + ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true); + else + ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false); + + // Binary mode + if (WriteMsg(Tag,Msg,"TYPE I") == false) + return ResultState::TRANSIENT_ERROR; + if (Tag >= 400) + { + _error->Error(_("TYPE failed, server said: %s"), Msg.c_str()); + return ResultState::FATAL_ERROR; + } + return ResultState::SUCCESSFUL; +} + /*}}}*/ +// FTPConn::ReadLine - Read a line from the server /*{{{*/ +// --------------------------------------------------------------------- +/* This performs a very simple buffered read. */ +bool FTPConn::ReadLine(string &Text) +{ + if (ServerFd->Fd() == -1) + return false; + + // Suck in a line + while (Len < sizeof(Buffer)) + { + // Scan the buffer for a new line + for (unsigned int I = 0; I != Len; I++) + { + // Escape some special chars + if (Buffer[I] == 0) + Buffer[I] = '?'; + + // End of line? + if (Buffer[I] != '\n') + continue; + + I++; + Text = string(Buffer,I); + memmove(Buffer,Buffer+I,Len - I); + Len -= I; + return true; + } + + // Wait for some data.. + if (WaitFd(ServerFd->Fd(), false, TimeOut) == false) + { + Close(); + return _error->Error(_("Connection timeout")); + } + + // Suck it back + int Res = ServerFd->Read(Buffer + Len, sizeof(Buffer) - Len); + if (Res == 0) + _error->Error(_("Server closed the connection")); + if (Res <= 0) + { + _error->Errno("read",_("Read error")); + Close(); + return false; + } + Len += Res; + } + + return _error->Error(_("A response overflowed the buffer.")); +} + /*}}}*/ +// FTPConn::ReadResp - Read a full response from the server /*{{{*/ +// --------------------------------------------------------------------- +/* This reads a reply code from the server, it handles both p */ +bool FTPConn::ReadResp(unsigned int &Ret,string &Text) +{ + // Grab the first line of the response + string Msg; + if (ReadLine(Msg) == false) + return false; + + // Get the ID code + char *End; + Ret = strtol(Msg.c_str(),&End,10); + if (End - Msg.c_str() != 3) + return _error->Error(_("Protocol corruption")); + + // All done ? + Text = Msg.c_str()+4; + if (*End == ' ') + { + if (Debug == true) + cerr << "<- '" << QuoteString(Text,"") << "'" << endl; + return true; + } + + if (*End != '-') + return _error->Error(_("Protocol corruption")); + + /* Okay, here we do the continued message trick. This is foolish, but + proftpd follows the protocol as specified and wu-ftpd doesn't, so + we filter. I wonder how many clients break if you use proftpd and + put a '- in the 3rd spot in the message? */ + char Leader[4]; + strncpy(Leader,Msg.c_str(),3); + Leader[3] = 0; + while (ReadLine(Msg) == true) + { + // Short, it must be using RFC continuation.. + if (Msg.length() < 4) + { + Text += Msg; + continue; + } + + // Oops, finished + if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == ' ') + { + Text += Msg.c_str()+4; + break; + } + + // This message has the wu-ftpd style reply code prefixed + if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == '-') + { + Text += Msg.c_str()+4; + continue; + } + + // Must be RFC style prefixing + Text += Msg; + } + + if (Debug == true && _error->PendingError() == false) + cerr << "<- '" << QuoteString(Text,"") << "'" << endl; + + return !_error->PendingError(); +} + /*}}}*/ +// FTPConn::WriteMsg - Send a message to the server /*{{{*/ +// --------------------------------------------------------------------- +/* Simple printf like function.. */ +bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...) +{ + va_list args; + va_start(args,Fmt); + + // sprintf the description + char S[400]; + vsnprintf(S,sizeof(S) - 4,Fmt,args); + strcat(S,"\r\n"); + va_end(args); + + if (Debug == true) + cerr << "-> '" << QuoteString(S,"") << "'" << endl; + + // Send it off + unsigned long Len = strlen(S); + unsigned long Start = 0; + while (Len != 0) + { + if (WaitFd(ServerFd->Fd(), true, TimeOut) == false) + { + Close(); + return _error->Error(_("Connection timeout")); + } + + int Res = ServerFd->Write(S + Start, Len); + if (Res <= 0) + { + _error->Errno("write",_("Write error")); + Close(); + return false; + } + + Len -= Res; + Start += Res; + } + + return ReadResp(Ret,Text); +} + /*}}}*/ +// FTPConn::GoPasv - Enter Passive mode /*{{{*/ +// --------------------------------------------------------------------- +/* Try to enter passive mode, the return code does not indicate if passive + mode could or could not be established, only if there was a fatal error. + We have to enter passive mode every time we make a data connection :| */ +bool FTPConn::GoPasv() +{ + /* The PASV command only works on IPv4 sockets, even though it could + in theory suppory IPv6 via an all zeros reply */ + if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET || + ForceExtended == true) + return ExtGoPasv(); + + if (PasvAddr != 0) + freeaddrinfo(PasvAddr); + PasvAddr = 0; + + // Try to enable pasv mode + unsigned int Tag; + string Msg; + if (WriteMsg(Tag,Msg,"PASV") == false) + return false; + + // Unsupported function + string::size_type Pos = Msg.find('('); + if (Tag >= 400) + return true; + + //wu-2.6.2(1) ftp server, returns + //227 Entering Passive Mode 193,219,28,140,150,111 + //without parentheses, let's try to cope with it. + //wget(1) and ftp(1) can. + if (Pos == string::npos) + Pos = Msg.rfind(' '); + else + ++Pos; + + // Still unsupported function + if (Pos == string::npos) + return true; + + // Scan it + unsigned a0,a1,a2,a3,p0,p1; + if (sscanf(Msg.c_str() + Pos,"%u,%u,%u,%u,%u,%u",&a0,&a1,&a2,&a3,&p0,&p1) != 6) + return true; + + /* Some evil servers return 0 to mean their addr. We can actually speak + to these servers natively using IPv6 */ + if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0) + { + // Get the IP in text form + char Name[NI_MAXHOST]; + char Service[NI_MAXSERV]; + getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen, + Name,sizeof(Name),Service,sizeof(Service), + NI_NUMERICHOST|NI_NUMERICSERV); + + struct addrinfo Hints; + memset(&Hints,0,sizeof(Hints)); + Hints.ai_socktype = SOCK_STREAM; + Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family; + Hints.ai_flags |= AI_NUMERICHOST; + + // Get a new passive address. + char Port[100]; + snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1); + if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0) + return true; + return true; + } + + struct addrinfo Hints; + memset(&Hints,0,sizeof(Hints)); + Hints.ai_socktype = SOCK_STREAM; + Hints.ai_family = AF_INET; + Hints.ai_flags |= AI_NUMERICHOST; + + // Get a new passive address. + char Port[100]; + snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1); + char Name[100]; + snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3); + if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0) + return true; + return true; +} + /*}}}*/ +// FTPConn::ExtGoPasv - Enter Extended Passive mode /*{{{*/ +// --------------------------------------------------------------------- +/* Try to enter extended passive mode. See GoPasv above and RFC 2428 */ +bool FTPConn::ExtGoPasv() +{ + if (PasvAddr != 0) + freeaddrinfo(PasvAddr); + PasvAddr = 0; + + // Try to enable pasv mode + unsigned int Tag; + string Msg; + if (WriteMsg(Tag,Msg,"EPSV") == false) + return false; + + // Unsupported function + string::size_type Pos = Msg.find('('); + if (Tag >= 400 || Pos == string::npos) + return true; + + // Scan it + string::const_iterator List[4]; + unsigned Count = 0; + Pos++; + for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); ++I) + { + if (*I != Msg[Pos]) + continue; + if (Count >= 4) + return true; + List[Count++] = I; + } + if (Count != 4) + return true; + + // Break it up .. + unsigned long Proto = 0; + unsigned long Port = 0; + string IP; + IP = string(List[1]+1,List[2]); + Port = atoi(string(List[2]+1,List[3]).c_str()); + if (IP.empty() == false) + Proto = atoi(string(List[0]+1,List[1]).c_str()); + + if (Port == 0) + return false; + + // String version of the port + char PStr[100]; + snprintf(PStr,sizeof(PStr),"%lu",Port); + + // Get the IP in text form + struct addrinfo Hints; + memset(&Hints,0,sizeof(Hints)); + Hints.ai_socktype = SOCK_STREAM; + Hints.ai_flags |= AI_NUMERICHOST; + + /* The RFC defined case, connect to the old IP/protocol using the + new port. */ + if (IP.empty() == true) + { + // Get the IP in text form + char Name[NI_MAXHOST]; + char Service[NI_MAXSERV]; + getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen, + Name,sizeof(Name),Service,sizeof(Service), + NI_NUMERICHOST|NI_NUMERICSERV); + IP = Name; + Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family; + } + else + { + // Get the family.. + Hints.ai_family = 0; + for (unsigned J = 0; AFMap[J].Family != 0; J++) + if (AFMap[J].IETFFamily == Proto) + Hints.ai_family = AFMap[J].Family; + if (Hints.ai_family == 0) + return true; + } + + // Get a new passive address. + if (getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr) != 0) + return true; + + return true; +} + /*}}}*/ +// FTPConn::Size - Return the size of a file /*{{{*/ +// --------------------------------------------------------------------- +/* Grab the file size from the server, 0 means no size or empty file */ +bool FTPConn::Size(const char *Path,unsigned long long &Size) +{ + // Query the size + unsigned int Tag; + string Msg; + Size = 0; + if (WriteMsg(Tag,Msg,"SIZE %s",Path) == false) + return false; + + char *End; + Size = strtoull(Msg.c_str(),&End,10); + if (Tag >= 400 || End == Msg.c_str()) + Size = 0; + return true; +} + /*}}}*/ +// FTPConn::ModTime - Return the modification time of the file /*{{{*/ +// --------------------------------------------------------------------- +/* Like Size no error is returned if the command is not supported. If the + command fails then time is set to the current time of day to fool + date checks. */ +bool FTPConn::ModTime(const char *Path, time_t &Time) +{ + Time = time(&Time); + + // Query the mod time + unsigned int Tag; + string Msg; + if (WriteMsg(Tag,Msg,"MDTM %s",Path) == false) + return false; + if (Tag >= 400 || Msg.empty() == true || isdigit(Msg[0]) == 0) + return true; + + // Parse it + return FTPMDTMStrToTime(Msg.c_str(), Time); +} + /*}}}*/ +// FTPConn::CreateDataFd - Get a data connection /*{{{*/ +// --------------------------------------------------------------------- +/* Create the data connection. Call FinalizeDataFd after this though.. */ +bool FTPConn::CreateDataFd() +{ + close(DataFd); + DataFd = -1; + + // Attempt to enter passive mode. + if (TryPassive == true) + { + if (GoPasv() == false) + return false; + + // Oops, didn't work out, don't bother trying again. + if (PasvAddr == 0) + TryPassive = false; + } + + // Passive mode? + if (PasvAddr != 0) + { + // Get a socket + if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype, + PasvAddr->ai_protocol)) < 0) + return _error->Errno("socket",_("Could not create a socket")); + + // Connect to the server + SetNonBlock(DataFd,true); + if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 && + errno != EINPROGRESS) + return _error->Errno("socket",_("Could not create a socket")); + + /* This implements a timeout for connect by opening the connection + nonblocking */ + if (WaitFd(DataFd,true,TimeOut) == false) + return _error->Error(_("Could not connect data socket, connection timed out")); + unsigned int Err; + unsigned int Len = sizeof(Err); + if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0) + return _error->Errno("getsockopt",_("Failed")); + if (Err != 0) + return _error->Error(_("Could not connect passive socket.")); + + return true; + } + + // Port mode :< + close(DataListenFd); + DataListenFd = -1; + + // Get the information for a listening socket. + struct addrinfo *BindAddr = NULL; + struct addrinfo Hints; + memset(&Hints,0,sizeof(Hints)); + Hints.ai_socktype = SOCK_STREAM; + Hints.ai_flags |= AI_PASSIVE; + Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family; + if (getaddrinfo(0,"0",&Hints,&BindAddr) != 0 || BindAddr == NULL) + return _error->Error(_("getaddrinfo was unable to get a listening socket")); + + // Construct the socket + if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype, + BindAddr->ai_protocol)) < 0) + { + freeaddrinfo(BindAddr); + return _error->Errno("socket",_("Could not create a socket")); + } + + // Bind and listen + if (::bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0) + { + freeaddrinfo(BindAddr); + return _error->Errno("bind",_("Could not bind a socket")); + } + freeaddrinfo(BindAddr); + if (listen(DataListenFd,1) < 0) + return _error->Errno("listen",_("Could not listen on the socket")); + SetNonBlock(DataListenFd,true); + + // Determine the name to send to the remote + struct sockaddr_storage Addr; + socklen_t AddrLen = sizeof(Addr); + if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0) + return _error->Errno("getsockname",_("Could not determine the socket's name")); + + + // Reverse the address. We need the server address and the data port. + char Name[NI_MAXHOST]; + char Service[NI_MAXSERV]; + char Service2[NI_MAXSERV]; + getnameinfo((struct sockaddr *)&Addr,AddrLen, + Name,sizeof(Name),Service,sizeof(Service), + NI_NUMERICHOST|NI_NUMERICSERV); + getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen, + Name,sizeof(Name),Service2,sizeof(Service2), + NI_NUMERICHOST|NI_NUMERICSERV); + + // Send off an IPv4 address in the old port format + if (((struct sockaddr *)&Addr)->sa_family == AF_INET && + ForceExtended == false) + { + // Convert the dots in the quad into commas + for (char *I = Name; *I != 0; I++) + if (*I == '.') + *I = ','; + unsigned long Port = atoi(Service); + + // Send the port command + unsigned int Tag; + string Msg; + if (WriteMsg(Tag,Msg,"PORT %s,%d,%d", + Name, + (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false) + return false; + if (Tag >= 400) + return _error->Error(_("Unable to send PORT command")); + return true; + } + + // Construct an EPRT command + unsigned Proto = 0; + for (unsigned J = 0; AFMap[J].Family != 0; J++) + if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family) + Proto = AFMap[J].IETFFamily; + if (Proto == 0) + return _error->Error(_("Unknown address family %u (AF_*)"), + ((struct sockaddr *)&Addr)->sa_family); + + // Send the EPRT command + unsigned int Tag; + string Msg; + if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false) + return false; + if (Tag >= 400) + return _error->Error(_("EPRT failed, server said: %s"),Msg.c_str()); + return true; +} + /*}}}*/ +// FTPConn::Finalize - Complete the Data connection /*{{{*/ +// --------------------------------------------------------------------- +/* If the connection is in port mode this waits for the other end to hook + up to us. */ +bool FTPConn::Finalize() +{ + // Passive mode? Do nothing + if (PasvAddr != 0) + return true; + + // Close any old socket.. + close(DataFd); + DataFd = -1; + + // Wait for someone to connect.. + if (WaitFd(DataListenFd,false,TimeOut) == false) + return _error->Error(_("Data socket connect timed out")); + + // Accept the connection + struct sockaddr_in Addr; + socklen_t Len = sizeof(Addr); + DataFd = accept(DataListenFd,(struct sockaddr *)&Addr,&Len); + if (DataFd < 0) + return _error->Errno("accept",_("Unable to accept connection")); + + close(DataListenFd); + DataListenFd = -1; + + return true; +} + /*}}}*/ +// FTPConn::Get - Get a file /*{{{*/ +// --------------------------------------------------------------------- +/* This opens a data connection, sends REST and RETR and then + transfers the file over. */ +bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, + Hashes &Hash,bool &Missing, unsigned long long MaximumSize, + pkgAcqMethod *Owner) +{ + Missing = false; + if (CreateDataFd() == false) + return false; + + unsigned int Tag; + string Msg; + if (Resume != 0) + { + if (WriteMsg(Tag,Msg,"REST %u",Resume) == false) + return false; + if (Tag >= 400) + Resume = 0; + } + + if (To.Truncate(Resume) == false) + return false; + + if (To.Seek(0) == false) + return false; + + if (Resume != 0) + { + if (Hash.AddFD(To,Resume) == false) + { + _error->Errno("read",_("Problem hashing file")); + return false; + } + } + + // Send the get command + if (WriteMsg(Tag,Msg,"RETR %s",Path) == false) + return false; + + if (Tag >= 400) + { + if (Tag == 550) + Missing = true; + return _error->Error(_("Unable to fetch file, server said '%s'"),Msg.c_str()); + } + + // Finish off the data connection + if (Finalize() == false) + return false; + + // Copy loop + unsigned char Buffer[4096]; + while (1) + { + // Wait for some data.. + if (WaitFd(DataFd,false,TimeOut) == false) + { + Close(); + return _error->Error(_("Data socket timed out")); + } + + // Read the data.. + int Res = read(DataFd,Buffer,sizeof(Buffer)); + if (Res == 0) + break; + if (Res < 0) + { + if (errno == EAGAIN) + continue; + break; + } + + Hash.Add(Buffer,Res); + if (To.Write(Buffer,Res) == false) + { + Close(); + return false; + } + + if (MaximumSize > 0 && To.Tell() > MaximumSize) + { + Owner->SetFailReason("MaximumSizeExceeded"); + return _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"), + To.Tell(), MaximumSize); + } + } + + // All done + close(DataFd); + DataFd = -1; + + // Read the closing message from the server + if (ReadResp(Tag,Msg) == false) + return false; + if (Tag >= 400) + return _error->Error(_("Data transfer failed, server said '%s'"),Msg.c_str()); + return true; +} + /*}}}*/ + +// FtpMethod::FtpMethod - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +FtpMethod::FtpMethod() : aptAuthConfMethod("ftp", "1.0", SendConfig | SendURIEncoded) +{ + SeccompFlags = aptMethod::BASE | aptMethod::NETWORK; + signal(SIGTERM,SigTerm); + signal(SIGINT,SigTerm); + + Server = 0; + FailFd = -1; +} + /*}}}*/ +// FtpMethod::SigTerm - Handle a fatal signal /*{{{*/ +// --------------------------------------------------------------------- +/* This closes and timestamps the open file. This is necessary to get + resume behavior on user abort */ +void FtpMethod::SigTerm(int) +{ + if (FailFd == -1) + _exit(100); + + // Timestamp + struct timeval times[2]; + times[0].tv_sec = FailTime; + times[1].tv_sec = FailTime; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(FailFile.c_str(), times); + + close(FailFd); + + _exit(100); +} + /*}}}*/ +// FtpMethod::Configuration - Handle a configuration message /*{{{*/ +// --------------------------------------------------------------------- +/* We stash the desired pipeline depth */ +bool FtpMethod::Configuration(string Message) +{ + if (aptAuthConfMethod::Configuration(Message) == false) + return false; + + TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut); + + return true; +} + /*}}}*/ +// FtpMethod::Fetch - Fetch a file /*{{{*/ +// --------------------------------------------------------------------- +/* Fetch a single file, called by the base class.. */ +bool FtpMethod::Fetch(FetchItem *Itm) +{ + URI Get(Itm->Uri); + auto const File = DecodeSendURI(Get.Path); + FetchResult Res; + Res.Filename = Itm->DestFile; + Res.IMSHit = false; + + MaybeAddAuthTo(Get); + + // Connect to the server + if (Server == 0 || Server->Comp(Get) == false) + { + delete Server; + Server = new FTPConn(Get); + } + + // Could not connect is a transient error.. + switch (Server->Open(this)) + { + case ResultState::TRANSIENT_ERROR: + Server->Close(); + Fail(true); + return true; + case ResultState::FATAL_ERROR: + Server->Close(); + Fail(false); + return true; + case ResultState::SUCCESSFUL: + break; + } + + // Get the files information + Status(_("Query")); + unsigned long long Size; + if (not Server->Size(File.c_str(), Size) || + not Server->ModTime(File.c_str(), FailTime)) + { + Fail(true); + return true; + } + Res.Size = Size; + + // See if it is an IMS hit + if (Itm->LastModified == FailTime) + { + Res.Size = 0; + Res.IMSHit = true; + URIDone(Res); + return true; + } + + // See if the file exists + struct stat Buf; + if (stat(Itm->DestFile.c_str(),&Buf) == 0) + { + if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime) + { + Res.Size = Buf.st_size; + Res.LastModified = Buf.st_mtime; + Res.ResumePoint = Buf.st_size; + URIDone(Res); + return true; + } + + // Resume? + if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size) + Res.ResumePoint = Buf.st_size; + } + + // Open the file + Hashes Hash(Itm->ExpectedHashes); + { + FileFd Fd(Itm->DestFile,FileFd::WriteAny); + if (_error->PendingError() == true) + return false; + + URIStart(Res); + + FailFile = Itm->DestFile; + (void)(FailFile.c_str()); // Make sure we don't do a malloc in the signal handler + FailFd = Fd.Fd(); + + bool Missing; + if (not Server->Get(File.c_str(), Fd, Res.ResumePoint, Hash, Missing, Itm->MaximumSize, this)) + { + Fd.Close(); + + // Timestamp + struct timeval times[2]; + times[0].tv_sec = FailTime; + times[1].tv_sec = FailTime; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(FailFile.c_str(), times); + + // If the file is missing we hard fail and delete the destfile + // otherwise transient fail + if (Missing == true) { + RemoveFile("ftp", FailFile); + return false; + } + Fail(true); + return true; + } + + Res.Size = Fd.Size(); + + // Timestamp + struct timeval times[2]; + times[0].tv_sec = FailTime; + times[1].tv_sec = FailTime; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(Fd.Name().c_str(), times); + FailFd = -1; + } + + Res.LastModified = FailTime; + Res.TakeHashes(Hash); + + URIDone(Res); + + return true; +} + /*}}}*/ + +int main(int, const char *argv[]) +{ + /* See if we should be come the http client - we do this for http + proxy urls */ + if (getenv("ftp_proxy") != 0) + { + URI Proxy(string(getenv("ftp_proxy"))); + + // Run the HTTP method + if (Proxy.Access == "http") + { + // Copy over the environment setting + char S[300]; + snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy")); + putenv(S); + putenv((char *)"no_proxy="); + + // Run the http method + string Path = flNotFile(argv[0]) + "http"; + execl(Path.c_str(),Path.c_str(),(char *)NULL); + cerr << _("Unable to invoke ") << Path << endl; + exit(100); + } + } + return FtpMethod().Run(); +} diff --git a/methods/ftp.h b/methods/ftp.h new file mode 100644 index 0000000..da9887b --- /dev/null +++ b/methods/ftp.h @@ -0,0 +1,91 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + FTP Acquire Method - This is the FTP acquire method for APT. + + ##################################################################### */ + /*}}}*/ +#ifndef APT_FTP_H +#define APT_FTP_H + +#include "aptmethod.h" +#include "connect.h" +#include <apt-pkg/strutl.h> + +#include <string> +#include <sys/socket.h> +#include <sys/types.h> +#include <time.h> + +class FTPConn +{ + char Buffer[1024*10]; + unsigned long Len; + std::unique_ptr<MethodFd> ServerFd; + int DataFd; + int DataListenFd; + URI ServerName; + bool ForceExtended; + bool TryPassive; + bool Debug; + + struct addrinfo *PasvAddr; + + // Generic Peer Address + struct sockaddr_storage PeerAddr; + socklen_t PeerAddrLen; + + // Generic Server Address (us) + struct sockaddr_storage ServerAddr; + socklen_t ServerAddrLen; + + // Private helper functions + bool ReadLine(std::string &Text); + ResultState Login(); + bool CreateDataFd(); + bool Finalize(); + + public: + + bool Comp(URI Other) {return Other.Host == ServerName.Host && Other.Port == ServerName.Port && Other.User == ServerName.User && Other.Password == ServerName.Password; }; + + // Raw connection IO + bool ReadResp(unsigned int &Ret,std::string &Text); + bool WriteMsg(unsigned int &Ret,std::string &Text,const char *Fmt,...); + + // Connection control + ResultState Open(aptMethod *Owner); + void Close(); + bool GoPasv(); + bool ExtGoPasv(); + + // Query + bool Size(const char *Path,unsigned long long &Size); + bool ModTime(const char *Path, time_t &Time); + bool Get(const char *Path,FileFd &To,unsigned long long Resume, + Hashes &MD5,bool &Missing, unsigned long long MaximumSize, + pkgAcqMethod *Owner); + + explicit FTPConn(URI Srv); + ~FTPConn(); +}; + +class FtpMethod : public aptAuthConfMethod +{ + virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; + virtual bool Configuration(std::string Message) APT_OVERRIDE; + + FTPConn *Server; + + static std::string FailFile; + static int FailFd; + static time_t FailTime; + static APT_NORETURN void SigTerm(int); + + public: + + FtpMethod(); +}; + +#endif diff --git a/methods/gpgv.cc b/methods/gpgv.cc new file mode 100644 index 0000000..267b43e --- /dev/null +++ b/methods/gpgv.cc @@ -0,0 +1,597 @@ +#include <config.h> + +#include "aptmethod.h" +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/gpgv.h> +#include <apt-pkg/strutl.h> + +#include <ctype.h> +#include <errno.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/wait.h> +#include <unistd.h> + +#include <algorithm> +#include <array> +#include <iostream> +#include <iterator> +#include <map> +#include <sstream> +#include <string> +#include <vector> + +#include <apti18n.h> + +using std::string; +using std::vector; + +#define GNUPGPREFIX "[GNUPG:]" +#define GNUPGBADSIG "[GNUPG:] BADSIG" +#define GNUPGERRSIG "[GNUPG:] ERRSIG" +#define GNUPGNOPUBKEY "[GNUPG:] NO_PUBKEY" +#define GNUPGVALIDSIG "[GNUPG:] VALIDSIG" +#define GNUPGGOODSIG "[GNUPG:] GOODSIG" +#define GNUPGEXPKEYSIG "[GNUPG:] EXPKEYSIG" +#define GNUPGEXPSIG "[GNUPG:] EXPSIG" +#define GNUPGREVKEYSIG "[GNUPG:] REVKEYSIG" +#define GNUPGNODATA "[GNUPG:] NODATA" +#define APTKEYWARNING "[APTKEY:] WARNING" +#define APTKEYERROR "[APTKEY:] ERROR" + +struct Digest { + enum class State { + Untrusted, + Weak, + Trusted, + } state; + char name[32]; + + State getState() const { + std::string optionUntrusted; + std::string optionWeak; + strprintf(optionUntrusted, "APT::Hashes::%s::Untrusted", name); + strprintf(optionWeak, "APT::Hashes::%s::Weak", name); + if (_config->FindB(optionUntrusted, false) == true) + return State::Untrusted; + if (_config->FindB(optionWeak, false) == true) + return State::Weak; + + return state; + } +}; + +static constexpr Digest Digests[] = { + {Digest::State::Untrusted, "Invalid digest"}, + {Digest::State::Untrusted, "MD5"}, + {Digest::State::Untrusted, "SHA1"}, + {Digest::State::Untrusted, "RIPE-MD/160"}, + {Digest::State::Trusted, "Reserved digest"}, + {Digest::State::Trusted, "Reserved digest"}, + {Digest::State::Trusted, "Reserved digest"}, + {Digest::State::Trusted, "Reserved digest"}, + {Digest::State::Trusted, "SHA256"}, + {Digest::State::Trusted, "SHA384"}, + {Digest::State::Trusted, "SHA512"}, + {Digest::State::Trusted, "SHA224"}, +}; + +static Digest FindDigest(std::string const & Digest) +{ + int id = atoi(Digest.c_str()); + if (id >= 0 && static_cast<unsigned>(id) < APT_ARRAY_SIZE(Digests)) + { + return Digests[id]; + } else { + return Digests[0]; + } +} + +struct Signer { + std::string key; + std::string note; +}; +static bool IsTheSameKey(std::string const &validsig, std::string const &goodsig) { + // VALIDSIG reports a fingerprint (40 = 24 + 16), GOODSIG can be longid (16) or + // fingerprint according to documentation in DETAILS.gz + if (goodsig.length() == 40 + strlen("GOODSIG ")) + return validsig.compare(0, 40, goodsig, strlen("GOODSIG "), 40) == 0; + return validsig.compare(24, 16, goodsig, strlen("GOODSIG "), 16) == 0; +} + +struct APT_HIDDEN SignersStorage { + std::vector<std::string> Good; + std::vector<std::string> Bad; + std::vector<std::string> Worthless; + // a worthless signature is a expired or revoked one + std::vector<Signer> SoonWorthless; + std::vector<std::string> NoPubKey; + std::vector<std::string> Valid; + std::vector<std::string> SignedBy; +}; +class GPGVMethod : public aptMethod +{ + private: + string VerifyGetSigners(const char *file, const char *outfile, + vector<string> const &keyFpts, + vector<string> const &keyFiles, + SignersStorage &Signers); + string VerifyGetSignersWithLegacy(const char *file, const char *outfile, + vector<string> const &keyFpts, + vector<string> const &keyFiles, + SignersStorage &Signers); + + protected: + virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE; + public: + GPGVMethod() : aptMethod("gpgv", "1.1", SingleInstance | SendConfig | SendURIEncoded){}; +}; +static void PushEntryWithKeyID(std::vector<std::string> &Signers, char * const buffer, bool const Debug) +{ + char * const msg = buffer + sizeof(GNUPGPREFIX); + char *p = msg; + // skip the message + while (*p && !isspace(*p)) + ++p; + // skip the separator whitespace + ++p; + // skip the hexdigit fingerprint + while (*p && isxdigit(*p)) + ++p; + // cut the rest from the message + *p = '\0'; + if (Debug == true) + std::clog << "Got " << msg << " !" << std::endl; + Signers.push_back(msg); +} +static void PushEntryWithUID(std::vector<std::string> &Signers, char * const buffer, bool const Debug) +{ + std::string msg = buffer + sizeof(GNUPGPREFIX); + auto const nuke = msg.find_last_not_of("\n\t\r"); + if (nuke != std::string::npos) + msg.erase(nuke + 1); + if (Debug == true) + std::clog << "Got " << msg << " !" << std::endl; + Signers.push_back(msg); +} +static void implodeVector(std::vector<std::string> const &vec, std::ostream &out, char const * const sep) +{ + if (vec.empty()) + return; + std::copy(vec.begin(), std::prev(vec.end()), std::ostream_iterator<std::string>(out, sep)); + out << *vec.rbegin(); + return; +} +string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, + vector<string> const &keyFpts, + vector<string> const &keyFiles, + SignersStorage &Signers) +{ + bool const Debug = DebugEnabled(); + + if (Debug == true) + std::clog << "inside VerifyGetSigners" << std::endl; + + int fd[2]; + + if (pipe(fd) < 0) + return "Couldn't create pipe"; + + pid_t pid = fork(); + if (pid < 0) + return string("Couldn't spawn new process") + strerror(errno); + else if (pid == 0) + { + std::ostringstream keys; + implodeVector(keyFiles, keys, ","); + setenv("APT_KEY_NO_LEGACY_KEYRING", "1", true); + ExecGPGV(outfile, file, 3, fd, keys.str()); + } + close(fd[1]); + + FILE *pipein = fdopen(fd[0], "r"); + + // Loop over the output of apt-key (which really is gnupg), and check the signatures. + std::vector<std::string> ErrSigners; + std::map<std::string, std::vector<std::string>> SubKeyMapping; + size_t buffersize = 0; + char *buffer = NULL; + bool gotNODATA = false; + while (1) + { + if (getline(&buffer, &buffersize, pipein) == -1) + break; + if (Debug == true) + std::clog << "Read: " << buffer << std::endl; + + // Push the data into three separate vectors, which + // we later concatenate. They're kept separate so + // if we improve the apt method communication stuff later + // it will be better. + if (strncmp(buffer, GNUPGBADSIG, sizeof(GNUPGBADSIG)-1) == 0) + PushEntryWithUID(Signers.Bad, buffer, Debug); + else if (strncmp(buffer, GNUPGERRSIG, sizeof(GNUPGERRSIG)-1) == 0) + PushEntryWithKeyID(ErrSigners, buffer, Debug); + else if (strncmp(buffer, GNUPGNOPUBKEY, sizeof(GNUPGNOPUBKEY)-1) == 0) + { + PushEntryWithKeyID(Signers.NoPubKey, buffer, Debug); + ErrSigners.erase(std::remove_if(ErrSigners.begin(), ErrSigners.end(), [&](std::string const &errsig) { + return errsig.compare(strlen("ERRSIG "), 16, buffer, sizeof(GNUPGNOPUBKEY), 16) == 0; }), ErrSigners.end()); + } + else if (strncmp(buffer, GNUPGNODATA, sizeof(GNUPGNODATA)-1) == 0) + gotNODATA = true; + else if (strncmp(buffer, GNUPGEXPKEYSIG, sizeof(GNUPGEXPKEYSIG)-1) == 0) + PushEntryWithUID(Signers.Worthless, buffer, Debug); + else if (strncmp(buffer, GNUPGEXPSIG, sizeof(GNUPGEXPSIG)-1) == 0) + PushEntryWithUID(Signers.Worthless, buffer, Debug); + else if (strncmp(buffer, GNUPGREVKEYSIG, sizeof(GNUPGREVKEYSIG)-1) == 0) + PushEntryWithUID(Signers.Worthless, buffer, Debug); + else if (strncmp(buffer, GNUPGGOODSIG, sizeof(GNUPGGOODSIG)-1) == 0) + PushEntryWithKeyID(Signers.Good, buffer, Debug); + else if (strncmp(buffer, GNUPGVALIDSIG, sizeof(GNUPGVALIDSIG)-1) == 0) + { + std::istringstream iss(buffer + sizeof(GNUPGVALIDSIG)); + vector<string> tokens{std::istream_iterator<string>{iss}, + std::istream_iterator<string>{}}; + auto const sig = tokens[0]; + // Reject weak digest algorithms + Digest digest = FindDigest(tokens[7]); + switch (digest.getState()) { + case Digest::State::Weak: + // Treat them like an expired key: For that a message about expiry + // is emitted, a VALIDSIG, but no GOODSIG. + Signers.SoonWorthless.push_back({sig, digest.name}); + if (Debug == true) + std::clog << "Got weak VALIDSIG, key ID: " << sig << std::endl; + break; + case Digest::State::Untrusted: + // Treat them like an expired key: For that a message about expiry + // is emitted, a VALIDSIG, but no GOODSIG. + Signers.Worthless.push_back(sig); + Signers.Good.erase(std::remove_if(Signers.Good.begin(), Signers.Good.end(), [&](std::string const &goodsig) { + return IsTheSameKey(sig, goodsig); }), Signers.Good.end()); + if (Debug == true) + std::clog << "Got untrusted VALIDSIG, key ID: " << sig << std::endl; + break; + + case Digest::State::Trusted: + if (Debug == true) + std::clog << "Got trusted VALIDSIG, key ID: " << sig << std::endl; + break; + } + + Signers.Valid.push_back(sig); + + if (tokens.size() > 9 && sig != tokens[9]) + SubKeyMapping[tokens[9]].emplace_back(sig); + } + else if (strncmp(buffer, APTKEYWARNING, sizeof(APTKEYWARNING)-1) == 0) + Warning(buffer + sizeof(APTKEYWARNING)); + else if (strncmp(buffer, APTKEYERROR, sizeof(APTKEYERROR)-1) == 0) + _error->Error("%s", buffer + sizeof(APTKEYERROR)); + } + fclose(pipein); + free(buffer); + std::move(ErrSigners.begin(), ErrSigners.end(), std::back_inserter(Signers.Worthless)); + + // apt-key has a --keyid parameter, but this requires gpg, so we call it without it + // and instead check after the fact which keyids where used for verification + if (keyFpts.empty() == false) + { + if (Debug == true) + { + std::clog << "GoodSigs needs to be limited to keyid(s): "; + implodeVector(keyFpts, std::clog, ", "); + std::clog << "\n"; + } + std::vector<std::string> filteredGood; + for (auto &&good: Signers.Good) + { + if (Debug == true) + std::clog << "Key " << good << " is good sig, is it also a valid and allowed one? "; + bool found = false; + for (auto l : keyFpts) + { + bool exactKey = false; + if (APT::String::Endswith(l, "!")) + { + exactKey = true; + l.erase(l.length() - 1); + } + if (IsTheSameKey(l, good)) + { + // GOODSIG might be "just" a longid, so we check VALIDSIG which is always a fingerprint + if (std::find(Signers.Valid.cbegin(), Signers.Valid.cend(), l) == Signers.Valid.cend()) + continue; + found = true; + Signers.SignedBy.push_back(l + "!"); + break; + } + else if (exactKey == false) + { + auto const primary = SubKeyMapping.find(l); + if (primary == SubKeyMapping.end()) + continue; + auto const validsubkeysig = std::find_if(primary->second.cbegin(), primary->second.cend(), [&](auto const subkey) { + return IsTheSameKey(subkey, good) && std::find(Signers.Valid.cbegin(), Signers.Valid.cend(), subkey) != Signers.Valid.cend(); + }); + if (validsubkeysig != primary->second.cend()) + { + found = true; + Signers.SignedBy.push_back(l); + Signers.SignedBy.push_back(*validsubkeysig + "!"); + break; + } + } + } + if (Debug) + std::clog << (found ? "yes" : "no") << "\n"; + if (found) + filteredGood.emplace_back(std::move(good)); + else + Signers.NoPubKey.emplace_back(std::move(good)); + } + Signers.Good= std::move(filteredGood); + } + else + { + // for gpg an expired key is valid, too, but we want only the valid & good ones + for (auto const &v : Signers.Valid) + if (std::any_of(Signers.Good.begin(), Signers.Good.end(), + [&v](std::string const &g) { return IsTheSameKey(v, g); })) + Signers.SignedBy.push_back(v + "!"); + for (auto sub : SubKeyMapping) + if (std::any_of(sub.second.begin(), sub.second.end(), + [&](std::string const &s) { + if (std::find(Signers.Valid.begin(), Signers.Valid.end(), s) == Signers.Valid.end()) + return false; + return std::any_of(Signers.Good.begin(), Signers.Good.end(), + [&s](std::string const &g) { return IsTheSameKey(s, g); }); + })) + Signers.SignedBy.push_back(sub.first); + } + std::sort(Signers.SignedBy.begin(), Signers.SignedBy.end()); + + int status; + waitpid(pid, &status, 0); + if (Debug == true) + { + ioprintf(std::clog, "gpgv exited with status %i\n", WEXITSTATUS(status)); + } + + if (Debug) + { + std::cerr << "Summary:\n Good: "; + implodeVector(Signers.Good, std::cerr, ", "); + std::cerr << "\n Valid: "; + implodeVector(Signers.Valid, std::cerr, ", "); + std::cerr << "\n Bad: "; + implodeVector(Signers.Bad, std::cerr, ", "); + std::cerr << "\n Worthless: "; + implodeVector(Signers.Worthless, std::cerr, ", "); + std::cerr << "\n SoonWorthless: "; + std::for_each(Signers.SoonWorthless.begin(), Signers.SoonWorthless.end(), [](Signer const &sig) { std::cerr << sig.key << ", "; }); + std::cerr << "\n NoPubKey: "; + implodeVector(Signers.NoPubKey, std::cerr, ", "); + std::cerr << "\n Signed-By: "; + implodeVector(Signers.SignedBy, std::cerr, ", "); + std::cerr << std::endl << " NODATA: " << (gotNODATA ? "yes" : "no") << std::endl; + } + + if (WEXITSTATUS(status) == 112) + { + // acquire system checks for "NODATA" to generate GPG errors (the others are only warnings) + std::string errmsg; + //TRANSLATORS: %s is a single techy word like 'NODATA' + strprintf(errmsg, _("Clearsigned file isn't valid, got '%s' (does the network require authentication?)"), "NODATA"); + return errmsg; + } + else if (gotNODATA) + { + // acquire system checks for "NODATA" to generate GPG errors (the others are only warnings) + std::string errmsg; + //TRANSLATORS: %s is a single techy word like 'NODATA' + strprintf(errmsg, _("Signed file isn't valid, got '%s' (does the network require authentication?)"), "NODATA"); + return errmsg; + } + else if (WEXITSTATUS(status) == 0) + { + if (keyFpts.empty() == false) + { + // gpgv will report success, but we want to enforce a certain keyring + // so if we haven't found the key the valid we found is in fact invalid + if (Signers.Good.empty()) + return _("At least one invalid signature was encountered."); + } + else + { + if (Signers.Good.empty()) + return _("Internal error: Good signature, but could not determine key fingerprint?!"); + } + return ""; + } + else if (WEXITSTATUS(status) == 1) + return _("At least one invalid signature was encountered."); + else if (WEXITSTATUS(status) == 111) + return _("Could not execute 'apt-key' to verify signature (is gnupg installed?)"); + else + return _("Unknown error executing apt-key"); +} +string GPGVMethod::VerifyGetSignersWithLegacy(const char *file, const char *outfile, + vector<string> const &keyFpts, + vector<string> const &keyFiles, + SignersStorage &Signers) +{ + string const msg = VerifyGetSigners(file, outfile, keyFpts, keyFiles, Signers); + if (_error->PendingError()) + return msg; + + // Bad signature always remains bad, no need to retry against trusted.gpg + if (!Signers.Bad.empty()) + return msg; + + // We do not have a key file pinned, did not find a good signature, but found + // missing keys - let's retry with trusted.gpg + if (keyFiles.empty() && Signers.Valid.empty() && !Signers.NoPubKey.empty()) + { + std::vector<std::string> legacyKeyFiles{_config->FindFile("Dir::Etc::trusted")}; + if (legacyKeyFiles[0].empty()) + return msg; + if (DebugEnabled()) + std::clog << "Retrying against " << legacyKeyFiles[0] << "\n"; + + SignersStorage legacySigners; + + string const legacyMsg = VerifyGetSigners(file, outfile, keyFpts, legacyKeyFiles, legacySigners); + if (_error->PendingError()) + return legacyMsg; + // Hooray, we found a key apparently, something verified as good or bad + if (!legacySigners.Valid.empty() || !legacySigners.Bad.empty()) + { + std::string warning; + strprintf(warning, + _("Key is stored in legacy trusted.gpg keyring (%s), see the DEPRECATION section in apt-key(8) for details."), + legacyKeyFiles[0].c_str()); + Warning(std::move(warning)); + Signers = std::move(legacySigners); + return legacyMsg; + } + + } + return msg; +} +static std::string GenerateKeyFile(std::string const key) +{ + FileFd fd; + GetTempFile("apt-key.XXXXXX.asc", false, &fd); + fd.Write(key.data(), key.size()); + return fd.Name(); +} + +bool GPGVMethod::URIAcquire(std::string const &Message, FetchItem *Itm) +{ + URI const Get(Itm->Uri); + std::string const Path = DecodeSendURI(Get.Host + Get.Path); // To account for relative paths + SignersStorage Signers; + + std::vector<std::string> keyFpts, keyFiles; + struct TemporaryFile + { + std::string name = ""; + ~TemporaryFile() { RemoveFile("~TemporaryFile", name); } + } tmpKey; + + std::string SignedBy = DeQuoteString(LookupTag(Message, "Signed-By")); + + if (SignedBy.find("-----BEGIN PGP PUBLIC KEY BLOCK-----") != std::string::npos) + { + tmpKey.name = GenerateKeyFile(SignedBy); + keyFiles.emplace_back(tmpKey.name); + } + else + { + for (auto &&key : VectorizeString(SignedBy, ',')) + if (key.empty() == false && key[0] == '/') + keyFiles.emplace_back(std::move(key)); + else + keyFpts.emplace_back(std::move(key)); + } + + // Run apt-key on file, extract contents and get the key ID of the signer + string const msg = VerifyGetSignersWithLegacy(Path.c_str(), Itm->DestFile.c_str(), keyFpts, keyFiles, Signers); + if (_error->PendingError()) + return false; + + // Check if all good signers are soon worthless and warn in that case + if (std::all_of(Signers.Good.begin(), Signers.Good.end(), [&](std::string const &good) { + return std::any_of(Signers.SoonWorthless.begin(), Signers.SoonWorthless.end(), [&](Signer const &weak) { + return IsTheSameKey(weak.key, good); + }); + })) + { + for (auto const & Signer : Signers.SoonWorthless) + { + std::string msg; + // TRANSLATORS: The second %s is the reason and is untranslated for repository owners. + strprintf(msg, _("Signature by key %s uses weak digest algorithm (%s)"), Signer.key.c_str(), Signer.note.c_str()); + Warning(std::move(msg)); + } + } + + if (Signers.Good.empty() || !Signers.Bad.empty() || !Signers.NoPubKey.empty()) + { + string errmsg; + // In this case, something bad probably happened, so we just go + // with what the other method gave us for an error message. + if (Signers.Bad.empty() && Signers.Worthless.empty() && Signers.NoPubKey.empty()) + errmsg = msg; + else + { + if (!Signers.Bad.empty()) + { + errmsg += _("The following signatures were invalid:\n"); + for (auto const &I : Signers.Bad) + errmsg.append(I).append("\n"); + } + if (!Signers.Worthless.empty()) + { + errmsg += _("The following signatures were invalid:\n"); + for (auto const &I : Signers.Worthless) + errmsg.append(I).append("\n"); + } + if (!Signers.NoPubKey.empty()) + { + errmsg += _("The following signatures couldn't be verified because the public key is not available:\n"); + for (auto const &I : Signers.NoPubKey) + errmsg.append(I).append("\n"); + } + } + // this is only fatal if we have no good sigs or if we have at + // least one bad signature. good signatures and NoPubKey signatures + // happen easily when a file is signed with multiple signatures + if (Signers.Good.empty() or !Signers.Bad.empty()) + return _error->Error("%s", errmsg.c_str()); + } + + std::unordered_map<std::string, std::string> fields; + fields.emplace("URI", Itm->Uri); + fields.emplace("Filename", Itm->DestFile); + if (Signers.SignedBy.empty() == false) + { + std::ostringstream out; + implodeVector(Signers.SignedBy, out, "\n"); + fields.emplace("Signed-By", out.str()); + } + { + // Just pass the raw output up, because passing it as a real data + // structure is too difficult with the method stuff. We keep it + // as three separate vectors for future extensibility. + std::vector<std::string> gpgvoutput; + std::move(Signers.Good.begin(), Signers.Good.end(), std::back_inserter(gpgvoutput)); + std::move(Signers.Bad.begin(), Signers.Bad.end(), std::back_inserter(gpgvoutput)); + std::move(Signers.NoPubKey.begin(), Signers.NoPubKey.end(), std::back_inserter(gpgvoutput)); + if (gpgvoutput.empty() == false) + { + std::ostringstream out; + implodeVector(gpgvoutput, out, "\n"); + fields.emplace("GPGVOutput", out.str()); + } + } + SendMessage("201 URI Done", std::move(fields)); + Dequeue(); + + if (DebugEnabled()) + std::clog << "apt-key succeeded\n"; + + return true; +} + + +int main() +{ + return GPGVMethod().Run(); +} diff --git a/methods/http.cc b/methods/http.cc new file mode 100644 index 0000000..b4519a8 --- /dev/null +++ b/methods/http.cc @@ -0,0 +1,1053 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + HTTP Acquire Method - This is the HTTP acquire method for APT. + + It uses HTTP/1.1 and many of the fancy options there-in, such as + pipelining, range, if-range and so on. + + It is based on a doubly buffered select loop. A groupe of requests are + fed into a single output buffer that is constantly fed out the + socket. This provides ideal pipelining as in many cases all of the + requests will fit into a single packet. The input socket is buffered + the same way and fed into the fd for the file (may be a pipe in future). + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/proxy.h> +#include <apt-pkg/string_view.h> +#include <apt-pkg/strutl.h> + +#include <chrono> +#include <cstring> +#include <iostream> +#include <sstream> +#include <arpa/inet.h> +#include <errno.h> +#include <signal.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/select.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <unistd.h> + +#include "config.h" +#include "connect.h" +#include "http.h" + +#include <apti18n.h> + +#ifdef HAVE_SYSTEMD +#include <systemd/sd-login.h> +#endif + /*}}}*/ +using namespace std; + +unsigned long long CircleBuf::BwReadLimit=0; +unsigned long long CircleBuf::BwTickReadData=0; +std::chrono::steady_clock::duration CircleBuf::BwReadTick{0}; +const unsigned int CircleBuf::BW_HZ=10; + +// CircleBuf::CircleBuf - Circular input buffer /*{{{*/ +// --------------------------------------------------------------------- +/* */ +CircleBuf::CircleBuf(HttpMethod const * const Owner, unsigned long long Size) + : Size(Size), Hash(NULL), TotalWriten(0) +{ + Buf = new unsigned char[Size]; + Reset(); + + CircleBuf::BwReadLimit = Owner->ConfigFindI("Dl-Limit", 0) * 1024; +} + /*}}}*/ +// CircleBuf::Reset - Reset to the default state /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void CircleBuf::Reset() +{ + InP = 0; + OutP = 0; + StrPos = 0; + TotalWriten = 0; + MaxGet = (unsigned long long)-1; + OutQueue = string(); + if (Hash != NULL) + { + delete Hash; + Hash = NULL; + } +} + /*}}}*/ +// CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/ +// --------------------------------------------------------------------- +/* This fills up the buffer with as much data as is in the FD, assuming it + is non-blocking.. */ +bool CircleBuf::Read(std::unique_ptr<MethodFd> const &Fd) +{ + size_t ReadThisCycle = 0; + while (1) + { + // Woops, buffer is full + if (InP - OutP == Size) + return true; + + // what's left to read in this tick + unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ; + + if(CircleBuf::BwReadLimit) { + auto const now = std::chrono::steady_clock::now().time_since_epoch(); + auto const d = now - CircleBuf::BwReadTick; + + auto const tickLen = std::chrono::microseconds(std::chrono::seconds(1)) / BW_HZ; + if(d > tickLen) { + CircleBuf::BwReadTick = now; + CircleBuf::BwTickReadData = 0; + } + + if(CircleBuf::BwTickReadData >= BwReadMax) { + usleep(tickLen.count()); + return true; + } + } + + // Write the buffer segment + ssize_t Res; + if(CircleBuf::BwReadLimit) { + Res = Fd->Read(Buf + (InP % Size), + BwReadMax > LeftRead() ? LeftRead() : BwReadMax); + } else + Res = Fd->Read(Buf + (InP % Size), LeftRead()); + + if(Res > 0 && BwReadLimit > 0) + CircleBuf::BwTickReadData += Res; + + if (Res == 0) + return ReadThisCycle != 0; + if (Res < 0) + { + if (errno == EAGAIN) + return true; + return false; + } + + InP += Res; + ReadThisCycle += Res; + } +} + /*}}}*/ +// CircleBuf::Read - Put the string into the buffer /*{{{*/ +// --------------------------------------------------------------------- +/* This will hold the string in and fill the buffer with it as it empties */ +bool CircleBuf::Read(string const &Data) +{ + OutQueue.append(Data); + FillOut(); + return true; +} + /*}}}*/ +// CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void CircleBuf::FillOut() +{ + if (OutQueue.empty() == true) + return; + while (1) + { + // Woops, buffer is full + if (InP - OutP == Size) + return; + + // Write the buffer segment + unsigned long long Sz = LeftRead(); + if (OutQueue.length() - StrPos < Sz) + Sz = OutQueue.length() - StrPos; + memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz); + + // Advance + StrPos += Sz; + InP += Sz; + if (OutQueue.length() == StrPos) + { + StrPos = 0; + OutQueue = ""; + return; + } + } +} + /*}}}*/ +// CircleBuf::Write - Write from the buffer into a FD /*{{{*/ +// --------------------------------------------------------------------- +/* This empties the buffer into the FD. */ +bool CircleBuf::Write(std::unique_ptr<MethodFd> const &Fd) +{ + while (1) + { + FillOut(); + + // Woops, buffer is empty + if (OutP == InP) + return true; + + if (OutP == MaxGet) + return true; + + // Write the buffer segment + ssize_t Res; + Res = Fd->Write(Buf + (OutP % Size), LeftWrite()); + + if (Res < 0) + { + if (errno == EAGAIN) + return true; + + return false; + } + + TotalWriten += Res; + + if (Hash != NULL) + Hash->Add(Buf + (OutP%Size),Res); + + OutP += Res; + } +} + /*}}}*/ +// CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/ +// --------------------------------------------------------------------- +/* This copies till the first empty line */ +bool CircleBuf::WriteTillEl(string &Data,bool Single) +{ + // We cheat and assume it is unneeded to have more than one buffer load + for (unsigned long long I = OutP; I < InP; I++) + { + if (Buf[I%Size] != '\n') + continue; + ++I; + + if (Single == false) + { + if (I < InP && Buf[I%Size] == '\r') + ++I; + if (I >= InP || Buf[I%Size] != '\n') + continue; + ++I; + } + + Data = ""; + while (OutP < I) + { + unsigned long long Sz = LeftWrite(); + if (Sz == 0) + return false; + if (I - OutP < Sz) + Sz = I - OutP; + Data += string((char *)(Buf + (OutP%Size)),Sz); + OutP += Sz; + } + return true; + } + return false; +} + /*}}}*/ +// CircleBuf::Write - Write from the buffer to a string /*{{{*/ +// --------------------------------------------------------------------- +/* This copies everything */ +bool CircleBuf::Write(string &Data) +{ + Data = std::string((char *)Buf + (OutP % Size), LeftWrite()); + OutP += LeftWrite(); + return true; +} + /*}}}*/ +CircleBuf::~CircleBuf() /*{{{*/ +{ + delete [] Buf; + delete Hash; +} + /*}}}*/ + +// UnwrapHTTPConnect - Does the HTTP CONNECT handshake /*{{{*/ +// --------------------------------------------------------------------- +/* Performs a TLS handshake on the socket */ +struct HttpConnectFd : public MethodFd +{ + std::unique_ptr<MethodFd> UnderlyingFd; + std::string Buffer; + + int Fd() APT_OVERRIDE { return UnderlyingFd->Fd(); } + + ssize_t Read(void *buf, size_t count) APT_OVERRIDE + { + if (!Buffer.empty()) + { + auto read = count < Buffer.size() ? count : Buffer.size(); + + memcpy(buf, Buffer.data(), read); + Buffer.erase(Buffer.begin(), Buffer.begin() + read); + return read; + } + + return UnderlyingFd->Read(buf, count); + } + ssize_t Write(void *buf, size_t count) APT_OVERRIDE + { + return UnderlyingFd->Write(buf, count); + } + + int Close() APT_OVERRIDE + { + return UnderlyingFd->Close(); + } + + bool HasPending() APT_OVERRIDE + { + return !Buffer.empty(); + } +}; + +static ResultState UnwrapHTTPConnect(std::string Host, int Port, URI Proxy, std::unique_ptr<MethodFd> &Fd, + unsigned long Timeout, aptAuthConfMethod *Owner) +{ + Owner->Status(_("Connecting to %s (%s)"), "HTTP proxy", URI::SiteOnly(Proxy).c_str()); + // The HTTP server expects a hostname with a trailing :port + std::stringstream Req; + std::string ProperHost; + + if (Host.find(':') != std::string::npos) + ProperHost = '[' + Host + ']'; + else + ProperHost = Host; + + // Build the connect + Req << "CONNECT " << Host << ":" << std::to_string(Port) << " HTTP/1.1\r\n"; + if (Proxy.Port != 0) + Req << "Host: " << ProperHost << ":" << std::to_string(Port) << "\r\n"; + else + Req << "Host: " << ProperHost << "\r\n"; + + Owner->MaybeAddAuthTo(Proxy); + if (Proxy.User.empty() == false || Proxy.Password.empty() == false) + Req << "Proxy-Authorization: Basic " + << Base64Encode(Proxy.User + ":" + Proxy.Password) << "\r\n"; + + Req << "User-Agent: " << Owner->ConfigFind("User-Agent", "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n"; + + Req << "\r\n"; + + CircleBuf In(dynamic_cast<HttpMethod *>(Owner), 4096); + CircleBuf Out(dynamic_cast<HttpMethod *>(Owner), 4096); + std::string Headers; + + if (Owner->DebugEnabled() == true) + cerr << Req.str() << endl; + Out.Read(Req.str()); + + // Writing from proxy + while (Out.WriteSpace()) + { + if (WaitFd(Fd->Fd(), true, Timeout) == false) + { + _error->Errno("select", "Writing to proxy failed"); + return ResultState::TRANSIENT_ERROR; + } + if (Out.Write(Fd) == false) + { + _error->Errno("write", "Writing to proxy failed"); + return ResultState::TRANSIENT_ERROR; + } + } + + while (In.ReadSpace()) + { + if (WaitFd(Fd->Fd(), false, Timeout) == false) + { + _error->Errno("select", "Reading from proxy failed"); + return ResultState::TRANSIENT_ERROR; + } + if (In.Read(Fd) == false) + { + _error->Errno("read", "Reading from proxy failed"); + return ResultState::TRANSIENT_ERROR; + } + + if (In.WriteTillEl(Headers)) + break; + } + + if (Owner->DebugEnabled() == true) + cerr << Headers << endl; + + if (!(APT::String::Startswith(Headers, "HTTP/1.0 200") || APT::String::Startswith(Headers, "HTTP/1.1 200"))) + { + _error->Error("Invalid response from proxy: %s", Headers.c_str()); + return ResultState::TRANSIENT_ERROR; + } + + if (In.WriteSpace()) + { + // Maybe there is actual data already read, if so we need to buffer it + std::unique_ptr<HttpConnectFd> NewFd(new HttpConnectFd()); + In.Write(NewFd->Buffer); + NewFd->UnderlyingFd = std::move(Fd); + Fd = std::move(NewFd); + } + + return ResultState::SUCCESSFUL; +} + /*}}}*/ + +// HttpServerState::HttpServerState - Constructor /*{{{*/ +HttpServerState::HttpServerState(URI Srv, HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, APT_BUFFER_SIZE), Out(Owner, 4 * 1024) +{ + TimeOut = Owner->ConfigFindI("Timeout", TimeOut); + ServerFd = MethodFd::FromFd(-1); + Reset(); +} + /*}}}*/ +// HttpServerState::Open - Open a connection to the server /*{{{*/ +// --------------------------------------------------------------------- +/* This opens a connection to the server. */ +ResultState HttpServerState::Open() +{ + // Use the already open connection if possible. + if (ServerFd->Fd() != -1) + return ResultState::SUCCESSFUL; + + Close(); + In.Reset(); + Out.Reset(); + Persistent = true; + + bool tls = (ServerName.Access == "https" || APT::String::Endswith(ServerName.Access, "+https")); + + // Determine the proxy setting + // Used to run AutoDetectProxy(ServerName) here, but we now send a Proxy + // header in the URI Acquire request and set "Acquire::"+uri.Access+"::proxy::"+uri.Host + // to it in BaseHttpMethod::Loop() + string SpecificProxy = Owner->ConfigFind("Proxy::" + ServerName.Host, ""); + if (!SpecificProxy.empty()) + { + if (SpecificProxy == "DIRECT") + Proxy = ""; + else + Proxy = SpecificProxy; + } + else + { + string DefProxy = Owner->ConfigFind("Proxy", ""); + if (!DefProxy.empty()) + { + Proxy = DefProxy; + } + else + { + char *result = getenv("http_proxy"); + Proxy = result ? result : ""; + if (tls == true) + { + char *result = getenv("https_proxy"); + if (result != nullptr) + { + Proxy = result; + } + } + } + } + + // Parse no_proxy, a , separated list of domains + if (getenv("no_proxy") != 0) + { + if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true) + Proxy = ""; + } + + if (Proxy.empty() == false) + Owner->AddProxyAuth(Proxy, ServerName); + + auto const DefaultService = tls ? "https" : "http"; + auto const DefaultPort = tls ? 443 : 80; + if (Proxy.Access == "socks5h") + { + auto result = Connect(Proxy.Host, Proxy.Port, "socks", 1080, ServerFd, TimeOut, Owner); + if (result != ResultState::SUCCESSFUL) + return result; + + result = UnwrapSocks(ServerName.Host, ServerName.Port == 0 ? DefaultPort : ServerName.Port, + Proxy, ServerFd, Owner->ConfigFindI("TimeOut", 30), Owner); + if (result != ResultState::SUCCESSFUL) + return result; + } + else + { + // Determine what host and port to use based on the proxy settings + int Port = 0; + string Host; + if (Proxy.empty() == true || Proxy.Host.empty() == true) + { + if (ServerName.Port != 0) + Port = ServerName.Port; + Host = ServerName.Host; + } + else if (Proxy.Access != "http" && Proxy.Access != "https") + { + _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str()); + return ResultState::FATAL_ERROR; + } + else + { + if (Proxy.Port != 0) + Port = Proxy.Port; + Host = Proxy.Host; + + if (Proxy.Access == "https" && Port == 0) + Port = 443; + } + auto result = Connect(Host, Port, DefaultService, DefaultPort, ServerFd, TimeOut, Owner); + if (result != ResultState::SUCCESSFUL) + return result; + if (Host == Proxy.Host && Proxy.Access == "https") + { + aptConfigWrapperForMethods ProxyConf{std::vector<std::string>{"http", "https"}}; + ProxyConf.setPostfixForMethodNames(Proxy.Host.c_str()); + result = UnwrapTLS(Proxy.Host, ServerFd, TimeOut, Owner, &ProxyConf); + if (result != ResultState::SUCCESSFUL) + return result; + } + if (Host == Proxy.Host && tls) + { + result = UnwrapHTTPConnect(ServerName.Host, ServerName.Port == 0 ? DefaultPort : ServerName.Port, Proxy, ServerFd, Owner->ConfigFindI("TimeOut", 30), Owner); + if (result != ResultState::SUCCESSFUL) + return result; + } + } + + if (tls) + return UnwrapTLS(ServerName.Host, ServerFd, TimeOut, Owner, Owner); + + return ResultState::SUCCESSFUL; +} + /*}}}*/ +// HttpServerState::Close - Close a connection to the server /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool HttpServerState::Close() +{ + ServerFd->Close(); + return true; +} + /*}}}*/ +// HttpServerState::RunData - Transfer the data from the socket /*{{{*/ +ResultState HttpServerState::RunData(RequestState &Req) +{ + Req.State = RequestState::Data; + + // Chunked transfer encoding is fun.. + if (Req.Encoding == RequestState::Chunked) + { + while (1) + { + // Grab the block size + ResultState Last = ResultState::SUCCESSFUL; + string Data; + In.Limit(-1); + do + { + if (In.WriteTillEl(Data,true) == true) + break; + } while ((Last = Go(false, Req)) == ResultState::SUCCESSFUL); + + if (Last != ResultState::SUCCESSFUL) + return Last; + + // See if we are done + unsigned long long Len = strtoull(Data.c_str(),0,16); + if (Len == 0) + { + In.Limit(-1); + + // We have to remove the entity trailer + Last = ResultState::SUCCESSFUL; + do + { + if (In.WriteTillEl(Data,true) == true && Data.length() <= 2) + break; + } while ((Last = Go(false, Req)) == ResultState::SUCCESSFUL); + return Last; + } + + // Transfer the block + In.Limit(Len); + while (Go(true, Req) == ResultState::SUCCESSFUL) + if (In.IsLimit() == true) + break; + + // Error + if (In.IsLimit() == false) + return ResultState::TRANSIENT_ERROR; + + // The server sends an extra new line before the next block specifier.. + In.Limit(-1); + Last = ResultState::SUCCESSFUL; + do + { + if (In.WriteTillEl(Data,true) == true) + break; + } while ((Last = Go(false, Req)) == ResultState::SUCCESSFUL); + if (Last != ResultState::SUCCESSFUL) + return Last; + } + } + else + { + /* Closes encoding is used when the server did not specify a size, the + loss of the connection means we are done */ + if (Req.JunkSize != 0) + In.Limit(Req.JunkSize); + else if (Req.DownloadSize != 0) + { + if (Req.MaximumSize != 0 && Req.DownloadSize > Req.MaximumSize) + { + Owner->SetFailReason("MaximumSizeExceeded"); + _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"), + Req.DownloadSize, Req.MaximumSize); + return ResultState::FATAL_ERROR; + } + In.Limit(Req.DownloadSize); + } + else if (Persistent == false) + In.Limit(-1); + + // Just transfer the whole block. + while (true) + { + if (In.IsLimit() == false) + { + auto const result = Go(true, Req); + if (result == ResultState::SUCCESSFUL) + continue; + return result; + } + + In.Limit(-1); + return _error->PendingError() ? ResultState::FATAL_ERROR : ResultState::SUCCESSFUL; + } + } + + if (Flush(&Req.File) == false) + return ResultState::TRANSIENT_ERROR; + return ResultState::SUCCESSFUL; +} + /*}}}*/ +ResultState HttpServerState::RunDataToDevNull(RequestState &Req) /*{{{*/ +{ + // no need to clean up if we discard the connection anyhow + if (Persistent == false) + return ResultState::SUCCESSFUL; + Req.File.Open("/dev/null", FileFd::WriteOnly); + return RunData(Req); +} + /*}}}*/ +bool HttpServerState::ReadHeaderLines(std::string &Data) /*{{{*/ +{ + return In.WriteTillEl(Data); +} + /*}}}*/ +ResultState HttpServerState::LoadNextResponse(bool const ToFile, RequestState &Req) /*{{{*/ +{ + return Go(ToFile, Req); +} + /*}}}*/ +bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/ +{ + return Out.Read(Data); +} + /*}}}*/ +APT_PURE bool HttpServerState::IsOpen() /*{{{*/ +{ + return (ServerFd->Fd() != -1); +} + /*}}}*/ +bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/ +{ + delete In.Hash; + In.Hash = new Hashes(ExpectedHashes); + return true; +} + /*}}}*/ +void HttpServerState::Reset() /*{{{*/ +{ + ServerState::Reset(); + ServerFd->Close(); +} + /*}}}*/ + +APT_PURE Hashes * HttpServerState::GetHashes() /*{{{*/ +{ + return In.Hash; +} + /*}}}*/ +// HttpServerState::Die - The server has closed the connection. /*{{{*/ +ResultState HttpServerState::Die(RequestState &Req) +{ + unsigned int LErrno = errno; + + Close(); + + switch (Req.State) + { + case RequestState::Data: + // We have read all data we could, or the connection is not persistent + if (In.IsLimit() == true || Persistent == false) + return ResultState::SUCCESSFUL; + break; + case RequestState::Header: + In.Limit(-1); + // We have read some headers, but we might also have read the content + // and an EOF and hence reached this point. This is fine. + if (In.WriteSpace()) + return ResultState::SUCCESSFUL; + break; + } + + // We have reached an actual error, tell the user about it. + if (LErrno == 0) + { + _error->Error(_("Error reading from server. Remote end closed connection")); + return ResultState::TRANSIENT_ERROR; + } + errno = LErrno; + _error->Errno("read", _("Error reading from server")); + + return ResultState::TRANSIENT_ERROR; +} + /*}}}*/ +// HttpServerState::Flush - Dump the buffer into the file /*{{{*/ +// --------------------------------------------------------------------- +/* This takes the current input buffer from the Server FD and writes it + into the file */ +bool HttpServerState::Flush(FileFd *const File, bool MustComplete) +{ + if (File != nullptr) + { + if (In.WriteSpace() == false) + return true; + + while (In.WriteSpace() == true) + { + if (In.Write(MethodFd::FromFd(File->Fd())) == false) + return _error->Errno("write",_("Error writing to file")); + if (In.IsLimit() == true) + return true; + } + + if (In.IsLimit() == true || Persistent == false || not MustComplete) + return true; + } + return false; +} + /*}}}*/ +// HttpServerState::Go - Run a single loop /*{{{*/ +// --------------------------------------------------------------------- +/* This runs the select loop over the server FDs, Output file FDs and + stdin. */ +ResultState HttpServerState::Go(bool ToFile, RequestState &Req) +{ + // Server has closed the connection + if (ServerFd->Fd() == -1 && (In.WriteSpace() == false || + ToFile == false)) + return ResultState::TRANSIENT_ERROR; + + // Record if we have data pending to read in the server, so that we can + // skip the wait in select(). This can happen if data has already been + // read into a methodfd's buffer - the TCP queue might be empty at that + // point. + bool ServerPending = ServerFd->HasPending(); + + fd_set rfds,wfds; + FD_ZERO(&rfds); + FD_ZERO(&wfds); + + /* Add the server. We only send more requests if the connection will + be persisting */ + if (Out.WriteSpace() == true && ServerFd->Fd() != -1 && Persistent == true) + FD_SET(ServerFd->Fd(), &wfds); + if (In.ReadSpace() == true && ServerFd->Fd() != -1) + FD_SET(ServerFd->Fd(), &rfds); + + // Add the file. Note that we need to add the file to the select and + // then write before we read from the server so we do not have content + // left to write if the server closes the connection when we read from it. + // + // An alternative would be to just flush the file in those circumstances + // and then return. Because otherwise we might end up blocking indefinitely + // in the select() call if we were to continue but all that was left to do + // was write to the local file. + if (In.WriteSpace() == true && ToFile == true && Req.File.IsOpen()) + FD_SET(Req.File.Fd(), &wfds); + + // Add stdin + if (Owner->ConfigFindB("DependOnSTDIN", true) == true) + FD_SET(STDIN_FILENO,&rfds); + + // Figure out the max fd + int MaxFd = Req.File.Fd(); + if (MaxFd < ServerFd->Fd()) + MaxFd = ServerFd->Fd(); + + // Select + struct timeval tv; + tv.tv_sec = ServerPending ? 0 : TimeOut; + tv.tv_usec = 0; + int Res = 0; + if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0) + { + if (errno == EINTR) + return ResultState::SUCCESSFUL; + _error->Errno("select", _("Select failed")); + return ResultState::TRANSIENT_ERROR; + } + + if (Res == 0 && not ServerPending) + { + _error->Error(_("Connection timed out")); + return ResultState::TRANSIENT_ERROR; + } + + // Flush any data before talking to the server, in case the server + // closed the connection, we want to be done writing. + if (Req.File.IsOpen() && FD_ISSET(Req.File.Fd(), &wfds)) + { + if (not Flush(&Req.File, false)) + return ResultState::TRANSIENT_ERROR; + } + + // Handle server IO + if (ServerPending || (ServerFd->Fd() != -1 && FD_ISSET(ServerFd->Fd(), &rfds))) + { + errno = 0; + if (In.Read(ServerFd) == false) + return Die(Req); + } + + // Send data to the file + if (In.WriteSpace() == true && ToFile == true && Req.File.IsOpen()) + { + if (not Flush(&Req.File, false)) + return ResultState::TRANSIENT_ERROR; + } + + if (ServerFd->Fd() != -1 && FD_ISSET(ServerFd->Fd(), &wfds)) + { + errno = 0; + if (Out.Write(ServerFd) == false) + return Die(Req); + } + + if (Req.MaximumSize > 0 && Req.File.IsOpen() && Req.File.Failed() == false && Req.File.Tell() > Req.MaximumSize) + { + Owner->SetFailReason("MaximumSizeExceeded"); + _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"), + Req.File.Tell(), Req.MaximumSize); + return ResultState::FATAL_ERROR; + } + + // Handle commands from APT + if (FD_ISSET(STDIN_FILENO,&rfds)) + { + if (Owner->Run(true) != -1) + exit(100); + } + + return ResultState::SUCCESSFUL; +} + /*}}}*/ + +// HttpMethod::SendReq - Send the HTTP request /*{{{*/ +// --------------------------------------------------------------------- +/* This places the http request in the outbound buffer */ +void HttpMethod::SendReq(FetchItem *Itm) +{ + URI Uri(Itm->Uri); + { + auto const plus = Binary.find('+'); + if (plus != std::string::npos) + Uri.Access = Binary.substr(plus + 1); + } + + // The HTTP server expects a hostname with a trailing :port + std::stringstream Req; + string ProperHost; + + if (Uri.Host.find(':') != string::npos) + ProperHost = '[' + Uri.Host + ']'; + else + ProperHost = Uri.Host; + + /* RFC 2616 §5.1.2 requires absolute URIs for requests to proxies, + but while its a must for all servers to accept absolute URIs, + it is assumed clients will sent an absolute path for non-proxies */ + std::string requesturi; + if ((Server->Proxy.Access != "http" && Server->Proxy.Access != "https") || APT::String::Endswith(Uri.Access, "https") || Server->Proxy.empty() == true || Server->Proxy.Host.empty()) + requesturi = Uri.Path; + else + requesturi = Uri; + + if (not _config->FindB("Acquire::Send-URI-Encoded", false)) + requesturi = URIEncode(requesturi); + + /* Build the request. No keep-alive is included as it is the default + in 1.1, can cause problems with proxies, and we are an HTTP/1.1 + client anyway. + C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */ + Req << "GET " << requesturi << " HTTP/1.1\r\n"; + if (Uri.Port != 0) + Req << "Host: " << ProperHost << ":" << std::to_string(Uri.Port) << "\r\n"; + else + Req << "Host: " << ProperHost << "\r\n"; + + // generate a cache control header (if needed) + if (ConfigFindB("No-Cache",false) == true) + Req << "Cache-Control: no-cache\r\n" + << "Pragma: no-cache\r\n"; + else if (Itm->IndexFile == true) + Req << "Cache-Control: max-age=" << std::to_string(ConfigFindI("Max-Age", 0)) << "\r\n"; + else if (ConfigFindB("No-Store", false) == true) + Req << "Cache-Control: no-store\r\n"; + + // If we ask for uncompressed files servers might respond with content- + // negotiation which lets us end up with compressed files we do not support, + // see 657029, 657560 and co, so if we have no extension on the request + // ask for text only. As a sidenote: If there is nothing to negotate servers + // seem to be nice and ignore it. + if (ConfigFindB("SendAccept", true) == true) + { + size_t const filepos = Itm->Uri.find_last_of('/'); + string const file = Itm->Uri.substr(filepos + 1); + if (flExtension(file) == file) + Req << "Accept: text/*\r\n"; + } + + // Check for a partial file and send if-queries accordingly + struct stat SBuf; + if (Server->RangesAllowed && stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) + Req << "Range: bytes=" << std::to_string(SBuf.st_size) << "-\r\n" + << "If-Range: " << TimeRFC1123(SBuf.st_mtime, false) << "\r\n"; + else if (Itm->LastModified != 0) + Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified, false).c_str() << "\r\n"; + + if ((Server->Proxy.Access == "http" || Server->Proxy.Access == "https") && + (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false)) + Req << "Proxy-Authorization: Basic " + << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n"; + + MaybeAddAuthTo(Uri); + if (Uri.User.empty() == false || Uri.Password.empty() == false) + Req << "Authorization: Basic " + << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n"; + + Req << "User-Agent: " << ConfigFind("User-Agent", + "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")"); + +#ifdef HAVE_SYSTEMD + if (ConfigFindB("User-Agent-Non-Interactive", false)) + { + using APT::operator""_sv; + char *unit = nullptr; + sd_pid_get_unit(getpid(), &unit); + if (unit != nullptr && *unit != '\0' && not APT::String::Startswith(unit, "user@") // user@ _is_ interactive + && "packagekit.service"_sv != unit // packagekit likely is interactive + && "dbus.service"_sv != unit) // aptdaemon and qapt don't have systemd services + Req << " non-interactive"; + + free(unit); + } +#endif + + Req << "\r\n"; + + // the famously typoed HTTP header field + auto const referrer = ConfigFind("Referer", ""); + if (referrer.empty() == false) + Req << "Referer: " << referrer << "\r\n"; + + Req << "\r\n"; + + if (Debug == true) + cerr << Req.str() << endl; + + Server->WriteResponse(Req.str()); +} + /*}}}*/ +std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/ +{ + return std::unique_ptr<ServerState>(new HttpServerState(uri, this)); +} + /*}}}*/ +void HttpMethod::RotateDNS() /*{{{*/ +{ + ::RotateDNS(); +} + /*}}}*/ +BaseHttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res, RequestState &Req)/*{{{*/ +{ + auto ret = BaseHttpMethod::DealWithHeaders(Res, Req); + if (ret != BaseHttpMethod::FILE_IS_OPEN) + return ret; + if (Req.File.Open(Queue->DestFile, FileFd::WriteAny) == false) + return ERROR_NOT_FROM_SERVER; + + FailFile = Queue->DestFile; + (void)(FailFile.c_str()); // Make sure we don't do a malloc in the signal handler + FailFd = Req.File.Fd(); + FailTime = Req.Date; + + if (Server->InitHashes(Queue->ExpectedHashes) == false || Req.AddPartialFileToHashes(Req.File) == false) + { + _error->Errno("read",_("Problem hashing file")); + return ERROR_NOT_FROM_SERVER; + } + if (Req.StartPos > 0) + Res.ResumePoint = Req.StartPos; + + return FILE_IS_OPEN; +} + /*}}}*/ +HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(std::move(pProg), "1.2", Pipeline | SendConfig | SendURIEncoded) /*{{{*/ +{ + SeccompFlags = aptMethod::BASE | aptMethod::NETWORK; + + auto addName = std::inserter(methodNames, methodNames.begin()); + if (Binary != "http") + addName = "http"; + auto const plus = Binary.find('+'); + if (plus != std::string::npos) + { + auto name2 = Binary.substr(plus + 1); + if (std::find(methodNames.begin(), methodNames.end(), name2) == methodNames.end()) + addName = std::move(name2); + addName = Binary.substr(0, plus); + } +} + /*}}}*/ + +int main(int, const char *argv[]) +{ + // ignore SIGPIPE, this can happen on write() if the socket + // closes the connection (this is dealt with via ServerDie()) + signal(SIGPIPE, SIG_IGN); + std::string Binary = flNotDir(argv[0]); + if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http") + Binary.append("+http"); + return HttpMethod(std::move(Binary)).Loop(); +} diff --git a/methods/http.h b/methods/http.h new file mode 100644 index 0000000..cae579a --- /dev/null +++ b/methods/http.h @@ -0,0 +1,142 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + HTTP Acquire Method - This is the HTTP acquire method for APT. + + ##################################################################### */ + /*}}}*/ + +#ifndef APT_HTTP_H +#define APT_HTTP_H + +#include <apt-pkg/strutl.h> + +#include <chrono> +#include <iostream> +#include <memory> +#include <string> +#include <sys/time.h> + +#include "basehttp.h" +#include "connect.h" + +using std::cout; +using std::endl; + +class FileFd; +class HttpMethod; +class Hashes; + +class CircleBuf +{ + unsigned char *Buf; + unsigned long long Size; + unsigned long long InP; + unsigned long long OutP; + std::string OutQueue; + unsigned long long StrPos; + unsigned long long MaxGet; + + static unsigned long long BwReadLimit; + static unsigned long long BwTickReadData; + static std::chrono::steady_clock::duration BwReadTick; + static const unsigned int BW_HZ; + + unsigned long long LeftRead() const + { + unsigned long long Sz = Size - (InP - OutP); + if (Sz > Size - (InP%Size)) + Sz = Size - (InP%Size); + return Sz; + } + unsigned long long LeftWrite() const + { + unsigned long long Sz = InP - OutP; + if (InP > MaxGet) + Sz = MaxGet - OutP; + if (Sz > Size - (OutP%Size)) + Sz = Size - (OutP%Size); + return Sz; + } + void FillOut(); + + public: + Hashes *Hash; + // total amount of data that got written so far + unsigned long long TotalWriten; + + // Read data in + bool Read(std::unique_ptr<MethodFd> const &Fd); + bool Read(std::string const &Data); + + // Write data out + bool Write(std::unique_ptr<MethodFd> const &Fd); + bool Write(std::string &Data); + bool WriteTillEl(std::string &Data,bool Single = false); + + // Control the write limit + void Limit(long long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;} + bool IsLimit() const {return MaxGet == OutP;}; + void Print() const {cout << MaxGet << ',' << OutP << endl;}; + + // Test for free space in the buffer + bool ReadSpace() const {return Size - (InP - OutP) > 0;}; + bool WriteSpace() const {return InP - OutP > 0;}; + + void Reset(); + + CircleBuf(HttpMethod const * const Owner, unsigned long long Size); + ~CircleBuf(); +}; + +struct HttpServerState: public ServerState +{ + // This is the connection itself. Output is data FROM the server + CircleBuf In; + CircleBuf Out; + std::unique_ptr<MethodFd> ServerFd; + + protected: + virtual bool ReadHeaderLines(std::string &Data) APT_OVERRIDE; + virtual ResultState LoadNextResponse(bool const ToFile, RequestState &Req) APT_OVERRIDE; + virtual bool WriteResponse(std::string const &Data) APT_OVERRIDE; + + public: + virtual void Reset() APT_OVERRIDE; + + virtual ResultState RunData(RequestState &Req) APT_OVERRIDE; + virtual ResultState RunDataToDevNull(RequestState &Req) APT_OVERRIDE; + + virtual ResultState Open() APT_OVERRIDE; + virtual bool IsOpen() APT_OVERRIDE; + virtual bool Close() APT_OVERRIDE; + virtual bool InitHashes(HashStringList const &ExpectedHashes) APT_OVERRIDE; + virtual Hashes * GetHashes() APT_OVERRIDE; + virtual ResultState Die(RequestState &Req) APT_OVERRIDE; + virtual bool Flush(FileFd *const File, bool MustComplete = true) APT_OVERRIDE; + virtual ResultState Go(bool ToFile, RequestState &Req) APT_OVERRIDE; + + HttpServerState(URI Srv, HttpMethod *Owner); + virtual ~HttpServerState() {Close();}; +}; + +class HttpMethod : public BaseHttpMethod +{ + public: + virtual void SendReq(FetchItem *Itm) APT_OVERRIDE; + + virtual std::unique_ptr<ServerState> CreateServerState(URI const &uri) APT_OVERRIDE; + virtual void RotateDNS() APT_OVERRIDE; + virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res, RequestState &Req) APT_OVERRIDE; + + protected: + std::string AutoDetectProxyCmd; + + public: + friend struct HttpServerState; + + explicit HttpMethod(std::string &&pProg); +}; + +#endif diff --git a/methods/mirror.cc b/methods/mirror.cc new file mode 100644 index 0000000..787e4c7 --- /dev/null +++ b/methods/mirror.cc @@ -0,0 +1,414 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Mirror URI – This method helps avoiding hardcoding of mirrors in the + sources.lists by looking up a list of mirrors first to which the + following requests are redirected. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include "aptmethod.h" +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/metaindex.h> +#include <apt-pkg/sourcelist.h> +#include <apt-pkg/strutl.h> + +#include <functional> +#include <random> +#include <string> +#include <unordered_map> + +#include <sys/utsname.h> + +#include <apti18n.h> + /*}}}*/ +constexpr char const *const disallowLocal[] = {"ftp", "http", "https"}; + +static void sortByLength(std::vector<std::string> &vec) /*{{{*/ +{ + // this ensures having mirror://foo/ and mirror://foo/bar/ works as expected + // by checking for the longest matches first + std::sort(vec.begin(), vec.end(), [](std::string const &a, std::string const &b) { + return a.length() > b.length(); + }); +} + /*}}}*/ +class MirrorMethod : public aptMethod /*{{{*/ +{ + std::mt19937 genrng; + std::vector<std::string> sourceslist; + std::unordered_map<std::string, std::string> msgCache; + enum MirrorFileState + { + REQUESTED, + FAILED, + AVAILABLE + }; + struct MirrorInfo + { + std::string uri; + unsigned long priority = std::numeric_limits<decltype(priority)>::max(); + decltype(genrng)::result_type seed = 0; + std::unordered_map<std::string, std::vector<std::string>> tags; + explicit MirrorInfo(std::string const &u, std::vector<std::string> &&ptags = {}) : uri(u) + { + for (auto &&tag : ptags) + { + auto const colonfound = tag.find(':'); + if (unlikely(colonfound == std::string::npos)) + continue; + auto name = tag.substr(0, colonfound); + auto value = tag.substr(colonfound + 1); + if (name == "arch") + tags["Architecture"].emplace_back(std::move(value)); + else if (name == "lang") + tags["Language"].emplace_back(std::move(value)); + else if (name == "priority") + priority = std::strtoul(value.c_str(), nullptr, 10); + else if (likely(name.empty() == false)) + { + if (name == "codename" || name == "suite") + tags["Release"].push_back(value); + name[0] = std::toupper(name[0]); + tags[std::move(name)].emplace_back(std::move(value)); + } + } + } + }; + struct MirrorListInfo + { + MirrorFileState state; + std::string baseuri; + std::vector<MirrorInfo> list; + }; + std::unordered_map<std::string, MirrorListInfo> mirrorfilestate; + + virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE; + + void RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message); + bool MirrorListFileReceived(MirrorListInfo &info, FetchItem *const Itm); + std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm); + void DealWithPendingItems(std::vector<std::string> const &baseuris, MirrorListInfo const &info, FetchItem *const Itm, std::function<void()> handler); + + public: + explicit MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests | SendURIEncoded), genrng(clock()) + { + SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY; + } +}; + /*}}}*/ +void MirrorMethod::RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message) /*{{{*/ +{ + std::unordered_map<std::string, std::string> matchers; + matchers.emplace("Architecture", LookupTag(Message, "Target-Architecture")); + matchers.emplace("Codename", LookupTag(Message, "Target-Codename")); + matchers.emplace("Component", LookupTag(Message, "Target-Component")); + matchers.emplace("Language", LookupTag(Message, "Target-Language")); + matchers.emplace("Release", LookupTag(Message, "Target-Release")); + matchers.emplace("Suite", LookupTag(Message, "Target-Suite")); + matchers.emplace("Type", LookupTag(Message, "Target-Type")); + decltype(info.list) possMirrors; + for (auto const &mirror : info.list) + { + bool failedMatch = false; + for (auto const &m : matchers) + { + if (m.second.empty()) + continue; + auto const tagsetiter = mirror.tags.find(m.first); + if (tagsetiter == mirror.tags.end()) + continue; + auto const tagset = tagsetiter->second; + if (tagset.empty() == false && std::find(tagset.begin(), tagset.end(), m.second) == tagset.end()) + { + failedMatch = true; + break; + } + } + if (failedMatch) + continue; + possMirrors.push_back(mirror); + } + for (auto &&mirror : possMirrors) + mirror.seed = genrng(); + std::sort(possMirrors.begin(), possMirrors.end(), [](MirrorInfo const &a, MirrorInfo const &b) { + if (a.priority != b.priority) + return a.priority < b.priority; + return a.seed < b.seed; + }); + std::string const path = Itm->Uri.substr(info.baseuri.length()); + std::string altMirrors; + std::unordered_map<std::string, std::string> fields; + fields.emplace("URI", Itm->Uri); + for (auto curMirror = possMirrors.cbegin(); curMirror != possMirrors.cend(); ++curMirror) + { + std::string mirror = curMirror->uri; + if (APT::String::Endswith(mirror, "/") == false) + mirror.append("/"); + mirror.append(path); + if (curMirror == possMirrors.cbegin()) + fields.emplace("New-URI", mirror); + else if (altMirrors.empty()) + altMirrors.append(mirror); + else + altMirrors.append("\n").append(mirror); + } + fields.emplace("Alternate-URIs", altMirrors); + SendMessage("103 Redirect", std::move(fields)); + + // Remove Itm from the queue, then delete + if (Queue == Itm) + Queue = Itm->Next; + else + { + FetchItem *previous = Queue; + while (previous->Next != Itm) + previous = previous->Next; + + previous->Next = Itm->Next; + } + delete Itm; +} + /*}}}*/ +void MirrorMethod::DealWithPendingItems(std::vector<std::string> const &baseuris, /*{{{*/ + MirrorListInfo const &info, FetchItem *const Itm, + std::function<void()> handler) +{ + FetchItem **LastItm = &Itm->Next; + while (*LastItm != nullptr) + LastItm = &((*LastItm)->Next); + while (Queue != Itm) + { + if (APT::String::Startswith(Queue->Uri, info.baseuri) == false || + std::any_of(baseuris.cbegin(), baseuris.cend(), [&](std::string const &b) { return APT::String::Startswith(Queue->Uri, b); })) + { + // move the item behind the aux file not related to it + *LastItm = Queue; + Queue = QueueBack = Queue->Next; + (*LastItm)->Next = nullptr; + LastItm = &((*LastItm)->Next); + } + else + { + handler(); + } + } + // now remove out trigger + QueueBack = Queue = Queue->Next; + delete Itm; +} + /*}}}*/ +bool MirrorMethod::MirrorListFileReceived(MirrorListInfo &info, FetchItem *const Itm) /*{{{*/ +{ + std::vector<std::string> baseuris; + for (auto const &i : mirrorfilestate) + if (info.baseuri.length() < i.second.baseuri.length() && + i.second.state == REQUESTED && + APT::String::Startswith(i.second.baseuri, info.baseuri)) + baseuris.push_back(i.second.baseuri); + sortByLength(baseuris); + + FileFd mirrorlist; + if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension)) + { + auto const accessColon = info.baseuri.find(':'); + auto access = info.baseuri.substr(0, accessColon); + std::string prefixAccess; + if (APT::String::Startswith(access, "mirror") == false) + { + auto const plus = info.baseuri.find('+'); + prefixAccess = info.baseuri.substr(0, plus); + access.erase(0, plus + 1); + } + std::vector<std::string> limitAccess; + // If the mirror file comes from an online source, allow only other online + // sources, not e.g. file:///. If the mirrorlist comes from there we can assume + // the admin knows what (s)he is doing through and not limit the options. + if (std::any_of(std::begin(disallowLocal), std::end(disallowLocal), + [&access](char const *const a) { return APT::String::Endswith(access, std::string("+") + a); }) || + access == "mirror") + { + std::copy(std::begin(disallowLocal), std::end(disallowLocal), std::back_inserter(limitAccess)); + } + std::string line; + while (mirrorlist.ReadLine(line)) + { + if (line.empty() || line[0] == '#') + continue; + auto const access = line.substr(0, line.find(':')); + if (limitAccess.empty() == false && std::find(limitAccess.begin(), limitAccess.end(), access) == limitAccess.end()) + continue; + auto const tab = line.find('\t'); + if (tab == std::string::npos) + { + if (prefixAccess.empty()) + info.list.emplace_back(std::move(line)); + else + info.list.emplace_back(prefixAccess + '+' + line); + } + else + { + auto uri = line.substr(0, tab); + if (prefixAccess.empty() == false) + uri = prefixAccess + '+' + uri; + auto tagline = line.substr(tab + 1); + std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' '); + auto tags = VectorizeString(tagline, ' '); + tags.erase(std::remove_if(tags.begin(), tags.end(), [](std::string const &a) { return a.empty(); }), tags.end()); + info.list.emplace_back(std::move(uri), std::move(tags)); + } + } + mirrorlist.Close(); + + if (info.list.empty()) + { + info.state = FAILED; + DealWithPendingItems(baseuris, info, Itm, [&]() { + std::string msg; + strprintf(msg, "Mirror list %s is empty for %s", Itm->DestFile.c_str(), Queue->Uri.c_str()); + Fail(msg, false); + }); + } + else + { + info.state = AVAILABLE; + DealWithPendingItems(baseuris, info, Itm, [&]() { + RedirectItem(info, Queue, msgCache[Queue->Uri]); + }); + msgCache.clear(); + } + } + else + { + info.state = FAILED; + DealWithPendingItems(baseuris, info, Itm, [&]() { + std::string msg; + strprintf(msg, "Downloading mirror file %s failed for %s", Itm->DestFile.c_str(), Queue->Uri.c_str()); + Fail(msg, false); + }); + } + return true; +} + /*}}}*/ +std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem *const Itm) /*{{{*/ +{ + if (APT::String::Startswith(Itm->Uri, Binary)) + { + std::string const repouri = LookupTag(Message, "Target-Repo-Uri"); + if (repouri.empty() == false && std::find(sourceslist.cbegin(), sourceslist.cend(), repouri) == sourceslist.cend()) + sourceslist.push_back(repouri); + } + if (sourceslist.empty()) + { + // read sources.list and find the matching base uri + pkgSourceList sl; + if (sl.ReadMainList() == false) + { + _error->Error(_("The list of sources could not be read.")); + return ""; + } + std::string const needle = Binary + ":"; + for (auto const &SL : sl) + { + std::string uristr = SL->GetURI(); + if (APT::String::Startswith(uristr, needle)) + sourceslist.push_back(uristr); + } + sortByLength(sourceslist); + } + for (auto uristr : sourceslist) + { + if (APT::String::Startswith(Itm->Uri, uristr)) + { + uristr.erase(uristr.length() - 1); // remove the ending '/' + auto const colon = uristr.find(':'); + if (unlikely(colon == std::string::npos)) + continue; + auto const plus = uristr.find("+"); + if (plus < colon) + { + // started as tor+mirror+http we want to get the file via tor+http + auto const access = uristr.substr(0, colon); + if (APT::String::Startswith(access, "mirror") == false) + { + uristr.erase(plus, strlen("mirror") + 1); + return uristr; + } + else + return uristr.substr(plus + 1); + } + else + { + uristr.replace(0, strlen("mirror"), "http"); + return uristr; + } + } + } + return ""; +} + /*}}}*/ +bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*/ +{ + auto mirrorinfo = mirrorfilestate.find(Itm->Uri); + if (mirrorinfo != mirrorfilestate.end()) + return MirrorListFileReceived(mirrorinfo->second, Itm); + + std::string const mirrorfileuri = GetMirrorFileURI(Message, Itm); + if (mirrorfileuri.empty()) + { + _error->Error("Couldn't determine mirror list to query for %s", Itm->Uri.c_str()); + return false; + } + if (DebugEnabled()) + std::clog << "Mirror-URI: " << mirrorfileuri << " for " << Itm->Uri << std::endl; + + // have we requested this mirror file already? + auto const state = mirrorfilestate.find(mirrorfileuri); + if (state == mirrorfilestate.end()) + { + msgCache[Itm->Uri] = Message; + MirrorListInfo info; + info.state = REQUESTED; + info.baseuri = mirrorfileuri + '/'; + auto const colon = info.baseuri.find(':'); + if (unlikely(colon == std::string::npos)) + return false; + info.baseuri.replace(0, colon, Binary); + mirrorfilestate[mirrorfileuri] = info; + std::unordered_map<std::string, std::string> fields; + fields.emplace("URI", Itm->Uri); + fields.emplace("MaximumSize", std::to_string(1 * 1024 * 1024)); //FIXME: 1 MB is enough for everyone + fields.emplace("Aux-ShortDesc", "Mirrorlist"); + fields.emplace("Aux-Description", mirrorfileuri + " Mirrorlist"); + fields.emplace("Aux-Uri", mirrorfileuri); + SendMessage("351 Aux Request", std::move(fields)); + return true; + } + + switch (state->second.state) + { + case REQUESTED: + // lets wait for the requested mirror file + msgCache[Itm->Uri] = Message; + return true; + case FAILED: + Fail("Downloading mirror file failed", false); + return true; + case AVAILABLE: + RedirectItem(state->second, Itm, Message); + return true; + } + return false; +} + /*}}}*/ + +int main(int, const char *argv[]) +{ + return MirrorMethod(flNotDir(argv[0])).Run(); +} diff --git a/methods/rfc2553emu.cc b/methods/rfc2553emu.cc new file mode 100644 index 0000000..41aa747 --- /dev/null +++ b/methods/rfc2553emu.cc @@ -0,0 +1,244 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + RFC 2553 Emulation - Provides emulation for RFC 2553 getaddrinfo, + freeaddrinfo and getnameinfo + + This is really C code, it just has a .cc extensions to play nicer with + the rest of APT. + + Originally written by Jason Gunthorpe <jgg@debian.org> and placed into + the Public Domain, do with it what you will. + + ##################################################################### */ + /*}}}*/ +#include <config.h> + +#include "rfc2553emu.h" +#include <arpa/inet.h> +#include <netinet/in.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#ifndef HAVE_GETADDRINFO +// getaddrinfo - Resolve a hostname /*{{{*/ +// --------------------------------------------------------------------- +/* */ +int getaddrinfo(const char *nodename, const char *servname, + const struct addrinfo *hints, + struct addrinfo **res) +{ + struct addrinfo **Result = res; + hostent *Addr; + unsigned int Port; + int Proto; + const char *End; + char **CurAddr; + + // Try to convert the service as a number + Port = htons(strtol(servname,(char **)&End,0)); + Proto = SOCK_STREAM; + + if (hints != 0 && hints->ai_socktype != 0) + Proto = hints->ai_socktype; + + // Not a number, must be a name. + if (End != servname + strlen(servname)) + { + struct servent *Srv = 0; + + // Do a lookup in the service database + if (hints == 0 || hints->ai_socktype == SOCK_STREAM) + Srv = getservbyname(servname,"tcp"); + if (hints != 0 && hints->ai_socktype == SOCK_DGRAM) + Srv = getservbyname(servname,"udp"); + if (Srv == 0) + return EAI_NONAME; + + // Get the right protocol + Port = Srv->s_port; + if (strcmp(Srv->s_proto,"tcp") == 0) + Proto = SOCK_STREAM; + else + { + if (strcmp(Srv->s_proto,"udp") == 0) + Proto = SOCK_DGRAM; + else + return EAI_NONAME; + } + + if (hints != 0 && hints->ai_socktype != Proto && + hints->ai_socktype != 0) + return EAI_SERVICE; + } + + // Hostname lookup, only if this is not a listening socket + if (hints != 0 && (hints->ai_flags & AI_PASSIVE) != AI_PASSIVE) + { + Addr = gethostbyname(nodename); + if (Addr == 0) + { + if (h_errno == TRY_AGAIN) + return EAI_AGAIN; + if (h_errno == NO_RECOVERY) + return EAI_FAIL; + return EAI_NONAME; + } + + // No A records + if (Addr->h_addr_list[0] == 0) + return EAI_NONAME; + + CurAddr = Addr->h_addr_list; + } + else + CurAddr = (char **)&End; // Fake! + + // Start constructing the linked list + *res = 0; + for (; *CurAddr != 0; CurAddr++) + { + // New result structure + *Result = (struct addrinfo *)calloc(sizeof(**Result),1); + if (*Result == 0) + { + freeaddrinfo(*res); + return EAI_MEMORY; + } + if (*res == 0) + *res = *Result; + + (*Result)->ai_family = AF_INET; + (*Result)->ai_socktype = Proto; + + // If we have the IPPROTO defines we can set the protocol field + #ifdef IPPROTO_TCP + if (Proto == SOCK_STREAM) + (*Result)->ai_protocol = IPPROTO_TCP; + if (Proto == SOCK_DGRAM) + (*Result)->ai_protocol = IPPROTO_UDP; + #endif + + // Allocate space for the address + (*Result)->ai_addrlen = sizeof(struct sockaddr_in); + (*Result)->ai_addr = (struct sockaddr *)calloc(sizeof(sockaddr_in),1); + if ((*Result)->ai_addr == 0) + { + freeaddrinfo(*res); + return EAI_MEMORY; + } + + // Set the address + ((struct sockaddr_in *)(*Result)->ai_addr)->sin_family = AF_INET; + ((struct sockaddr_in *)(*Result)->ai_addr)->sin_port = Port; + + if (hints != 0 && (hints->ai_flags & AI_PASSIVE) != AI_PASSIVE) + ((struct sockaddr_in *)(*Result)->ai_addr)->sin_addr = *(in_addr *)(*CurAddr); + else + { + // Already zerod by calloc. + break; + } + + Result = &(*Result)->ai_next; + } + + return 0; +} + /*}}}*/ +// freeaddrinfo - Free the result of getaddrinfo /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void freeaddrinfo(struct addrinfo *ai) +{ + while (ai != 0) + { + free(ai->ai_addr); + ai = ai->ai_next; + free(ai); + } +} + /*}}}*/ +#endif // HAVE_GETADDRINFO + +#ifndef HAVE_GETNAMEINFO +// getnameinfo - Convert a sockaddr to a string /*{{{*/ +// --------------------------------------------------------------------- +/* */ +int getnameinfo(const struct sockaddr *sa, socklen_t salen, + char *host, size_t hostlen, + char *serv, size_t servlen, + int flags) +{ + struct sockaddr_in *sin = (struct sockaddr_in *)sa; + + // This routine only supports internet addresses + if (sa->sa_family != AF_INET) + return EAI_ADDRFAMILY; + + if (host != 0) + { + // Try to resolve the hostname + if ((flags & NI_NUMERICHOST) != NI_NUMERICHOST) + { + struct hostent *Ent = gethostbyaddr((char *)&sin->sin_addr,sizeof(sin->sin_addr), + AF_INET); + if (Ent != 0) + strncpy(host,Ent->h_name,hostlen); + else + { + if ((flags & NI_NAMEREQD) == NI_NAMEREQD) + { + if (h_errno == TRY_AGAIN) + return EAI_AGAIN; + if (h_errno == NO_RECOVERY) + return EAI_FAIL; + return EAI_NONAME; + } + + flags |= NI_NUMERICHOST; + } + } + + // Resolve as a plain numeric + if ((flags & NI_NUMERICHOST) == NI_NUMERICHOST) + { + strncpy(host,inet_ntoa(sin->sin_addr),hostlen); + } + } + + if (serv != 0) + { + // Try to resolve the hostname + if ((flags & NI_NUMERICSERV) != NI_NUMERICSERV) + { + struct servent *Ent; + if ((flags & NI_DATAGRAM) == NI_DATAGRAM) + Ent = getservbyport(ntohs(sin->sin_port),"udp"); + else + Ent = getservbyport(ntohs(sin->sin_port),"tcp"); + + if (Ent != 0) + strncpy(serv,Ent->s_name,servlen); + else + { + if ((flags & NI_NAMEREQD) == NI_NAMEREQD) + return EAI_NONAME; + + flags |= NI_NUMERICSERV; + } + } + + // Resolve as a plain numeric + if ((flags & NI_NUMERICSERV) == NI_NUMERICSERV) + { + snprintf(serv,servlen,"%u",ntohs(sin->sin_port)); + } + } + + return 0; +} + /*}}}*/ +#endif // HAVE_GETNAMEINFO diff --git a/methods/rfc2553emu.h b/methods/rfc2553emu.h new file mode 100644 index 0000000..65d744d --- /dev/null +++ b/methods/rfc2553emu.h @@ -0,0 +1,112 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + RFC 2553 Emulation - Provides emulation for RFC 2553 getaddrinfo, + freeaddrinfo and getnameinfo + + These functions are necessary to write portable protocol independent + networking. They transparently support IPv4, IPv6 and probably many + other protocols too. This implementation is needed when the host does + not support these standards. It implements a simple wrapper that + basically supports only IPv4. + + Perfect emulation is not provided, but it is passable.. + + Originally written by Jason Gunthorpe <jgg@debian.org> and placed into + the Public Domain, do with it what you will. + + ##################################################################### */ + /*}}}*/ +#ifndef RFC2553EMU_H +#define RFC2553EMU_H + +#include <netdb.h> +#include <sys/socket.h> +#include <sys/types.h> + +// Autosense getaddrinfo +#if defined(AI_PASSIVE) && defined(EAI_NONAME) +#define HAVE_GETADDRINFO +#endif + +// Autosense getnameinfo +#if defined(NI_NUMERICHOST) +#define HAVE_GETNAMEINFO +#endif + +// getaddrinfo support? +#ifndef HAVE_GETADDRINFO + // Renamed to advoid type clashing.. (for debugging) + struct addrinfo_emu + { + int ai_flags; /* AI_PASSIVE, AI_CANONNAME, AI_NUMERICHOST */ + int ai_family; /* PF_xxx */ + int ai_socktype; /* SOCK_xxx */ + int ai_protocol; /* 0 or IPPROTO_xxx for IPv4 and IPv6 */ + size_t ai_addrlen; /* length of ai_addr */ + char *ai_canonname; /* canonical name for nodename */ + struct sockaddr *ai_addr; /* binary address */ + struct addrinfo_emu *ai_next; /* next structure in linked list */ + }; + #define addrinfo addrinfo_emu + + int getaddrinfo(const char *nodename, const char *servname, + const struct addrinfo *hints, + struct addrinfo **res); + void freeaddrinfo(struct addrinfo *ai); + + #ifndef AI_PASSIVE + #define AI_PASSIVE (1<<1) + #endif + + #ifndef EAI_NONAME + #define EAI_NONAME -1 + #define EAI_AGAIN -2 + #define EAI_FAIL -3 + #define EAI_NODATA -4 + #define EAI_FAMILY -5 + #define EAI_SOCKTYPE -6 + #define EAI_SERVICE -7 + #define EAI_ADDRFAMILY -8 + #define EAI_SYSTEM -10 + #define EAI_MEMORY -11 + #endif + + /* If we don't have getaddrinfo then we probably don't have + sockaddr_storage either (same RFC) so we definitely will not be + doing any IPv6 stuff. Do not use the members of this structure to + retain portability, cast to a sockaddr. */ + #define sockaddr_storage sockaddr_in +#endif + +// getnameinfo support (glibc2.0 has getaddrinfo only) +#ifndef HAVE_GETNAMEINFO + + int getnameinfo(const struct sockaddr *sa, socklen_t salen, + char *host, size_t hostlen, + char *serv, size_t servlen, + int flags); + + #ifndef NI_MAXHOST + #define NI_MAXHOST 1025 + #define NI_MAXSERV 32 + #endif + + #ifndef NI_NUMERICHOST + #define NI_NUMERICHOST (1<<0) + #define NI_NUMERICSERV (1<<1) +// #define NI_NOFQDN (1<<2) + #define NI_NAMEREQD (1<<3) + #define NI_DATAGRAM (1<<4) + #endif + + #define sockaddr_storage sockaddr_in +#endif + +// Glibc 2.0.7 misses this one +#ifndef AI_NUMERICHOST +#define AI_NUMERICHOST 0 +#endif + +#endif diff --git a/methods/rred.cc b/methods/rred.cc new file mode 100644 index 0000000..3453bd3 --- /dev/null +++ b/methods/rred.cc @@ -0,0 +1,885 @@ +// Copyright (c) 2014 Anthony Towns +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. + +#include <config.h> + +#ifndef APT_EXCLUDE_RRED_METHOD_CODE +#include "aptmethod.h" +#include <apt-pkg/configuration.h> +#include <apt-pkg/init.h> +#endif + +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/strutl.h> + +#include <apt-private/private-cmndline.h> + +#include <iostream> +#include <list> +#include <string> +#include <vector> +#include <stddef.h> + +#include <cassert> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/time.h> + +#include <apti18n.h> + +#ifndef APT_MEMBLOCK_SIZE +#define APT_MEMBLOCK_SIZE (512*1024) +#endif + +static bool ShowHelp(CommandLine &) +{ + std::cout << + "Usage: rred [options] -t input output patch-1 … patch-N\n" + " rred [options] -f patch-1 … patch-N < input > output\n" + " rred [options] patch-1 … patch-N > merged-patch\n" + "\n" + "The main use of this binary is by APTs acquire system, a mode reached\n" + "by calling it without any arguments and driven via messages on stdin.\n" + "\n" + "For the propose of testing as well as simpler direct usage the above\n" + "mentioned modes to work with \"reversed restricted ed\" patches as well.\n" + "\n" + "The arguments used above are:\n" + "* input: denotes a file you want to patch.\n" + "* output: a file you want to store the patched content in.\n" + "* patch-1 … patch-N: One or more files containing a patch.\n" + "* merged-patch: All changes by patch-1 … patch-N in one patch.\n" + "\n" + "This rred supports the commands 'a', 'c' and 'd', both single as well\n" + "as multi line. Other commands are not supported (hence 'restricted').\n" + "The command to patch the last line must appear first in the patch\n" + "(hence 'reversed'). Such a patch can e.g. be produced with 'diff --ed'.\n" + ; + return true; +} + +class MemBlock { + char *start; + size_t size; + char *free; + MemBlock *next = nullptr; + + explicit MemBlock(size_t size) : size(size) + { + free = start = new char[size]; + } + + size_t avail(void) { return size - (free - start); } + + public: + + MemBlock() : MemBlock(APT_MEMBLOCK_SIZE) {} + + ~MemBlock() { + delete [] start; + delete next; + } + + void clear(void) { + free = start; + if (next) + next->clear(); + } + + char *add_easy(char *src, size_t len, char *last) + { + if (last) { + for (MemBlock *k = this; k; k = k->next) { + if (k->free == last) { + if (len <= k->avail()) { + char * const n = k->add(src, len); + assert(last == n); // we checked already that the block is big enough, so a new one shouldn't be used + return (last == n) ? nullptr : n; + } else { + break; + } + } else if (last >= start && last < free) { + break; + } + } + } + return add(src, len); + } + + char *add(char *src, size_t len) { + if (len > avail()) { + if (!next) { + if (len > APT_MEMBLOCK_SIZE) { + next = new MemBlock(len); + } else { + next = new MemBlock(); + } + } + return next->add(src, len); + } + char *dst = free; + free += len; + memcpy(dst, src, len); + return dst; + } +}; + +struct Change { + /* Ordering: + * + * 1. write out <offset> lines unchanged + * 2. skip <del_cnt> lines from source + * 3. write out <add_cnt> lines (<add>/<add_len>) + */ + size_t offset; + size_t del_cnt; + size_t add_cnt; /* lines */ + size_t add_len; /* bytes */ + char *add; + + explicit Change(size_t off) + { + offset = off; + del_cnt = add_cnt = add_len = 0; + add = NULL; + } + + /* actually, don't write <lines> lines from <add> */ + bool skip_lines(size_t lines) + { + while (lines > 0) { + char *s = (char*) memchr(add, '\n', add_len); + if (s == nullptr) + return _error->Error("No line left in add_len data to skip (1)"); + s++; + add_len -= (s - add); + add_cnt--; + lines--; + if (add_len == 0) { + add = nullptr; + if (add_cnt != 0 || lines != 0) + return _error->Error("No line left in add_len data to skip (2)"); + } else { + add = s; + if (add_cnt == 0) + return _error->Error("No line left in add_len data to skip (3)"); + } + } + return true; + } +}; + +class FileChanges { + std::list<struct Change> changes; + std::list<struct Change>::iterator where; + size_t pos; // line number is as far left of iterator as possible + + bool pos_is_okay(void) const + { +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + // this isn't unsafe, it is just a moderately expensive check we want to avoid normally + size_t cpos = 0; + std::list<struct Change>::const_iterator x; + for (x = changes.begin(); x != where; ++x) { + assert(x != changes.end()); + cpos += x->offset + x->add_cnt; + } + return cpos == pos; +#else + return true; +#endif + } + + public: + FileChanges() { + where = changes.end(); + pos = 0; + } + + std::list<struct Change>::iterator begin(void) { return changes.begin(); } + std::list<struct Change>::iterator end(void) { return changes.end(); } + + std::list<struct Change>::reverse_iterator rbegin(void) { return changes.rbegin(); } + std::list<struct Change>::reverse_iterator rend(void) { return changes.rend(); } + + bool add_change(Change c) { + assert(pos_is_okay()); + if (not go_to_change_for(c.offset) || + pos + where->offset != c.offset) + return false; + if (c.del_cnt > 0) + if (not delete_lines(c.del_cnt)) + return false; + if (pos + where->offset != c.offset) + return false; + if (c.add_len > 0) { + assert(pos_is_okay()); + if (where->add_len > 0) + if (not new_change()) + return false; + if (where->add_len != 0 || where->add_cnt != 0) + return false; + + where->add_len = c.add_len; + where->add_cnt = c.add_cnt; + where->add = c.add; + } + assert(pos_is_okay()); + if (not merge()) + return false; + return pos_is_okay(); + } + + private: + bool merge(void) + { + while (where->offset == 0 && where != changes.begin()) { + if (not left()) + return false; + } + std::list<struct Change>::iterator next = where; + ++next; + + while (next != changes.end() && next->offset == 0) { + where->del_cnt += next->del_cnt; + next->del_cnt = 0; + if (next->add == NULL) { + next = changes.erase(next); + } else if (where->add == NULL) { + where->add = next->add; + where->add_len = next->add_len; + where->add_cnt = next->add_cnt; + next = changes.erase(next); + } else { + ++next; + } + } + return true; + } + + bool go_to_change_for(size_t line) + { + while(where != changes.end()) { + if (line < pos) { + if (not left()) + return false; + continue; + } + if (pos + where->offset + where->add_cnt <= line) { + if (not right()) + return false; + continue; + } + // line is somewhere in this slot + if (line < pos + where->offset) { + break; + } else if (line == pos + where->offset) { + return true; + } else { + if (not split(line - pos)) + return false; + return right(); + } + } + /* it goes before this patch */ + return insert(line-pos); + } + + bool new_change(void) { return insert(where->offset); } + + bool insert(size_t offset) + { + assert(pos_is_okay()); + if (where != changes.end() && offset > where->offset) + return false; + if (where != changes.end()) + where->offset -= offset; + changes.insert(where, Change(offset)); + --where; + return pos_is_okay(); + } + + bool split(size_t offset) + { + assert(pos_is_okay()); + if (where->offset >= offset || offset >= where->offset + where->add_cnt) + return false; + + size_t keep_lines = offset - where->offset; + + Change before(*where); + + where->del_cnt = 0; + where->offset = 0; + if (not where->skip_lines(keep_lines)) + return false; + + before.add_cnt = keep_lines; + before.add_len -= where->add_len; + + changes.insert(where, before); + --where; + return pos_is_okay(); + } + + bool delete_lines(size_t cnt) + { + assert(pos_is_okay()); + std::list<struct Change>::iterator x = where; + while (cnt > 0) + { + size_t del; + del = x->add_cnt; + if (del > cnt) + del = cnt; + if (not x->skip_lines(del)) + return false; + cnt -= del; + + ++x; + if (x == changes.end()) { + del = cnt; + } else { + del = x->offset; + if (del > cnt) + del = cnt; + x->offset -= del; + } + where->del_cnt += del; + cnt -= del; + } + return pos_is_okay(); + } + + bool left(void) { + assert(pos_is_okay()); + --where; + pos -= where->offset + where->add_cnt; + return pos_is_okay(); + } + + bool right(void) { + assert(pos_is_okay()); + pos += where->offset + where->add_cnt; + ++where; + return pos_is_okay(); + } +}; + +class Patch { + FileChanges filechanges; + MemBlock add_text; + + static bool retry_fwrite(char *b, size_t l, FileFd &f, Hashes * const start_hash, Hashes * const end_hash = nullptr) APT_NONNULL(1) + { + if (f.Write(b, l) == false) + return false; + if (start_hash) + start_hash->Add((unsigned char*)b, l); + if (end_hash) + end_hash->Add((unsigned char*)b, l); + return true; + } + + static void dump_rest(FileFd &o, FileFd &i, + Hashes * const start_hash, Hashes * const end_hash) + { + char buffer[APT_MEMBLOCK_SIZE]; + unsigned long long l = 0; + while (i.Read(buffer, sizeof(buffer), &l)) { + if (l ==0 || !retry_fwrite(buffer, l, o, start_hash, end_hash)) + break; + } + } + + static void dump_lines(FileFd &o, FileFd &i, size_t n, + Hashes * const start_hash, Hashes * const end_hash) + { + char buffer[APT_MEMBLOCK_SIZE]; + while (n > 0) { + if (i.ReadLine(buffer, sizeof(buffer)) == NULL) + buffer[0] = '\0'; + size_t const l = strlen(buffer); + if (l == 0 || buffer[l-1] == '\n') + n--; + retry_fwrite(buffer, l, o, start_hash, end_hash); + } + } + + static void skip_lines(FileFd &i, int n, Hashes * const start_hash) + { + char buffer[APT_MEMBLOCK_SIZE]; + while (n > 0) { + if (i.ReadLine(buffer, sizeof(buffer)) == NULL) + buffer[0] = '\0'; + size_t const l = strlen(buffer); + if (l == 0 || buffer[l-1] == '\n') + n--; + if (start_hash) + start_hash->Add((unsigned char*)buffer, l); + } + } + + static void dump_mem(FileFd &o, char *p, size_t s, Hashes *hash) APT_NONNULL(2) { + retry_fwrite(p, s, o, nullptr, hash); + } + + public: + + bool read_diff(FileFd &f, Hashes * const h) + { + char buffer[APT_MEMBLOCK_SIZE]; + bool cmdwanted = true; + + Change ch(std::numeric_limits<size_t>::max()); + if (f.ReadLine(buffer, sizeof(buffer)) == nullptr) + { + if (f.Eof()) + return true; + return _error->Error("Reading first line of patchfile %s failed", f.Name().c_str()); + } + do { + if (h != NULL) + h->Add(buffer); + if (cmdwanted) { + char *m, *c; + size_t s, e; + errno = 0; + s = strtoul(buffer, &m, 10); + if (unlikely(m == buffer || s == std::numeric_limits<unsigned long>::max() || errno != 0)) + return _error->Error("Parsing patchfile %s failed: Expected an effected line start", f.Name().c_str()); + else if (*m == ',') { + ++m; + e = strtol(m, &c, 10); + if (unlikely(m == c || e == std::numeric_limits<unsigned long>::max() || errno != 0)) + return _error->Error("Parsing patchfile %s failed: Expected an effected line end", f.Name().c_str()); + if (unlikely(e < s)) + return _error->Error("Parsing patchfile %s failed: Effected lines end %lu is before start %lu", f.Name().c_str(), e, s); + } else { + e = s; + c = m; + } + if (s > ch.offset) + return _error->Error("Parsing patchfile %s failed: Effected line is after previous effected line", f.Name().c_str()); + switch(*c) { + case 'a': + cmdwanted = false; + ch.add = NULL; + ch.add_cnt = 0; + ch.add_len = 0; + ch.offset = s; + ch.del_cnt = 0; + break; + case 'c': + if (unlikely(s == 0)) + return _error->Error("Parsing patchfile %s failed: Change command can't effect line zero", f.Name().c_str()); + cmdwanted = false; + ch.add = NULL; + ch.add_cnt = 0; + ch.add_len = 0; + ch.offset = s - 1; + ch.del_cnt = e - s + 1; + break; + case 'd': + if (unlikely(s == 0)) + return _error->Error("Parsing patchfile %s failed: Delete command can't effect line zero", f.Name().c_str()); + ch.offset = s - 1; + ch.del_cnt = e - s + 1; + ch.add = NULL; + ch.add_cnt = 0; + ch.add_len = 0; + if (not filechanges.add_change(ch)) + return _error->Error("Parsing patchfile %s failed: Delete command could not be added to changes", f.Name().c_str()); + break; + default: + return _error->Error("Parsing patchfile %s failed: Unknown command", f.Name().c_str()); + } + } else { /* !cmdwanted */ + if (strcmp(buffer, ".\n") == 0) { + cmdwanted = true; + if (not filechanges.add_change(ch)) + return _error->Error("Parsing patchfile %s failed: Data couldn't be added for command (1)", f.Name().c_str()); + } else { + char *last = NULL; + char *add; + size_t l; + if (ch.add) + last = ch.add + ch.add_len; + l = strlen(buffer); + add = add_text.add_easy(buffer, l, last); + if (!add) { + ch.add_len += l; + ch.add_cnt++; + } else { + if (ch.add) { + if (not filechanges.add_change(ch)) + return _error->Error("Parsing patchfile %s failed: Data couldn't be added for command (2)", f.Name().c_str()); + ch.del_cnt = 0; + } + ch.offset += ch.add_cnt; + ch.add = add; + ch.add_len = l; + ch.add_cnt = 1; + } + } + } + } while(f.ReadLine(buffer, sizeof(buffer))); + return true; + } + + void write_diff(FileFd &f) + { + unsigned long long line = 0; + std::list<struct Change>::reverse_iterator ch; + for (ch = filechanges.rbegin(); ch != filechanges.rend(); ++ch) { + line += ch->offset + ch->del_cnt; + } + + for (ch = filechanges.rbegin(); ch != filechanges.rend(); ++ch) { + std::list<struct Change>::reverse_iterator mg_i, mg_e = ch; + while (ch->del_cnt == 0 && ch->offset == 0) + { + ++ch; + if (unlikely(ch == filechanges.rend())) + return; + } + line -= ch->del_cnt; + std::string buf; + if (ch->add_cnt > 0) { + if (ch->del_cnt == 0) { + strprintf(buf, "%llua\n", line); + } else if (ch->del_cnt == 1) { + strprintf(buf, "%lluc\n", line+1); + } else { + strprintf(buf, "%llu,%lluc\n", line+1, line+ch->del_cnt); + } + f.Write(buf.c_str(), buf.length()); + + mg_i = ch; + do { + dump_mem(f, mg_i->add, mg_i->add_len, NULL); + } while (mg_i-- != mg_e); + + buf = ".\n"; + f.Write(buf.c_str(), buf.length()); + } else if (ch->del_cnt == 1) { + strprintf(buf, "%llud\n", line+1); + f.Write(buf.c_str(), buf.length()); + } else if (ch->del_cnt > 1) { + strprintf(buf, "%llu,%llud\n", line+1, line+ch->del_cnt); + f.Write(buf.c_str(), buf.length()); + } + line -= ch->offset; + } + } + + void apply_against_file(FileFd &out, FileFd &in, + Hashes * const start_hash = nullptr, Hashes * const end_hash = nullptr) + { + std::list<struct Change>::iterator ch; + for (ch = filechanges.begin(); ch != filechanges.end(); ++ch) { + dump_lines(out, in, ch->offset, start_hash, end_hash); + skip_lines(in, ch->del_cnt, start_hash); + if (ch->add_len != 0) + dump_mem(out, ch->add, ch->add_len, end_hash); + } + dump_rest(out, in, start_hash, end_hash); + out.Flush(); + } +}; + +#ifndef APT_EXCLUDE_RRED_METHOD_CODE +class RredMethod : public aptMethod { + private: + bool Debug; + + struct PDiffFile { + std::string FileName; + HashStringList ExpectedHashes; + PDiffFile(std::string const &FileName, HashStringList const &ExpectedHashes) : + FileName(FileName), ExpectedHashes(ExpectedHashes) {} + }; + + HashStringList ReadExpectedHashesForPatch(unsigned int const patch, std::string const &Message) + { + HashStringList ExpectedHashes; + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + { + std::string tagname; + strprintf(tagname, "Patch-%d-%s-Hash", patch, *type); + std::string const hashsum = LookupTag(Message, tagname.c_str()); + if (hashsum.empty() == false) + ExpectedHashes.push_back(HashString(*type, hashsum)); + } + return ExpectedHashes; + } + + protected: + virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE { + Debug = DebugEnabled(); + URI Get(Itm->Uri); + std::string Path = DecodeSendURI(Get.Host + Get.Path); // rred:/path - no host + + FetchResult Res; + Res.Filename = Itm->DestFile; + if (Itm->Uri.empty()) + { + Path = Itm->DestFile; + Itm->DestFile.append(".result"); + } else + URIStart(Res); + + std::vector<PDiffFile> patchfiles; + Patch patch; + + HashStringList StartHashes; + for (char const * const * type = HashString::SupportedHashes(); *type != nullptr; ++type) + { + std::string tagname; + strprintf(tagname, "Start-%s-Hash", *type); + std::string const hashsum = LookupTag(Message, tagname.c_str()); + if (hashsum.empty() == false) + StartHashes.push_back(HashString(*type, hashsum)); + } + + if (FileExists(Path + ".ed") == true) + { + HashStringList const ExpectedHashes = ReadExpectedHashesForPatch(0, Message); + std::string const FileName = Path + ".ed"; + if (ExpectedHashes.usable() == false) + return _error->Error("No hashes found for uncompressed patch: %s", FileName.c_str()); + patchfiles.push_back(PDiffFile(FileName, ExpectedHashes)); + } + else + { + _error->PushToStack(); + std::vector<std::string> patches = GetListOfFilesInDir(flNotFile(Path), "gz", true, false); + _error->RevertToStack(); + + std::string const baseName = Path + ".ed."; + unsigned int seen_patches = 0; + for (std::vector<std::string>::const_iterator p = patches.begin(); + p != patches.end(); ++p) + { + if (p->compare(0, baseName.length(), baseName) == 0) + { + HashStringList const ExpectedHashes = ReadExpectedHashesForPatch(seen_patches, Message); + if (ExpectedHashes.usable() == false) + return _error->Error("No hashes found for uncompressed patch %d: %s", seen_patches, p->c_str()); + patchfiles.push_back(PDiffFile(*p, ExpectedHashes)); + ++seen_patches; + } + } + } + + std::string patch_name; + for (std::vector<PDiffFile>::iterator I = patchfiles.begin(); + I != patchfiles.end(); + ++I) + { + patch_name = I->FileName; + if (Debug == true) + std::clog << "Patching " << Path << " with " << patch_name + << std::endl; + + FileFd p; + Hashes patch_hash(I->ExpectedHashes); + // all patches are compressed, even if the name doesn't reflect it + if (p.Open(patch_name, FileFd::ReadOnly, FileFd::Gzip) == false || + patch.read_diff(p, &patch_hash) == false) + { + _error->DumpErrors(std::cerr, GlobalError::DEBUG, false); + return false; + } + p.Close(); + HashStringList const hsl = patch_hash.GetHashStringList(); + if (hsl != I->ExpectedHashes) + return _error->Error("Hash Sum mismatch for uncompressed patch %s", patch_name.c_str()); + } + + if (Debug == true) + std::clog << "Applying patches against " << Path + << " and writing results to " << Itm->DestFile + << std::endl; + + FileFd inp, out; + if (inp.Open(Path, FileFd::ReadOnly, FileFd::Extension) == false) + { + if (Debug == true) + std::clog << "FAILED to open inp " << Path << std::endl; + return _error->Error("Failed to open inp %s", Path.c_str()); + } + if (out.Open(Itm->DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Empty | FileFd::BufferedWrite, FileFd::Extension) == false) + { + if (Debug == true) + std::clog << "FAILED to open out " << Itm->DestFile << std::endl; + return _error->Error("Failed to open out %s", Itm->DestFile.c_str()); + } + + Hashes end_hash(Itm->ExpectedHashes); + if (StartHashes.usable()) + { + Hashes start_hash(StartHashes); + patch.apply_against_file(out, inp, &start_hash, &end_hash); + if (start_hash.GetHashStringList() != StartHashes) + _error->Error("The input file hadn't the expected hash!"); + } + else + patch.apply_against_file(out, inp, nullptr, &end_hash); + + out.Close(); + inp.Close(); + + if (_error->PendingError() == true) { + if (Debug == true) + std::clog << "FAILED to read or write files" << std::endl; + return false; + } + + if (Debug == true) { + std::clog << "rred: finished file patching of " << Path << "." << std::endl; + } + + struct stat bufbase, bufpatch; + if (stat(Path.c_str(), &bufbase) != 0 || + stat(patch_name.c_str(), &bufpatch) != 0) + return _error->Errno("stat", _("Failed to stat %s"), Path.c_str()); + + struct timeval times[2]; + times[0].tv_sec = bufbase.st_atime; + times[1].tv_sec = bufpatch.st_mtime; + times[0].tv_usec = times[1].tv_usec = 0; + if (utimes(Itm->DestFile.c_str(), times) != 0) + return _error->Errno("utimes",_("Failed to set modification time")); + + if (stat(Itm->DestFile.c_str(), &bufbase) != 0) + return _error->Errno("stat", _("Failed to stat %s"), Itm->DestFile.c_str()); + + Res.LastModified = bufbase.st_mtime; + Res.Size = bufbase.st_size; + Res.TakeHashes(end_hash); + URIDone(Res); + + return true; + } + + public: + RredMethod() : aptMethod("rred", "2.0", SendConfig | SendURIEncoded), Debug(false) + { + SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY; + } +}; + +static const APT::Configuration::Compressor *FindCompressor(std::vector<APT::Configuration::Compressor> const &compressors, std::string const &name) /*{{{*/ +{ + APT::Configuration::Compressor const * compressor = nullptr; + for (auto const & c : compressors) + { + if (compressor != nullptr && c.Cost >= compressor->Cost) + continue; + if (c.Name == name || c.Extension == name || (!c.Extension.empty() && c.Extension.substr(1) == name)) + compressor = &c; + } + return compressor; +} + /*}}}*/ +static std::vector<aptDispatchWithHelp> GetCommands() +{ + return {{nullptr, nullptr, nullptr}}; +} +int main(int argc, const char *argv[]) +{ + if (argc <= 1) + return RredMethod().Run(); + + CommandLine CmdL; + auto const Cmds = ParseCommandLine(CmdL, APT_CMD::RRED, &_config, nullptr, argc, argv, &ShowHelp, &GetCommands); + + FileFd input, output; + unsigned int argi = 0; + auto const argmax = CmdL.FileSize(); + bool const quiet = _config->FindI("quiet", 0) >= 2; + + std::string const compressorName = _config->Find("Rred::Compress", ""); + auto const compressors = APT::Configuration::getCompressors(); + APT::Configuration::Compressor const * compressor = nullptr; + if (not compressorName.empty()) + { + compressor = FindCompressor(compressors, compressorName); + if (compressor == nullptr) + { + std::cerr << "E: Could not find compressor: " << compressorName << '\n'; + return 101; + } + } + + bool just_diff = false; + if (_config->FindB("Rred::T", false)) + { + if (argmax < 3) + { + std::cerr << "E: Not enough filenames given on the command line for mode 't'\n"; + return 101; + } + if (not quiet) + std::clog << "Patching " << CmdL.FileList[0] << " into " << CmdL.FileList[1] << "\n"; + input.Open(CmdL.FileList[0], FileFd::ReadOnly,FileFd::Extension); + if (compressor == nullptr) + output.Open(CmdL.FileList[1], FileFd::WriteOnly | FileFd::Create | FileFd::Empty | FileFd::BufferedWrite, FileFd::Extension); + else + output.Open(CmdL.FileList[1], FileFd::WriteOnly | FileFd::Create | FileFd::Empty | FileFd::BufferedWrite, *compressor); + argi = 2; + } + else + { + if (compressor == nullptr) + output.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly | FileFd::Create | FileFd::BufferedWrite); + else + output.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly | FileFd::Create | FileFd::BufferedWrite, *compressor); + if (_config->FindB("Rred::F", false)) + input.OpenDescriptor(STDIN_FILENO, FileFd::ReadOnly); + else + just_diff = true; + } + + if (argi + 1 > argmax) + { + std::cerr << "E: At least one patch needs to be given on the command line\n"; + return 101; + } + + Patch merged_patch; + for (; argi < argmax; ++argi) + { + FileFd patch; + if (not patch.Open(CmdL.FileList[argi], FileFd::ReadOnly, FileFd::Extension)) + { + _error->DumpErrors(std::cerr); + return 1; + } + if (not merged_patch.read_diff(patch, nullptr)) + { + _error->DumpErrors(std::cerr); + return 2; + } + } + + if (just_diff) + merged_patch.write_diff(output); + else + merged_patch.apply_against_file(output, input); + + output.Close(); + input.Close(); + return DispatchCommandLine(CmdL, {}); +} +#endif diff --git a/methods/rsh.cc b/methods/rsh.cc new file mode 100644 index 0000000..f5e1279 --- /dev/null +++ b/methods/rsh.cc @@ -0,0 +1,561 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + RSH method - Transfer files via rsh compatible program + + Written by Ben Collins <bcollins@debian.org>, Copyright (c) 2000 + + This file stated: + Licensed under the GNU General Public License v2 [no exception clauses] + + We believe that this was intended to be not a statement against future + versions of the GPL, but meant to exclude the Qt license exception in + place in APT until that time. + + We received permission from Ben in 2021 to relicense under GPL-2+, + contributions from Adam Heath and Daniel Hartwig may still have to + be considered GPL-2 for the time being. + + Other contributions are GPL-2+ + + See https://lists.debian.org/deity/2021/04/msg00013.html for details + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/strutl.h> + +#include "rsh.h" +#include <errno.h> +#include <signal.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <unistd.h> + +#include <apti18n.h> + /*}}}*/ + +unsigned long TimeOut = 30; +Configuration::Item const *RshOptions = 0; +time_t RSHMethod::FailTime = 0; +std::string RSHMethod::FailFile; +int RSHMethod::FailFd = -1; + +// RSHConn::RSHConn - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +RSHConn::RSHConn(std::string const &pProg, URI Srv) : Len(0), WriteFd(-1), ReadFd(-1), + ServerName(Srv), Prog(pProg), Process(-1) { + Buffer[0] = '\0'; +} + /*}}}*/ +// RSHConn::RSHConn - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +RSHConn::~RSHConn() +{ + Close(); +} + /*}}}*/ +// RSHConn::Close - Forcibly terminate the connection /*{{{*/ +// --------------------------------------------------------------------- +/* Often this is called when things have gone wrong to indicate that the + connection is no longer usable. */ +void RSHConn::Close() +{ + if (Process == -1) + return; + + close(WriteFd); + close(ReadFd); + kill(Process,SIGINT); + ExecWait(Process,"",true); + WriteFd = -1; + ReadFd = -1; + Process = -1; +} + /*}}}*/ +// RSHConn::Open - Connect to a host /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool RSHConn::Open() +{ + // Use the already open connection if possible. + if (Process != -1) + return true; + + if (Connect(ServerName.Host,ServerName.Port,ServerName.User) == false) + return false; + + return true; +} + /*}}}*/ +// RSHConn::Connect - Fire up rsh and connect /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool RSHConn::Connect(std::string const &Host, unsigned int Port, std::string const &User) +{ + char *PortStr = NULL; + if (Port != 0) + { + if (asprintf (&PortStr, "%d", Port) == -1 || PortStr == NULL) + return _error->Errno("asprintf", _("Failed")); + } + + // Create the pipes + int Pipes[4] = {-1,-1,-1,-1}; + if (pipe(Pipes) != 0 || pipe(Pipes+2) != 0) + { + _error->Errno("pipe",_("Failed to create IPC pipe to subprocess")); + for (int I = 0; I != 4; I++) + close(Pipes[I]); + return false; + } + for (int I = 0; I != 4; I++) + SetCloseExec(Pipes[I],true); + + Process = ExecFork(); + + // The child + if (Process == 0) + { + const char *Args[400]; + unsigned int i = 0; + + dup2(Pipes[1],STDOUT_FILENO); + dup2(Pipes[2],STDIN_FILENO); + + // Probably should do + // dup2(open("/dev/null",O_RDONLY),STDERR_FILENO); + + Args[i++] = Prog.c_str(); + + // Insert user-supplied command line options + Configuration::Item const *Opts = RshOptions; + if (Opts != 0) + { + Opts = Opts->Child; + for (; Opts != 0; Opts = Opts->Next) + { + if (Opts->Value.empty() == true) + continue; + Args[i++] = Opts->Value.c_str(); + } + } + + if (User.empty() == false) { + Args[i++] = "-l"; + Args[i++] = User.c_str(); + } + if (PortStr != NULL) { + Args[i++] = "-p"; + Args[i++] = PortStr; + } + if (Host.empty() == false) { + Args[i++] = Host.c_str(); + } + Args[i++] = "/bin/sh"; + Args[i] = 0; + execvp(Args[0],(char **)Args); + exit(100); + } + + if (PortStr != NULL) + free(PortStr); + + ReadFd = Pipes[0]; + WriteFd = Pipes[3]; + SetNonBlock(Pipes[0],true); + SetNonBlock(Pipes[3],true); + close(Pipes[1]); + close(Pipes[2]); + + return true; +} +bool RSHConn::Connect(std::string const &Host, std::string const &User) +{ + return Connect(Host, 0, User); +} + /*}}}*/ +// RSHConn::ReadLine - Very simple buffered read with timeout /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool RSHConn::ReadLine(std::string &Text) +{ + if (Process == -1 || ReadFd == -1) + return false; + + // Suck in a line + while (Len < sizeof(Buffer)) + { + // Scan the buffer for a new line + for (unsigned int I = 0; I != Len; I++) + { + // Escape some special chars + if (Buffer[I] == 0) + Buffer[I] = '?'; + + // End of line? + if (Buffer[I] != '\n') + continue; + + I++; + Text = std::string(Buffer,I); + memmove(Buffer,Buffer+I,Len - I); + Len -= I; + return true; + } + + // Wait for some data.. + if (WaitFd(ReadFd,false,TimeOut) == false) + { + Close(); + return _error->Error(_("Connection timeout")); + } + + // Suck it back + int Res = read(ReadFd,Buffer + Len,sizeof(Buffer) - Len); + if (Res <= 0) + { + _error->Errno("read",_("Read error")); + Close(); + return false; + } + Len += Res; + } + + return _error->Error(_("A response overflowed the buffer.")); +} + /*}}}*/ +// RSHConn::WriteMsg - Send a message with optional remote sync. /*{{{*/ +// --------------------------------------------------------------------- +/* The remote sync flag appends a || echo which will insert blank line + once the command completes. */ +bool RSHConn::WriteMsg(std::string &Text,bool Sync,const char *Fmt,...) +{ + va_list args; + va_start(args,Fmt); + + // sprintf into a buffer + char Tmp[1024]; + vsnprintf(Tmp,sizeof(Tmp),Fmt,args); + va_end(args); + + // concat to create the real msg + std::string Msg; + if (Sync == true) + Msg = std::string(Tmp) + " 2> /dev/null || echo\n"; + else + Msg = std::string(Tmp) + " 2> /dev/null\n"; + + // Send it off + const char *S = Msg.c_str(); + unsigned long Len = strlen(S); + unsigned long Start = 0; + while (Len != 0) + { + if (WaitFd(WriteFd,true,TimeOut) == false) + { + + Close(); + return _error->Error(_("Connection timeout")); + } + + int Res = write(WriteFd,S + Start,Len); + if (Res <= 0) + { + _error->Errno("write",_("Write error")); + Close(); + return false; + } + + Len -= Res; + Start += Res; + } + + if (Sync == true) + return ReadLine(Text); + return true; +} + /*}}}*/ +// RSHConn::Size - Return the size of the file /*{{{*/ +// --------------------------------------------------------------------- +/* Right now for successful transfer the file size must be known in + advance. */ +bool RSHConn::Size(const char *Path,unsigned long long &Size) +{ + // Query the size + std::string Msg; + Size = 0; + + if (WriteMsg(Msg,true,"find %s -follow -printf '%%s\\n'",Path) == false) + return false; + + // FIXME: Sense if the bad reply is due to a File Not Found. + + char *End; + Size = strtoull(Msg.c_str(),&End,10); + if (End == Msg.c_str()) + return _error->Error(_("File not found")); + return true; +} + /*}}}*/ +// RSHConn::ModTime - Get the modification time in UTC /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool RSHConn::ModTime(const char *Path, time_t &Time) +{ + Time = time(&Time); + // Query the mod time + std::string Msg; + + if (WriteMsg(Msg,true,"TZ=UTC find %s -follow -printf '%%TY%%Tm%%Td%%TH%%TM%%TS\\n'",Path) == false) + return false; + + // Parse it + return FTPMDTMStrToTime(Msg.c_str(), Time); +} + /*}}}*/ +// RSHConn::Get - Get a file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool RSHConn::Get(const char *Path,FileFd &To,unsigned long long Resume, + Hashes &Hash,bool &Missing, unsigned long long Size) +{ + Missing = false; + + // Round to a 2048 byte block + Resume = Resume - (Resume % 2048); + + if (To.Truncate(Resume) == false) + return false; + if (To.Seek(0) == false) + return false; + + if (Resume != 0) { + if (Hash.AddFD(To,Resume) == false) { + _error->Errno("read",_("Problem hashing file")); + return false; + } + } + + // FIXME: Detect file-not openable type errors. + std::string Jnk; + if (WriteMsg(Jnk,false,"dd if=%s bs=2048 skip=%u", Path, Resume / 2048) == false) + return false; + + // Copy loop + unsigned long long MyLen = Resume; + unsigned char Buffer[4096]; + while (MyLen < Size) + { + // Wait for some data.. + if (WaitFd(ReadFd,false,TimeOut) == false) + { + Close(); + return _error->Error(_("Data socket timed out")); + } + + // Read the data.. + int Res = read(ReadFd,Buffer,sizeof(Buffer)); + if (Res == 0) + { + Close(); + return _error->Error(_("Connection closed prematurely")); + } + + if (Res < 0) + { + if (errno == EAGAIN) + continue; + break; + } + MyLen += Res; + + Hash.Add(Buffer,Res); + if (To.Write(Buffer,Res) == false) + { + Close(); + return false; + } + } + + return true; +} + /*}}}*/ + +// RSHMethod::RSHMethod - Constructor /*{{{*/ +RSHMethod::RSHMethod(std::string &&pProg) : aptMethod(std::move(pProg),"1.0",SendConfig | SendURIEncoded) +{ + signal(SIGTERM,SigTerm); + signal(SIGINT,SigTerm); + Server = 0; + FailFd = -1; +} + /*}}}*/ +// RSHMethod::Configuration - Handle a configuration message /*{{{*/ +// --------------------------------------------------------------------- +bool RSHMethod::Configuration(std::string Message) +{ + // enabling privilege dropping for this method requires configuration… + // … which is otherwise lifted straight from root, so use it by default. + _config->Set(std::string("Binary::") + Binary + "::APT::Sandbox::User", ""); + + if (aptMethod::Configuration(Message) == false) + return false; + + std::string const timeconf = std::string("Acquire::") + Binary + "::Timeout"; + TimeOut = _config->FindI(timeconf, TimeOut); + std::string const optsconf = std::string("Acquire::") + Binary + "::Options"; + RshOptions = _config->Tree(optsconf.c_str()); + + return true; +} + /*}}}*/ +// RSHMethod::SigTerm - Clean up and timestamp the files on exit /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void RSHMethod::SigTerm(int) +{ + if (FailFd == -1) + _exit(100); + + // Transfer the modification times + struct timeval times[2]; + times[0].tv_sec = FailTime; + times[1].tv_sec = FailTime; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(FailFile.c_str(), times); + close(FailFd); + + _exit(100); +} + /*}}}*/ +// RSHMethod::Fetch - Fetch a URI /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool RSHMethod::Fetch(FetchItem *Itm) +{ + URI Get(Itm->Uri); + auto const File = DecodeSendURI(Get.Path); + FetchResult Res; + Res.Filename = Itm->DestFile; + Res.IMSHit = false; + + // Connect to the server + if (Server == 0 || Server->Comp(Get) == false) { + delete Server; + Server = new RSHConn(Binary, Get); + } + + // Could not connect is a transient error.. + if (Server->Open() == false) { + Server->Close(); + Fail(true); + return true; + } + + // We say this mainly because the pause here is for the + // ssh connection that is still going + Status(_("Connecting to %s"), Get.Host.c_str()); + + // Get the files information + unsigned long long Size; + if (not Server->Size(File.c_str(), Size) || + not Server->ModTime(File.c_str(), FailTime)) + { + //Fail(true); + //_error->Error(_("File not found")); // Will be handled by Size + return false; + } + Res.Size = Size; + + // See if it is an IMS hit + if (Itm->LastModified == FailTime) { + Res.Size = 0; + Res.IMSHit = true; + URIDone(Res); + return true; + } + + // See if the file exists + struct stat Buf; + if (stat(Itm->DestFile.c_str(),&Buf) == 0) { + if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime) { + Res.Size = Buf.st_size; + Res.LastModified = Buf.st_mtime; + Res.ResumePoint = Buf.st_size; + URIDone(Res); + return true; + } + + // Resume? + if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size) + Res.ResumePoint = Buf.st_size; + } + + // Open the file + Hashes Hash(Itm->ExpectedHashes); + { + FileFd Fd(Itm->DestFile,FileFd::WriteAny); + if (_error->PendingError() == true) + return false; + + URIStart(Res); + + FailFile = Itm->DestFile; + (void)(FailFile.c_str()); // Make sure we don't do a malloc in the signal handler + FailFd = Fd.Fd(); + + bool Missing; + if (not Server->Get(File.c_str(), Fd, Res.ResumePoint, Hash, Missing, Res.Size)) + { + Fd.Close(); + + // Timestamp + struct timeval times[2]; + times[0].tv_sec = FailTime; + times[1].tv_sec = FailTime; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(FailFile.c_str(), times); + + // If the file is missing we hard fail otherwise transient fail + if (Missing == true) + return false; + Fail(true); + return true; + } + + Res.Size = Fd.Size(); + struct timeval times[2]; + times[0].tv_sec = FailTime; + times[1].tv_sec = FailTime; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(Fd.Name().c_str(), times); + FailFd = -1; + } + + Res.LastModified = FailTime; + Res.TakeHashes(Hash); + + URIDone(Res); + + return true; +} + /*}}}*/ + +int main(int, const char *argv[]) +{ + return RSHMethod(flNotDir(argv[0])).Run(); +} diff --git a/methods/rsh.h b/methods/rsh.h new file mode 100644 index 0000000..7545d58 --- /dev/null +++ b/methods/rsh.h @@ -0,0 +1,75 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + RSH method - Transfer files via rsh compatible program + + ##################################################################### */ + /*}}}*/ +#ifndef APT_RSH_H +#define APT_RSH_H + +#include <string> +#include <time.h> + +#include <apt-pkg/strutl.h> + +class Hashes; +class FileFd; + +class RSHConn +{ + char Buffer[1024*10]; + unsigned long Len; + int WriteFd; + int ReadFd; + URI ServerName; + std::string const Prog; + + // Private helper functions + bool ReadLine(std::string &Text); + + public: + + pid_t Process; + + // Raw connection IO + bool WriteMsg(std::string &Text,bool Sync,const char *Fmt,...); + bool Connect(std::string const &Host, std::string const &User); + bool Connect(std::string const &Host, unsigned int Port, std::string const &User); + bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; + + // Connection control + bool Open(); + void Close(); + + // Query + bool Size(const char *Path,unsigned long long &Size); + bool ModTime(const char *Path, time_t &Time); + bool Get(const char *Path,FileFd &To,unsigned long long Resume, + Hashes &Hash,bool &Missing, unsigned long long Size); + + RSHConn(std::string const &Prog, URI Srv); + ~RSHConn(); +}; + +#include "aptmethod.h" + +class RSHMethod : public aptMethod +{ + virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; + virtual bool Configuration(std::string Message) APT_OVERRIDE; + + RSHConn *Server; + + static std::string FailFile; + static int FailFd; + static time_t FailTime; + static APT_NORETURN void SigTerm(int); + + public: + + explicit RSHMethod(std::string &&Prog); +}; + +#endif diff --git a/methods/store.cc b/methods/store.cc new file mode 100644 index 0000000..8ffc7b9 --- /dev/null +++ b/methods/store.cc @@ -0,0 +1,147 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Store method - Takes a file URI and stores its content (for which it will + calculate the hashes) in the given destination. The input file will be + extracted based on its file extension (or with the given compressor if + called with one of the compatible symlinks) and potentially recompressed + based on the file extension of the destination filename. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include "aptmethod.h" +#include <apt-pkg/aptconfiguration.h> +#include <apt-pkg/configuration.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/strutl.h> + +#include <string> +#include <vector> +#include <string.h> +#include <sys/stat.h> +#include <sys/time.h> + +#include <apti18n.h> + /*}}}*/ + +class StoreMethod : public aptMethod +{ + virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; + + public: + + explicit StoreMethod(std::string &&pProg) : aptMethod(std::move(pProg),"1.2",SingleInstance | SendConfig | SendURIEncoded) + { + SeccompFlags = aptMethod::BASE; + if (Binary != "store") + methodNames.insert(methodNames.begin(), "store"); + } +}; + +static bool OpenFileWithCompressorByName(FileFd &fileFd, std::string const &Filename, unsigned int const Mode, std::string const &Name) +{ + if (Name == "store") + return fileFd.Open(Filename, Mode, FileFd::Extension); + + std::vector<APT::Configuration::Compressor> const compressors = APT::Configuration::getCompressors(); + std::vector<APT::Configuration::Compressor>::const_iterator compressor = compressors.begin(); + for (; compressor != compressors.end(); ++compressor) + if (compressor->Name == Name) + break; + if (compressor == compressors.end()) + return _error->Error("Extraction of file %s requires unknown compressor %s", Filename.c_str(), Name.c_str()); + return fileFd.Open(Filename, Mode, *compressor); +} + + + /*}}}*/ +bool StoreMethod::Fetch(FetchItem *Itm) /*{{{*/ +{ + URI Get(Itm->Uri); + std::string Path = DecodeSendURI(Get.Host + Get.Path); // To account for relative paths + + FetchResult Res; + Res.Filename = Itm->DestFile; + URIStart(Res); + + // Open the source and destination files + FileFd From; + if (_config->FindB("Method::Compress", false) == false) + { + if (OpenFileWithCompressorByName(From, Path, FileFd::ReadOnly, Binary) == false) + return false; + if(From.IsCompressed() && From.FileSize() == 0) + return _error->Error(_("Empty files can't be valid archives")); + } + else + From.Open(Path, FileFd::ReadOnly, FileFd::Extension); + if (From.IsOpen() == false || From.Failed() == true) + return false; + + FileFd To; + if (Itm->DestFile != "/dev/null" && Itm->DestFile != Path) + { + if (_config->FindB("Method::Compress", false) == false) + To.Open(Itm->DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Atomic, FileFd::Extension); + else if (OpenFileWithCompressorByName(To, Itm->DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Empty, Binary) == false) + return false; + + if (To.IsOpen() == false || To.Failed() == true) + return false; + To.EraseOnFailure(); + } + + // Read data from source, generate checksums and write + Hashes Hash(Itm->ExpectedHashes); + bool Failed = false; + Res.Size = 0; + while (1) + { + unsigned char Buffer[4*1024]; + unsigned long long Count = 0; + + if (!From.Read(Buffer,sizeof(Buffer),&Count)) + { + if (To.IsOpen()) + To.OpFail(); + return false; + } + if (Count == 0) + break; + Res.Size += Count; + + Hash.Add(Buffer,Count); + if (To.IsOpen() && To.Write(Buffer,Count) == false) + { + Failed = true; + break; + } + } + + From.Close(); + To.Close(); + + if (Failed == true) + return false; + + if (TransferModificationTimes(Path.c_str(), Itm->DestFile.c_str(), Res.LastModified) == false) + return false; + + // Return a Done response + Res.TakeHashes(Hash); + + URIDone(Res); + return true; +} + /*}}}*/ + +int main(int, char *argv[]) +{ + return StoreMethod(flNotDir(argv[0])).Run(); +} |