summaryrefslogtreecommitdiffstats
path: root/methods/mirror.cc
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 09:00:48 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 09:00:48 +0000
commit851b6a097165af4d51c0db01b5e05256e5006896 (patch)
tree5f7c388ec894a7806c49a99f3bdb605d0b299a7c /methods/mirror.cc
parentInitial commit. (diff)
downloadapt-851b6a097165af4d51c0db01b5e05256e5006896.tar.xz
apt-851b6a097165af4d51c0db01b5e05256e5006896.zip
Adding upstream version 2.6.1.upstream/2.6.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'methods/mirror.cc')
-rw-r--r--methods/mirror.cc414
1 files changed, 414 insertions, 0 deletions
diff --git a/methods/mirror.cc b/methods/mirror.cc
new file mode 100644
index 0000000..787e4c7
--- /dev/null
+++ b/methods/mirror.cc
@@ -0,0 +1,414 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ Mirror URI – This method helps avoiding hardcoding of mirrors in the
+ sources.lists by looking up a list of mirrors first to which the
+ following requests are redirected.
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include "aptmethod.h"
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/metaindex.h>
+#include <apt-pkg/sourcelist.h>
+#include <apt-pkg/strutl.h>
+
+#include <functional>
+#include <random>
+#include <string>
+#include <unordered_map>
+
+#include <sys/utsname.h>
+
+#include <apti18n.h>
+ /*}}}*/
+constexpr char const *const disallowLocal[] = {"ftp", "http", "https"};
+
+static void sortByLength(std::vector<std::string> &vec) /*{{{*/
+{
+ // this ensures having mirror://foo/ and mirror://foo/bar/ works as expected
+ // by checking for the longest matches first
+ std::sort(vec.begin(), vec.end(), [](std::string const &a, std::string const &b) {
+ return a.length() > b.length();
+ });
+}
+ /*}}}*/
+class MirrorMethod : public aptMethod /*{{{*/
+{
+ std::mt19937 genrng;
+ std::vector<std::string> sourceslist;
+ std::unordered_map<std::string, std::string> msgCache;
+ enum MirrorFileState
+ {
+ REQUESTED,
+ FAILED,
+ AVAILABLE
+ };
+ struct MirrorInfo
+ {
+ std::string uri;
+ unsigned long priority = std::numeric_limits<decltype(priority)>::max();
+ decltype(genrng)::result_type seed = 0;
+ std::unordered_map<std::string, std::vector<std::string>> tags;
+ explicit MirrorInfo(std::string const &u, std::vector<std::string> &&ptags = {}) : uri(u)
+ {
+ for (auto &&tag : ptags)
+ {
+ auto const colonfound = tag.find(':');
+ if (unlikely(colonfound == std::string::npos))
+ continue;
+ auto name = tag.substr(0, colonfound);
+ auto value = tag.substr(colonfound + 1);
+ if (name == "arch")
+ tags["Architecture"].emplace_back(std::move(value));
+ else if (name == "lang")
+ tags["Language"].emplace_back(std::move(value));
+ else if (name == "priority")
+ priority = std::strtoul(value.c_str(), nullptr, 10);
+ else if (likely(name.empty() == false))
+ {
+ if (name == "codename" || name == "suite")
+ tags["Release"].push_back(value);
+ name[0] = std::toupper(name[0]);
+ tags[std::move(name)].emplace_back(std::move(value));
+ }
+ }
+ }
+ };
+ struct MirrorListInfo
+ {
+ MirrorFileState state;
+ std::string baseuri;
+ std::vector<MirrorInfo> list;
+ };
+ std::unordered_map<std::string, MirrorListInfo> mirrorfilestate;
+
+ virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE;
+
+ void RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message);
+ bool MirrorListFileReceived(MirrorListInfo &info, FetchItem *const Itm);
+ std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm);
+ void DealWithPendingItems(std::vector<std::string> const &baseuris, MirrorListInfo const &info, FetchItem *const Itm, std::function<void()> handler);
+
+ public:
+ explicit MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests | SendURIEncoded), genrng(clock())
+ {
+ SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY;
+ }
+};
+ /*}}}*/
+void MirrorMethod::RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message) /*{{{*/
+{
+ std::unordered_map<std::string, std::string> matchers;
+ matchers.emplace("Architecture", LookupTag(Message, "Target-Architecture"));
+ matchers.emplace("Codename", LookupTag(Message, "Target-Codename"));
+ matchers.emplace("Component", LookupTag(Message, "Target-Component"));
+ matchers.emplace("Language", LookupTag(Message, "Target-Language"));
+ matchers.emplace("Release", LookupTag(Message, "Target-Release"));
+ matchers.emplace("Suite", LookupTag(Message, "Target-Suite"));
+ matchers.emplace("Type", LookupTag(Message, "Target-Type"));
+ decltype(info.list) possMirrors;
+ for (auto const &mirror : info.list)
+ {
+ bool failedMatch = false;
+ for (auto const &m : matchers)
+ {
+ if (m.second.empty())
+ continue;
+ auto const tagsetiter = mirror.tags.find(m.first);
+ if (tagsetiter == mirror.tags.end())
+ continue;
+ auto const tagset = tagsetiter->second;
+ if (tagset.empty() == false && std::find(tagset.begin(), tagset.end(), m.second) == tagset.end())
+ {
+ failedMatch = true;
+ break;
+ }
+ }
+ if (failedMatch)
+ continue;
+ possMirrors.push_back(mirror);
+ }
+ for (auto &&mirror : possMirrors)
+ mirror.seed = genrng();
+ std::sort(possMirrors.begin(), possMirrors.end(), [](MirrorInfo const &a, MirrorInfo const &b) {
+ if (a.priority != b.priority)
+ return a.priority < b.priority;
+ return a.seed < b.seed;
+ });
+ std::string const path = Itm->Uri.substr(info.baseuri.length());
+ std::string altMirrors;
+ std::unordered_map<std::string, std::string> fields;
+ fields.emplace("URI", Itm->Uri);
+ for (auto curMirror = possMirrors.cbegin(); curMirror != possMirrors.cend(); ++curMirror)
+ {
+ std::string mirror = curMirror->uri;
+ if (APT::String::Endswith(mirror, "/") == false)
+ mirror.append("/");
+ mirror.append(path);
+ if (curMirror == possMirrors.cbegin())
+ fields.emplace("New-URI", mirror);
+ else if (altMirrors.empty())
+ altMirrors.append(mirror);
+ else
+ altMirrors.append("\n").append(mirror);
+ }
+ fields.emplace("Alternate-URIs", altMirrors);
+ SendMessage("103 Redirect", std::move(fields));
+
+ // Remove Itm from the queue, then delete
+ if (Queue == Itm)
+ Queue = Itm->Next;
+ else
+ {
+ FetchItem *previous = Queue;
+ while (previous->Next != Itm)
+ previous = previous->Next;
+
+ previous->Next = Itm->Next;
+ }
+ delete Itm;
+}
+ /*}}}*/
+void MirrorMethod::DealWithPendingItems(std::vector<std::string> const &baseuris, /*{{{*/
+ MirrorListInfo const &info, FetchItem *const Itm,
+ std::function<void()> handler)
+{
+ FetchItem **LastItm = &Itm->Next;
+ while (*LastItm != nullptr)
+ LastItm = &((*LastItm)->Next);
+ while (Queue != Itm)
+ {
+ if (APT::String::Startswith(Queue->Uri, info.baseuri) == false ||
+ std::any_of(baseuris.cbegin(), baseuris.cend(), [&](std::string const &b) { return APT::String::Startswith(Queue->Uri, b); }))
+ {
+ // move the item behind the aux file not related to it
+ *LastItm = Queue;
+ Queue = QueueBack = Queue->Next;
+ (*LastItm)->Next = nullptr;
+ LastItm = &((*LastItm)->Next);
+ }
+ else
+ {
+ handler();
+ }
+ }
+ // now remove out trigger
+ QueueBack = Queue = Queue->Next;
+ delete Itm;
+}
+ /*}}}*/
+bool MirrorMethod::MirrorListFileReceived(MirrorListInfo &info, FetchItem *const Itm) /*{{{*/
+{
+ std::vector<std::string> baseuris;
+ for (auto const &i : mirrorfilestate)
+ if (info.baseuri.length() < i.second.baseuri.length() &&
+ i.second.state == REQUESTED &&
+ APT::String::Startswith(i.second.baseuri, info.baseuri))
+ baseuris.push_back(i.second.baseuri);
+ sortByLength(baseuris);
+
+ FileFd mirrorlist;
+ if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension))
+ {
+ auto const accessColon = info.baseuri.find(':');
+ auto access = info.baseuri.substr(0, accessColon);
+ std::string prefixAccess;
+ if (APT::String::Startswith(access, "mirror") == false)
+ {
+ auto const plus = info.baseuri.find('+');
+ prefixAccess = info.baseuri.substr(0, plus);
+ access.erase(0, plus + 1);
+ }
+ std::vector<std::string> limitAccess;
+ // If the mirror file comes from an online source, allow only other online
+ // sources, not e.g. file:///. If the mirrorlist comes from there we can assume
+ // the admin knows what (s)he is doing through and not limit the options.
+ if (std::any_of(std::begin(disallowLocal), std::end(disallowLocal),
+ [&access](char const *const a) { return APT::String::Endswith(access, std::string("+") + a); }) ||
+ access == "mirror")
+ {
+ std::copy(std::begin(disallowLocal), std::end(disallowLocal), std::back_inserter(limitAccess));
+ }
+ std::string line;
+ while (mirrorlist.ReadLine(line))
+ {
+ if (line.empty() || line[0] == '#')
+ continue;
+ auto const access = line.substr(0, line.find(':'));
+ if (limitAccess.empty() == false && std::find(limitAccess.begin(), limitAccess.end(), access) == limitAccess.end())
+ continue;
+ auto const tab = line.find('\t');
+ if (tab == std::string::npos)
+ {
+ if (prefixAccess.empty())
+ info.list.emplace_back(std::move(line));
+ else
+ info.list.emplace_back(prefixAccess + '+' + line);
+ }
+ else
+ {
+ auto uri = line.substr(0, tab);
+ if (prefixAccess.empty() == false)
+ uri = prefixAccess + '+' + uri;
+ auto tagline = line.substr(tab + 1);
+ std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' ');
+ auto tags = VectorizeString(tagline, ' ');
+ tags.erase(std::remove_if(tags.begin(), tags.end(), [](std::string const &a) { return a.empty(); }), tags.end());
+ info.list.emplace_back(std::move(uri), std::move(tags));
+ }
+ }
+ mirrorlist.Close();
+
+ if (info.list.empty())
+ {
+ info.state = FAILED;
+ DealWithPendingItems(baseuris, info, Itm, [&]() {
+ std::string msg;
+ strprintf(msg, "Mirror list %s is empty for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
+ Fail(msg, false);
+ });
+ }
+ else
+ {
+ info.state = AVAILABLE;
+ DealWithPendingItems(baseuris, info, Itm, [&]() {
+ RedirectItem(info, Queue, msgCache[Queue->Uri]);
+ });
+ msgCache.clear();
+ }
+ }
+ else
+ {
+ info.state = FAILED;
+ DealWithPendingItems(baseuris, info, Itm, [&]() {
+ std::string msg;
+ strprintf(msg, "Downloading mirror file %s failed for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
+ Fail(msg, false);
+ });
+ }
+ return true;
+}
+ /*}}}*/
+std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem *const Itm) /*{{{*/
+{
+ if (APT::String::Startswith(Itm->Uri, Binary))
+ {
+ std::string const repouri = LookupTag(Message, "Target-Repo-Uri");
+ if (repouri.empty() == false && std::find(sourceslist.cbegin(), sourceslist.cend(), repouri) == sourceslist.cend())
+ sourceslist.push_back(repouri);
+ }
+ if (sourceslist.empty())
+ {
+ // read sources.list and find the matching base uri
+ pkgSourceList sl;
+ if (sl.ReadMainList() == false)
+ {
+ _error->Error(_("The list of sources could not be read."));
+ return "";
+ }
+ std::string const needle = Binary + ":";
+ for (auto const &SL : sl)
+ {
+ std::string uristr = SL->GetURI();
+ if (APT::String::Startswith(uristr, needle))
+ sourceslist.push_back(uristr);
+ }
+ sortByLength(sourceslist);
+ }
+ for (auto uristr : sourceslist)
+ {
+ if (APT::String::Startswith(Itm->Uri, uristr))
+ {
+ uristr.erase(uristr.length() - 1); // remove the ending '/'
+ auto const colon = uristr.find(':');
+ if (unlikely(colon == std::string::npos))
+ continue;
+ auto const plus = uristr.find("+");
+ if (plus < colon)
+ {
+ // started as tor+mirror+http we want to get the file via tor+http
+ auto const access = uristr.substr(0, colon);
+ if (APT::String::Startswith(access, "mirror") == false)
+ {
+ uristr.erase(plus, strlen("mirror") + 1);
+ return uristr;
+ }
+ else
+ return uristr.substr(plus + 1);
+ }
+ else
+ {
+ uristr.replace(0, strlen("mirror"), "http");
+ return uristr;
+ }
+ }
+ }
+ return "";
+}
+ /*}}}*/
+bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*/
+{
+ auto mirrorinfo = mirrorfilestate.find(Itm->Uri);
+ if (mirrorinfo != mirrorfilestate.end())
+ return MirrorListFileReceived(mirrorinfo->second, Itm);
+
+ std::string const mirrorfileuri = GetMirrorFileURI(Message, Itm);
+ if (mirrorfileuri.empty())
+ {
+ _error->Error("Couldn't determine mirror list to query for %s", Itm->Uri.c_str());
+ return false;
+ }
+ if (DebugEnabled())
+ std::clog << "Mirror-URI: " << mirrorfileuri << " for " << Itm->Uri << std::endl;
+
+ // have we requested this mirror file already?
+ auto const state = mirrorfilestate.find(mirrorfileuri);
+ if (state == mirrorfilestate.end())
+ {
+ msgCache[Itm->Uri] = Message;
+ MirrorListInfo info;
+ info.state = REQUESTED;
+ info.baseuri = mirrorfileuri + '/';
+ auto const colon = info.baseuri.find(':');
+ if (unlikely(colon == std::string::npos))
+ return false;
+ info.baseuri.replace(0, colon, Binary);
+ mirrorfilestate[mirrorfileuri] = info;
+ std::unordered_map<std::string, std::string> fields;
+ fields.emplace("URI", Itm->Uri);
+ fields.emplace("MaximumSize", std::to_string(1 * 1024 * 1024)); //FIXME: 1 MB is enough for everyone
+ fields.emplace("Aux-ShortDesc", "Mirrorlist");
+ fields.emplace("Aux-Description", mirrorfileuri + " Mirrorlist");
+ fields.emplace("Aux-Uri", mirrorfileuri);
+ SendMessage("351 Aux Request", std::move(fields));
+ return true;
+ }
+
+ switch (state->second.state)
+ {
+ case REQUESTED:
+ // lets wait for the requested mirror file
+ msgCache[Itm->Uri] = Message;
+ return true;
+ case FAILED:
+ Fail("Downloading mirror file failed", false);
+ return true;
+ case AVAILABLE:
+ RedirectItem(state->second, Itm, Message);
+ return true;
+ }
+ return false;
+}
+ /*}}}*/
+
+int main(int, const char *argv[])
+{
+ return MirrorMethod(flNotDir(argv[0])).Run();
+}