Adding upstream version 2.7.12.upstream/2.7.12

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 414 insertions, 0 deletions

diff --git a/methods/mirror.cc b/methods/mirror.cc
new file mode 100644
index 0000000..787e4c7
--- /dev/null
+++ b/methods/mirror.cc
@@ -0,0 +1,414 @@
+// -*- mode: cpp; mode: fold -*-
+// Description								/*{{{*/
+/* ######################################################################
+
+   Mirror URI – This method helps avoiding hardcoding of mirrors in the
+   sources.lists by looking up a list of mirrors first to which the
+   following requests are redirected.
+
+   ##################################################################### */
+									/*}}}*/
+// Include Files							/*{{{*/
+#include <config.h>
+
+#include "aptmethod.h"
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/metaindex.h>
+#include <apt-pkg/sourcelist.h>
+#include <apt-pkg/strutl.h>
+
+#include <functional>
+#include <random>
+#include <string>
+#include <unordered_map>
+
+#include <sys/utsname.h>
+
+#include <apti18n.h>
+									/*}}}*/
+constexpr char const *const disallowLocal[] = {"ftp", "http", "https"};
+
+static void sortByLength(std::vector<std::string> &vec)			/*{{{*/
+{
+   // this ensures having mirror://foo/ and mirror://foo/bar/ works as expected
+   // by checking for the longest matches first
+   std::sort(vec.begin(), vec.end(), [](std::string const &a, std::string const &b) {
+      return a.length() > b.length();
+   });
+}
+									/*}}}*/
+class MirrorMethod : public aptMethod /*{{{*/
+{
+   std::mt19937 genrng;
+   std::vector<std::string> sourceslist;
+   std::unordered_map<std::string, std::string> msgCache;
+   enum MirrorFileState
+   {
+      REQUESTED,
+      FAILED,
+      AVAILABLE
+   };
+   struct MirrorInfo
+   {
+      std::string uri;
+      unsigned long priority = std::numeric_limits<decltype(priority)>::max();
+      decltype(genrng)::result_type seed = 0;
+      std::unordered_map<std::string, std::vector<std::string>> tags;
+      explicit MirrorInfo(std::string const &u, std::vector<std::string> &&ptags = {}) : uri(u)
+      {
+	 for (auto &&tag : ptags)
+	 {
+	    auto const colonfound = tag.find(':');
+	    if (unlikely(colonfound == std::string::npos))
+	       continue;
+	    auto name = tag.substr(0, colonfound);
+	    auto value = tag.substr(colonfound + 1);
+	    if (name == "arch")
+	       tags["Architecture"].emplace_back(std::move(value));
+	    else if (name == "lang")
+	       tags["Language"].emplace_back(std::move(value));
+	    else if (name == "priority")
+	       priority = std::strtoul(value.c_str(), nullptr, 10);
+	    else if (likely(name.empty() == false))
+	    {
+	       if (name == "codename" || name == "suite")
+		  tags["Release"].push_back(value);
+	       name[0] = std::toupper(name[0]);
+	       tags[std::move(name)].emplace_back(std::move(value));
+	    }
+	 }
+      }
+   };
+   struct MirrorListInfo
+   {
+      MirrorFileState state;
+      std::string baseuri;
+      std::vector<MirrorInfo> list;
+   };
+   std::unordered_map<std::string, MirrorListInfo> mirrorfilestate;
+
+   virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE;
+
+   void RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message);
+   bool MirrorListFileReceived(MirrorListInfo &info, FetchItem *const Itm);
+   std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm);
+   void DealWithPendingItems(std::vector<std::string> const &baseuris, MirrorListInfo const &info, FetchItem *const Itm, std::function<void()> handler);
+
+   public:
+   explicit MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests | SendURIEncoded), genrng(clock())
+   {
+      SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY;
+   }
+};
+									/*}}}*/
+void MirrorMethod::RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message) /*{{{*/
+{
+   std::unordered_map<std::string, std::string> matchers;
+   matchers.emplace("Architecture", LookupTag(Message, "Target-Architecture"));
+   matchers.emplace("Codename", LookupTag(Message, "Target-Codename"));
+   matchers.emplace("Component", LookupTag(Message, "Target-Component"));
+   matchers.emplace("Language", LookupTag(Message, "Target-Language"));
+   matchers.emplace("Release", LookupTag(Message, "Target-Release"));
+   matchers.emplace("Suite", LookupTag(Message, "Target-Suite"));
+   matchers.emplace("Type", LookupTag(Message, "Target-Type"));
+   decltype(info.list) possMirrors;
+   for (auto const &mirror : info.list)
+   {
+      bool failedMatch = false;
+      for (auto const &m : matchers)
+      {
+	 if (m.second.empty())
+	    continue;
+	 auto const tagsetiter = mirror.tags.find(m.first);
+	 if (tagsetiter == mirror.tags.end())
+	    continue;
+	 auto const tagset = tagsetiter->second;
+	 if (tagset.empty() == false && std::find(tagset.begin(), tagset.end(), m.second) == tagset.end())
+	 {
+	    failedMatch = true;
+	    break;
+	 }
+      }
+      if (failedMatch)
+	 continue;
+      possMirrors.push_back(mirror);
+   }
+   for (auto &&mirror : possMirrors)
+      mirror.seed = genrng();
+   std::sort(possMirrors.begin(), possMirrors.end(), [](MirrorInfo const &a, MirrorInfo const &b) {
+      if (a.priority != b.priority)
+	 return a.priority < b.priority;
+      return a.seed < b.seed;
+   });
+   std::string const path = Itm->Uri.substr(info.baseuri.length());
+   std::string altMirrors;
+   std::unordered_map<std::string, std::string> fields;
+   fields.emplace("URI", Itm->Uri);
+   for (auto curMirror = possMirrors.cbegin(); curMirror != possMirrors.cend(); ++curMirror)
+   {
+      std::string mirror = curMirror->uri;
+      if (APT::String::Endswith(mirror, "/") == false)
+	 mirror.append("/");
+      mirror.append(path);
+      if (curMirror == possMirrors.cbegin())
+	 fields.emplace("New-URI", mirror);
+      else if (altMirrors.empty())
+	 altMirrors.append(mirror);
+      else
+	 altMirrors.append("\n").append(mirror);
+   }
+   fields.emplace("Alternate-URIs", altMirrors);
+   SendMessage("103 Redirect", std::move(fields));
+
+   // Remove Itm from the queue, then delete
+   if (Queue == Itm)
+      Queue = Itm->Next;
+   else
+   {
+      FetchItem *previous = Queue;
+      while (previous->Next != Itm)
+	 previous = previous->Next;
+
+      previous->Next = Itm->Next;
+   }
+   delete Itm;
+}
+									/*}}}*/
+void MirrorMethod::DealWithPendingItems(std::vector<std::string> const &baseuris, /*{{{*/
+					MirrorListInfo const &info, FetchItem *const Itm,
+					std::function<void()> handler)
+{
+   FetchItem **LastItm = &Itm->Next;
+   while (*LastItm != nullptr)
+      LastItm = &((*LastItm)->Next);
+   while (Queue != Itm)
+   {
+      if (APT::String::Startswith(Queue->Uri, info.baseuri) == false ||
+	  std::any_of(baseuris.cbegin(), baseuris.cend(), [&](std::string const &b) { return APT::String::Startswith(Queue->Uri, b); }))
+      {
+	 // move the item behind the aux file not related to it
+	 *LastItm = Queue;
+	 Queue = QueueBack = Queue->Next;
+	 (*LastItm)->Next = nullptr;
+	 LastItm = &((*LastItm)->Next);
+      }
+      else
+      {
+	 handler();
+      }
+   }
+   // now remove out trigger
+   QueueBack = Queue = Queue->Next;
+   delete Itm;
+}
+									/*}}}*/
+bool MirrorMethod::MirrorListFileReceived(MirrorListInfo &info, FetchItem *const Itm) /*{{{*/
+{
+   std::vector<std::string> baseuris;
+   for (auto const &i : mirrorfilestate)
+      if (info.baseuri.length() < i.second.baseuri.length() &&
+	  i.second.state == REQUESTED &&
+	  APT::String::Startswith(i.second.baseuri, info.baseuri))
+	 baseuris.push_back(i.second.baseuri);
+   sortByLength(baseuris);
+
+   FileFd mirrorlist;
+   if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension))
+   {
+      auto const accessColon = info.baseuri.find(':');
+      auto access = info.baseuri.substr(0, accessColon);
+      std::string prefixAccess;
+      if (APT::String::Startswith(access, "mirror") == false)
+      {
+	 auto const plus = info.baseuri.find('+');
+	 prefixAccess = info.baseuri.substr(0, plus);
+	 access.erase(0, plus + 1);
+      }
+      std::vector<std::string> limitAccess;
+      // If the mirror file comes from an online source, allow only other online
+      // sources, not e.g. file:///. If the mirrorlist comes from there we can assume
+      // the admin knows what (s)he is doing through and not limit the options.
+      if (std::any_of(std::begin(disallowLocal), std::end(disallowLocal),
+		      [&access](char const *const a) { return APT::String::Endswith(access, std::string("+") + a); }) ||
+	  access == "mirror")
+      {
+	 std::copy(std::begin(disallowLocal), std::end(disallowLocal), std::back_inserter(limitAccess));
+      }
+      std::string line;
+      while (mirrorlist.ReadLine(line))
+      {
+	 if (line.empty() || line[0] == '#')
+	    continue;
+	 auto const access = line.substr(0, line.find(':'));
+	 if (limitAccess.empty() == false && std::find(limitAccess.begin(), limitAccess.end(), access) == limitAccess.end())
+	    continue;
+	 auto const tab = line.find('\t');
+	 if (tab == std::string::npos)
+	 {
+	    if (prefixAccess.empty())
+	       info.list.emplace_back(std::move(line));
+	    else
+	       info.list.emplace_back(prefixAccess + '+' + line);
+	 }
+	 else
+	 {
+	    auto uri = line.substr(0, tab);
+	    if (prefixAccess.empty() == false)
+	       uri = prefixAccess + '+' + uri;
+	    auto tagline = line.substr(tab + 1);
+	    std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' ');
+	    auto tags = VectorizeString(tagline, ' ');
+	    tags.erase(std::remove_if(tags.begin(), tags.end(), [](std::string const &a) { return a.empty(); }), tags.end());
+	    info.list.emplace_back(std::move(uri), std::move(tags));
+	 }
+      }
+      mirrorlist.Close();
+
+      if (info.list.empty())
+      {
+	 info.state = FAILED;
+	 DealWithPendingItems(baseuris, info, Itm, [&]() {
+	    std::string msg;
+	    strprintf(msg, "Mirror list %s is empty for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
+	    Fail(msg, false);
+	 });
+      }
+      else
+      {
+	 info.state = AVAILABLE;
+	 DealWithPendingItems(baseuris, info, Itm, [&]() {
+	    RedirectItem(info, Queue, msgCache[Queue->Uri]);
+	 });
+	 msgCache.clear();
+      }
+   }
+   else
+   {
+      info.state = FAILED;
+      DealWithPendingItems(baseuris, info, Itm, [&]() {
+	 std::string msg;
+	 strprintf(msg, "Downloading mirror file %s failed for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
+	 Fail(msg, false);
+      });
+   }
+   return true;
+}
+									/*}}}*/
+std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem *const Itm) /*{{{*/
+{
+   if (APT::String::Startswith(Itm->Uri, Binary))
+   {
+      std::string const repouri = LookupTag(Message, "Target-Repo-Uri");
+      if (repouri.empty() == false && std::find(sourceslist.cbegin(), sourceslist.cend(), repouri) == sourceslist.cend())
+	 sourceslist.push_back(repouri);
+   }
+   if (sourceslist.empty())
+   {
+      // read sources.list and find the matching base uri
+      pkgSourceList sl;
+      if (sl.ReadMainList() == false)
+      {
+	 _error->Error(_("The list of sources could not be read."));
+	 return "";
+      }
+      std::string const needle = Binary + ":";
+      for (auto const &SL : sl)
+      {
+	 std::string uristr = SL->GetURI();
+	 if (APT::String::Startswith(uristr, needle))
+	    sourceslist.push_back(uristr);
+      }
+      sortByLength(sourceslist);
+   }
+   for (auto uristr : sourceslist)
+   {
+      if (APT::String::Startswith(Itm->Uri, uristr))
+      {
+	 uristr.erase(uristr.length() - 1); // remove the ending '/'
+	 auto const colon = uristr.find(':');
+	 if (unlikely(colon == std::string::npos))
+	    continue;
+	 auto const plus = uristr.find("+");
+	 if (plus < colon)
+	 {
+	    // started as tor+mirror+http we want to get the file via tor+http
+	    auto const access = uristr.substr(0, colon);
+	    if (APT::String::Startswith(access, "mirror") == false)
+	    {
+	       uristr.erase(plus, strlen("mirror") + 1);
+	       return uristr;
+	    }
+	    else
+	       return uristr.substr(plus + 1);
+	 }
+	 else
+	 {
+	    uristr.replace(0, strlen("mirror"), "http");
+	    return uristr;
+	 }
+      }
+   }
+   return "";
+}
+									/*}}}*/
+bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*/
+{
+   auto mirrorinfo = mirrorfilestate.find(Itm->Uri);
+   if (mirrorinfo != mirrorfilestate.end())
+      return MirrorListFileReceived(mirrorinfo->second, Itm);
+
+   std::string const mirrorfileuri = GetMirrorFileURI(Message, Itm);
+   if (mirrorfileuri.empty())
+   {
+      _error->Error("Couldn't determine mirror list to query for %s", Itm->Uri.c_str());
+      return false;
+   }
+   if (DebugEnabled())
+      std::clog << "Mirror-URI: " << mirrorfileuri << " for " << Itm->Uri << std::endl;
+
+   // have we requested this mirror file already?
+   auto const state = mirrorfilestate.find(mirrorfileuri);
+   if (state == mirrorfilestate.end())
+   {
+      msgCache[Itm->Uri] = Message;
+      MirrorListInfo info;
+      info.state = REQUESTED;
+      info.baseuri = mirrorfileuri + '/';
+      auto const colon = info.baseuri.find(':');
+      if (unlikely(colon == std::string::npos))
+	 return false;
+      info.baseuri.replace(0, colon, Binary);
+      mirrorfilestate[mirrorfileuri] = info;
+      std::unordered_map<std::string, std::string> fields;
+      fields.emplace("URI", Itm->Uri);
+      fields.emplace("MaximumSize", std::to_string(1 * 1024 * 1024)); //FIXME: 1 MB is enough for everyone
+      fields.emplace("Aux-ShortDesc", "Mirrorlist");
+      fields.emplace("Aux-Description", mirrorfileuri + " Mirrorlist");
+      fields.emplace("Aux-Uri", mirrorfileuri);
+      SendMessage("351 Aux Request", std::move(fields));
+      return true;
+   }
+
+   switch (state->second.state)
+   {
+   case REQUESTED:
+      // lets wait for the requested mirror file
+      msgCache[Itm->Uri] = Message;
+      return true;
+   case FAILED:
+      Fail("Downloading mirror file failed", false);
+      return true;
+   case AVAILABLE:
+      RedirectItem(state->second, Itm, Message);
+      return true;
+   }
+   return false;
+}
+									/*}}}*/
+
+int main(int, const char *argv[])
+{
+   return MirrorMethod(flNotDir(argv[0])).Run();
+}