421 lines
14 KiB
C++
421 lines
14 KiB
C++
// -*- mode: cpp; mode: fold -*-
|
||
// Description /*{{{*/
|
||
/* ######################################################################
|
||
|
||
Mirror URI – This method helps avoiding hardcoding of mirrors in the
|
||
sources.lists by looking up a list of mirrors first to which the
|
||
following requests are redirected.
|
||
|
||
##################################################################### */
|
||
/*}}}*/
|
||
// Include Files /*{{{*/
|
||
#include <config.h>
|
||
|
||
#include "aptmethod.h"
|
||
#include <apt-pkg/configuration.h>
|
||
#include <apt-pkg/error.h>
|
||
#include <apt-pkg/fileutl.h>
|
||
#include <apt-pkg/metaindex.h>
|
||
#include <apt-pkg/sourcelist.h>
|
||
#include <apt-pkg/strutl.h>
|
||
|
||
#include <functional>
|
||
#include <random>
|
||
#include <string>
|
||
#include <unordered_map>
|
||
|
||
#include <sys/utsname.h>
|
||
|
||
#include <apti18n.h>
|
||
/*}}}*/
|
||
constexpr char const *const disallowLocal[] = {"ftp", "http", "https"};
|
||
|
||
static void sortByLength(std::vector<std::string> &vec) /*{{{*/
|
||
{
|
||
// this ensures having mirror://foo/ and mirror://foo/bar/ works as expected
|
||
// by checking for the longest matches first
|
||
std::sort(vec.begin(), vec.end(), [](std::string const &a, std::string const &b) {
|
||
return a.length() > b.length();
|
||
});
|
||
}
|
||
/*}}}*/
|
||
class MirrorMethod : public aptMethod /*{{{*/
|
||
{
|
||
std::mt19937 genrng;
|
||
std::vector<std::string> sourceslist;
|
||
std::unordered_map<std::string, std::string> msgCache;
|
||
enum MirrorFileState
|
||
{
|
||
REQUESTED,
|
||
FAILED,
|
||
AVAILABLE
|
||
};
|
||
struct MirrorInfo
|
||
{
|
||
std::string uri;
|
||
unsigned long priority = std::numeric_limits<decltype(priority)>::max();
|
||
decltype(genrng)::result_type seed = 0;
|
||
std::unordered_map<std::string, std::vector<std::string>> tags;
|
||
explicit MirrorInfo(std::string const &u, std::vector<std::string> &&ptags = {}) : uri(u)
|
||
{
|
||
for (auto &&tag : ptags)
|
||
{
|
||
auto const colonfound = tag.find(':');
|
||
if (unlikely(colonfound == std::string::npos))
|
||
continue;
|
||
auto name = tag.substr(0, colonfound);
|
||
auto value = tag.substr(colonfound + 1);
|
||
if (name == "arch")
|
||
tags["Architecture"].emplace_back(std::move(value));
|
||
else if (name == "lang")
|
||
tags["Language"].emplace_back(std::move(value));
|
||
else if (name == "priority")
|
||
priority = std::strtoul(value.c_str(), nullptr, 10);
|
||
else if (likely(name.empty() == false))
|
||
{
|
||
if (name == "codename" || name == "suite")
|
||
tags["Release"].push_back(value);
|
||
name[0] = std::toupper(name[0]);
|
||
tags[std::move(name)].emplace_back(std::move(value));
|
||
}
|
||
}
|
||
}
|
||
};
|
||
struct MirrorListInfo
|
||
{
|
||
MirrorFileState state;
|
||
std::string baseuri;
|
||
std::vector<MirrorInfo> list;
|
||
};
|
||
std::unordered_map<std::string, MirrorListInfo> mirrorfilestate;
|
||
|
||
bool URIAcquire(std::string const &Message, FetchItem *Itm) override;
|
||
|
||
void RedirectItem(MirrorListInfo const &info, FetchItem *Itm, std::string const &Message);
|
||
bool MirrorListFileReceived(MirrorListInfo &info, FetchItem *Itm);
|
||
std::string GetMirrorFileURI(std::string const &Message, FetchItem *Itm);
|
||
void DealWithPendingItems(std::vector<std::string> const &baseuris, MirrorListInfo const &info, FetchItem *Itm, std::function<void()> handler);
|
||
|
||
public:
|
||
explicit MirrorMethod(std::string pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests | SendURIEncoded), genrng(clock())
|
||
{
|
||
SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY;
|
||
}
|
||
};
|
||
/*}}}*/
|
||
void MirrorMethod::RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message) /*{{{*/
|
||
{
|
||
std::unordered_map<std::string, std::string> matchers;
|
||
matchers.emplace("Architecture", LookupTag(Message, "Target-Architecture"));
|
||
matchers.emplace("Codename", LookupTag(Message, "Target-Codename"));
|
||
matchers.emplace("Component", LookupTag(Message, "Target-Component"));
|
||
matchers.emplace("Language", LookupTag(Message, "Target-Language"));
|
||
matchers.emplace("Release", LookupTag(Message, "Target-Release"));
|
||
matchers.emplace("Suite", LookupTag(Message, "Target-Suite"));
|
||
matchers.emplace("Type", LookupTag(Message, "Target-Type"));
|
||
decltype(info.list) possMirrors;
|
||
for (auto const &mirror : info.list)
|
||
{
|
||
bool failedMatch = false;
|
||
for (auto const &m : matchers)
|
||
{
|
||
if (m.second.empty())
|
||
continue;
|
||
auto const tagsetiter = mirror.tags.find(m.first);
|
||
if (tagsetiter == mirror.tags.end())
|
||
continue;
|
||
auto const tagset = tagsetiter->second;
|
||
if (tagset.empty() == false && std::find(tagset.begin(), tagset.end(), m.second) == tagset.end())
|
||
{
|
||
failedMatch = true;
|
||
break;
|
||
}
|
||
}
|
||
if (failedMatch)
|
||
continue;
|
||
possMirrors.push_back(mirror);
|
||
}
|
||
for (auto &&mirror : possMirrors)
|
||
mirror.seed = genrng();
|
||
std::sort(possMirrors.begin(), possMirrors.end(), [](MirrorInfo const &a, MirrorInfo const &b) {
|
||
if (a.priority != b.priority)
|
||
return a.priority < b.priority;
|
||
return a.seed < b.seed;
|
||
});
|
||
std::string const path = Itm->Uri.substr(info.baseuri.length());
|
||
std::string altMirrors;
|
||
std::unordered_map<std::string, std::string> fields;
|
||
fields.emplace("URI", Itm->Uri);
|
||
for (auto curMirror = possMirrors.cbegin(); curMirror != possMirrors.cend(); ++curMirror)
|
||
{
|
||
std::string mirror = curMirror->uri;
|
||
if (APT::String::Endswith(mirror, "/") == false)
|
||
mirror.append("/");
|
||
mirror.append(path);
|
||
if (curMirror == possMirrors.cbegin())
|
||
fields.emplace("New-URI", mirror);
|
||
else if (altMirrors.empty())
|
||
altMirrors.append(mirror);
|
||
else
|
||
altMirrors.append("\n").append(mirror);
|
||
}
|
||
fields.emplace("Alternate-URIs", altMirrors);
|
||
SendMessage("103 Redirect", std::move(fields));
|
||
|
||
// Remove Itm from the queue, then delete
|
||
if (Queue == Itm)
|
||
Queue = Itm->Next;
|
||
else
|
||
{
|
||
FetchItem *previous = Queue;
|
||
while (previous->Next != Itm)
|
||
previous = previous->Next;
|
||
|
||
previous->Next = Itm->Next;
|
||
}
|
||
delete Itm;
|
||
}
|
||
/*}}}*/
|
||
void MirrorMethod::DealWithPendingItems(std::vector<std::string> const &baseuris, /*{{{*/
|
||
MirrorListInfo const &info, FetchItem *const Itm,
|
||
std::function<void()> handler)
|
||
{
|
||
FetchItem **LastItm = &Itm->Next;
|
||
while (*LastItm != nullptr)
|
||
LastItm = &((*LastItm)->Next);
|
||
while (Queue != Itm)
|
||
{
|
||
if (APT::String::Startswith(Queue->Uri, info.baseuri) == false ||
|
||
std::any_of(baseuris.cbegin(), baseuris.cend(), [&](std::string const &b) { return APT::String::Startswith(Queue->Uri, b); }))
|
||
{
|
||
// move the item behind the aux file not related to it
|
||
*LastItm = Queue;
|
||
Queue = QueueBack = Queue->Next;
|
||
(*LastItm)->Next = nullptr;
|
||
LastItm = &((*LastItm)->Next);
|
||
}
|
||
else
|
||
{
|
||
handler();
|
||
}
|
||
}
|
||
// now remove out trigger
|
||
QueueBack = Queue = Queue->Next;
|
||
delete Itm;
|
||
}
|
||
/*}}}*/
|
||
bool MirrorMethod::MirrorListFileReceived(MirrorListInfo &info, FetchItem *const Itm) /*{{{*/
|
||
{
|
||
std::vector<std::string> baseuris;
|
||
for (auto const &i : mirrorfilestate)
|
||
if (info.baseuri.length() < i.second.baseuri.length() &&
|
||
i.second.state == REQUESTED &&
|
||
APT::String::Startswith(i.second.baseuri, info.baseuri))
|
||
baseuris.push_back(i.second.baseuri);
|
||
sortByLength(baseuris);
|
||
|
||
FileFd mirrorlist;
|
||
if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension))
|
||
{
|
||
auto const accessColon = info.baseuri.find(':');
|
||
auto access = info.baseuri.substr(0, accessColon);
|
||
std::string prefixAccess;
|
||
if (APT::String::Startswith(access, "mirror") == false)
|
||
{
|
||
auto const plus = info.baseuri.find('+');
|
||
prefixAccess = info.baseuri.substr(0, plus);
|
||
access.erase(0, plus + 1);
|
||
}
|
||
std::vector<std::string> limitAccess;
|
||
// If the mirror file comes from an online source, allow only other online
|
||
// sources, not e.g. file:///. If the mirrorlist comes from there we can assume
|
||
// the admin knows what (s)he is doing through and not limit the options.
|
||
if (std::any_of(std::begin(disallowLocal), std::end(disallowLocal),
|
||
[&access](char const *const a) { return APT::String::Endswith(access, std::string("+") + a); }) ||
|
||
access == "mirror")
|
||
{
|
||
std::copy(std::begin(disallowLocal), std::end(disallowLocal), std::back_inserter(limitAccess));
|
||
}
|
||
std::string line;
|
||
while (mirrorlist.ReadLine(line))
|
||
{
|
||
if (line.empty() || line[0] == '#')
|
||
continue;
|
||
auto const access = line.substr(0, line.find(':'));
|
||
if (limitAccess.empty() == false && std::find(limitAccess.begin(), limitAccess.end(), access) == limitAccess.end())
|
||
continue;
|
||
auto const tab = line.find('\t');
|
||
if (tab == std::string::npos)
|
||
{
|
||
if (prefixAccess.empty())
|
||
info.list.emplace_back(std::move(line));
|
||
else
|
||
info.list.emplace_back(prefixAccess + '+' + line);
|
||
}
|
||
else
|
||
{
|
||
auto uri = line.substr(0, tab);
|
||
if (prefixAccess.empty() == false)
|
||
uri = prefixAccess + '+' + uri;
|
||
auto tagline = line.substr(tab + 1);
|
||
std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' ');
|
||
auto tags = VectorizeString(tagline, ' ');
|
||
tags.erase(std::remove_if(tags.begin(), tags.end(), [](std::string const &a) { return a.empty(); }), tags.end());
|
||
info.list.emplace_back(std::move(uri), std::move(tags));
|
||
}
|
||
}
|
||
mirrorlist.Close();
|
||
|
||
if (info.list.empty())
|
||
{
|
||
info.state = FAILED;
|
||
DealWithPendingItems(baseuris, info, Itm, [&]() {
|
||
std::string msg;
|
||
strprintf(msg, "Mirror list %s is empty for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
|
||
Fail(msg, false);
|
||
});
|
||
}
|
||
else
|
||
{
|
||
info.state = AVAILABLE;
|
||
DealWithPendingItems(baseuris, info, Itm, [&]() {
|
||
RedirectItem(info, Queue, msgCache[Queue->Uri]);
|
||
});
|
||
msgCache.clear();
|
||
}
|
||
}
|
||
else
|
||
{
|
||
info.state = FAILED;
|
||
DealWithPendingItems(baseuris, info, Itm, [&]() {
|
||
std::string msg;
|
||
strprintf(msg, "Downloading mirror file %s failed for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
|
||
Fail(msg, false);
|
||
});
|
||
}
|
||
return true;
|
||
}
|
||
/*}}}*/
|
||
std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem *const Itm) /*{{{*/
|
||
{
|
||
if (APT::String::Startswith(Itm->Uri, Binary))
|
||
{
|
||
std::string const repouri = LookupTag(Message, "Target-Repo-Uri");
|
||
if (repouri.empty() == false && std::find(sourceslist.cbegin(), sourceslist.cend(), repouri) == sourceslist.cend())
|
||
sourceslist.push_back(repouri);
|
||
}
|
||
if (sourceslist.empty())
|
||
{
|
||
// read sources.list and find the matching base uri
|
||
pkgSourceList sl;
|
||
if (sl.ReadMainList() == false)
|
||
{
|
||
_error->Error(_("The list of sources could not be read."));
|
||
return "";
|
||
}
|
||
std::string const needle = Binary + ":";
|
||
for (auto const &SL : sl)
|
||
{
|
||
std::string uristr = SL->GetURI();
|
||
if (APT::String::Startswith(uristr, needle))
|
||
sourceslist.push_back(uristr);
|
||
}
|
||
sortByLength(sourceslist);
|
||
}
|
||
for (auto uristr : sourceslist)
|
||
{
|
||
if (APT::String::Startswith(Itm->Uri, uristr))
|
||
{
|
||
if (::URI uri{uristr}; uri.Path.length() > 1 && APT::String::Endswith(uri.Path, "/"))
|
||
{
|
||
uri.Path.erase(uri.Path.length() - 1); // remove the ending '/'
|
||
uristr = uri;
|
||
}
|
||
auto const colon = uristr.find(':');
|
||
if (unlikely(colon == std::string::npos))
|
||
continue;
|
||
auto const plus = uristr.find("+");
|
||
if (plus < colon)
|
||
{
|
||
// started as tor+mirror+http we want to get the file via tor+http
|
||
auto const access = uristr.substr(0, colon);
|
||
if (APT::String::Startswith(access, "mirror") == false)
|
||
{
|
||
uristr.erase(plus, strlen("mirror") + 1);
|
||
return uristr;
|
||
}
|
||
else
|
||
return uristr.substr(plus + 1);
|
||
}
|
||
else
|
||
{
|
||
uristr.replace(0, strlen("mirror"), "http");
|
||
return uristr;
|
||
}
|
||
}
|
||
}
|
||
return "";
|
||
}
|
||
/*}}}*/
|
||
bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*/
|
||
{
|
||
auto mirrorinfo = mirrorfilestate.find(Itm->Uri);
|
||
if (mirrorinfo != mirrorfilestate.end())
|
||
return MirrorListFileReceived(mirrorinfo->second, Itm);
|
||
|
||
std::string const mirrorfileuri = GetMirrorFileURI(Message, Itm);
|
||
if (mirrorfileuri.empty())
|
||
{
|
||
_error->Error("Couldn't determine mirror list to query for %s", Itm->Uri.c_str());
|
||
return false;
|
||
}
|
||
if (DebugEnabled())
|
||
std::clog << "Mirror-URI: " << mirrorfileuri << " for " << Itm->Uri << std::endl;
|
||
|
||
// have we requested this mirror file already?
|
||
auto const state = mirrorfilestate.find(mirrorfileuri);
|
||
if (state == mirrorfilestate.end())
|
||
{
|
||
msgCache[Itm->Uri] = Message;
|
||
MirrorListInfo info;
|
||
info.state = REQUESTED;
|
||
if (not APT::String::Endswith(mirrorfileuri, "/"))
|
||
info.baseuri = mirrorfileuri + '/';
|
||
else
|
||
info.baseuri = mirrorfileuri;
|
||
auto const colon = info.baseuri.find(':');
|
||
if (unlikely(colon == std::string::npos))
|
||
return false;
|
||
info.baseuri.replace(0, colon, Binary);
|
||
mirrorfilestate[mirrorfileuri] = info;
|
||
std::unordered_map<std::string, std::string> fields;
|
||
fields.emplace("URI", Itm->Uri);
|
||
fields.emplace("MaximumSize", std::to_string(1 * 1024 * 1024)); //FIXME: 1 MB is enough for everyone
|
||
fields.emplace("Aux-ShortDesc", "Mirrorlist");
|
||
fields.emplace("Aux-Description", mirrorfileuri + " Mirrorlist");
|
||
fields.emplace("Aux-Uri", mirrorfileuri);
|
||
SendMessage("351 Aux Request", std::move(fields));
|
||
return true;
|
||
}
|
||
|
||
switch (state->second.state)
|
||
{
|
||
case REQUESTED:
|
||
// lets wait for the requested mirror file
|
||
msgCache[Itm->Uri] = Message;
|
||
return true;
|
||
case FAILED:
|
||
Fail("Downloading mirror file failed", false);
|
||
return true;
|
||
case AVAILABLE:
|
||
RedirectItem(state->second, Itm, Message);
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
/*}}}*/
|
||
|
||
int main(int, const char *argv[])
|
||
{
|
||
return MirrorMethod(std::string{flNotDir(argv[0])}).Run();
|
||
}
|