summaryrefslogtreecommitdiffstats
path: root/ftparchive/writer.cc
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--ftparchive/writer.cc1181
1 files changed, 1181 insertions, 0 deletions
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc
new file mode 100644
index 0000000..ae427b1
--- /dev/null
+++ b/ftparchive/writer.cc
@@ -0,0 +1,1181 @@
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+/* ######################################################################
+
+ Writer
+
+ The file writer classes. These write various types of output, sources,
+ packages and contents.
+
+ ##################################################################### */
+ /*}}}*/
+// Include Files /*{{{*/
+#include <config.h>
+
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/debfile.h>
+#include <apt-pkg/deblistparser.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/gpgv.h>
+#include <apt-pkg/hashes.h>
+#include <apt-pkg/pkgcache.h>
+#include <apt-pkg/strutl.h>
+#include <apt-pkg/tagfile-keys.h>
+#include <apt-pkg/tagfile.h>
+
+#include <algorithm>
+#include <ctime>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <sstream>
+#include <utility>
+#include <ctype.h>
+#include <fnmatch.h>
+#include <ftw.h>
+#include <locale.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "apt-ftparchive.h"
+#include "byhash.h"
+#include "cachedb.h"
+#include "multicompress.h"
+#include "writer.h"
+
+#include <apti18n.h>
+ /*}}}*/
+using namespace std;
+FTWScanner *FTWScanner::Owner;
+
+// ConfigToDoHashes - which hashes to generate /*{{{*/
+static void SingleConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf, unsigned int const Flag)
+{
+ if (_config->FindB(Conf, (DoHashes & Flag) == Flag) == true)
+ DoHashes |= Flag;
+ else
+ DoHashes &= ~Flag;
+}
+static void ConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf)
+{
+ SingleConfigToDoHashes(DoHashes, Conf + "::MD5", Hashes::MD5SUM);
+ SingleConfigToDoHashes(DoHashes, Conf + "::SHA1", Hashes::SHA1SUM);
+ SingleConfigToDoHashes(DoHashes, Conf + "::SHA256", Hashes::SHA256SUM);
+ SingleConfigToDoHashes(DoHashes, Conf + "::SHA512", Hashes::SHA512SUM);
+}
+ /*}}}*/
+
+// FTWScanner::FTWScanner - Constructor /*{{{*/
+FTWScanner::FTWScanner(FileFd * const GivenOutput, string const &Arch, bool const IncludeArchAll)
+ : Arch(Arch), IncludeArchAll(IncludeArchAll), DoHashes(~0)
+{
+ if (GivenOutput == NULL)
+ {
+ Output = new FileFd;
+ OwnsOutput = true;
+ Output->OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
+ }
+ else
+ {
+ Output = GivenOutput;
+ OwnsOutput = false;
+ }
+ ErrorPrinted = false;
+ NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
+ ConfigToDoHashes(DoHashes, "APT::FTPArchive");
+}
+ /*}}}*/
+FTWScanner::~FTWScanner()
+{
+ if (Output != NULL && OwnsOutput)
+ delete Output;
+}
+// FTWScanner::Scanner - FTW Scanner /*{{{*/
+// ---------------------------------------------------------------------
+/* This is the FTW scanner, it processes each directory element in the
+ directory tree. */
+int FTWScanner::ScannerFTW(const char *File,const struct stat * /*sb*/,int Flag)
+{
+ if (Flag == FTW_DNR)
+ {
+ Owner->NewLine(1);
+ ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
+ }
+ if (Flag == FTW_NS)
+ {
+ Owner->NewLine(1);
+ ioprintf(c1out, _("W: Unable to stat %s\n"), File);
+ }
+ if (Flag != FTW_F)
+ return 0;
+
+ return ScannerFile(File, true);
+}
+ /*}}}*/
+static bool FileMatchesPatterns(char const *const File, std::vector<std::string> const &Patterns) /*{{{*/
+{
+ const char *LastComponent = strrchr(File, '/');
+ if (LastComponent == nullptr)
+ LastComponent = File;
+ else
+ ++LastComponent;
+
+ return std::any_of(Patterns.cbegin(), Patterns.cend(), [&](std::string const &pattern) {
+ return fnmatch(pattern.c_str(), LastComponent, 0) == 0;
+ });
+}
+ /*}}}*/
+int FTWScanner::ScannerFile(const char *const File, bool const ReadLink) /*{{{*/
+{
+ if (FileMatchesPatterns(File, Owner->Patterns) == false)
+ return 0;
+
+ Owner->FilesToProcess.emplace_back(File, ReadLink);
+ return 0;
+}
+ /*}}}*/
+int FTWScanner::ProcessFile(const char *const File, bool const ReadLink) /*{{{*/
+{
+ /* Process it. If the file is a link then resolve it into an absolute
+ name.. This works best if the directory components the scanner are
+ given are not links themselves. */
+ char Jnk[2];
+ char *RealPath = NULL;
+ Owner->OriginalPath = File;
+ if (ReadLink &&
+ readlink(File,Jnk,sizeof(Jnk)) != -1 &&
+ (RealPath = realpath(File,NULL)) != 0)
+ {
+ Owner->DoPackage(RealPath);
+ free(RealPath);
+ }
+ else
+ Owner->DoPackage(File);
+
+ if (_error->empty() == false)
+ {
+ // Print any errors or warnings found
+ string Err;
+ bool SeenPath = false;
+ while (_error->empty() == false)
+ {
+ Owner->NewLine(1);
+
+ bool const Type = _error->PopMessage(Err);
+ if (Type == true)
+ cerr << _("E: ") << Err << endl;
+ else
+ cerr << _("W: ") << Err << endl;
+
+ if (Err.find(File) != string::npos)
+ SeenPath = true;
+ }
+
+ if (SeenPath == false)
+ cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
+ return 0;
+ }
+
+ return 0;
+}
+ /*}}}*/
+// FTWScanner::RecursiveScan - Just scan a directory tree /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool FTWScanner::RecursiveScan(string const &Dir)
+{
+ /* If noprefix is set then jam the scan root in, so we don't generate
+ link followed paths out of control */
+ if (InternalPrefix.empty() == true)
+ {
+ char *RealPath = nullptr;
+ if ((RealPath = realpath(Dir.c_str(), nullptr)) == 0)
+ return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
+ InternalPrefix = RealPath;
+ free(RealPath);
+ }
+
+ // Do recursive directory searching
+ Owner = this;
+ int const Res = ftw(Dir.c_str(),ScannerFTW,30);
+
+ // Error treewalking?
+ if (Res != 0)
+ {
+ if (_error->PendingError() == false)
+ _error->Errno("ftw",_("Tree walking failed"));
+ return false;
+ }
+
+ using PairType = decltype(*FilesToProcess.cbegin());
+ std::sort(FilesToProcess.begin(), FilesToProcess.end(), [](PairType a, PairType b) {
+ return a.first < b.first;
+ });
+ if (not std::all_of(FilesToProcess.cbegin(), FilesToProcess.cend(), [](auto &&it) { return ProcessFile(it.first.c_str(), it.second) == 0; }))
+ return false;
+ FilesToProcess.clear();
+ return true;
+}
+ /*}}}*/
+// FTWScanner::LoadFileList - Load the file list from a file /*{{{*/
+// ---------------------------------------------------------------------
+/* This is an alternative to using FTW to locate files, it reads the list
+ of files from another file. */
+bool FTWScanner::LoadFileList(string const &Dir, string const &File)
+{
+ /* If noprefix is set then jam the scan root in, so we don't generate
+ link followed paths out of control */
+ if (InternalPrefix.empty() == true)
+ {
+ char *RealPath = nullptr;
+ if ((RealPath = realpath(Dir.c_str(), nullptr)) == 0)
+ return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
+ InternalPrefix = RealPath;
+ free(RealPath);
+ }
+
+ Owner = this;
+ FILE *List = fopen(File.c_str(),"r");
+ if (List == 0)
+ return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
+
+ /* We are a tad tricky here.. We prefix the buffer with the directory
+ name, that way if we need a full path with just use line.. Sneaky and
+ fully evil. */
+ char Line[1000];
+ char *FileStart;
+ if (Dir.empty() == true || Dir.end()[-1] != '/')
+ FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
+ else
+ FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
+ while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
+ {
+ char *FileName = _strstrip(FileStart);
+ if (FileName[0] == 0)
+ continue;
+
+ if (FileName[0] != '/')
+ {
+ if (FileName != FileStart)
+ memmove(FileStart,FileName,strlen(FileStart));
+ FileName = Line;
+ }
+
+#if 0
+ struct stat St;
+ int Flag = FTW_F;
+ if (stat(FileName,&St) != 0)
+ Flag = FTW_NS;
+#endif
+ if (FileMatchesPatterns(FileName, Patterns) == false)
+ continue;
+
+ if (ProcessFile(FileName, false) != 0)
+ break;
+ }
+
+ fclose(List);
+ return true;
+}
+ /*}}}*/
+// FTWScanner::Delink - Delink symlinks /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
+ unsigned long long &DeLinkBytes,
+ unsigned long long const &FileSize)
+{
+ // See if this isn't an internally prefix'd file name.
+ if (InternalPrefix.empty() == false &&
+ InternalPrefix.length() < FileName.length() &&
+ stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
+ InternalPrefix.begin(),InternalPrefix.end()) != 0)
+ {
+ if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
+ {
+ // Tidy up the display
+ if (DeLinkBytes == 0)
+ cout << endl;
+
+ NewLine(1);
+ ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
+ SizeToStr(FileSize).c_str());
+ c1out << flush;
+
+ if (NoLinkAct == false)
+ {
+ char OldLink[400];
+ if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
+ _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
+ else
+ {
+ if (RemoveFile("FTWScanner::Delink", OriginalPath))
+ {
+ if (link(FileName.c_str(),OriginalPath) != 0)
+ {
+ // Panic! Restore the symlink
+ if (symlink(OldLink,OriginalPath) != 0)
+ _error->Errno("symlink", "failed to restore symlink");
+ return _error->Errno("link",_("*** Failed to link %s to %s"),
+ FileName.c_str(),
+ OriginalPath);
+ }
+ }
+ }
+ }
+
+ DeLinkBytes += FileSize;
+ if (DeLinkBytes/1024 >= DeLinkLimit)
+ ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
+ }
+
+ FileName = OriginalPath;
+ }
+
+ return true;
+}
+ /*}}}*/
+// FTWScanner::SetExts - Set extensions to support /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool FTWScanner::SetExts(string const &Vals)
+{
+ ClearPatterns();
+ string::size_type Start = 0;
+ while (Start <= Vals.length()-1)
+ {
+ string::size_type const Space = Vals.find(' ',Start);
+ string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
+ if ( Arch.empty() == false )
+ {
+ AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
+ if (IncludeArchAll == true && Arch != "all")
+ AddPattern(string("*_all") + Vals.substr(Start, Length));
+ }
+ else
+ AddPattern(string("*") + Vals.substr(Start, Length));
+
+ Start += Length + 1;
+ }
+
+ return true;
+}
+ /*}}}*/
+
+// PackagesWriter::PackagesWriter - Constructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+PackagesWriter::PackagesWriter(FileFd * const GivenOutput, TranslationWriter * const transWriter,
+ string const &DB,string const &Overrides,string const &ExtOverrides,
+ string const &Arch, bool const IncludeArchAll) :
+ FTWScanner(GivenOutput, Arch, IncludeArchAll), Db(DB), Stats(Db.Stats), TransWriter(transWriter)
+{
+ SetExts(".deb .ddeb .udeb");
+ DeLinkLimit = 0;
+
+ // Process the command line options
+ ConfigToDoHashes(DoHashes, "APT::FTPArchive::Packages");
+ DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
+ DoContents = _config->FindB("APT::FTPArchive::Contents",true);
+ NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
+ LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
+
+ if (Db.Loaded() == false)
+ DoContents = false;
+
+ // Read the override file
+ if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
+ return;
+ else
+ NoOverride = true;
+
+ if (ExtOverrides.empty() == false)
+ Over.ReadExtraOverride(ExtOverrides);
+
+ _error->DumpErrors();
+}
+ /*}}}*/
+// PackagesWriter::DoPackage - Process a single package /*{{{*/
+// ---------------------------------------------------------------------
+/* This method takes a package and gets its control information and
+ MD5, SHA1 and SHA256 then writes out a control record with the proper fields
+ rewritten and the path/size/hash appended. */
+bool PackagesWriter::DoPackage(string FileName)
+{
+ // Pull all the data we need form the DB
+ if (Db.GetFileInfo(FileName,
+ true, /* DoControl */
+ DoContents,
+ true, /* GenContentsOnly */
+ false, /* DoSource */
+ DoHashes, DoAlwaysStat) == false)
+ {
+ return false;
+ }
+
+ unsigned long long FileSize = Db.GetFileSize();
+ if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
+ return false;
+
+ // Lookup the override information
+ pkgTagSection &Tags = Db.Control.Section;
+ auto const Package = Tags.Find(pkgTagSection::Key::Package).to_string();
+ string Architecture;
+ // if we generate a Packages file for a given arch, we use it to
+ // look for overrides. if we run in "simple" mode without the
+ // "Architectures" variable in the config we use the architecture value
+ // from the deb file
+ if(Arch != "")
+ Architecture = Arch;
+ else
+ Architecture = Tags.Find(pkgTagSection::Key::Architecture).to_string();
+ unique_ptr<Override::Item> OverItem(Over.GetItem(Package, Architecture));
+
+ if (Package.empty() == true)
+ return _error->Error(_("Archive had no package field"));
+
+ // If we need to do any rewriting of the header do it now..
+ if (OverItem.get() == 0)
+ {
+ if (NoOverride == false)
+ {
+ NewLine(1);
+ ioprintf(c1out, _(" %s has no override entry\n"), Package.c_str());
+ }
+
+ OverItem = unique_ptr<Override::Item>(new Override::Item);
+ OverItem->FieldOverride["Section"] = Tags.Find(pkgTagSection::Key::Section).to_string();
+ OverItem->Priority = Tags.Find(pkgTagSection::Key::Priority).to_string();
+ }
+
+ // Strip the DirStrip prefix from the FileName and add the PathPrefix
+ string NewFileName;
+ if (DirStrip.empty() == false &&
+ FileName.length() > DirStrip.length() &&
+ stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
+ DirStrip.begin(),DirStrip.end()) == 0)
+ NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
+ else
+ NewFileName = FileName;
+ if (PathPrefix.empty() == false)
+ NewFileName = flCombine(PathPrefix,NewFileName);
+
+ /* Configuration says we don't want to include the long Description
+ in the package file - instead we want to ship a separated file */
+ string desc;
+ if (LongDescription == false) {
+ desc = Tags.Find(pkgTagSection::Key::Description).to_string().append("\n");
+ OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
+ }
+
+ // This lists all the changes to the fields we are going to make.
+ std::vector<pkgTagSection::Tag> Changes;
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Size", std::to_string(FileSize)));
+
+ for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
+ {
+ if (hs->HashType() == "MD5Sum")
+ Changes.push_back(pkgTagSection::Tag::Rewrite("MD5sum", hs->HashValue()));
+ else if (hs->HashType() == "Checksum-FileSize")
+ continue;
+ else
+ Changes.push_back(pkgTagSection::Tag::Rewrite(hs->HashType(), hs->HashValue()));
+ }
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", NewFileName));
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", OverItem->Priority));
+ Changes.push_back(pkgTagSection::Tag::Remove("Status"));
+
+ string DescriptionMd5;
+ if (LongDescription == false) {
+ Hashes descmd5(Hashes::MD5SUM);
+ descmd5.Add(desc.c_str());
+ DescriptionMd5 = descmd5.GetHashString(Hashes::MD5SUM).HashValue();
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Description-md5", DescriptionMd5));
+ if (TransWriter != NULL)
+ TransWriter->DoPackage(Package, desc, DescriptionMd5);
+ }
+
+ // Rewrite the maintainer field if necessary
+ bool MaintFailed;
+ string NewMaint = OverItem->SwapMaint(Tags.Find(pkgTagSection::Key::Maintainer).to_string(), MaintFailed);
+ if (MaintFailed == true)
+ {
+ if (NoOverride == false)
+ {
+ NewLine(1);
+ ioprintf(c1out, _(" %s maintainer is %s not %s\n"),
+ Package.c_str(), Tags.Find(pkgTagSection::Key::Maintainer).to_string().c_str(), OverItem->OldMaint.c_str());
+ }
+ }
+
+ if (NewMaint.empty() == false)
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint));
+
+ for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
+ I != OverItem->FieldOverride.end(); ++I)
+ Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
+
+ // Rewrite and store the fields.
+ if (Tags.Write(*Output, TFRewritePackageOrder, Changes) == false ||
+ Output->Write("\n", 1) == false)
+ return false;
+
+ return Db.Finish();
+}
+ /*}}}*/
+PackagesWriter::~PackagesWriter() /*{{{*/
+{
+}
+ /*}}}*/
+
+// TranslationWriter::TranslationWriter - Constructor /*{{{*/
+// ---------------------------------------------------------------------
+/* Create a Translation-Master file for this Packages file */
+TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
+ mode_t const &Permissions) : Comp(NULL), Output(NULL)
+{
+ if (File.empty() == true)
+ return;
+
+ Comp = new MultiCompress(File, TransCompress, Permissions);
+ Output = &Comp->Input;
+}
+ /*}}}*/
+// TranslationWriter::DoPackage - Process a single package /*{{{*/
+// ---------------------------------------------------------------------
+/* Create a Translation-Master file for this Packages file */
+bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
+ string const &MD5)
+{
+ if (Output == NULL)
+ return true;
+
+ // Different archs can include different versions and therefore
+ // different descriptions - so we need to check for both name and md5.
+ string const Record = Pkg + ":" + MD5;
+
+ if (Included.find(Record) != Included.end())
+ return true;
+
+ std::string out;
+ strprintf(out, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
+ Pkg.c_str(), MD5.c_str(), Desc.c_str());
+ Output->Write(out.c_str(), out.length());
+
+ Included.insert(Record);
+ return true;
+}
+ /*}}}*/
+// TranslationWriter::~TranslationWriter - Destructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+TranslationWriter::~TranslationWriter()
+{
+ if (Comp != NULL)
+ delete Comp;
+}
+ /*}}}*/
+
+// SourcesWriter::SourcesWriter - Constructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+SourcesWriter::SourcesWriter(FileFd * const GivenOutput, string const &DB, string const &BOverrides,string const &SOverrides,
+ string const &ExtOverrides) :
+ FTWScanner(GivenOutput), Db(DB), Stats(Db.Stats)
+{
+ AddPattern("*.dsc");
+ DeLinkLimit = 0;
+ Buffer = 0;
+ BufSize = 0;
+
+ // Process the command line options
+ ConfigToDoHashes(DoHashes, "APT::FTPArchive::Sources");
+ NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
+ DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
+
+ // Read the override file
+ if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
+ return;
+ else
+ NoOverride = true;
+
+ // WTF?? The logic above: if we can't read binary overrides, don't even try
+ // reading source overrides. if we can read binary overrides, then say there
+ // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
+
+ if (ExtOverrides.empty() == false)
+ SOver.ReadExtraOverride(ExtOverrides);
+
+ if (SOverrides.empty() == false && FileExists(SOverrides) == true)
+ SOver.ReadOverride(SOverrides,true);
+}
+ /*}}}*/
+// SourcesWriter::DoPackage - Process a single package /*{{{*/
+static std::string getDscHash(unsigned int const DoHashes,
+ Hashes::SupportedHashes const DoIt, pkgTagSection &Tags, pkgTagSection::Key const FieldKey,
+ HashString const * const Hash, unsigned long long Size, std::string const &FileName)
+{
+ if ((DoHashes & DoIt) != DoIt || not Tags.Exists(FieldKey) || Hash == nullptr)
+ return "";
+ std::ostringstream out;
+ out << "\n " << Hash->HashValue() << " " << std::to_string(Size) << " " << FileName
+ << "\n " << Tags.Find(FieldKey).to_string();
+ return out.str();
+}
+bool SourcesWriter::DoPackage(string FileName)
+{
+ // Pull all the data we need form the DB
+ if (Db.GetFileInfo(FileName,
+ false, /* DoControl */
+ false, /* DoContents */
+ false, /* GenContentsOnly */
+ true, /* DoSource */
+ DoHashes, DoAlwaysStat) == false)
+ {
+ return false;
+ }
+
+ // we need to perform a "write" here (this is what finish is doing)
+ // because the call to Db.GetFileInfo() in the loop will change
+ // the "db cursor"
+ Db.Finish();
+
+ pkgTagSection Tags;
+ if (Tags.Scan(Db.Dsc.Data.c_str(), Db.Dsc.Data.length()) == false)
+ return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
+
+ if (not Tags.Exists(pkgTagSection::Key::Source))
+ return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str());
+ Tags.Trim();
+
+ // Lookup the override information, finding first the best priority.
+ string BestPrio;
+ auto const Bins = Tags.Find(pkgTagSection::Key::Binary);
+ char Buffer[Bins.length() + 1];
+ unique_ptr<Override::Item> OverItem(nullptr);
+ if (Bins.empty() == false)
+ {
+ strncpy(Buffer, Bins.data(), Bins.length());
+ Buffer[Bins.length()] = '\0';
+
+ // Ignore too-long errors.
+ char *BinList[400];
+ TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
+
+ // Look at all the binaries
+ unsigned char BestPrioV = pkgCache::State::Extra;
+ for (unsigned I = 0; BinList[I] != 0; I++)
+ {
+ unique_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
+ if (Itm.get() == 0)
+ continue;
+
+ unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
+ if (NewPrioV < BestPrioV || BestPrio.empty() == true)
+ {
+ BestPrioV = NewPrioV;
+ BestPrio = Itm->Priority;
+ }
+
+ if (OverItem.get() == 0)
+ OverItem = std::move(Itm);
+ }
+ }
+
+ // If we need to do any rewriting of the header do it now..
+ if (OverItem.get() == 0)
+ {
+ if (NoOverride == false)
+ {
+ NewLine(1);
+ ioprintf(c1out, _(" %s has no override entry\n"), Tags.Find(pkgTagSection::Key::Source).to_string().c_str());
+ }
+
+ OverItem.reset(new Override::Item);
+ }
+
+ struct stat St;
+ if (stat(FileName.c_str(), &St) != 0)
+ return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
+
+ auto const Package = Tags.Find(pkgTagSection::Key::Source).to_string();
+ unique_ptr<Override::Item> SOverItem(SOver.GetItem(Package));
+ // const unique_ptr<Override::Item> autoSOverItem(SOverItem);
+ if (SOverItem.get() == 0)
+ {
+ ioprintf(c1out, _(" %s has no source override entry\n"), Package.c_str());
+ SOverItem = unique_ptr<Override::Item>(BOver.GetItem(Package));
+ if (SOverItem.get() == 0)
+ {
+ ioprintf(c1out, _(" %s has no binary override entry either\n"), Package.c_str());
+ SOverItem = unique_ptr<Override::Item>(new Override::Item);
+ *SOverItem = *OverItem;
+ }
+ }
+
+ // Add the dsc to the files hash list
+ string const strippedName = flNotDir(FileName);
+ std::string const Files = getDscHash(DoHashes, Hashes::MD5SUM, Tags, pkgTagSection::Key::Files, Db.HashesList.find("MD5Sum"), St.st_size, strippedName);
+ std::string ChecksumsSha1 = getDscHash(DoHashes, Hashes::SHA1SUM, Tags, pkgTagSection::Key::Checksums_Sha1, Db.HashesList.find("SHA1"), St.st_size, strippedName);
+ std::string ChecksumsSha256 = getDscHash(DoHashes, Hashes::SHA256SUM, Tags, pkgTagSection::Key::Checksums_Sha256, Db.HashesList.find("SHA256"), St.st_size, strippedName);
+ std::string ChecksumsSha512 = getDscHash(DoHashes, Hashes::SHA512SUM, Tags, pkgTagSection::Key::Checksums_Sha512, Db.HashesList.find("SHA512"), St.st_size, strippedName);
+
+ // Strip the DirStrip prefix from the FileName and add the PathPrefix
+ string NewFileName;
+ if (DirStrip.empty() == false &&
+ FileName.length() > DirStrip.length() &&
+ stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
+ NewFileName = string(OriginalPath + DirStrip.length());
+ else
+ NewFileName = OriginalPath;
+ if (PathPrefix.empty() == false)
+ NewFileName = flCombine(PathPrefix,NewFileName);
+
+ string Directory = flNotFile(OriginalPath);
+
+ // Perform operation over all of the files
+ string ParseJnk;
+ const char *C = Files.c_str();
+ char *RealPath = NULL;
+ for (;isspace(*C); C++);
+ while (*C != 0)
+ {
+ // Parse each of the elements
+ if (ParseQuoteWord(C,ParseJnk) == false ||
+ ParseQuoteWord(C,ParseJnk) == false ||
+ ParseQuoteWord(C,ParseJnk) == false)
+ return _error->Error("Error parsing file record");
+
+ string OriginalPath = Directory + ParseJnk;
+
+ // Add missing hashes to source files
+ if (((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM && not Tags.Exists(pkgTagSection::Key::Checksums_Sha1)) ||
+ ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM && not Tags.Exists(pkgTagSection::Key::Checksums_Sha256)) ||
+ ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM && not Tags.Exists(pkgTagSection::Key::Checksums_Sha512)))
+ {
+ if (Db.GetFileInfo(OriginalPath,
+ false, /* DoControl */
+ false, /* DoContents */
+ false, /* GenContentsOnly */
+ false, /* DoSource */
+ DoHashes,
+ DoAlwaysStat) == false)
+ {
+ return _error->Error("Error getting file info");
+ }
+
+ for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
+ {
+ if (hs->HashType() == "MD5Sum" || hs->HashType() == "Checksum-FileSize")
+ continue;
+ pkgTagSection::Key fieldkey;
+ std::string * out;
+ if (hs->HashType() == "SHA1")
+ {
+ fieldkey = pkgTagSection::Key::Checksums_Sha1;
+ out = &ChecksumsSha1;
+ }
+ else if (hs->HashType() == "SHA256")
+ {
+ fieldkey = pkgTagSection::Key::Checksums_Sha256;
+ out = &ChecksumsSha256;
+ }
+ else if (hs->HashType() == "SHA512")
+ {
+ fieldkey = pkgTagSection::Key::Checksums_Sha512;
+ out = &ChecksumsSha512;
+ }
+ else
+ {
+ _error->Warning("Ignoring unknown Checksumtype %s in SourcesWriter::DoPackages", hs->HashType().c_str());
+ continue;
+ }
+ if (Tags.Exists(fieldkey))
+ continue;
+ std::ostringstream streamout;
+ streamout << "\n " << hs->HashValue() << " " << std::to_string(Db.GetFileSize()) << " " << ParseJnk;
+ out->append(streamout.str());
+ }
+
+ // write back the GetFileInfo() stats data
+ Db.Finish();
+ }
+
+ // Perform the delinking operation
+ char Jnk[2];
+
+ if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
+ (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
+ {
+ string RP = RealPath;
+ free(RealPath);
+ if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
+ return false;
+ }
+ }
+
+ Directory = flNotFile(NewFileName);
+ if (Directory.length() > 2)
+ Directory.erase(Directory.end()-1);
+
+ // This lists all the changes to the fields we are going to make.
+ // (5 hardcoded + checksums + maintainer + end marker)
+ std::vector<pkgTagSection::Tag> Changes;
+
+ Changes.push_back(pkgTagSection::Tag::Remove("Source"));
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Package", Package));
+ if (Files.empty() == false)
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Files", Files));
+ else
+ Changes.push_back(pkgTagSection::Tag::Remove("Files"));
+ if (ChecksumsSha1.empty() == false)
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha1", ChecksumsSha1));
+ else
+ Changes.push_back(pkgTagSection::Tag::Remove("Checksums-Sha1"));
+ if (ChecksumsSha256.empty() == false)
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha256", ChecksumsSha256));
+ else
+ Changes.push_back(pkgTagSection::Tag::Remove("Checksums-Sha256"));
+ if (ChecksumsSha512.empty() == false)
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha512", ChecksumsSha512));
+ else
+ Changes.push_back(pkgTagSection::Tag::Remove("Checksums-Sha512"));
+ if (Directory != "./")
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Directory));
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", BestPrio));
+ Changes.push_back(pkgTagSection::Tag::Remove("Status"));
+
+ // Rewrite the maintainer field if necessary
+ bool MaintFailed;
+ string NewMaint = OverItem->SwapMaint(Tags.Find(pkgTagSection::Key::Maintainer).to_string(), MaintFailed);
+ if (MaintFailed == true)
+ {
+ if (NoOverride == false)
+ {
+ NewLine(1);
+ ioprintf(c1out, _(" %s maintainer is %s not %s\n"), Package.c_str(),
+ Tags.Find(pkgTagSection::Key::Maintainer).to_string().c_str(), OverItem->OldMaint.c_str());
+ }
+ }
+ if (NewMaint.empty() == false)
+ Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint.c_str()));
+
+ for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
+ I != SOverItem->FieldOverride.end(); ++I)
+ Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
+
+ // Rewrite and store the fields.
+ if (Tags.Write(*Output, TFRewriteSourceOrder, Changes) == false ||
+ Output->Write("\n", 1) == false)
+ return false;
+
+ Stats.Packages++;
+
+ return true;
+}
+ /*}}}*/
+
+// ContentsWriter::ContentsWriter - Constructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+ContentsWriter::ContentsWriter(FileFd * const GivenOutput, string const &DB,
+ string const &Arch, bool const IncludeArchAll) :
+ FTWScanner(GivenOutput, Arch, IncludeArchAll), Db(DB), Stats(Db.Stats)
+
+{
+ SetExts(".deb");
+}
+ /*}}}*/
+// ContentsWriter::DoPackage - Process a single package /*{{{*/
+// ---------------------------------------------------------------------
+/* If Package is the empty string the control record will be parsed to
+ determine what the package name is. */
+bool ContentsWriter::DoPackage(string FileName, string Package)
+{
+ if (!Db.GetFileInfo(FileName,
+ Package.empty(), /* DoControl */
+ true, /* DoContents */
+ false, /* GenContentsOnly */
+ false, /* DoSource */
+ 0, /* DoHashes */
+ false /* checkMtime */))
+ {
+ return false;
+ }
+
+ // Parse the package name
+ if (Package.empty() == true)
+ {
+ Package = Db.Control.Section.Find(pkgTagSection::Key::Package).to_string();
+ }
+
+ Db.Contents.Add(Gen,Package);
+
+ return Db.Finish();
+}
+ /*}}}*/
+// ContentsWriter::ReadFromPkgs - Read from a packages file /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
+{
+ MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
+ if (_error->PendingError() == true)
+ return false;
+
+ // Open the package file
+ FileFd Fd;
+ if (Pkgs.OpenOld(Fd) == false)
+ return false;
+
+ pkgTagFile Tags(&Fd);
+ if (_error->PendingError() == true)
+ return false;
+
+ // Parse.
+ pkgTagSection Section;
+ while (Tags.Step(Section) == true)
+ {
+ auto File = flCombine(Prefix, Section.Find(pkgTagSection::Key::Filename).to_string());
+ auto Package = flCombine(Section.Find(pkgTagSection::Key::Section).to_string(), Section.Find(pkgTagSection::Key::Package).to_string());
+ DoPackage(std::move(File), std::move(Package));
+ if (_error->empty() == false)
+ {
+ _error->Error("Errors apply to file '%s'",File.c_str());
+ _error->DumpErrors();
+ }
+ }
+
+ // Tidy the compressor
+ Fd.Close();
+
+ return true;
+}
+
+ /*}}}*/
+
+// ReleaseWriter::ReleaseWriter - Constructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+static std::string formatUTCDateTime(time_t const now)
+{
+ bool const NumericTimezone = _config->FindB("APT::FTPArchive::Release::NumericTimezone", true);
+ // TimeRFC1123 uses GMT to satisfy HTTP/1.1
+ std::string datetime = TimeRFC1123(now, NumericTimezone);
+ if (NumericTimezone == false)
+ {
+ auto const lastspace = datetime.rfind(' ');
+ if (likely(lastspace != std::string::npos))
+ datetime.replace(lastspace + 1, 3, "UTC");
+ }
+ return datetime;
+}
+ReleaseWriter::ReleaseWriter(FileFd * const GivenOutput, string const &/*DB*/) : FTWScanner(GivenOutput)
+{
+ if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
+ {
+ AddPattern("Packages");
+ AddPattern("Packages.*");
+ AddPattern("Translation-*");
+ AddPattern("Sources");
+ AddPattern("Sources.*");
+ AddPattern("Release");
+ AddPattern("Contents-*");
+ AddPattern("Index");
+ AddPattern("Index.*");
+ AddPattern("icons-*.tar");
+ AddPattern("icons-*.tar.*");
+ AddPattern("Components-*.yml");
+ AddPattern("Components-*.yml.*");
+ AddPattern("md5sum.txt");
+ }
+ AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
+
+ time_t const now = time(NULL);
+ time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
+
+ map<string,bool> BoolFields;
+ map<string,string> Fields;
+ Fields["Origin"] = "";
+ Fields["Label"] = "";
+ Fields["Suite"] = "";
+ Fields["Version"] = "";
+ Fields["Codename"] = "";
+ Fields["Date"] = formatUTCDateTime(now);
+ if (validuntil != now)
+ Fields["Valid-Until"] = formatUTCDateTime(validuntil);
+ Fields["Architectures"] = "";
+ Fields["Components"] = "";
+ Fields["Description"] = "";
+ Fields["Signed-By"] = "";
+ BoolFields["Acquire-By-Hash"] = _config->FindB("APT::FTPArchive::DoByHash", false);
+ BoolFields["NotAutomatic"] = false;
+ BoolFields["ButAutomaticUpgrades"] = false;
+
+ // Read configuration for string fields, but don't output them
+ for (auto &&I : Fields)
+ {
+ string Config = string("APT::FTPArchive::Release::") + I.first;
+ I.second = _config->Find(Config, I.second);
+ }
+
+ // Read configuration for bool fields, and add them to Fields if true
+ for (auto &&I : BoolFields)
+ {
+ string Config = string("APT::FTPArchive::Release::") + I.first;
+ I.second = _config->FindB(Config, I.second);
+ if (I.second)
+ Fields[I.first] = "yes";
+ }
+
+ // All configuration read and stored in Fields; output
+ for (auto &&I : Fields)
+ {
+ if (I.second.empty())
+ continue;
+ std::string const out = I.first + ": " + I.second + "\n";
+ Output->Write(out.c_str(), out.length());
+ }
+
+ ConfigToDoHashes(DoHashes, "APT::FTPArchive::Release");
+}
+ /*}}}*/
+// ReleaseWriter::DoPackage - Process a single package /*{{{*/
+// ---------------------------------------------------------------------
+bool ReleaseWriter::DoPackage(string FileName)
+{
+ // Strip the DirStrip prefix from the FileName and add the PathPrefix
+ string NewFileName;
+ if (DirStrip.empty() == false &&
+ FileName.length() > DirStrip.length() &&
+ stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
+ DirStrip.begin(),DirStrip.end()) == 0)
+ {
+ NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
+ while (NewFileName[0] == '/')
+ NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
+ }
+ else
+ NewFileName = FileName;
+
+ if (PathPrefix.empty() == false)
+ NewFileName = flCombine(PathPrefix,NewFileName);
+
+ FileFd fd(FileName, FileFd::ReadOnly);
+
+ if (!fd.IsOpen())
+ {
+ return false;
+ }
+
+ CheckSums[NewFileName].size = fd.Size();
+
+ Hashes hs(DoHashes);
+ hs.AddFD(fd);
+ CheckSums[NewFileName].Hashes = hs.GetHashStringList();
+ fd.Close();
+
+ // FIXME: wrong layer in the code(?)
+ // FIXME2: symlink instead of create a copy
+ if (_config->FindB("APT::FTPArchive::DoByHash", false) == true)
+ {
+ std::string Input = FileName;
+ HashStringList hsl = hs.GetHashStringList();
+ for(HashStringList::const_iterator h = hsl.begin();
+ h != hsl.end(); ++h)
+ {
+ if (!h->usable())
+ continue;
+ if (flNotDir(FileName) == "Release" || flNotDir(FileName) == "InRelease")
+ continue;
+
+ std::string ByHashOutputFile = GenByHashFilename(Input, *h);
+ std::string ByHashOutputDir = flNotFile(ByHashOutputFile);
+ if(!CreateDirectory(flNotFile(Input), ByHashOutputDir))
+ return _error->Warning("can not create dir %s", flNotFile(ByHashOutputFile).c_str());
+
+ // write new hashes
+ FileFd In(Input, FileFd::ReadOnly);
+ FileFd Out(ByHashOutputFile, FileFd::WriteEmpty);
+ if(!CopyFile(In, Out))
+ return _error->Warning("failed to copy %s %s", Input.c_str(), ByHashOutputFile.c_str());
+ }
+ }
+
+ return true;
+}
+
+ /*}}}*/
+// ReleaseWriter::Finish - Output the checksums /*{{{*/
+// ---------------------------------------------------------------------
+static void printChecksumTypeRecord(FileFd &Output, char const * const Type, map<string, ReleaseWriter::CheckSum> const &CheckSums)
+{
+ {
+ std::string out;
+ strprintf(out, "%s:\n", Type);
+ Output.Write(out.c_str(), out.length());
+ }
+ for(map<string,ReleaseWriter::CheckSum>::const_iterator I = CheckSums.begin();
+ I != CheckSums.end(); ++I)
+ {
+ HashString const * const hs = I->second.Hashes.find(Type);
+ if (hs == NULL)
+ continue;
+ std::string out;
+ strprintf(out, " %s %16llu %s\n",
+ hs->HashValue().c_str(),
+ (*I).second.size,
+ (*I).first.c_str());
+ Output.Write(out.c_str(), out.length());
+ }
+}
+void ReleaseWriter::Finish()
+{
+ if ((DoHashes & Hashes::MD5SUM) == Hashes::MD5SUM)
+ printChecksumTypeRecord(*Output, "MD5Sum", CheckSums);
+ if ((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM)
+ printChecksumTypeRecord(*Output, "SHA1", CheckSums);
+ if ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM)
+ printChecksumTypeRecord(*Output, "SHA256", CheckSums);
+ if ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM)
+ printChecksumTypeRecord(*Output, "SHA512", CheckSums);
+
+ // go by-hash cleanup
+ map<string,ReleaseWriter::CheckSum>::const_iterator prev = CheckSums.begin();
+ if (_config->FindB("APT::FTPArchive::DoByHash", false) == true)
+ {
+ for(map<string,ReleaseWriter::CheckSum>::const_iterator I = CheckSums.begin();
+ I != CheckSums.end(); ++I)
+ {
+ if (I->first == "Release" || I->first == "InRelease")
+ continue;
+
+ // keep iterating until we find a new subdir
+ if(flNotFile(I->first) == flNotFile(prev->first))
+ continue;
+
+ // clean that subdir up
+ int keepFiles = _config->FindI("APT::FTPArchive::By-Hash-Keep", 3);
+ // calculate how many compressors are used (the amount of files
+ // in that subdir generated for this run)
+ keepFiles *= std::distance(prev, I);
+ prev = I;
+
+ HashStringList hsl = prev->second.Hashes;
+ for(HashStringList::const_iterator h = hsl.begin();
+ h != hsl.end(); ++h)
+ {
+
+ if (!h->usable())
+ continue;
+
+ std::string RealFilename = DirStrip+"/"+prev->first;
+ std::string ByHashOutputFile = GenByHashFilename(RealFilename, *h);
+ DeleteAllButMostRecent(flNotFile(ByHashOutputFile), keepFiles);
+ }
+ }
+ }
+}