diff options
Diffstat (limited to 'ftparchive/writer.cc')
| -rw-r--r-- | ftparchive/writer.cc | 1181 |
1 files changed, 1181 insertions, 0 deletions
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc new file mode 100644 index 0000000..ae427b1 --- /dev/null +++ b/ftparchive/writer.cc @@ -0,0 +1,1181 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Writer + + The file writer classes. These write various types of output, sources, + packages and contents. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include <config.h> + +#include <apt-pkg/configuration.h> +#include <apt-pkg/debfile.h> +#include <apt-pkg/deblistparser.h> +#include <apt-pkg/error.h> +#include <apt-pkg/fileutl.h> +#include <apt-pkg/gpgv.h> +#include <apt-pkg/hashes.h> +#include <apt-pkg/pkgcache.h> +#include <apt-pkg/strutl.h> +#include <apt-pkg/tagfile-keys.h> +#include <apt-pkg/tagfile.h> + +#include <algorithm> +#include <ctime> +#include <iomanip> +#include <iostream> +#include <memory> +#include <sstream> +#include <utility> +#include <ctype.h> +#include <fnmatch.h> +#include <ftw.h> +#include <locale.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "apt-ftparchive.h" +#include "byhash.h" +#include "cachedb.h" +#include "multicompress.h" +#include "writer.h" + +#include <apti18n.h> + /*}}}*/ +using namespace std; +FTWScanner *FTWScanner::Owner; + +// ConfigToDoHashes - which hashes to generate /*{{{*/ +static void SingleConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf, unsigned int const Flag) +{ + if (_config->FindB(Conf, (DoHashes & Flag) == Flag) == true) + DoHashes |= Flag; + else + DoHashes &= ~Flag; +} +static void ConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf) +{ + SingleConfigToDoHashes(DoHashes, Conf + "::MD5", Hashes::MD5SUM); + SingleConfigToDoHashes(DoHashes, Conf + "::SHA1", Hashes::SHA1SUM); + SingleConfigToDoHashes(DoHashes, Conf + "::SHA256", Hashes::SHA256SUM); + SingleConfigToDoHashes(DoHashes, Conf + "::SHA512", Hashes::SHA512SUM); +} + /*}}}*/ + +// FTWScanner::FTWScanner - Constructor /*{{{*/ +FTWScanner::FTWScanner(FileFd * const GivenOutput, string const &Arch, bool const IncludeArchAll) + : Arch(Arch), IncludeArchAll(IncludeArchAll), DoHashes(~0) +{ + if (GivenOutput == NULL) + { + Output = new FileFd; + OwnsOutput = true; + Output->OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false); + } + else + { + Output = GivenOutput; + OwnsOutput = false; + } + ErrorPrinted = false; + NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true); + ConfigToDoHashes(DoHashes, "APT::FTPArchive"); +} + /*}}}*/ +FTWScanner::~FTWScanner() +{ + if (Output != NULL && OwnsOutput) + delete Output; +} +// FTWScanner::Scanner - FTW Scanner /*{{{*/ +// --------------------------------------------------------------------- +/* This is the FTW scanner, it processes each directory element in the + directory tree. */ +int FTWScanner::ScannerFTW(const char *File,const struct stat * /*sb*/,int Flag) +{ + if (Flag == FTW_DNR) + { + Owner->NewLine(1); + ioprintf(c1out, _("W: Unable to read directory %s\n"), File); + } + if (Flag == FTW_NS) + { + Owner->NewLine(1); + ioprintf(c1out, _("W: Unable to stat %s\n"), File); + } + if (Flag != FTW_F) + return 0; + + return ScannerFile(File, true); +} + /*}}}*/ +static bool FileMatchesPatterns(char const *const File, std::vector<std::string> const &Patterns) /*{{{*/ +{ + const char *LastComponent = strrchr(File, '/'); + if (LastComponent == nullptr) + LastComponent = File; + else + ++LastComponent; + + return std::any_of(Patterns.cbegin(), Patterns.cend(), [&](std::string const &pattern) { + return fnmatch(pattern.c_str(), LastComponent, 0) == 0; + }); +} + /*}}}*/ +int FTWScanner::ScannerFile(const char *const File, bool const ReadLink) /*{{{*/ +{ + if (FileMatchesPatterns(File, Owner->Patterns) == false) + return 0; + + Owner->FilesToProcess.emplace_back(File, ReadLink); + return 0; +} + /*}}}*/ +int FTWScanner::ProcessFile(const char *const File, bool const ReadLink) /*{{{*/ +{ + /* Process it. If the file is a link then resolve it into an absolute + name.. This works best if the directory components the scanner are + given are not links themselves. */ + char Jnk[2]; + char *RealPath = NULL; + Owner->OriginalPath = File; + if (ReadLink && + readlink(File,Jnk,sizeof(Jnk)) != -1 && + (RealPath = realpath(File,NULL)) != 0) + { + Owner->DoPackage(RealPath); + free(RealPath); + } + else + Owner->DoPackage(File); + + if (_error->empty() == false) + { + // Print any errors or warnings found + string Err; + bool SeenPath = false; + while (_error->empty() == false) + { + Owner->NewLine(1); + + bool const Type = _error->PopMessage(Err); + if (Type == true) + cerr << _("E: ") << Err << endl; + else + cerr << _("W: ") << Err << endl; + + if (Err.find(File) != string::npos) + SeenPath = true; + } + + if (SeenPath == false) + cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl; + return 0; + } + + return 0; +} + /*}}}*/ +// FTWScanner::RecursiveScan - Just scan a directory tree /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool FTWScanner::RecursiveScan(string const &Dir) +{ + /* If noprefix is set then jam the scan root in, so we don't generate + link followed paths out of control */ + if (InternalPrefix.empty() == true) + { + char *RealPath = nullptr; + if ((RealPath = realpath(Dir.c_str(), nullptr)) == 0) + return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str()); + InternalPrefix = RealPath; + free(RealPath); + } + + // Do recursive directory searching + Owner = this; + int const Res = ftw(Dir.c_str(),ScannerFTW,30); + + // Error treewalking? + if (Res != 0) + { + if (_error->PendingError() == false) + _error->Errno("ftw",_("Tree walking failed")); + return false; + } + + using PairType = decltype(*FilesToProcess.cbegin()); + std::sort(FilesToProcess.begin(), FilesToProcess.end(), [](PairType a, PairType b) { + return a.first < b.first; + }); + if (not std::all_of(FilesToProcess.cbegin(), FilesToProcess.cend(), [](auto &&it) { return ProcessFile(it.first.c_str(), it.second) == 0; })) + return false; + FilesToProcess.clear(); + return true; +} + /*}}}*/ +// FTWScanner::LoadFileList - Load the file list from a file /*{{{*/ +// --------------------------------------------------------------------- +/* This is an alternative to using FTW to locate files, it reads the list + of files from another file. */ +bool FTWScanner::LoadFileList(string const &Dir, string const &File) +{ + /* If noprefix is set then jam the scan root in, so we don't generate + link followed paths out of control */ + if (InternalPrefix.empty() == true) + { + char *RealPath = nullptr; + if ((RealPath = realpath(Dir.c_str(), nullptr)) == 0) + return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str()); + InternalPrefix = RealPath; + free(RealPath); + } + + Owner = this; + FILE *List = fopen(File.c_str(),"r"); + if (List == 0) + return _error->Errno("fopen",_("Failed to open %s"),File.c_str()); + + /* We are a tad tricky here.. We prefix the buffer with the directory + name, that way if we need a full path with just use line.. Sneaky and + fully evil. */ + char Line[1000]; + char *FileStart; + if (Dir.empty() == true || Dir.end()[-1] != '/') + FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str()); + else + FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str()); + while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0) + { + char *FileName = _strstrip(FileStart); + if (FileName[0] == 0) + continue; + + if (FileName[0] != '/') + { + if (FileName != FileStart) + memmove(FileStart,FileName,strlen(FileStart)); + FileName = Line; + } + +#if 0 + struct stat St; + int Flag = FTW_F; + if (stat(FileName,&St) != 0) + Flag = FTW_NS; +#endif + if (FileMatchesPatterns(FileName, Patterns) == false) + continue; + + if (ProcessFile(FileName, false) != 0) + break; + } + + fclose(List); + return true; +} + /*}}}*/ +// FTWScanner::Delink - Delink symlinks /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool FTWScanner::Delink(string &FileName,const char *OriginalPath, + unsigned long long &DeLinkBytes, + unsigned long long const &FileSize) +{ + // See if this isn't an internally prefix'd file name. + if (InternalPrefix.empty() == false && + InternalPrefix.length() < FileName.length() && + stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(), + InternalPrefix.begin(),InternalPrefix.end()) != 0) + { + if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit) + { + // Tidy up the display + if (DeLinkBytes == 0) + cout << endl; + + NewLine(1); + ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()), + SizeToStr(FileSize).c_str()); + c1out << flush; + + if (NoLinkAct == false) + { + char OldLink[400]; + if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1) + _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath); + else + { + if (RemoveFile("FTWScanner::Delink", OriginalPath)) + { + if (link(FileName.c_str(),OriginalPath) != 0) + { + // Panic! Restore the symlink + if (symlink(OldLink,OriginalPath) != 0) + _error->Errno("symlink", "failed to restore symlink"); + return _error->Errno("link",_("*** Failed to link %s to %s"), + FileName.c_str(), + OriginalPath); + } + } + } + } + + DeLinkBytes += FileSize; + if (DeLinkBytes/1024 >= DeLinkLimit) + ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str()); + } + + FileName = OriginalPath; + } + + return true; +} + /*}}}*/ +// FTWScanner::SetExts - Set extensions to support /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool FTWScanner::SetExts(string const &Vals) +{ + ClearPatterns(); + string::size_type Start = 0; + while (Start <= Vals.length()-1) + { + string::size_type const Space = Vals.find(' ',Start); + string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start; + if ( Arch.empty() == false ) + { + AddPattern(string("*_") + Arch + Vals.substr(Start, Length)); + if (IncludeArchAll == true && Arch != "all") + AddPattern(string("*_all") + Vals.substr(Start, Length)); + } + else + AddPattern(string("*") + Vals.substr(Start, Length)); + + Start += Length + 1; + } + + return true; +} + /*}}}*/ + +// PackagesWriter::PackagesWriter - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +PackagesWriter::PackagesWriter(FileFd * const GivenOutput, TranslationWriter * const transWriter, + string const &DB,string const &Overrides,string const &ExtOverrides, + string const &Arch, bool const IncludeArchAll) : + FTWScanner(GivenOutput, Arch, IncludeArchAll), Db(DB), Stats(Db.Stats), TransWriter(transWriter) +{ + SetExts(".deb .ddeb .udeb"); + DeLinkLimit = 0; + + // Process the command line options + ConfigToDoHashes(DoHashes, "APT::FTPArchive::Packages"); + DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false); + DoContents = _config->FindB("APT::FTPArchive::Contents",true); + NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false); + LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true); + + if (Db.Loaded() == false) + DoContents = false; + + // Read the override file + if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false) + return; + else + NoOverride = true; + + if (ExtOverrides.empty() == false) + Over.ReadExtraOverride(ExtOverrides); + + _error->DumpErrors(); +} + /*}}}*/ +// PackagesWriter::DoPackage - Process a single package /*{{{*/ +// --------------------------------------------------------------------- +/* This method takes a package and gets its control information and + MD5, SHA1 and SHA256 then writes out a control record with the proper fields + rewritten and the path/size/hash appended. */ +bool PackagesWriter::DoPackage(string FileName) +{ + // Pull all the data we need form the DB + if (Db.GetFileInfo(FileName, + true, /* DoControl */ + DoContents, + true, /* GenContentsOnly */ + false, /* DoSource */ + DoHashes, DoAlwaysStat) == false) + { + return false; + } + + unsigned long long FileSize = Db.GetFileSize(); + if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false) + return false; + + // Lookup the override information + pkgTagSection &Tags = Db.Control.Section; + auto const Package = Tags.Find(pkgTagSection::Key::Package).to_string(); + string Architecture; + // if we generate a Packages file for a given arch, we use it to + // look for overrides. if we run in "simple" mode without the + // "Architectures" variable in the config we use the architecture value + // from the deb file + if(Arch != "") + Architecture = Arch; + else + Architecture = Tags.Find(pkgTagSection::Key::Architecture).to_string(); + unique_ptr<Override::Item> OverItem(Over.GetItem(Package, Architecture)); + + if (Package.empty() == true) + return _error->Error(_("Archive had no package field")); + + // If we need to do any rewriting of the header do it now.. + if (OverItem.get() == 0) + { + if (NoOverride == false) + { + NewLine(1); + ioprintf(c1out, _(" %s has no override entry\n"), Package.c_str()); + } + + OverItem = unique_ptr<Override::Item>(new Override::Item); + OverItem->FieldOverride["Section"] = Tags.Find(pkgTagSection::Key::Section).to_string(); + OverItem->Priority = Tags.Find(pkgTagSection::Key::Priority).to_string(); + } + + // Strip the DirStrip prefix from the FileName and add the PathPrefix + string NewFileName; + if (DirStrip.empty() == false && + FileName.length() > DirStrip.length() && + stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(), + DirStrip.begin(),DirStrip.end()) == 0) + NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end()); + else + NewFileName = FileName; + if (PathPrefix.empty() == false) + NewFileName = flCombine(PathPrefix,NewFileName); + + /* Configuration says we don't want to include the long Description + in the package file - instead we want to ship a separated file */ + string desc; + if (LongDescription == false) { + desc = Tags.Find(pkgTagSection::Key::Description).to_string().append("\n"); + OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str(); + } + + // This lists all the changes to the fields we are going to make. + std::vector<pkgTagSection::Tag> Changes; + Changes.push_back(pkgTagSection::Tag::Rewrite("Size", std::to_string(FileSize))); + + for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs) + { + if (hs->HashType() == "MD5Sum") + Changes.push_back(pkgTagSection::Tag::Rewrite("MD5sum", hs->HashValue())); + else if (hs->HashType() == "Checksum-FileSize") + continue; + else + Changes.push_back(pkgTagSection::Tag::Rewrite(hs->HashType(), hs->HashValue())); + } + Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", NewFileName)); + Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", OverItem->Priority)); + Changes.push_back(pkgTagSection::Tag::Remove("Status")); + + string DescriptionMd5; + if (LongDescription == false) { + Hashes descmd5(Hashes::MD5SUM); + descmd5.Add(desc.c_str()); + DescriptionMd5 = descmd5.GetHashString(Hashes::MD5SUM).HashValue(); + Changes.push_back(pkgTagSection::Tag::Rewrite("Description-md5", DescriptionMd5)); + if (TransWriter != NULL) + TransWriter->DoPackage(Package, desc, DescriptionMd5); + } + + // Rewrite the maintainer field if necessary + bool MaintFailed; + string NewMaint = OverItem->SwapMaint(Tags.Find(pkgTagSection::Key::Maintainer).to_string(), MaintFailed); + if (MaintFailed == true) + { + if (NoOverride == false) + { + NewLine(1); + ioprintf(c1out, _(" %s maintainer is %s not %s\n"), + Package.c_str(), Tags.Find(pkgTagSection::Key::Maintainer).to_string().c_str(), OverItem->OldMaint.c_str()); + } + } + + if (NewMaint.empty() == false) + Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint)); + + for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin(); + I != OverItem->FieldOverride.end(); ++I) + Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second)); + + // Rewrite and store the fields. + if (Tags.Write(*Output, TFRewritePackageOrder, Changes) == false || + Output->Write("\n", 1) == false) + return false; + + return Db.Finish(); +} + /*}}}*/ +PackagesWriter::~PackagesWriter() /*{{{*/ +{ +} + /*}}}*/ + +// TranslationWriter::TranslationWriter - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Create a Translation-Master file for this Packages file */ +TranslationWriter::TranslationWriter(string const &File, string const &TransCompress, + mode_t const &Permissions) : Comp(NULL), Output(NULL) +{ + if (File.empty() == true) + return; + + Comp = new MultiCompress(File, TransCompress, Permissions); + Output = &Comp->Input; +} + /*}}}*/ +// TranslationWriter::DoPackage - Process a single package /*{{{*/ +// --------------------------------------------------------------------- +/* Create a Translation-Master file for this Packages file */ +bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc, + string const &MD5) +{ + if (Output == NULL) + return true; + + // Different archs can include different versions and therefore + // different descriptions - so we need to check for both name and md5. + string const Record = Pkg + ":" + MD5; + + if (Included.find(Record) != Included.end()) + return true; + + std::string out; + strprintf(out, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n", + Pkg.c_str(), MD5.c_str(), Desc.c_str()); + Output->Write(out.c_str(), out.length()); + + Included.insert(Record); + return true; +} + /*}}}*/ +// TranslationWriter::~TranslationWriter - Destructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +TranslationWriter::~TranslationWriter() +{ + if (Comp != NULL) + delete Comp; +} + /*}}}*/ + +// SourcesWriter::SourcesWriter - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +SourcesWriter::SourcesWriter(FileFd * const GivenOutput, string const &DB, string const &BOverrides,string const &SOverrides, + string const &ExtOverrides) : + FTWScanner(GivenOutput), Db(DB), Stats(Db.Stats) +{ + AddPattern("*.dsc"); + DeLinkLimit = 0; + Buffer = 0; + BufSize = 0; + + // Process the command line options + ConfigToDoHashes(DoHashes, "APT::FTPArchive::Sources"); + NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false); + DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false); + + // Read the override file + if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false) + return; + else + NoOverride = true; + + // WTF?? The logic above: if we can't read binary overrides, don't even try + // reading source overrides. if we can read binary overrides, then say there + // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28 + + if (ExtOverrides.empty() == false) + SOver.ReadExtraOverride(ExtOverrides); + + if (SOverrides.empty() == false && FileExists(SOverrides) == true) + SOver.ReadOverride(SOverrides,true); +} + /*}}}*/ +// SourcesWriter::DoPackage - Process a single package /*{{{*/ +static std::string getDscHash(unsigned int const DoHashes, + Hashes::SupportedHashes const DoIt, pkgTagSection &Tags, pkgTagSection::Key const FieldKey, + HashString const * const Hash, unsigned long long Size, std::string const &FileName) +{ + if ((DoHashes & DoIt) != DoIt || not Tags.Exists(FieldKey) || Hash == nullptr) + return ""; + std::ostringstream out; + out << "\n " << Hash->HashValue() << " " << std::to_string(Size) << " " << FileName + << "\n " << Tags.Find(FieldKey).to_string(); + return out.str(); +} +bool SourcesWriter::DoPackage(string FileName) +{ + // Pull all the data we need form the DB + if (Db.GetFileInfo(FileName, + false, /* DoControl */ + false, /* DoContents */ + false, /* GenContentsOnly */ + true, /* DoSource */ + DoHashes, DoAlwaysStat) == false) + { + return false; + } + + // we need to perform a "write" here (this is what finish is doing) + // because the call to Db.GetFileInfo() in the loop will change + // the "db cursor" + Db.Finish(); + + pkgTagSection Tags; + if (Tags.Scan(Db.Dsc.Data.c_str(), Db.Dsc.Data.length()) == false) + return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str()); + + if (not Tags.Exists(pkgTagSection::Key::Source)) + return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str()); + Tags.Trim(); + + // Lookup the override information, finding first the best priority. + string BestPrio; + auto const Bins = Tags.Find(pkgTagSection::Key::Binary); + char Buffer[Bins.length() + 1]; + unique_ptr<Override::Item> OverItem(nullptr); + if (Bins.empty() == false) + { + strncpy(Buffer, Bins.data(), Bins.length()); + Buffer[Bins.length()] = '\0'; + + // Ignore too-long errors. + char *BinList[400]; + TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0])); + + // Look at all the binaries + unsigned char BestPrioV = pkgCache::State::Extra; + for (unsigned I = 0; BinList[I] != 0; I++) + { + unique_ptr<Override::Item> Itm(BOver.GetItem(BinList[I])); + if (Itm.get() == 0) + continue; + + unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority); + if (NewPrioV < BestPrioV || BestPrio.empty() == true) + { + BestPrioV = NewPrioV; + BestPrio = Itm->Priority; + } + + if (OverItem.get() == 0) + OverItem = std::move(Itm); + } + } + + // If we need to do any rewriting of the header do it now.. + if (OverItem.get() == 0) + { + if (NoOverride == false) + { + NewLine(1); + ioprintf(c1out, _(" %s has no override entry\n"), Tags.Find(pkgTagSection::Key::Source).to_string().c_str()); + } + + OverItem.reset(new Override::Item); + } + + struct stat St; + if (stat(FileName.c_str(), &St) != 0) + return _error->Errno("fstat","Failed to stat %s",FileName.c_str()); + + auto const Package = Tags.Find(pkgTagSection::Key::Source).to_string(); + unique_ptr<Override::Item> SOverItem(SOver.GetItem(Package)); + // const unique_ptr<Override::Item> autoSOverItem(SOverItem); + if (SOverItem.get() == 0) + { + ioprintf(c1out, _(" %s has no source override entry\n"), Package.c_str()); + SOverItem = unique_ptr<Override::Item>(BOver.GetItem(Package)); + if (SOverItem.get() == 0) + { + ioprintf(c1out, _(" %s has no binary override entry either\n"), Package.c_str()); + SOverItem = unique_ptr<Override::Item>(new Override::Item); + *SOverItem = *OverItem; + } + } + + // Add the dsc to the files hash list + string const strippedName = flNotDir(FileName); + std::string const Files = getDscHash(DoHashes, Hashes::MD5SUM, Tags, pkgTagSection::Key::Files, Db.HashesList.find("MD5Sum"), St.st_size, strippedName); + std::string ChecksumsSha1 = getDscHash(DoHashes, Hashes::SHA1SUM, Tags, pkgTagSection::Key::Checksums_Sha1, Db.HashesList.find("SHA1"), St.st_size, strippedName); + std::string ChecksumsSha256 = getDscHash(DoHashes, Hashes::SHA256SUM, Tags, pkgTagSection::Key::Checksums_Sha256, Db.HashesList.find("SHA256"), St.st_size, strippedName); + std::string ChecksumsSha512 = getDscHash(DoHashes, Hashes::SHA512SUM, Tags, pkgTagSection::Key::Checksums_Sha512, Db.HashesList.find("SHA512"), St.st_size, strippedName); + + // Strip the DirStrip prefix from the FileName and add the PathPrefix + string NewFileName; + if (DirStrip.empty() == false && + FileName.length() > DirStrip.length() && + stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0) + NewFileName = string(OriginalPath + DirStrip.length()); + else + NewFileName = OriginalPath; + if (PathPrefix.empty() == false) + NewFileName = flCombine(PathPrefix,NewFileName); + + string Directory = flNotFile(OriginalPath); + + // Perform operation over all of the files + string ParseJnk; + const char *C = Files.c_str(); + char *RealPath = NULL; + for (;isspace(*C); C++); + while (*C != 0) + { + // Parse each of the elements + if (ParseQuoteWord(C,ParseJnk) == false || + ParseQuoteWord(C,ParseJnk) == false || + ParseQuoteWord(C,ParseJnk) == false) + return _error->Error("Error parsing file record"); + + string OriginalPath = Directory + ParseJnk; + + // Add missing hashes to source files + if (((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM && not Tags.Exists(pkgTagSection::Key::Checksums_Sha1)) || + ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM && not Tags.Exists(pkgTagSection::Key::Checksums_Sha256)) || + ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM && not Tags.Exists(pkgTagSection::Key::Checksums_Sha512))) + { + if (Db.GetFileInfo(OriginalPath, + false, /* DoControl */ + false, /* DoContents */ + false, /* GenContentsOnly */ + false, /* DoSource */ + DoHashes, + DoAlwaysStat) == false) + { + return _error->Error("Error getting file info"); + } + + for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs) + { + if (hs->HashType() == "MD5Sum" || hs->HashType() == "Checksum-FileSize") + continue; + pkgTagSection::Key fieldkey; + std::string * out; + if (hs->HashType() == "SHA1") + { + fieldkey = pkgTagSection::Key::Checksums_Sha1; + out = &ChecksumsSha1; + } + else if (hs->HashType() == "SHA256") + { + fieldkey = pkgTagSection::Key::Checksums_Sha256; + out = &ChecksumsSha256; + } + else if (hs->HashType() == "SHA512") + { + fieldkey = pkgTagSection::Key::Checksums_Sha512; + out = &ChecksumsSha512; + } + else + { + _error->Warning("Ignoring unknown Checksumtype %s in SourcesWriter::DoPackages", hs->HashType().c_str()); + continue; + } + if (Tags.Exists(fieldkey)) + continue; + std::ostringstream streamout; + streamout << "\n " << hs->HashValue() << " " << std::to_string(Db.GetFileSize()) << " " << ParseJnk; + out->append(streamout.str()); + } + + // write back the GetFileInfo() stats data + Db.Finish(); + } + + // Perform the delinking operation + char Jnk[2]; + + if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 && + (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0) + { + string RP = RealPath; + free(RealPath); + if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false) + return false; + } + } + + Directory = flNotFile(NewFileName); + if (Directory.length() > 2) + Directory.erase(Directory.end()-1); + + // This lists all the changes to the fields we are going to make. + // (5 hardcoded + checksums + maintainer + end marker) + std::vector<pkgTagSection::Tag> Changes; + + Changes.push_back(pkgTagSection::Tag::Remove("Source")); + Changes.push_back(pkgTagSection::Tag::Rewrite("Package", Package)); + if (Files.empty() == false) + Changes.push_back(pkgTagSection::Tag::Rewrite("Files", Files)); + else + Changes.push_back(pkgTagSection::Tag::Remove("Files")); + if (ChecksumsSha1.empty() == false) + Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha1", ChecksumsSha1)); + else + Changes.push_back(pkgTagSection::Tag::Remove("Checksums-Sha1")); + if (ChecksumsSha256.empty() == false) + Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha256", ChecksumsSha256)); + else + Changes.push_back(pkgTagSection::Tag::Remove("Checksums-Sha256")); + if (ChecksumsSha512.empty() == false) + Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha512", ChecksumsSha512)); + else + Changes.push_back(pkgTagSection::Tag::Remove("Checksums-Sha512")); + if (Directory != "./") + Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Directory)); + Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", BestPrio)); + Changes.push_back(pkgTagSection::Tag::Remove("Status")); + + // Rewrite the maintainer field if necessary + bool MaintFailed; + string NewMaint = OverItem->SwapMaint(Tags.Find(pkgTagSection::Key::Maintainer).to_string(), MaintFailed); + if (MaintFailed == true) + { + if (NoOverride == false) + { + NewLine(1); + ioprintf(c1out, _(" %s maintainer is %s not %s\n"), Package.c_str(), + Tags.Find(pkgTagSection::Key::Maintainer).to_string().c_str(), OverItem->OldMaint.c_str()); + } + } + if (NewMaint.empty() == false) + Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint.c_str())); + + for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin(); + I != SOverItem->FieldOverride.end(); ++I) + Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second)); + + // Rewrite and store the fields. + if (Tags.Write(*Output, TFRewriteSourceOrder, Changes) == false || + Output->Write("\n", 1) == false) + return false; + + Stats.Packages++; + + return true; +} + /*}}}*/ + +// ContentsWriter::ContentsWriter - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +ContentsWriter::ContentsWriter(FileFd * const GivenOutput, string const &DB, + string const &Arch, bool const IncludeArchAll) : + FTWScanner(GivenOutput, Arch, IncludeArchAll), Db(DB), Stats(Db.Stats) + +{ + SetExts(".deb"); +} + /*}}}*/ +// ContentsWriter::DoPackage - Process a single package /*{{{*/ +// --------------------------------------------------------------------- +/* If Package is the empty string the control record will be parsed to + determine what the package name is. */ +bool ContentsWriter::DoPackage(string FileName, string Package) +{ + if (!Db.GetFileInfo(FileName, + Package.empty(), /* DoControl */ + true, /* DoContents */ + false, /* GenContentsOnly */ + false, /* DoSource */ + 0, /* DoHashes */ + false /* checkMtime */)) + { + return false; + } + + // Parse the package name + if (Package.empty() == true) + { + Package = Db.Control.Section.Find(pkgTagSection::Key::Package).to_string(); + } + + Db.Contents.Add(Gen,Package); + + return Db.Finish(); +} + /*}}}*/ +// ContentsWriter::ReadFromPkgs - Read from a packages file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress) +{ + MultiCompress Pkgs(PkgFile,PkgCompress,0,false); + if (_error->PendingError() == true) + return false; + + // Open the package file + FileFd Fd; + if (Pkgs.OpenOld(Fd) == false) + return false; + + pkgTagFile Tags(&Fd); + if (_error->PendingError() == true) + return false; + + // Parse. + pkgTagSection Section; + while (Tags.Step(Section) == true) + { + auto File = flCombine(Prefix, Section.Find(pkgTagSection::Key::Filename).to_string()); + auto Package = flCombine(Section.Find(pkgTagSection::Key::Section).to_string(), Section.Find(pkgTagSection::Key::Package).to_string()); + DoPackage(std::move(File), std::move(Package)); + if (_error->empty() == false) + { + _error->Error("Errors apply to file '%s'",File.c_str()); + _error->DumpErrors(); + } + } + + // Tidy the compressor + Fd.Close(); + + return true; +} + + /*}}}*/ + +// ReleaseWriter::ReleaseWriter - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +static std::string formatUTCDateTime(time_t const now) +{ + bool const NumericTimezone = _config->FindB("APT::FTPArchive::Release::NumericTimezone", true); + // TimeRFC1123 uses GMT to satisfy HTTP/1.1 + std::string datetime = TimeRFC1123(now, NumericTimezone); + if (NumericTimezone == false) + { + auto const lastspace = datetime.rfind(' '); + if (likely(lastspace != std::string::npos)) + datetime.replace(lastspace + 1, 3, "UTC"); + } + return datetime; +} +ReleaseWriter::ReleaseWriter(FileFd * const GivenOutput, string const &/*DB*/) : FTWScanner(GivenOutput) +{ + if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true) + { + AddPattern("Packages"); + AddPattern("Packages.*"); + AddPattern("Translation-*"); + AddPattern("Sources"); + AddPattern("Sources.*"); + AddPattern("Release"); + AddPattern("Contents-*"); + AddPattern("Index"); + AddPattern("Index.*"); + AddPattern("icons-*.tar"); + AddPattern("icons-*.tar.*"); + AddPattern("Components-*.yml"); + AddPattern("Components-*.yml.*"); + AddPattern("md5sum.txt"); + } + AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns")); + + time_t const now = time(NULL); + time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0); + + map<string,bool> BoolFields; + map<string,string> Fields; + Fields["Origin"] = ""; + Fields["Label"] = ""; + Fields["Suite"] = ""; + Fields["Version"] = ""; + Fields["Codename"] = ""; + Fields["Date"] = formatUTCDateTime(now); + if (validuntil != now) + Fields["Valid-Until"] = formatUTCDateTime(validuntil); + Fields["Architectures"] = ""; + Fields["Components"] = ""; + Fields["Description"] = ""; + Fields["Signed-By"] = ""; + BoolFields["Acquire-By-Hash"] = _config->FindB("APT::FTPArchive::DoByHash", false); + BoolFields["NotAutomatic"] = false; + BoolFields["ButAutomaticUpgrades"] = false; + + // Read configuration for string fields, but don't output them + for (auto &&I : Fields) + { + string Config = string("APT::FTPArchive::Release::") + I.first; + I.second = _config->Find(Config, I.second); + } + + // Read configuration for bool fields, and add them to Fields if true + for (auto &&I : BoolFields) + { + string Config = string("APT::FTPArchive::Release::") + I.first; + I.second = _config->FindB(Config, I.second); + if (I.second) + Fields[I.first] = "yes"; + } + + // All configuration read and stored in Fields; output + for (auto &&I : Fields) + { + if (I.second.empty()) + continue; + std::string const out = I.first + ": " + I.second + "\n"; + Output->Write(out.c_str(), out.length()); + } + + ConfigToDoHashes(DoHashes, "APT::FTPArchive::Release"); +} + /*}}}*/ +// ReleaseWriter::DoPackage - Process a single package /*{{{*/ +// --------------------------------------------------------------------- +bool ReleaseWriter::DoPackage(string FileName) +{ + // Strip the DirStrip prefix from the FileName and add the PathPrefix + string NewFileName; + if (DirStrip.empty() == false && + FileName.length() > DirStrip.length() && + stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(), + DirStrip.begin(),DirStrip.end()) == 0) + { + NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end()); + while (NewFileName[0] == '/') + NewFileName = string(NewFileName.begin() + 1,NewFileName.end()); + } + else + NewFileName = FileName; + + if (PathPrefix.empty() == false) + NewFileName = flCombine(PathPrefix,NewFileName); + + FileFd fd(FileName, FileFd::ReadOnly); + + if (!fd.IsOpen()) + { + return false; + } + + CheckSums[NewFileName].size = fd.Size(); + + Hashes hs(DoHashes); + hs.AddFD(fd); + CheckSums[NewFileName].Hashes = hs.GetHashStringList(); + fd.Close(); + + // FIXME: wrong layer in the code(?) + // FIXME2: symlink instead of create a copy + if (_config->FindB("APT::FTPArchive::DoByHash", false) == true) + { + std::string Input = FileName; + HashStringList hsl = hs.GetHashStringList(); + for(HashStringList::const_iterator h = hsl.begin(); + h != hsl.end(); ++h) + { + if (!h->usable()) + continue; + if (flNotDir(FileName) == "Release" || flNotDir(FileName) == "InRelease") + continue; + + std::string ByHashOutputFile = GenByHashFilename(Input, *h); + std::string ByHashOutputDir = flNotFile(ByHashOutputFile); + if(!CreateDirectory(flNotFile(Input), ByHashOutputDir)) + return _error->Warning("can not create dir %s", flNotFile(ByHashOutputFile).c_str()); + + // write new hashes + FileFd In(Input, FileFd::ReadOnly); + FileFd Out(ByHashOutputFile, FileFd::WriteEmpty); + if(!CopyFile(In, Out)) + return _error->Warning("failed to copy %s %s", Input.c_str(), ByHashOutputFile.c_str()); + } + } + + return true; +} + + /*}}}*/ +// ReleaseWriter::Finish - Output the checksums /*{{{*/ +// --------------------------------------------------------------------- +static void printChecksumTypeRecord(FileFd &Output, char const * const Type, map<string, ReleaseWriter::CheckSum> const &CheckSums) +{ + { + std::string out; + strprintf(out, "%s:\n", Type); + Output.Write(out.c_str(), out.length()); + } + for(map<string,ReleaseWriter::CheckSum>::const_iterator I = CheckSums.begin(); + I != CheckSums.end(); ++I) + { + HashString const * const hs = I->second.Hashes.find(Type); + if (hs == NULL) + continue; + std::string out; + strprintf(out, " %s %16llu %s\n", + hs->HashValue().c_str(), + (*I).second.size, + (*I).first.c_str()); + Output.Write(out.c_str(), out.length()); + } +} +void ReleaseWriter::Finish() +{ + if ((DoHashes & Hashes::MD5SUM) == Hashes::MD5SUM) + printChecksumTypeRecord(*Output, "MD5Sum", CheckSums); + if ((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM) + printChecksumTypeRecord(*Output, "SHA1", CheckSums); + if ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM) + printChecksumTypeRecord(*Output, "SHA256", CheckSums); + if ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM) + printChecksumTypeRecord(*Output, "SHA512", CheckSums); + + // go by-hash cleanup + map<string,ReleaseWriter::CheckSum>::const_iterator prev = CheckSums.begin(); + if (_config->FindB("APT::FTPArchive::DoByHash", false) == true) + { + for(map<string,ReleaseWriter::CheckSum>::const_iterator I = CheckSums.begin(); + I != CheckSums.end(); ++I) + { + if (I->first == "Release" || I->first == "InRelease") + continue; + + // keep iterating until we find a new subdir + if(flNotFile(I->first) == flNotFile(prev->first)) + continue; + + // clean that subdir up + int keepFiles = _config->FindI("APT::FTPArchive::By-Hash-Keep", 3); + // calculate how many compressors are used (the amount of files + // in that subdir generated for this run) + keepFiles *= std::distance(prev, I); + prev = I; + + HashStringList hsl = prev->second.Hashes; + for(HashStringList::const_iterator h = hsl.begin(); + h != hsl.end(); ++h) + { + + if (!h->usable()) + continue; + + std::string RealFilename = DirStrip+"/"+prev->first; + std::string ByHashOutputFile = GenByHashFilename(RealFilename, *h); + DeleteAllButMostRecent(flNotFile(ByHashOutputFile), keepFiles); + } + } + } +} |