Adding upstream version 1.8.2.3.upstream/1.8.2.3upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1058 insertions, 0 deletions

diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
new file mode 100644
index 0000000..077701c
--- /dev/null
+++ b/ftparchive/apt-ftparchive.cc
@@ -0,0 +1,1058 @@
+// -*- mode: cpp; mode: fold -*-
+// Description								/*{{{*/
+/* ######################################################################
+
+   apt-ftparchive - Efficient work-alike for dpkg-scanpackages
+
+   Let contents be disabled from the conf
+   
+   ##################################################################### */
+									/*}}}*/
+// Include Files							/*{{{*/
+#include <config.h>
+
+#include <apt-pkg/cmndline.h>
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/init.h>
+#include <apt-pkg/strutl.h>
+
+#include <apt-private/private-cmndline.h>
+#include <apt-private/private-main.h>
+#include <apt-private/private-output.h>
+
+#include <algorithm>
+#include <chrono>
+#include <climits>
+#include <functional>
+#include <iostream>
+#include <string>
+#include <vector>
+#include <locale.h>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <time.h>
+#include <math.h>
+
+#include "apt-ftparchive.h"
+#include "cachedb.h"
+#include "multicompress.h"
+#include "override.h"
+#include "writer.h"
+
+#include <apti18n.h>
+									/*}}}*/
+
+using namespace std;
+unsigned Quiet = 0;
+
+static struct timeval GetTimevalFromSteadyClock()			/*{{{*/
+{
+   auto const Time = std::chrono::steady_clock::now().time_since_epoch();
+   auto const Time_sec = std::chrono::duration_cast<std::chrono::seconds>(Time);
+   auto const Time_usec = std::chrono::duration_cast<std::chrono::microseconds>(Time - Time_sec);
+   return { Time_sec.count(), Time_usec.count() };
+}
+									/*}}}*/
+
+// struct PackageMap - List of all package files in the config file	/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+struct PackageMap
+{
+   // General Stuff
+   string BaseDir;
+   string InternalPrefix;
+   string FLFile;
+   string PkgExt;
+   string SrcExt;
+   
+   // Stuff for the Package File
+   string PkgFile;
+   string BinCacheDB;
+   string SrcCacheDB;
+   string BinOverride;
+   string ExtraOverride;
+
+   // We generate for this given arch
+   string Arch;
+   bool IncludeArchAll;
+   
+   // Stuff for the Source File
+   string SrcFile;
+   string SrcOverride;
+   string SrcExtraOverride;
+
+   // Translation master file
+   bool LongDesc;
+   TranslationWriter *TransWriter;
+
+   // Contents 
+   string Contents;
+   string ContentsHead;
+   
+   // Random things
+   string Tag;
+   string PkgCompress;
+   string CntCompress;
+   string SrcCompress;
+   string PathPrefix;
+   unsigned int DeLinkLimit;
+   mode_t Permissions;
+   
+   bool ContentsDone;
+   bool PkgDone;
+   bool SrcDone;
+   time_t ContentsMTime;
+   
+   struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
+   {
+      inline bool operator() (const PackageMap &x,const PackageMap &y)
+      {return x.ContentsMTime < y.ContentsMTime;};
+   };
+    
+   struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
+   {
+      inline bool operator() (const PackageMap &x,const PackageMap &y)
+      {return x.BinCacheDB < y.BinCacheDB;};
+   };  
+
+   struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
+   {
+      inline bool operator() (const PackageMap &x,const PackageMap &y)
+      {return x.SrcCacheDB < y.SrcCacheDB;};
+   };
+   
+   void GetGeneral(Configuration &Setup,Configuration &Block);
+   bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
+   bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
+   bool GenContents(Configuration &Setup,
+		    vector<PackageMap>::iterator Begin,
+		    vector<PackageMap>::iterator End,
+		    unsigned long &Left);
+   
+   PackageMap() : IncludeArchAll(true), LongDesc(true), TransWriter(NULL),
+		  DeLinkLimit(0), Permissions(1), ContentsDone(false),
+		  PkgDone(false), SrcDone(false), ContentsMTime(0) {};
+};
+									/*}}}*/
+
+// PackageMap::GetGeneral - Common per-section definitions		/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
+{
+   PathPrefix = Block.Find("PathPrefix");
+   
+   if (Block.FindB("External-Links",true) == false)
+      DeLinkLimit = Setup.FindI("Default::DeLinkLimit", std::numeric_limits<unsigned int>::max());
+   else
+      DeLinkLimit = 0;
+   
+   PkgCompress = Block.Find("Packages::Compress",
+			    Setup.Find("Default::Packages::Compress",". gzip").c_str());
+   CntCompress = Block.Find("Contents::Compress",
+			    Setup.Find("Default::Contents::Compress",". gzip").c_str());
+   SrcCompress = Block.Find("Sources::Compress",
+			    Setup.Find("Default::Sources::Compress",". gzip").c_str());
+   
+   SrcExt = Block.Find("Sources::Extensions",
+		       Setup.Find("Default::Sources::Extensions",".dsc").c_str());
+   PkgExt = Block.Find("Packages::Extensions",
+		       Setup.Find("Default::Packages::Extensions",".deb").c_str());
+   
+   Permissions = Setup.FindI("Default::FileMode",0644);
+
+   if (FLFile.empty() == false)
+      FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
+   
+   if (Contents == " ")
+      Contents= string();   
+}
+									/*}}}*/
+// PackageMap::GenPackages - Actually generate a Package file		/*{{{*/
+// ---------------------------------------------------------------------
+/* This generates the Package File described by this object. */
+bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
+{   
+   if (PkgFile.empty() == true)
+      return true;
+   
+   string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
+   string OverrideDir = Setup.FindDir("Dir::OverrideDir");
+   string CacheDir = Setup.FindDir("Dir::CacheDir");
+
+   struct timeval StartTime = GetTimevalFromSteadyClock();
+
+   PkgDone = true;
+   
+   // Create a package writer object.
+   MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
+		      PkgCompress,Permissions);
+   PackagesWriter Packages(&Comp.Input, TransWriter, flCombine(CacheDir,BinCacheDB),
+			   flCombine(OverrideDir,BinOverride),
+			   flCombine(OverrideDir,ExtraOverride),
+			   Arch, IncludeArchAll);
+   if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
+      return _error->Error(_("Package extension list is too long"));
+   if (_error->PendingError() == true)
+      return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
+   
+   Packages.PathPrefix = PathPrefix;
+   Packages.DirStrip = ArchiveDir;
+   Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
+
+   Packages.LongDescription = LongDesc;
+
+   Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
+   Packages.DeLinkLimit = DeLinkLimit;
+
+   if (_error->PendingError() == true)
+      return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
+   
+   c0out << ' ' << BaseDir << ":" << flush;
+   
+   // Do recursive directory searching
+   if (FLFile.empty() == true)
+   {
+      if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
+	 return false;
+   }
+   else
+   {
+      if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
+	 return false;
+   }
+   
+   Packages.Output = 0;      // Just in case
+   
+   // Finish compressing
+   unsigned long long Size;
+   if (Comp.Finalize(Size) == false)
+   {
+      c0out << endl;
+      return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
+   }
+   
+   if (Size != 0)
+      c0out << " New "
+             << SizeToStr(Size) << "B ";
+   else
+      c0out << ' ';
+   
+   struct timeval NewTime = GetTimevalFromSteadyClock();
+   std::chrono::duration<double> Delta =
+      std::chrono::seconds(NewTime.tv_sec - StartTime.tv_sec) +
+      std::chrono::microseconds(NewTime.tv_sec - StartTime.tv_usec);
+
+   c0out << Packages.Stats.Packages << " files " <<
+/*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
+      SizeToStr(Packages.Stats.Bytes) << "B " <<
+      TimeToStr(llround(Delta.count())) << endl;
+
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
+   
+   Stats.Add(Packages.Stats);
+   Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
+   
+   return !_error->PendingError();
+}
+
+									/*}}}*/
+// PackageMap::GenSources - Actually generate a Source file		/*{{{*/
+// ---------------------------------------------------------------------
+/* This generates the Sources File described by this object. */
+bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
+{   
+   if (SrcFile.empty() == true)
+      return true;
+   
+   string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
+   string OverrideDir = Setup.FindDir("Dir::OverrideDir");
+   string CacheDir = Setup.FindDir("Dir::CacheDir");
+
+   struct timeval StartTime = GetTimevalFromSteadyClock();
+   SrcDone = true;
+   
+   // Create a package writer object.
+   MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
+		      SrcCompress,Permissions);
+   SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
+			 flCombine(OverrideDir,BinOverride),
+			 flCombine(OverrideDir,SrcOverride),
+			 flCombine(OverrideDir,SrcExtraOverride));
+   if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
+      return _error->Error(_("Source extension list is too long"));
+   if (_error->PendingError() == true)
+      return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
+   
+   Sources.PathPrefix = PathPrefix;
+   Sources.DirStrip = ArchiveDir;
+   Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
+
+   Sources.DeLinkLimit = DeLinkLimit;
+   Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
+
+   if (_error->PendingError() == true)
+      return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
+
+   c0out << ' ' << BaseDir << ":" << flush;
+   
+   // Do recursive directory searching
+   if (FLFile.empty() == true)
+   {
+      if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
+	 return false;
+   }   
+   else
+   {
+      if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
+	 return false;
+   }
+   Sources.Output = 0;      // Just in case
+   
+   // Finish compressing
+   unsigned long long Size;
+   if (Comp.Finalize(Size) == false)
+   {
+      c0out << endl;
+      return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
+   }
+      
+   if (Size != 0)
+      c0out << " New "
+             << SizeToStr(Size) << "B ";
+   else
+      c0out << ' ';
+   
+   struct timeval NewTime = GetTimevalFromSteadyClock();
+   std::chrono::duration<double> Delta =
+      std::chrono::seconds(NewTime.tv_sec - StartTime.tv_sec) +
+      std::chrono::microseconds(NewTime.tv_sec - StartTime.tv_usec);
+   
+   c0out << Sources.Stats.Packages << " pkgs in " <<
+      TimeToStr(llround(Delta.count())) << endl;
+
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
+
+   Stats.Add(Sources.Stats);
+   Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
+   
+   return !_error->PendingError();
+}
+									/*}}}*/
+// PackageMap::GenContents - Actually generate a Contents file		/*{{{*/
+// ---------------------------------------------------------------------
+/* This generates the contents file partially described by this object.
+   It searches the given iterator range for other package files that map
+   into this contents file and includes their data as well when building. */
+bool PackageMap::GenContents(Configuration &Setup,
+			     vector<PackageMap>::iterator Begin,
+			     vector<PackageMap>::iterator End,
+			     unsigned long &Left)
+{
+   if (Contents.empty() == true)
+      return true;
+   
+   if (Left == 0)
+      return true;
+   
+   string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
+   string CacheDir = Setup.FindDir("Dir::CacheDir");
+   string OverrideDir = Setup.FindDir("Dir::OverrideDir");
+
+   struct timeval StartTime = GetTimevalFromSteadyClock();
+
+   // Create a package writer object.
+   MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
+		      CntCompress,Permissions);
+   Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
+   ContentsWriter Contents(&Comp.Input, "", Arch, IncludeArchAll);
+   if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
+      return _error->Error(_("Package extension list is too long"));
+   if (_error->PendingError() == true)
+      return false;
+
+   if (_error->PendingError() == true)
+      return false;
+
+   // Write the header out.
+   if (ContentsHead.empty() == false)
+   {
+      FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
+      if (_error->PendingError() == true)
+	 return false;
+
+      unsigned long long Size = Head.Size();
+      unsigned char Buf[4096];
+      while (Size != 0)
+      {
+	 unsigned long long ToRead = Size;
+	 if (Size > sizeof(Buf))
+	    ToRead = sizeof(Buf);
+
+	 if (Head.Read(Buf,ToRead) == false)
+	    return false;
+
+	 if (Comp.Input.Write(Buf, ToRead) == false)
+	    return _error->Errno("fwrite",_("Error writing header to contents file"));
+
+	 Size -= ToRead;
+      }
+   }
+
+   /* Go over all the package file records and parse all the package
+      files associated with this contents file into one great big honking
+      memory structure, then dump the sorted version */
+   c0out << ' ' << this->Contents << ":" << flush;
+   for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
+   {
+      if (I->Contents != this->Contents)
+	 continue;
+      
+      Contents.Prefix = ArchiveDir;
+      Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
+      Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
+			    I->PkgCompress);
+      
+      I->ContentsDone = true;	    
+   }
+   
+   Contents.Finish();
+   
+   // Finish compressing
+   unsigned long long Size;
+   if (Comp.Finalize(Size) == false || _error->PendingError() == true)
+   {
+      c0out << endl;
+      return _error->Error(_("Error processing contents %s"),
+			   this->Contents.c_str());
+   }
+   
+   if (Size != 0)
+   {
+      c0out << " New " << SizeToStr(Size) << "B ";
+      if (Left > Size)
+	 Left -= Size;
+      else
+	 Left = 0;
+   }
+   else
+      c0out << ' ';
+   
+   struct timeval NewTime = GetTimevalFromSteadyClock();
+   std::chrono::duration<double> Delta =
+      std::chrono::seconds(NewTime.tv_sec - StartTime.tv_sec) +
+      std::chrono::microseconds(NewTime.tv_sec - StartTime.tv_usec);
+   
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
+
+   c0out << Contents.Stats.Packages << " files " <<
+      SizeToStr(Contents.Stats.Bytes) << "B " <<
+      TimeToStr(llround(Delta.count())) << endl;
+   
+   return true;
+}
+									/*}}}*/
+
+// LoadTree - Load a 'tree' section from the Generate Config		/*{{{*/
+// ---------------------------------------------------------------------
+/* This populates the PkgList with all the possible permutations of the
+   section/arch lists. */
+static void LoadTree(vector<PackageMap> &PkgList, std::vector<TranslationWriter*> &TransList, Configuration &Setup)
+{   
+   // Load the defaults
+   string DDir = Setup.Find("TreeDefault::Directory",
+			    "$(DIST)/$(SECTION)/binary-$(ARCH)/");
+   string DSDir = Setup.Find("TreeDefault::SrcDirectory",
+			    "$(DIST)/$(SECTION)/source/");
+   string DPkg = Setup.Find("TreeDefault::Packages",
+			    "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
+   string DTrans = Setup.Find("TreeDefault::Translation",
+			    "$(DIST)/$(SECTION)/i18n/Translation-en");
+   string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
+			    "$(DIST)/$(SECTION)/");
+   string DContents = Setup.Find("TreeDefault::Contents",
+			    "$(DIST)/$(SECTION)/Contents-$(ARCH)");
+   string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
+   string DBCache = Setup.Find("TreeDefault::BinCacheDB",
+			       "packages-$(ARCH).db");
+   string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
+			       "sources-$(SECTION).db");
+   string DSources = Setup.Find("TreeDefault::Sources",
+				"$(DIST)/$(SECTION)/source/Sources");
+   string DFLFile = Setup.Find("TreeDefault::FileList", "");
+   string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
+
+   mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
+
+   bool const LongDescription = Setup.FindB("Default::LongDescription",
+					_config->FindB("APT::FTPArchive::LongDescription", true));
+   string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
+   bool const ConfIncludeArchAllExists = _config->Exists("APT::FTPArchive::IncludeArchitectureAll");
+   bool const ConfIncludeArchAll = _config->FindB("APT::FTPArchive::IncludeArchitectureAll", true);
+
+   // Process 'tree' type sections
+   const Configuration::Item *Top = Setup.Tree("tree");
+   for (Top = (Top == 0?0:Top->Child); Top != 0;)
+   {
+      Configuration Block(Top);
+      string Dist = Top->Tag;
+
+      // Parse the sections
+      string Tmp = Block.Find("Sections");
+      const char *Sections = Tmp.c_str();
+      string Section;
+      while (ParseQuoteWord(Sections,Section) == true)
+      {
+	 struct SubstVar Vars[] = {{"$(DIST)",&Dist},
+					 {"$(SECTION)",&Section},
+					 {"$(ARCH)",nullptr},
+					 {nullptr, nullptr}};
+	 mode_t const Perms = Block.FindI("FileMode", Permissions);
+	 bool const LongDesc = Block.FindB("LongDescription", LongDescription);
+	 TranslationWriter *TransWriter = nullptr;
+
+	 std::string Tmp2 = Block.Find("Architectures");
+	 std::transform(Tmp2.begin(), Tmp2.end(), Tmp2.begin(), ::tolower);
+	 std::vector<std::string> const Archs = VectorizeString(Tmp2, ' ');
+	 bool IncludeArchAll;
+	 if (ConfIncludeArchAllExists == true)
+	    IncludeArchAll = ConfIncludeArchAll;
+	 else
+	    IncludeArchAll = std::find(Archs.begin(), Archs.end(), "all") == Archs.end();
+	 for (auto const& Arch: Archs)
+	 {
+	    if (Arch.empty()) continue;
+	    Vars[2].Contents = &Arch;
+	    PackageMap Itm;
+	    Itm.Permissions = Perms;
+	    Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
+	    Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
+
+	    if (Arch == "source")
+	    {
+	       Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
+	       Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
+	       Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
+	       Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
+	       Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
+	       Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
+	       Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
+	    }
+	    else
+	    {
+	       Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
+	       Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
+	       Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
+	       Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
+	       Itm.Arch = Arch;
+	       Itm.IncludeArchAll = IncludeArchAll;
+	       Itm.LongDesc = LongDesc;
+	       if (TransWriter == NULL && DTrans.empty() == false && LongDesc == false && DTrans != "/dev/null")
+	       {
+		  string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
+			SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
+		  string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
+		  TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
+		  TransList.push_back(TransWriter);
+	       }
+	       Itm.TransWriter = TransWriter;
+	       Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
+	       Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
+	       Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
+	       Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
+	    }
+
+	    Itm.GetGeneral(Setup,Block);
+	    PkgList.push_back(Itm);
+	 }
+      }
+
+      Top = Top->Next;
+   }
+}
+									/*}}}*/
+static void UnloadTree(std::vector<TranslationWriter*> const &Trans)		/*{{{*/
+{
+   for (std::vector<TranslationWriter*>::const_reverse_iterator T = Trans.rbegin(); T != Trans.rend(); ++T)
+      delete *T;
+}
+									/*}}}*/
+// LoadBinDir - Load a 'bindirectory' section from the Generate Config	/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
+{
+   mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
+
+   // Process 'bindirectory' type sections
+   const Configuration::Item *Top = Setup.Tree("bindirectory");
+   for (Top = (Top == 0?0:Top->Child); Top != 0;)
+   {
+      Configuration Block(Top);
+      
+      PackageMap Itm;
+      Itm.PkgFile = Block.Find("Packages");
+      Itm.SrcFile = Block.Find("Sources");
+      Itm.BinCacheDB = Block.Find("BinCacheDB");
+      Itm.SrcCacheDB = Block.Find("SrcCacheDB");
+      Itm.BinOverride = Block.Find("BinOverride");
+      Itm.ExtraOverride = Block.Find("ExtraOverride");
+      Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
+      Itm.SrcOverride = Block.Find("SrcOverride");
+      Itm.BaseDir = Top->Tag;
+      Itm.FLFile = Block.Find("FileList");
+      Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
+      Itm.Contents = Block.Find("Contents");
+      Itm.ContentsHead = Block.Find("Contents::Header");
+      Itm.Permissions = Block.FindI("FileMode", Permissions);
+      
+      Itm.GetGeneral(Setup,Block);
+      PkgList.push_back(Itm);
+      
+      Top = Top->Next;
+   }      
+}
+									/*}}}*/
+
+static bool ShowHelp(CommandLine &)					/*{{{*/
+{
+   std::cout <<
+    _("Usage: apt-ftparchive [options] command\n"
+      "Commands: packages binarypath [overridefile [pathprefix]]\n"
+      "          sources srcpath [overridefile [pathprefix]]\n"
+      "          contents path\n"
+      "          release path\n"
+      "          generate config [groups]\n"
+      "          clean config\n"
+      "\n"
+      "apt-ftparchive generates index files for Debian archives. It supports\n"
+      "many styles of generation from fully automated to functional replacements\n"
+      "for dpkg-scanpackages and dpkg-scansources\n"
+      "\n"
+      "apt-ftparchive generates Package files from a tree of .debs. The\n"
+      "Package file contains the contents of all the control fields from\n"
+      "each package as well as the MD5 hash and filesize. An override file\n"
+      "is supported to force the value of Priority and Section.\n"
+      "\n"
+      "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
+      "The --source-override option can be used to specify a src override file\n"
+      "\n"
+      "The 'packages' and 'sources' command should be run in the root of the\n"
+      "tree. BinaryPath should point to the base of the recursive search and \n"
+      "override file should contain the override flags. Pathprefix is\n"
+      "appended to the filename fields if present. Example usage from the \n"
+      "Debian archive:\n"
+      "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
+      "               dists/potato/main/binary-i386/Packages\n"
+      "\n"
+      "Options:\n"
+      "  -h    This help text\n"
+      "  --md5 Control MD5 generation\n"
+      "  -s=?  Source override file\n"
+      "  -q    Quiet\n"
+      "  -d=?  Select the optional caching database\n"
+      "  --no-delink Enable delinking debug mode\n"
+      "  --contents  Control contents file generation\n"
+      "  -c=?  Read this configuration file\n"
+      "  -o=?  Set an arbitrary configuration option") << endl;
+   return true;
+}
+									/*}}}*/
+// SimpleGenPackages - Generate a Packages file for a directory tree	/*{{{*/
+// ---------------------------------------------------------------------
+/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
+static bool SimpleGenPackages(CommandLine &CmdL)
+{
+   if (CmdL.FileSize() < 2)
+      return ShowHelp(CmdL);
+   
+   string Override;
+   if (CmdL.FileSize() >= 3)
+      Override = CmdL.FileList[2];
+   
+   // Create a package writer object.
+   PackagesWriter Packages(NULL, NULL, _config->Find("APT::FTPArchive::DB"),
+			   Override, "", _config->Find("APT::FTPArchive::Architecture"),
+			   _config->FindB("APT::FTPArchive::IncludeArchitectureAll", true));
+   if (_error->PendingError() == true)
+      return false;
+   
+   if (CmdL.FileSize() >= 4)
+      Packages.PathPrefix = CmdL.FileList[3];
+   
+   // Do recursive directory searching
+   if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
+      return false;
+
+   // Give some stats if asked for
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
+
+   return true;
+}
+									/*}}}*/
+// SimpleGenContents - Generate a Contents listing			/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+static bool SimpleGenContents(CommandLine &CmdL)
+{
+   if (CmdL.FileSize() < 2)
+      return ShowHelp(CmdL);
+   
+   // Create a package writer object.
+   ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
+   if (_error->PendingError() == true)
+      return false;
+   
+   // Do recursive directory searching
+   if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
+      return false;
+
+   Contents.Finish();
+   
+   return true;
+}
+									/*}}}*/
+// SimpleGenSources - Generate a Sources file for a directory tree	/*{{{*/
+// ---------------------------------------------------------------------
+/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
+static bool SimpleGenSources(CommandLine &CmdL)
+{
+   if (CmdL.FileSize() < 2)
+      return ShowHelp(CmdL);
+   
+   string Override;
+   if (CmdL.FileSize() >= 3)
+      Override = CmdL.FileList[2];
+   
+   string SOverride;
+   if (Override.empty() == false)
+      SOverride = Override + ".src";
+   
+   SOverride = _config->Find("APT::FTPArchive::SourceOverride",
+			     SOverride.c_str());
+       
+   // Create a package writer object.
+   SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
+   if (_error->PendingError() == true)
+      return false;
+   
+   if (CmdL.FileSize() >= 4)
+      Sources.PathPrefix = CmdL.FileList[3];
+   
+   // Do recursive directory searching
+   if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
+      return false;
+
+   // Give some stats if asked for
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
+
+   return true;
+}
+									/*}}}*/
+// SimpleGenRelease - Generate a Release file for a directory tree	/*{{{*/
+// ---------------------------------------------------------------------
+static bool SimpleGenRelease(CommandLine &CmdL)
+{
+   if (CmdL.FileSize() < 2)
+      return ShowHelp(CmdL);
+
+   string Dir = CmdL.FileList[1];
+
+   ReleaseWriter Release(NULL, "");
+   Release.DirStrip = Dir;
+
+   if (_error->PendingError() == true)
+      return false;
+
+   if (Release.RecursiveScan(Dir) == false)
+      return false;
+
+   Release.Finish();
+
+   return true;
+}
+
+									/*}}}*/
+// DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
+// ---------------------------------------------------------------------
+static bool DoGeneratePackagesAndSources(Configuration &Setup,
+					 vector<PackageMap> &PkgList,
+					 struct CacheDB::Stats &SrcStats,
+					 struct CacheDB::Stats &Stats,
+					 CommandLine &CmdL)
+{
+   if (CmdL.FileSize() <= 2)
+   {
+      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
+	 if (I->GenPackages(Setup,Stats) == false)
+	    _error->DumpErrors();
+      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
+	 if (I->GenSources(Setup,SrcStats) == false)
+	    _error->DumpErrors();
+   }
+   else
+   {
+      // Make a choice list out of the package list..
+      RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
+      RxChoiceList *End = List;
+      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
+      {
+	 End->UserData = &(*I);
+	 End->Str = I->BaseDir.c_str();
+	 End++;
+	 
+	 End->UserData = &(*I);
+	 End->Str = I->Tag.c_str();
+	 End++;	 
+      }
+      End->Str = 0;
+     
+      // Regex it
+      if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
+      {
+	 delete [] List;
+	 return _error->Error(_("No selections matched"));
+      }
+      _error->DumpErrors();
+      
+      // Do the generation for Packages
+      for (End = List; End->Str != 0; ++End)
+      {
+	 if (End->Hit == false)
+	    continue;
+	 
+	 PackageMap * const I = static_cast<PackageMap *>(End->UserData);
+	 if (I->PkgDone == true)
+	    continue;
+	 if (I->GenPackages(Setup,Stats) == false)
+	    _error->DumpErrors();
+      }
+      
+      // Do the generation for Sources
+      for (End = List; End->Str != 0; ++End)
+      {
+	 if (End->Hit == false)
+	    continue;
+	 
+	 PackageMap * const I = static_cast<PackageMap *>(End->UserData);
+	 if (I->SrcDone == true)
+	    continue;
+	 if (I->GenSources(Setup,SrcStats) == false)
+	    _error->DumpErrors();
+      }
+      
+      delete [] List;
+   }
+   return true;
+}
+
+                                                                        /*}}}*/
+// DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
+// ---------------------------------------------------------------------
+static bool DoGenerateContents(Configuration &Setup,
+			       vector<PackageMap> &PkgList,
+			       CommandLine &CmdL)
+{
+   c1out << "Packages done, Starting contents." << endl;
+
+   // Sort the contents file list by date
+   string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
+   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
+   {
+      struct stat A;
+      if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
+				 I->CntCompress,A) == false)
+	 time(&I->ContentsMTime);
+      else
+	 I->ContentsMTime = A.st_mtime;
+   }
+   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
+   
+   /* Now for Contents.. The process here is to do a make-like dependency
+      check. Each contents file is verified to be newer than the package files
+      that describe the debs it indexes. Since the package files contain 
+      hashes of the .debs this means they have not changed either so the 
+      contents must be up to date. */
+   unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",
+                                                 std::numeric_limits<unsigned int>::max())*1024;
+   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
+   {
+      // This record is not relevant
+      if (I->ContentsDone == true ||
+	  I->Contents.empty() == true)
+	 continue;
+
+      // Do not do everything if the user specified sections.
+      if (CmdL.FileSize() > 2 && I->PkgDone == false)
+	 continue;
+
+      struct stat A,B;
+      if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
+      {
+	 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
+	 {
+	    _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
+	    continue;
+	 }
+	 
+	 if (A.st_mtime > B.st_mtime)
+	    continue;
+      }
+      
+      if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
+			 MaxContentsChange) == false)
+	 _error->DumpErrors();
+      
+      // Hit the limit?
+      if (MaxContentsChange == 0)
+      {
+	 c1out << "Hit contents update byte limit" << endl;
+	 break;
+      }      
+   }
+
+   return true;
+}
+
+									/*}}}*/
+// Generate - Full generate, using a config file			/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+static bool Generate(CommandLine &CmdL)
+{
+   struct CacheDB::Stats SrcStats;
+   if (CmdL.FileSize() < 2)
+      return ShowHelp(CmdL);
+
+   struct timeval StartTime = GetTimevalFromSteadyClock();
+   struct CacheDB::Stats Stats;
+   
+   // Read the configuration file.
+   Configuration Setup;
+   if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
+      return false;
+
+   vector<PackageMap> PkgList;
+   std::vector<TranslationWriter*> TransList;
+   LoadTree(PkgList, TransList, Setup);
+   LoadBinDir(PkgList,Setup);
+
+   // Sort by cache DB to improve IO locality.
+   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
+   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
+
+   // Generate packages
+   if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
+   {
+      if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
+      {
+	 UnloadTree(TransList);
+         return false;
+      }
+   } else {
+      c1out << "Skipping Packages/Sources generation" << endl;
+   }
+
+   // do Contents if needed
+   if (_config->FindB("APT::FTPArchive::Contents", true) == true)
+      if (DoGenerateContents(Setup, PkgList, CmdL) == false)
+      {
+	 UnloadTree(TransList);
+	 return false;
+      }
+
+   struct timeval NewTime = GetTimevalFromSteadyClock();
+   std::chrono::duration<double> Delta =
+      std::chrono::seconds(NewTime.tv_sec - StartTime.tv_sec) +
+      std::chrono::microseconds(NewTime.tv_sec - StartTime.tv_usec);
+   c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
+         << " archives. Took " << TimeToStr(llround(Delta.count())) << endl;
+
+   UnloadTree(TransList);
+   return true;
+}
+
+                                                                        /*}}}*/
+// Clean - Clean out the databases					/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+static bool Clean(CommandLine &CmdL)
+{
+   if (CmdL.FileSize() != 2)
+      return ShowHelp(CmdL);
+
+   // Read the configuration file.
+   Configuration Setup;
+   if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
+      return false;
+   // we don't need translation creation here
+   Setup.Set("TreeDefault::Translation", "/dev/null");
+
+   vector<PackageMap> PkgList;
+   std::vector<TranslationWriter*> TransList;
+   LoadTree(PkgList, TransList, Setup);
+   LoadBinDir(PkgList,Setup);
+
+   // Sort by cache DB to improve IO locality.
+   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
+   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
+
+   string CacheDir = Setup.FindDir("Dir::CacheDir");
+
+   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
+   {
+      if(I->BinCacheDB != "")
+         c0out << I->BinCacheDB << endl;
+      if(I->SrcCacheDB != "")
+         c0out << I->SrcCacheDB << endl;
+      CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
+      CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
+      if (DB.Clean() == false)
+	 _error->DumpErrors();
+      if (DB_SRC.Clean() == false)
+	 _error->DumpErrors();
+
+      I = std::find_if(I, PkgList.end(),
+	    [&](PackageMap const &PM) { return PM.BinCacheDB != I->BinCacheDB || PM.SrcCacheDB != I->SrcCacheDB;
+      });
+   }
+
+   return true;
+}
+									/*}}}*/
+
+static std::vector<aptDispatchWithHelp> GetCommands()			/*{{{*/
+{
+   return {
+      {"packages",&SimpleGenPackages, nullptr},
+      {"contents",&SimpleGenContents, nullptr},
+      {"sources",&SimpleGenSources, nullptr},
+      {"release",&SimpleGenRelease, nullptr},
+      {"generate",&Generate, nullptr},
+      {"clean",&Clean, nullptr},
+      {nullptr, nullptr, nullptr}
+   };
+}
+									/*}}}*/
+int main(int argc, const char *argv[])					/*{{{*/
+{
+   // Parse the command line and initialize the package library
+   CommandLine CmdL;
+   auto const Cmds = ParseCommandLine(CmdL, APT_CMD::APT_FTPARCHIVE, &_config, NULL, argc, argv, ShowHelp, &GetCommands);
+
+   _config->CndSet("quiet",0);
+   Quiet = _config->FindI("quiet",0);
+   InitOutput(clog.rdbuf());
+
+   return DispatchCommandLine(CmdL, Cmds);
+}
+									/*}}}*/