summaryrefslogtreecommitdiffstats
path: root/remoterepository.c
diff options
context:
space:
mode:
Diffstat (limited to 'remoterepository.c')
-rw-r--r--remoterepository.c2103
1 files changed, 2103 insertions, 0 deletions
diff --git a/remoterepository.c b/remoterepository.c
new file mode 100644
index 0000000..ec1d726
--- /dev/null
+++ b/remoterepository.c
@@ -0,0 +1,2103 @@
+/* This file is part of "reprepro"
+ * Copyright (C) 2003,2004,2005,2007,2008,2009,2012 Bernhard R. Link
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA
+ */
+
+#include <config.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <unistd.h>
+#include <assert.h>
+#include <string.h>
+#include <errno.h>
+#include <dirent.h>
+
+#include "globals.h"
+#include "error.h"
+#include "ignore.h"
+#include "filecntl.h"
+#include "checksums.h"
+#include "mprintf.h"
+#include "dirs.h"
+#include "chunks.h"
+#include "names.h"
+#include "aptmethod.h"
+#include "signature.h"
+#include "readtextfile.h"
+#include "uncompression.h"
+#include "diffindex.h"
+#include "rredpatch.h"
+#include "remoterepository.h"
+
+/* This is code to handle lists from remote repositories.
+ Those are stored in the lists/ (or --listdir) directory
+ and needs some maintaince:
+
+ - cleaning (unneeded) lists from that directory
+ - deciding what to download from a remote repository
+ (needs knowledge what is there and what is there)
+ - in the future: implement ed to use remote .diffs
+*/
+
+struct remote_repository {
+ struct remote_repository *next, *prev;
+
+ /* repository is determined by pattern name currently.
+ * That might change if there is some safe way to combine
+ * some. (note that method options might make equally looking
+ * repositories different ones, so that is hard to decide).
+ *
+ * This is possible as pattern is not modifyable in options
+ * or method by the using distribution.
+ */
+ const char *name;
+ const char *method;
+ const char *fallback;
+ const struct strlist *config;
+
+ struct aptmethod *download;
+
+ struct remote_distribution *distributions;
+};
+static struct remote_repository *repositories = NULL;
+
+struct remote_distribution {
+ struct remote_distribution *next;
+
+ /* repository and suite uniquely identify it,
+ as the only thing the distribution can change is the suite.
+ Currently most of the other fields would also fit in the
+ remote_repository structure, but I plan to add new patters
+ allowing this by distribution...
+ */
+ struct remote_repository *repository;
+ char *suite;
+
+ /* flat repository */
+ bool flat; bool flatnonflatwarned;
+ char *suite_base_dir;
+
+ /* if true, do not download or check Release file */
+ bool ignorerelease;
+ /* hashes to ignore */
+ bool ignorehashes[cs_hashCOUNT];
+
+ /* linked list of key descriptions to check against, each must match */
+ struct signature_requirement *verify;
+
+ /* local copy of InRelease, Release and Release.gpg file,
+ * only set if available */
+ char *inreleasefile;
+ char *releasefile;
+ char *releasegpgfile;
+ const char *usedreleasefile;
+
+ /* filenames and checksums from the Release file */
+ struct checksumsarray remotefiles;
+
+ /* the index files we need */
+ struct remote_index *indices;
+
+ /* InRelease failed or requested not to be used */
+ bool noinrelease;
+};
+
+struct remote_index {
+ /* next index in remote distribution */
+ struct remote_index *next;
+
+ struct remote_distribution *from;
+
+ /* what to download? .gz better than .bz2? and so on */
+ struct encoding_preferences downloadas;
+
+ /* remote filename as to be found in Release file*/
+ char *filename_in_release;
+
+ /* the name without suffix in the lists/ dir */
+ char *cachefilename;
+ /* the basename of the above */
+ const char *cachebasename;
+
+ /* index in checksums for the different types, -1 = not avail */
+ int ofs[c_COUNT], diff_ofs;
+
+ /* index in requested download methods so we can continue later */
+ int lasttriedencoding;
+ /* the compression to be tried currently */
+ enum compression compression;
+
+ /* the old uncompressed file, so that it is only deleted
+ * when needed, to avoid losing it for a patch run */
+ /*@dependant@*/struct cachedlistfile *olduncompressed;
+ struct checksums *oldchecksums;
+
+ /* if using pdiffs, the content of the Packages.diff/Index: */
+ struct diffindex *diffindex;
+ /* the last patch queued to be applied */
+ char *patchfilename;
+ /*@dependant@*/const struct diffindex_patch *selectedpatch;
+ bool deletecompressedpatch;
+
+ bool queued;
+ bool needed;
+ bool got;
+};
+
+#define MAXPARTS 5
+struct cachedlistfile {
+ struct cachedlistfile *next;
+ const char *basefilename;
+ unsigned int partcount;
+ const char *parts[MAXPARTS];
+ /* might be used by some rule */
+ bool needed, deleted;
+ char fullfilename[];
+};
+
+
+static void remote_index_free(/*@only@*/struct remote_index *i) {
+ if (i == NULL)
+ return;
+ free(i->cachefilename);
+ free(i->patchfilename);
+ free(i->filename_in_release);
+ diffindex_free(i->diffindex);
+ checksums_free(i->oldchecksums);
+ free(i);
+}
+
+static void remote_distribution_free(/*@only@*/struct remote_distribution *d) {
+ if (d == NULL)
+ return;
+ free(d->suite);
+ signature_requirements_free(d->verify);
+ free(d->inreleasefile);
+ free(d->releasefile);
+ free(d->releasegpgfile);
+ free(d->suite_base_dir);
+ checksumsarray_done(&d->remotefiles);
+ while (d->indices != NULL) {
+ struct remote_index *h = d->indices;
+ d->indices = h->next;
+ remote_index_free(h);
+ }
+ free(d);
+}
+
+void remote_repository_free(struct remote_repository *remote) {
+ if (remote == NULL)
+ return;
+ while (remote->distributions != NULL) {
+ struct remote_distribution *h = remote->distributions;
+ remote->distributions = h->next;
+ remote_distribution_free(h);
+ }
+ if (remote->next != NULL)
+ remote->next->prev = remote->prev;
+ if (remote->prev != NULL)
+ remote->prev->next = remote->next;
+ free(remote);
+ return;
+}
+
+void cachedlistfile_freelist(struct cachedlistfile *c) {
+ while (c != NULL) {
+ struct cachedlistfile *n = c->next;
+ free(c);
+ c = n;
+ }
+}
+
+void cachedlistfile_deleteunneeded(const struct cachedlistfile *c) {
+ for (; c != NULL ; c = c->next) {
+ if (c->needed)
+ continue;
+ if (verbose >= 0)
+ printf("deleting %s\n", c->fullfilename);
+ deletefile(c->fullfilename);
+ }
+}
+
+static /*@null@*/ struct cachedlistfile *cachedlistfile_new(const char *basefilename, size_t len, size_t listdirlen) {
+ struct cachedlistfile *c;
+ size_t l;
+ char *p;
+ char ch;
+
+ c = malloc(sizeof(struct cachedlistfile) + listdirlen + 2*len + 3);
+ if (FAILEDTOALLOC(c))
+ return NULL;
+ c->next = NULL;
+ c->needed = false;
+ c->deleted = false;
+ p = c->fullfilename;
+ assert ((size_t)(p - (char*)c) <= sizeof(struct cachedlistfile));
+ memcpy(p, global.listdir, listdirlen);
+ p += listdirlen;
+ *(p++) = '/';
+ assert ((size_t)(p - c->fullfilename) == listdirlen + 1);
+ c->basefilename = p;
+ memcpy(p, basefilename, len); p += len;
+ *(p++) = '\0';
+ assert ((size_t)(p - c->fullfilename) == listdirlen + len + 2);
+
+ c->parts[0] = p;
+ c->partcount = 1;
+ l = len;
+ while (l-- > 0 && (ch = *(basefilename++)) != '\0') {
+ if (ch == '_') {
+ *(p++) = '\0';
+ if (c->partcount < MAXPARTS)
+ c->parts[c->partcount] = p;
+ c->partcount++;
+ } else if (ch == '%') {
+ char first, second;
+
+ if (len <= 1) {
+ c->partcount = 0;
+ return c;
+ }
+ first = *(basefilename++);
+ second = *(basefilename++);
+ if (first >= '0' && first <= '9')
+ *p = (first - '0') << 4;
+ else if (first >= 'a' && first <= 'f')
+ *p = (first - 'a' + 10) << 4;
+ else {
+ c->partcount = 0;
+ return c;
+ }
+ if (second >= '0' && second <= '9')
+ *p |= (second - '0');
+ else if (second >= 'a' && second <= 'f')
+ *p |= (second - 'a' + 10);
+ else {
+ c->partcount = 0;
+ return c;
+ }
+ p++;
+ } else
+ *(p++) = ch;
+ }
+ *(p++) = '\0';
+ assert ((size_t)(p - c->fullfilename) <= listdirlen + 2*len + 3);
+ return c;
+}
+
+retvalue cachedlists_scandir(/*@out@*/struct cachedlistfile **cachedfiles_p) {
+ struct cachedlistfile *cachedfiles = NULL, **next_p;
+ struct dirent *r;
+ size_t listdirlen = strlen(global.listdir);
+ DIR *dir;
+
+ // TODO: check if it is always created before...
+ dir = opendir(global.listdir);
+ if (dir == NULL) {
+ int e = errno;
+ fprintf(stderr,
+"Error %d opening directory '%s': %s!\n",
+ e, global.listdir, strerror(e));
+ return RET_ERRNO(e);
+ }
+ next_p = &cachedfiles;
+ while (true) {
+ size_t namelen;
+ int e;
+
+ errno = 0;
+ r = readdir(dir);
+ if (r == NULL) {
+ e = errno;
+ if (e == 0)
+ break;
+ /* this should not happen... */
+ e = errno;
+ fprintf(stderr, "Error %d reading dir '%s': %s!\n",
+ e, global.listdir, strerror(e));
+ (void)closedir(dir);
+ cachedlistfile_freelist(cachedfiles);
+ return RET_ERRNO(e);
+ }
+ namelen = _D_EXACT_NAMLEN(r);
+ if (namelen == 1 && r->d_name[0] == '.')
+ continue;
+ if (namelen == 2 && r->d_name[0] == '.' && r->d_name[1] == '.')
+ continue;
+ *next_p = cachedlistfile_new(r->d_name, namelen, listdirlen);
+ if (FAILEDTOALLOC(*next_p)) {
+ (void)closedir(dir);
+ cachedlistfile_freelist(cachedfiles);
+ return RET_ERROR_OOM;
+ }
+ next_p = &(*next_p)->next;
+ }
+ if (closedir(dir) != 0) {
+ int e = errno;
+ fprintf(stderr, "Error %d closing directory '%s': %s!\n",
+ e, global.listdir, strerror(e));
+ cachedlistfile_freelist(cachedfiles);
+ return RET_ERRNO(e);
+ }
+ *cachedfiles_p = cachedfiles;
+ return RET_OK;
+}
+
+static retvalue cachedlistfile_delete(struct cachedlistfile *old) {
+ int e;
+ if (old->deleted)
+ return RET_OK;
+ e = deletefile(old->fullfilename);
+ if (e != 0)
+ return RET_ERRNO(e);
+ old->deleted = true;
+ return RET_OK;
+}
+
+struct remote_repository *remote_repository_prepare(const char *name, const char *method, const char *fallback, const struct strlist *config) {
+ struct remote_repository *n;
+
+ /* calling code ensures no two with the same name are created,
+ * so just create it... */
+
+ n = zNEW(struct remote_repository);
+ if (FAILEDTOALLOC(n))
+ return NULL;
+ n->name = name;
+ n->method = method;
+ n->fallback = fallback;
+ n->config = config;
+
+ n->next = repositories;
+ if (n->next != NULL)
+ n->next->prev = n;
+ repositories = n;
+
+ return n;
+}
+
+/* This escaping is quite harsh, but so nothing bad can happen... */
+static inline size_t escapedlen(const char *p) {
+ size_t l = 0;
+ if (*p == '-') {
+ l = 3;
+ p++;
+ }
+ while (*p != '\0') {
+ if ((*p < 'A' || *p > 'Z') && (*p < 'a' || *p > 'z') &&
+ (*p < '0' || *p > '9') && *p != '-')
+ l +=3;
+ else
+ l++;
+ p++;
+ }
+ return l;
+}
+
+static inline char *escapedcopy(char *dest, const char *orig) {
+ static char hex[16] = "0123456789ABCDEF";
+ if (*orig == '-') {
+ orig++;
+ *dest = '%'; dest++;
+ *dest = '2'; dest++;
+ *dest = 'D'; dest++;
+ }
+ while (*orig != '\0') {
+ if ((*orig < 'A' || *orig > 'Z')
+ && (*orig < 'a' || *orig > 'z')
+ && (*orig < '0' || *orig > '9')
+ && *orig != '-') {
+ *dest = '%'; dest++;
+ *dest = hex[(*orig >> 4)& 0xF ]; dest++;
+ *dest = hex[*orig & 0xF ]; dest++;
+ } else {
+ *dest = *orig;
+ dest++;
+ }
+ orig++;
+ }
+ return dest;
+}
+
+char *genlistsfilename(const char *type, unsigned int count, ...) {
+ const char *fields[count];
+ unsigned int i;
+ size_t listdir_len, type_len, len;
+ char *result, *p;
+ va_list ap;
+
+ len = 0;
+ va_start(ap, count);
+ for (i = 0 ; i < count ; i++) {
+ fields[i] = va_arg(ap, const char*);
+ assert (fields[i] != NULL);
+ len += escapedlen(fields[i]) + 1;
+ }
+ /* check sentinel */
+ assert (va_arg(ap, const char*) == NULL);
+ va_end(ap);
+ listdir_len = strlen(global.listdir);
+ if (type != NULL)
+ type_len = strlen(type);
+ else
+ type_len = 0;
+
+ result = malloc(listdir_len + type_len + len + 2);
+ if (FAILEDTOALLOC(result))
+ return NULL;
+ memcpy(result, global.listdir, listdir_len);
+ p = result + listdir_len;
+ *(p++) = '/';
+ for (i = 0 ; i < count ; i++) {
+ p = escapedcopy(p, fields[i]);
+ *(p++) = '_';
+ }
+ assert ((size_t)(p - result) == listdir_len + len + 1);
+ if (type != NULL)
+ memcpy(p, type, type_len + 1);
+ else
+ *(--p) = '\0';
+ return result;
+}
+
+void cachedlistfile_need(struct cachedlistfile *list, const char *type, unsigned int count, ...) {
+ struct cachedlistfile *file;
+ const char *fields[count];
+ unsigned int i;
+ va_list ap;
+
+ va_start(ap, count);
+ for (i = 0 ; i < count ; i++) {
+ fields[i] = va_arg(ap, const char*);
+ assert (fields[i] != NULL);
+ }
+ /* check sentinel */
+ assert (va_arg(ap, const char*) == NULL);
+ va_end(ap);
+
+ for (file = list ; file != NULL ; file = file->next) {
+ if (file->partcount != count + 1)
+ continue;
+ i = 0;
+ while (i < count && strcmp(file->parts[i], fields[i]) == 0)
+ i++;
+ if (i < count)
+ continue;
+ if (strcmp(type, file->parts[i]) != 0)
+ continue;
+ file->needed = true;
+ }
+}
+
+retvalue remote_distribution_prepare(struct remote_repository *repository, const char *suite, bool ignorerelease, bool getinrelease, const char *verifyrelease, bool flat, bool *ignorehashes, struct remote_distribution **out_p) {
+ struct remote_distribution *n, **last;
+ enum checksumtype cs;
+
+ for (last = &repository->distributions ; (n = *last) != NULL
+ ; last = &n->next) {
+ if (strcmp(n->suite, suite) != 0)
+ continue;
+ if (n->flat != flat) {
+ if (verbose >= 0 && !n->flatnonflatwarned &&
+ !IGNORABLE(flatandnonflat))
+ fprintf(stderr,
+"Warning: From the same remote repository '%s', distribution '%s'\n"
+"is requested both flat and non-flat. While this is possible\n"
+"(having %s/dists/%s and %s/%s), it is unlikely.\n"
+"To no longer see this message, use --ignore=flatandnonflat.\n",
+ repository->method, suite,
+ repository->method, suite,
+ repository->method, suite);
+ n->flatnonflatwarned = true;
+ continue;
+ }
+ break;
+ }
+
+ if (*last != NULL) {
+ n = *last;
+ assert (n->flat == flat);
+
+ if ((n->ignorerelease && !ignorerelease) ||
+ (!n->ignorerelease && ignorerelease)) {
+ // TODO a hint which two are at fault would be nice,
+ // but how to get the information...
+ if (verbose >= 0)
+ fprintf(stderr,
+"Warning: I was told to both ignore Release files for Suite '%s'\n"
+"from remote repository '%s' and to not ignore it. Going to not ignore!\n",
+ suite, repository->name);
+ n->ignorerelease = false;
+ }
+ if ((n->noinrelease && getinrelease) ||
+ (!n->noinrelease && !getinrelease)) {
+ if (verbose >= 0)
+ fprintf(stderr,
+"Warning: Conflicting GetInRelease values for Suite '%s'\n"
+"from remote repository '%s'. Resolving to get InRelease files!\n",
+ suite, repository->name);
+ n->noinrelease = false;
+ }
+ for (cs = cs_md5sum ; cs < cs_hashCOUNT ; cs++) {
+ if ((n->ignorehashes[cs] && !ignorehashes[cs]) ||
+ (!n->ignorehashes[cs] && ignorehashes[cs])) {
+ // TODO dito
+ if (verbose >= 0)
+ fprintf(stderr,
+"Warning: I was told to both ignore '%s' for Suite '%s'\n"
+"from remote repository '%s' and to not ignore it. Going to not ignore!\n",
+ suite,
+ release_checksum_names[cs],
+ repository->name);
+ n->ignorehashes[cs] = false;
+ }
+ }
+ if (verifyrelease != NULL) {
+ retvalue r;
+
+ r = signature_requirement_add(&n->verify,
+ verifyrelease);
+ if (RET_WAS_ERROR(r))
+ return r;
+ }
+ *out_p = n;
+ return RET_OK;
+ }
+
+ n = zNEW(struct remote_distribution);
+ if (FAILEDTOALLOC(n))
+ return RET_ERROR_OOM;
+ n->repository = repository;
+ n->suite = strdup(suite);
+ n->ignorerelease = ignorerelease;
+ n->noinrelease = !getinrelease;
+ if (verifyrelease != NULL) {
+ retvalue r;
+
+ r = signature_requirement_add(&n->verify, verifyrelease);
+ if (RET_WAS_ERROR(r)) {
+ remote_distribution_free(n);
+ return r;
+ }
+ }
+ memcpy(n->ignorehashes, ignorehashes, sizeof(bool [cs_hashCOUNT]));
+ n->flat = flat;
+ if (flat)
+ n->suite_base_dir = strdup(suite);
+ else
+ n->suite_base_dir = calc_dirconcat("dists", suite);
+ if (FAILEDTOALLOC(n->suite) ||
+ FAILEDTOALLOC(n->suite_base_dir)) {
+ remote_distribution_free(n);
+ return RET_ERROR_OOM;
+ }
+ /* ignorerelease can be unset later, so always calculate the filename */
+ if (flat)
+ n->inreleasefile = genlistsfilename("InRelease", 3,
+ repository->name, suite, "flat",
+ ENDOFARGUMENTS);
+ else
+ n->inreleasefile = genlistsfilename("InRelease", 2,
+ repository->name, suite, ENDOFARGUMENTS);
+ if (FAILEDTOALLOC(n->inreleasefile)) {
+ remote_distribution_free(n);
+ return RET_ERROR_OOM;
+ }
+ if (flat)
+ n->releasefile = genlistsfilename("Release", 3,
+ repository->name, suite, "flat",
+ ENDOFARGUMENTS);
+ else
+ n->releasefile = genlistsfilename("Release", 2,
+ repository->name, suite, ENDOFARGUMENTS);
+ if (FAILEDTOALLOC(n->releasefile)) {
+ remote_distribution_free(n);
+ return RET_ERROR_OOM;
+ }
+ n->releasegpgfile = calc_addsuffix(n->releasefile, "gpg");
+ if (FAILEDTOALLOC(n->releasefile)) {
+ remote_distribution_free(n);
+ return RET_ERROR_OOM;
+ }
+ *last = n;
+ *out_p = n;
+ return RET_OK;
+}
+
+static retvalue copytoplace(const char *gotfilename, const char *wantedfilename, const char *method, struct checksums **checksums_p) {
+ retvalue r;
+ struct checksums *checksums = NULL;
+
+ /* if the file is somewhere else, copy it: */
+ if (strcmp(gotfilename, wantedfilename) != 0) {
+ /* never link index files, but copy them */
+ if (verbose > 1)
+ fprintf(stderr,
+"Copy file '%s' to '%s'...\n", gotfilename, wantedfilename);
+ r = checksums_copyfile(wantedfilename, gotfilename, false,
+ &checksums);
+ if (r == RET_ERROR_EXIST) {
+ fprintf(stderr,
+"Unexpected error: '%s' exists while it should not!\n",
+ wantedfilename);
+ }
+ if (r == RET_NOTHING) {
+ fprintf(stderr,
+"Cannot open '%s', obtained from '%s' method.\n",
+ gotfilename, method);
+ r = RET_ERROR_MISSING;
+ }
+ if (RET_WAS_ERROR(r)) {
+ return r;
+ }
+ }
+ if (checksums_p == NULL)
+ checksums_free(checksums);
+ else
+ *checksums_p = checksums;
+ return RET_OK;
+}
+
+static retvalue enqueue_old_release_files(struct remote_distribution *d);
+
+/* handle a downloaded Release or Release.gpg file:
+ * no checksums to test, nothing to trigger, as they have to be all
+ * read at once to decide what is new and what actually needs downloading */
+static retvalue release_callback(enum queue_action action, void *privdata, void *privdata2, UNUSED(const char *uri), const char *gotfilename, const char *wantedfilename, UNUSED(/*@null@*/const struct checksums *checksums), const char *methodname) {
+ struct remote_distribution *d = privdata;
+ retvalue r;
+
+ /* if the InRelease file cannot be got,
+ * try Release (and Release.gpg if checking) instead */
+ if (action == qa_error && privdata2 == d->inreleasefile) {
+ assert (!d->noinrelease);
+
+ return enqueue_old_release_files(d);
+ }
+
+ if (action != qa_got)
+ return RET_ERROR;
+
+ r = copytoplace(gotfilename, wantedfilename, methodname, NULL);
+ if (RET_WAS_ERROR(r))
+ return r;
+ return r;
+}
+
+static retvalue enqueue_old_release_files(struct remote_distribution *d) {
+ retvalue r;
+
+ d->noinrelease = true;
+ r = aptmethod_enqueueindex(d->repository->download,
+ d->suite_base_dir, "Release", "",
+ d->releasefile, "",
+ release_callback, d, NULL);
+ if (RET_WAS_ERROR(r))
+ return r;
+ if (d->verify != NULL) {
+ r = aptmethod_enqueueindex(d->repository->download,
+ d->suite_base_dir, "Release", ".gpg",
+ d->releasegpgfile, "",
+ release_callback, d, NULL);
+ if (RET_WAS_ERROR(r))
+ return r;
+ }
+ return RET_OK;
+}
+
+static retvalue remote_distribution_enqueuemetalists(struct remote_distribution *d) {
+ struct remote_repository *repository = d->repository;
+
+ assert (repository->download != NULL);
+
+ if (d->ignorerelease)
+ return RET_NOTHING;
+
+ (void)unlink(d->inreleasefile);
+ (void)unlink(d->releasefile);
+ if (d->verify != NULL) {
+ (void)unlink(d->releasegpgfile);
+ }
+
+ if (d->noinrelease)
+ return enqueue_old_release_files(d);
+ else
+ return aptmethod_enqueueindex(repository->download,
+ d->suite_base_dir, "InRelease", "", d->inreleasefile,
+ "", release_callback, d, d->inreleasefile);
+}
+
+retvalue remote_startup(struct aptmethodrun *run) {
+ struct remote_repository *rr;
+ retvalue r;
+
+ if (interrupted())
+ return RET_ERROR_INTERRUPTED;
+
+ for (rr = repositories ; rr != NULL ; rr = rr->next) {
+ assert (rr->download == NULL);
+
+ r = aptmethod_newmethod(run,
+ rr->method, rr->fallback,
+ rr->config, &rr->download);
+ if (RET_WAS_ERROR(r))
+ return r;
+ }
+ return RET_OK;
+}
+
+static void find_index(const struct strlist *files, struct remote_index *ri) {
+ const char *filename = ri->filename_in_release;
+ size_t len = strlen(filename);
+ int i;
+ enum compression c;
+
+ for (i = 0 ; i < files->count ; i++) {
+ const char *value = files->values[i];
+
+ if (strncmp(value, filename, len) != 0)
+ continue;
+
+ value += len;
+
+ if (*value == '\0') {
+ ri->ofs[c_none] = i;
+ continue;
+ }
+ if (*value != '.')
+ continue;
+ if (strcmp(value, ".diff/Index") == 0) {
+ ri->diff_ofs = i;
+ continue;
+ }
+
+ for (c = 0 ; c < c_COUNT ; c++)
+ if (strcmp(value, uncompression_suffix[c]) == 0) {
+ ri->ofs[c] = i;
+ break;
+ }
+ }
+}
+
+/* get a strlist with the md5sums of a Release-file */
+static inline retvalue release_getchecksums(const char *releasefile, const char *chunk, const bool ignorehash[cs_hashCOUNT], struct checksumsarray *out) {
+ retvalue r;
+ struct strlist files[cs_hashCOUNT];
+ enum checksumtype cs;
+ bool foundanything = false;
+
+ for (cs = cs_md5sum ; cs < cs_hashCOUNT ; cs++) {
+ if (ignorehash[cs]) {
+ strlist_init(&files[cs]);
+ continue;
+ }
+ assert (release_checksum_names[cs] != NULL);
+ r = chunk_getextralinelist(chunk, release_checksum_names[cs],
+ &files[cs]);
+ if (RET_WAS_ERROR(r)) {
+ while (cs-- > cs_md5sum) {
+ strlist_done(&files[cs]);
+ }
+ return r;
+ } else if (r == RET_NOTHING)
+ strlist_init(&files[cs]);
+ else
+ foundanything = true;
+ }
+
+ if (!foundanything) {
+ fprintf(stderr, "Missing checksums in Release file '%s'!\n",
+ releasefile);
+ return RET_ERROR;
+ }
+
+ r = checksumsarray_parse(out, files, releasefile);
+ for (cs = cs_md5sum ; cs < cs_hashCOUNT ; cs++) {
+ strlist_done(&files[cs]);
+ }
+ return r;
+}
+
+static retvalue process_remoterelease(struct remote_distribution *rd) {
+ struct remote_repository *rr = rd->repository;
+ struct remote_index *ri;
+ retvalue r;
+ char *releasedata;
+ size_t releaselen;
+
+ if (!rd->noinrelease) {
+ r = signature_check_inline(rd->verify,
+ rd->inreleasefile, &releasedata);
+ assert (r != RET_NOTHING);
+ if (r == RET_NOTHING)
+ r = RET_ERROR_BADSIG;
+ if (r == RET_ERROR_BADSIG) {
+ fprintf(stderr,
+"Error: Not enough signatures found for remote repository %s (%s %s)!\n",
+ rr->name, rr->method, rd->suite);
+ r = RET_ERROR_BADSIG;
+ }
+ if (RET_WAS_ERROR(r))
+ return r;
+ rd->usedreleasefile = rd->inreleasefile;
+ } else {
+ r = readtextfile(rd->releasefile, rd->releasefile,
+ &releasedata, &releaselen);
+ assert (r != RET_NOTHING);
+ if (RET_WAS_ERROR(r))
+ return r;
+ rd->usedreleasefile = rd->releasefile;
+
+ if (rd->verify != NULL) {
+ r = signature_check(rd->verify,
+ rd->releasegpgfile, rd->releasefile,
+ releasedata, releaselen);
+ assert (r != RET_NOTHING);
+ if (r == RET_NOTHING)
+ r = RET_ERROR_BADSIG;
+ if (r == RET_ERROR_BADSIG) {
+ fprintf(stderr,
+"Error: Not enough signatures found for remote repository %s (%s %s)!\n",
+ rr->name, rr->method, rd->suite);
+ r = RET_ERROR_BADSIG;
+ }
+ if (RET_WAS_ERROR(r)) {
+ free(releasedata);
+ return r;
+ }
+ }
+ }
+ r = release_getchecksums(rd->usedreleasefile, releasedata,
+ rd->ignorehashes, &rd->remotefiles);
+ free(releasedata);
+ if (RET_WAS_ERROR(r))
+ return r;
+
+ /* Check for our files in there */
+ for (ri = rd->indices ; ri != NULL ; ri = ri->next) {
+ find_index(&rd->remotefiles.names, ri);
+ }
+ // TODO: move checking if not exists at all to here?
+ return RET_OK;
+}
+
+retvalue remote_preparemetalists(struct aptmethodrun *run, bool nodownload) {
+ struct remote_repository *rr;
+ struct remote_distribution *rd;
+ retvalue r;
+
+ if (!nodownload) {
+ for (rr = repositories ; rr != NULL ; rr = rr->next) {
+ for (rd = rr->distributions ; rd != NULL ;
+ rd = rd->next) {
+ r = remote_distribution_enqueuemetalists(rd);
+ if (RET_WAS_ERROR(r))
+ return r;
+ }
+ }
+ r = aptmethod_download(run);
+ if (RET_WAS_ERROR(r))
+ return r;
+ }
+ for (rr = repositories ; rr != NULL ; rr = rr->next) {
+ for (rd = rr->distributions ; rd != NULL ; rd = rd->next) {
+ if (!rd->ignorerelease) {
+ if (nodownload)
+ if (!isregularfile(rd->inreleasefile))
+ rd->noinrelease = true;
+ r = process_remoterelease(rd);
+ if (RET_WAS_ERROR(r))
+ return r;
+ }
+ }
+ }
+ return RET_OK;
+}
+
+bool remote_index_isnew(/*@null@*/const struct remote_index *ri, struct donefile *done) {
+ const char *basefilename;
+ struct checksums *checksums;
+ bool hashes_missing, improves;
+
+ /* files without uncompressed checksum cannot be tested */
+ if (ri->ofs[c_none] < 0)
+ return true;
+ /* if not there or the wrong files comes next, then something
+ * has changed and we better reload everything */
+ if (!donefile_nextindex(done, &basefilename, &checksums))
+ return true;
+ if (strcmp(basefilename, ri->cachebasename) != 0) {
+ checksums_free(checksums);
+ return true;
+ }
+ /* otherwise check if the file checksums match */
+ if (!checksums_check(checksums,
+ ri->from->remotefiles.checksums[ri->ofs[c_none]],
+ &hashes_missing)) {
+ checksums_free(checksums);
+ return true;
+ }
+ if (hashes_missing) {
+ /* if Release has checksums we do not yet know about,
+ * process it to make sure those match as well */
+ checksums_free(checksums);
+ return true;
+ }
+ if (!checksums_check(ri->from->remotefiles.checksums[ri->ofs[c_none]],
+ checksums, &improves)) {
+ /* this should not happen, but ... */
+ checksums_free(checksums);
+ return true;
+ }
+ if (improves) {
+ /* assume this is our file and add the other hashes so they
+ * will show up in the file again the next time.
+ * This is a bit unelegant in mixing stuff, but otherwise this
+ * will cause redownloading when remote adds more hashes.
+ * The only downside of mixing can reject files that have the
+ * same recorded hashes as a previously processed files.
+ * But that is quite inlikely unless on attack, so getting some
+ * hint in that case cannot harm.*/
+ (void)checksums_combine(&ri->from->remotefiles.checksums[
+ ri->ofs[c_none]], checksums, NULL);
+ }
+ checksums_free(checksums);
+ return false;
+}
+
+static inline void remote_index_oldfiles(struct remote_index *ri, /*@null@*/struct cachedlistfile *oldfiles, /*@out@*/struct cachedlistfile *old[c_COUNT]) {
+ struct cachedlistfile *o;
+ size_t l;
+ enum compression c;
+
+ for (c = 0 ; c < c_COUNT ; c++)
+ old[c] = NULL;
+
+ l = strlen(ri->cachebasename);
+ for (o = oldfiles ; o != NULL ; o = o->next) {
+ if (o->deleted)
+ continue;
+ if (strncmp(o->basefilename, ri->cachebasename, l) != 0)
+ continue;
+ for (c = 0 ; c < c_COUNT ; c++)
+ if (strcmp(o->basefilename + l,
+ uncompression_suffix[c]) == 0) {
+ old[c] = o;
+ o->needed = true;
+ break;
+ }
+ if (strcmp(o->basefilename + l, ".diffindex") == 0)
+ (void)cachedlistfile_delete(o);
+ if (strncmp(o->basefilename + l, ".diff-", 6) == 0)
+ (void)cachedlistfile_delete(o);
+ }
+}
+
+static inline void remote_index_delete_oldfiles(struct remote_index *ri, /*@null@*/struct cachedlistfile *oldfiles) {
+ struct cachedlistfile *o;
+ size_t l;
+
+ l = strlen(ri->cachebasename);
+ for (o = oldfiles ; o != NULL ; o = o->next) {
+ if (o->deleted)
+ continue;
+ if (strncmp(o->basefilename, ri->cachebasename, l) != 0)
+ continue;
+ (void)cachedlistfile_delete(o);
+ }
+}
+
+static queue_callback index_callback;
+static queue_callback diff_callback;
+
+static retvalue queue_next_without_release(struct remote_distribution *rd, struct remote_index *ri) {
+ const struct encoding_preferences *downloadas;
+ static const struct encoding_preferences defaultdownloadas = {
+ .count = 5,
+ .requested = {
+ { .diff = false, .force = false, .compression = c_gzip },
+ { .diff = false, .force = false, .compression = c_bzip2 },
+ { .diff = false, .force = false, .compression = c_none },
+ { .diff = false, .force = false, .compression = c_lzma },
+ { .diff = false, .force = false, .compression = c_xz }
+ }
+ };
+ int e;
+
+ if (ri->downloadas.count == 0)
+ downloadas = &defaultdownloadas;
+ else
+ downloadas = &ri->downloadas;
+
+ for (e = ri->lasttriedencoding + 1 ; e < downloadas->count ; e++) {
+ enum compression c = downloadas->requested[e].compression;
+
+ if (downloadas->requested[e].diff)
+ continue;
+ if (uncompression_supported(c)) {
+ ri->lasttriedencoding = e;
+ ri->compression = c;
+ return aptmethod_enqueueindex(rd->repository->download,
+ rd->suite_base_dir,
+ ri->filename_in_release,
+ uncompression_suffix[c],
+ ri->cachefilename,
+ uncompression_suffix[c],
+ index_callback, ri, NULL);
+ }
+ }
+ if (ri->lasttriedencoding < 0)
+ fprintf(stderr,
+"ERROR: no supported compressions in DownloadListsAs for '%s' by '%s'!\n",
+ rd->suite, rd->repository->method);
+ ri->lasttriedencoding = e;
+ return RET_ERROR;
+}
+
+static inline retvalue find_requested_encoding(struct remote_index *ri, const char *releasefile) {
+ int e;
+ enum compression c, stopat,
+ /* the most-preferred requested but unsupported */
+ unsupported = c_COUNT,
+ /* the best unrequested but supported */
+ unrequested = c_COUNT;
+
+ if (ri->downloadas.count > 0) {
+ bool found = false;
+ for (e = ri->lasttriedencoding + 1 ;
+ e < ri->downloadas.count ;
+ e++) {
+ struct compression_preference req;
+
+ req = ri->downloadas.requested[e];
+
+ if (req.diff) {
+ if (ri->olduncompressed == NULL)
+ continue;
+ assert (ri->ofs[c_none] >= 0);
+ if (!req.force && ri->diff_ofs < 0)
+ continue;
+ ri->compression = c_COUNT;
+ ri->lasttriedencoding = e;
+ return RET_OK;
+ }
+ if (ri->ofs[req.compression] < 0 &&
+ (!req.force || ri->ofs[c_none] < 0))
+ continue;
+ if (uncompression_supported(req.compression)) {
+ ri->compression = req.compression;
+ ri->lasttriedencoding = e;
+ return RET_OK;
+ } else if (unsupported == c_COUNT)
+ unsupported = req.compression;
+ }
+ if (ri->lasttriedencoding > -1) {
+ /* we already tried something, and nothing else
+ * is available, so give up */
+ ri->lasttriedencoding = e;
+ return RET_ERROR;
+ }
+
+ /* nothing that is both requested by the user and supported
+ * and listed in the Release file found, check what is there
+ * to get a meaningful error message */
+
+ for (c = 0 ; c < c_COUNT ; c++) {
+ if (ri->ofs[c] < 0)
+ continue;
+ found = true;
+ if (uncompression_supported(c))
+ unrequested = c;
+ }
+
+ if (!found) {
+ // TODO: might be nice to check for not-yet-even
+ // known about compressions and say they are not
+ // yet know yet instead then here...
+ fprintf(stderr,
+"Could not find '%s' within '%s'\n",
+ ri->filename_in_release, releasefile);
+ return RET_ERROR_WRONG_MD5;
+ }
+
+ if (unsupported != c_COUNT && unrequested != c_COUNT) {
+ fprintf(stderr,
+"Error: '%s' only lists unusable or unrequested compressions of '%s'.\n"
+"Try e.g the '%s' option (or check what it is set to) to make more useable.\n"
+"Or change your DownloadListsAs to request e.g. '%s'.\n",
+ releasefile, ri->filename_in_release,
+ uncompression_option[unsupported],
+ uncompression_config[unrequested]);
+ return RET_ERROR;
+ }
+ if (unsupported != c_COUNT) {
+ fprintf(stderr,
+"Error: '%s' only lists unusable compressions of '%s'.\n"
+"Try e.g the '%s' option (or check what it is set to) to make more useable.\n",
+ releasefile, ri->filename_in_release,
+ uncompression_option[unsupported]);
+ return RET_ERROR;
+ }
+ if (unrequested != c_COUNT) {
+ fprintf(stderr,
+"Error: '%s' only lists unrequested compressions of '%s'.\n"
+"Try changing your DownloadListsAs to request e.g. '%s'.\n",
+ releasefile, ri->filename_in_release,
+ uncompression_config[unrequested]);
+ return RET_ERROR;
+ }
+ fprintf(stderr,
+"Error: '%s' lists no requested and usable compressions of '%s'.\n",
+ releasefile, ri->filename_in_release);
+ return RET_ERROR;
+ }
+ /* When nothing specified, use the newest compression.
+ * This might make it slow on older computers (and perhaps
+ * on relatively new ones, too), but usually bandwidth costs
+ * and your time not.
+ * And you can always configure it to prefer a faster one...
+ */
+
+ /* ri->lasttriedencoding -1 means nothing tried,
+ * 0 means Package.diff was tried,
+ * 1 means nothing c_COUNT - 1 was already tried,
+ * 2 means nothing c_COUNT - 2 was already tried,
+ * and so on...*/
+
+ if (ri->lasttriedencoding < 0) {
+ if (ri->olduncompressed != NULL && ri->diff_ofs >= 0) {
+ ri->compression = c_COUNT;
+ ri->lasttriedencoding = 0;
+ return RET_OK;
+ }
+ stopat = c_COUNT;
+ } else
+ stopat = c_COUNT - ri->lasttriedencoding;
+
+ ri->compression = c_COUNT;
+ for (c = 0 ; c < stopat ; c++) {
+ if (ri->ofs[c] < 0)
+ continue;
+ if (uncompression_supported(c))
+ ri->compression = c;
+ else
+ unsupported = c;
+ }
+ if (ri->compression == c_COUNT) {
+ if (ri->lasttriedencoding > -1) {
+ /* not the first try, no error message needed */
+ ri->lasttriedencoding = c_COUNT;
+ return RET_ERROR;
+ }
+ if (unsupported != c_COUNT) {
+ fprintf(stderr,
+"Error: '%s' only lists unusable compressions of '%s'.\n"
+"Try e.g the '%s' option (or check what it is set to) to enable more.\n",
+ releasefile, ri->filename_in_release,
+ uncompression_option[unsupported]);
+ return RET_ERROR;
+ }
+ fprintf(stderr,
+"Could not find '%s' within '%s'\n",
+ ri->filename_in_release, releasefile);
+ return RET_ERROR_WRONG_MD5;
+
+ }
+ ri->lasttriedencoding = c_COUNT - ri->compression;
+ return RET_OK;
+}
+
+static inline retvalue remove_old_uncompressed(struct remote_index *ri) {
+ retvalue r;
+
+ if (ri->olduncompressed != NULL) {
+ r = cachedlistfile_delete(ri->olduncompressed);
+ ri->olduncompressed = NULL;
+ return r;
+ } else
+ return RET_NOTHING;
+}
+
+static retvalue queue_next_encoding(struct remote_distribution *rd, struct remote_index *ri);
+
+// TODO: check if this still makes sense.
+// (might be left over to support switching from older versions
+// of reprepro that also put compressed files there)
+static inline retvalue reuse_old_compressed_index(struct remote_distribution *rd, struct remote_index *ri, enum compression c, const char *oldfullfilename) {
+ retvalue r;
+
+ r = uncompress_file(oldfullfilename, ri->cachefilename, c);
+ assert (r != RET_NOTHING);
+ if (RET_WAS_ERROR(r))
+ return r;
+ if (ri->ofs[c_none] >= 0) {
+ r = checksums_test(ri->cachefilename,
+ rd->remotefiles.checksums[ri->ofs[c_none]],
+ &rd->remotefiles.checksums[ri->ofs[c_none]]);
+ if (r == RET_ERROR_WRONG_MD5) {
+ fprintf(stderr,
+"Error: File '%s' looked correct according to '%s',\n"
+"but after unpacking '%s' looks wrong.\n"
+"Something is seriously broken!\n",
+ oldfullfilename, rd->usedreleasefile,
+ ri->cachefilename);
+ }
+ if (r == RET_NOTHING) {
+ fprintf(stderr,
+"File '%s' mysteriously vanished!\n", ri->cachefilename);
+ r = RET_ERROR_MISSING;
+ }
+ if (RET_WAS_ERROR(r))
+ return r;
+ }
+ /* already there, nothing to do to get it... */
+ ri->queued = true;
+ ri->got = true;
+ return RET_OK;
+}
+
+static inline retvalue queueindex(struct remote_distribution *rd, struct remote_index *ri, bool nodownload, /*@null@*/struct cachedlistfile *oldfiles) {
+ enum compression c;
+ retvalue r;
+ struct cachedlistfile *old[c_COUNT];
+
+ if (rd->ignorerelease) {
+ ri->queued = true;
+ if (nodownload) {
+ ri->got = true;
+ return RET_OK;
+ }
+
+ /* as there is no way to know which are current,
+ * just delete everything */
+ remote_index_delete_oldfiles(ri, oldfiles);
+
+ return queue_next_without_release(rd, ri);
+ }
+
+ /* check if this file is still available from an earlier download */
+ remote_index_oldfiles(ri, oldfiles, old);
+ ri->olduncompressed = NULL;
+ ri->oldchecksums = NULL;
+ if (ri->ofs[c_none] < 0 && old[c_none] != NULL) {
+ /* if we know not what it should be,
+ * we canot use the old... */
+ r = cachedlistfile_delete(old[c_none]);
+ if (RET_WAS_ERROR(r))
+ return r;
+ old[c_none] = NULL;
+ } else if (old[c_none] != NULL) {
+ bool improves;
+ int uo = ri->ofs[c_none];
+ struct checksums **wanted_p = &rd->remotefiles.checksums[uo];
+
+ r = checksums_read(old[c_none]->fullfilename,
+ &ri->oldchecksums);
+ if (r == RET_NOTHING) {
+ fprintf(stderr, "File '%s' mysteriously vanished!\n",
+ old[c_none]->fullfilename);
+ r = RET_ERROR_MISSING;
+ }
+ if (RET_WAS_ERROR(r))
+ return r;
+ if (checksums_check(*wanted_p, ri->oldchecksums, &improves)) {
+ /* already there, nothing to do to get it... */
+ ri->queued = true;
+ ri->got = true;
+ if (improves)
+ r = checksums_combine(wanted_p,
+ ri->oldchecksums, NULL);
+ else
+ r = RET_OK;
+ checksums_free(ri->oldchecksums);
+ ri->oldchecksums = NULL;
+ return r;
+ }
+ ri->olduncompressed = old[c_none];
+ old[c_none] = NULL;
+ }
+
+ assert (old[c_none] == NULL);
+
+ /* make sure everything old is deleted or check if it can be used */
+ for (c = 0 ; c < c_COUNT ; c++) {
+ if (old[c] == NULL)
+ continue;
+ if (c != c_none && ri->ofs[c] >= 0) {
+ /* check if it can be used */
+ r = checksums_test(old[c]->fullfilename,
+ rd->remotefiles.checksums[ri->ofs[c]],
+ &rd->remotefiles.checksums[ri->ofs[c]]);
+ if (r == RET_ERROR_WRONG_MD5)
+ r = RET_NOTHING;
+ if (RET_WAS_ERROR(r))
+ return r;
+ if (RET_IS_OK(r)) {
+ r = remove_old_uncompressed(ri);
+ if (RET_WAS_ERROR(r))
+ return r;
+ assert (old[c_none] == NULL);
+ return reuse_old_compressed_index(rd, ri, c,
+ old[c]->fullfilename);
+ }
+ }
+ r = cachedlistfile_delete(old[c]);
+ if (RET_WAS_ERROR(r))
+ return r;
+ old[c] = NULL;
+ }
+
+ /* nothing found, we'll have to download: */
+
+ if (nodownload) {
+ if (ri->olduncompressed != NULL)
+ fprintf(stderr,
+"Error: '%s' does not match Release file, try without --nolistsdownload to download new one!\n",
+ ri->cachefilename);
+ else
+ fprintf(stderr,
+"Error: Missing '%s', try without --nolistsdownload to download it!\n",
+ ri->cachefilename);
+ return RET_ERROR_MISSING;
+ }
+
+ return queue_next_encoding(rd, ri);
+}
+
+static retvalue queue_next_encoding(struct remote_distribution *rd, struct remote_index *ri) {
+ struct remote_repository *rr = rd->repository;
+ retvalue r;
+
+ if (rd->ignorerelease)
+ return queue_next_without_release(rd, ri);
+
+ r = find_requested_encoding(ri, rd->usedreleasefile);
+ assert (r != RET_NOTHING);
+ if (RET_WAS_ERROR(r))
+ return r;
+
+ assert (ri->compression <= c_COUNT);
+
+ /* check if downloading a .diff/Index (aka .pdiff) is requested */
+ if (ri->compression == c_COUNT) {
+ assert (ri->olduncompressed != NULL);
+ assert (ri->oldchecksums != NULL);
+
+ ri->queued = true;
+ return aptmethod_enqueueindex(rr->download, rd->suite_base_dir,
+ ri->filename_in_release, ".diff/Index",
+ ri->cachefilename, ".diffindex",
+ diff_callback, ri, NULL);
+ }
+
+ assert (ri->compression < c_COUNT);
+ assert (uncompression_supported(ri->compression));
+
+ if (ri->compression == c_none) {
+ r = remove_old_uncompressed(ri);
+ if (RET_WAS_ERROR(r))
+ return r;
+ }
+/* as those checksums might be overwritten with completed data,
+ * this assumes that the uncompressed checksums for one index is never
+ * the compressed checksum for another... */
+
+ ri->queued = true;
+ return aptmethod_enqueueindex(rr->download, rd->suite_base_dir,
+ ri->filename_in_release,
+ uncompression_suffix[ri->compression],
+ ri->cachefilename,
+ uncompression_suffix[ri->compression],
+ index_callback, ri, NULL);
+}
+
+
+static retvalue remote_distribution_enqueuelists(struct remote_distribution *rd, bool nodownload, struct cachedlistfile *oldfiles) {
+ struct remote_index *ri;
+ retvalue r;
+
+ /* check what to get for the requested indicies */
+ for (ri = rd->indices ; ri != NULL ; ri = ri->next) {
+ if (ri->queued)
+ continue;
+ if (!ri->needed) {
+ /* if we do not know anything about it,
+ * it cannot have got marked as old
+ * or otherwise as unneeded */
+ assert (!rd->ignorerelease);
+ continue;
+ }
+ r = queueindex(rd, ri, nodownload, oldfiles);
+ if (RET_WAS_ERROR(r))
+ return r;
+ }
+ return RET_OK;
+}
+
+retvalue remote_preparelists(struct aptmethodrun *run, bool nodownload) {
+ struct remote_repository *rr;
+ struct remote_distribution *rd;
+ retvalue r;
+ struct cachedlistfile *oldfiles;
+
+ r = cachedlists_scandir(&oldfiles);
+ if (RET_WAS_ERROR(r))
+ return r;
+ if (r == RET_NOTHING)
+ oldfiles = NULL;
+
+ for (rr = repositories ; rr != NULL ; rr = rr->next) {
+ for (rd = rr->distributions ; rd != NULL
+ ; rd = rd->next) {
+ r = remote_distribution_enqueuelists(rd,
+ nodownload, oldfiles);
+ if (RET_WAS_ERROR(r)) {
+ cachedlistfile_freelist(oldfiles);
+ return r;
+ }
+ }
+ }
+ r = aptmethod_download(run);
+ if (RET_WAS_ERROR(r)) {
+ cachedlistfile_freelist(oldfiles);
+ return r;
+ }
+
+ cachedlistfile_freelist(oldfiles);
+ return RET_OK;
+}
+
+static struct remote_index *addindex(struct remote_distribution *rd, /*@only@*/char *cachefilename, /*@only@*/char *filename, /*@null@*/const struct encoding_preferences *downloadas) {
+ struct remote_index *ri, **last;
+ enum compression c;
+ const char *cachebasename;
+
+ if (FAILEDTOALLOC(cachefilename) || FAILEDTOALLOC(filename))
+ return NULL;
+
+ cachebasename = dirs_basename(cachefilename);
+ last = &rd->indices;
+ while (*last != NULL && strcmp((*last)->cachebasename, cachebasename) != 0)
+ last = &(*last)->next;
+ if (*last != NULL) {
+ ri = *last;
+ // TODO: perhaps try to calculate some form of intersections
+ // instead of just using the shorter one...
+ if (downloadas != NULL &&
+ (ri->downloadas.count == 0
+ || ri->downloadas.count > downloadas->count))
+ ri->downloadas = *downloadas;
+ free(cachefilename); free(filename);
+ return ri;
+ }
+
+ ri = zNEW(struct remote_index);
+ if (FAILEDTOALLOC(ri)) {
+ free(cachefilename); free(filename);
+ return NULL;
+ }
+
+ *last = ri;
+ ri->from = rd;
+ ri->cachefilename = cachefilename;
+ ri->cachebasename = cachebasename;
+ ri->filename_in_release = filename;
+ if (downloadas != NULL)
+ ri->downloadas = *downloadas;
+ for (c = 0 ; c < c_COUNT ; c++)
+ ri->ofs[c] = -1;
+ ri->diff_ofs = -1;
+ ri->lasttriedencoding = -1;
+ return ri;
+}
+
+struct remote_index *remote_index(struct remote_distribution *rd, const char *architecture, const char *component, packagetype_t packagetype, /*@null@*/const struct encoding_preferences *downloadas) {
+ char *cachefilename, *filename_in_release;
+
+ assert (!rd->flat);
+ if (packagetype == pt_deb) {
+ filename_in_release = mprintf(
+"%s/binary-%s/Packages",
+ component, architecture);
+ cachefilename = genlistsfilename("Packages", 4,
+ rd->repository->name, rd->suite,
+ component, architecture, ENDOFARGUMENTS);
+ } else if (packagetype == pt_udeb) {
+ filename_in_release = mprintf(
+"%s/debian-installer/binary-%s/Packages",
+ component, architecture);
+ cachefilename = genlistsfilename("uPackages", 4,
+ rd->repository->name, rd->suite,
+ component, architecture, ENDOFARGUMENTS);
+ } else if (packagetype == pt_dsc) {
+ filename_in_release = mprintf(
+"%s/source/Sources",
+ component);
+ cachefilename = genlistsfilename("Sources", 3,
+ rd->repository->name, rd->suite,
+ component, ENDOFARGUMENTS);
+ } else {
+ assert ("Unexpected package type" == NULL);
+ }
+ return addindex(rd, cachefilename, filename_in_release, downloadas);
+}
+
+void cachedlistfile_need_index(struct cachedlistfile *list, const char *repository, const char *suite, const char *architecture, const char *component, packagetype_t packagetype) {
+ if (packagetype == pt_deb) {
+ cachedlistfile_need(list, "Packages", 4,
+ repository, suite,
+ component, architecture, ENDOFARGUMENTS);
+ } else if (packagetype == pt_udeb) {
+ cachedlistfile_need(list, "uPackages", 4,
+ repository, suite,
+ component, architecture, ENDOFARGUMENTS);
+ } else if (packagetype == pt_dsc) {
+ cachedlistfile_need(list, "Sources", 3,
+ repository, suite,
+ component, ENDOFARGUMENTS);
+ }
+}
+
+struct remote_index *remote_flat_index(struct remote_distribution *rd, packagetype_t packagetype, /*@null@*/const struct encoding_preferences *downloadas) {
+ char *cachefilename, *filename_in_release;
+
+ assert (rd->flat);
+ if (packagetype == pt_deb) {
+ filename_in_release = strdup("Packages");
+ cachefilename = genlistsfilename("Packages", 2,
+ rd->repository->name, rd->suite,
+ ENDOFARGUMENTS);
+ } else if (packagetype == pt_dsc) {
+ filename_in_release = strdup("Sources");
+ cachefilename = genlistsfilename("Sources", 2,
+ rd->repository->name, rd->suite,
+ ENDOFARGUMENTS);
+ } else {
+ assert ("Unexpected package type" == NULL);
+ }
+ return addindex(rd, cachefilename, filename_in_release, downloadas);
+}
+
+void cachedlistfile_need_flat_index(struct cachedlistfile *list, const char *repository, const char *suite, packagetype_t packagetype) {
+ if (packagetype == pt_deb) {
+ cachedlistfile_need(list, "Packages", 2,
+ repository, suite, ENDOFARGUMENTS);
+ } else if (packagetype == pt_dsc) {
+ cachedlistfile_need(list, "Sources", 1,
+ repository, suite, ENDOFARGUMENTS);
+ }
+}
+
+const char *remote_index_file(const struct remote_index *ri) {
+ assert (ri->needed && ri->queued && ri->got);
+ return ri->cachefilename;
+}
+const char *remote_index_basefile(const struct remote_index *ri) {
+ assert (ri->needed && ri->queued);
+ return ri->cachebasename;
+}
+
+struct aptmethod *remote_aptmethod(const struct remote_distribution *rd) {
+ return rd->repository->download;
+}
+
+void remote_index_markdone(const struct remote_index *ri, struct markdonefile *done) {
+ if (ri->ofs[c_none] < 0)
+ return;
+ markdone_index(done, ri->cachebasename,
+ ri->from->remotefiles.checksums[ri->ofs[c_none]]);
+}
+void remote_index_needed(struct remote_index *ri) {
+ ri->needed = true;
+}
+
+static retvalue indexfile_mark_got(struct remote_distribution *rd, struct remote_index *ri, /*@null@*/const struct checksums *gotchecksums) {
+ struct checksums **checksums_p;
+
+ if (!rd->ignorerelease && ri->ofs[c_none] >= 0) {
+ checksums_p = &rd->remotefiles.checksums[ri->ofs[c_none]];
+ bool matches, improves;
+
+ // TODO: this no longer calculates all the checksums if
+ // the Release does not contain more and the apt method
+ // returned not all (but all that are in Release).
+ // This will then cause the done file not containing all
+ // checksums. (but if the Release not contain them, this
+ // does not harm, does it?)
+
+ if (gotchecksums != NULL) {
+ matches = checksums_check(*checksums_p, gotchecksums,
+ &improves);
+ /* that should have been tested earlier */
+ assert (matches);
+ if (! matches)
+ return RET_ERROR_WRONG_MD5;
+ if (improves) {
+ retvalue r;
+
+ r = checksums_combine(checksums_p,
+ gotchecksums, NULL);
+ if (RET_WAS_ERROR(r))
+ return r;
+ }
+ }
+ }
+ ri->got = true;
+ return RET_OK;
+}
+
+static retvalue indexfile_unpacked(void *privdata, const char *compressed, bool failed) {
+ struct remote_index *ri = privdata;
+ struct remote_distribution *rd = ri->from;
+ retvalue r;
+ struct checksums *readchecksums = NULL;
+
+ if (failed) {
+ // TODO: check if alternative can be used...
+ return RET_ERROR;
+ }
+
+ /* file got uncompressed, check if it has the correct checksum */
+
+ /* even with a Release file, an old-style one might
+ * not list the checksums for the uncompressed indices */
+ if (!rd->ignorerelease && ri->ofs[c_none] >= 0) {
+ int ofs = ri->ofs[c_none];
+ const struct checksums *wantedchecksums =
+ rd->remotefiles.checksums[ofs];
+ bool matches, missing = false;
+
+ r = checksums_read(ri->cachefilename, &readchecksums);
+ if (r == RET_NOTHING) {
+ fprintf(stderr,
+"Cannot open '%s', though it should just have been unpacked from '%s'!\n",
+ ri->cachefilename,
+ compressed);
+ r = RET_ERROR_MISSING;
+ }
+ if (RET_WAS_ERROR(r))
+ return r;
+ missing = false;
+ matches = checksums_check(readchecksums,
+ wantedchecksums, &missing);
+ assert (!missing);
+ if (!matches) {
+ fprintf(stderr,
+"Wrong checksum of uncompressed content of '%s':\n", compressed);
+ checksums_printdifferences(stderr,
+ wantedchecksums,
+ readchecksums);
+ checksums_free(readchecksums);
+ return RET_ERROR_WRONG_MD5;
+ }
+ /* if the compressed file was downloaded or copied, delete it.
+ * This is only done if we know the uncompressed checksum, so
+ * that less downloading is needed (though as apt no longer
+ * supports such archieves, they are unlikely anyway). */
+
+ if (strncmp(ri->cachefilename, compressed,
+ strlen(ri->cachefilename)) == 0) {
+ (void)unlink(compressed);
+ }
+ }
+ r = indexfile_mark_got(rd, ri, readchecksums);
+ checksums_free(readchecksums);
+ if (RET_WAS_ERROR(r))
+ return r;
+ return RET_OK;
+}
+
+/* *checksums_p must be either NULL or gotchecksums list all known checksums */
+static inline retvalue check_checksums(const char *methodname, const char *uri, const char *gotfilename, const struct checksums *wantedchecksums, /*@null@*/const struct checksums *gotchecksums, struct checksums **checksums_p) {
+ bool matches, missing = false;
+ struct checksums *readchecksums = NULL;
+ retvalue r;
+
+ if (gotchecksums == NULL) {
+ matches = true;
+ missing = true;
+ } else
+ matches = checksums_check(gotchecksums,
+ wantedchecksums, &missing);
+ /* if the apt method did not generate all checksums
+ * we want to check, we'll have to do so: */
+ if (matches && missing) {
+ /* we assume that everything we know how to
+ * extract from a Release file is something
+ * we know how to calculate out of a file */
+ assert (checksums_p == NULL || *checksums_p == NULL);
+ r = checksums_read(gotfilename, &readchecksums);
+ if (r == RET_NOTHING) {
+ fprintf(stderr,
+"Cannot open '%s', though apt-method '%s' claims it is there!\n",
+ gotfilename, methodname);
+ r = RET_ERROR_MISSING;
+ }
+ if (RET_WAS_ERROR(r))
+ return r;
+ gotchecksums = readchecksums;
+ missing = false;
+ matches = checksums_check(gotchecksums,
+ wantedchecksums, &missing);
+ assert (!missing);
+ }
+ if (!matches) {
+ fprintf(stderr, "Wrong checksum during receive of '%s':\n",
+ uri);
+ checksums_printdifferences(stderr,
+ wantedchecksums,
+ gotchecksums);
+ checksums_free(readchecksums);
+ return RET_ERROR_WRONG_MD5;
+ }
+ if (checksums_p == NULL)
+ checksums_free(readchecksums);
+ else if (readchecksums != NULL)
+ *checksums_p = readchecksums;
+ return RET_OK;
+}
+
+static retvalue index_callback(enum queue_action action, void *privdata, UNUSED(void *privdata2), const char *uri, const char *gotfilename, const char *wantedfilename, /*@null@*/const struct checksums *gotchecksums, const char *methodname) {
+ struct remote_index *ri = privdata;
+ struct remote_distribution *rd = ri->from;
+ struct checksums *readchecksums = NULL;
+ retvalue r;
+
+ if (action == qa_error)
+ return queue_next_encoding(rd, ri);
+ if (action != qa_got)
+ return RET_ERROR;
+
+ if (ri->compression == c_none) {
+ assert (strcmp(wantedfilename, ri->cachefilename) == 0);
+ r = copytoplace(gotfilename, wantedfilename, methodname,
+ &readchecksums);
+ if (RET_WAS_ERROR(r))
+ return r;
+ gotfilename = wantedfilename;
+ if (readchecksums != NULL)
+ gotchecksums = readchecksums;
+ }
+
+ if (!rd->ignorerelease && ri->ofs[ri->compression] >= 0) {
+ int ofs = ri->ofs[ri->compression];
+ const struct checksums *wantedchecksums =
+ rd->remotefiles.checksums[ofs];
+
+ r = check_checksums(methodname, uri, gotfilename,
+ wantedchecksums, gotchecksums, &readchecksums);
+ if (RET_WAS_ERROR(r)) {
+ checksums_free(readchecksums);
+ return r;
+ }
+ if (readchecksums != NULL)
+ gotchecksums = readchecksums;
+ }
+
+ if (ri->compression == c_none) {
+ assert (strcmp(gotfilename, wantedfilename) == 0);
+ r = indexfile_mark_got(rd, ri, gotchecksums);
+ checksums_free(readchecksums);
+ if (RET_WAS_ERROR(r))
+ return r;
+ return RET_OK;
+ } else {
+ checksums_free(readchecksums);
+ r = remove_old_uncompressed(ri);
+ if (RET_WAS_ERROR(r))
+ return r;
+ r = uncompress_queue_file(gotfilename, ri->cachefilename,
+ ri->compression,
+ indexfile_unpacked, privdata);
+ if (RET_WAS_ERROR(r))
+ return r;
+ return RET_OK;
+ }
+}
+
+static queue_callback diff_got_callback;
+
+static retvalue queue_next_diff(struct remote_index *ri) {
+ struct remote_distribution *rd = ri->from;
+ struct remote_repository *rr = rd->repository;
+ int i;
+ retvalue r;
+
+ for (i = 0 ; i < ri->diffindex->patchcount ; i++) {
+ bool improves;
+ struct diffindex_patch *p = &ri->diffindex->patches[i];
+ char *patchsuffix, *c;
+
+ if (p->done || p->frompackages == NULL)
+ continue;
+
+ if (!checksums_check(ri->oldchecksums, p->frompackages,
+ &improves))
+ continue;
+ /* p->frompackages should only have sha1 and oldchecksums
+ * should definitely list a sha1 hash */
+ assert (!improves);
+
+ p->done = true;
+
+ free(ri->patchfilename);
+ ri->patchfilename = mprintf("%s.diff-%s", ri->cachefilename,
+ p->name);
+ if (FAILEDTOALLOC(ri->patchfilename))
+ return RET_ERROR_OOM;
+ c = ri->patchfilename + strlen(ri->cachefilename);
+ while (*c != '\0') {
+ if ((*c < '0' || *c > '9')
+ && (*c < 'A' || *c > 'Z')
+ && (*c < 'a' || *c > 'z')
+ && *c != '.' && *c != '-')
+ *c = '_';
+ c++;
+ }
+ ri->selectedpatch = p;
+ patchsuffix = mprintf(".diff/%s.gz", p->name);
+ if (FAILEDTOALLOC(patchsuffix))
+ return RET_ERROR_OOM;
+
+ /* found a matching patch, tell the downloader we want it */
+ r = aptmethod_enqueueindex(rr->download, rd->suite_base_dir,
+ ri->filename_in_release,
+ patchsuffix,
+ ri->patchfilename, ".gz",
+ diff_got_callback, ri, p);
+ free(patchsuffix);
+ return r;
+ }
+ /* no patch matches, try next possibility... */
+ fprintf(stderr, "Error: available '%s' not listed in '%s.diffindex'.\n",
+ ri->cachefilename, ri->cachefilename);
+ return queue_next_encoding(rd, ri);
+}
+
+static retvalue diff_uncompressed(void *privdata, const char *compressed, bool failed) {
+ struct remote_index *ri = privdata;
+ struct remote_distribution *rd = ri->from;
+ const struct diffindex_patch *p = ri->selectedpatch;
+ char *tempfilename;
+ struct rred_patch *rp;
+ FILE *f;
+ int i;
+ retvalue r;
+ bool dummy;
+
+ if (ri->deletecompressedpatch)
+ (void)unlink(compressed);
+ if (failed)
+ return RET_ERROR;
+
+ r = checksums_test(ri->patchfilename, p->checksums, NULL);
+ if (r == RET_NOTHING) {
+ fprintf(stderr, "Mysteriously vanished file '%s'!\n",
+ ri->patchfilename);
+ r = RET_ERROR_MISSING;
+ }
+ if (r == RET_ERROR_WRONG_MD5)
+ fprintf(stderr, "Corrupted package diff '%s'!\n",
+ ri->patchfilename);
+ if (RET_WAS_ERROR(r))
+ return r;
+
+ r = patch_load(ri->patchfilename,
+ checksums_getfilesize(p->checksums), &rp);
+ ASSERT_NOT_NOTHING(r);
+ if (RET_WAS_ERROR(r))
+ return r;
+
+ tempfilename = calc_addsuffix(ri->cachefilename, "tmp");
+ if (FAILEDTOALLOC(tempfilename)) {
+ patch_free(rp);
+ return RET_ERROR_OOM;
+ }
+ (void)unlink(tempfilename);
+ i = rename(ri->cachefilename, tempfilename);
+ if (i != 0) {
+ int e = errno;
+ fprintf(stderr, "Error %d moving '%s' to '%s': %s\n",
+ e, ri->cachefilename, tempfilename,
+ strerror(e));
+ free(tempfilename);
+ patch_free(rp);
+ return RET_ERRNO(e);
+ }
+ f = fopen(ri->cachefilename, "w");
+ if (f == NULL) {
+ int e = errno;
+ fprintf(stderr, "Error %d creating '%s': %s\n",
+ e, ri->cachefilename, strerror(e));
+ (void)unlink(tempfilename);
+ ri->olduncompressed->deleted = true;
+ ri->olduncompressed = NULL;
+ free(tempfilename);
+ patch_free(rp);
+ return RET_ERRNO(e);
+ }
+ r = patch_file(f, tempfilename, patch_getconstmodifications(rp));
+ (void)unlink(tempfilename);
+ (void)unlink(ri->patchfilename);
+ free(ri->patchfilename);
+ ri->patchfilename = NULL;
+ free(tempfilename);
+ patch_free(rp);
+ if (RET_WAS_ERROR(r)) {
+ (void)fclose(f);
+ remove_old_uncompressed(ri);
+ // TODO: fall back to downloading at once?
+ return r;
+ }
+ i = ferror(f);
+ if (i != 0) {
+ int e = errno;
+ (void)fclose(f);
+ fprintf(stderr, "Error %d writing to '%s': %s\n",
+ e, ri->cachefilename, strerror(e));
+ remove_old_uncompressed(ri);
+ return RET_ERRNO(e);
+ }
+ i = fclose(f);
+ if (i != 0) {
+ int e = errno;
+ fprintf(stderr, "Error %d writing to '%s': %s\n",
+ e, ri->cachefilename, strerror(e));
+ remove_old_uncompressed(ri);
+ return RET_ERRNO(e);
+ }
+ checksums_free(ri->oldchecksums);
+ ri->oldchecksums = NULL;
+ r = checksums_read(ri->cachefilename, &ri->oldchecksums);
+ if (r == RET_NOTHING) {
+ fprintf(stderr, "Myteriously vanished file '%s'!\n",
+ ri->cachefilename);
+ r = RET_ERROR;
+ }
+ if (RET_WAS_ERROR(r))
+ return r;
+ if (checksums_check(ri->oldchecksums,
+ rd->remotefiles.checksums[ri->ofs[c_none]],
+ &dummy)) {
+ ri->olduncompressed->deleted = true;
+ ri->olduncompressed = NULL;
+ /* we have a winner */
+ return indexfile_mark_got(rd, ri, ri->oldchecksums);
+ }
+ /* let's see what patch we need next */
+ return queue_next_diff(ri);
+}
+
+static retvalue diff_got_callback(enum queue_action action, void *privdata, UNUSED(void *privdata2), UNUSED(const char *uri), const char *gotfilename, const char *wantedfilename, UNUSED(/*@null@*/const struct checksums *gotchecksums), UNUSED(const char *methodname)) {
+ struct remote_index *ri = privdata;
+ retvalue r;
+
+ if (action == qa_error)
+ return queue_next_encoding(ri->from, ri);
+ if (action != qa_got)
+ return RET_ERROR;
+
+ ri->deletecompressedpatch = strcmp(gotfilename, wantedfilename) == 0;
+ r = uncompress_queue_file(gotfilename, ri->patchfilename,
+ c_gzip, diff_uncompressed, ri);
+ if (RET_WAS_ERROR(r))
+ (void)unlink(gotfilename);
+ return r;
+}
+
+static retvalue diff_callback(enum queue_action action, void *privdata, UNUSED(void *privdata2), const char *uri, const char *gotfilename, const char *wantedfilename, /*@null@*/const struct checksums *gotchecksums, const char *methodname) {
+ struct remote_index *ri = privdata;
+ struct remote_distribution *rd = ri->from;
+ struct checksums *readchecksums = NULL;
+ int ofs;
+ retvalue r;
+
+ if (action == qa_error)
+ return queue_next_encoding(rd, ri);
+ if (action != qa_got)
+ return RET_ERROR;
+
+ r = copytoplace(gotfilename, wantedfilename, methodname,
+ &readchecksums);
+ if (RET_WAS_ERROR(r))
+ return r;
+ if (readchecksums != NULL)
+ gotchecksums = readchecksums;
+
+ ofs = ri->diff_ofs;
+ if (ofs >= 0) {
+ const struct checksums *wantedchecksums =
+ rd->remotefiles.checksums[ofs];
+ bool matches, missing = false;
+
+ if (gotchecksums == NULL) {
+ matches = true;
+ missing = true;
+ } else
+ matches = checksums_check(gotchecksums,
+ wantedchecksums, &missing);
+ /* if the apt method did not generate all checksums
+ * we want to check, we'll have to do so: */
+ if (matches && missing) {
+ /* we assume that everything we know how to
+ * extract from a Release file is something
+ * we know how to calculate out of a file */
+ assert (readchecksums == NULL);
+ r = checksums_read(gotfilename, &readchecksums);
+ if (r == RET_NOTHING) {
+ fprintf(stderr,
+"Cannot open '%s', though apt-method '%s' claims it is there!\n",
+ gotfilename, methodname);
+ r = RET_ERROR_MISSING;
+ }
+ if (RET_WAS_ERROR(r))
+ return r;
+ gotchecksums = readchecksums;
+ missing = false;
+ matches = checksums_check(gotchecksums,
+ wantedchecksums, &missing);
+ assert (!missing);
+ }
+ if (!matches) {
+ fprintf(stderr,
+"Wrong checksum during receive of '%s':\n", uri);
+ checksums_printdifferences(stderr,
+ wantedchecksums, gotchecksums);
+ checksums_free(readchecksums);
+ return RET_ERROR_WRONG_MD5;
+ }
+ }
+ checksums_free(readchecksums);
+ r = diffindex_read(wantedfilename, &ri->diffindex);
+ ASSERT_NOT_NOTHING(r);
+ if (RET_WAS_ERROR(r))
+ return queue_next_encoding(rd, ri);
+ if (ri->ofs[c_none] >= 0) {
+ bool dummy;
+ if (!checksums_check(rd->remotefiles.checksums[
+ ri->ofs[c_none]],
+ ri->diffindex->destination, &dummy)) {
+ fprintf(stderr,
+"'%s' does not match file requested in '%s'. Aborting diff processing...\n",
+ gotfilename, rd->usedreleasefile);
+ /* as this is claimed to be a common error
+ * (outdated .diff/Index file), proceed with
+ * other requested way to retrieve index file */
+ return queue_next_encoding(rd, ri);
+ }
+ }
+ return queue_next_diff(ri);
+}