diff options
Diffstat (limited to 'remoterepository.c')
| -rw-r--r-- | remoterepository.c | 2103 |
1 files changed, 2103 insertions, 0 deletions
diff --git a/remoterepository.c b/remoterepository.c new file mode 100644 index 0000000..ec1d726 --- /dev/null +++ b/remoterepository.c @@ -0,0 +1,2103 @@ +/* This file is part of "reprepro" + * Copyright (C) 2003,2004,2005,2007,2008,2009,2012 Bernhard R. Link + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA + */ + +#include <config.h> + +#include <stdlib.h> +#include <stdio.h> +#include <stdarg.h> +#include <unistd.h> +#include <assert.h> +#include <string.h> +#include <errno.h> +#include <dirent.h> + +#include "globals.h" +#include "error.h" +#include "ignore.h" +#include "filecntl.h" +#include "checksums.h" +#include "mprintf.h" +#include "dirs.h" +#include "chunks.h" +#include "names.h" +#include "aptmethod.h" +#include "signature.h" +#include "readtextfile.h" +#include "uncompression.h" +#include "diffindex.h" +#include "rredpatch.h" +#include "remoterepository.h" + +/* This is code to handle lists from remote repositories. + Those are stored in the lists/ (or --listdir) directory + and needs some maintaince: + + - cleaning (unneeded) lists from that directory + - deciding what to download from a remote repository + (needs knowledge what is there and what is there) + - in the future: implement ed to use remote .diffs +*/ + +struct remote_repository { + struct remote_repository *next, *prev; + + /* repository is determined by pattern name currently. + * That might change if there is some safe way to combine + * some. (note that method options might make equally looking + * repositories different ones, so that is hard to decide). + * + * This is possible as pattern is not modifyable in options + * or method by the using distribution. + */ + const char *name; + const char *method; + const char *fallback; + const struct strlist *config; + + struct aptmethod *download; + + struct remote_distribution *distributions; +}; +static struct remote_repository *repositories = NULL; + +struct remote_distribution { + struct remote_distribution *next; + + /* repository and suite uniquely identify it, + as the only thing the distribution can change is the suite. + Currently most of the other fields would also fit in the + remote_repository structure, but I plan to add new patters + allowing this by distribution... + */ + struct remote_repository *repository; + char *suite; + + /* flat repository */ + bool flat; bool flatnonflatwarned; + char *suite_base_dir; + + /* if true, do not download or check Release file */ + bool ignorerelease; + /* hashes to ignore */ + bool ignorehashes[cs_hashCOUNT]; + + /* linked list of key descriptions to check against, each must match */ + struct signature_requirement *verify; + + /* local copy of InRelease, Release and Release.gpg file, + * only set if available */ + char *inreleasefile; + char *releasefile; + char *releasegpgfile; + const char *usedreleasefile; + + /* filenames and checksums from the Release file */ + struct checksumsarray remotefiles; + + /* the index files we need */ + struct remote_index *indices; + + /* InRelease failed or requested not to be used */ + bool noinrelease; +}; + +struct remote_index { + /* next index in remote distribution */ + struct remote_index *next; + + struct remote_distribution *from; + + /* what to download? .gz better than .bz2? and so on */ + struct encoding_preferences downloadas; + + /* remote filename as to be found in Release file*/ + char *filename_in_release; + + /* the name without suffix in the lists/ dir */ + char *cachefilename; + /* the basename of the above */ + const char *cachebasename; + + /* index in checksums for the different types, -1 = not avail */ + int ofs[c_COUNT], diff_ofs; + + /* index in requested download methods so we can continue later */ + int lasttriedencoding; + /* the compression to be tried currently */ + enum compression compression; + + /* the old uncompressed file, so that it is only deleted + * when needed, to avoid losing it for a patch run */ + /*@dependant@*/struct cachedlistfile *olduncompressed; + struct checksums *oldchecksums; + + /* if using pdiffs, the content of the Packages.diff/Index: */ + struct diffindex *diffindex; + /* the last patch queued to be applied */ + char *patchfilename; + /*@dependant@*/const struct diffindex_patch *selectedpatch; + bool deletecompressedpatch; + + bool queued; + bool needed; + bool got; +}; + +#define MAXPARTS 5 +struct cachedlistfile { + struct cachedlistfile *next; + const char *basefilename; + unsigned int partcount; + const char *parts[MAXPARTS]; + /* might be used by some rule */ + bool needed, deleted; + char fullfilename[]; +}; + + +static void remote_index_free(/*@only@*/struct remote_index *i) { + if (i == NULL) + return; + free(i->cachefilename); + free(i->patchfilename); + free(i->filename_in_release); + diffindex_free(i->diffindex); + checksums_free(i->oldchecksums); + free(i); +} + +static void remote_distribution_free(/*@only@*/struct remote_distribution *d) { + if (d == NULL) + return; + free(d->suite); + signature_requirements_free(d->verify); + free(d->inreleasefile); + free(d->releasefile); + free(d->releasegpgfile); + free(d->suite_base_dir); + checksumsarray_done(&d->remotefiles); + while (d->indices != NULL) { + struct remote_index *h = d->indices; + d->indices = h->next; + remote_index_free(h); + } + free(d); +} + +void remote_repository_free(struct remote_repository *remote) { + if (remote == NULL) + return; + while (remote->distributions != NULL) { + struct remote_distribution *h = remote->distributions; + remote->distributions = h->next; + remote_distribution_free(h); + } + if (remote->next != NULL) + remote->next->prev = remote->prev; + if (remote->prev != NULL) + remote->prev->next = remote->next; + free(remote); + return; +} + +void cachedlistfile_freelist(struct cachedlistfile *c) { + while (c != NULL) { + struct cachedlistfile *n = c->next; + free(c); + c = n; + } +} + +void cachedlistfile_deleteunneeded(const struct cachedlistfile *c) { + for (; c != NULL ; c = c->next) { + if (c->needed) + continue; + if (verbose >= 0) + printf("deleting %s\n", c->fullfilename); + deletefile(c->fullfilename); + } +} + +static /*@null@*/ struct cachedlistfile *cachedlistfile_new(const char *basefilename, size_t len, size_t listdirlen) { + struct cachedlistfile *c; + size_t l; + char *p; + char ch; + + c = malloc(sizeof(struct cachedlistfile) + listdirlen + 2*len + 3); + if (FAILEDTOALLOC(c)) + return NULL; + c->next = NULL; + c->needed = false; + c->deleted = false; + p = c->fullfilename; + assert ((size_t)(p - (char*)c) <= sizeof(struct cachedlistfile)); + memcpy(p, global.listdir, listdirlen); + p += listdirlen; + *(p++) = '/'; + assert ((size_t)(p - c->fullfilename) == listdirlen + 1); + c->basefilename = p; + memcpy(p, basefilename, len); p += len; + *(p++) = '\0'; + assert ((size_t)(p - c->fullfilename) == listdirlen + len + 2); + + c->parts[0] = p; + c->partcount = 1; + l = len; + while (l-- > 0 && (ch = *(basefilename++)) != '\0') { + if (ch == '_') { + *(p++) = '\0'; + if (c->partcount < MAXPARTS) + c->parts[c->partcount] = p; + c->partcount++; + } else if (ch == '%') { + char first, second; + + if (len <= 1) { + c->partcount = 0; + return c; + } + first = *(basefilename++); + second = *(basefilename++); + if (first >= '0' && first <= '9') + *p = (first - '0') << 4; + else if (first >= 'a' && first <= 'f') + *p = (first - 'a' + 10) << 4; + else { + c->partcount = 0; + return c; + } + if (second >= '0' && second <= '9') + *p |= (second - '0'); + else if (second >= 'a' && second <= 'f') + *p |= (second - 'a' + 10); + else { + c->partcount = 0; + return c; + } + p++; + } else + *(p++) = ch; + } + *(p++) = '\0'; + assert ((size_t)(p - c->fullfilename) <= listdirlen + 2*len + 3); + return c; +} + +retvalue cachedlists_scandir(/*@out@*/struct cachedlistfile **cachedfiles_p) { + struct cachedlistfile *cachedfiles = NULL, **next_p; + struct dirent *r; + size_t listdirlen = strlen(global.listdir); + DIR *dir; + + // TODO: check if it is always created before... + dir = opendir(global.listdir); + if (dir == NULL) { + int e = errno; + fprintf(stderr, +"Error %d opening directory '%s': %s!\n", + e, global.listdir, strerror(e)); + return RET_ERRNO(e); + } + next_p = &cachedfiles; + while (true) { + size_t namelen; + int e; + + errno = 0; + r = readdir(dir); + if (r == NULL) { + e = errno; + if (e == 0) + break; + /* this should not happen... */ + e = errno; + fprintf(stderr, "Error %d reading dir '%s': %s!\n", + e, global.listdir, strerror(e)); + (void)closedir(dir); + cachedlistfile_freelist(cachedfiles); + return RET_ERRNO(e); + } + namelen = _D_EXACT_NAMLEN(r); + if (namelen == 1 && r->d_name[0] == '.') + continue; + if (namelen == 2 && r->d_name[0] == '.' && r->d_name[1] == '.') + continue; + *next_p = cachedlistfile_new(r->d_name, namelen, listdirlen); + if (FAILEDTOALLOC(*next_p)) { + (void)closedir(dir); + cachedlistfile_freelist(cachedfiles); + return RET_ERROR_OOM; + } + next_p = &(*next_p)->next; + } + if (closedir(dir) != 0) { + int e = errno; + fprintf(stderr, "Error %d closing directory '%s': %s!\n", + e, global.listdir, strerror(e)); + cachedlistfile_freelist(cachedfiles); + return RET_ERRNO(e); + } + *cachedfiles_p = cachedfiles; + return RET_OK; +} + +static retvalue cachedlistfile_delete(struct cachedlistfile *old) { + int e; + if (old->deleted) + return RET_OK; + e = deletefile(old->fullfilename); + if (e != 0) + return RET_ERRNO(e); + old->deleted = true; + return RET_OK; +} + +struct remote_repository *remote_repository_prepare(const char *name, const char *method, const char *fallback, const struct strlist *config) { + struct remote_repository *n; + + /* calling code ensures no two with the same name are created, + * so just create it... */ + + n = zNEW(struct remote_repository); + if (FAILEDTOALLOC(n)) + return NULL; + n->name = name; + n->method = method; + n->fallback = fallback; + n->config = config; + + n->next = repositories; + if (n->next != NULL) + n->next->prev = n; + repositories = n; + + return n; +} + +/* This escaping is quite harsh, but so nothing bad can happen... */ +static inline size_t escapedlen(const char *p) { + size_t l = 0; + if (*p == '-') { + l = 3; + p++; + } + while (*p != '\0') { + if ((*p < 'A' || *p > 'Z') && (*p < 'a' || *p > 'z') && + (*p < '0' || *p > '9') && *p != '-') + l +=3; + else + l++; + p++; + } + return l; +} + +static inline char *escapedcopy(char *dest, const char *orig) { + static char hex[16] = "0123456789ABCDEF"; + if (*orig == '-') { + orig++; + *dest = '%'; dest++; + *dest = '2'; dest++; + *dest = 'D'; dest++; + } + while (*orig != '\0') { + if ((*orig < 'A' || *orig > 'Z') + && (*orig < 'a' || *orig > 'z') + && (*orig < '0' || *orig > '9') + && *orig != '-') { + *dest = '%'; dest++; + *dest = hex[(*orig >> 4)& 0xF ]; dest++; + *dest = hex[*orig & 0xF ]; dest++; + } else { + *dest = *orig; + dest++; + } + orig++; + } + return dest; +} + +char *genlistsfilename(const char *type, unsigned int count, ...) { + const char *fields[count]; + unsigned int i; + size_t listdir_len, type_len, len; + char *result, *p; + va_list ap; + + len = 0; + va_start(ap, count); + for (i = 0 ; i < count ; i++) { + fields[i] = va_arg(ap, const char*); + assert (fields[i] != NULL); + len += escapedlen(fields[i]) + 1; + } + /* check sentinel */ + assert (va_arg(ap, const char*) == NULL); + va_end(ap); + listdir_len = strlen(global.listdir); + if (type != NULL) + type_len = strlen(type); + else + type_len = 0; + + result = malloc(listdir_len + type_len + len + 2); + if (FAILEDTOALLOC(result)) + return NULL; + memcpy(result, global.listdir, listdir_len); + p = result + listdir_len; + *(p++) = '/'; + for (i = 0 ; i < count ; i++) { + p = escapedcopy(p, fields[i]); + *(p++) = '_'; + } + assert ((size_t)(p - result) == listdir_len + len + 1); + if (type != NULL) + memcpy(p, type, type_len + 1); + else + *(--p) = '\0'; + return result; +} + +void cachedlistfile_need(struct cachedlistfile *list, const char *type, unsigned int count, ...) { + struct cachedlistfile *file; + const char *fields[count]; + unsigned int i; + va_list ap; + + va_start(ap, count); + for (i = 0 ; i < count ; i++) { + fields[i] = va_arg(ap, const char*); + assert (fields[i] != NULL); + } + /* check sentinel */ + assert (va_arg(ap, const char*) == NULL); + va_end(ap); + + for (file = list ; file != NULL ; file = file->next) { + if (file->partcount != count + 1) + continue; + i = 0; + while (i < count && strcmp(file->parts[i], fields[i]) == 0) + i++; + if (i < count) + continue; + if (strcmp(type, file->parts[i]) != 0) + continue; + file->needed = true; + } +} + +retvalue remote_distribution_prepare(struct remote_repository *repository, const char *suite, bool ignorerelease, bool getinrelease, const char *verifyrelease, bool flat, bool *ignorehashes, struct remote_distribution **out_p) { + struct remote_distribution *n, **last; + enum checksumtype cs; + + for (last = &repository->distributions ; (n = *last) != NULL + ; last = &n->next) { + if (strcmp(n->suite, suite) != 0) + continue; + if (n->flat != flat) { + if (verbose >= 0 && !n->flatnonflatwarned && + !IGNORABLE(flatandnonflat)) + fprintf(stderr, +"Warning: From the same remote repository '%s', distribution '%s'\n" +"is requested both flat and non-flat. While this is possible\n" +"(having %s/dists/%s and %s/%s), it is unlikely.\n" +"To no longer see this message, use --ignore=flatandnonflat.\n", + repository->method, suite, + repository->method, suite, + repository->method, suite); + n->flatnonflatwarned = true; + continue; + } + break; + } + + if (*last != NULL) { + n = *last; + assert (n->flat == flat); + + if ((n->ignorerelease && !ignorerelease) || + (!n->ignorerelease && ignorerelease)) { + // TODO a hint which two are at fault would be nice, + // but how to get the information... + if (verbose >= 0) + fprintf(stderr, +"Warning: I was told to both ignore Release files for Suite '%s'\n" +"from remote repository '%s' and to not ignore it. Going to not ignore!\n", + suite, repository->name); + n->ignorerelease = false; + } + if ((n->noinrelease && getinrelease) || + (!n->noinrelease && !getinrelease)) { + if (verbose >= 0) + fprintf(stderr, +"Warning: Conflicting GetInRelease values for Suite '%s'\n" +"from remote repository '%s'. Resolving to get InRelease files!\n", + suite, repository->name); + n->noinrelease = false; + } + for (cs = cs_md5sum ; cs < cs_hashCOUNT ; cs++) { + if ((n->ignorehashes[cs] && !ignorehashes[cs]) || + (!n->ignorehashes[cs] && ignorehashes[cs])) { + // TODO dito + if (verbose >= 0) + fprintf(stderr, +"Warning: I was told to both ignore '%s' for Suite '%s'\n" +"from remote repository '%s' and to not ignore it. Going to not ignore!\n", + suite, + release_checksum_names[cs], + repository->name); + n->ignorehashes[cs] = false; + } + } + if (verifyrelease != NULL) { + retvalue r; + + r = signature_requirement_add(&n->verify, + verifyrelease); + if (RET_WAS_ERROR(r)) + return r; + } + *out_p = n; + return RET_OK; + } + + n = zNEW(struct remote_distribution); + if (FAILEDTOALLOC(n)) + return RET_ERROR_OOM; + n->repository = repository; + n->suite = strdup(suite); + n->ignorerelease = ignorerelease; + n->noinrelease = !getinrelease; + if (verifyrelease != NULL) { + retvalue r; + + r = signature_requirement_add(&n->verify, verifyrelease); + if (RET_WAS_ERROR(r)) { + remote_distribution_free(n); + return r; + } + } + memcpy(n->ignorehashes, ignorehashes, sizeof(bool [cs_hashCOUNT])); + n->flat = flat; + if (flat) + n->suite_base_dir = strdup(suite); + else + n->suite_base_dir = calc_dirconcat("dists", suite); + if (FAILEDTOALLOC(n->suite) || + FAILEDTOALLOC(n->suite_base_dir)) { + remote_distribution_free(n); + return RET_ERROR_OOM; + } + /* ignorerelease can be unset later, so always calculate the filename */ + if (flat) + n->inreleasefile = genlistsfilename("InRelease", 3, + repository->name, suite, "flat", + ENDOFARGUMENTS); + else + n->inreleasefile = genlistsfilename("InRelease", 2, + repository->name, suite, ENDOFARGUMENTS); + if (FAILEDTOALLOC(n->inreleasefile)) { + remote_distribution_free(n); + return RET_ERROR_OOM; + } + if (flat) + n->releasefile = genlistsfilename("Release", 3, + repository->name, suite, "flat", + ENDOFARGUMENTS); + else + n->releasefile = genlistsfilename("Release", 2, + repository->name, suite, ENDOFARGUMENTS); + if (FAILEDTOALLOC(n->releasefile)) { + remote_distribution_free(n); + return RET_ERROR_OOM; + } + n->releasegpgfile = calc_addsuffix(n->releasefile, "gpg"); + if (FAILEDTOALLOC(n->releasefile)) { + remote_distribution_free(n); + return RET_ERROR_OOM; + } + *last = n; + *out_p = n; + return RET_OK; +} + +static retvalue copytoplace(const char *gotfilename, const char *wantedfilename, const char *method, struct checksums **checksums_p) { + retvalue r; + struct checksums *checksums = NULL; + + /* if the file is somewhere else, copy it: */ + if (strcmp(gotfilename, wantedfilename) != 0) { + /* never link index files, but copy them */ + if (verbose > 1) + fprintf(stderr, +"Copy file '%s' to '%s'...\n", gotfilename, wantedfilename); + r = checksums_copyfile(wantedfilename, gotfilename, false, + &checksums); + if (r == RET_ERROR_EXIST) { + fprintf(stderr, +"Unexpected error: '%s' exists while it should not!\n", + wantedfilename); + } + if (r == RET_NOTHING) { + fprintf(stderr, +"Cannot open '%s', obtained from '%s' method.\n", + gotfilename, method); + r = RET_ERROR_MISSING; + } + if (RET_WAS_ERROR(r)) { + return r; + } + } + if (checksums_p == NULL) + checksums_free(checksums); + else + *checksums_p = checksums; + return RET_OK; +} + +static retvalue enqueue_old_release_files(struct remote_distribution *d); + +/* handle a downloaded Release or Release.gpg file: + * no checksums to test, nothing to trigger, as they have to be all + * read at once to decide what is new and what actually needs downloading */ +static retvalue release_callback(enum queue_action action, void *privdata, void *privdata2, UNUSED(const char *uri), const char *gotfilename, const char *wantedfilename, UNUSED(/*@null@*/const struct checksums *checksums), const char *methodname) { + struct remote_distribution *d = privdata; + retvalue r; + + /* if the InRelease file cannot be got, + * try Release (and Release.gpg if checking) instead */ + if (action == qa_error && privdata2 == d->inreleasefile) { + assert (!d->noinrelease); + + return enqueue_old_release_files(d); + } + + if (action != qa_got) + return RET_ERROR; + + r = copytoplace(gotfilename, wantedfilename, methodname, NULL); + if (RET_WAS_ERROR(r)) + return r; + return r; +} + +static retvalue enqueue_old_release_files(struct remote_distribution *d) { + retvalue r; + + d->noinrelease = true; + r = aptmethod_enqueueindex(d->repository->download, + d->suite_base_dir, "Release", "", + d->releasefile, "", + release_callback, d, NULL); + if (RET_WAS_ERROR(r)) + return r; + if (d->verify != NULL) { + r = aptmethod_enqueueindex(d->repository->download, + d->suite_base_dir, "Release", ".gpg", + d->releasegpgfile, "", + release_callback, d, NULL); + if (RET_WAS_ERROR(r)) + return r; + } + return RET_OK; +} + +static retvalue remote_distribution_enqueuemetalists(struct remote_distribution *d) { + struct remote_repository *repository = d->repository; + + assert (repository->download != NULL); + + if (d->ignorerelease) + return RET_NOTHING; + + (void)unlink(d->inreleasefile); + (void)unlink(d->releasefile); + if (d->verify != NULL) { + (void)unlink(d->releasegpgfile); + } + + if (d->noinrelease) + return enqueue_old_release_files(d); + else + return aptmethod_enqueueindex(repository->download, + d->suite_base_dir, "InRelease", "", d->inreleasefile, + "", release_callback, d, d->inreleasefile); +} + +retvalue remote_startup(struct aptmethodrun *run) { + struct remote_repository *rr; + retvalue r; + + if (interrupted()) + return RET_ERROR_INTERRUPTED; + + for (rr = repositories ; rr != NULL ; rr = rr->next) { + assert (rr->download == NULL); + + r = aptmethod_newmethod(run, + rr->method, rr->fallback, + rr->config, &rr->download); + if (RET_WAS_ERROR(r)) + return r; + } + return RET_OK; +} + +static void find_index(const struct strlist *files, struct remote_index *ri) { + const char *filename = ri->filename_in_release; + size_t len = strlen(filename); + int i; + enum compression c; + + for (i = 0 ; i < files->count ; i++) { + const char *value = files->values[i]; + + if (strncmp(value, filename, len) != 0) + continue; + + value += len; + + if (*value == '\0') { + ri->ofs[c_none] = i; + continue; + } + if (*value != '.') + continue; + if (strcmp(value, ".diff/Index") == 0) { + ri->diff_ofs = i; + continue; + } + + for (c = 0 ; c < c_COUNT ; c++) + if (strcmp(value, uncompression_suffix[c]) == 0) { + ri->ofs[c] = i; + break; + } + } +} + +/* get a strlist with the md5sums of a Release-file */ +static inline retvalue release_getchecksums(const char *releasefile, const char *chunk, const bool ignorehash[cs_hashCOUNT], struct checksumsarray *out) { + retvalue r; + struct strlist files[cs_hashCOUNT]; + enum checksumtype cs; + bool foundanything = false; + + for (cs = cs_md5sum ; cs < cs_hashCOUNT ; cs++) { + if (ignorehash[cs]) { + strlist_init(&files[cs]); + continue; + } + assert (release_checksum_names[cs] != NULL); + r = chunk_getextralinelist(chunk, release_checksum_names[cs], + &files[cs]); + if (RET_WAS_ERROR(r)) { + while (cs-- > cs_md5sum) { + strlist_done(&files[cs]); + } + return r; + } else if (r == RET_NOTHING) + strlist_init(&files[cs]); + else + foundanything = true; + } + + if (!foundanything) { + fprintf(stderr, "Missing checksums in Release file '%s'!\n", + releasefile); + return RET_ERROR; + } + + r = checksumsarray_parse(out, files, releasefile); + for (cs = cs_md5sum ; cs < cs_hashCOUNT ; cs++) { + strlist_done(&files[cs]); + } + return r; +} + +static retvalue process_remoterelease(struct remote_distribution *rd) { + struct remote_repository *rr = rd->repository; + struct remote_index *ri; + retvalue r; + char *releasedata; + size_t releaselen; + + if (!rd->noinrelease) { + r = signature_check_inline(rd->verify, + rd->inreleasefile, &releasedata); + assert (r != RET_NOTHING); + if (r == RET_NOTHING) + r = RET_ERROR_BADSIG; + if (r == RET_ERROR_BADSIG) { + fprintf(stderr, +"Error: Not enough signatures found for remote repository %s (%s %s)!\n", + rr->name, rr->method, rd->suite); + r = RET_ERROR_BADSIG; + } + if (RET_WAS_ERROR(r)) + return r; + rd->usedreleasefile = rd->inreleasefile; + } else { + r = readtextfile(rd->releasefile, rd->releasefile, + &releasedata, &releaselen); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) + return r; + rd->usedreleasefile = rd->releasefile; + + if (rd->verify != NULL) { + r = signature_check(rd->verify, + rd->releasegpgfile, rd->releasefile, + releasedata, releaselen); + assert (r != RET_NOTHING); + if (r == RET_NOTHING) + r = RET_ERROR_BADSIG; + if (r == RET_ERROR_BADSIG) { + fprintf(stderr, +"Error: Not enough signatures found for remote repository %s (%s %s)!\n", + rr->name, rr->method, rd->suite); + r = RET_ERROR_BADSIG; + } + if (RET_WAS_ERROR(r)) { + free(releasedata); + return r; + } + } + } + r = release_getchecksums(rd->usedreleasefile, releasedata, + rd->ignorehashes, &rd->remotefiles); + free(releasedata); + if (RET_WAS_ERROR(r)) + return r; + + /* Check for our files in there */ + for (ri = rd->indices ; ri != NULL ; ri = ri->next) { + find_index(&rd->remotefiles.names, ri); + } + // TODO: move checking if not exists at all to here? + return RET_OK; +} + +retvalue remote_preparemetalists(struct aptmethodrun *run, bool nodownload) { + struct remote_repository *rr; + struct remote_distribution *rd; + retvalue r; + + if (!nodownload) { + for (rr = repositories ; rr != NULL ; rr = rr->next) { + for (rd = rr->distributions ; rd != NULL ; + rd = rd->next) { + r = remote_distribution_enqueuemetalists(rd); + if (RET_WAS_ERROR(r)) + return r; + } + } + r = aptmethod_download(run); + if (RET_WAS_ERROR(r)) + return r; + } + for (rr = repositories ; rr != NULL ; rr = rr->next) { + for (rd = rr->distributions ; rd != NULL ; rd = rd->next) { + if (!rd->ignorerelease) { + if (nodownload) + if (!isregularfile(rd->inreleasefile)) + rd->noinrelease = true; + r = process_remoterelease(rd); + if (RET_WAS_ERROR(r)) + return r; + } + } + } + return RET_OK; +} + +bool remote_index_isnew(/*@null@*/const struct remote_index *ri, struct donefile *done) { + const char *basefilename; + struct checksums *checksums; + bool hashes_missing, improves; + + /* files without uncompressed checksum cannot be tested */ + if (ri->ofs[c_none] < 0) + return true; + /* if not there or the wrong files comes next, then something + * has changed and we better reload everything */ + if (!donefile_nextindex(done, &basefilename, &checksums)) + return true; + if (strcmp(basefilename, ri->cachebasename) != 0) { + checksums_free(checksums); + return true; + } + /* otherwise check if the file checksums match */ + if (!checksums_check(checksums, + ri->from->remotefiles.checksums[ri->ofs[c_none]], + &hashes_missing)) { + checksums_free(checksums); + return true; + } + if (hashes_missing) { + /* if Release has checksums we do not yet know about, + * process it to make sure those match as well */ + checksums_free(checksums); + return true; + } + if (!checksums_check(ri->from->remotefiles.checksums[ri->ofs[c_none]], + checksums, &improves)) { + /* this should not happen, but ... */ + checksums_free(checksums); + return true; + } + if (improves) { + /* assume this is our file and add the other hashes so they + * will show up in the file again the next time. + * This is a bit unelegant in mixing stuff, but otherwise this + * will cause redownloading when remote adds more hashes. + * The only downside of mixing can reject files that have the + * same recorded hashes as a previously processed files. + * But that is quite inlikely unless on attack, so getting some + * hint in that case cannot harm.*/ + (void)checksums_combine(&ri->from->remotefiles.checksums[ + ri->ofs[c_none]], checksums, NULL); + } + checksums_free(checksums); + return false; +} + +static inline void remote_index_oldfiles(struct remote_index *ri, /*@null@*/struct cachedlistfile *oldfiles, /*@out@*/struct cachedlistfile *old[c_COUNT]) { + struct cachedlistfile *o; + size_t l; + enum compression c; + + for (c = 0 ; c < c_COUNT ; c++) + old[c] = NULL; + + l = strlen(ri->cachebasename); + for (o = oldfiles ; o != NULL ; o = o->next) { + if (o->deleted) + continue; + if (strncmp(o->basefilename, ri->cachebasename, l) != 0) + continue; + for (c = 0 ; c < c_COUNT ; c++) + if (strcmp(o->basefilename + l, + uncompression_suffix[c]) == 0) { + old[c] = o; + o->needed = true; + break; + } + if (strcmp(o->basefilename + l, ".diffindex") == 0) + (void)cachedlistfile_delete(o); + if (strncmp(o->basefilename + l, ".diff-", 6) == 0) + (void)cachedlistfile_delete(o); + } +} + +static inline void remote_index_delete_oldfiles(struct remote_index *ri, /*@null@*/struct cachedlistfile *oldfiles) { + struct cachedlistfile *o; + size_t l; + + l = strlen(ri->cachebasename); + for (o = oldfiles ; o != NULL ; o = o->next) { + if (o->deleted) + continue; + if (strncmp(o->basefilename, ri->cachebasename, l) != 0) + continue; + (void)cachedlistfile_delete(o); + } +} + +static queue_callback index_callback; +static queue_callback diff_callback; + +static retvalue queue_next_without_release(struct remote_distribution *rd, struct remote_index *ri) { + const struct encoding_preferences *downloadas; + static const struct encoding_preferences defaultdownloadas = { + .count = 5, + .requested = { + { .diff = false, .force = false, .compression = c_gzip }, + { .diff = false, .force = false, .compression = c_bzip2 }, + { .diff = false, .force = false, .compression = c_none }, + { .diff = false, .force = false, .compression = c_lzma }, + { .diff = false, .force = false, .compression = c_xz } + } + }; + int e; + + if (ri->downloadas.count == 0) + downloadas = &defaultdownloadas; + else + downloadas = &ri->downloadas; + + for (e = ri->lasttriedencoding + 1 ; e < downloadas->count ; e++) { + enum compression c = downloadas->requested[e].compression; + + if (downloadas->requested[e].diff) + continue; + if (uncompression_supported(c)) { + ri->lasttriedencoding = e; + ri->compression = c; + return aptmethod_enqueueindex(rd->repository->download, + rd->suite_base_dir, + ri->filename_in_release, + uncompression_suffix[c], + ri->cachefilename, + uncompression_suffix[c], + index_callback, ri, NULL); + } + } + if (ri->lasttriedencoding < 0) + fprintf(stderr, +"ERROR: no supported compressions in DownloadListsAs for '%s' by '%s'!\n", + rd->suite, rd->repository->method); + ri->lasttriedencoding = e; + return RET_ERROR; +} + +static inline retvalue find_requested_encoding(struct remote_index *ri, const char *releasefile) { + int e; + enum compression c, stopat, + /* the most-preferred requested but unsupported */ + unsupported = c_COUNT, + /* the best unrequested but supported */ + unrequested = c_COUNT; + + if (ri->downloadas.count > 0) { + bool found = false; + for (e = ri->lasttriedencoding + 1 ; + e < ri->downloadas.count ; + e++) { + struct compression_preference req; + + req = ri->downloadas.requested[e]; + + if (req.diff) { + if (ri->olduncompressed == NULL) + continue; + assert (ri->ofs[c_none] >= 0); + if (!req.force && ri->diff_ofs < 0) + continue; + ri->compression = c_COUNT; + ri->lasttriedencoding = e; + return RET_OK; + } + if (ri->ofs[req.compression] < 0 && + (!req.force || ri->ofs[c_none] < 0)) + continue; + if (uncompression_supported(req.compression)) { + ri->compression = req.compression; + ri->lasttriedencoding = e; + return RET_OK; + } else if (unsupported == c_COUNT) + unsupported = req.compression; + } + if (ri->lasttriedencoding > -1) { + /* we already tried something, and nothing else + * is available, so give up */ + ri->lasttriedencoding = e; + return RET_ERROR; + } + + /* nothing that is both requested by the user and supported + * and listed in the Release file found, check what is there + * to get a meaningful error message */ + + for (c = 0 ; c < c_COUNT ; c++) { + if (ri->ofs[c] < 0) + continue; + found = true; + if (uncompression_supported(c)) + unrequested = c; + } + + if (!found) { + // TODO: might be nice to check for not-yet-even + // known about compressions and say they are not + // yet know yet instead then here... + fprintf(stderr, +"Could not find '%s' within '%s'\n", + ri->filename_in_release, releasefile); + return RET_ERROR_WRONG_MD5; + } + + if (unsupported != c_COUNT && unrequested != c_COUNT) { + fprintf(stderr, +"Error: '%s' only lists unusable or unrequested compressions of '%s'.\n" +"Try e.g the '%s' option (or check what it is set to) to make more useable.\n" +"Or change your DownloadListsAs to request e.g. '%s'.\n", + releasefile, ri->filename_in_release, + uncompression_option[unsupported], + uncompression_config[unrequested]); + return RET_ERROR; + } + if (unsupported != c_COUNT) { + fprintf(stderr, +"Error: '%s' only lists unusable compressions of '%s'.\n" +"Try e.g the '%s' option (or check what it is set to) to make more useable.\n", + releasefile, ri->filename_in_release, + uncompression_option[unsupported]); + return RET_ERROR; + } + if (unrequested != c_COUNT) { + fprintf(stderr, +"Error: '%s' only lists unrequested compressions of '%s'.\n" +"Try changing your DownloadListsAs to request e.g. '%s'.\n", + releasefile, ri->filename_in_release, + uncompression_config[unrequested]); + return RET_ERROR; + } + fprintf(stderr, +"Error: '%s' lists no requested and usable compressions of '%s'.\n", + releasefile, ri->filename_in_release); + return RET_ERROR; + } + /* When nothing specified, use the newest compression. + * This might make it slow on older computers (and perhaps + * on relatively new ones, too), but usually bandwidth costs + * and your time not. + * And you can always configure it to prefer a faster one... + */ + + /* ri->lasttriedencoding -1 means nothing tried, + * 0 means Package.diff was tried, + * 1 means nothing c_COUNT - 1 was already tried, + * 2 means nothing c_COUNT - 2 was already tried, + * and so on...*/ + + if (ri->lasttriedencoding < 0) { + if (ri->olduncompressed != NULL && ri->diff_ofs >= 0) { + ri->compression = c_COUNT; + ri->lasttriedencoding = 0; + return RET_OK; + } + stopat = c_COUNT; + } else + stopat = c_COUNT - ri->lasttriedencoding; + + ri->compression = c_COUNT; + for (c = 0 ; c < stopat ; c++) { + if (ri->ofs[c] < 0) + continue; + if (uncompression_supported(c)) + ri->compression = c; + else + unsupported = c; + } + if (ri->compression == c_COUNT) { + if (ri->lasttriedencoding > -1) { + /* not the first try, no error message needed */ + ri->lasttriedencoding = c_COUNT; + return RET_ERROR; + } + if (unsupported != c_COUNT) { + fprintf(stderr, +"Error: '%s' only lists unusable compressions of '%s'.\n" +"Try e.g the '%s' option (or check what it is set to) to enable more.\n", + releasefile, ri->filename_in_release, + uncompression_option[unsupported]); + return RET_ERROR; + } + fprintf(stderr, +"Could not find '%s' within '%s'\n", + ri->filename_in_release, releasefile); + return RET_ERROR_WRONG_MD5; + + } + ri->lasttriedencoding = c_COUNT - ri->compression; + return RET_OK; +} + +static inline retvalue remove_old_uncompressed(struct remote_index *ri) { + retvalue r; + + if (ri->olduncompressed != NULL) { + r = cachedlistfile_delete(ri->olduncompressed); + ri->olduncompressed = NULL; + return r; + } else + return RET_NOTHING; +} + +static retvalue queue_next_encoding(struct remote_distribution *rd, struct remote_index *ri); + +// TODO: check if this still makes sense. +// (might be left over to support switching from older versions +// of reprepro that also put compressed files there) +static inline retvalue reuse_old_compressed_index(struct remote_distribution *rd, struct remote_index *ri, enum compression c, const char *oldfullfilename) { + retvalue r; + + r = uncompress_file(oldfullfilename, ri->cachefilename, c); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) + return r; + if (ri->ofs[c_none] >= 0) { + r = checksums_test(ri->cachefilename, + rd->remotefiles.checksums[ri->ofs[c_none]], + &rd->remotefiles.checksums[ri->ofs[c_none]]); + if (r == RET_ERROR_WRONG_MD5) { + fprintf(stderr, +"Error: File '%s' looked correct according to '%s',\n" +"but after unpacking '%s' looks wrong.\n" +"Something is seriously broken!\n", + oldfullfilename, rd->usedreleasefile, + ri->cachefilename); + } + if (r == RET_NOTHING) { + fprintf(stderr, +"File '%s' mysteriously vanished!\n", ri->cachefilename); + r = RET_ERROR_MISSING; + } + if (RET_WAS_ERROR(r)) + return r; + } + /* already there, nothing to do to get it... */ + ri->queued = true; + ri->got = true; + return RET_OK; +} + +static inline retvalue queueindex(struct remote_distribution *rd, struct remote_index *ri, bool nodownload, /*@null@*/struct cachedlistfile *oldfiles) { + enum compression c; + retvalue r; + struct cachedlistfile *old[c_COUNT]; + + if (rd->ignorerelease) { + ri->queued = true; + if (nodownload) { + ri->got = true; + return RET_OK; + } + + /* as there is no way to know which are current, + * just delete everything */ + remote_index_delete_oldfiles(ri, oldfiles); + + return queue_next_without_release(rd, ri); + } + + /* check if this file is still available from an earlier download */ + remote_index_oldfiles(ri, oldfiles, old); + ri->olduncompressed = NULL; + ri->oldchecksums = NULL; + if (ri->ofs[c_none] < 0 && old[c_none] != NULL) { + /* if we know not what it should be, + * we canot use the old... */ + r = cachedlistfile_delete(old[c_none]); + if (RET_WAS_ERROR(r)) + return r; + old[c_none] = NULL; + } else if (old[c_none] != NULL) { + bool improves; + int uo = ri->ofs[c_none]; + struct checksums **wanted_p = &rd->remotefiles.checksums[uo]; + + r = checksums_read(old[c_none]->fullfilename, + &ri->oldchecksums); + if (r == RET_NOTHING) { + fprintf(stderr, "File '%s' mysteriously vanished!\n", + old[c_none]->fullfilename); + r = RET_ERROR_MISSING; + } + if (RET_WAS_ERROR(r)) + return r; + if (checksums_check(*wanted_p, ri->oldchecksums, &improves)) { + /* already there, nothing to do to get it... */ + ri->queued = true; + ri->got = true; + if (improves) + r = checksums_combine(wanted_p, + ri->oldchecksums, NULL); + else + r = RET_OK; + checksums_free(ri->oldchecksums); + ri->oldchecksums = NULL; + return r; + } + ri->olduncompressed = old[c_none]; + old[c_none] = NULL; + } + + assert (old[c_none] == NULL); + + /* make sure everything old is deleted or check if it can be used */ + for (c = 0 ; c < c_COUNT ; c++) { + if (old[c] == NULL) + continue; + if (c != c_none && ri->ofs[c] >= 0) { + /* check if it can be used */ + r = checksums_test(old[c]->fullfilename, + rd->remotefiles.checksums[ri->ofs[c]], + &rd->remotefiles.checksums[ri->ofs[c]]); + if (r == RET_ERROR_WRONG_MD5) + r = RET_NOTHING; + if (RET_WAS_ERROR(r)) + return r; + if (RET_IS_OK(r)) { + r = remove_old_uncompressed(ri); + if (RET_WAS_ERROR(r)) + return r; + assert (old[c_none] == NULL); + return reuse_old_compressed_index(rd, ri, c, + old[c]->fullfilename); + } + } + r = cachedlistfile_delete(old[c]); + if (RET_WAS_ERROR(r)) + return r; + old[c] = NULL; + } + + /* nothing found, we'll have to download: */ + + if (nodownload) { + if (ri->olduncompressed != NULL) + fprintf(stderr, +"Error: '%s' does not match Release file, try without --nolistsdownload to download new one!\n", + ri->cachefilename); + else + fprintf(stderr, +"Error: Missing '%s', try without --nolistsdownload to download it!\n", + ri->cachefilename); + return RET_ERROR_MISSING; + } + + return queue_next_encoding(rd, ri); +} + +static retvalue queue_next_encoding(struct remote_distribution *rd, struct remote_index *ri) { + struct remote_repository *rr = rd->repository; + retvalue r; + + if (rd->ignorerelease) + return queue_next_without_release(rd, ri); + + r = find_requested_encoding(ri, rd->usedreleasefile); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) + return r; + + assert (ri->compression <= c_COUNT); + + /* check if downloading a .diff/Index (aka .pdiff) is requested */ + if (ri->compression == c_COUNT) { + assert (ri->olduncompressed != NULL); + assert (ri->oldchecksums != NULL); + + ri->queued = true; + return aptmethod_enqueueindex(rr->download, rd->suite_base_dir, + ri->filename_in_release, ".diff/Index", + ri->cachefilename, ".diffindex", + diff_callback, ri, NULL); + } + + assert (ri->compression < c_COUNT); + assert (uncompression_supported(ri->compression)); + + if (ri->compression == c_none) { + r = remove_old_uncompressed(ri); + if (RET_WAS_ERROR(r)) + return r; + } +/* as those checksums might be overwritten with completed data, + * this assumes that the uncompressed checksums for one index is never + * the compressed checksum for another... */ + + ri->queued = true; + return aptmethod_enqueueindex(rr->download, rd->suite_base_dir, + ri->filename_in_release, + uncompression_suffix[ri->compression], + ri->cachefilename, + uncompression_suffix[ri->compression], + index_callback, ri, NULL); +} + + +static retvalue remote_distribution_enqueuelists(struct remote_distribution *rd, bool nodownload, struct cachedlistfile *oldfiles) { + struct remote_index *ri; + retvalue r; + + /* check what to get for the requested indicies */ + for (ri = rd->indices ; ri != NULL ; ri = ri->next) { + if (ri->queued) + continue; + if (!ri->needed) { + /* if we do not know anything about it, + * it cannot have got marked as old + * or otherwise as unneeded */ + assert (!rd->ignorerelease); + continue; + } + r = queueindex(rd, ri, nodownload, oldfiles); + if (RET_WAS_ERROR(r)) + return r; + } + return RET_OK; +} + +retvalue remote_preparelists(struct aptmethodrun *run, bool nodownload) { + struct remote_repository *rr; + struct remote_distribution *rd; + retvalue r; + struct cachedlistfile *oldfiles; + + r = cachedlists_scandir(&oldfiles); + if (RET_WAS_ERROR(r)) + return r; + if (r == RET_NOTHING) + oldfiles = NULL; + + for (rr = repositories ; rr != NULL ; rr = rr->next) { + for (rd = rr->distributions ; rd != NULL + ; rd = rd->next) { + r = remote_distribution_enqueuelists(rd, + nodownload, oldfiles); + if (RET_WAS_ERROR(r)) { + cachedlistfile_freelist(oldfiles); + return r; + } + } + } + r = aptmethod_download(run); + if (RET_WAS_ERROR(r)) { + cachedlistfile_freelist(oldfiles); + return r; + } + + cachedlistfile_freelist(oldfiles); + return RET_OK; +} + +static struct remote_index *addindex(struct remote_distribution *rd, /*@only@*/char *cachefilename, /*@only@*/char *filename, /*@null@*/const struct encoding_preferences *downloadas) { + struct remote_index *ri, **last; + enum compression c; + const char *cachebasename; + + if (FAILEDTOALLOC(cachefilename) || FAILEDTOALLOC(filename)) + return NULL; + + cachebasename = dirs_basename(cachefilename); + last = &rd->indices; + while (*last != NULL && strcmp((*last)->cachebasename, cachebasename) != 0) + last = &(*last)->next; + if (*last != NULL) { + ri = *last; + // TODO: perhaps try to calculate some form of intersections + // instead of just using the shorter one... + if (downloadas != NULL && + (ri->downloadas.count == 0 + || ri->downloadas.count > downloadas->count)) + ri->downloadas = *downloadas; + free(cachefilename); free(filename); + return ri; + } + + ri = zNEW(struct remote_index); + if (FAILEDTOALLOC(ri)) { + free(cachefilename); free(filename); + return NULL; + } + + *last = ri; + ri->from = rd; + ri->cachefilename = cachefilename; + ri->cachebasename = cachebasename; + ri->filename_in_release = filename; + if (downloadas != NULL) + ri->downloadas = *downloadas; + for (c = 0 ; c < c_COUNT ; c++) + ri->ofs[c] = -1; + ri->diff_ofs = -1; + ri->lasttriedencoding = -1; + return ri; +} + +struct remote_index *remote_index(struct remote_distribution *rd, const char *architecture, const char *component, packagetype_t packagetype, /*@null@*/const struct encoding_preferences *downloadas) { + char *cachefilename, *filename_in_release; + + assert (!rd->flat); + if (packagetype == pt_deb) { + filename_in_release = mprintf( +"%s/binary-%s/Packages", + component, architecture); + cachefilename = genlistsfilename("Packages", 4, + rd->repository->name, rd->suite, + component, architecture, ENDOFARGUMENTS); + } else if (packagetype == pt_udeb) { + filename_in_release = mprintf( +"%s/debian-installer/binary-%s/Packages", + component, architecture); + cachefilename = genlistsfilename("uPackages", 4, + rd->repository->name, rd->suite, + component, architecture, ENDOFARGUMENTS); + } else if (packagetype == pt_dsc) { + filename_in_release = mprintf( +"%s/source/Sources", + component); + cachefilename = genlistsfilename("Sources", 3, + rd->repository->name, rd->suite, + component, ENDOFARGUMENTS); + } else { + assert ("Unexpected package type" == NULL); + } + return addindex(rd, cachefilename, filename_in_release, downloadas); +} + +void cachedlistfile_need_index(struct cachedlistfile *list, const char *repository, const char *suite, const char *architecture, const char *component, packagetype_t packagetype) { + if (packagetype == pt_deb) { + cachedlistfile_need(list, "Packages", 4, + repository, suite, + component, architecture, ENDOFARGUMENTS); + } else if (packagetype == pt_udeb) { + cachedlistfile_need(list, "uPackages", 4, + repository, suite, + component, architecture, ENDOFARGUMENTS); + } else if (packagetype == pt_dsc) { + cachedlistfile_need(list, "Sources", 3, + repository, suite, + component, ENDOFARGUMENTS); + } +} + +struct remote_index *remote_flat_index(struct remote_distribution *rd, packagetype_t packagetype, /*@null@*/const struct encoding_preferences *downloadas) { + char *cachefilename, *filename_in_release; + + assert (rd->flat); + if (packagetype == pt_deb) { + filename_in_release = strdup("Packages"); + cachefilename = genlistsfilename("Packages", 2, + rd->repository->name, rd->suite, + ENDOFARGUMENTS); + } else if (packagetype == pt_dsc) { + filename_in_release = strdup("Sources"); + cachefilename = genlistsfilename("Sources", 2, + rd->repository->name, rd->suite, + ENDOFARGUMENTS); + } else { + assert ("Unexpected package type" == NULL); + } + return addindex(rd, cachefilename, filename_in_release, downloadas); +} + +void cachedlistfile_need_flat_index(struct cachedlistfile *list, const char *repository, const char *suite, packagetype_t packagetype) { + if (packagetype == pt_deb) { + cachedlistfile_need(list, "Packages", 2, + repository, suite, ENDOFARGUMENTS); + } else if (packagetype == pt_dsc) { + cachedlistfile_need(list, "Sources", 1, + repository, suite, ENDOFARGUMENTS); + } +} + +const char *remote_index_file(const struct remote_index *ri) { + assert (ri->needed && ri->queued && ri->got); + return ri->cachefilename; +} +const char *remote_index_basefile(const struct remote_index *ri) { + assert (ri->needed && ri->queued); + return ri->cachebasename; +} + +struct aptmethod *remote_aptmethod(const struct remote_distribution *rd) { + return rd->repository->download; +} + +void remote_index_markdone(const struct remote_index *ri, struct markdonefile *done) { + if (ri->ofs[c_none] < 0) + return; + markdone_index(done, ri->cachebasename, + ri->from->remotefiles.checksums[ri->ofs[c_none]]); +} +void remote_index_needed(struct remote_index *ri) { + ri->needed = true; +} + +static retvalue indexfile_mark_got(struct remote_distribution *rd, struct remote_index *ri, /*@null@*/const struct checksums *gotchecksums) { + struct checksums **checksums_p; + + if (!rd->ignorerelease && ri->ofs[c_none] >= 0) { + checksums_p = &rd->remotefiles.checksums[ri->ofs[c_none]]; + bool matches, improves; + + // TODO: this no longer calculates all the checksums if + // the Release does not contain more and the apt method + // returned not all (but all that are in Release). + // This will then cause the done file not containing all + // checksums. (but if the Release not contain them, this + // does not harm, does it?) + + if (gotchecksums != NULL) { + matches = checksums_check(*checksums_p, gotchecksums, + &improves); + /* that should have been tested earlier */ + assert (matches); + if (! matches) + return RET_ERROR_WRONG_MD5; + if (improves) { + retvalue r; + + r = checksums_combine(checksums_p, + gotchecksums, NULL); + if (RET_WAS_ERROR(r)) + return r; + } + } + } + ri->got = true; + return RET_OK; +} + +static retvalue indexfile_unpacked(void *privdata, const char *compressed, bool failed) { + struct remote_index *ri = privdata; + struct remote_distribution *rd = ri->from; + retvalue r; + struct checksums *readchecksums = NULL; + + if (failed) { + // TODO: check if alternative can be used... + return RET_ERROR; + } + + /* file got uncompressed, check if it has the correct checksum */ + + /* even with a Release file, an old-style one might + * not list the checksums for the uncompressed indices */ + if (!rd->ignorerelease && ri->ofs[c_none] >= 0) { + int ofs = ri->ofs[c_none]; + const struct checksums *wantedchecksums = + rd->remotefiles.checksums[ofs]; + bool matches, missing = false; + + r = checksums_read(ri->cachefilename, &readchecksums); + if (r == RET_NOTHING) { + fprintf(stderr, +"Cannot open '%s', though it should just have been unpacked from '%s'!\n", + ri->cachefilename, + compressed); + r = RET_ERROR_MISSING; + } + if (RET_WAS_ERROR(r)) + return r; + missing = false; + matches = checksums_check(readchecksums, + wantedchecksums, &missing); + assert (!missing); + if (!matches) { + fprintf(stderr, +"Wrong checksum of uncompressed content of '%s':\n", compressed); + checksums_printdifferences(stderr, + wantedchecksums, + readchecksums); + checksums_free(readchecksums); + return RET_ERROR_WRONG_MD5; + } + /* if the compressed file was downloaded or copied, delete it. + * This is only done if we know the uncompressed checksum, so + * that less downloading is needed (though as apt no longer + * supports such archieves, they are unlikely anyway). */ + + if (strncmp(ri->cachefilename, compressed, + strlen(ri->cachefilename)) == 0) { + (void)unlink(compressed); + } + } + r = indexfile_mark_got(rd, ri, readchecksums); + checksums_free(readchecksums); + if (RET_WAS_ERROR(r)) + return r; + return RET_OK; +} + +/* *checksums_p must be either NULL or gotchecksums list all known checksums */ +static inline retvalue check_checksums(const char *methodname, const char *uri, const char *gotfilename, const struct checksums *wantedchecksums, /*@null@*/const struct checksums *gotchecksums, struct checksums **checksums_p) { + bool matches, missing = false; + struct checksums *readchecksums = NULL; + retvalue r; + + if (gotchecksums == NULL) { + matches = true; + missing = true; + } else + matches = checksums_check(gotchecksums, + wantedchecksums, &missing); + /* if the apt method did not generate all checksums + * we want to check, we'll have to do so: */ + if (matches && missing) { + /* we assume that everything we know how to + * extract from a Release file is something + * we know how to calculate out of a file */ + assert (checksums_p == NULL || *checksums_p == NULL); + r = checksums_read(gotfilename, &readchecksums); + if (r == RET_NOTHING) { + fprintf(stderr, +"Cannot open '%s', though apt-method '%s' claims it is there!\n", + gotfilename, methodname); + r = RET_ERROR_MISSING; + } + if (RET_WAS_ERROR(r)) + return r; + gotchecksums = readchecksums; + missing = false; + matches = checksums_check(gotchecksums, + wantedchecksums, &missing); + assert (!missing); + } + if (!matches) { + fprintf(stderr, "Wrong checksum during receive of '%s':\n", + uri); + checksums_printdifferences(stderr, + wantedchecksums, + gotchecksums); + checksums_free(readchecksums); + return RET_ERROR_WRONG_MD5; + } + if (checksums_p == NULL) + checksums_free(readchecksums); + else if (readchecksums != NULL) + *checksums_p = readchecksums; + return RET_OK; +} + +static retvalue index_callback(enum queue_action action, void *privdata, UNUSED(void *privdata2), const char *uri, const char *gotfilename, const char *wantedfilename, /*@null@*/const struct checksums *gotchecksums, const char *methodname) { + struct remote_index *ri = privdata; + struct remote_distribution *rd = ri->from; + struct checksums *readchecksums = NULL; + retvalue r; + + if (action == qa_error) + return queue_next_encoding(rd, ri); + if (action != qa_got) + return RET_ERROR; + + if (ri->compression == c_none) { + assert (strcmp(wantedfilename, ri->cachefilename) == 0); + r = copytoplace(gotfilename, wantedfilename, methodname, + &readchecksums); + if (RET_WAS_ERROR(r)) + return r; + gotfilename = wantedfilename; + if (readchecksums != NULL) + gotchecksums = readchecksums; + } + + if (!rd->ignorerelease && ri->ofs[ri->compression] >= 0) { + int ofs = ri->ofs[ri->compression]; + const struct checksums *wantedchecksums = + rd->remotefiles.checksums[ofs]; + + r = check_checksums(methodname, uri, gotfilename, + wantedchecksums, gotchecksums, &readchecksums); + if (RET_WAS_ERROR(r)) { + checksums_free(readchecksums); + return r; + } + if (readchecksums != NULL) + gotchecksums = readchecksums; + } + + if (ri->compression == c_none) { + assert (strcmp(gotfilename, wantedfilename) == 0); + r = indexfile_mark_got(rd, ri, gotchecksums); + checksums_free(readchecksums); + if (RET_WAS_ERROR(r)) + return r; + return RET_OK; + } else { + checksums_free(readchecksums); + r = remove_old_uncompressed(ri); + if (RET_WAS_ERROR(r)) + return r; + r = uncompress_queue_file(gotfilename, ri->cachefilename, + ri->compression, + indexfile_unpacked, privdata); + if (RET_WAS_ERROR(r)) + return r; + return RET_OK; + } +} + +static queue_callback diff_got_callback; + +static retvalue queue_next_diff(struct remote_index *ri) { + struct remote_distribution *rd = ri->from; + struct remote_repository *rr = rd->repository; + int i; + retvalue r; + + for (i = 0 ; i < ri->diffindex->patchcount ; i++) { + bool improves; + struct diffindex_patch *p = &ri->diffindex->patches[i]; + char *patchsuffix, *c; + + if (p->done || p->frompackages == NULL) + continue; + + if (!checksums_check(ri->oldchecksums, p->frompackages, + &improves)) + continue; + /* p->frompackages should only have sha1 and oldchecksums + * should definitely list a sha1 hash */ + assert (!improves); + + p->done = true; + + free(ri->patchfilename); + ri->patchfilename = mprintf("%s.diff-%s", ri->cachefilename, + p->name); + if (FAILEDTOALLOC(ri->patchfilename)) + return RET_ERROR_OOM; + c = ri->patchfilename + strlen(ri->cachefilename); + while (*c != '\0') { + if ((*c < '0' || *c > '9') + && (*c < 'A' || *c > 'Z') + && (*c < 'a' || *c > 'z') + && *c != '.' && *c != '-') + *c = '_'; + c++; + } + ri->selectedpatch = p; + patchsuffix = mprintf(".diff/%s.gz", p->name); + if (FAILEDTOALLOC(patchsuffix)) + return RET_ERROR_OOM; + + /* found a matching patch, tell the downloader we want it */ + r = aptmethod_enqueueindex(rr->download, rd->suite_base_dir, + ri->filename_in_release, + patchsuffix, + ri->patchfilename, ".gz", + diff_got_callback, ri, p); + free(patchsuffix); + return r; + } + /* no patch matches, try next possibility... */ + fprintf(stderr, "Error: available '%s' not listed in '%s.diffindex'.\n", + ri->cachefilename, ri->cachefilename); + return queue_next_encoding(rd, ri); +} + +static retvalue diff_uncompressed(void *privdata, const char *compressed, bool failed) { + struct remote_index *ri = privdata; + struct remote_distribution *rd = ri->from; + const struct diffindex_patch *p = ri->selectedpatch; + char *tempfilename; + struct rred_patch *rp; + FILE *f; + int i; + retvalue r; + bool dummy; + + if (ri->deletecompressedpatch) + (void)unlink(compressed); + if (failed) + return RET_ERROR; + + r = checksums_test(ri->patchfilename, p->checksums, NULL); + if (r == RET_NOTHING) { + fprintf(stderr, "Mysteriously vanished file '%s'!\n", + ri->patchfilename); + r = RET_ERROR_MISSING; + } + if (r == RET_ERROR_WRONG_MD5) + fprintf(stderr, "Corrupted package diff '%s'!\n", + ri->patchfilename); + if (RET_WAS_ERROR(r)) + return r; + + r = patch_load(ri->patchfilename, + checksums_getfilesize(p->checksums), &rp); + ASSERT_NOT_NOTHING(r); + if (RET_WAS_ERROR(r)) + return r; + + tempfilename = calc_addsuffix(ri->cachefilename, "tmp"); + if (FAILEDTOALLOC(tempfilename)) { + patch_free(rp); + return RET_ERROR_OOM; + } + (void)unlink(tempfilename); + i = rename(ri->cachefilename, tempfilename); + if (i != 0) { + int e = errno; + fprintf(stderr, "Error %d moving '%s' to '%s': %s\n", + e, ri->cachefilename, tempfilename, + strerror(e)); + free(tempfilename); + patch_free(rp); + return RET_ERRNO(e); + } + f = fopen(ri->cachefilename, "w"); + if (f == NULL) { + int e = errno; + fprintf(stderr, "Error %d creating '%s': %s\n", + e, ri->cachefilename, strerror(e)); + (void)unlink(tempfilename); + ri->olduncompressed->deleted = true; + ri->olduncompressed = NULL; + free(tempfilename); + patch_free(rp); + return RET_ERRNO(e); + } + r = patch_file(f, tempfilename, patch_getconstmodifications(rp)); + (void)unlink(tempfilename); + (void)unlink(ri->patchfilename); + free(ri->patchfilename); + ri->patchfilename = NULL; + free(tempfilename); + patch_free(rp); + if (RET_WAS_ERROR(r)) { + (void)fclose(f); + remove_old_uncompressed(ri); + // TODO: fall back to downloading at once? + return r; + } + i = ferror(f); + if (i != 0) { + int e = errno; + (void)fclose(f); + fprintf(stderr, "Error %d writing to '%s': %s\n", + e, ri->cachefilename, strerror(e)); + remove_old_uncompressed(ri); + return RET_ERRNO(e); + } + i = fclose(f); + if (i != 0) { + int e = errno; + fprintf(stderr, "Error %d writing to '%s': %s\n", + e, ri->cachefilename, strerror(e)); + remove_old_uncompressed(ri); + return RET_ERRNO(e); + } + checksums_free(ri->oldchecksums); + ri->oldchecksums = NULL; + r = checksums_read(ri->cachefilename, &ri->oldchecksums); + if (r == RET_NOTHING) { + fprintf(stderr, "Myteriously vanished file '%s'!\n", + ri->cachefilename); + r = RET_ERROR; + } + if (RET_WAS_ERROR(r)) + return r; + if (checksums_check(ri->oldchecksums, + rd->remotefiles.checksums[ri->ofs[c_none]], + &dummy)) { + ri->olduncompressed->deleted = true; + ri->olduncompressed = NULL; + /* we have a winner */ + return indexfile_mark_got(rd, ri, ri->oldchecksums); + } + /* let's see what patch we need next */ + return queue_next_diff(ri); +} + +static retvalue diff_got_callback(enum queue_action action, void *privdata, UNUSED(void *privdata2), UNUSED(const char *uri), const char *gotfilename, const char *wantedfilename, UNUSED(/*@null@*/const struct checksums *gotchecksums), UNUSED(const char *methodname)) { + struct remote_index *ri = privdata; + retvalue r; + + if (action == qa_error) + return queue_next_encoding(ri->from, ri); + if (action != qa_got) + return RET_ERROR; + + ri->deletecompressedpatch = strcmp(gotfilename, wantedfilename) == 0; + r = uncompress_queue_file(gotfilename, ri->patchfilename, + c_gzip, diff_uncompressed, ri); + if (RET_WAS_ERROR(r)) + (void)unlink(gotfilename); + return r; +} + +static retvalue diff_callback(enum queue_action action, void *privdata, UNUSED(void *privdata2), const char *uri, const char *gotfilename, const char *wantedfilename, /*@null@*/const struct checksums *gotchecksums, const char *methodname) { + struct remote_index *ri = privdata; + struct remote_distribution *rd = ri->from; + struct checksums *readchecksums = NULL; + int ofs; + retvalue r; + + if (action == qa_error) + return queue_next_encoding(rd, ri); + if (action != qa_got) + return RET_ERROR; + + r = copytoplace(gotfilename, wantedfilename, methodname, + &readchecksums); + if (RET_WAS_ERROR(r)) + return r; + if (readchecksums != NULL) + gotchecksums = readchecksums; + + ofs = ri->diff_ofs; + if (ofs >= 0) { + const struct checksums *wantedchecksums = + rd->remotefiles.checksums[ofs]; + bool matches, missing = false; + + if (gotchecksums == NULL) { + matches = true; + missing = true; + } else + matches = checksums_check(gotchecksums, + wantedchecksums, &missing); + /* if the apt method did not generate all checksums + * we want to check, we'll have to do so: */ + if (matches && missing) { + /* we assume that everything we know how to + * extract from a Release file is something + * we know how to calculate out of a file */ + assert (readchecksums == NULL); + r = checksums_read(gotfilename, &readchecksums); + if (r == RET_NOTHING) { + fprintf(stderr, +"Cannot open '%s', though apt-method '%s' claims it is there!\n", + gotfilename, methodname); + r = RET_ERROR_MISSING; + } + if (RET_WAS_ERROR(r)) + return r; + gotchecksums = readchecksums; + missing = false; + matches = checksums_check(gotchecksums, + wantedchecksums, &missing); + assert (!missing); + } + if (!matches) { + fprintf(stderr, +"Wrong checksum during receive of '%s':\n", uri); + checksums_printdifferences(stderr, + wantedchecksums, gotchecksums); + checksums_free(readchecksums); + return RET_ERROR_WRONG_MD5; + } + } + checksums_free(readchecksums); + r = diffindex_read(wantedfilename, &ri->diffindex); + ASSERT_NOT_NOTHING(r); + if (RET_WAS_ERROR(r)) + return queue_next_encoding(rd, ri); + if (ri->ofs[c_none] >= 0) { + bool dummy; + if (!checksums_check(rd->remotefiles.checksums[ + ri->ofs[c_none]], + ri->diffindex->destination, &dummy)) { + fprintf(stderr, +"'%s' does not match file requested in '%s'. Aborting diff processing...\n", + gotfilename, rd->usedreleasefile); + /* as this is claimed to be a common error + * (outdated .diff/Index file), proceed with + * other requested way to retrieve index file */ + return queue_next_encoding(rd, ri); + } + } + return queue_next_diff(ri); +} |