From 4b8a0f3f3dcf60dac2ce308ea08d413a535af29f Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Mon, 15 Apr 2024 21:12:14 +0200 Subject: Adding upstream version 5.4.4. Signed-off-by: Daniel Baumann --- pool.c | 869 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 869 insertions(+) create mode 100644 pool.c (limited to 'pool.c') diff --git a/pool.c b/pool.c new file mode 100644 index 0000000..c651ae0 --- /dev/null +++ b/pool.c @@ -0,0 +1,869 @@ +/* This file is part of "reprepro" + * Copyright (C) 2008,2009,2012 Bernhard R. Link + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "error.h" +#include "ignore.h" +#include "mprintf.h" +#include "atoms.h" +#include "strlist.h" +#include "dirs.h" +#include "pool.h" +#include "reference.h" +#include "files.h" +#include "sources.h" +#include "outhook.h" + +/* for now save them only in memory. In later times some way to store + * them on disk would be nice */ + +static component_t reserved_components = 0; +static void **file_changes_per_component = NULL; +static void *legacy_file_changes = NULL; +bool pool_havedereferenced = false; +bool pool_havedeleted = false; + +#define pl_ADDED 1 +#define pl_UNREFERENCED 2 +#define pl_DELETED 4 + +static int legacy_compare(const void *a, const void *b) { + const char *v1 = a, *v2 = b; + v1++; + v2++; + return strcmp(v1, v2); +} + +struct source_node { + void *file_changes; + char sourcename[]; +}; + +static int source_node_compare(const void *a, const void *b) { + const struct source_node *v1 = a, *v2 = b; + return strcmp(v1->sourcename, v2->sourcename); +} + +static retvalue split_filekey(const char *filekey, /*@out@*/component_t *component_p, /*@out@*/struct source_node **node_p, /*@out@*/const char **basename_p) { + const char *p, *lastp, *source; + struct source_node *node; + component_t c; + + if (unlikely(memcmp(filekey, "pool/", 5) != 0)) + return RET_NOTHING; + lastp = filekey + 4; + filekey = lastp + 1; + /* components can include slashes, so look for the first valid component + * followed by something looking like a proper directory. + * This might missdetect the component, but as it only is used for + * the current run it will hopefully always detect the same place + * (and all that is important is that it is the same place) */ + while (true) { + p = strchr(lastp + 1, '/'); + if (unlikely(p == NULL)) + return RET_NOTHING; + lastp = p; + c = component_find_l(filekey, (size_t)(p - filekey)); + if (unlikely(!atom_defined(c))) + continue; + p++; + if (p[0] != '\0' && p[1] == '/' && p[0] != '/' && p[2] == p[0]) { + p += 2; + if (unlikely(p[0] == 'l' && p[1] == 'i' && p[2] == 'b')) + continue; + source = p; + break; + } else if (p[0] == 'l' && p[1] == 'i' && p[2] == 'b' && p[3] != '\0' + && p[4] == '/' && p[5] == 'l' && p[6] == 'i' + && p[7] == 'b' && p[3] != '/' && p[8] == p[3]) { + source = p + 5; + break; + } else + continue; + } + p = strchr(source, '/'); + if (unlikely(p == NULL)) + return RET_NOTHING; + node = malloc(sizeof(struct source_node) + (p - source) + 1); + if (FAILEDTOALLOC(node)) + return RET_ERROR_OOM; + node->file_changes = NULL; + memcpy(node->sourcename, source, p - source); + node->sourcename[p - source] = '\0'; + p++; + *basename_p = p; + *node_p = node; + *component_p = c; + return RET_OK; +} + +/* name can be either basename (in a source directory) or a full + * filekey (in legacy fallback mode) */ +static retvalue remember_name(void **root_p, const char *name, char mode, char mode_and) { + char **p; + + p = tsearch(name - 1, root_p, legacy_compare); + if (FAILEDTOALLOC(p)) + return RET_ERROR_OOM; + if (*p == name - 1) { + size_t l = strlen(name); + *p = malloc(l + 2); + if (FAILEDTOALLOC(*p)) + return RET_ERROR_OOM; + **p = mode; + memcpy((*p) + 1, name, l + 1); + } else { + **p &= mode_and; + **p |= mode; + } + return RET_OK; +} + +static retvalue remember_filekey(const char *filekey, char mode, char mode_and) { + retvalue r; + component_t c; + struct source_node *node, **found; + const char *basefilename; + + r = split_filekey(filekey, &c, &node, &basefilename); + if (RET_WAS_ERROR(r)) + return r; + if (r == RET_OK) { + assert (atom_defined(c)); + if (c > reserved_components) { + void ** h; + + assert (c <= components_count()); + + h = realloc(file_changes_per_component, + sizeof(void*) * (c + 1)); + if (FAILEDTOALLOC(h)) + return RET_ERROR_OOM; + file_changes_per_component = h; + while (reserved_components < c) { + h[++reserved_components] = NULL; + } + } + assert (file_changes_per_component != NULL); + found = tsearch(node, &file_changes_per_component[c], + source_node_compare); + if (FAILEDTOALLOC(found)) + return RET_ERROR_OOM; + if (*found != node) { + free(node); + node = *found; + } + return remember_name(&node->file_changes, basefilename, + mode, mode_and); + } + fprintf(stderr, "Warning: strange filekey '%s'!\n", filekey); + return remember_name(&legacy_file_changes, filekey, mode, mode_and); +} + +retvalue pool_dereferenced(const char *filekey) { + pool_havedereferenced = true; + return remember_filekey(filekey, pl_UNREFERENCED, 0xFF); +}; + +retvalue pool_markadded(const char *filekey) { + return remember_filekey(filekey, pl_ADDED, ~pl_DELETED); +}; + +/* so much code, just for the case the morguedir is on another partition than + * the pool dir... */ + +static inline retvalue copyfile(const char *source, const char *destination, int outfd, off_t length) { + int infd, err; + ssize_t readbytes; + void *buffer; + size_t bufsize = 1024*1024; + + buffer = malloc(bufsize); + if (FAILEDTOALLOC(buffer)) { + (void)close(outfd); + (void)unlink(destination); + bufsize = 16*1024; + buffer = malloc(bufsize); + if (FAILEDTOALLOC(buffer)) + return RET_ERROR_OOM; + } + + infd = open(source, O_RDONLY|O_NOCTTY); + if (infd < 0) { + int en = errno; + + fprintf(stderr, +"error %d opening file %s to be copied into the morgue: %s\n", + en, source, strerror(en)); + free(buffer); + (void)close(outfd); + (void)unlink(destination); + return RET_ERRNO(en); + } + while ((readbytes = read(infd, buffer, bufsize)) > 0) { + const char *start = buffer; + + if ((off_t)readbytes > length) { + fprintf(stderr, +"Mismatch of sizes of '%s': files is larger than expected!\n", + destination); + break; + } + while (readbytes > 0) { + ssize_t written; + + written = write(outfd, start, readbytes); + if (written > 0) { + assert (written <= readbytes); + readbytes -= written; + start += written; + } else if (written < 0) { + int en = errno; + + (void)close(infd); + (void)close(outfd); + (void)unlink(destination); + free(buffer); + + fprintf(stderr, +"error %d writing to morgue file %s: %s\n", + en, destination, strerror(en)); + return RET_ERRNO(en); + } + } + } + free(buffer); + if (readbytes == 0) { + err = close(infd); + if (err != 0) + readbytes = -1; + infd = -1; + } + if (readbytes < 0) { + int en = errno; + fprintf(stderr, +"error %d reading file %s to be copied into the morgue: %s\n", + en, source, strerror(en)); + if (infd >= 0) + (void)close(infd); + (void)close(outfd); + (void)unlink(destination); + return RET_ERRNO(en); + } + if (infd >= 0) + (void)close(infd); + err = close(outfd); + if (err != 0) { + int en = errno; + + fprintf(stderr, "error %d writing to morgue file %s: %s\n", + en, destination, strerror(en)); + (void)unlink(destination); + return RET_ERRNO(en); + } + return RET_OK; +} + +static inline retvalue morgue_name(const char *filekey, char **name_p, int *fd_p) { + const char *name = dirs_basename(filekey); + char *firsttry = calc_dirconcat(global.morguedir, name); + int fd, en, number; + retvalue r; + + if (FAILEDTOALLOC(firsttry)) + return RET_ERROR_OOM; + + fd = open(firsttry, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY, 0666); + if (fd >= 0) { + assert (fd > 2); + *name_p = firsttry; + *fd_p = fd; + return RET_OK; + } + en = errno; + if (en == ENOENT) { + r = dirs_make_recursive(global.morguedir); + if (RET_WAS_ERROR(r)) { + free(firsttry); + return r; + } + /* try again */ + fd = open(firsttry, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY, 0666); + if (fd >= 0) { + assert (fd > 2); + *name_p = firsttry; + *fd_p = fd; + return RET_OK; + } + en = errno; + } + if (en != EEXIST) { + fprintf(stderr, "error %d creating morgue-file %s: %s\n", + en, firsttry, strerror(en)); + free(firsttry); + return RET_ERRNO(en); + } + /* file exists, try names with -number appended: */ + for (number = 1 ; number < 1000 ; number++) { + char *try = mprintf("%s-%d", firsttry, number); + + if (FAILEDTOALLOC(try)) { + free(firsttry); + return RET_ERROR_OOM; + } + fd = open(try, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY, 0666); + if (fd >= 0) { + assert (fd > 2); + free(firsttry); + *name_p = try; + *fd_p = fd; + return RET_OK; + } + free(try); + } + free(firsttry); + fprintf(stderr, "Could not create a new file '%s' in morguedir '%s'!\n", + name, global.morguedir); + return RET_ERROR; +} + +/* if file not there, return RET_NOTHING */ +static inline retvalue movefiletomorgue(const char *filekey, const char *filename, bool new) { + char *morguefilename = NULL; + int err; + retvalue r; + + if (!new && global.morguedir != NULL) { + int morguefd = -1; + struct stat s; + + r = morgue_name(filekey, &morguefilename, &morguefd); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) + return r; + err = lstat(filename, &s); + if (err != 0) { + int en = errno; + if (errno == ENOENT) { + (void)close(morguefd); + (void)unlink(morguefilename); + free(morguefilename); + return RET_NOTHING; + } + fprintf(stderr, +"error %d looking at file %s to be moved into the morgue: %s\n", + en, filename, strerror(en)); + (void)close(morguefd); + (void)unlink(morguefilename); + free(morguefilename); + return RET_ERRNO(en); + } + if (S_ISLNK(s.st_mode)) { + /* no need to copy a symbolic link: */ + (void)close(morguefd); + (void)unlink(morguefilename); + free(morguefilename); + morguefilename = NULL; + } else if (S_ISREG(s.st_mode)) { + err = rename(filename, morguefilename); + if (err == 0) { + (void)close(morguefd); + free(morguefilename); + return RET_OK; + } + r = copyfile(filename, morguefilename, morguefd, + s.st_size); + if (RET_WAS_ERROR(r)) { + free(morguefilename); + return r; + } + } else { + fprintf(stderr, +"Strange (non-regular) file '%s' in the pool.\nPlease delete manually!\n", + filename); + (void)close(morguefd); + (void)unlink(morguefilename); + free(morguefilename); + morguefilename = NULL; + return RET_ERROR; + } + } + err = unlink(filename); + if (err != 0) { + int en = errno; + if (errno == ENOENT) + return RET_NOTHING; + fprintf(stderr, "error %d while unlinking %s: %s\n", + en, filename, strerror(en)); + if (morguefilename != NULL) { + (void)unlink(morguefilename); + free(morguefilename); + } + return RET_ERRNO(en); + } else { + free(morguefilename); + return RET_OK; + } +} + +/* delete the file and possible parent directories, + * if not new and morguedir set, first move/copy there */ +static retvalue deletepoolfile(const char *filekey, bool new) { + char *filename; + retvalue r; + + if (interrupted()) + return RET_ERROR_INTERRUPTED; + if (!new) + outhook_send("POOLDELETE", filekey, NULL, NULL); + filename = files_calcfullfilename(filekey); + if (FAILEDTOALLOC(filename)) + return RET_ERROR_OOM; + /* move to morgue or simply delete: */ + r = movefiletomorgue(filekey, filename, new); + if (r == RET_NOTHING) { + fprintf(stderr, "%s not found, forgetting anyway\n", filename); + } + if (!RET_IS_OK(r)) { + free(filename); + return r; + } + if (!global.keepdirectories) { + /* try to delete parent directories, until one gives + * errors (hopefully because it still contains files) */ + size_t fixedpartlen = strlen(global.outdir); + char *p; + int err, en; + + while ((p = strrchr(filename, '/')) != NULL) { + /* do not try to remove parts of the mirrordir */ + if ((size_t)(p-filename) <= fixedpartlen+1) + break; + *p ='\0'; + /* try to rmdir the directory, this will + * fail if there are still other files or directories + * in it: */ + err = rmdir(filename); + if (err == 0) { + if (verbose > 1) { + printf( +"removed now empty directory %s\n", + filename); + } + } else { + en = errno; + if (en != ENOTEMPTY) { + //TODO: check here if only some + //other error was first and it + //is not empty so we do not have + //to remove it anyway... + fprintf(stderr, +"ignoring error %d trying to rmdir %s: %s\n", en, filename, strerror(en)); + } + /* parent directories will contain this one + * thus not be empty, in other words: + * everything's done */ + break; + } + } + + } + free(filename); + return RET_OK; +} + + +retvalue pool_delete(const char *filekey) { + retvalue r; + + if (verbose >= 1) + printf("deleting and forgetting %s\n", filekey); + + r = deletepoolfile(filekey, false); + if (RET_WAS_ERROR(r)) + return r; + + return files_remove(filekey); +} + +/* called from files_remove: */ +retvalue pool_markdeleted(const char *filekey) { + pool_havedeleted = true; + return remember_filekey(filekey, pl_DELETED, ~pl_UNREFERENCED); +}; + +/* libc's twalk misses a callback_data pointer, so we need some temporary + * global variables: */ +static retvalue result; +static bool first, onlycount; +static long woulddelete_count; +static component_t current_component; +static const char *sourcename = NULL; + +static void removeifunreferenced(const void *nodep, const VISIT which, UNUSED(const int depth)) { + char *node; const char *filekey; + retvalue r; + + if (which != leaf && which != postorder) + return; + + if (interrupted()) + return; + + node = *(char **)nodep; + filekey = node + 1; + if ((*node & pl_UNREFERENCED) == 0) + return; + r = references_isused(filekey); + if (r != RET_NOTHING) + return; + + if (onlycount) { + woulddelete_count++; + return; + } + + if (verbose >= 0 && first) { + printf("Deleting files no longer referenced...\n"); + first = false; + } + if (verbose >= 1) + printf("deleting and forgetting %s\n", filekey); + r = deletepoolfile(filekey, (*node & pl_ADDED) != 0); + RET_UPDATE(result, r); + if (!RET_WAS_ERROR(r)) { + r = files_removesilent(filekey); + RET_UPDATE(result, r); + if (!RET_WAS_ERROR(r)) + *node &= ~pl_UNREFERENCED; + if (RET_IS_OK(r)) + *node |= pl_DELETED; + } +} + + +static void removeifunreferenced2(const void *nodep, const VISIT which, UNUSED(const int depth)) { + char *node; + char *filekey; + retvalue r; + + if (which != leaf && which != postorder) + return; + + if (interrupted()) + return; + + node = *(char **)nodep; + if ((*node & pl_UNREFERENCED) == 0) + return; + filekey = calc_filekey(current_component, sourcename, node + 1); + r = references_isused(filekey); + if (r != RET_NOTHING) { + free(filekey); + return; + } + if (onlycount) { + woulddelete_count++; + free(filekey); + return; + } + if (verbose >= 0 && first) { + printf("Deleting files no longer referenced...\n"); + first = false; + } + if (verbose >= 1) + printf("deleting and forgetting %s\n", filekey); + r = deletepoolfile(filekey, (*node & pl_ADDED) != 0); + RET_UPDATE(result, r); + if (!RET_WAS_ERROR(r)) { + r = files_removesilent(filekey); + RET_UPDATE(result, r); + if (!RET_WAS_ERROR(r)) + *node &= ~pl_UNREFERENCED; + if (RET_IS_OK(r)) + *node |= pl_DELETED; + } + RET_UPDATE(result, r); + free(filekey); +} + +static void removeunreferenced_from_component(const void *nodep, const VISIT which, UNUSED(const int depth)) { + struct source_node *node; + + if (which != leaf && which != postorder) + return; + + if (interrupted()) + return; + + node = *(struct source_node **)nodep; + sourcename = node->sourcename; + twalk(node->file_changes, removeifunreferenced2); +} + +retvalue pool_removeunreferenced(bool delete) { + component_t c; + + if (!delete && verbose <= 0) + return RET_NOTHING; + + result = RET_NOTHING; + first = true; + onlycount = !delete; + woulddelete_count = 0; + for (c = 1 ; c <= reserved_components ; c++) { + assert (file_changes_per_component != NULL); + current_component = c; + twalk(file_changes_per_component[c], + removeunreferenced_from_component); + } + twalk(legacy_file_changes, removeifunreferenced); + if (interrupted()) + result = RET_ERROR_INTERRUPTED; + if (!delete && woulddelete_count > 0) { + printf( +"%lu files lost their last reference.\n" +"(dumpunreferenced lists such files, use deleteunreferenced to delete them.)\n", + woulddelete_count); + } + return result; +} + +static void removeunusednew(const void *nodep, const VISIT which, UNUSED(const int depth)) { + char *node; const char *filekey; + retvalue r; + + if (which != leaf && which != postorder) + return; + + if (interrupted()) + return; + + node = *(char **)nodep; + filekey = node + 1; + /* only look at newly added and not already deleted */ + if ((*node & (pl_ADDED|pl_DELETED)) != pl_ADDED) + return; + r = references_isused(filekey); + if (r != RET_NOTHING) + return; + + if (onlycount) { + woulddelete_count++; + return; + } + + if (verbose >= 0 && first) { + printf( +"Deleting files just added to the pool but not used.\n" +"(to avoid use --keepunusednewfiles next time)\n"); + first = false; + } + if (verbose >= 1) + printf("deleting and forgetting %s\n", filekey); + r = deletepoolfile(filekey, true); + RET_UPDATE(result, r); + if (!RET_WAS_ERROR(r)) { + r = files_removesilent(filekey); + RET_UPDATE(result, r); + /* don't remove pl_ADDED here, otherwise the hook + * script will be told to remove something not added */ + if (!RET_WAS_ERROR(r)) + *node &= ~pl_UNREFERENCED; + if (RET_IS_OK(r)) + *node |= pl_DELETED; + } +} + + +static void removeunusednew2(const void *nodep, const VISIT which, UNUSED(const int depth)) { + char *node; + char *filekey; + retvalue r; + + if (which != leaf && which != postorder) + return; + + if (interrupted()) + return; + + node = *(char **)nodep; + /* only look at newly added and not already deleted */ + if ((*node & (pl_ADDED|pl_DELETED)) != pl_ADDED) + return; + filekey = calc_filekey(current_component, sourcename, node + 1); + r = references_isused(filekey); + if (r != RET_NOTHING) { + free(filekey); + return; + } + if (onlycount) { + woulddelete_count++; + free(filekey); + return; + } + if (verbose >= 0 && first) { + printf( +"Deleting files just added to the pool but not used.\n" +"(to avoid use --keepunusednewfiles next time)\n"); + first = false; + } + if (verbose >= 1) + printf("deleting and forgetting %s\n", filekey); + r = deletepoolfile(filekey, true); + RET_UPDATE(result, r); + if (!RET_WAS_ERROR(r)) { + r = files_removesilent(filekey); + RET_UPDATE(result, r); + /* don't remove pl_ADDED here, otherwise the hook + * script will be told to remove something not added */ + if (!RET_WAS_ERROR(r)) + *node &= ~pl_UNREFERENCED; + if (RET_IS_OK(r)) + *node |= pl_DELETED; + } + RET_UPDATE(result, r); + free(filekey); +} + +static void removeunusednew_from_component(const void *nodep, const VISIT which, UNUSED(const int depth)) { + struct source_node *node; + + if (which != leaf && which != postorder) + return; + + if (interrupted()) + return; + + node = *(struct source_node **)nodep; + sourcename = node->sourcename; + twalk(node->file_changes, removeunusednew2); +} + +void pool_tidyadded(bool delete) { + component_t c; + + if (!delete && verbose < 0) + return; + + result = RET_NOTHING; + first = true; + onlycount = !delete; + woulddelete_count = 0; + for (c = 1 ; c <= reserved_components ; c++) { + assert (file_changes_per_component != NULL); + current_component = c; + twalk(file_changes_per_component[c], + removeunusednew_from_component); + } + // this should not really happen at all, but better safe then sorry: + twalk(legacy_file_changes, removeunusednew); + if (!delete && woulddelete_count > 0) { + printf( +"%lu files were added but not used.\n" +"The next deleteunreferenced call will delete them.\n", + woulddelete_count); + } + return; + +} + +static void reportnewlegacyfiles(const void *nodep, const VISIT which, UNUSED(const int depth)) { + char *node; + + if (which != leaf && which != postorder) + return; + + node = *(char **)nodep; + /* only look at newly added and not already deleted */ + if ((*node & (pl_ADDED|pl_DELETED)) != pl_ADDED) + return; + outhook_sendpool(atom_unknown, NULL, node + 1); +} + + +static void reportnewproperfiles(const void *nodep, const VISIT which, UNUSED(const int depth)) { + char *node; + + if (which != leaf && which != postorder) + return; + + node = *(char **)nodep; + /* only look at newly added and not already deleted */ + if ((*node & (pl_ADDED|pl_DELETED)) != pl_ADDED) + return; + outhook_sendpool(current_component, sourcename, node + 1); +} + +static void reportnewfiles(const void *nodep, const VISIT which, UNUSED(const int depth)) { + struct source_node *node; + + if (which != leaf && which != postorder) + return; + + node = *(struct source_node **)nodep; + sourcename = node->sourcename; + twalk(node->file_changes, reportnewproperfiles); +} + +void pool_sendnewfiles(void) { + component_t c; + + for (c = 1 ; c <= reserved_components ; c++) { + assert (file_changes_per_component != NULL); + current_component = c; + twalk(file_changes_per_component[c], + reportnewfiles); + } + twalk(legacy_file_changes, reportnewlegacyfiles); + return; + +} + +#ifdef HAVE_TDESTROY +static void sourcename_free(void *n) { + struct source_node *node = n; + + tdestroy(node->file_changes, free); + free(node); +} +#endif + +void pool_free(void) { +#ifdef HAVE_TDESTROY + component_t c; + + for (c = 1 ; c <= reserved_components ; c++) { + tdestroy(file_changes_per_component[c], sourcename_free); + } + reserved_components = 0; + free(file_changes_per_component); + file_changes_per_component = NULL; + tdestroy(legacy_file_changes, free); + legacy_file_changes = NULL; +#endif +} -- cgit v1.2.3