Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 421 insertions, 0 deletions

diff --git a/src/lib-index/mail-cache-private.h b/src/lib-index/mail-cache-private.h
new file mode 100644
index 0000000..c2fee17
--- /dev/null
+++ b/src/lib-index/mail-cache-private.h
@@ -0,0 +1,421 @@
+#ifndef MAIL_CACHE_PRIVATE_H
+#define MAIL_CACHE_PRIVATE_H
+
+#include "file-dotlock.h"
+#include "mail-index-private.h"
+#include "mail-cache.h"
+
+#define MAIL_CACHE_MAJOR_VERSION 1
+#define MAIL_CACHE_MINOR_VERSION 1
+
+#define MAIL_CACHE_LOCK_TIMEOUT 10
+#define MAIL_CACHE_LOCK_CHANGE_TIMEOUT 300
+
+#define MAIL_CACHE_MAX_WRITE_BUFFER (1024*256)
+
+#define MAIL_CACHE_IS_UNUSABLE(cache) \
+	((cache)->hdr == NULL)
+
+struct mail_cache_header {
+	/* Major version is increased only when you can't have backwards
+	   compatibility. If the field doesn't match MAIL_CACHE_MAJOR_VERSION,
+	   don't even try to read it. */
+	uint8_t major_version;
+	/* If this isn't the same as sizeof(uoff_t), the cache file can't be
+	   safely used with the current implementation. */
+	uint8_t compat_sizeof_uoff_t;
+	/* Minor version is increased when the file format changes in a
+	   backwards compatible way. */
+	uint8_t minor_version;
+	uint8_t unused;
+
+	/* Unique index file ID, which must match the main index's indexid.
+	   See mail_index_header.indexid. */
+	uint32_t indexid;
+	/* Cache file sequence. Increased on every purge. This must match the
+	   main index's reset_id for "cache" extension or the cache offsets
+	   aren't valid. When creating the first cache file, use the current
+	   UNIX timestamp as the file_seq. */
+	uint32_t file_seq;
+
+	/* Number of cache records that are linked inside the cache file,
+	   instead of being directly pointed from the main index. */
+	uint32_t continued_record_count;
+
+	/* Number of messages cached in this file. This does not include
+	   the continuation records.
+
+	   NOTE: <=v2.1 used this for hole offset, so we can't fully
+	   rely on it */
+	uint32_t record_count;
+	/* Currently unused. */
+	uint32_t backwards_compat_used_file_size;
+	/* Number of already expunged messages that currently have cache
+	   content in this file. */
+	uint32_t deleted_record_count;
+
+	/* Offset to the first mail_cache_header_fields. */
+	uint32_t field_header_offset;
+};
+
+struct mail_cache_header_fields {
+	/* Offset to the updated version of this header. Use
+	   mail_index_offset_to_uint32() to decode it. */
+	uint32_t next_offset;
+	/* Full size of this header. */
+	uint32_t size;
+	/* Number of fields in this header. */
+	uint32_t fields_count;
+
+#if 0
+	/* Last time the field was accessed. Not updated more often than
+	   once a day. This field may be overwritten later on, which in theory
+	   could cause reading to see a partially updated (corrupted) value.
+	   Don't fully trust this field unless it was read while cache is
+	   locked. */
+	uint32_t last_used[fields_count];
+	/* (uint32_t)-1 for variable sized fields */
+	uint32_t size[fields_count];
+	/* enum mail_cache_field_type */
+	uint8_t type[fields_count];
+	/* enum mail_cache_decision_type. This field can be overwritten
+	   later on to update the caching decision. */
+	uint8_t decision[fields_count];
+	/* NUL-separated list of field names */
+	char name[fields_count][];
+#endif
+};
+
+/* Macros to return offsets to the fields in mail_cache_header_fields. */
+#define MAIL_CACHE_FIELD_LAST_USED() \
+	(sizeof(uint32_t) * 3)
+#define MAIL_CACHE_FIELD_SIZE(count) \
+	(MAIL_CACHE_FIELD_LAST_USED() + sizeof(uint32_t) * (count))
+#define MAIL_CACHE_FIELD_TYPE(count) \
+	(MAIL_CACHE_FIELD_SIZE(count) + sizeof(uint32_t) * (count))
+#define MAIL_CACHE_FIELD_DECISION(count) \
+	(MAIL_CACHE_FIELD_TYPE(count) + sizeof(uint8_t) * (count))
+#define MAIL_CACHE_FIELD_NAMES(count) \
+	(MAIL_CACHE_FIELD_DECISION(count) + sizeof(uint8_t) * (count))
+
+struct mail_cache_record {
+	uint32_t prev_offset;
+	uint32_t size; /* full record size, including this header */
+	/* array of { uint32_t field; [ uint32_t size; ] { .. } } */
+};
+
+struct mail_cache_field_private {
+	struct mail_cache_field field;
+
+	/* Highest message UID whose cache field of this type have been
+	   accessed within this session. This is used to track whether messages
+	   are accessed in non-ascending order, which indicates an IMAP client
+	   that doesn't have a local cache. That will result in the caching
+	   decision to change from TEMP to YES. */
+	uint32_t uid_highwater;
+
+	/* Unused fields aren't written to cache file */
+	bool used:1;
+	/* field.decision is pending a write to cache file header. If the
+	   cache header is read from disk, don't overwrite it. */
+	bool decision_dirty:1;
+};
+
+struct mail_cache {
+	struct mail_index *index;
+	struct event *event;
+	/* Registered "cache" extension ID */
+	uint32_t ext_id;
+
+	char *filepath;
+	int fd;
+
+	struct dotlock_settings dotlock_settings;
+	struct file_lock *file_lock;
+
+	/* Cache file's inode, device and size when it was last fstat()ed. */
+	ino_t st_ino;
+	dev_t st_dev;
+	uoff_t last_stat_size;
+
+	/* Used to avoid logging mmap() errors too rapidly. */
+	time_t last_mmap_error_time;
+
+	/* a) mmaping the whole file */
+	void *mmap_base;
+	/* b) using file cache */
+	struct file_cache *file_cache;
+	/* c) using small read() calls with MAIL_INDEX_OPEN_FLAG_SAVEONLY */
+	uoff_t read_offset;
+	buffer_t *read_buf;
+	/* Size of the cache file as currently mapped to memory. Used for all
+	   of a), b), and c). */
+	size_t mmap_length;
+	/* mail_cache_map() increases this always. Used only for asserts. */
+	unsigned int remap_counter;
+	/* Linked list of all cache views. */
+	struct mail_cache_view *views;
+
+	/* mmap_disable=no: hdr points to data / NULL when cache is invalid.
+	   mmap_disable=yes: hdr points to hdr_ro_copy. this is needed because
+	   cache invalidation can zero the data any time */
+	const struct mail_cache_header *hdr;
+	struct mail_cache_header hdr_ro_copy;
+	/* hdr_copy gets updated when cache is locked and written when
+	   unlocking and hdr_modified=TRUE */
+	struct mail_cache_header hdr_copy;
+	/* If non-0, the offset for the last seen mail_cache_header_fields.
+	   Used as a cache to avoid reading through multiple next_offset
+	   pointers. */
+	uint32_t last_field_header_offset;
+
+	/* Memory pool used for permanent field allocations. Currently this
+	   means mail_cache_field.name and field_name_hash. */
+	pool_t field_pool;
+	/* Size of fields[] and field_file_map[] */
+	unsigned int fields_count;
+	/* All the registered cache fields. */
+	struct mail_cache_field_private *fields;
+	/* mail_cache_field.idx -> file-specific header index. The reverse
+	   of this is file_field_map[]. */
+	uint32_t *field_file_map;
+	/* mail_cache_field.name -> mail_cache_field.idx */
+	HASH_TABLE(char *, void *) field_name_hash; /* name -> idx */
+
+	/* file-specific header index -> mail_cache_fields.idx. The reverse
+	   of this is field_file_map[]. */
+	unsigned int *file_field_map;
+	/* Size of file_field_map[] */
+	unsigned int file_fields_count;
+
+	/* mail_cache_purge_later() sets these values to trigger purging on
+	   the next index sync. need_purge_file_seq is set to the current
+	   cache file_seq. If at sync time the file_seq differs, it means
+	   the cache was already purged and another purge isn't necessary. */
+	uint32_t need_purge_file_seq;
+	/* Human-readable reason for purging. Used for debugging and events. */
+	char *need_purge_reason;
+
+	/* Cache has been opened (or it doesn't exist). */
+	bool opened:1;
+	/* Cache has been locked with mail_cache_lock(). */
+	bool locked:1;
+	/* TRUE if the last lock attempt failed. The next locking attempt will
+	   be non-blocking to avoid unnecessarily waiting on a cache that has
+	   been locked for a long time. Since cache isn't strictly required,
+	   this could avoid unnecessarily long waits with some edge cases. */
+	bool last_lock_failed:1;
+	/* cache->hdr_copy has been modified. This must be used only while
+	   cache is locked. */
+	bool hdr_modified:1;
+	/* At least one of the cache fields' last_used or cache decision has
+	   changed. mail_cache_header_fields_update() will be used to overwrite
+	   these to the latest mail_cache_header_fields. */
+	bool field_header_write_pending:1;
+	/* Cache is currently being purged. */
+	bool purging:1;
+	/* Access the cache file by reading as little as possible from it
+	   (as opposed to mmap()ing it or using file-cache.h API to cache
+	   larger parts of it). This is used with MAIL_INDEX_OPEN_FLAG_SAVEONLY
+	   to avoid unnecessary cache reads. */
+	bool map_with_read:1;
+};
+
+struct mail_cache_loop_track {
+	/* we're looping if size_sum > (max_offset-min_offset) */
+	uoff_t min_offset, max_offset;
+	uoff_t size_sum;
+};
+
+struct mail_cache_missing_reason_cache {
+	uint32_t highest_checked_seq;
+	uint32_t highest_seq_with_cache;
+
+	uint32_t reset_id;
+	uint32_t log_file_head_seq;
+	uoff_t log_file_head_offset;
+};
+
+struct mail_cache_view {
+	struct mail_cache *cache;
+	struct mail_cache_view *prev, *next;
+	struct mail_index_view *view, *trans_view;
+
+	struct mail_cache_transaction_ctx *transaction;
+	/* mail_cache_add() has been called for some of the messages between
+	   trans_seq1..trans_seq2 in an uncommitted transaction. Check also
+	   the transaction contents when looking up cache fields for these
+	   mails. */
+	uint32_t trans_seq1, trans_seq2;
+
+	/* Used to avoid infinite loops in case cache records point to each
+	   others, causing a loop. FIXME: New cache files no longer support
+	   overwriting existing data, so this could be removed and replaced
+	   with a simple check that prev_offset is always smaller than the
+	   current record's offset. */
+	struct mail_cache_loop_track loop_track;
+	/* Used for optimizing mail_cache_get_missing_reason() */
+	struct mail_cache_missing_reason_cache reason_cache;
+
+	/* if cached_exists_buf[field] == cached_exists_value, it's cached.
+	   this allows us to avoid constantly clearing the whole buffer.
+	   it needs to be cleared only when cached_exists_value is wrapped. */
+	buffer_t *cached_exists_buf;
+	uint8_t cached_exists_value;
+	uint32_t cached_exists_seq;
+
+	/* mail_cache_view_update_cache_decisions() has been used to disable
+	   updating cache decisions. */
+	bool no_decision_updates:1;
+};
+
+/* mail_cache_lookup_iter_next() returns the next found field. */
+struct mail_cache_iterate_field {
+	/* mail_cache_field.idx */
+	unsigned int field_idx;
+	/* Size of data */
+	unsigned int size;
+	/* Cache field content in the field type-specific format */
+	const void *data;
+	/* Offset to data in cache file */
+	uoff_t offset;
+};
+
+struct mail_cache_lookup_iterate_ctx {
+	struct mail_cache_view *view;
+	/* This must match mail_cache.remap_counter or the iterator is
+	   invalid. */
+	unsigned int remap_counter;
+	/* Message sequence as given to mail_cache_lookup_iter_init() */
+	uint32_t seq;
+
+	/* Pointer to current cache record being iterated. This may point
+	   to the cache file or uncommitted transaction. */
+	const struct mail_cache_record *rec;
+	/* Iterator's current position in the cache record. Starts from
+	   sizeof(mail_cache_record). */
+	unsigned int pos;
+	/* Copy of rec->size */
+	unsigned int rec_size;
+	/* Cache file offset to the beginning of rec, or 0 if it points to
+	   an uncommitted transaction. */
+	uint32_t offset;
+
+	/* Used to loop through all changes in the uncommited transaction,
+	   in case there are multiple changes to the same message. */
+	unsigned int trans_next_idx;
+
+	/* Cache has become unusable. Stop the iteration. */
+	bool stop:1;
+	/* I/O error or lock timeout occurred during iteration. Normally there
+	   is no locking during iteration, but it may happen while cache is
+	   being purged to wait for the purging to finish before cache can be
+	   accessed again. */
+	bool failed:1;
+	/* Iteration has finished returning changes from uncommitted
+	   transaction's in-memory buffer. */
+	bool memory_appends_checked:1;
+	/* Iteration has finished returning changes from uncommitted
+	   transaction that were already written to cache file, but not
+	   to main index. */
+	bool disk_appends_checked:1;
+	/* TRUE if the field index numbers in rec as the internal
+	   mail_cache_field.idx (instead of the file-specific indexes).
+	   This indicates that the rec points to uncommited transaction's
+	   in-memory buffer. */
+	bool inmemory_field_idx:1;
+};
+
+/* Explicitly lock the cache file. Returns -1 if error / timed out,
+   1 if ok, 0 if cache is broken/doesn't exist */
+int mail_cache_lock(struct mail_cache *cache);
+/* Flush pending header updates and unlock. Returns -1 if cache is / just got
+   corrupted, 0 if ok. */
+int mail_cache_flush_and_unlock(struct mail_cache *cache);
+/* Unlock the cache without any header updates. */
+void mail_cache_unlock(struct mail_cache *cache);
+
+int mail_cache_write(struct mail_cache *cache, const void *data, size_t size,
+		     uoff_t offset);
+int mail_cache_append(struct mail_cache *cache, const void *data, size_t size,
+		      uint32_t *offset);
+
+int mail_cache_header_fields_read(struct mail_cache *cache);
+int mail_cache_header_fields_update(struct mail_cache *cache);
+void mail_cache_header_fields_get(struct mail_cache *cache, buffer_t *dest);
+int mail_cache_header_fields_get_next_offset(struct mail_cache *cache,
+					     uint32_t *offset_r);
+void mail_cache_expunge_count(struct mail_cache *cache, unsigned int count);
+
+uint32_t mail_cache_lookup_cur_offset(struct mail_index_view *view,
+				      uint32_t seq, uint32_t *reset_id_r);
+int mail_cache_get_record(struct mail_cache *cache, uint32_t offset,
+			  const struct mail_cache_record **rec_r);
+uint32_t mail_cache_get_first_new_seq(struct mail_index_view *view);
+
+/* Returns TRUE if offset..size area has been tracked before.
+   Returns FALSE if the area may or may not have been tracked before,
+   but we don't know for sure yet. */
+bool mail_cache_track_loops(struct mail_cache_loop_track *loop_track,
+			    uoff_t offset, uoff_t size);
+
+/* Iterate through a message's cached fields. */
+void mail_cache_lookup_iter_init(struct mail_cache_view *view, uint32_t seq,
+				 struct mail_cache_lookup_iterate_ctx *ctx_r);
+/* Returns 1 if field was returned, 0 if end of fields, or -1 if error.
+   Note that this may trigger re-reading and reallocating cache fields. */
+int mail_cache_lookup_iter_next(struct mail_cache_lookup_iterate_ctx *ctx,
+				struct mail_cache_iterate_field *field_r);
+const struct mail_cache_record *
+mail_cache_transaction_lookup_rec(struct mail_cache_transaction_ctx *ctx,
+				  unsigned int seq,
+				  unsigned int *trans_next_idx);
+bool mail_cache_transactions_have_changes(struct mail_cache *cache);
+
+/* Return data from the specified position in the cache file. Returns 1 if
+   successful, 0 if offset/size points outside the cache file, -1 if I/O
+   error. */
+int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size,
+		   const void **data_r);
+/* Map the whole cache file into memory. Returns 1 if ok, 0 if corrupted
+   (and deleted), -1 if I/O error. */
+int mail_cache_map_all(struct mail_cache *cache);
+void mail_cache_file_close(struct mail_cache *cache);
+int mail_cache_reopen(struct mail_cache *cache);
+int mail_cache_sync_reset_id(struct mail_cache *cache);
+
+/* Notify the decision handling code that field was looked up for seq.
+   This should be called even for fields that aren't currently in cache file.
+   This is used to update caching decisions for fields that already exist
+   in the cache file. */
+void mail_cache_decision_state_update(struct mail_cache_view *view,
+				      uint32_t seq, unsigned int field);
+const char *mail_cache_decision_to_string(enum mail_cache_decision_type dec);
+struct event_passthrough *
+mail_cache_decision_changed_event(struct mail_cache *cache, struct event *event,
+				  unsigned int field);
+
+struct mail_cache_purge_drop_ctx {
+	struct mail_cache *cache;
+	time_t max_yes_downgrade_time;
+	time_t max_temp_drop_time;
+};
+enum mail_cache_purge_drop_decision {
+	MAIL_CACHE_PURGE_DROP_DECISION_NONE,
+	MAIL_CACHE_PURGE_DROP_DECISION_DROP,
+	MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP,
+};
+void mail_cache_purge_drop_init(struct mail_cache *cache,
+				const struct mail_index_header *hdr,
+				struct mail_cache_purge_drop_ctx *ctx_r);
+enum mail_cache_purge_drop_decision
+mail_cache_purge_drop_test(struct mail_cache_purge_drop_ctx *ctx,
+			   unsigned int field);
+
+int mail_cache_expunge_handler(struct mail_index_sync_map_ctx *sync_ctx,
+			       const void *data, void **sync_context);
+
+void mail_cache_set_syscall_error(struct mail_cache *cache,
+				  const char *function) ATTR_COLD;
+
+#endif