1 files changed, 203 insertions, 48 deletions

diff --git a/src/cache.c b/src/cache.c
index 9f12f10..32f2e47 100644
--- a/src/cache.c
+++ b/src/cache.c
@@ -77,6 +77,7 @@ struct cache_appctx {
 	unsigned int rem_data;           /* Remaining bytes for the last data block (HTX only, 0 means process next block) */
 	unsigned int send_notmodified:1; /* In case of conditional request, we might want to send a "304 Not Modified" response instead of the stored data. */
 	unsigned int unused:31;
+	/* 4 bytes hole here */
 	struct shared_block *next;       /* The next block of data to be sent for this cache entry. */
 };
 
@@ -193,7 +194,7 @@ struct cache_entry {
 	unsigned int latest_validation;     /* latest validation date */
 	unsigned int expire;      /* expiration date (wall clock time) */
 	unsigned int age;         /* Origin server "Age" header value */
-
+	unsigned int body_size;         /* Size of the body */
 	int refcount;
 
 	struct eb32_node eb;     /* ebtree node used to hold the cache object */
@@ -231,8 +232,8 @@ DECLARE_STATIC_POOL(pool_head_cache_st, "cache_st", sizeof(struct cache_st));
 
 static struct eb32_node *insert_entry(struct cache *cache, struct cache_tree *tree, struct cache_entry *new_entry);
 static void delete_entry(struct cache_entry *del_entry);
-static void release_entry_locked(struct cache_tree *cache, struct cache_entry *entry);
-static void release_entry_unlocked(struct cache_tree *cache, struct cache_entry *entry);
+static inline void release_entry_locked(struct cache_tree *cache, struct cache_entry *entry);
+static inline void release_entry_unlocked(struct cache_tree *cache, struct cache_entry *entry);
 
 /*
  * Find a cache_entry in the <cache>'s tree that has the hash <hash>.
@@ -753,6 +754,7 @@ cache_store_http_payload(struct stream *s, struct filter *filter, struct http_ms
 	struct htx_blk *blk;
 	struct shared_block *fb;
 	struct htx_ret htxret;
+	size_t data_len = 0;
 	unsigned int orig_len, to_forward;
 	int ret;
 
@@ -789,6 +791,7 @@ cache_store_http_payload(struct stream *s, struct filter *filter, struct http_ms
 				chunk_memcat(&trash, (char *)&info, sizeof(info));
 				chunk_istcat(&trash, v);
 				to_forward += v.len;
+				data_len += v.len;
 				len -= v.len;
 				break;
 
@@ -817,6 +820,8 @@ cache_store_http_payload(struct stream *s, struct filter *filter, struct http_ms
 		goto no_cache;
 	}
 
+	/* disguise below to shut a warning on */
+	DISGUISE((struct cache_entry *)st->first_block->data)->body_size += data_len;
 	ret = shctx_row_data_append(shctx, st->first_block,
 				    (unsigned char *)b_head(&trash), b_data(&trash));
 	if (ret < 0)
@@ -1133,7 +1138,7 @@ static int http_check_vary_header(struct htx *htx, unsigned int *vary_signature)
  * "vary" on the accept-encoding value.
  * Returns 0 if we found a known encoding in the response, -1 otherwise.
  */
-static int set_secondary_key_encoding(struct htx *htx, char *secondary_key)
+static int set_secondary_key_encoding(struct htx *htx, unsigned int vary_signature, char *secondary_key)
 {
 	unsigned int resp_encoding_bitmap = 0;
 	const struct vary_hashing_information *info = vary_information;
@@ -1143,6 +1148,11 @@ static int set_secondary_key_encoding(struct htx *htx, char *secondary_key)
 	unsigned int encoding_value;
 	struct http_hdr_ctx ctx = { .blk = NULL };
 
+	/* We must not set the accept encoding part of the secondary signature
+	 * if the response does not vary on 'Accept Encoding'. */
+	if (!(vary_signature & VARY_ACCEPT_ENCODING))
+		return 0;
+
 	/* Look for the accept-encoding part of the secondary_key. */
 	while (count < hash_info_count && info->value != VARY_ACCEPT_ENCODING) {
 		offset += info->hash_length;
@@ -1404,7 +1414,7 @@ enum act_return http_action_store_cache(struct act_rule *rule, struct proxy *px,
 	 * We will not cache a response that has an unknown encoding (not
 	 * explicitly supported in parse_encoding_value function). */
 	if (cache->vary_processing_enabled && vary_signature)
-		if (set_secondary_key_encoding(htx, object->secondary_key))
+		if (set_secondary_key_encoding(htx, vary_signature, object->secondary_key))
 		    goto out;
 
 	if (!shctx_row_reserve_hot(shctx, first, trash.data)) {
@@ -1480,8 +1490,7 @@ static unsigned int htx_cache_dump_blk(struct appctx *appctx, struct htx *htx, e
 	unsigned int max, total;
 	uint32_t blksz;
 
-	max = htx_get_max_blksz(htx,
-				channel_htx_recv_max(sc_ic(appctx_sc(appctx)), htx));
+	max = htx_free_data_space(htx);
 	if (!max)
 		return 0;
 	blksz = ((type == HTX_BLK_HDR || type == HTX_BLK_TLR)
@@ -1521,14 +1530,14 @@ static unsigned int htx_cache_dump_data_blk(struct appctx *appctx, struct htx *h
 	struct cache_appctx *ctx = appctx->svcctx;
 	struct cache_flt_conf *cconf = appctx->rule->arg.act.p[0];
 	struct shared_context *shctx = shctx_ptr(cconf->c.cache);
-	unsigned int max, total, rem_data;
+	unsigned int max, total, rem_data, data_len;
 	uint32_t blksz;
 
-	max = htx_get_max_blksz(htx,
-				channel_htx_recv_max(sc_ic(appctx_sc(appctx)), htx));
+	max = htx_free_data_space(htx);
 	if (!max)
 		return 0;
 
+	data_len = 0;
 	rem_data = 0;
 	if (ctx->rem_data) {
 		blksz = ctx->rem_data;
@@ -1551,6 +1560,7 @@ static unsigned int htx_cache_dump_data_blk(struct appctx *appctx, struct htx *h
 		offset += sz;
 		blksz  -= sz;
 		total  += sz;
+		data_len += sz;
 		if (sz < max)
 			break;
 		if (blksz || offset == shctx->block_size) {
@@ -1563,6 +1573,7 @@ static unsigned int htx_cache_dump_data_blk(struct appctx *appctx, struct htx *h
 	ctx->next     = shblk;
 	ctx->sent    += total;
 	ctx->rem_data = rem_data + blksz;
+	appctx->to_forward -= data_len;
 	return total;
 }
 
@@ -1619,6 +1630,108 @@ static size_t htx_cache_dump_msg(struct appctx *appctx, struct htx *htx, unsigne
 	return total;
 }
 
+static unsigned int ff_cache_dump_data_blk(struct appctx *appctx, struct buffer *buf, unsigned int len,
+					   uint32_t info, struct shared_block *shblk, unsigned int offset)
+{
+	struct cache_appctx *ctx = appctx->svcctx;
+	struct cache_flt_conf *cconf = appctx->rule->arg.act.p[0];
+	struct shared_context *shctx = shctx_ptr(cconf->c.cache);
+	unsigned int total, rem_data, data_len;
+	uint32_t blksz;
+
+	total = 0;
+	data_len = 0;
+	rem_data = 0;
+	if (ctx->rem_data)
+		blksz = ctx->rem_data;
+	else {
+		blksz = (info & 0xfffffff);
+		ctx->sent += 4;
+	}
+	if (blksz > len) {
+		rem_data = blksz - len;
+		blksz = len;
+	}
+
+	while (blksz) {
+		size_t sz;
+
+		len = MIN(blksz, shctx->block_size - offset);
+		sz = b_putblk(buf, (char *)(shblk->data + offset), len);
+		offset += sz;
+		blksz  -= sz;
+		total  += sz;
+		data_len += sz;
+		if (sz < len)
+			break;
+		if (blksz || offset == shctx->block_size) {
+			shblk = LIST_NEXT(&shblk->list, typeof(shblk), list);
+			offset = 0;
+		}
+	}
+
+	ctx->offset   = offset;
+	ctx->next     = shblk;
+	ctx->sent    += total;
+	ctx->rem_data = rem_data + blksz;
+	appctx->to_forward -= data_len;
+	return total;
+}
+
+static size_t ff_cache_dump_msg(struct appctx *appctx, struct buffer *buf, unsigned int len)
+{
+	struct cache_appctx *ctx = appctx->svcctx;
+	struct cache_entry *cache_ptr = ctx->entry;
+	struct shared_block *first = block_ptr(cache_ptr);
+	struct cache_flt_conf *cconf = appctx->rule->arg.act.p[0];
+	struct shared_context *shctx = shctx_ptr(cconf->c.cache);
+	struct shared_block   *shblk;
+	unsigned int offset, sz;
+	unsigned int ret, total = 0;
+
+	while (len && (ctx->sent != first->len - sizeof(*cache_ptr))) {
+		enum htx_blk_type type;
+		uint32_t info;
+
+		shblk  = ctx->next;
+		offset = ctx->offset;
+		if (ctx->rem_data) {
+			type = HTX_BLK_DATA;
+			info = 0;
+			goto add_data_blk;
+		}
+
+		/* Get info of the next HTX block. May be split on 2 shblk */
+		sz = MIN(4, shctx->block_size - offset);
+		memcpy((char *)&info, (const char *)shblk->data + offset, sz);
+		offset += sz;
+		if (sz < 4) {
+			shblk = LIST_NEXT(&shblk->list, typeof(shblk), list);
+			memcpy(((char *)&info)+sz, (const char *)shblk->data, 4 - sz);
+			offset = (4 - sz);
+		}
+
+		/* Get payload of the next HTX block and insert it. */
+		type = (info >> 28);
+		if (type == HTX_BLK_DATA) {
+		  add_data_blk:
+			ret = ff_cache_dump_data_blk(appctx, buf, len, info, shblk, offset);
+		}
+		else
+			ret = 0;
+
+		if (!ret)
+			break;
+		total += ret;
+		len   -= ret;
+
+		if (ctx->rem_data)
+			break;
+	}
+
+	return total;
+}
+
 static int htx_cache_add_age_hdr(struct appctx *appctx, struct htx *htx)
 {
 	struct cache_appctx *ctx = appctx->svcctx;
@@ -1637,31 +1750,58 @@ static int htx_cache_add_age_hdr(struct appctx *appctx, struct htx *htx)
 	return 1;
 }
 
+static size_t http_cache_fastfwd(struct appctx *appctx, struct buffer *buf, size_t count, unsigned int flags)
+{
+	struct cache_appctx *ctx = appctx->svcctx;
+	struct cache_entry *cache_ptr = ctx->entry;
+	struct shared_block *first = block_ptr(cache_ptr);
+	size_t ret;
+
+	BUG_ON(!appctx->to_forward || count > appctx->to_forward);
+
+	ret = ff_cache_dump_msg(appctx, buf, count);
+
+	if (!appctx->to_forward) {
+		se_fl_clr(appctx->sedesc, SE_FL_MAY_FASTFWD_PROD);
+		applet_fl_clr(appctx, APPCTX_FL_FASTFWD);
+		if (ctx->sent == first->len - sizeof(*cache_ptr)) {
+			applet_set_eoi(appctx);
+			applet_set_eos(appctx);
+			appctx->st0 = HTX_CACHE_END;
+		}
+	}
+	return ret;
+}
+
 static void http_cache_io_handler(struct appctx *appctx)
 {
 	struct cache_appctx *ctx = appctx->svcctx;
 	struct cache_entry *cache_ptr = ctx->entry;
 	struct shared_block *first = block_ptr(cache_ptr);
-	struct stconn *sc = appctx_sc(appctx);
-	struct channel *req = sc_oc(sc);
-	struct channel *res = sc_ic(sc);
-	struct htx *req_htx, *res_htx;
+	struct htx *res_htx = NULL;
 	struct buffer *errmsg;
 	unsigned int len;
-	size_t ret, total = 0;
+	size_t ret;
 
-	res_htx = htx_from_buf(&res->buf);
-	total = res_htx->data;
+	if (applet_fl_test(appctx, APPCTX_FL_OUTBLK_ALLOC|APPCTX_FL_OUTBLK_FULL))
+		goto exit;
 
-	if (unlikely(se_fl_test(appctx->sedesc, (SE_FL_EOS|SE_FL_ERROR|SE_FL_SHR|SE_FL_SHW))))
-		goto out;
+	if (applet_fl_test(appctx, APPCTX_FL_FASTFWD) && se_fl_test(appctx->sedesc, SE_FL_MAY_FASTFWD_PROD))
+		goto exit;
 
-	/* Check if the input buffer is available. */
-	if (!b_size(&res->buf)) {
-		sc_need_room(sc, 0);
-		goto out;
+	if (!appctx_get_buf(appctx, &appctx->outbuf)) {
+		goto exit;
+	}
+
+	if (unlikely(applet_fl_test(appctx, APPCTX_FL_EOS|APPCTX_FL_ERROR))) {
+		goto exit;
 	}
 
+	res_htx = htx_from_buf(&appctx->outbuf);
+
+	len = first->len - sizeof(*cache_ptr) - ctx->sent;
+	res_htx = htx_from_buf(&appctx->outbuf);
+
 	if (appctx->st0 == HTX_CACHE_INIT) {
 		ctx->next = block_ptr(cache_ptr);
 		ctx->offset = sizeof(*cache_ptr);
@@ -1671,8 +1811,13 @@ static void http_cache_io_handler(struct appctx *appctx)
 	}
 
 	if (appctx->st0 == HTX_CACHE_HEADER) {
+		struct ist meth;
+
+		if (unlikely(applet_fl_test(appctx, APPCTX_FL_INBLK_ALLOC))) {
+			goto exit;
+		}
+
 		/* Headers must be dump at once. Otherwise it is an error */
-		len = first->len - sizeof(*cache_ptr) - ctx->sent;
 		ret = htx_cache_dump_msg(appctx, res_htx, len, HTX_BLK_EOH);
 		if (!ret || (htx_get_tail_type(res_htx) != HTX_BLK_EOH) ||
 		    !htx_cache_add_age_hdr(appctx, res_htx))
@@ -1689,60 +1834,66 @@ static void http_cache_io_handler(struct appctx *appctx)
 
 		/* Skip response body for HEAD requests or in case of "304 Not
 		 * Modified" response. */
-		if (__sc_strm(sc)->txn->meth == HTTP_METH_HEAD || ctx->send_notmodified)
+		meth = htx_sl_req_meth(http_get_stline(htxbuf(&appctx->inbuf)));
+		if (find_http_meth(istptr(meth), istlen(meth)) == HTTP_METH_HEAD || ctx->send_notmodified)
 			appctx->st0 = HTX_CACHE_EOM;
-		else
+		else {
+			if (!(global.tune.no_zero_copy_fwd & NO_ZERO_COPY_FWD_APPLET))
+				se_fl_set(appctx->sedesc, SE_FL_MAY_FASTFWD_PROD);
+
+			appctx->to_forward = cache_ptr->body_size;
+			len = first->len - sizeof(*cache_ptr) - ctx->sent;
 			appctx->st0 = HTX_CACHE_DATA;
+		}
 	}
 
 	if (appctx->st0 == HTX_CACHE_DATA) {
-		len = first->len - sizeof(*cache_ptr) - ctx->sent;
 		if (len) {
 			ret = htx_cache_dump_msg(appctx, res_htx, len, HTX_BLK_UNUSED);
 			if (ret < len) {
-				sc_need_room(sc, channel_htx_recv_max(res, res_htx) + 1);
+				applet_fl_set(appctx, APPCTX_FL_OUTBLK_FULL);
 				goto out;
 			}
 		}
+		BUG_ON(appctx->to_forward);
 		appctx->st0 = HTX_CACHE_EOM;
 	}
 
 	if (appctx->st0 == HTX_CACHE_EOM) {
 		 /* no more data are expected. */
 		res_htx->flags |= HTX_FL_EOM;
-		se_fl_set(appctx->sedesc, SE_FL_EOI);
-
+		applet_set_eoi(appctx);
+		se_fl_clr(appctx->sedesc, SE_FL_MAY_FASTFWD_PROD);
+		applet_fl_clr(appctx, APPCTX_FL_FASTFWD);
 		appctx->st0 = HTX_CACHE_END;
 	}
 
   end:
-	if (appctx->st0 == HTX_CACHE_END)
-		se_fl_set(appctx->sedesc, SE_FL_EOS);
+	if (appctx->st0 == HTX_CACHE_END) {
+		applet_set_eos(appctx);
+	}
 
   out:
-	total = res_htx->data - total;
-	if (total)
-		channel_add_input(res, total);
-	htx_to_buf(res_htx, &res->buf);
+	if (res_htx)
+		htx_to_buf(res_htx, &appctx->outbuf);
 
+  exit:
 	/* eat the whole request */
-	if (co_data(req)) {
-		req_htx = htx_from_buf(&req->buf);
-		co_htx_skip(req, req_htx, co_data(req));
-		htx_to_buf(req_htx, &req->buf);
-	}
+	b_reset(&appctx->inbuf);
+	applet_fl_clr(appctx, APPCTX_FL_INBLK_FULL);
+	appctx->sedesc->iobuf.flags &= ~IOBUF_FL_FF_BLOCKED;
 	return;
 
   error:
 	/* Sent and HTTP error 500 */
-	b_reset(&res->buf);
+	b_reset(&appctx->outbuf);
 	errmsg = &http_err_chunks[HTTP_ERR_500];
-	res->buf.data = b_data(errmsg);
-	memcpy(res->buf.area, b_head(errmsg), b_data(errmsg));
-	res_htx = htx_from_buf(&res->buf);
+	appctx->outbuf.data = b_data(errmsg);
+	memcpy(appctx->outbuf.area, b_head(errmsg), b_data(errmsg));
+	res_htx = htx_from_buf(&appctx->outbuf);
 
-	total = 0;
-	se_fl_set(appctx->sedesc, SE_FL_ERROR);
+	applet_set_eos(appctx);
+	applet_set_error(appctx);
 	appctx->st0 = HTX_CACHE_END;
 	goto end;
 }
@@ -2324,7 +2475,7 @@ int post_check_cache()
 	list_for_each_entry_safe(cache_config, back, &caches_config, list) {
 
 		ret_shctx = shctx_init(&shctx, cache_config->maxblocks, CACHE_BLOCKSIZE,
-		                       cache_config->maxobjsz, sizeof(struct cache));
+		                       cache_config->maxobjsz, sizeof(struct cache), cache_config->id);
 
 		if (ret_shctx <= 0) {
 			if (ret_shctx == SHCTX_E_INIT_LOCK)
@@ -2995,9 +3146,13 @@ struct applet http_cache_applet = {
 	.obj_type = OBJ_TYPE_APPLET,
 	.name = "<CACHE>", /* used for logging */
 	.fct = http_cache_io_handler,
+	.rcv_buf = appctx_htx_rcv_buf,
+	.snd_buf = appctx_htx_snd_buf,
+	.fastfwd = http_cache_fastfwd,
 	.release = http_cache_applet_release,
 };
 
+
 /* config parsers for this section */
 REGISTER_CONFIG_SECTION("cache", cfg_parse_cache, cfg_post_parse_section_cache);
 REGISTER_POST_CHECK(post_check_cache);