summaryrefslogtreecommitdiffstats
path: root/examples/rgw/rgw-cache
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:54:28 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:54:28 +0000
commite6918187568dbd01842d8d1d2c808ce16a894239 (patch)
tree64f88b554b444a49f656b6c656111a145cbbaa28 /examples/rgw/rgw-cache
parentInitial commit. (diff)
downloadceph-e6918187568dbd01842d8d1d2c808ce16a894239.tar.xz
ceph-e6918187568dbd01842d8d1d2c808ce16a894239.zip
Adding upstream version 18.2.2.upstream/18.2.2
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'examples/rgw/rgw-cache')
-rw-r--r--examples/rgw/rgw-cache/nginx-default.conf136
-rw-r--r--examples/rgw/rgw-cache/nginx-lua-file.lua26
-rw-r--r--examples/rgw/rgw-cache/nginx-noprefetch.conf101
-rw-r--r--examples/rgw/rgw-cache/nginx-slicing.conf137
-rw-r--r--examples/rgw/rgw-cache/nginx.conf57
5 files changed, 457 insertions, 0 deletions
diff --git a/examples/rgw/rgw-cache/nginx-default.conf b/examples/rgw/rgw-cache/nginx-default.conf
new file mode 100644
index 000000000..c22d15531
--- /dev/null
+++ b/examples/rgw/rgw-cache/nginx-default.conf
@@ -0,0 +1,136 @@
+#config cache size and path to the cache directory, you should make sure that the user that is running nginx have permissions to access the cache directory
+#max_size means that Nginx will not cache more than 20G, It should be tuned to a larger number if the /data/cache is bigger
+proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off;
+upstream rgws {
+ # List of all rgws (ips or resolvable names)
+ server rgw1:8000 max_fails=2 fail_timeout=5s;
+ server rgw2:8000 max_fails=2 fail_timeout=5s;
+ server rgw3:8000 max_fails=2 fail_timeout=5s;
+}
+server {
+ listen 80;
+ server_name cacher;
+ location /authentication {
+ internal;
+ client_max_body_size 0;
+ proxy_pass http://rgws$request_uri;
+ proxy_pass_request_body off;
+ proxy_set_header Host $host;
+ # setting x-rgw-auth allow the RGW the ability to only authorize the request without fetching the obj data
+ proxy_set_header x-rgw-auth "yes";
+ proxy_set_header Authorization $http_authorization;
+ proxy_http_version 1.1;
+ proxy_method $request_method;
+ # Do not convert HEAD requests into GET requests
+ proxy_cache_convert_head off;
+ error_page 404 = @outage;
+ proxy_intercept_errors on;
+ if ($request_uri = "/") {
+ return 200;
+ }
+ # URI included with question mark is not being cached
+ if ($request_uri ~* (\?)) {
+ return 200;
+ }
+ if ($request_method = "PUT") {
+ return 200;
+ }
+ if ($request_method = "POST") {
+ return 200;
+ }
+ if ($request_method = "HEAD") {
+ return 200;
+ }
+ if ($request_method = "COPY") {
+ return 200;
+ }
+ if ($request_method = "DELETE") {
+ return 200;
+ }
+ if ($http_if_match) {
+ return 200;
+ }
+ if ($http_authorization !~* "aws4_request") {
+ return 200;
+ }
+ }
+ location @outage{
+ return 403;
+ }
+ location / {
+ auth_request /authentication;
+ proxy_pass http://rgws;
+ set $authvar '';
+ # if $do_not_cache is not empty the request would not be cached, this is relevant for list op for example
+ set $do_not_cache '';
+ # the IP or name of the RGWs
+ rewrite_by_lua_file /etc/nginx/nginx-lua-file.lua;
+ #proxy_set_header Authorization $http_authorization;
+ # my cache configured at the top of the file
+ proxy_cache mycache;
+ proxy_cache_lock_timeout 0s;
+ proxy_cache_lock_age 1000s;
+ proxy_http_version 1.1;
+ set $date $aws_auth_date;
+ # Getting 403 if this header not set
+ proxy_set_header Host $host;
+ # Cache all 200 OK's for 1 day
+ proxy_cache_valid 200 206 1d;
+ # Use stale cache file in all errors from upstream if we can
+ proxy_cache_use_stale updating;
+ proxy_cache_background_update on;
+ # Try to check if etag have changed, if yes, do not re-fetch from rgw the object
+ proxy_cache_revalidate on;
+ # Lock the cache so that only one request can populate it at a time
+ proxy_cache_lock on;
+ # prevent conversion of head requests to get requests
+ proxy_cache_convert_head off;
+ # Listing all buckets should not be cached
+ if ($request_uri = "/") {
+ set $do_not_cache "no";
+ set $date $http_x_amz_date;
+ }
+ # URI including question mark are not supported to prevent bucket listing cache
+ if ($request_uri ~* (\?)) {
+ set $do_not_cache "no";
+ set $date $http_x_amz_date;
+ }
+ # Only aws4 requests are being cached - As the aws auth module supporting only aws v2
+ if ($http_authorization !~* "aws4_request") {
+ set $date $http_x_amz_date;
+ }
+ if ($request_method = "PUT") {
+ set $date $http_x_amz_date;
+ }
+ if ($request_method = "POST") {
+ set $date $http_x_amz_date;
+ }
+ if ($request_method = "HEAD") {
+ set $do_not_cache "no";
+ set $date $http_x_amz_date;
+ }
+ if ($request_method = "COPY") {
+ set $do_not_cache "no";
+ set $date $http_x_amz_date;
+ }
+ if ($http_if_match) {
+ #set $do_not_cache "no";
+ set $date $http_x_amz_date;
+ set $myrange $http_range;
+ }
+ if ($request_method = "DELETE") {
+ set $do_not_cache "no";
+ set $date $http_x_amz_date;
+ }
+ proxy_set_header if_match $http_if_match;
+ proxy_set_header Range $myrange;
+ # Use the original x-amz-date if the aws auth module didn't create one
+ proxy_set_header x-amz-date $date;
+ proxy_set_header X-Amz-Cache $authvar;
+ proxy_no_cache $do_not_cache;
+ proxy_set_header Authorization $awsauthfour;
+ # This is on which content the nginx to use for hashing the cache keys
+ proxy_cache_key "$request_uri$request_method$request_body$myrange";
+ client_max_body_size 0;
+ }
+}
diff --git a/examples/rgw/rgw-cache/nginx-lua-file.lua b/examples/rgw/rgw-cache/nginx-lua-file.lua
new file mode 100644
index 000000000..efaf42230
--- /dev/null
+++ b/examples/rgw/rgw-cache/nginx-lua-file.lua
@@ -0,0 +1,26 @@
+local check = ngx.req.get_headers()["AUTHORIZATION"]
+local uri = ngx.var.request_uri
+local ngx_re = require "ngx.re"
+local hdrs = ngx.req.get_headers()
+--Take all signedheaders names, this for creating the X-Amz-Cache which is necessary to override range header to be able to readahead an object
+local res, err = ngx_re.split(check,"SignedHeaders=")
+local res2, err2 = ngx_re.split(res[2],",")
+local res3, err3 = ngx_re.split(res2[1],";")
+local t = {}
+local concathdrs = string.char(0x00)
+for i = 1, #res3, 1 do
+ if hdrs[res3[i]] ~= nil then
+--0xB1 is the separator between header name and value
+ t[i] = res3[i] .. string.char(0xB1) .. hdrs[res3[i]]
+--0xB2 is the separator between headers
+ concathdrs = concathdrs .. string.char(0xB2) .. t[i]
+ end
+end
+-- check if the authorization header is not empty
+if check ~= nil then
+ local xamzcache = concathdrs:sub(2)
+ xamzcache = xamzcache .. string.char(0xB2) .. "Authorization" .. string.char(0xB1) .. check
+ if xamzcache:find("aws4_request") ~= nil and uri ~= "/" and uri:find("?") == nil and hdrs["if-match"] == nil then
+ ngx.var.authvar = xamzcache
+ end
+end
diff --git a/examples/rgw/rgw-cache/nginx-noprefetch.conf b/examples/rgw/rgw-cache/nginx-noprefetch.conf
new file mode 100644
index 000000000..02294415a
--- /dev/null
+++ b/examples/rgw/rgw-cache/nginx-noprefetch.conf
@@ -0,0 +1,101 @@
+#config cache size and path to the cache directory, you should make sure that the user that is running nginx have permissions to access the cache directory
+#max_size means that Nginx will not cache more than 20G, It should be tuned to a larger number if the /data/cache is bigger
+proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off;
+upstream rgws {
+ # List of all rgws (ips or resolvable names)
+ server rgw1:8000 max_fails=2 fail_timeout=5s;
+ server rgw2:8000 max_fails=2 fail_timeout=5s;
+ server rgw3:8000 max_fails=2 fail_timeout=5s;
+}
+server {
+ listen 80;
+ server_name cacher;
+ location /authentication {
+ internal;
+ client_max_body_size 0;
+ proxy_pass http://rgws$request_uri;
+ proxy_pass_request_body off;
+ proxy_set_header Host $host;
+ # setting x-rgw-auth allow the RGW the ability to only authorize the request without fetching the obj data
+ proxy_set_header x-rgw-auth "yes";
+ proxy_set_header Authorization $http_authorization;
+ proxy_http_version 1.1;
+ proxy_method $request_method;
+ # Do not convert HEAD requests into GET requests
+ proxy_cache_convert_head off;
+ error_page 404 = @outage;
+ proxy_intercept_errors on;
+ if ($request_uri = "/") {
+ return 200;
+ }
+ # URI included with question mark is not being cached
+ if ($request_uri ~* (\?)) {
+ return 200;
+ }
+ if ($request_method = "PUT") {
+ return 200;
+ }
+ if ($request_method = "POST") {
+ return 200;
+ }
+ if ($request_method = "HEAD") {
+ return 200;
+ }
+ if ($request_method = "COPY") {
+ return 200;
+ }
+ if ($request_method = "DELETE") {
+ return 200;
+ }
+ if ($http_if_match) {
+ return 200;
+ }
+ if ($http_authorization !~* "aws4_request") {
+ return 200;
+ }
+ }
+ location @outage{
+ return 403;
+ }
+ location / {
+ auth_request /authentication;
+ proxy_pass http://rgws;
+ # if $do_not_cache is not empty the request would not be cached, this is relevant for list op for example
+ set $do_not_cache '';
+ # the IP or name of the RGWs
+ #proxy_set_header Authorization $http_authorization;
+ # my cache configured at the top of the file
+ proxy_cache mycache;
+ proxy_cache_lock_timeout 0s;
+ proxy_cache_lock_age 1000s;
+ proxy_http_version 1.1;
+ # Getting 403 if this header not set
+ proxy_set_header Host $host;
+ # Cache all 200 OK's for 1 day
+ proxy_cache_valid 200 206 1d;
+ # Use stale cache file in all errors from upstream if we can
+ proxy_cache_use_stale updating;
+ proxy_cache_background_update on;
+ # Try to check if etag have changed, if yes, do not re-fetch from rgw the object
+ proxy_cache_revalidate on;
+ # Lock the cache so that only one request can populate it at a time
+ proxy_cache_lock on;
+ # prevent conversion of head requests to get requests
+ proxy_cache_convert_head off;
+ # Listing all buckets should not be cached
+ if ($request_uri = "/") {
+ set $do_not_cache "no";
+ }
+ # URI including question mark are not supported to prevent bucket listing cache
+ if ($request_uri ~* (\?)) {
+ set $do_not_cache "no";
+ }
+ # Use the original x-amz-date if the aws auth module didn't create one
+ proxy_no_cache $do_not_cache;
+ proxy_set_header Authorization $http_authorization;
+ proxy_set_header Range $http_range;
+ # This is on which content the nginx to use for hashing the cache keys
+ proxy_cache_key "$request_uri$request_method$request_body$http_range";
+ client_max_body_size 0;
+ }
+}
diff --git a/examples/rgw/rgw-cache/nginx-slicing.conf b/examples/rgw/rgw-cache/nginx-slicing.conf
new file mode 100644
index 000000000..3de4c67fc
--- /dev/null
+++ b/examples/rgw/rgw-cache/nginx-slicing.conf
@@ -0,0 +1,137 @@
+#config cache size and path to the cache directory, you should make sure that the user that is running nginx have permissions to access the cache directory
+#max_size means that Nginx will not cache more than 20G, It should be tuned to a larger number if the /data/cache is bigger
+proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off;
+upstream rgws {
+ # List of all rgws (ips or resolvable names)
+ server rgw1:8000 max_fails=2 fail_timeout=5s;
+ server rgw2:8000 max_fails=2 fail_timeout=5s;
+ server rgw3:8000 max_fails=2 fail_timeout=5s;
+}
+server {
+ listen 80;
+ server_name cacher;
+ location /authentication {
+ internal;
+ client_max_body_size 0;
+ proxy_pass http://rgws$request_uri;
+ proxy_pass_request_body off;
+ proxy_set_header Host $host;
+ # setting x-rgw-auth allow the RGW the ability to only authorize the request without fetching the obj data
+ proxy_set_header x-rgw-auth "yes";
+ proxy_set_header Authorization $http_authorization;
+ proxy_http_version 1.1;
+ proxy_method $request_method;
+ # Do not convert HEAD requests into GET requests
+ proxy_cache_convert_head off;
+ error_page 404 = @outage;
+ proxy_intercept_errors on;
+ if ($request_uri = "/") {
+ return 200;
+ }
+ # URI included with question mark is not being cached
+ if ($request_uri ~* (\?)) {
+ return 200;
+ }
+ if ($request_method = "PUT") {
+ return 200;
+ }
+ if ($request_method = "POST") {
+ return 200;
+ }
+ if ($request_method = "HEAD") {
+ return 200;
+ }
+ if ($request_method = "COPY") {
+ return 200;
+ }
+ if ($request_method = "DELETE") {
+ return 200;
+ }
+ if ($http_if_match) {
+ return 200;
+ }
+ if ($http_authorization !~* "aws4_request") {
+ return 200;
+ }
+ }
+ location @outage{
+ return 403;
+ }
+ location / {
+ slice 1m;
+ auth_request /authentication;
+ proxy_set_header Range $slice_range;
+ proxy_pass http://rgws;
+ set $authvar '';
+ # if $do_not_cache is not empty the request would not be cached, this is relevant for list op for example
+ set $do_not_cache '';
+ # the IP or name of the RGWs
+ rewrite_by_lua_file /etc/nginx/nginx-lua-file.lua;
+ #proxy_set_header Authorization $http_authorization;
+ # my cache configured at the top of the file
+ proxy_cache mycache;
+ proxy_cache_lock_timeout 0s;
+ proxy_cache_lock_age 1000s;
+ proxy_http_version 1.1;
+ set $date $aws_auth_date;
+ # Getting 403 if this header not set
+ proxy_set_header Host $host;
+ # Cache all 200 OK's for 1 day
+ proxy_cache_valid 200 206 1d;
+ # Use stale cache file in all errors from upstream if we can
+ proxy_cache_use_stale updating;
+ proxy_cache_background_update on;
+ # Try to check if etag have changed, if yes, do not re-fetch from rgw the object
+ proxy_cache_revalidate on;
+ # Lock the cache so that only one request can populate it at a time
+ proxy_cache_lock on;
+ # prevent conversion of head requests to get requests
+ proxy_cache_convert_head off;
+ # Listing all buckets should not be cached
+ if ($request_uri = "/") {
+ set $do_not_cache "no";
+ set $date $http_x_amz_date;
+ }
+ # URI including question mark are not supported to prevent bucket listing cache
+ if ($request_uri ~* (\?)) {
+ set $do_not_cache "no";
+ set $date $http_x_amz_date;
+ }
+ # Only aws4 requests are being cached - As the aws auth module supporting only aws v2
+ if ($http_authorization !~* "aws4_request") {
+ set $date $http_x_amz_date;
+ }
+ if ($request_method = "PUT") {
+ set $date $http_x_amz_date;
+ }
+ if ($request_method = "POST") {
+ set $date $http_x_amz_date;
+ }
+ if ($request_method = "HEAD") {
+ set $do_not_cache "no";
+ set $date $http_x_amz_date;
+ }
+ if ($request_method = "COPY") {
+ set $do_not_cache "no";
+ set $date $http_x_amz_date;
+ }
+ if ($http_if_match) {
+ #set $do_not_cache "no";
+ set $date $http_x_amz_date;
+ set $myrange $slice_range;
+ }
+ if ($request_method = "DELETE") {
+ set $do_not_cache "no";
+ set $date $http_x_amz_date;
+ }
+ proxy_set_header if_match $http_if_match;
+ # Use the original x-amz-date if the aws auth module didn't create one
+ proxy_set_header x-amz-date $date;
+ proxy_set_header X-Amz-Cache $authvar;
+ proxy_no_cache $do_not_cache;
+ proxy_set_header Authorization $awsauthfour;
+ # This is on which content the nginx to use for hashing the cache keys
+ proxy_cache_key "$request_uri$request_method$request_body$slice_range";
+ client_max_body_size 0;
+ }
+}
diff --git a/examples/rgw/rgw-cache/nginx.conf b/examples/rgw/rgw-cache/nginx.conf
new file mode 100644
index 000000000..a478db1dc
--- /dev/null
+++ b/examples/rgw/rgw-cache/nginx.conf
@@ -0,0 +1,57 @@
+
+user nginx;
+#Process per core
+worker_processes auto;
+pid /var/run/nginx.pid;
+events {
+#Number of connections per worker
+ worker_connections 1024;
+}
+
+
+http {
+ types_hash_max_size 4096;
+ lua_package_path '/usr/local/openresty/lualib/?.lua;;';
+ aws_auth $aws_token {
+ # access key and secret key of the cache
+ # Please substitute with the access key and secret key of the amz-cache cap user
+ access_key cache;
+ secret_key cache;
+ service s3;
+ region us-east-1;
+ }
+ # This map is used to choose the original authorization header if the aws_auth module refuse to create one
+ map $aws_token $awsauth {
+ default $http_authorization;
+ ~. $aws_token; # Regular expression to match any value
+ }
+ map $request_uri $awsauthtwo {
+ "/" $http_authorization;
+ "~\?" $http_authorization;
+ default $awsauth;
+ }
+ map $request_method $awsauththree {
+ default $awsauthtwo;
+ "PUT" $http_authorization;
+ "HEAD" $http_authorization;
+ "POST" $http_authorization;
+ "DELETE" $http_authorization;
+ "COPY" $http_authorization;
+ }
+ map $http_if_match $awsauthfour {
+ ~. $http_authorization; # Regular expression to match any value
+ default $awsauththree;
+ }
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+ log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+ '$status $body_bytes_sent "$http_referer" '
+ '"$http_user_agent" "$http_x_forwarded_for"';
+ error_log /var/log/nginx/error.log;
+ access_log /var/log/nginx/access.log main;
+
+ sendfile on;
+ tcp_nodelay on;
+ keepalive_timeout 65;
+ include /etc/nginx/conf.d/*.conf;
+}