diff options
Diffstat (limited to 'examples/rgw/rgw-cache')
| -rw-r--r-- | examples/rgw/rgw-cache/nginx-default.conf | 136 | ||||
| -rw-r--r-- | examples/rgw/rgw-cache/nginx-lua-file.lua | 26 | ||||
| -rw-r--r-- | examples/rgw/rgw-cache/nginx-noprefetch.conf | 101 | ||||
| -rw-r--r-- | examples/rgw/rgw-cache/nginx-slicing.conf | 137 | ||||
| -rw-r--r-- | examples/rgw/rgw-cache/nginx.conf | 57 |
5 files changed, 457 insertions, 0 deletions
diff --git a/examples/rgw/rgw-cache/nginx-default.conf b/examples/rgw/rgw-cache/nginx-default.conf new file mode 100644 index 000000000..c22d15531 --- /dev/null +++ b/examples/rgw/rgw-cache/nginx-default.conf @@ -0,0 +1,136 @@ +#config cache size and path to the cache directory, you should make sure that the user that is running nginx have permissions to access the cache directory +#max_size means that Nginx will not cache more than 20G, It should be tuned to a larger number if the /data/cache is bigger +proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off; +upstream rgws { + # List of all rgws (ips or resolvable names) + server rgw1:8000 max_fails=2 fail_timeout=5s; + server rgw2:8000 max_fails=2 fail_timeout=5s; + server rgw3:8000 max_fails=2 fail_timeout=5s; +} +server { + listen 80; + server_name cacher; + location /authentication { + internal; + client_max_body_size 0; + proxy_pass http://rgws$request_uri; + proxy_pass_request_body off; + proxy_set_header Host $host; + # setting x-rgw-auth allow the RGW the ability to only authorize the request without fetching the obj data + proxy_set_header x-rgw-auth "yes"; + proxy_set_header Authorization $http_authorization; + proxy_http_version 1.1; + proxy_method $request_method; + # Do not convert HEAD requests into GET requests + proxy_cache_convert_head off; + error_page 404 = @outage; + proxy_intercept_errors on; + if ($request_uri = "/") { + return 200; + } + # URI included with question mark is not being cached + if ($request_uri ~* (\?)) { + return 200; + } + if ($request_method = "PUT") { + return 200; + } + if ($request_method = "POST") { + return 200; + } + if ($request_method = "HEAD") { + return 200; + } + if ($request_method = "COPY") { + return 200; + } + if ($request_method = "DELETE") { + return 200; + } + if ($http_if_match) { + return 200; + } + if ($http_authorization !~* "aws4_request") { + return 200; + } + } + location @outage{ + return 403; + } + location / { + auth_request /authentication; + proxy_pass http://rgws; + set $authvar ''; + # if $do_not_cache is not empty the request would not be cached, this is relevant for list op for example + set $do_not_cache ''; + # the IP or name of the RGWs + rewrite_by_lua_file /etc/nginx/nginx-lua-file.lua; + #proxy_set_header Authorization $http_authorization; + # my cache configured at the top of the file + proxy_cache mycache; + proxy_cache_lock_timeout 0s; + proxy_cache_lock_age 1000s; + proxy_http_version 1.1; + set $date $aws_auth_date; + # Getting 403 if this header not set + proxy_set_header Host $host; + # Cache all 200 OK's for 1 day + proxy_cache_valid 200 206 1d; + # Use stale cache file in all errors from upstream if we can + proxy_cache_use_stale updating; + proxy_cache_background_update on; + # Try to check if etag have changed, if yes, do not re-fetch from rgw the object + proxy_cache_revalidate on; + # Lock the cache so that only one request can populate it at a time + proxy_cache_lock on; + # prevent conversion of head requests to get requests + proxy_cache_convert_head off; + # Listing all buckets should not be cached + if ($request_uri = "/") { + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + # URI including question mark are not supported to prevent bucket listing cache + if ($request_uri ~* (\?)) { + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + # Only aws4 requests are being cached - As the aws auth module supporting only aws v2 + if ($http_authorization !~* "aws4_request") { + set $date $http_x_amz_date; + } + if ($request_method = "PUT") { + set $date $http_x_amz_date; + } + if ($request_method = "POST") { + set $date $http_x_amz_date; + } + if ($request_method = "HEAD") { + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + if ($request_method = "COPY") { + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + if ($http_if_match) { + #set $do_not_cache "no"; + set $date $http_x_amz_date; + set $myrange $http_range; + } + if ($request_method = "DELETE") { + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + proxy_set_header if_match $http_if_match; + proxy_set_header Range $myrange; + # Use the original x-amz-date if the aws auth module didn't create one + proxy_set_header x-amz-date $date; + proxy_set_header X-Amz-Cache $authvar; + proxy_no_cache $do_not_cache; + proxy_set_header Authorization $awsauthfour; + # This is on which content the nginx to use for hashing the cache keys + proxy_cache_key "$request_uri$request_method$request_body$myrange"; + client_max_body_size 0; + } +} diff --git a/examples/rgw/rgw-cache/nginx-lua-file.lua b/examples/rgw/rgw-cache/nginx-lua-file.lua new file mode 100644 index 000000000..efaf42230 --- /dev/null +++ b/examples/rgw/rgw-cache/nginx-lua-file.lua @@ -0,0 +1,26 @@ +local check = ngx.req.get_headers()["AUTHORIZATION"] +local uri = ngx.var.request_uri +local ngx_re = require "ngx.re" +local hdrs = ngx.req.get_headers() +--Take all signedheaders names, this for creating the X-Amz-Cache which is necessary to override range header to be able to readahead an object +local res, err = ngx_re.split(check,"SignedHeaders=") +local res2, err2 = ngx_re.split(res[2],",") +local res3, err3 = ngx_re.split(res2[1],";") +local t = {} +local concathdrs = string.char(0x00) +for i = 1, #res3, 1 do + if hdrs[res3[i]] ~= nil then +--0xB1 is the separator between header name and value + t[i] = res3[i] .. string.char(0xB1) .. hdrs[res3[i]] +--0xB2 is the separator between headers + concathdrs = concathdrs .. string.char(0xB2) .. t[i] + end +end +-- check if the authorization header is not empty +if check ~= nil then + local xamzcache = concathdrs:sub(2) + xamzcache = xamzcache .. string.char(0xB2) .. "Authorization" .. string.char(0xB1) .. check + if xamzcache:find("aws4_request") ~= nil and uri ~= "/" and uri:find("?") == nil and hdrs["if-match"] == nil then + ngx.var.authvar = xamzcache + end +end diff --git a/examples/rgw/rgw-cache/nginx-noprefetch.conf b/examples/rgw/rgw-cache/nginx-noprefetch.conf new file mode 100644 index 000000000..02294415a --- /dev/null +++ b/examples/rgw/rgw-cache/nginx-noprefetch.conf @@ -0,0 +1,101 @@ +#config cache size and path to the cache directory, you should make sure that the user that is running nginx have permissions to access the cache directory +#max_size means that Nginx will not cache more than 20G, It should be tuned to a larger number if the /data/cache is bigger +proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off; +upstream rgws { + # List of all rgws (ips or resolvable names) + server rgw1:8000 max_fails=2 fail_timeout=5s; + server rgw2:8000 max_fails=2 fail_timeout=5s; + server rgw3:8000 max_fails=2 fail_timeout=5s; +} +server { + listen 80; + server_name cacher; + location /authentication { + internal; + client_max_body_size 0; + proxy_pass http://rgws$request_uri; + proxy_pass_request_body off; + proxy_set_header Host $host; + # setting x-rgw-auth allow the RGW the ability to only authorize the request without fetching the obj data + proxy_set_header x-rgw-auth "yes"; + proxy_set_header Authorization $http_authorization; + proxy_http_version 1.1; + proxy_method $request_method; + # Do not convert HEAD requests into GET requests + proxy_cache_convert_head off; + error_page 404 = @outage; + proxy_intercept_errors on; + if ($request_uri = "/") { + return 200; + } + # URI included with question mark is not being cached + if ($request_uri ~* (\?)) { + return 200; + } + if ($request_method = "PUT") { + return 200; + } + if ($request_method = "POST") { + return 200; + } + if ($request_method = "HEAD") { + return 200; + } + if ($request_method = "COPY") { + return 200; + } + if ($request_method = "DELETE") { + return 200; + } + if ($http_if_match) { + return 200; + } + if ($http_authorization !~* "aws4_request") { + return 200; + } + } + location @outage{ + return 403; + } + location / { + auth_request /authentication; + proxy_pass http://rgws; + # if $do_not_cache is not empty the request would not be cached, this is relevant for list op for example + set $do_not_cache ''; + # the IP or name of the RGWs + #proxy_set_header Authorization $http_authorization; + # my cache configured at the top of the file + proxy_cache mycache; + proxy_cache_lock_timeout 0s; + proxy_cache_lock_age 1000s; + proxy_http_version 1.1; + # Getting 403 if this header not set + proxy_set_header Host $host; + # Cache all 200 OK's for 1 day + proxy_cache_valid 200 206 1d; + # Use stale cache file in all errors from upstream if we can + proxy_cache_use_stale updating; + proxy_cache_background_update on; + # Try to check if etag have changed, if yes, do not re-fetch from rgw the object + proxy_cache_revalidate on; + # Lock the cache so that only one request can populate it at a time + proxy_cache_lock on; + # prevent conversion of head requests to get requests + proxy_cache_convert_head off; + # Listing all buckets should not be cached + if ($request_uri = "/") { + set $do_not_cache "no"; + } + # URI including question mark are not supported to prevent bucket listing cache + if ($request_uri ~* (\?)) { + set $do_not_cache "no"; + } + # Use the original x-amz-date if the aws auth module didn't create one + proxy_no_cache $do_not_cache; + proxy_set_header Authorization $http_authorization; + proxy_set_header Range $http_range; + # This is on which content the nginx to use for hashing the cache keys + proxy_cache_key "$request_uri$request_method$request_body$http_range"; + client_max_body_size 0; + } +} diff --git a/examples/rgw/rgw-cache/nginx-slicing.conf b/examples/rgw/rgw-cache/nginx-slicing.conf new file mode 100644 index 000000000..3de4c67fc --- /dev/null +++ b/examples/rgw/rgw-cache/nginx-slicing.conf @@ -0,0 +1,137 @@ +#config cache size and path to the cache directory, you should make sure that the user that is running nginx have permissions to access the cache directory +#max_size means that Nginx will not cache more than 20G, It should be tuned to a larger number if the /data/cache is bigger +proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off; +upstream rgws { + # List of all rgws (ips or resolvable names) + server rgw1:8000 max_fails=2 fail_timeout=5s; + server rgw2:8000 max_fails=2 fail_timeout=5s; + server rgw3:8000 max_fails=2 fail_timeout=5s; +} +server { + listen 80; + server_name cacher; + location /authentication { + internal; + client_max_body_size 0; + proxy_pass http://rgws$request_uri; + proxy_pass_request_body off; + proxy_set_header Host $host; + # setting x-rgw-auth allow the RGW the ability to only authorize the request without fetching the obj data + proxy_set_header x-rgw-auth "yes"; + proxy_set_header Authorization $http_authorization; + proxy_http_version 1.1; + proxy_method $request_method; + # Do not convert HEAD requests into GET requests + proxy_cache_convert_head off; + error_page 404 = @outage; + proxy_intercept_errors on; + if ($request_uri = "/") { + return 200; + } + # URI included with question mark is not being cached + if ($request_uri ~* (\?)) { + return 200; + } + if ($request_method = "PUT") { + return 200; + } + if ($request_method = "POST") { + return 200; + } + if ($request_method = "HEAD") { + return 200; + } + if ($request_method = "COPY") { + return 200; + } + if ($request_method = "DELETE") { + return 200; + } + if ($http_if_match) { + return 200; + } + if ($http_authorization !~* "aws4_request") { + return 200; + } + } + location @outage{ + return 403; + } + location / { + slice 1m; + auth_request /authentication; + proxy_set_header Range $slice_range; + proxy_pass http://rgws; + set $authvar ''; + # if $do_not_cache is not empty the request would not be cached, this is relevant for list op for example + set $do_not_cache ''; + # the IP or name of the RGWs + rewrite_by_lua_file /etc/nginx/nginx-lua-file.lua; + #proxy_set_header Authorization $http_authorization; + # my cache configured at the top of the file + proxy_cache mycache; + proxy_cache_lock_timeout 0s; + proxy_cache_lock_age 1000s; + proxy_http_version 1.1; + set $date $aws_auth_date; + # Getting 403 if this header not set + proxy_set_header Host $host; + # Cache all 200 OK's for 1 day + proxy_cache_valid 200 206 1d; + # Use stale cache file in all errors from upstream if we can + proxy_cache_use_stale updating; + proxy_cache_background_update on; + # Try to check if etag have changed, if yes, do not re-fetch from rgw the object + proxy_cache_revalidate on; + # Lock the cache so that only one request can populate it at a time + proxy_cache_lock on; + # prevent conversion of head requests to get requests + proxy_cache_convert_head off; + # Listing all buckets should not be cached + if ($request_uri = "/") { + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + # URI including question mark are not supported to prevent bucket listing cache + if ($request_uri ~* (\?)) { + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + # Only aws4 requests are being cached - As the aws auth module supporting only aws v2 + if ($http_authorization !~* "aws4_request") { + set $date $http_x_amz_date; + } + if ($request_method = "PUT") { + set $date $http_x_amz_date; + } + if ($request_method = "POST") { + set $date $http_x_amz_date; + } + if ($request_method = "HEAD") { + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + if ($request_method = "COPY") { + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + if ($http_if_match) { + #set $do_not_cache "no"; + set $date $http_x_amz_date; + set $myrange $slice_range; + } + if ($request_method = "DELETE") { + set $do_not_cache "no"; + set $date $http_x_amz_date; + } + proxy_set_header if_match $http_if_match; + # Use the original x-amz-date if the aws auth module didn't create one + proxy_set_header x-amz-date $date; + proxy_set_header X-Amz-Cache $authvar; + proxy_no_cache $do_not_cache; + proxy_set_header Authorization $awsauthfour; + # This is on which content the nginx to use for hashing the cache keys + proxy_cache_key "$request_uri$request_method$request_body$slice_range"; + client_max_body_size 0; + } +} diff --git a/examples/rgw/rgw-cache/nginx.conf b/examples/rgw/rgw-cache/nginx.conf new file mode 100644 index 000000000..a478db1dc --- /dev/null +++ b/examples/rgw/rgw-cache/nginx.conf @@ -0,0 +1,57 @@ + +user nginx; +#Process per core +worker_processes auto; +pid /var/run/nginx.pid; +events { +#Number of connections per worker + worker_connections 1024; +} + + +http { + types_hash_max_size 4096; + lua_package_path '/usr/local/openresty/lualib/?.lua;;'; + aws_auth $aws_token { + # access key and secret key of the cache + # Please substitute with the access key and secret key of the amz-cache cap user + access_key cache; + secret_key cache; + service s3; + region us-east-1; + } + # This map is used to choose the original authorization header if the aws_auth module refuse to create one + map $aws_token $awsauth { + default $http_authorization; + ~. $aws_token; # Regular expression to match any value + } + map $request_uri $awsauthtwo { + "/" $http_authorization; + "~\?" $http_authorization; + default $awsauth; + } + map $request_method $awsauththree { + default $awsauthtwo; + "PUT" $http_authorization; + "HEAD" $http_authorization; + "POST" $http_authorization; + "DELETE" $http_authorization; + "COPY" $http_authorization; + } + map $http_if_match $awsauthfour { + ~. $http_authorization; # Regular expression to match any value + default $awsauththree; + } + include /etc/nginx/mime.types; + default_type application/octet-stream; + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + error_log /var/log/nginx/error.log; + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nodelay on; + keepalive_timeout 65; + include /etc/nginx/conf.d/*.conf; +} |