#config cache size and path to the cache directory, you should make sure that the user that is running nginx have permissions to access the cache directory
#max_size means that Nginx will not cache more than 20G, It should be tuned to a larger number if the /data/cache is bigger
proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off;
upstream rgws {
    # List of all rgws (ips or resolvable names)
    server rgw1:8000 max_fails=2 fail_timeout=5s;
    server rgw2:8000 max_fails=2 fail_timeout=5s;
    server rgw3:8000 max_fails=2 fail_timeout=5s;
}
server {
    listen       80;
    server_name  cacher;
    location /authentication {
        internal;
        client_max_body_size 0;
        proxy_pass http://rgws$request_uri;
        proxy_pass_request_body off;
        proxy_set_header Host $host;
        # setting x-rgw-auth allow the RGW the ability to only authorize the request without fetching the obj data 
        proxy_set_header x-rgw-auth "yes";
        proxy_set_header Authorization $http_authorization;
        proxy_http_version 1.1;
        proxy_method $request_method;
        # Do not convert HEAD requests into GET requests
        proxy_cache_convert_head off;
        error_page 404 = @outage;
        proxy_intercept_errors on;
        if ($request_uri = "/") {
            return 200;
        }
        # URI included with question mark is not being cached
        if ($request_uri ~* (\?)) {
            return 200;
        }
        if ($request_method = "PUT") {
            return 200;
        }
        if ($request_method = "POST") {
            return 200;
        }
        if ($request_method = "HEAD") {
            return 200;
        }
        if ($request_method = "COPY") {
            return 200;
        }
        if ($request_method = "DELETE") {
            return 200;
        }
        if ($http_if_match) {
            return 200;
        }
        if ($http_authorization !~* "aws4_request") {
            return 200;
        }
    }
    location @outage{
        return 403;
    }
    location / {
        auth_request /authentication;
        proxy_pass http://rgws;
        # if $do_not_cache is not empty the request would not be cached, this is relevant for list op for example
        set $do_not_cache '';
        # the IP or name of the RGWs
        #proxy_set_header Authorization $http_authorization;
        # my cache configured at the top of the file
        proxy_cache mycache;
        proxy_cache_lock_timeout 0s;
        proxy_cache_lock_age 1000s;
        proxy_http_version 1.1;
        # Getting 403 if this header not set
        proxy_set_header Host $host;
        # Cache all 200 OK's for 1 day
        proxy_cache_valid 200 206 1d;
        # Use stale cache file in all errors from upstream if we can
        proxy_cache_use_stale updating;
        proxy_cache_background_update on;
        # Try to check if etag have changed, if yes, do not re-fetch from rgw the object
        proxy_cache_revalidate on;
        # Lock the cache so that only one request can populate it at a time
        proxy_cache_lock on;
        # prevent convertion of head requests to get requests
        proxy_cache_convert_head off;
        # Listing all buckets should not be cached 
        if ($request_uri = "/") {
            set $do_not_cache "no";
        }
        # URI including question mark are not supported to prevent bucket listing cache
        if ($request_uri ~* (\?)) {
            set $do_not_cache "no";
        }
        # Use the original x-amz-date if the aws auth module didn't create one 
        proxy_no_cache $do_not_cache;
        proxy_set_header Authorization $http_authorization;
        proxy_set_header Range $http_range;
        # This is on which content the nginx to use for hashing the cache keys 
        proxy_cache_key "$request_uri$request_method$request_body$http_range";
        client_max_body_size 0;
    }
}