lightning fast with varnish
TRANSCRIPT
apt-get install apt-transport-https curl https://repo.varnish-cache.org/GPG-key.txt | apt-key add - echo "deb https://repo.varnish-cache.org/debian/ jessie varnish-4.1"\ >> /etc/apt/sources.list.d/varnish-cache.list apt-get update apt-get install varnish
Install on Linux (Debian)
There's stuff for Ubuntu, RHEL
& CentOS too
-a :80 \ -a :81,PROXY \ -T localhost:6082 \ -f /etc/varnish/default.vcl \ -S /etc/varnish/secret \ -s malloc,3g \ -j unix,user=myUser
Simple config
✓Doesn't cache when cookies are present ✓Doesn't cache when cookies are set ✓Doesn't cache when Autorization headers are present
State
Avoids caching user-
specific contentAffects hit
rate
Expiration
Expires: Sat, 09 Sep 2017 14:30:00 GMT
Cache-control: public, max-age=3600, s-maxage=86400
Cache-control: private, no-cache, no-store
Expiration precedence
1. beresp.ttl (VCL) 2. s-maxage 3. max-age 4. expires 5. 120 seconds by default
Conditional requests
HTTP/1.1 200 OK Host: localhost Etag: 7c9d70604c6061da9bb9377d3f00eb27 Content-type: text/html; charset=UTF-8
Hello world output
GET /if_none_match.php HTTP/1.1 Host: localhost User-Agent: curl/7.48.0
Conditional requests
HTTP/1.0 304 Not Modified Host: localhost Etag: 7c9d70604c6061da9bb9377d3f00eb27
GET /if_none_match.php HTTP/1.1 Host: localhost User-Agent: curl/7.48.0 If-None-Match: 7c9d70604c6061da9bb9377d3f00eb27
Conditional requests
HTTP/1.1 200 OK Host: localhost Last-Modified: Fri, 22 Jul 2016 10:11:16 GMT Content-type: text/html; charset=UTF-8
Hello world output
GET /if_none_match.php HTTP/1.1 Host: localhost User-Agent: curl/7.48.0
Conditional requests
HTTP/1.0 304 Not Modified Host: localhost Last-Modified: Fri, 22 Jul 2016 10:11:16 GMT
GET /if_none_match.php HTTP/1.1 Host: localhost User-Agent: curl/7.48.0 If-Last-Modified: Fri, 22 Jul 2016 10:11:16 GMT
Cache variationsGET / HTTP/1.1 Host: localhost Accept-Language: nl
HTTP/1.1 200 OK Host: localhost Vary: Accept-Language
Hallo, deze pagina is in het Nederlands geschreven.
✓Only GET & HEAD ✓No Cookie ✓No Authorization
Built-in VCL (frontend)
Lookup in cache
Otherwise pass and don't
cache
✓No Set-Cookie ✓TTL > 0 ✓No "no-cache", "private", "no-store"
Built-in VCL (backend)
Store in cacheOtherwise
blacklist for 120s
sub vcl_recv { if (req.method == "PRI") { /* We do not support SPDY or HTTP/2.0 */ return (synth(405)); } if (req.method != "GET" && req.method != "HEAD" && req.method != "PUT" && req.method != "POST" && req.method != "TRACE" && req.method != "OPTIONS" && req.method != "DELETE") { /* Non-RFC2616 or CONNECT which is weird. */ return (pipe); }
if (req.method != "GET" && req.method != "HEAD") { /* We only deal with GET and HEAD by default */ return (pass); } if (req.http.Authorization || req.http.Cookie) { /* Not cacheable by default */ return (pass); } return (hash); }
sub vcl_pipe { # By default Connection: close is set on all piped requests, to stop # connection reuse from sending future requests directly to the # (potentially) wrong backend. If you do want this to happen, you can undo # it here. # unset bereq.http.connection; return (pipe); }
sub vcl_pass { return (fetch); }
sub vcl_hash { hash_data(req.url); if (req.http.host) { hash_data(req.http.host); } else { hash_data(server.ip); } return (lookup); }
sub vcl_purge { return (synth(200, "Purged")); }
sub vcl_hit { if (obj.ttl >= 0s) { // A pure unadultered hit, deliver it return (deliver); } if (obj.ttl + obj.grace > 0s) { // Object is in grace, deliver it // Automatically triggers a background fetch return (deliver); } // fetch & deliver once we get the result return (miss); }
sub vcl_miss { return (fetch); }
sub vcl_deliver { return (deliver); }
sub vcl_backend_fetch { return (fetch); }
sub vcl_backend_response { if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Surrogate-control ~ "no-store" || (!beresp.http.Surrogate-Control && beresp.http.Cache-Control ~ "no-cache|no-store|private") || beresp.http.Vary == "*") { /* * Mark as "Hit-For-Pass" for the next 2 minutes */ set beresp.ttl = 120s; set beresp.uncacheable = true; } return (deliver); }
vcl 4.0;import std;
sub vcl_recv { set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
set req.url = std.querysort(req.url);
if (req.url ~ "\#") { set req.url = regsub(req.url, "\#.*$", ""); }
if (req.url ~ "\?$") { set req.url = regsub(req.url, "\?$", ""); }
if (req.restarts == 0) { if (req.http.Accept-Encoding) { if (req.http.User-Agent ~ "MSIE 6") { unset req.http.Accept-Encoding; } elsif (req.http.Accept-Encoding ~ "gzip") { set req.http.Accept-Encoding = "gzip"; } elsif (req.http.Accept-Encoding ~ "deflate") { set req.http.Accept-Encoding = "deflate"; } else { unset req.http.Accept-Encoding; } } }}
sub vcl_recv { if (req.url ~ "^/status\.php$" || req.url ~ "^/update\.php$" || req.url ~ "^/admin$" || req.url ~ "^/admin/.*$" || req.url ~ "^/user$" || req.url ~ "^/user/.*$" || req.url ~ "^/flag/.*$" || req.url ~ "^.*/ajax/.*$" || req.url ~ "^.*/ahah/.*$") { return (pass); }}
URL blacklist
vcl 4.0;
sub vcl_recv { set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "_ga=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "_gat=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "utmctr=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "utmcmd.=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "utmccn.=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "__gads=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "__atuv.=[^;]+(; )?", ""); set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");
if (req.http.cookie ~ "^\s*$") { unset req.http.cookie; }}
Remove tracking cookies
vcl 4.0;
sub vcl_recv { if (req.http.Cookie) { set req.http.Cookie = ";" + req.http.Cookie; set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";"); set req.http.Cookie = regsuball(req.http.Cookie, ";(PHPSESSID)=", "; \1="); set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", ""); set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
if (req.http.cookie ~ "^\s*$") { unset req.http.cookie; } }}
Only keep session cookie
sub vcl_recv { if (req.http.Cookie) { set req.http.Cookie = ";" + req.http.Cookie; set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";"); set req.http.Cookie = regsuball(req.http.Cookie, ";(language)=", "; \1="); set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", ""); set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
if (req.http.cookie ~ "^\s*$") { unset req.http.cookie; return(pass); }
return(hash); }}
sub vcl_hash { hash_data(regsub( req.http.Cookie, "^.*language=([^;]*);*.*$", "\1" ));}
Language cookie cache variation
sub vcl_hash { hash_data(req.http.Accept-Language);}
HTTP/1.1 200 OK Host: localhost Vary: Accept-Language
Hash language
Or just use
✓Don't throw everything in a session ✓Use dedicated cookies ✓Strip off tracking cookies in VCL ✓Match cookies in VCL & make decisions ✓Perform cache variations on cookie values ✓Only use session cookies when you really have to ✓Use JWT to store session state client-side
Cookie guidelines
<esi:include src="/header" />
Edge Side Includes
✓Placeholder ✓Parsed by Varnish ✓Output is a composition of blocks ✓State per block ✓TTL per block
sub vcl_recv { set req.http.Surrogate-Capability = "key=ESI/1.0";}
sub vcl_backend_response { if (beresp.http.Surrogate-Control ~ "ESI/1.0") { unset beresp.http.Surrogate-Control; set beresp.do_esi = true; }}
Edge Side Includes
<esi:include src="/header" />
Choose wisely
<hx:include src="/header"></hx:include>
ESI, parsed by Varnish
HInclude, parsed by Javascript
acl purge { "localhost"; "127.0.0.1"; "::1";}
sub vcl_recv { if (req.method == "PURGE") { if (!client.ip ~ purge) { return (synth(405, “Not allowed.”)); } return (purge); }}
Purging
acl purge { "localhost"; "127.0.0.1"; "::1";}
sub vcl_backend_response { set beresp.http.x-url = bereq.url; set beresp.http.x-host = bereq.http.host; }
sub vcl_deliver { unset resp.http.x-url; unset resp.http.x-host; }
sub vcl_recv { if (req.method == "PURGE") { if (!client.ip ~ purge) { return (synth(405, "Not allowed")); } if(req.http.x-purge-regex) { ban("obj.http.x-host == " + req.http.host + " && obj.http.x-url ~ " + req.http.x-purge-regex); } else { ban("obj.http.x-host == " + req.http.host + " && obj.http.x-url == " + req.url); } return (synth(200, "Purged")); }}
Banning
Banningcurl -XPURGE "http://example.com/products"
curl -XPURGE -H "x-purge-regex:/products" \ "http://example.com"
varnishadm> ban obj.http.x-host == example.com && obj.http.x-url ~ ^/product/[0-9]+/details