[GeneralDiscussion] server outage, caching changes

On 13 Jan 2010 at 7:36, Simon Michael wrote:

> Niall, thanks a lot, I had forgotten these pages. This really deserves  
> to be prominent in the zope & plone docs. I'll reread and explore  
> those suggestions more. What makes you prefer nginx + varnish so much  
> to apache (+ varnish, perhaps) ? Note my zope with just two threads  
> and 5000 target cache size likes to get up around 700M. I have a $20/ 
> mo 1G vps from prgmr.com to try, which seems a pretty good deal..

My reasons were purely from benchmarking - nginx had a narrower 
result variance even though it was slightly slower on average than 
its next competitor which in my mind meant it was more predictable 
and therefore more desirable. BTW nginx is many times faster than 
Apache when acting as a simple frontend, so much so that I simply 
wouldn't advise using Apache at all. In the end though, my nginx does 
very little except handling a few static pages, some gzip compression 
of merged CSS and Javascript which Zope cannot currently gzip (this 
makes a BIG difference to bandwidth usage when using Plone) and of 
course HTTPS. I don't think I have the magic nginx config for this 
public, so here it is:

	# Plone's ResourceRegistries can't gzip their compressed output
	# so we need to patch it here. Makes a huge difference to first
	# load times so it's worth the extra server load
	gzip             on;
	gzip_http_version 1.0;
	gzip_buffers     16 8k;
	gzip_min_length  1000;
	gzip_proxied     any;
	gzip_types       application/x-javascript text/css;
	gzip_disable     "MSIE [1-6]\.";
	gzip_vary        on;

If you do follow the nginx route, let me know if you have trouble 
figuring out how to correctly handle 503 errors. It took me weeks of 
experimentation :(

BTW I've attached my varnish default.vcl below. You'll note that it 
manages to "mount" cached copies of Google Finance and the RePeC 
bibliographic database as subdirectories of my websites - this allows 
one to work around cross-domain scripting in AJAX. Such is the power 
of varnish!

I have a low end 1.2Ghz Celeron D dedicated server, yet a nginx + 
varnish config will easily max out my 100Mbit network port before 
maxing out my CPU. I'd doubt though that my CPU would handle a 1Gbit 
max out though.

HTH,
Niall

# This VCL config file is adapted from template.vcl in 
http://pypi.python.org/pypi/plone.recipe.varnish
backend default {
	.host = "localhost";
	.port = "6100";
	.first_byte_timeout = 300s; /* varnish v2.0.3 or later only */
}

backend repec {
        .host = "ideas.repec.org";
        .port = "80";
}

backend google {
	.host = "209.85.229.106"; /*www.google.com";*/
	.port = "80";
}

/* Only permit cluster to purge files from cache */
acl purge {
	"dedi1.nedprod.com";
	"us1.nedproductions.biz";
	"localhost";
}

sub vcl_recv {
	set req.grace = 20s; /* Only enable if you don't mind slightly stale 
content */

	/* Before anything else we need to fix gzip compression */
	if (req.http.Accept-Encoding) {
		if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
			# No point in compressing these
			remove req.http.Accept-Encoding;
		} else if (req.http.Accept-Encoding ~ "gzip") {
			set req.http.Accept-Encoding = "gzip";
		} else if (req.http.Accept-Encoding ~ "deflate") {
			set req.http.Accept-Encoding = "deflate";
		} else {
			# unknown algorithm
			remove req.http.Accept-Encoding;
		}
	}

	if (req.request == "PURGE") {
		if (!client.ip ~ purge) {
			error 405 "Not allowed.";
		}
		/* Always purge by URL rather than going via vcl_hash
		   as it hashes other factors which break purging */
		purge_url(req.url);
		error 200 "Purged";
	}

	/* Rewrite all requests to /repec/cgi-bin/authorref.cgi to 
http://ideas.repec.org/cgi-bin/authorref.cgi */
	if (req.url ~ "^/repec/cgi-bin/authorref.cgi\?handle=" || req.url ~ 
"^/repec/cgi-bin/ref.cgi\?handle=") {
		set req.http.host = "ideas.repec.org";
		set req.url = regsub(req.url, "^/repec", "");
		set req.backend = repec;
		remove req.http.Cookie;
		lookup;
	} else if(req.url ~ "^/googlefinance/finance/converter\?") {
		set req.http.host = "www.google.com";
		set req.url = regsub(req.url, "^/googlefinance", "");
		set req.backend = google;
		remove req.http.Cookie;
		lookup;
	} else {
		set req.backend = default;
		if (req.http.X-Forwarded-Proto == "https" ) {
			set req.http.X-Forwarded-Port = "443";
		} else {
			set req.http.X-Forwarded-Port = "80";
		}
		if (req.http.host ~ "^(www\.|ipv6\.)?([-0-9a-zA-Z]+)\.([a-zA-
Z]+)$") {
			set req.http.host = regsub(req.http.host, "^(www\.|ipv6\.)?([-0-9a-
zA-Z]+)\.([a-zA-Z]+)$", "\1\2.\3");
			set req.url = "/VirtualHostBase/" req.http.X-Forwarded-Proto
				regsub(req.http.host, "^(www\.|ipv6\.)?([-0-9a-zA-Z]+)\.([a-zA-
Z]+)$", "/\1\2.\3:")
				req.http.X-Forwarded-Port
				regsub(req.http.host, "^(www\.|ipv6\.)?([-0-9a-zA-Z]+)\.([a-zA-
Z]+)$", "/\2.\3/\2.\3/VirtualHostRoot")
				req.url;
		}
	}

	if (req.request != "GET" &&
		req.request != "HEAD" &&
		req.request != "PUT" &&
		req.request != "POST" &&
		req.request != "TRACE" &&
		req.request != "OPTIONS" &&
		req.request != "DELETE") {
		/* Non-RFC2616 or CONNECT which is weird. */
		pipe;
	}

	if (req.request != "GET" && req.request != "HEAD") {
		/* We only deal with GET and HEAD by default */
		pass;
	}

	if (req.http.Cookie) {
	        # We only care about the "__ac.*" cookies, used for 
authentication and special persistent p_* cookies.
	        if (req.http.Cookie ~ "__ac.*" ) {
	                pass;
		}
		# Else strip all cookies
		remove req.http.Cookie;
        }

	if (req.http.If-None-Match) {
		pass;
	}

	if (req.url ~ "createObject") {
		pass;
	}

	lookup;
}

sub vcl_pipe {
	# This is not necessary if you do not do any request rewriting.
	set req.http.connection = "close";
}

sub vcl_hash {
	# Normally it hashes on URL and Host but we rewrite the host
	# into a VirtualHostBase URL. Therefore we can hash on URL alone.
	set req.hash += req.url;

	# One needs to include compression state normalised above
	if (req.http.Accept-Encoding) {
		set req.hash += req.http.Accept-Encoding;
	}

	# Differentiate based on login cookie too
	#set req.hash += req.http.cookie;

	return (hash);
}

sub vcl_hit {
	if (req.request == "PURGE") {
		purge_url(req.url);
		error 200 "Purged";
	}
}

sub vcl_miss {
	if (req.request == "PURGE") {
		error 404 "Not in cache";
	}
}

sub vcl_fetch {
	set req.grace = 20s; /* Only enable if you don't mind slightly stale 
content */
	if (req.url ~ "\.svgz$") {
		# Add a Content-Encoding to match compressed SVG
		set obj.http.Content-Type = "image/svg+xml";
		set obj.http.Content-Encoding = "gzip";
		remove obj.http.Content-Length;
		set obj.ttl = 86400s;
		set obj.http.Cache-Control = "max-age=3600";
		deliver;
	}
	if (req.http.host == "ideas.repec.org" || req.http.host == 
"www.google.com") {
		set obj.http.Content-Type = "text/html; charset=utf-8"; /* Correct 
the wrong response */
		set obj.ttl = 86400s;
		set obj.http.Cache-Control = "max-age=3600";
		deliver;
	}
	if (obj.http.Set-Cookie) {
		pass;
	}
	if (req.http.Authorization && !obj.http.Cache-Control ~ "public") {
		pass;
	}
	/* Only use this if you wish to override Plone's CacheFu */
	if (obj.ttl < 3600s) {
		if (obj.http.Cache-Control ~ "(private|no-cache|no-store)") {
			set obj.ttl = 60s; /* Caching everything anonymous for 60s is 
handy for being slashdotted :) */
		} else {
			set obj.ttl = 3600s;
		}
	}
}


sub vcl_error {
    set obj.http.Content-Type = "text/html; charset=utf-8";
    synthetic {"
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html>
  <head>
    <title>"} obj.status " " obj.response {"</title>
  </head>
  <body>
    <div style="background-color:yellow;">
      <h1>This website is unavailable</h1>
      <p>If you are seeing this page, either maintenance is being 
performed
      or something really bad has happened. Try returning in a few 
minutes.</p>
      <h2>Error "} obj.status " " obj.response {"</h2>
      <p>"} obj.response {"</p>
      <h3>Guru Meditation:</h3>
      <p>XID: "} req.xid {"</p>
      <address>
         <a href="http://www.nedproductions.biz/">ned Productions 
Ltd.</a>
      </address>
    </div>
    <div style="position:fixed;top:0;left:0;width:100%;height:100%;z-
index:-1;">
    <img alt="" src="/static/BBCTestCard.jpg" 
style="width:100%;height:100%" /></div>
  </body>
</html>
"};
    return (deliver);
}









#Below is a commented-out copy of the default VCL logic.  If you
#redefine any of these subroutines, the built-in logic will be
#appended to your code.
#
#sub vcl_recv {
#    if (req.request != "GET" &&
#      req.request != "HEAD" &&
#      req.request != "PUT" &&
#      req.request != "POST" &&
#      req.request != "TRACE" &&
#      req.request != "OPTIONS" &&
#      req.request != "DELETE") {
#        /* Non-RFC2616 or CONNECT which is weird. */
#        return (pipe);
#    }
#    if (req.request != "GET" && req.request != "HEAD") {
#        /* We only deal with GET and HEAD by default */
#        return (pass);
#    }
#    if (req.http.Authorization || req.http.Cookie) {
#        /* Not cacheable by default */
#        return (pass);
#    }
#    return (lookup);
#}
#
#sub vcl_pipe {
#    return (pipe);
#}
#
#sub vcl_pass {
#    return (pass);
#}
#
#sub vcl_hash {
#    set req.hash += req.url;
#    if (req.http.host) {
#        set req.hash += req.http.host;
#    } else {
#        set req.hash += server.ip;
#    }
#    return (hash);
#}
#
#sub vcl_hit {
#    if (!obj.cacheable) {
#        return (pass);
#    }
#    return (deliver);
#}
#
#sub vcl_miss {
#    return (fetch);
#}
#
#sub vcl_fetch {
#    if (!obj.cacheable) {
#        return (pass);
#    }
#    if (obj.http.Set-Cookie) {
#        return (pass);
#    }
#    set obj.prefetch =  -30s;
#    return (deliver);
#}
#
#sub vcl_deliver {
#    return (deliver);
#}
#
#sub vcl_discard {
#    /* XXX: Do not redefine vcl_discard{}, it is not yet supported 
*/
#    return (discard);
#}
#
#sub vcl_prefetch {
#    /* XXX: Do not redefine vcl_prefetch{}, it is not yet supported 
*/
#    return (fetch);
#}
#
#sub vcl_timeout {
#    /* XXX: Do not redefine vcl_timeout{}, it is not yet supported 
*/
#    return (discard);
#}
#
#sub vcl_error {
#    set obj.http.Content-Type = "text/html; charset=utf-8";
#    synthetic {"
#<?xml version="1.0" encoding="utf-8"?>
#<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
# "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
#<html>
#  <head>
#    <title>"} obj.status " " obj.response {"</title>
#  </head>
#  <body>
#    <h1>Error "} obj.status " " obj.response {"</h1>
#    <p>"} obj.response {"</p>
#    <h3>Guru Meditation:</h3>
#    <p>XID: "} req.xid {"</p>
#    <address>
#       <a href="http://www.varnish-cache.org/">Varnish</a>
#    </address>
#  </body>
#</html>
#"};
#    return (deliver);
#}


--
forwarded from http://zwiki.org/GeneralDiscussion#msg4B4DEE77.13979.5A70BC9@s_sourceforge.nedprod.com