Re: add support to httpd for serving static brotli encoded files

On 2026/08/01 11:41, Moviuro wrote:
> On 26-02-25 09:46:36, Adam Mullins wrote:
> > 
> > Here is another unified diff against -current. Compared to the first it:
> > - Reorders SRVFLAG_BITS as suggested by Lloyd and adds an entry for
> >   brotli-static.
> > - Edits comments, code, and whitespace to conform to style(9).
> > - Removes some obvious comments and trims the length of the others.
> > - Sorts variable declarations to conform; ie largest to smallest.
> > - Renames the helper function from server_file_encoded_path() to
> >   find_compressed_path(). I wanted to avoid 'encoded' even though
> >   that is the name of the HTTP header because it could be confused for
> >   URL encoding. I also dropped the server_file prefix, following the
> >   lead of some other internal functions in server_file.c.
> 
> Hi there,
> 
> I suggest the following instead: overload gzip-static to handle all statically
> compressed files that can be found on disk. Current state of the art is
> gzip (~100%) + zstd (82.1%) + brotli (95.72%), so let's handle those [0].
> 
> For all available encodings sent by the client, if they are supported by httpd,
> pick the smallest available file that is also not older than the
> source/original.
> 
> Overloading gzip-static means that there's no new flag to manage in the config,
> the current setups continue working, and changes are available immediately when
> a supported compressed file is available on disk. When a new compression
> algorithm (dcb, dcz) becomes available, we just add it to the struct - one-line
> patches.

Seems risky to add new extensions under an existing config. What happens
if someone already has an unrelated file that gets picked up by this?
Especially if adding more extensions later? Explicit config would make
more sense.

> Trying to work on those was also a fun way to find out that brotli(1) truncates
> nanoseconds on OpenBSD, causing httpd(8) to ignore .br files [1]. Here's a patch
> against usr.sbin/httpd collected on 2026-07-31 (server_file.c,v 1.84).
> 
> diff --git i/usr.sbin/httpd/httpd.conf.5 w/usr.sbin/httpd/httpd.conf.5
> index c28869275df..2500e294f73 100644
> --- i/usr.sbin/httpd/httpd.conf.5
> +++ w/usr.sbin/httpd/httpd.conf.5
> @@ -480,11 +480,18 @@ features in use
>  .Pq omitted when TLS client verification is not in use .
>  .El
>  .It Ic gzip-static
> -Enable static gzip compression to save bandwidth.
> +Enable static compression to save bandwidth.
>  .Pp
>  If gzip encoding is accepted and if the requested file exists with
>  an additional .gz suffix, use the compressed file instead and deliver
>  it with content encoding gzip.
> +.Pp
> +gzip-static is overloaded to also work with brotli- (.br) and zstd- (.zst)
> +compressed files. The smallest file supported by the client will be used.
> +.Pp
> +Additionally, compressed files are only used if they are not older than the
> +requested file on disk. Some archiving software may not correctly copy mtime
> +and cause compressed files to be ignored. Cf. touch(1) -r or stat(1) -f "%Fm".
>  .It Ic header Ar option
>  Manipulate HTTP response headers.
>  Multiple
> diff --git i/usr.sbin/httpd/server_file.c w/usr.sbin/httpd/server_file.c
> index 5eca7f3fc5f..82e463bc58a 100644
> --- i/usr.sbin/httpd/server_file.c
> +++ w/usr.sbin/httpd/server_file.c
> @@ -168,44 +168,71 @@ server_file_access(struct httpd *env, struct client *clt,
>  			    fd, &st, r->kv_value));
>  	}
>  
> -	/* change path to path.gz if necessary. */
> +	/* change path to the smallest compressed file if necessary. */
>  	if (srv_conf->flags & SRVFLAG_GZIP_STATIC) {
>  		struct http_descriptor	*req = clt->clt_descreq;
>  		struct http_descriptor	*resp = clt->clt_descresp;
> -		struct stat		 gzst;
> -		int			 gzfd;
> -		char			 gzpath[PATH_MAX];
> +		struct stat		 compst, bestst;
> +		int			 compfd, bestfd = -1;
> +		char			 comppath[PATH_MAX];
> +		const char		*bestenc = NULL;
> +		off_t			 bestsize = st.st_size;
> +		size_t			 i;
> +
> +		struct {
> +			const char	*enc;
> +			const char	*ext;
> +		} comp[] = {
> +			{ "zstd", ".zst" },
> +			{ "br",   ".br" },
> +			{ "gzip", ".gz" }
> +		};
>  
>  		/* check Accept-Encoding header */
>  		key.kv_key = "Accept-Encoding";
>  		r = kv_find(&req->http_headers, &key);
>  
> -		if (r != NULL && strstr(r->kv_value, "gzip") != NULL) {
> -			/* append ".gz" to path and check existence */
> -			ret = snprintf(gzpath, sizeof(gzpath), "%s.gz", path);
> -			if (ret < 0 || (size_t)ret >= sizeof(gzpath)) {
> -				close(fd);
> -				return (500);
> -			}
> +		if (r != NULL) {
> +			for (i = 0; i < (sizeof(comp) / sizeof(comp[0])); i++) {
> +				/* check if client supports our compression */
> +				if (strstr(r->kv_value, comp[i].enc) == NULL)
> +					continue;
>  
> -			if ((gzfd = open(gzpath, O_RDONLY)) != -1) {
> -				/* .gz must be a file, and not older */
> -				if (fstat(gzfd, &gzst) != -1 &&
> -				    S_ISREG(gzst.st_mode) &&
> -				    timespeccmp(&gzst.st_mtim, &st.st_mtim,
> -				    >=)) {
> -					kv_add(&resp->http_headers,
> -					    "Content-Encoding", "gzip");
> -					/* Use original file timestamp */
> -					gzst.st_mtim = st.st_mtim;
> -					st = gzst;
> -					close(fd);
> -					fd = gzfd;
> +				/* check if a matching compressed path exists */
> +				ret = snprintf(comppath, sizeof(comppath), "%s%s",
> +				    path, comp[i].ext);
> +				if (ret < 0 || (size_t)ret >= sizeof(comppath))
> +					continue;
> +
> +				if ((compfd = open(comppath, O_RDONLY)) == -1)
> +					continue;
> +
> +				/* compressed path must be a file, and not
> +				 * older, and smaller than what we have */
> +				if (fstat(compfd, &compst) != -1 &&
> +				    S_ISREG(compst.st_mode) &&
> +				    timespeccmp(&compst.st_mtim, &st.st_mtim, >=) &&
> +				    compst.st_size < bestsize) {
> +					if (bestfd != -1)
> +						close(bestfd);
> +					bestfd = compfd;
> +					bestsize = compst.st_size;
> +					bestst = compst;
> +					bestenc = comp[i].enc;
>  				} else {
> -					close(gzfd);
> +					close(compfd);
>  				}
>  			}
>  		}
> +
> +		if (bestfd != -1) {
> +			kv_add(&resp->http_headers, "Content-Encoding", bestenc);
> +			/* Use original file timestamp */
> +			bestst.st_mtim = st.st_mtim;
> +			st = bestst;
> +			close(fd);
> +			fd = bestfd;
> +		}
>  	}
>  
>  	return (server_file_request(env, clt, media, fd, &st));
> 
> 
> [0] https://caniuse.com/?search=accept-encoding
> [1] https://github.com/google/brotli/issues/1515
>