fileserver: read etags from precomputed files (#6222)

Author: armadi1809

caddytest/integration/caddyfile_adapt/file_server_etag_file_extensions.caddyfiletest

@@ -0,0 +1,40 @@
+:8080 {
+	root * ./
+	file_server {
+		etag_file_extensions .b3sum .sha256
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":8080"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "vars",
+									"root": "./"
+								},
+								{
+									"etag_file_extensions": [
+										".b3sum",
+										".sha256"
+									],
+									"handler": "file_server",
+									"hide": [
+										"./Caddyfile"
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

modules/caddyhttp/fileserver/caddyfile.go

@@ -164,6 +164,13 @@ func (fsrv *FileServer) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 			}
 			fsrv.PassThru = true
 
+		case "etag_file_extensions":
+			etagFileExtensions := d.RemainingArgs()
+			if len(etagFileExtensions) == 0 {
+				return d.ArgErr()
+			}
+			fsrv.EtagFileExtensions = etagFileExtensions
+
 		default:
 			return d.Errf("unknown subdirective '%s'", d.Val())
 		}

modules/caddyhttp/fileserver/staticfiles.go

@@ -161,6 +161,12 @@ type FileServer struct {
 	PrecompressedOrder []string `json:"precompressed_order,omitempty"`
 	precompressors     map[string]encode.Precompressed
 
+	// List of file extensions to try to read Etags from.
+	// If set, file Etags will be read from sidecar files
+	// with any of these suffixes, instead of generating
+	// our own Etag.
+	EtagFileExtensions []string `json:"etag_file_extensions,omitempty"`
+
 	fsmap caddy.FileSystems
 
 	logger *zap.Logger
@@ -396,6 +402,14 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 		w.Header().Del("Accept-Ranges")
 		w.Header().Add("Vary", "Accept-Encoding")
 
+		// try to get the etag from pre computed files if an etag suffix list was provided
+		if etag == "" && fsrv.EtagFileExtensions != nil {
+			etag, err = fsrv.getEtagFromFile(fileSystem, compressedFilename)
+			if err != nil {
+				return err
+			}
+		}
+
 		// don't assign info = compressedInfo because sidecars are kind
 		// of transparent; however we do need to set the Etag:
 		// https://caddy.community/t/gzipped-sidecar-file-wrong-same-etag/16793
@@ -420,7 +434,13 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 			return err // error is already structured
 		}
 		defer file.Close()
-
+		// try to get the etag from pre computed files if an etag suffix list was provided
+		if etag == "" && fsrv.EtagFileExtensions != nil {
+			etag, err = fsrv.getEtagFromFile(fileSystem, filename)
+			if err != nil {
+				return err
+			}
+		}
 		if etag == "" {
 			etag = calculateEtag(info)
 		}
@@ -639,6 +659,22 @@ func calculateEtag(d os.FileInfo) string {
 	return `"` + t + s + `"`
 }
 
+// Finds the first corresponding etag file for a given file in the file system and return its content
+func (fsrv *FileServer) getEtagFromFile(fileSystem fs.FS, filename string) (string, error) {
+	for _, suffix := range fsrv.EtagFileExtensions {
+		etagFilename := filename + suffix
+		etag, err := fs.ReadFile(fileSystem, etagFilename)
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			return "", fmt.Errorf("cannot read etag from file %s: %v", etagFilename, err)
+		}
+		return string(etag), nil
+	}
+	return "", nil
+}
+
 // redirect performs a redirect to a given path. The 'toPath' parameter
 // MUST be solely a path, and MUST NOT include a query.
 func redirect(w http.ResponseWriter, r *http.Request, toPath string) error {