Skip to content

Commit 387545a

Browse files
tgeoghegantgeoghegan
authored
metrics: Record request metrics on HTTP errors (#5979)
1 parent b49ec05 commit 387545a

File tree

2 files changed

+142
-11
lines changed

2 files changed

+142
-11
lines changed

‎modules/caddyhttp/metrics.go

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package caddyhttp
22

33
import (
44
"context"
5+
"errors"
56
"net/http"
67
"sync"
78
"time"
@@ -137,22 +138,33 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
137138
err := h.mh.ServeHTTP(wrec, r, next)
138139
dur := time.Since(start).Seconds()
139140
httpMetrics.requestCount.With(labels).Inc()
141+
142+
observeRequest := func(status int) {
143+
// If the code hasn't been set yet, and we didn't encounter an error, we're
144+
// probably falling through with an empty handler.
145+
if statusLabels["code"] == "" {
146+
// we still sanitize it, even though it's likely to be 0. A 200 is
147+
// returned on fallthrough so we want to reflect that.
148+
statusLabels["code"] = metrics.SanitizeCode(status)
149+
}
150+
151+
httpMetrics.requestDuration.With(statusLabels).Observe(dur)
152+
httpMetrics.requestSize.With(statusLabels).Observe(float64(computeApproximateRequestSize(r)))
153+
httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size()))
154+
}
155+
140156
if err != nil {
157+
var handlerErr HandlerError
158+
if errors.As(err, &handlerErr) {
159+
observeRequest(handlerErr.StatusCode)
160+
}
161+
141162
httpMetrics.requestErrors.With(labels).Inc()
142-
return err
143-
}
144163

145-
// If the code hasn't been set yet, and we didn't encounter an error, we're
146-
// probably falling through with an empty handler.
147-
if statusLabels["code"] == "" {
148-
// we still sanitize it, even though it's likely to be 0. A 200 is
149-
// returned on fallthrough so we want to reflect that.
150-
statusLabels["code"] = metrics.SanitizeCode(wrec.Status())
164+
return err
151165
}
152166

153-
httpMetrics.requestDuration.With(statusLabels).Observe(dur)
154-
httpMetrics.requestSize.With(statusLabels).Observe(float64(computeApproximateRequestSize(r)))
155-
httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size()))
167+
observeRequest(wrec.Status())
156168

157169
return nil
158170
}

‎modules/caddyhttp/metrics_test.go

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ import (
55
"errors"
66
"net/http"
77
"net/http/httptest"
8+
"strings"
89
"testing"
910

11+
"github.com/prometheus/client_golang/prometheus"
1012
"github.com/prometheus/client_golang/prometheus/testutil"
1113
)
1214

@@ -75,6 +77,123 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
7577
if actual := w.Result().Header; len(actual) != 0 {
7678
t.Errorf("Not empty: expected headers to be empty, but got %#v", actual)
7779
}
80+
81+
// handler returning an error with an HTTP status
82+
mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
83+
return Error(http.StatusTooManyRequests, nil)
84+
})
85+
86+
ih = newMetricsInstrumentedHandler("foo", mh)
87+
88+
r = httptest.NewRequest("GET", "/", nil)
89+
w = httptest.NewRecorder()
90+
91+
if err := ih.ServeHTTP(w, r, nil); err == nil {
92+
t.Errorf("expected error to be propagated")
93+
}
94+
95+
expected := `
96+
# HELP caddy_http_request_duration_seconds Histogram of round-trip request durations.
97+
# TYPE caddy_http_request_duration_seconds histogram
98+
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.005"} 1
99+
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.01"} 1
100+
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.025"} 1
101+
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.05"} 1
102+
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.1"} 1
103+
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.25"} 1
104+
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.5"} 1
105+
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1"} 1
106+
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="2.5"} 1
107+
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="5"} 1
108+
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="10"} 1
109+
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="+Inf"} 1
110+
caddy_http_request_duration_seconds_count{code="429",handler="foo",method="GET",server="UNKNOWN"} 1
111+
# HELP caddy_http_request_size_bytes Total size of the request. Includes body
112+
# TYPE caddy_http_request_size_bytes histogram
113+
caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="256"} 1
114+
caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="1024"} 1
115+
caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="4096"} 1
116+
caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="16384"} 1
117+
caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="65536"} 1
118+
caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="262144"} 1
119+
caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
120+
caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
121+
caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="+Inf"} 1
122+
caddy_http_request_size_bytes_sum{code="200",handler="bar",method="GET",server="UNKNOWN"} 23
123+
caddy_http_request_size_bytes_count{code="200",handler="bar",method="GET",server="UNKNOWN"} 1
124+
caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="256"} 1
125+
caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="1024"} 1
126+
caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="4096"} 1
127+
caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="16384"} 1
128+
caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="65536"} 1
129+
caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="262144"} 1
130+
caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
131+
caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
132+
caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="+Inf"} 1
133+
caddy_http_request_size_bytes_sum{code="200",handler="empty",method="GET",server="UNKNOWN"} 23
134+
caddy_http_request_size_bytes_count{code="200",handler="empty",method="GET",server="UNKNOWN"} 1
135+
caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="256"} 1
136+
caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1024"} 1
137+
caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="4096"} 1
138+
caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="16384"} 1
139+
caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="65536"} 1
140+
caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="262144"} 1
141+
caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
142+
caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
143+
caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="+Inf"} 1
144+
caddy_http_request_size_bytes_sum{code="429",handler="foo",method="GET",server="UNKNOWN"} 23
145+
caddy_http_request_size_bytes_count{code="429",handler="foo",method="GET",server="UNKNOWN"} 1
146+
# HELP caddy_http_response_size_bytes Size of the returned response.
147+
# TYPE caddy_http_response_size_bytes histogram
148+
caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="256"} 1
149+
caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="1024"} 1
150+
caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="4096"} 1
151+
caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="16384"} 1
152+
caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="65536"} 1
153+
caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="262144"} 1
154+
caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
155+
caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
156+
caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="+Inf"} 1
157+
caddy_http_response_size_bytes_sum{code="200",handler="bar",method="GET",server="UNKNOWN"} 12
158+
caddy_http_response_size_bytes_count{code="200",handler="bar",method="GET",server="UNKNOWN"} 1
159+
caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="256"} 1
160+
caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="1024"} 1
161+
caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="4096"} 1
162+
caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="16384"} 1
163+
caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="65536"} 1
164+
caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="262144"} 1
165+
caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
166+
caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
167+
caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="+Inf"} 1
168+
caddy_http_response_size_bytes_sum{code="200",handler="empty",method="GET",server="UNKNOWN"} 0
169+
caddy_http_response_size_bytes_count{code="200",handler="empty",method="GET",server="UNKNOWN"} 1
170+
caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="256"} 1
171+
caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1024"} 1
172+
caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="4096"} 1
173+
caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="16384"} 1
174+
caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="65536"} 1
175+
caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="262144"} 1
176+
caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
177+
caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
178+
caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="+Inf"} 1
179+
caddy_http_response_size_bytes_sum{code="429",handler="foo",method="GET",server="UNKNOWN"} 0
180+
caddy_http_response_size_bytes_count{code="429",handler="foo",method="GET",server="UNKNOWN"} 1
181+
# HELP caddy_http_request_errors_total Number of requests resulting in middleware errors.
182+
# TYPE caddy_http_request_errors_total counter
183+
caddy_http_request_errors_total{handler="bar",server="UNKNOWN"} 1
184+
caddy_http_request_errors_total{handler="foo",server="UNKNOWN"} 1
185+
`
186+
if err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected),
187+
"caddy_http_request_size_bytes",
188+
"caddy_http_response_size_bytes",
189+
// caddy_http_request_duration_seconds_sum will vary based on how long the test took to run,
190+
// so we check just the _bucket and _count metrics
191+
"caddy_http_request_duration_seconds_bucket",
192+
"caddy_http_request_duration_seconds_count",
193+
"caddy_http_request_errors_total",
194+
); err != nil {
195+
t.Errorf("received unexpected error: %s", err)
196+
}
78197
}
79198

80199
type middlewareHandlerFunc func(http.ResponseWriter, *http.Request, Handler) error

0 commit comments

Comments
 (0)