feat(metrics): label http requests and bytes with a resource

Commit `bdd5983`

Antonio Mika · 2026-10-25 22:57:04 -0400 EDT

parent eea11fa

feat(metrics): label http requests and bytes with a resource

4 files changed, +199, -68

M pkg/apps/pgs/site_metrics_test.go

M pkg/apps/pgs/web.go

M pkg/shared/router/metrics.go

M pkg/shared/router/metrics_test.go

M pkg/apps/pgs/site_metrics_test.go

+26, -12

......@@ -26,7 +26,7 @@ func TestSiteMetricsCountCacheHits(t *testing.T) {
	cfg.Domain = "pgs.test"
	httpCache, _ := NewPgsHttpCache(cfg, newWebRouter(cfg))
	reg := prometheus.NewRegistry()
	h := router.NewUserMetrics("pgs", reg, router.UserMetricsOpts{SiteUser: siteUser(cfg)}).Handler(httpCache)
	h := router.NewUserMetrics("pgs", reg, router.UserMetricsOpts{Owner: siteProject(cfg)}).Handler(httpCache)

	var statuses []string
	for range 2 {

......@@ -42,20 +42,34 @@ func TestSiteMetricsCountCacheHits(t *testing.T) {
	if err != nil {
		t.Fatal(err)
	}
	var count float64
	for _, family := range families {
		if family.GetName() != "pico_http_requests_total" {
			continue
		}
		for _, m := range family.GetMetric() {
			for _, l := range m.GetLabel() {
				if l.GetName() == "user" && l.GetValue() == user.Name {
					count += m.GetCounter().GetValue()
	count := func(name string, labels map[string]string) float64 {
		var total float64
		for _, family := range families {
			if family.GetName() != name {
				continue
			}
		metrics:
			for _, m := range family.GetMetric() {
				matched := 0
				for _, l := range m.GetLabel() {
					if want, ok := labels[l.GetName()]; ok {
						if want != l.GetValue() {
							continue metrics
						}
						matched++
					}
				}
				if matched == len(labels) {
					total += m.GetCounter().GetValue()
				}
			}
		}
		return total
	}
	if got := count("pico_http_requests_total", map[string]string{"user": user.Name}); got != 2 {
		t.Errorf("requests for %s = %v, want 2", user.Name, got)
	}
	if count != 2 {
		t.Errorf("requests for %s = %v, want 2", user.Name, count)
	if got := count("pico_http_requests_total", map[string]string{"user": user.Name, "resource": "test"}); got != 2 {
		t.Errorf("requests for %s/test = %v, want 2", user.Name, got)
	}
}

M pkg/apps/pgs/web.go

+10, -7

......@@ -134,23 +134,23 @@ func NewPgsHttpCache(cfg *Config, upstream http.Handler) (*httpcache.HttpCache,
	return httpCache, cache
}

// NewSiteMetrics counts requests for each site owner. The owner comes from
// the host, so responses served from a cache count too.
// NewSiteMetrics counts requests for each site owner and project. Both come
// from the host, so responses served from a cache count too.
func NewSiteMetrics(cfg *Config, app string, trustSuccess bool) *router.UserMetrics {
	return router.NewUserMetrics(app, prometheus.DefaultRegisterer, router.UserMetricsOpts{
		SiteUser:     siteUser(cfg),
		Owner:        siteProject(cfg),
		TrustSuccess: trustSuccess,
	})
}

func siteUser(cfg *Config) func(r *http.Request) string {
	return func(r *http.Request) string {
func siteProject(cfg *Config) func(r *http.Request) (string, string) {
	return func(r *http.Request) (string, string) {
		subdomain := router.GetSubdomainFromRequest(r, cfg.Domain, cfg.TxtPrefix)
		if subdomain == "" {
			return ""
			return "", ""
		}
		props, _ := router.GetProjectFromSubdomain(subdomain)
		return props.Username
		return props.Username, props.ProjectName
	}
}

......@@ -523,6 +523,9 @@ func (web *WebRouter) ServeAsset(fname string, opts *storage.ImgProcessOpts, has
		return
	}
	router.SetUser(r, site.user.Name)
	// The name from the host, so cache hits for it count under the same
	// project.
	router.SetResource(r, props.ProjectName)
	user, project, bucket := site.user, site.project, site.bucket

	logger = logger.With(

M pkg/shared/router/metrics.go

+83, -38

......@@ -6,6 +6,7 @@ import (
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"

	lru "github.com/hashicorp/golang-lru/v2"

......@@ -16,26 +17,28 @@ import (
// UserMetricsOpts tells UserMetrics who a request belongs to when no
// handler called SetUser, such as a response served from a cache.
type UserMetricsOpts struct {
	// SiteUser names the owner of the site a request is for, without
	// checking that the user exists.
	SiteUser func(r *http.Request) string
	// TrustSuccess counts a SiteUser as real once its site returns a
	// success, for a proxy that can't look users up. Only safe when every
	// route on a site's host fails for a site that doesn't exist.
	// Owner names the user and resource a request is for, read from the
	// host without checking that either exists.
	Owner func(r *http.Request) (user, resource string)
	// TrustSuccess counts an Owner as real once it returns a success, for
	// a proxy that can't look users up. Only safe when every route on a
	// site's host fails for a site that doesn't exist.
	TrustSuccess bool
}

// UserMetrics counts HTTP requests, latency and response bytes for each
// site owner. Every app shares the metric names and sets the app label, so
// site owner. Requests and bytes also carry the resource they were for,
// such as a pgs project, which the latency histogram leaves out to keep its
// series down. Every app shares the metric names and sets the app label, so
// queries can aggregate across services.
type UserMetrics struct {
	requests *prometheus.CounterVec
	duration *prometheus.HistogramVec
	bytes    *prometheus.CounterVec
	opts     UserMetricsOpts
	// known holds the users a handler or a successful response confirmed.
	// A name from the Host header alone never becomes a label, so made-up
	// subdomains can't add series.
	// known holds the users and resources a handler or a successful
	// response confirmed. A name from the Host header alone never becomes a
	// label, so made-up subdomains can't add series.
	known *lru.Cache[string, struct{}]
}

......@@ -49,7 +52,7 @@ func NewUserMetrics(app string, reg prometheus.Registerer, opts UserMetricsOpts)
			ConstLabels: prometheus.Labels{"app": app},
			Name:        "requests_total",
			Help:        "Number of HTTP requests handled.",
		}, []string{"user", "status", "method"}),
		}, []string{"user", "resource", "status", "method"}),
		duration: auto.NewHistogramVec(prometheus.HistogramOpts{
			Namespace:   "pico",
			Subsystem:   "http",

......@@ -64,62 +67,104 @@ func NewUserMetrics(app string, reg prometheus.Registerer, opts UserMetricsOpts)
			ConstLabels: prometheus.Labels{"app": app},
			Name:        "response_bytes_total",
			Help:        "Number of response body bytes sent.",
		}, []string{"user"}),
		}, []string{"user", "resource"}),
		opts:  opts,
		known: known,
	}
}

type ctxUserKey struct{}
type owner struct {
	user     string
	resource string
}

type ctxOwnerKey struct{}

// SetUser records name as the user a request belongs to. Handlers call it
// once they've found the user in the database.
func SetUser(r *http.Request, name string) {
	if user, ok := r.Context().Value(ctxUserKey{}).(*string); ok {
		*user = name
	if o, ok := r.Context().Value(ctxOwnerKey{}).(*owner); ok {
		o.user = name
	}
}

// SetResource records name as what a request is for within the user's
// account, such as a pgs project. Handlers call it alongside SetUser once
// they've found it.
func SetResource(r *http.Request, name string) {
	if o, ok := r.Context().Value(ctxOwnerKey{}).(*owner); ok {
		o.resource = name
	}
}

func (m *UserMetrics) Handler(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		var user string
		var o owner
		mw := &metricsWriter{ResponseWriter: w}
		next.ServeHTTP(mw, r.WithContext(context.WithValue(r.Context(), ctxUserKey{}, &user)))
		next.ServeHTTP(mw, r.WithContext(context.WithValue(r.Context(), ctxOwnerKey{}, &o)))

		status := mw.Status()
		m.Observe(m.resolveUser(r, user, status), status, r.Method, time.Since(start), mw.bytes)
		user, resource := m.resolve(r, o, status)
		m.Observe(user, resource, status, r.Method, time.Since(start), mw.bytes)
	})
}

// Observe records one request for servers that can't use Handler, such as
// tuns's gin logger. user must already be confirmed to exist. A negative
// size, which gin reports when nothing was written, counts as zero bytes.
func (m *UserMetrics) Observe(user string, status int, method string, elapsed time.Duration, bytes int64) {
	labels := []string{user, strconv.Itoa(status), metricMethod(method)}
	m.requests.WithLabelValues(labels...).Inc()
	m.duration.WithLabelValues(labels...).Observe(elapsed.Seconds())
	m.bytes.WithLabelValues(user).Add(float64(max(bytes, 0)))
// tuns's gin logger. user and resource must already be confirmed to
// exist, and resource may be empty. A negative size, which gin reports when
// nothing was written, counts as zero bytes.
func (m *UserMetrics) Observe(user, resource string, status int, method string, elapsed time.Duration, bytes int64) {
	if user == "" {
		resource = ""
	}
	code, verb := strconv.Itoa(status), metricMethod(method)
	m.requests.WithLabelValues(user, resource, code, verb).Inc()
	m.duration.WithLabelValues(user, code, verb).Observe(elapsed.Seconds())
	m.bytes.WithLabelValues(user, resource).Add(float64(max(bytes, 0)))
}

func (m *UserMetrics) resolveUser(r *http.Request, user string, status int) string {
	if user != "" {
		m.known.Add(user, struct{}{})
		return user
func (m *UserMetrics) resolve(r *http.Request, o owner, status int) (string, string) {
	var hostUser, hostResource string
	if m.opts.Owner != nil {
		hostUser, hostResource = m.opts.Owner(r)
	}
	if m.opts.SiteUser == nil {
		return ""
	}
	user = m.opts.SiteUser(r)
	trusted := m.opts.TrustSuccess && (status >= 200 && status < 300 || status == http.StatusNotModified)

	user := m.confirm(o.user, hostUser, trusted)
	if user == "" {
		return "", ""
	}
	// Resources are keyed under their owner, and user names can't hold a
	// slash.
	prefix := user + "/"
	set, fromHost := "", ""
	if o.resource != "" {
		set = prefix + o.resource
	}
	if hostUser == user && hostResource != "" {
		fromHost = prefix + hostResource
	}
	resource := strings.TrimPrefix(m.confirm(set, fromHost, trusted), prefix)
	return user, resource
}

// confirm returns the name a handler set, or else the name from the host
// once it's known or trusted.
func (m *UserMetrics) confirm(set, fromHost string, trusted bool) string {
	if set != "" {
		m.known.Add(set, struct{}{})
		return set
	}
	if fromHost == "" {
		return ""
	}
	if m.known.Contains(user) {
		return user
	if m.known.Contains(fromHost) {
		return fromHost
	}
	if m.opts.TrustSuccess && (status >= 200 && status < 300 || status == http.StatusNotModified) {
		m.known.Add(user, struct{}{})
		return user
	if trusted {
		m.known.Add(fromHost, struct{}{})
		return fromHost
	}
	return ""
}

M pkg/shared/router/metrics_test.go

+80, -11

......@@ -9,13 +9,15 @@ import (
	"github.com/prometheus/client_golang/prometheus"
)

// metricValue reads a counter, or a histogram's count, from reg.
// metricValue sums the counters, or histogram counts, in reg that match
// labels.
func metricValue(t *testing.T, reg *prometheus.Registry, name string, labels map[string]string) float64 {
	t.Helper()
	families, err := reg.Gather()
	if err != nil {
		t.Fatalf("gather: %v", err)
	}
	var total float64
	for _, family := range families {
		if family.GetName() != name {
			continue

......@@ -28,12 +30,20 @@ func metricValue(t *testing.T, reg *prometheus.Registry, name string, labels map
				}
			}
			if h := m.GetHistogram(); h != nil {
				return float64(h.GetSampleCount())
				total += float64(h.GetSampleCount())
			} else {
				total += m.GetCounter().GetValue()
			}
			return m.GetCounter().GetValue()
		}
	}
	return 0
	return total
}

// hostOwner reads "user-project.example.com" the way pgs reads its
// subdomains.
func hostOwner(r *http.Request) (string, string) {
	user, resource, _ := strings.Cut(strings.Split(r.Host, ".")[0], "-")
	return user, resource
}

func serve(h http.Handler, method, target string) *httptest.ResponseRecorder {

......@@ -67,9 +77,7 @@ func TestUserMetricsSetUser(t *testing.T) {

func TestUserMetricsSiteUser(t *testing.T) {
	reg := prometheus.NewRegistry()
	m := NewUserMetrics("test", reg, UserMetricsOpts{
		SiteUser: func(r *http.Request) string { return strings.Split(r.Host, ".")[0] },
	})
	m := NewUserMetrics("test", reg, UserMetricsOpts{Owner: hostOwner})
	// The handler confirms alice on her first request; later ones stand in
	// for cache hits that never reach it.
	confirmed := false

......@@ -98,7 +106,7 @@ func TestUserMetricsSiteUser(t *testing.T) {
func TestUserMetricsTrustSuccess(t *testing.T) {
	reg := prometheus.NewRegistry()
	m := NewUserMetrics("test", reg, UserMetricsOpts{
		SiteUser:     func(r *http.Request) string { return strings.Split(r.Host, ".")[0] },
		Owner:        hostOwner,
		TrustSuccess: true,
	})
	h := m.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

......@@ -107,7 +115,7 @@ func TestUserMetricsTrustSuccess(t *testing.T) {
		}
	}))

	serve(h, http.MethodGet, "http://bob.example.com/")
	serve(h, http.MethodGet, "http://bob-docs.example.com/")
	serve(h, http.MethodGet, "http://ghost.example.com/")

	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "bob", "status": "200"}); got != 1 {

......@@ -116,6 +124,64 @@ func TestUserMetricsTrustSuccess(t *testing.T) {
	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "", "status": "404"}); got != 1 {
		t.Errorf("unlabeled 404s = %v, want 1", got)
	}
	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "bob", "resource": "docs"}); got != 1 {
		t.Errorf("bob/docs requests = %v, want 1", got)
	}
}

func TestUserMetricsSetResource(t *testing.T) {
	reg := prometheus.NewRegistry()
	m := NewUserMetrics("test", reg, UserMetricsOpts{})
	h := m.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		SetUser(r, "alice")
		if r.URL.Path == "/blog" {
			SetResource(r, "blog")
		}
		_, _ = w.Write([]byte("hello"))
	}))

	serve(h, http.MethodGet, "/blog")
	serve(h, http.MethodGet, "/")

	labels := map[string]string{"app": "test", "user": "alice", "resource": "blog"}
	if got := metricValue(t, reg, "pico_http_requests_total", labels); got != 1 {
		t.Errorf("alice/blog requests = %v, want 1", got)
	}
	if got := metricValue(t, reg, "pico_http_response_bytes_total", labels); got != 5 {
		t.Errorf("alice/blog bytes = %v, want 5", got)
	}
	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "alice"}); got != 2 {
		t.Errorf("alice requests = %v, want 2", got)
	}
}

func TestUserMetricsResourceFromHost(t *testing.T) {
	reg := prometheus.NewRegistry()
	m := NewUserMetrics("test", reg, UserMetricsOpts{Owner: hostOwner})
	// The handler confirms alice's blog on its first request; later ones
	// stand in for cache hits that never reach it.
	confirmed := false
	h := m.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if !confirmed {
			SetUser(r, "alice")
			SetResource(r, "blog")
			confirmed = true
		}
	}))

	serve(h, http.MethodGet, "http://alice-blog.example.com/")
	serve(h, http.MethodGet, "http://alice-blog.example.com/")
	serve(h, http.MethodGet, "http://alice-ghost.example.com/")

	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "alice", "resource": "blog"}); got != 2 {
		t.Errorf("alice/blog requests = %v, want 2", got)
	}
	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "alice", "resource": ""}); got != 1 {
		t.Errorf("alice requests without a resource = %v, want 1", got)
	}
	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "alice"}); got != 3 {
		t.Errorf("alice requests = %v, want 3", got)
	}
}

func TestUserMetricsMethod(t *testing.T) {

......@@ -133,8 +199,8 @@ func TestUserMetricsObserve(t *testing.T) {
	reg := prometheus.NewRegistry()
	m := NewUserMetrics("test", reg, UserMetricsOpts{})

	m.Observe("carol", http.StatusOK, http.MethodGet, 0, 7)
	m.Observe("carol", http.StatusNotModified, "MADEUP", 0, -1)
	m.Observe("carol", "app", http.StatusOK, http.MethodGet, 0, 7)
	m.Observe("carol", "", http.StatusNotModified, "MADEUP", 0, -1)

	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "carol", "status": "200", "method": "GET"}); got != 1 {
		t.Errorf("200 requests = %v, want 1", got)

......@@ -145,6 +211,9 @@ func TestUserMetricsObserve(t *testing.T) {
	if got := metricValue(t, reg, "pico_http_response_bytes_total", map[string]string{"user": "carol"}); got != 7 {
		t.Errorf("bytes = %v, want 7", got)
	}
	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "carol", "resource": "app"}); got != 1 {
		t.Errorf("carol/app requests = %v, want 1", got)
	}
}

func TestUserMetricsHijack(t *testing.T) {

...	...		@@ -134,23 +134,23 @@ func NewPgsHttpCache(cfg Config, upstream http.Handler) (httpcache.HttpCache,
134	134		return httpCache, cache
135	135		}
136	136
137		-	// NewSiteMetrics counts requests for each site owner. The owner comes from
138		-	// the host, so responses served from a cache count too.
	137	+	// NewSiteMetrics counts requests for each site owner and project. Both come
	138	+	// from the host, so responses served from a cache count too.
139	139		func NewSiteMetrics(cfg Config, app string, trustSuccess bool) router.UserMetrics {
140	140		return router.NewUserMetrics(app, prometheus.DefaultRegisterer, router.UserMetricsOpts{
141		-	SiteUser: siteUser(cfg),
	141	+	Owner: siteProject(cfg),
142	142		TrustSuccess: trustSuccess,
143	143		})
144	144		}
145	145
146		-	func siteUser(cfg Config) func(r http.Request) string {
147		-	return func(r *http.Request) string {
	146	+	func siteProject(cfg Config) func(r http.Request) (string, string) {
	147	+	return func(r *http.Request) (string, string) {
148	148		subdomain := router.GetSubdomainFromRequest(r, cfg.Domain, cfg.TxtPrefix)
149	149		if subdomain == "" {
150		-	return ""
	150	+	return "", ""
151	151		}
152	152		props, _ := router.GetProjectFromSubdomain(subdomain)
153		-	return props.Username
	153	+	return props.Username, props.ProjectName
154	154		}
155	155		}
156	156

...	...		@@ -523,6 +523,9 @@ func (web WebRouter) ServeAsset(fname string, opts storage.ImgProcessOpts, has
523	523		return
524	524		}
525	525		router.SetUser(r, site.user.Name)
	526	+	// The name from the host, so cache hits for it count under the same
	527	+	// project.
	528	+	router.SetResource(r, props.ProjectName)
526	529		user, project, bucket := site.user, site.project, site.bucket
527	530
528	531		logger = logger.With(

...	...		@@ -64,62 +67,104 @@ func NewUserMetrics(app string, reg prometheus.Registerer, opts UserMetricsOpts)
64	67		ConstLabels: prometheus.Labels{"app": app},
65	68		Name: "response_bytes_total",
66	69		Help: "Number of response body bytes sent.",
67		-	}, []string{"user"}),
	70	+	}, []string{"user", "resource"}),
68	71		opts: opts,
69	72		known: known,
70	73		}
71	74		}
72	75
73		-	type ctxUserKey struct{}
	76	+	type owner struct {
	77	+	user string
	78	+	resource string
	79	+	}
	80	+
	81	+	type ctxOwnerKey struct{}
74	82
75	83		// SetUser records name as the user a request belongs to. Handlers call it
76	84		// once they've found the user in the database.
77	85		func SetUser(r *http.Request, name string) {
78		-	if user, ok := r.Context().Value(ctxUserKey{}).(*string); ok {
79		-	*user = name
	86	+	if o, ok := r.Context().Value(ctxOwnerKey{}).(*owner); ok {
	87	+	o.user = name
	88	+	}
	89	+	}
	90	+
	91	+	// SetResource records name as what a request is for within the user's
	92	+	// account, such as a pgs project. Handlers call it alongside SetUser once
	93	+	// they've found it.
	94	+	func SetResource(r *http.Request, name string) {
	95	+	if o, ok := r.Context().Value(ctxOwnerKey{}).(*owner); ok {
	96	+	o.resource = name
80	97		}
81	98		}
82	99
83	100		func (m *UserMetrics) Handler(next http.Handler) http.Handler {
84	101		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
85	102		start := time.Now()
86		-	var user string
	103	+	var o owner
87	104		mw := &metricsWriter{ResponseWriter: w}
88		-	next.ServeHTTP(mw, r.WithContext(context.WithValue(r.Context(), ctxUserKey{}, &user)))
	105	+	next.ServeHTTP(mw, r.WithContext(context.WithValue(r.Context(), ctxOwnerKey{}, &o)))
89	106
90	107		status := mw.Status()
91		-	m.Observe(m.resolveUser(r, user, status), status, r.Method, time.Since(start), mw.bytes)
	108	+	user, resource := m.resolve(r, o, status)
	109	+	m.Observe(user, resource, status, r.Method, time.Since(start), mw.bytes)
92	110		})
93	111		}
94	112
95	113		// Observe records one request for servers that can't use Handler, such as
96		-	// tuns's gin logger. user must already be confirmed to exist. A negative
97		-	// size, which gin reports when nothing was written, counts as zero bytes.
98		-	func (m *UserMetrics) Observe(user string, status int, method string, elapsed time.Duration, bytes int64) {
99		-	labels := []string{user, strconv.Itoa(status), metricMethod(method)}
100		-	m.requests.WithLabelValues(labels...).Inc()
101		-	m.duration.WithLabelValues(labels...).Observe(elapsed.Seconds())
102		-	m.bytes.WithLabelValues(user).Add(float64(max(bytes, 0)))
	114	+	// tuns's gin logger. user and resource must already be confirmed to
	115	+	// exist, and resource may be empty. A negative size, which gin reports when
	116	+	// nothing was written, counts as zero bytes.
	117	+	func (m *UserMetrics) Observe(user, resource string, status int, method string, elapsed time.Duration, bytes int64) {
	118	+	if user == "" {
	119	+	resource = ""
	120	+	}
	121	+	code, verb := strconv.Itoa(status), metricMethod(method)
	122	+	m.requests.WithLabelValues(user, resource, code, verb).Inc()
	123	+	m.duration.WithLabelValues(user, code, verb).Observe(elapsed.Seconds())
	124	+	m.bytes.WithLabelValues(user, resource).Add(float64(max(bytes, 0)))
103	125		}
104	126
105		-	func (m UserMetrics) resolveUser(r http.Request, user string, status int) string {
106		-	if user != "" {
107		-	m.known.Add(user, struct{}{})
108		-	return user
	127	+	func (m UserMetrics) resolve(r http.Request, o owner, status int) (string, string) {
	128	+	var hostUser, hostResource string
	129	+	if m.opts.Owner != nil {
	130	+	hostUser, hostResource = m.opts.Owner(r)
109	131		}
110		-	if m.opts.SiteUser == nil {
111		-	return ""
112		-	}
113		-	user = m.opts.SiteUser(r)
	132	+	trusted := m.opts.TrustSuccess && (status >= 200 && status < 300 \|\| status == http.StatusNotModified)
	133	+
	134	+	user := m.confirm(o.user, hostUser, trusted)
114	135		if user == "" {
	136	+	return "", ""
	137	+	}
	138	+	// Resources are keyed under their owner, and user names can't hold a
	139	+	// slash.
	140	+	prefix := user + "/"
	141	+	set, fromHost := "", ""
	142	+	if o.resource != "" {
	143	+	set = prefix + o.resource
	144	+	}
	145	+	if hostUser == user && hostResource != "" {
	146	+	fromHost = prefix + hostResource
	147	+	}
	148	+	resource := strings.TrimPrefix(m.confirm(set, fromHost, trusted), prefix)
	149	+	return user, resource
	150	+	}
	151	+
	152	+	// confirm returns the name a handler set, or else the name from the host
	153	+	// once it's known or trusted.
	154	+	func (m *UserMetrics) confirm(set, fromHost string, trusted bool) string {
	155	+	if set != "" {
	156	+	m.known.Add(set, struct{}{})
	157	+	return set
	158	+	}
	159	+	if fromHost == "" {
115	160		return ""
116	161		}
117		-	if m.known.Contains(user) {
118		-	return user
	162	+	if m.known.Contains(fromHost) {
	163	+	return fromHost
119	164		}
120		-	if m.opts.TrustSuccess && (status >= 200 && status < 300 \|\| status == http.StatusNotModified) {
121		-	m.known.Add(user, struct{}{})
122		-	return user
	165	+	if trusted {
	166	+	m.known.Add(fromHost, struct{}{})
	167	+	return fromHost
123	168		}
124	169		return ""
125	170		}

...	...		@@ -116,6 +124,64 @@ func TestUserMetricsTrustSuccess(t *testing.T) {
116	124		if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "", "status": "404"}); got != 1 {
117	125		t.Errorf("unlabeled 404s = %v, want 1", got)
118	126		}
	127	+	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "bob", "resource": "docs"}); got != 1 {
	128	+	t.Errorf("bob/docs requests = %v, want 1", got)
	129	+	}
	130	+	}
	131	+
	132	+	func TestUserMetricsSetResource(t *testing.T) {
	133	+	reg := prometheus.NewRegistry()
	134	+	m := NewUserMetrics("test", reg, UserMetricsOpts{})
	135	+	h := m.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	136	+	SetUser(r, "alice")
	137	+	if r.URL.Path == "/blog" {
	138	+	SetResource(r, "blog")
	139	+	}
	140	+	_, _ = w.Write([]byte("hello"))
	141	+	}))
	142	+
	143	+	serve(h, http.MethodGet, "/blog")
	144	+	serve(h, http.MethodGet, "/")
	145	+
	146	+	labels := map[string]string{"app": "test", "user": "alice", "resource": "blog"}
	147	+	if got := metricValue(t, reg, "pico_http_requests_total", labels); got != 1 {
	148	+	t.Errorf("alice/blog requests = %v, want 1", got)
	149	+	}
	150	+	if got := metricValue(t, reg, "pico_http_response_bytes_total", labels); got != 5 {
	151	+	t.Errorf("alice/blog bytes = %v, want 5", got)
	152	+	}
	153	+	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "alice"}); got != 2 {
	154	+	t.Errorf("alice requests = %v, want 2", got)
	155	+	}
	156	+	}
	157	+
	158	+	func TestUserMetricsResourceFromHost(t *testing.T) {
	159	+	reg := prometheus.NewRegistry()
	160	+	m := NewUserMetrics("test", reg, UserMetricsOpts{Owner: hostOwner})
	161	+	// The handler confirms alice's blog on its first request; later ones
	162	+	// stand in for cache hits that never reach it.
	163	+	confirmed := false
	164	+	h := m.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	165	+	if !confirmed {
	166	+	SetUser(r, "alice")
	167	+	SetResource(r, "blog")
	168	+	confirmed = true
	169	+	}
	170	+	}))
	171	+
	172	+	serve(h, http.MethodGet, "http://alice-blog.example.com/")
	173	+	serve(h, http.MethodGet, "http://alice-blog.example.com/")
	174	+	serve(h, http.MethodGet, "http://alice-ghost.example.com/")
	175	+
	176	+	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "alice", "resource": "blog"}); got != 2 {
	177	+	t.Errorf("alice/blog requests = %v, want 2", got)
	178	+	}
	179	+	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "alice", "resource": ""}); got != 1 {
	180	+	t.Errorf("alice requests without a resource = %v, want 1", got)
	181	+	}
	182	+	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "alice"}); got != 3 {
	183	+	t.Errorf("alice requests = %v, want 3", got)
	184	+	}
119	185		}
120	186
121	187		func TestUserMetricsMethod(t *testing.T) {

...	...		@@ -145,6 +211,9 @@ func TestUserMetricsObserve(t *testing.T) {
145	211		if got := metricValue(t, reg, "pico_http_response_bytes_total", map[string]string{"user": "carol"}); got != 7 {
146	212		t.Errorf("bytes = %v, want 7", got)
147	213		}
	214	+	if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "carol", "resource": "app"}); got != 1 {
	215	+	t.Errorf("carol/app requests = %v, want 1", got)
	216	+	}
148	217		}
149	218
150	219		func TestUserMetricsHijack(t *testing.T) {

...	...		@@ -26,7 +26,7 @@ func TestSiteMetricsCountCacheHits(t *testing.T) {
26	26		cfg.Domain = "pgs.test"
27	27		httpCache, _ := NewPgsHttpCache(cfg, newWebRouter(cfg))
28	28		reg := prometheus.NewRegistry()
29		-	h := router.NewUserMetrics("pgs", reg, router.UserMetricsOpts{SiteUser: siteUser(cfg)}).Handler(httpCache)
	29	+	h := router.NewUserMetrics("pgs", reg, router.UserMetricsOpts{Owner: siteProject(cfg)}).Handler(httpCache)
30	30
31	31		var statuses []string
32	32		for range 2 {

...	...		@@ -42,20 +42,34 @@ func TestSiteMetricsCountCacheHits(t *testing.T) {
42	42		if err != nil {
43	43		t.Fatal(err)
44	44		}
45		-	var count float64
46		-	for _, family := range families {
47		-	if family.GetName() != "pico_http_requests_total" {
48		-	continue
49		-	}
50		-	for _, m := range family.GetMetric() {
51		-	for _, l := range m.GetLabel() {
52		-	if l.GetName() == "user" && l.GetValue() == user.Name {
53		-	count += m.GetCounter().GetValue()
	45	+	count := func(name string, labels map[string]string) float64 {
	46	+	var total float64
	47	+	for _, family := range families {
	48	+	if family.GetName() != name {
	49	+	continue
	50	+	}
	51	+	metrics:
	52	+	for _, m := range family.GetMetric() {
	53	+	matched := 0
	54	+	for _, l := range m.GetLabel() {
	55	+	if want, ok := labels[l.GetName()]; ok {
	56	+	if want != l.GetValue() {
	57	+	continue metrics
	58	+	}
	59	+	matched++
	60	+	}
	61	+	}
	62	+	if matched == len(labels) {
	63	+	total += m.GetCounter().GetValue()
54	64		}
55	65		}
56	66		}
	67	+	return total
	68	+	}
	69	+	if got := count("pico_http_requests_total", map[string]string{"user": user.Name}); got != 2 {
	70	+	t.Errorf("requests for %s = %v, want 2", user.Name, got)
57	71		}
58		-	if count != 2 {
59		-	t.Errorf("requests for %s = %v, want 2", user.Name, count)
	72	+	if got := count("pico_http_requests_total", map[string]string{"user": user.Name, "resource": "test"}); got != 2 {
	73	+	t.Errorf("requests for %s/test = %v, want 2", user.Name, got)
60	74		}
61	75		}

...	...		@@ -6,6 +6,7 @@ import (
6	6		"net"
7	7		"net/http"
8	8		"strconv"
	9	+	"strings"
9	10		"time"
10	11
11	12		lru "github.com/hashicorp/golang-lru/v2"

...	...		@@ -16,26 +17,28 @@ import (
16	17		// UserMetricsOpts tells UserMetrics who a request belongs to when no
17	18		// handler called SetUser, such as a response served from a cache.
18	19		type UserMetricsOpts struct {
19		-	// SiteUser names the owner of the site a request is for, without
20		-	// checking that the user exists.
21		-	SiteUser func(r *http.Request) string
22		-	// TrustSuccess counts a SiteUser as real once its site returns a
23		-	// success, for a proxy that can't look users up. Only safe when every
24		-	// route on a site's host fails for a site that doesn't exist.
	20	+	// Owner names the user and resource a request is for, read from the
	21	+	// host without checking that either exists.
	22	+	Owner func(r *http.Request) (user, resource string)
	23	+	// TrustSuccess counts an Owner as real once it returns a success, for
	24	+	// a proxy that can't look users up. Only safe when every route on a
	25	+	// site's host fails for a site that doesn't exist.
25	26		TrustSuccess bool
26	27		}
27	28
28	29		// UserMetrics counts HTTP requests, latency and response bytes for each
29		-	// site owner. Every app shares the metric names and sets the app label, so
	30	+	// site owner. Requests and bytes also carry the resource they were for,
	31	+	// such as a pgs project, which the latency histogram leaves out to keep its
	32	+	// series down. Every app shares the metric names and sets the app label, so
30	33		// queries can aggregate across services.
31	34		type UserMetrics struct {
32	35		requests *prometheus.CounterVec
33	36		duration *prometheus.HistogramVec
34	37		bytes *prometheus.CounterVec
35	38		opts UserMetricsOpts
36		-	// known holds the users a handler or a successful response confirmed.
37		-	// A name from the Host header alone never becomes a label, so made-up
38		-	// subdomains can't add series.
	39	+	// known holds the users and resources a handler or a successful
	40	+	// response confirmed. A name from the Host header alone never becomes a
	41	+	// label, so made-up subdomains can't add series.
39	42		known *lru.Cache[string, struct{}]
40	43		}
41	44

...	...		@@ -49,7 +52,7 @@ func NewUserMetrics(app string, reg prometheus.Registerer, opts UserMetricsOpts)
49	52		ConstLabels: prometheus.Labels{"app": app},
50	53		Name: "requests_total",
51	54		Help: "Number of HTTP requests handled.",
52		-	}, []string{"user", "status", "method"}),
	55	+	}, []string{"user", "resource", "status", "method"}),
53	56		duration: auto.NewHistogramVec(prometheus.HistogramOpts{
54	57		Namespace: "pico",
55	58		Subsystem: "http",

...	...		@@ -9,13 +9,15 @@ import (
9	9		"github.com/prometheus/client_golang/prometheus"
10	10		)
11	11
12		-	// metricValue reads a counter, or a histogram's count, from reg.
	12	+	// metricValue sums the counters, or histogram counts, in reg that match
	13	+	// labels.
13	14		func metricValue(t testing.T, reg prometheus.Registry, name string, labels map[string]string) float64 {
14	15		t.Helper()
15	16		families, err := reg.Gather()
16	17		if err != nil {
17	18		t.Fatalf("gather: %v", err)
18	19		}
	20	+	var total float64
19	21		for _, family := range families {
20	22		if family.GetName() != name {
21	23		continue

...	...		@@ -28,12 +30,20 @@ func metricValue(t testing.T, reg prometheus.Registry, name string, labels map
28	30		}
29	31		}
30	32		if h := m.GetHistogram(); h != nil {
31		-	return float64(h.GetSampleCount())
	33	+	total += float64(h.GetSampleCount())
	34	+	} else {
	35	+	total += m.GetCounter().GetValue()
32	36		}
33		-	return m.GetCounter().GetValue()
34	37		}
35	38		}
36		-	return 0
	39	+	return total
	40	+	}
	41	+
	42	+	// hostOwner reads "user-project.example.com" the way pgs reads its
	43	+	// subdomains.
	44	+	func hostOwner(r *http.Request) (string, string) {
	45	+	user, resource, _ := strings.Cut(strings.Split(r.Host, ".")[0], "-")
	46	+	return user, resource
37	47		}
38	48
39	49		func serve(h http.Handler, method, target string) *httptest.ResponseRecorder {

...	...		@@ -67,9 +77,7 @@ func TestUserMetricsSetUser(t *testing.T) {
67	77
68	78		func TestUserMetricsSiteUser(t *testing.T) {
69	79		reg := prometheus.NewRegistry()
70		-	m := NewUserMetrics("test", reg, UserMetricsOpts{
71		-	SiteUser: func(r *http.Request) string { return strings.Split(r.Host, ".")[0] },
72		-	})
	80	+	m := NewUserMetrics("test", reg, UserMetricsOpts{Owner: hostOwner})
73	81		// The handler confirms alice on her first request; later ones stand in
74	82		// for cache hits that never reach it.
75	83		confirmed := false

...	...		@@ -98,7 +106,7 @@ func TestUserMetricsSiteUser(t *testing.T) {
98	106		func TestUserMetricsTrustSuccess(t *testing.T) {
99	107		reg := prometheus.NewRegistry()
100	108		m := NewUserMetrics("test", reg, UserMetricsOpts{
101		-	SiteUser: func(r *http.Request) string { return strings.Split(r.Host, ".")[0] },
	109	+	Owner: hostOwner,
102	110		TrustSuccess: true,
103	111		})
104	112		h := m.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

...	...		@@ -107,7 +115,7 @@ func TestUserMetricsTrustSuccess(t *testing.T) {
107	115		}
108	116		}))
109	117
110		-	serve(h, http.MethodGet, "http://bob.example.com/")
	118	+	serve(h, http.MethodGet, "http://bob-docs.example.com/")
111	119		serve(h, http.MethodGet, "http://ghost.example.com/")
112	120
113	121		if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "bob", "status": "200"}); got != 1 {

...	...		@@ -133,8 +199,8 @@ func TestUserMetricsObserve(t *testing.T) {
133	199		reg := prometheus.NewRegistry()
134	200		m := NewUserMetrics("test", reg, UserMetricsOpts{})
135	201
136		-	m.Observe("carol", http.StatusOK, http.MethodGet, 0, 7)
137		-	m.Observe("carol", http.StatusNotModified, "MADEUP", 0, -1)
	202	+	m.Observe("carol", "app", http.StatusOK, http.MethodGet, 0, 7)
	203	+	m.Observe("carol", "", http.StatusNotModified, "MADEUP", 0, -1)
138	204
139	205		if got := metricValue(t, reg, "pico_http_requests_total", map[string]string{"user": "carol", "status": "200", "method": "GET"}); got != 1 {
140	206		t.Errorf("200 requests = %v, want 1", got)

Commit bdd5983

Commit `bdd5983`