package obs import ( "database/sql" "sync" "github.com/prometheus/client_golang/prometheus" ) const ( axisUsername = "username" axisAddress = "address" routeLDAPBind = "ldap_bind" resultLocked = "locked" resultLimited = "limited" ) type Table interface { Len() int Evictions() uint64 } type lazy struct { mu sync.Mutex refusals *prometheus.CounterVec tables *backoffTables } func (r *Registry) refused(pairs ...[]string) *Counter { if r == nil { return nil } r.lazy.mu.Lock() defer r.lazy.mu.Unlock() if r.lazy.refusals == nil { r.lazy.refusals = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "tug_backoff_refusals_total", Help: "Requests refused by a backoff or a rate limit, by its axis and the route it guards.", }, []string{"axis", "route"}) r.reg.MustRegister(r.lazy.refusals) } return children(r.lazy.refusals, pairs) } func pairs(axis string, routes []string) [][]string { out := make([][]string, len(routes)) for i, route := range routes { out[i] = []string{axis, route} } return out } func (r *Registry) PasswordChecks(paths, results []string) func(path, result string) { c := r.Counter("tug_password_checks_total", "Password checks on the web, by path and result.", Label{Name: "path", Values: paths}, Label{Name: "result", Values: results}) locked := r.refused(pairs(axisUsername, paths)...) return func(path, result string) { c.Inc(path, result) if result == resultLocked { locked.Inc(axisUsername, path) } } } func (r *Registry) AddressRefusals(classes []string) func(class string) { c := r.refused(pairs(axisAddress, classes)...) return func(class string) { c.Inc(axisAddress, class) } } func (r *Registry) ConcurrencyWaits(gates []string) func(gate string) { c := r.Counter("tug_concurrency_waits_total", "Requests that found their concurrency gate full and waited for a slot, by gate.", Label{Name: "gate", Values: gates}) return func(gate string) { c.Inc(gate) } } func (r *Registry) ConcurrencyRefusals(gates []string) func(gate string) { c := r.Counter("tug_concurrency_refusals_total", "Requests refused because their concurrency gate stayed full, by gate.", Label{Name: "gate", Values: gates}) return func(gate string) { c.Inc(gate) } } func (r *Registry) HTTPResponses(hosts, codes []string) func(host, code string) { c := r.Counter("tug_http_responses_total", "Responses on the routed listeners, by host class and status class.", Label{Name: "host_class", Values: hosts}, Label{Name: "code_class", Values: codes}) return func(host, code string) { c.Inc(host, code) } } func (r *Registry) UnknownClientAddresses() func() { c := r.Counter("tug_http_unknown_client_address_total", "Routed requests whose X-Forwarded-For named no usable client address.") return func() { c.Inc() } } type backoffTables struct { mu sync.Mutex byAxis map[string][]Table entries *prometheus.Desc evictions *prometheus.Desc } func (r *Registry) table(axis string, t Table) { if r == nil || t == nil { return } r.lazy.mu.Lock() if r.lazy.tables == nil { r.lazy.tables = &backoffTables{ byAxis: map[string][]Table{}, entries: prometheus.NewDesc("tug_backoff_entries", "Keys a backoff or rate-limit table tracks, by axis.", []string{"axis"}, nil), evictions: prometheus.NewDesc("tug_backoff_evictions_total", "Keys a full backoff or rate-limit table evicted to admit another, by axis.", []string{"axis"}, nil), } r.reg.MustRegister(r.lazy.tables) } tables := r.lazy.tables r.lazy.mu.Unlock() tables.mu.Lock() defer tables.mu.Unlock() tables.byAxis[axis] = append(tables.byAxis[axis], t) } func (r *Registry) UsernameTable(t Table) { r.table(axisUsername, t) } func (r *Registry) AddressTable(t Table) { r.table(axisAddress, t) } func (b *backoffTables) Describe(ch chan<- *prometheus.Desc) { ch <- b.entries ch <- b.evictions } func (b *backoffTables) Collect(ch chan<- prometheus.Metric) { b.mu.Lock() defer b.mu.Unlock() for _, axis := range []string{axisUsername, axisAddress} { var entries int var evictions uint64 for _, t := range b.byAxis[axis] { entries += t.Len() evictions += t.Evictions() } ch <- prometheus.MustNewConstMetric(b.entries, prometheus.GaugeValue, float64(entries), axis) ch <- prometheus.MustNewConstMetric(b.evictions, prometheus.CounterValue, float64(evictions), axis) } } type readCollector struct { descs []*prometheus.Desc collect func(ch chan<- prometheus.Metric) } func (c readCollector) Describe(ch chan<- *prometheus.Desc) { for _, d := range c.descs { ch <- d } } func (c readCollector) Collect(ch chan<- prometheus.Metric) { c.collect(ch) } func (r *Registry) PasswordKDF(read func() (inUse, slots int, waits, busy uint64)) { if r == nil || read == nil { return } inUse := prometheus.NewDesc("tug_password_kdf_in_use", "Password key derivations running now.", nil, nil) slots := prometheus.NewDesc("tug_password_kdf_slots", "Password key derivations allowed to run at once.", nil, nil) waits := prometheus.NewDesc("tug_password_kdf_waits_total", "Password checks and hashes that found every slot taken and waited.", nil, nil) busy := prometheus.NewDesc("tug_password_kdf_busy_total", "Password checks and hashes refused as busy because no slot freed in time.", nil, nil) r.reg.MustRegister(readCollector{descs: []*prometheus.Desc{inUse, slots, waits, busy}, collect: func(ch chan<- prometheus.Metric) { u, s, w, b := read() ch <- prometheus.MustNewConstMetric(inUse, prometheus.GaugeValue, float64(u)) ch <- prometheus.MustNewConstMetric(slots, prometheus.GaugeValue, float64(s)) ch <- prometheus.MustNewConstMetric(waits, prometheus.CounterValue, float64(w)) ch <- prometheus.MustNewConstMetric(busy, prometheus.CounterValue, float64(b)) }}) } func (r *Registry) SQLitePools(read func() (tug, dex sql.DBStats)) { if r == nil || read == nil { return } conns := prometheus.NewDesc("tug_sqlite_connections", "Connections in a SQLite pool, by pool and state.", []string{"pool", "state"}, nil) r.reg.MustRegister(readCollector{descs: []*prometheus.Desc{conns}, collect: func(ch chan<- prometheus.Metric) { tug, dex := read() for _, p := range []struct { pool string stats sql.DBStats }{{"tug", tug}, {"dex", dex}} { ch <- prometheus.MustNewConstMetric(conns, prometheus.GaugeValue, float64(p.stats.OpenConnections), p.pool, "open") ch <- prometheus.MustNewConstMetric(conns, prometheus.GaugeValue, float64(p.stats.InUse), p.pool, "in_use") ch <- prometheus.MustNewConstMetric(conns, prometheus.GaugeValue, float64(p.stats.Idle), p.pool, "idle") } }}) } func (r *Registry) LastSeenDropped(read func() uint64) { if r == nil || read == nil { return } r.reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "tug_session_last_seen_dropped_total", Help: "Last-seen updates dropped because the pending batch was full.", }, func() float64 { return float64(read()) })) }