1package obs
2
3import (
4 "database/sql"
5 "sync"
6
7 "github.com/prometheus/client_golang/prometheus"
8)
9
10const (
11 axisUsername = "username"
12 axisAddress = "address"
13 routeLDAPBind = "ldap_bind"
14 resultLocked = "locked"
15 resultLimited = "limited"
16)
17
18type Table interface {
19 Len() int
20 Evictions() uint64
21}
22
23type lazy struct {
24 mu sync.Mutex
25 refusals *prometheus.CounterVec
26 tables *backoffTables
27}
28
29func (r *Registry) refused(pairs ...[]string) *Counter {
30 if r == nil {
31 return nil
32 }
33 r.lazy.mu.Lock()
34 defer r.lazy.mu.Unlock()
35 if r.lazy.refusals == nil {
36 r.lazy.refusals = prometheus.NewCounterVec(prometheus.CounterOpts{
37 Name: "tug_backoff_refusals_total",
38 Help: "Requests refused by a backoff or a rate limit, by its axis and the route it guards.",
39 }, []string{"axis", "route"})
40 r.reg.MustRegister(r.lazy.refusals)
41 }
42 return children(r.lazy.refusals, pairs)
43}
44
45func pairs(axis string, routes []string) [][]string {
46 out := make([][]string, len(routes))
47 for i, route := range routes {
48 out[i] = []string{axis, route}
49 }
50 return out
51}
52
53func (r *Registry) PasswordChecks(paths, results []string) func(path, result string) {
54 c := r.Counter("tug_password_checks_total", "Password checks on the web, by path and result.",
55 Label{Name: "path", Values: paths}, Label{Name: "result", Values: results})
56 locked := r.refused(pairs(axisUsername, paths)...)
57 return func(path, result string) {
58 c.Inc(path, result)
59 if result == resultLocked {
60 locked.Inc(axisUsername, path)
61 }
62 }
63}
64
65func (r *Registry) AddressRefusals(classes []string) func(class string) {
66 c := r.refused(pairs(axisAddress, classes)...)
67 return func(class string) { c.Inc(axisAddress, class) }
68}
69
70func (r *Registry) ConcurrencyWaits(gates []string) func(gate string) {
71 c := r.Counter("tug_concurrency_waits_total", "Requests that found their concurrency gate full and waited for a slot, by gate.", Label{Name: "gate", Values: gates})
72 return func(gate string) { c.Inc(gate) }
73}
74
75func (r *Registry) ConcurrencyRefusals(gates []string) func(gate string) {
76 c := r.Counter("tug_concurrency_refusals_total", "Requests refused because their concurrency gate stayed full, by gate.", Label{Name: "gate", Values: gates})
77 return func(gate string) { c.Inc(gate) }
78}
79
80func (r *Registry) HTTPResponses(hosts, codes []string) func(host, code string) {
81 c := r.Counter("tug_http_responses_total", "Responses on the routed listeners, by host class and status class.",
82 Label{Name: "host_class", Values: hosts}, Label{Name: "code_class", Values: codes})
83 return func(host, code string) { c.Inc(host, code) }
84}
85
86func (r *Registry) UnknownClientAddresses() func() {
87 c := r.Counter("tug_http_unknown_client_address_total", "Routed requests whose X-Forwarded-For named no usable client address.")
88 return func() { c.Inc() }
89}
90
91type backoffTables struct {
92 mu sync.Mutex
93 byAxis map[string][]Table
94 entries *prometheus.Desc
95 evictions *prometheus.Desc
96}
97
98func (r *Registry) table(axis string, t Table) {
99 if r == nil || t == nil {
100 return
101 }
102 r.lazy.mu.Lock()
103 if r.lazy.tables == nil {
104 r.lazy.tables = &backoffTables{
105 byAxis: map[string][]Table{},
106 entries: prometheus.NewDesc("tug_backoff_entries",
107 "Keys a backoff or rate-limit table tracks, by axis.", []string{"axis"}, nil),
108 evictions: prometheus.NewDesc("tug_backoff_evictions_total",
109 "Keys a full backoff or rate-limit table evicted to admit another, by axis.", []string{"axis"}, nil),
110 }
111 r.reg.MustRegister(r.lazy.tables)
112 }
113 tables := r.lazy.tables
114 r.lazy.mu.Unlock()
115 tables.mu.Lock()
116 defer tables.mu.Unlock()
117 tables.byAxis[axis] = append(tables.byAxis[axis], t)
118}
119
120func (r *Registry) UsernameTable(t Table) { r.table(axisUsername, t) }
121
122func (r *Registry) AddressTable(t Table) { r.table(axisAddress, t) }
123
124func (b *backoffTables) Describe(ch chan<- *prometheus.Desc) {
125 ch <- b.entries
126 ch <- b.evictions
127}
128
129func (b *backoffTables) Collect(ch chan<- prometheus.Metric) {
130 b.mu.Lock()
131 defer b.mu.Unlock()
132 for _, axis := range []string{axisUsername, axisAddress} {
133 var entries int
134 var evictions uint64
135 for _, t := range b.byAxis[axis] {
136 entries += t.Len()
137 evictions += t.Evictions()
138 }
139 ch <- prometheus.MustNewConstMetric(b.entries, prometheus.GaugeValue, float64(entries), axis)
140 ch <- prometheus.MustNewConstMetric(b.evictions, prometheus.CounterValue, float64(evictions), axis)
141 }
142}
143
144type readCollector struct {
145 descs []*prometheus.Desc
146 collect func(ch chan<- prometheus.Metric)
147}
148
149func (c readCollector) Describe(ch chan<- *prometheus.Desc) {
150 for _, d := range c.descs {
151 ch <- d
152 }
153}
154
155func (c readCollector) Collect(ch chan<- prometheus.Metric) { c.collect(ch) }
156
157func (r *Registry) PasswordKDF(read func() (inUse, slots int, waits, busy uint64)) {
158 if r == nil || read == nil {
159 return
160 }
161 inUse := prometheus.NewDesc("tug_password_kdf_in_use", "Password key derivations running now.", nil, nil)
162 slots := prometheus.NewDesc("tug_password_kdf_slots", "Password key derivations allowed to run at once.", nil, nil)
163 waits := prometheus.NewDesc("tug_password_kdf_waits_total", "Password checks and hashes that found every slot taken and waited.", nil, nil)
164 busy := prometheus.NewDesc("tug_password_kdf_busy_total", "Password checks and hashes refused as busy because no slot freed in time.", nil, nil)
165 r.reg.MustRegister(readCollector{descs: []*prometheus.Desc{inUse, slots, waits, busy}, collect: func(ch chan<- prometheus.Metric) {
166 u, s, w, b := read()
167 ch <- prometheus.MustNewConstMetric(inUse, prometheus.GaugeValue, float64(u))
168 ch <- prometheus.MustNewConstMetric(slots, prometheus.GaugeValue, float64(s))
169 ch <- prometheus.MustNewConstMetric(waits, prometheus.CounterValue, float64(w))
170 ch <- prometheus.MustNewConstMetric(busy, prometheus.CounterValue, float64(b))
171 }})
172}
173
174func (r *Registry) SQLitePools(read func() (tug, dex sql.DBStats)) {
175 if r == nil || read == nil {
176 return
177 }
178 conns := prometheus.NewDesc("tug_sqlite_connections", "Connections in a SQLite pool, by pool and state.", []string{"pool", "state"}, nil)
179 r.reg.MustRegister(readCollector{descs: []*prometheus.Desc{conns}, collect: func(ch chan<- prometheus.Metric) {
180 tug, dex := read()
181 for _, p := range []struct {
182 pool string
183 stats sql.DBStats
184 }{{"tug", tug}, {"dex", dex}} {
185 ch <- prometheus.MustNewConstMetric(conns, prometheus.GaugeValue, float64(p.stats.OpenConnections), p.pool, "open")
186 ch <- prometheus.MustNewConstMetric(conns, prometheus.GaugeValue, float64(p.stats.InUse), p.pool, "in_use")
187 ch <- prometheus.MustNewConstMetric(conns, prometheus.GaugeValue, float64(p.stats.Idle), p.pool, "idle")
188 }
189 }})
190}
191
192func (r *Registry) LastSeenDropped(read func() uint64) {
193 if r == nil || read == nil {
194 return
195 }
196 r.reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
197 Name: "tug_session_last_seen_dropped_total",
198 Help: "Last-seen updates dropped because the pending batch was full.",
199 }, func() float64 { return float64(read()) }))
200}