Coverage for app/backend/src/couchers/metrics.py: 95%
239 statements
« prev ^ index » next coverage.py v7.14.1, created at 2026-06-14 11:44 +0000
1import threading
2import time
3from collections.abc import Callable
4from concurrent.futures import ThreadPoolExecutor
5from datetime import datetime, timedelta
6from typing import Any, cast
8from opentelemetry import trace
9from prometheus_client import (
10 CONTENT_TYPE_LATEST,
11 CollectorRegistry,
12 Counter,
13 Gauge,
14 Histogram,
15 exposition,
16 generate_latest,
17 multiprocess,
18)
19from prometheus_client.registry import CollectorRegistry
20from sqlalchemy import Engine, and_, case, select
21from sqlalchemy.pool import QueuePool
22from sqlalchemy.sql import distinct, func
23from sqlalchemy.sql.selectable import Select
25from couchers import experimentation
26from couchers.config import config
27from couchers.db import session_scope
28from couchers.helpers.completed_profile import has_completed_profile_expression
29from couchers.materialized_views import ClusterSubscriptionCount
30from couchers.models import (
31 BackgroundJob,
32 ClientPlatform,
33 Cluster,
34 EventOccurrenceAttendee,
35 HostingStatus,
36 HostRequest,
37 Message,
38 Node,
39 NodeType,
40 NonvisibleUserAccessType,
41 NonvisibleUserState,
42 Reference,
43 User,
44 UserActivity,
45)
46from couchers.models.moderation import (
47 ModerationAction,
48 ModerationObjectType,
49 ModerationQueueItem,
50 ModerationState,
51 ModerationTrigger,
52 ModerationVisibility,
53)
54from couchers.perf import PerfResult
56tracer = trace.get_tracer(__name__)
58registry: CollectorRegistry = CollectorRegistry()
59multiprocess.MultiProcessCollector(registry) # type: ignore[no-untyped-call]
61_INF: float = float("inf")
63# Dense from 1ms to ~300ms where most calls land, sparse out to 10min for long background jobs.
64MACHINE_DURATION_SECONDS: tuple[float, ...] = (
65 0.001,
66 0.0025,
67 0.005,
68 0.0075,
69 0.01,
70 0.015,
71 0.02,
72 0.03,
73 0.04,
74 0.05,
75 0.06,
76 0.075,
77 0.1,
78 0.125,
79 0.15,
80 0.2,
81 0.25,
82 0.3,
83 0.4,
84 0.5,
85 0.75,
86 1.0,
87 1.5,
88 2.0,
89 3.0,
90 5.0,
91 7.5,
92 10.0,
93 15.0,
94 30.0,
95 60,
96 120,
97 300,
98 600,
99 _INF,
100)
102start_time_gauge: Gauge = Gauge(
103 "couchers_start_time_seconds",
104 "Unix timestamp of when the process started",
105 multiprocess_mode="max",
106)
107start_time_gauge.set(time.time())
109commit_timestamp_gauge: Gauge = Gauge(
110 "couchers_commit_timestamp_seconds",
111 "Unix timestamp of the deployed commit, 0 if not a CI build",
112 multiprocess_mode="max",
113)
114# left at its default of 0 when COMMIT_TIMESTAMP is empty (i.e. not a CI build)
115if config.COMMIT_TIMESTAMP: 115 ↛ 118line 115 didn't jump to line 118 because the condition on line 115 was always true
116 commit_timestamp_gauge.set(datetime.fromisoformat(config.COMMIT_TIMESTAMP).timestamp())
118jobs_duration_histogram: Histogram = Histogram(
119 "couchers_background_jobs_seconds",
120 "Durations of background jobs",
121 labelnames=["job", "status", "attempt", "exception"],
122 buckets=MACHINE_DURATION_SECONDS,
123)
126def observe_in_jobs_duration_histogram(
127 job_type: str, job_state: str, try_count: int, exception_name: str, duration_s: float
128) -> None:
129 jobs_duration_histogram.labels(job_type, job_state, str(try_count), exception_name).observe(duration_s)
132jobs_queued_histogram: Histogram = Histogram(
133 "couchers_background_jobs_queued_seconds",
134 "Time background job spent queued before being picked up",
135 labelnames=["priority"],
136 buckets=(
137 0.01,
138 0.05,
139 0.1,
140 0.5,
141 1.0,
142 2.5,
143 5.0,
144 10,
145 20,
146 30,
147 40,
148 50,
149 60,
150 90,
151 120,
152 180,
153 240,
154 300,
155 360,
156 420,
157 480,
158 540,
159 600,
160 720,
161 900,
162 1800,
163 3600,
164 _INF,
165 ),
166)
169servicer_duration_histogram: Histogram = Histogram(
170 "couchers_servicer_duration_seconds",
171 "Durations of processing gRPC calls",
172 labelnames=["method", "logged_in", "code", "exception"],
173 buckets=MACHINE_DURATION_SECONDS,
174)
177def observe_in_servicer_duration_histogram(
178 method: str, user_id: Any, status_code: str, exception_type: str, duration_s: float
179) -> None:
180 servicer_duration_histogram.labels(method, user_id is not None, status_code, exception_type).observe(duration_s)
183servicer_setup_errors_counter: Counter = Counter(
184 "couchers_servicer_setup_errors_total",
185 "Number of unexpected errors raised during gRPC interceptor setup, before the handler is invoked",
186 labelnames=["method", "exception"],
187)
190def observe_in_servicer_setup_errors_counter(method: str, exception_type: str) -> None:
191 servicer_setup_errors_counter.labels(method, exception_type).inc()
194# Per-request resource accounting (see couchers/perf.py), labelled by method only to keep cardinality modest. The
195# histogram _sum gives the cost rate per endpoint via rate() (DB-seconds/sec, CPU-seconds/sec); the buckets give the
196# per-call distribution.
197servicer_db_time_histogram: Histogram = Histogram(
198 "couchers_servicer_db_time_seconds",
199 "Time spent in DB cursor execution per gRPC call",
200 labelnames=["method"],
201 buckets=MACHINE_DURATION_SECONDS,
202)
203servicer_cpu_time_histogram: Histogram = Histogram(
204 "couchers_servicer_cpu_seconds",
205 "Backend thread CPU time per gRPC call",
206 labelnames=["method"],
207 buckets=MACHINE_DURATION_SECONDS,
208)
209# Fibonacci bucket boundaries: roughly exponential, good resolution for an unbounded value
210servicer_db_query_count_histogram: Histogram = Histogram(
211 "couchers_servicer_db_query_count",
212 "Number of SQL statements executed per gRPC call",
213 labelnames=["method"],
214 buckets=(1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, _INF),
215)
216servicer_db_write_query_count_histogram: Histogram = Histogram(
217 "couchers_servicer_db_write_query_count",
218 "Number of INSERT/UPDATE/DELETE statements executed per gRPC call",
219 labelnames=["method"],
220 buckets=(1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, _INF),
221)
224def observe_in_servicer_perf_histograms(method: str, perf: PerfResult | None) -> None:
225 if perf is None: 225 ↛ 226line 225 didn't jump to line 226 because the condition on line 225 was never true
226 return
227 servicer_db_time_histogram.labels(method).observe(perf.db_time_ms / 1000)
228 servicer_cpu_time_histogram.labels(method).observe(perf.cpu_ms / 1000)
229 servicer_db_query_count_histogram.labels(method).observe(perf.db_query_count)
230 servicer_db_write_query_count_histogram.labels(method).observe(perf.db_write_query_count)
233# Auth/setup phase (everything before the handler body), same db-vs-cpu split as the handler-body histograms above.
234servicer_setup_db_time_histogram: Histogram = Histogram(
235 "couchers_servicer_setup_db_time_seconds",
236 "Time spent in DB cursor execution during the auth/setup phase per gRPC call",
237 labelnames=["method"],
238 buckets=MACHINE_DURATION_SECONDS,
239)
240servicer_setup_cpu_time_histogram: Histogram = Histogram(
241 "couchers_servicer_setup_cpu_seconds",
242 "Backend thread CPU time during the auth/setup phase per gRPC call",
243 labelnames=["method"],
244 buckets=MACHINE_DURATION_SECONDS,
245)
248def observe_in_servicer_setup_histogram(method: str, perf: PerfResult | None) -> None:
249 if perf is None: 249 ↛ 250line 249 didn't jump to line 250 because the condition on line 249 was never true
250 return
251 servicer_setup_db_time_histogram.labels(method).observe(perf.db_time_ms / 1000)
252 servicer_setup_cpu_time_histogram.labels(method).observe(perf.cpu_ms / 1000)
255servicer_pool_wait_histogram: Histogram = Histogram(
256 "couchers_servicer_pool_wait_seconds",
257 "Time spent waiting to check out a DB connection from the pool per gRPC call",
258 labelnames=["method"],
259 buckets=MACHINE_DURATION_SECONDS,
260)
263def observe_in_servicer_pool_wait_histogram(method: str, pool_wait_s: float) -> None:
264 servicer_pool_wait_histogram.labels(method).observe(pool_wait_s)
267# Separate diagnostic, not part of the additive duration pie: "serialize" runs after the duration window closes.
268servicer_serde_histogram: Histogram = Histogram(
269 "couchers_servicer_serde_seconds",
270 "Protobuf request deserialization / response serialization time per gRPC call",
271 labelnames=["method", "direction"],
272 buckets=MACHINE_DURATION_SECONDS,
273)
276def observe_in_servicer_serde_histogram(method: str, direction: str, serde_s: float) -> None:
277 servicer_serde_histogram.labels(method, direction).observe(serde_s)
280# liveall keeps one series per worker pid (and drops dead workers), so these also show load balance across workers.
281# Updated from inside each worker since the /metrics scrape runs in the parent, which has neither pool.
282grpc_in_flight_gauge: Gauge = Gauge(
283 "couchers_grpc_in_flight",
284 "Outstanding gRPC calls (running plus queued for a server thread), per worker process",
285 multiprocess_mode="liveall",
286)
287grpc_threadpool_queue_depth_gauge: Gauge = Gauge(
288 "couchers_grpc_threadpool_queue_depth",
289 "gRPC calls queued waiting for a free server thread, per worker process",
290 multiprocess_mode="liveall",
291)
292db_pool_checked_out_gauge: Gauge = Gauge(
293 "couchers_db_pool_checked_out",
294 "Checked-out DB connections, per worker process",
295 multiprocess_mode="liveall",
296)
299def start_worker_resource_sampler(executor: ThreadPoolExecutor, engine: Engine, interval: float = 1.0) -> None:
300 def sample() -> None:
301 while True:
302 # _work_queue is private but stable: tasks gRPC has submitted that no thread has picked up yet
303 grpc_threadpool_queue_depth_gauge.set(executor._work_queue.qsize())
304 db_pool_checked_out_gauge.set(cast(QueuePool, engine.pool).checkedout())
305 time.sleep(interval)
307 threading.Thread(target=sample, daemon=True, name="resource-sampler").start()
310supervised_children_alive_gauge: Gauge = Gauge(
311 "couchers_supervised_children_alive",
312 "Child processes (API workers, background workers, scheduler) the supervisor currently sees alive",
313 multiprocess_mode="mostrecent",
314)
317# Simple count of API calls, broken down by method and the client platform header. Cheap (a counter, no buckets) and
318# answers "how much traffic comes from each platform".
319api_calls_counter: Counter = Counter(
320 "couchers_api_calls_total",
321 "Number of gRPC API calls",
322 labelnames=["method", "platform"],
323)
326def observe_api_call(method: str, client_platform: ClientPlatform | None) -> None:
327 api_calls_counter.labels(method, client_platform.name if client_platform is not None else "unknown").inc()
330# list of gauge names and function to execute to set value to
331# the python prometheus client does not support Gauge.set_function, so instead we hack around it and set each gauge just
332# before collection with this
333_set_hacky_gauges_funcs: list[tuple[Gauge, Callable[[], Any]]] = []
336def _make_gauge_from_query(name: str, description: str, statement: Select[Any]) -> Gauge:
337 """
338 Given a name, description and statement that is a sqlalchemy statement, creates a gauge from it
340 statement should be a sqlalchemy SELECT statement that returns a single number
341 """
343 def f() -> Any:
344 with tracer.start_as_current_span(f"metric.{name}"):
345 with session_scope() as session:
346 return session.execute(statement).scalar_one()
348 gauge = Gauge(name, description, multiprocess_mode="mostrecent")
349 _set_hacky_gauges_funcs.append((gauge, f))
350 return gauge
353# list of labeled gauges and the function to populate their label values just before collection
354_set_hacky_labeled_gauges_funcs: list[tuple[Gauge, Callable[[Gauge], None]]] = []
357def _make_labeled_gauge_from_query(
358 name: str,
359 description: str,
360 labelname: str,
361 statement: Select[Any],
362 default_label_values: list[str] | None = None,
363) -> Gauge:
364 """
365 Given a name, description, label name and statement, creates a gauge with one label set from the statement.
367 statement should be a sqlalchemy SELECT statement that returns rows of (label_value, count).
369 default_label_values, if given, are seeded to zero before the query results are applied, so that label
370 values with no matching rows are still emitted.
371 """
373 gauge = Gauge(name, description, labelnames=[labelname], multiprocess_mode="mostrecent")
375 def f(g: Gauge) -> None:
376 with tracer.start_as_current_span(f"metric.{name}"):
377 with session_scope() as session:
378 rows = session.execute(statement).all()
379 for label_value in default_label_values or []:
380 g.labels(label_value).set(0)
381 for label_value, count in rows:
382 g.labels(str(label_value)).set(count)
384 _set_hacky_labeled_gauges_funcs.append((gauge, f))
385 return gauge
388_active_user_periods: list[tuple[str, str, timedelta]] = [
389 ("5m", "5 min", timedelta(minutes=5)),
390 ("24h", "24 hours", timedelta(hours=24)),
391 ("1month", "1 month", timedelta(weeks=4)),
392 ("3month", "3 months", timedelta(weeks=13)),
393 ("6month", "6 months", timedelta(weeks=26)),
394 ("12month", "12 months", timedelta(days=365)),
395]
397active_users_gauges: list[Gauge] = [
398 _make_gauge_from_query(
399 f"couchers_active_users_{name}",
400 f"Number of active users in the last {description}",
401 (select(func.count()).select_from(User).where(User.is_visible).where(User.last_active > func.now() - interval)),
402 )
403 for name, description, interval in _active_user_periods
404]
406users_gauge: Gauge = _make_gauge_from_query(
407 "couchers_users", "Total number of users", select(func.count()).select_from(User).where(User.is_visible)
408)
410# Number of users per community, labeled by community name. Only includes communities at the region level or
411# broader (world, macroregion, region).
412users_per_community_gauge: Gauge = _make_labeled_gauge_from_query(
413 "couchers_users_per_community",
414 "Number of users per community, for regions and broader",
415 "community",
416 (
417 select(Cluster.name, func.coalesce(ClusterSubscriptionCount.count, 0))
418 .select_from(Node)
419 .join(Cluster, and_(Cluster.parent_node_id == Node.id, Cluster.is_official_cluster))
420 .outerjoin(ClusterSubscriptionCount, ClusterSubscriptionCount.cluster_id == Cluster.id)
421 .where(Node.node_type <= NodeType.region)
422 ),
423)
425# Number of users bucketed by how recently they were last active.
426_active_users_buckets: list[tuple[str, timedelta | None]] = [
427 ("<1d", timedelta(days=1)),
428 ("1d-1w", timedelta(days=7)),
429 ("1w-1m", timedelta(weeks=4)),
430 ("1m-6m", timedelta(weeks=26)),
431 ("6m-12m", timedelta(days=365)),
432 ("12m-24m", timedelta(days=730)),
433 ("24m+", None),
434]
435_active_users_age = func.now() - User.last_active
436active_users_by_recency_gauge: Gauge = _make_labeled_gauge_from_query(
437 "couchers_active_users_by_recency",
438 "Number of users bucketed by how recently they were last active",
439 "period",
440 (
441 select(
442 case(
443 *[(_active_users_age < interval, label) for label, interval in _active_users_buckets if interval],
444 else_=_active_users_buckets[-1][0],
445 ).label("period"),
446 func.count(),
447 )
448 .select_from(User)
449 .where(User.is_visible)
450 .group_by("period")
451 ),
452 default_label_values=[label for label, _ in _active_users_buckets],
453)
455# Window for the per-platform daily-active-user metrics. Kept to 24h so the user_activity scan stays cheap (an index
456# scan of just the last day's rows), letting these gauges be computed inline on every scrape.
457_ACTIVE_USERS_BY_PLATFORM_WINDOW = timedelta(hours=24)
458# Platforms counted as "mobile" for the mobile-share fraction (native apps plus the mobile web viewport).
459_MOBILE_PLATFORMS = [ClientPlatform.web_mobile, ClientPlatform.app_ios, ClientPlatform.app_android]
462def active_users_by_platform_statement() -> Select[Any]:
463 # one scan of the last 24h of user_activity: distinct active users in total, the mobile subset (for the share
464 # fraction), and a breakdown per platform. client_platform is set from a header the client explicitly sends; it's
465 # null for some other client (e.g. an API key script) or activity from before the header existed, so the
466 # per-platform counts don't sum to the total and "mobile" needs its own union count rather than summing labels.
467 distinct_users = func.count(distinct(UserActivity.user_id))
468 return (
469 select(
470 distinct_users.label("total"),
471 distinct_users.filter(UserActivity.client_platform.in_(_MOBILE_PLATFORMS)).label("mobile"),
472 *[
473 distinct_users.filter(UserActivity.client_platform == platform).label(platform.name)
474 for platform in ClientPlatform
475 ],
476 )
477 .select_from(UserActivity)
478 .join(User, User.id == UserActivity.user_id)
479 .where(User.is_visible)
480 .where(UserActivity.period > func.now() - _ACTIVE_USERS_BY_PLATFORM_WINDOW)
481 )
484# Distinct active users in the last 24h, split by client platform.
485active_users_by_platform_gauge: Gauge = Gauge(
486 "couchers_active_users_by_platform",
487 "Distinct active users in the last 24h, split by client platform (web_desktop, web_mobile, app_ios, app_android)",
488 labelnames=["platform"],
489 multiprocess_mode="mostrecent",
490)
492# Fraction of the last 24h's distinct active users who had any mobile activity. The headline "mobile is key" number.
493active_users_mobile_fraction_gauge: Gauge = Gauge(
494 "couchers_active_users_mobile_fraction",
495 "Fraction of distinct active users in the last 24h with any mobile activity (web_mobile, app_ios, app_android)",
496 multiprocess_mode="mostrecent",
497)
500def _set_active_users_by_platform(gauge: Gauge) -> None:
501 with tracer.start_as_current_span("metric.couchers_active_users_by_platform"):
502 with session_scope() as session:
503 row = session.execute(active_users_by_platform_statement()).one()._mapping
504 for platform in ClientPlatform:
505 gauge.labels(platform.name).set(row[platform.name])
506 total = row["total"]
507 active_users_mobile_fraction_gauge.set(row["mobile"] / total if total else 0.0)
510_set_hacky_labeled_gauges_funcs.append((active_users_by_platform_gauge, _set_active_users_by_platform))
512man_gauge: Gauge = _make_gauge_from_query(
513 "couchers_users_man",
514 "Total number of users with gender 'Man'",
515 select(func.count()).select_from(User).where(User.is_visible).where(User.gender == "Man"),
516)
518woman_gauge: Gauge = _make_gauge_from_query(
519 "couchers_users_woman",
520 "Total number of users with gender 'Woman'",
521 select(func.count()).select_from(User).where(User.is_visible).where(User.gender == "Woman"),
522)
524nonbinary_gauge: Gauge = _make_gauge_from_query(
525 "couchers_users_nonbinary",
526 "Total number of users with gender 'Non-binary'",
527 select(func.count()).select_from(User).where(User.is_visible).where(User.gender == "Non-binary"),
528)
530can_host_gauge: Gauge = _make_gauge_from_query(
531 "couchers_users_can_host",
532 "Total number of users with hosting status 'can_host'",
533 select(func.count()).select_from(User).where(User.is_visible).where(User.hosting_status == HostingStatus.can_host),
534)
536cant_host_gauge: Gauge = _make_gauge_from_query(
537 "couchers_users_cant_host",
538 "Total number of users with hosting status 'cant_host'",
539 select(func.count()).select_from(User).where(User.is_visible).where(User.hosting_status == HostingStatus.cant_host),
540)
542maybe_gauge: Gauge = _make_gauge_from_query(
543 "couchers_users_maybe",
544 "Total number of users with hosting status 'maybe'",
545 select(func.count()).select_from(User).where(User.is_visible).where(User.hosting_status == HostingStatus.maybe),
546)
548completed_profile_gauge: Gauge = _make_gauge_from_query(
549 "couchers_users_completed_profile",
550 "Total number of users with a completed profile",
551 select(func.count()).select_from(User).where(User.is_visible).where(has_completed_profile_expression()),
552)
554completed_my_home_gauge: Gauge = _make_gauge_from_query(
555 "couchers_users_completed_my_home",
556 "Total number of users with a completed my home section",
557 select(func.count()).select_from(User).where(User.is_visible).where(User.has_completed_my_home),
558)
560sent_message_gauge: Gauge = _make_gauge_from_query(
561 "couchers_users_sent_message",
562 "Total number of users who have sent a message",
563 (select(func.count(distinct(Message.author_id))).join(User, User.id == Message.author_id).where(User.is_visible)),
564)
566sent_request_gauge: Gauge = _make_gauge_from_query(
567 "couchers_users_sent_request",
568 "Total number of users who have sent a host request",
569 (
570 select(func.count(distinct(HostRequest.initiator_user_id)))
571 .join(User, User.id == HostRequest.initiator_user_id)
572 .where(User.is_visible)
573 ),
574)
576has_reference_gauge: Gauge = _make_gauge_from_query(
577 "couchers_users_has_reference",
578 "Total number of users who have a reference",
579 (
580 select(func.count(distinct(Reference.to_user_id)))
581 .join(User, User.id == Reference.to_user_id)
582 .where(User.is_visible)
583 ),
584)
586rsvpd_to_event_gauge: Gauge = _make_gauge_from_query(
587 "couchers_users_rsvpd_to_event",
588 "Total number of users who have RSVPd to an event",
589 (
590 select(func.count(distinct(EventOccurrenceAttendee.user_id)))
591 .join(User, User.id == EventOccurrenceAttendee.user_id)
592 .where(User.is_visible)
593 ),
594)
596background_jobs_ready_to_execute_gauge: Gauge = _make_gauge_from_query(
597 "couchers_background_jobs_ready_to_execute",
598 "Total number of background jobs ready to execute",
599 select(func.count()).select_from(BackgroundJob).where(BackgroundJob.ready_for_retry),
600)
602background_jobs_serialization_errors_counter: Counter = Counter(
603 "couchers_background_jobs_serialization_errors_total",
604 "Number of times a bg worker has a serialization error",
605)
607background_jobs_no_jobs_counter: Counter = Counter(
608 "couchers_background_jobs_no_jobs_total",
609 "Number of times a bg worker tries to grab a job but there is none",
610)
612background_jobs_got_job_counter: Counter = Counter(
613 "couchers_background_jobs_got_job_total",
614 "Number of times a bg worker grabbed a job",
615)
618signup_initiations_counter: Counter = Counter(
619 "couchers_signup_initiations_total",
620 "Number of initiated signups",
621)
622signup_completions_counter: Counter = Counter(
623 "couchers_signup_completions_total",
624 "Number of completed signups",
625 labelnames=["gender"],
626)
627# Per-step signup funnel counters. Each fires once, the first time a signup flow satisfies the given gate, so
628# that step_total/initiations_total gives the fraction of signups that reached that step. Unlabeled to match
629# signup_initiations_counter for clean ratios.
630signup_account_filled_counter: Counter = Counter(
631 "couchers_signup_account_filled_total",
632 "Number of signup flows that filled in their account details",
633)
634signup_email_verified_counter: Counter = Counter(
635 "couchers_signup_email_verified_total",
636 "Number of signup flows that verified their email address",
637)
638signup_guidelines_accepted_counter: Counter = Counter(
639 "couchers_signup_guidelines_accepted_total",
640 "Number of signup flows that accepted the community guidelines",
641)
642signup_motivations_filled_counter: Counter = Counter(
643 "couchers_signup_motivations_filled_total",
644 "Number of signup flows that filled in their motivations",
645)
646signup_time_histogram: Histogram = Histogram(
647 "couchers_signup_time_seconds",
648 "Time taken for a user to sign up",
649 labelnames=["gender"],
650 buckets=(30, 60, 90, 120, 180, 240, 300, 360, 420, 480, 540, 600, 900, 1200, 1800, 3600, 7200, _INF),
651)
653logins_counter: Counter = Counter(
654 "couchers_logins_total",
655 "Number of logins",
656 labelnames=["gender"],
657)
659password_reset_initiations_counter: Counter = Counter(
660 "couchers_password_reset_initiations_total",
661 "Number of password reset initiations",
662)
663password_reset_completions_counter: Counter = Counter(
664 "couchers_password_reset_completions_total",
665 "Number of password reset completions",
666)
668account_deletion_initiations_counter: Counter = Counter(
669 "couchers_account_deletion_initiations_total",
670 "Number of account deletion initiations",
671 labelnames=["gender"],
672)
673account_deletion_completions_counter: Counter = Counter(
674 "couchers_account_deletion_completions_total",
675 "Number of account deletion completions",
676 labelnames=["gender"],
677)
678account_recoveries_counter: Counter = Counter(
679 "couchers_account_recoveries_total",
680 "Number of account recoveries",
681 labelnames=["gender"],
682)
684strong_verification_initiations_counter: Counter = Counter(
685 "couchers_strong_verification_initiations_total",
686 "Number of strong verification initiations",
687 labelnames=["gender"],
688)
689strong_verification_completions_counter: Counter = Counter(
690 "couchers_strong_verification_completions_total",
691 "Number of strong verification completions",
692)
693strong_verification_data_deletions_counter: Counter = Counter(
694 "couchers_strong_verification_data_deletions_total",
695 "Number of strong verification data deletions",
696 labelnames=["gender"],
697)
699host_requests_sent_counter: Counter = Counter(
700 "couchers_host_requests_total",
701 "Number of host requests sent",
702 labelnames=["from_gender", "to_gender"],
703)
704host_request_responses_counter: Counter = Counter(
705 "couchers_host_requests_responses_total",
706 "Number of responses to host requests",
707 labelnames=["responder_gender", "other_gender", "response_type"],
708)
710sent_messages_counter: Counter = Counter(
711 "couchers_sent_messages_total",
712 "Number of messages sent",
713 labelnames=["gender", "message_type"],
714)
717push_notification_counter: Counter = Counter(
718 "couchers_push_notification_total",
719 "Number of push notification delivery attempts",
720 labelnames=["platform", "outcome"],
721)
722emails_counter: Counter = Counter(
723 "couchers_emails_total",
724 "Number of emails sent",
725)
728antibots_assessed_counter: Counter = Counter(
729 "couchers_antibots_assessed_total",
730 "Number of times an antibot assessment is created",
731 labelnames=["action"],
732)
734antibot_score_histogram: Histogram = Histogram(
735 "couchers_antibot_score",
736 "Score of antibot assessments",
737 labelnames=["action"],
738 buckets=tuple(x / 20 for x in range(0, 21)),
739)
741host_request_first_response_histogram: Histogram = Histogram(
742 "couchers_host_request_first_response_seconds",
743 "Response time to host requests",
744 labelnames=["host_gender", "surfer_gender", "response_type"],
745 buckets=(
746 1 * 60, # 1m
747 2 * 60, # 2m
748 5 * 60, # 5m
749 10 * 60, # 10m
750 15 * 60, # 15m
751 30 * 60, # 30m
752 45 * 60, # 45m
753 3_600, # 1h
754 2 * 3_600, # 2h
755 3 * 3_600, # 3h
756 6 * 3_600, # 6h
757 12 * 3_600, # 12h
758 86_400, # 24h
759 2 * 86_400, # 2d
760 5 * 86_400, # 4d
761 602_000, # 1w
762 2 * 602_000, # 2w
763 3 * 602_000, # 3w
764 4 * 602_000, # 4w
765 _INF,
766 ),
767)
768account_age_on_host_request_create_histogram: Histogram = Histogram(
769 "couchers_account_age_on_host_request_create_histogram_seconds",
770 "Age of account sending a host request",
771 labelnames=["surfer_gender", "host_gender"],
772 buckets=(
773 5 * 60, # 5m
774 10 * 60, # 10m
775 15 * 60, # 15m
776 30 * 60, # 30m
777 45 * 60, # 45m
778 3_600, # 1h
779 2 * 3_600, # 2h
780 3 * 3_600, # 3h
781 6 * 3_600, # 6h
782 12 * 3_600, # 12h
783 86_400, # 24h
784 2 * 86_400, # 2d
785 3 * 86_400, # 3d
786 4 * 86_400, # 4d
787 5 * 86_400, # 5d
788 6 * 86_400, # 6d
789 602_000, # 1w
790 2 * 602_000, # 2w
791 3 * 602_000, # 3w
792 4 * 602_000, # 4w
793 5 * 602_000, # 5w
794 10 * 602_000, # 10w
795 25 * 602_000, # 25w
796 52 * 602_000, # 52w
797 104 * 602_000, # 104w
798 _INF,
799 ),
800)
803# =============================================================================
804# Moderation metrics
805# =============================================================================
807# Gauges: Queue lengths
808moderation_queue_length_gauge: Gauge = _make_gauge_from_query(
809 "couchers_moderation_queue_length",
810 "Total number of unresolved items in the moderation queue",
811 select(func.count()).select_from(ModerationQueueItem).where(ModerationQueueItem.resolved_by_log_id.is_(None)),
812)
814moderation_queue_length_by_trigger_gauges: list[Gauge] = [
815 _make_gauge_from_query(
816 f"couchers_moderation_queue_length_{trigger.name.lower()}",
817 f"Number of unresolved items in the moderation queue with trigger {trigger.name}",
818 select(func.count())
819 .select_from(ModerationQueueItem)
820 .where(ModerationQueueItem.resolved_by_log_id.is_(None))
821 .where(ModerationQueueItem.trigger == trigger),
822 )
823 for trigger in ModerationTrigger
824]
826moderation_queue_length_by_object_type_gauges: list[Gauge] = [
827 _make_gauge_from_query(
828 f"couchers_moderation_queue_length_{object_type.name.lower()}",
829 f"Number of unresolved items in the moderation queue for {object_type.name}",
830 select(func.count())
831 .select_from(ModerationQueueItem)
832 .join(ModerationState, ModerationQueueItem.moderation_state_id == ModerationState.id)
833 .where(ModerationQueueItem.resolved_by_log_id.is_(None))
834 .where(ModerationState.object_type == object_type),
835 )
836 for object_type in ModerationObjectType
837]
839# Gauges: Items in each visibility state by object type
840moderation_visibility_gauges: list[Gauge] = [
841 _make_gauge_from_query(
842 f"couchers_moderation_items_{object_type.name.lower()}_{visibility.name.lower()}",
843 f"Number of {object_type.name} items with visibility {visibility.name}",
844 select(func.count())
845 .select_from(ModerationState)
846 .where(ModerationState.object_type == object_type)
847 .where(ModerationState.visibility == visibility),
848 )
849 for object_type in ModerationObjectType
850 for visibility in ModerationVisibility
851]
853# Counters: Moderation actions taken
854moderation_actions_counter: Counter = Counter(
855 "couchers_moderation_actions_total",
856 "Number of moderation actions taken",
857 labelnames=["action", "object_type"],
858)
861def observe_moderation_action(action: ModerationAction, object_type: ModerationObjectType) -> None:
862 moderation_actions_counter.labels(action.name, object_type.name).inc()
865# Counters: Visibility state transitions
866moderation_visibility_transitions_counter: Counter = Counter(
867 "couchers_moderation_visibility_transitions_total",
868 "Number of visibility state transitions",
869 labelnames=["from_visibility", "to_visibility", "object_type"],
870)
873def observe_moderation_visibility_transition(
874 from_visibility: ModerationVisibility, to_visibility: ModerationVisibility, object_type: ModerationObjectType
875) -> None:
876 moderation_visibility_transitions_counter.labels(from_visibility.name, to_visibility.name, object_type.name).inc()
879# Counters: Auto-approved items
880moderation_auto_approved_counter: Counter = Counter(
881 "couchers_moderation_auto_approved_total",
882 "Number of items that were auto-approved",
883)
886# Counters: Queue items created
887moderation_queue_items_created_counter: Counter = Counter(
888 "couchers_moderation_queue_items_created_total",
889 "Number of moderation queue items created",
890 labelnames=["trigger", "object_type"],
891)
894def observe_moderation_queue_item_created(trigger: ModerationTrigger, object_type: ModerationObjectType) -> None:
895 moderation_queue_items_created_counter.labels(trigger.name, object_type.name).inc()
898# Counters: Queue items resolved
899moderation_queue_items_resolved_counter: Counter = Counter(
900 "couchers_moderation_queue_items_resolved_total",
901 "Number of moderation queue items resolved",
902 labelnames=["trigger", "action", "object_type"],
903)
906def observe_moderation_queue_item_resolved(
907 trigger: ModerationTrigger, action: ModerationAction, object_type: ModerationObjectType
908) -> None:
909 moderation_queue_items_resolved_counter.labels(trigger.name, action.name, object_type.name).inc()
912# Histogram: Time to resolve queue items
913moderation_queue_resolution_time_histogram: Histogram = Histogram(
914 "couchers_moderation_queue_resolution_seconds",
915 "Time taken to resolve moderation queue items",
916 labelnames=["trigger", "action", "object_type"],
917 buckets=(
918 0.1,
919 0.25,
920 0.5,
921 1,
922 2.5,
923 5,
924 10,
925 30,
926 60,
927 5 * 60,
928 15 * 60,
929 30 * 60,
930 3_600,
931 2 * 3_600,
932 6 * 3_600,
933 12 * 3_600,
934 86_400,
935 2 * 86_400,
936 3 * 86_400,
937 7 * 86_400,
938 14 * 86_400,
939 30 * 86_400,
940 _INF,
941 ),
942)
945def observe_moderation_queue_resolution_time(
946 trigger: ModerationTrigger, action: ModerationAction, object_type: ModerationObjectType, duration_s: float
947) -> None:
948 moderation_queue_resolution_time_histogram.labels(trigger.name, action.name, object_type.name).observe(duration_s)
951nonvisible_user_access_counter: Counter = Counter(
952 "couchers_nonvisible_user_access_total",
953 "Number of access events involving nonvisible (banned/shadowed/deleted) users",
954 labelnames=["access_type", "target_state"],
955)
958def observe_nonvisible_user_access(access_type: NonvisibleUserAccessType, target_state: NonvisibleUserState) -> None:
959 nonvisible_user_access_counter.labels(access_type.name, target_state.name).inc()
962postcards_sent_counter: Counter = Counter(
963 "couchers_postcards_sent_total",
964 "Number of postcards sent via MyPostcard",
965 labelnames=["country_code"],
966)
969# Native app / OTA update metrics. Bucket layout is minute-resolution at the low end (watch an OTA
970# rolling out), dense around the OTA (~28d) and store (~91d) windows, and sparse past it for stragglers.
971_NATIVE_AGE_BUCKETS: tuple[float, ...] = (
972 60,
973 5 * 60,
974 15 * 60,
975 30 * 60,
976 3_600,
977 2 * 3_600,
978 6 * 3_600,
979 12 * 3_600,
980 86_400,
981 2 * 86_400,
982 3 * 86_400,
983 5 * 86_400,
984 7 * 86_400,
985 10 * 86_400,
986 14 * 86_400,
987 21 * 86_400,
988 28 * 86_400,
989 35 * 86_400,
990 45 * 86_400,
991 60 * 86_400,
992 75 * 86_400,
993 91 * 86_400,
994 120 * 86_400,
995 150 * 86_400,
996 180 * 86_400,
997 270 * 86_400,
998 365 * 86_400,
999 730 * 86_400,
1000 _INF,
1001)
1003native_bundle_age_histogram: Histogram = Histogram(
1004 "couchers_native_bundle_age_seconds",
1005 "Age of the OTA bundle reported by the client at CheckNativeStatus, by platform and launch source",
1006 labelnames=["platform", "is_ota_launch"],
1007 buckets=_NATIVE_AGE_BUCKETS,
1008)
1011def observe_native_bundle_age(platform: str, is_ota_launch: bool, age_s: float) -> None:
1012 native_bundle_age_histogram.labels(platform or "unknown", "true" if is_ota_launch else "false").observe(age_s)
1015native_binary_age_histogram: Histogram = Histogram(
1016 "couchers_native_binary_age_seconds",
1017 "Age of the embedded native binary reported by the client at CheckNativeStatus, by platform",
1018 labelnames=["platform"],
1019 buckets=_NATIVE_AGE_BUCKETS,
1020)
1023def observe_native_binary_age(platform: str, age_s: float) -> None:
1024 native_binary_age_histogram.labels(platform or "unknown").observe(age_s)
1027native_update_decisions_counter: Counter = Counter(
1028 "couchers_native_update_decisions_total",
1029 "CheckNativeStatus decisions, by platform / action / severity",
1030 labelnames=["platform", "action", "severity"],
1031)
1034def observe_native_update_decision(platform: str, action: str, severity: str) -> None:
1035 native_update_decisions_counter.labels(platform or "unknown", action, severity).inc()
1038native_banned_bundle_hits_counter: Counter = Counter(
1039 "couchers_native_banned_bundle_hits_total",
1040 "CheckNativeStatus calls from a device running a banned OTA bundle, by platform",
1041 labelnames=["platform"],
1042)
1045def observe_native_banned_bundle_hit(platform: str) -> None:
1046 native_banned_bundle_hits_counter.labels(platform or "unknown").inc()
1049native_ota_manifest_requests_counter: Counter = Counter(
1050 "couchers_native_ota_manifest_requests_total",
1051 "GetNativeUpdateManifest requests, by platform and result (served, no_update, no_match)",
1052 labelnames=["platform", "result"],
1053)
1056def observe_native_ota_manifest_request(platform: str, result: str) -> None:
1057 native_ota_manifest_requests_counter.labels(platform or "unknown", result).inc()
1060# One increment per CheckNativeStatus, labeled by build/bundle identity, to see the live mix of
1061# versions and bundles running in the fleet.
1062native_client_checkins_counter: Counter = Counter(
1063 "couchers_native_client_checkins_total",
1064 "CheckNativeStatus calls, labeled by build/bundle identity",
1065 labelnames=[
1066 "platform",
1067 "is_ota_launch",
1068 "embedded_display_version",
1069 "embedded_runtime_version",
1070 "ota_display_version",
1071 "ota_update_id",
1072 ],
1073)
1076def observe_native_client_checkin(
1077 platform: str,
1078 is_ota_launch: bool,
1079 embedded_display_version: str,
1080 embedded_runtime_version: str,
1081 ota_display_version: str,
1082 ota_update_id: str,
1083) -> None:
1084 native_client_checkins_counter.labels(
1085 platform or "unknown",
1086 "true" if is_ota_launch else "false",
1087 embedded_display_version or "unknown",
1088 embedded_runtime_version or "unknown",
1089 ota_display_version or "none",
1090 ota_update_id or "none",
1091 ).inc()
1094# Recomputed at scrape time via the hacky-gauge mechanism, so it reflects live age. 0 when disabled
1095# or never pulled.
1096def _feature_flags_staleness_seconds() -> float:
1097 return experimentation.seconds_since_last_fetch() or 0.0
1100feature_flags_staleness_gauge: Gauge = Gauge(
1101 "couchers_feature_flags_staleness_seconds",
1102 "Seconds since feature flags were last successfully fetched from GrowthBook",
1103 multiprocess_mode="mostrecent",
1104)
1105_set_hacky_gauges_funcs.append((feature_flags_staleness_gauge, _feature_flags_staleness_seconds))
1108feature_flag_evaluations_counter: Counter = Counter(
1109 "couchers_feature_flag_evaluations_total",
1110 "Number of feature flag evaluations, by flag key, evaluation source, and resolved value",
1111 labelnames=["flag_key", "source", "value"],
1112)
1114_MAX_FLAG_VALUE_LABEL_LEN = 32
1117def _stringify_flag_value(value: Any) -> str:
1118 if isinstance(value, bool):
1119 return "true" if value else "false"
1120 if isinstance(value, (int, float, str)):
1121 s = str(value)
1122 return s if len(s) <= _MAX_FLAG_VALUE_LABEL_LEN else f"<{type(value).__name__}>"
1123 if value is None: 1123 ↛ 1125line 1123 didn't jump to line 1125 because the condition on line 1123 was always true
1124 return "None"
1125 return f"<{type(value).__name__}>"
1128def observe_feature_flag_evaluation(flag_key: str, source: str, value: Any) -> None:
1129 feature_flag_evaluations_counter.labels(flag_key, source, _stringify_flag_value(value)).inc()
1132def create_prometheus_server(port: int) -> Any:
1133 """custom start method to fix problem descrbied in https://github.com/prometheus/client_python/issues/155"""
1135 def app(environ: Any, start_response: Any) -> Any:
1136 # set hacky gauges
1137 for gauge, f in _set_hacky_gauges_funcs:
1138 gauge.set(f())
1139 for gauge, labeled_f in _set_hacky_labeled_gauges_funcs:
1140 labeled_f(gauge)
1142 data = generate_latest(registry)
1143 start_response("200 OK", [("Content-type", CONTENT_TYPE_LATEST), ("Content-Length", str(len(data)))])
1144 return [data]
1146 httpd = exposition.make_server( # type: ignore[attr-defined]
1147 "", port, app, exposition.ThreadingWSGIServer, handler_class=exposition._SilentHandler
1148 )
1149 t = threading.Thread(target=httpd.serve_forever)
1150 t.daemon = True
1151 t.start()
1152 return httpd