{"service":"symphony-status","family":"rle","environment":"prod","project":"symphony-prod-509117","region":"us-central1","status":"degraded","checked_at":"2026-11-15T12:07:17.717199+00:00","elapsed_ms":571.6,"summary":{"services":7,"healthy":6,"unhealthy":1,"probes":7,"passed":6,"failed":1},"services":{"rle-api":{"family":"rle","description":"the rle backend, Deployment symphony-api on the symphony-rle cluster, serving the rle API under https://rle.tastelabs.com/api/v1/rle","url":null,"healthy":true,"probes":{"deployment":{"path":"symphony-rle/symphony-api/symphony-api","tells":"the deployment serves at or above its disruption floor on its cluster","url":null,"auth":"gke-api","ok":true,"http_status":null,"latency_ms":84.1,"response":{"replicas":{"desired":2,"available":2},"conditions":[{"type":"Available","status":"True","lastUpdateTime":"2026-11-15T02:04:44Z","lastTransitionTime":"2026-11-15T02:04:44Z","reason":"MinimumReplicasAvailable","message":"Deployment has minimum availability."},{"type":"Progressing","status":"True","lastUpdateTime":"2026-11-15T02:10:42Z","lastTransitionTime":"2026-09-19T19:24:38Z","reason":"NewReplicaSetAvailable","message":"ReplicaSet \"symphony-api-5545986c64\" has successfully progressed."}]},"error":null}}},"rle-console":{"family":"rle","description":"the rle console at https://rle.tastelabs.com behind IAP, the Cloud Run symphony-platform-v2 service read at its own health route","url":"https://symphony-platform-v2-296871097007.us-central1.run.app","healthy":true,"probes":{"health":{"path":"/api/health","tells":"the console serves","url":"https://symphony-platform-v2-296871097007.us-central1.run.app/api/health","auth":"identity-token","ok":true,"http_status":200,"latency_ms":26.1,"response":{"status":"ok","service":"rle-console"},"error":null}}},"rle-maestro":{"family":"rle","description":"the maestro build fleet: Spot MIG workers draining the rle build queue, health read off the rle api's own board","url":null,"healthy":true,"probes":{"board":{"path":"symphony-rle/symphony-api/symphony-api:80/api/v1/rle/status/maestro","tells":"the api's own reading of the component, relayed whole","url":null,"auth":"gke-service-proxy","ok":true,"http_status":null,"latency_ms":160.6,"response":{"key":"maestro","group":"compute","label":"Maestro build fleet","state":"operational","latency_ms":45,"detail":"0 live workers, 0 building, 0 queued, 20/20 recent builds succeeded"},"error":null}}},"rle-records-publisher":{"family":"rle","description":"the records publisher moving settled rollouts from the bucket to the lab stores on Hugging Face, health read off the rle api's own board","url":null,"healthy":false,"probes":{"board":{"path":"symphony-rle/symphony-api/symphony-api:80/api/v1/rle/status/publisher","tells":"the api's own reading of the component, relayed whole","url":null,"auth":"gke-service-proxy","ok":false,"http_status":null,"latency_ms":229.1,"response":{"key":"publisher","group":"storage","label":"Records publisher","state":"down","latency_ms":97,"detail":"records not reaching the hub: 2206 unit(s) owed, newest landing 24572s ago, 135 held behind a definition not landed"},"error":"records not reaching the hub: 2206 unit(s) owed, newest landing 24572s ago, 135 held behind a definition not landed"}}},"rle-site-proxy":{"family":"rle","description":"the public preview service at https://preview.rle.tastelabs.com, rollout previews and site snapshots served from their stores","url":"https://symphony-site-proxy-296871097007.us-central1.run.app","healthy":true,"probes":{"health":{"path":"/health","tells":"the proxy serves","url":"https://symphony-site-proxy-296871097007.us-central1.run.app/health","auth":"identity-token","ok":true,"http_status":200,"latency_ms":25.5,"response":{"status":"ok"},"error":null}}},"rle-openai-egress-proxy":{"family":"rle","description":"the static-IP egress proxy the codex runtime reaches OpenAI through, Cloud Run symphony-openai-egress-proxy on internal ingress (absent where the environment mocks it)","url":"https://symphony-openai-egress-proxy-296871097007.us-central1.run.app","healthy":true,"probes":{"health":{"path":"/health","tells":"the egress proxy serves","url":"https://symphony-openai-egress-proxy-296871097007.us-central1.run.app/health","auth":"identity-token","ok":true,"http_status":200,"latency_ms":26.8,"response":{"status":"ok"},"error":null}}},"symphony-status":{"family":"rle","description":"the rle status service at https://status.rle.tastelabs.com and its own history database, rle-status on the rle family's Cloud SQL instance","url":null,"healthy":true,"probes":{"history":{"path":"symphony-prod-509117:us-central1:symphony-rle/rle-status","tells":"the family's status database on its Cloud SQL instance answers","url":null,"auth":"cloud-sql-iam","ok":true,"http_status":null,"latency_ms":20.4,"response":{"instance":"symphony-prod-509117:us-central1:symphony-rle","database":"rle-status"},"error":null}}}},"queues":{"queues":{"rle-pool":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":2016.0,"memory_used_gib":0.0,"memory_quota_gib":7616.0},"rle-unified":{"pending":0,"inadmissible":0,"admitted":19,"cpu_used":85.5,"cpu_quota":0.0,"memory_used_gib":323.0,"memory_quota_gib":0.0},"rle-user-ander-194c28":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-andre-a540bd":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-andy-wong-68f05a":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-angela-f4e41f":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-bautista-c25265":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-caio-acb2d7":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-daniel-cffd93":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-eliana-c10db2":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-fausto-40d471":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-hamidah-9c405f":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-jayce-386e29":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-kobe-db8f87":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-lautaro-2b5e20":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-maria-b-2318b6":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-mariia-p-5d151c":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-nicole-65310b":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-remi-65a0ab":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-rle-gen-import-slop-benc-64edd7":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-symphony-ci-72a8e6":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0},"rle-user-tina-e9ad07":{"pending":0,"inadmissible":0,"admitted":0,"cpu_used":0.0,"cpu_quota":0.0,"memory_used_gib":0.0,"memory_quota_gib":0.0}},"error":null,"elapsed_ms":330.9},"sampler":{"interval_s":60.0,"last_sample_at":"2026-11-15T12:06:24.216782+00:00","last_sample_error":null}}