{ "version": 1, "generated_from": "9d26ab2", "entries": [ { "id": "agent.backend_missing", "kind": "problem", "title": "agent.backend_missing", "category": "configuration", "severity": "error", "detection": "static", "detects": "No runner backend selected.", "fix": "No runner backend selected.", "verify": null, "setting": "agent.backend", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.backend_unknown", "kind": "problem", "title": "agent.backend_unknown", "category": "configuration", "severity": "error", "detection": "static", "detects": "Not a backend. They are `docker`, `podman` and `process`.", "fix": "Not a backend. They are `docker`, `podman` and `process`.", "verify": null, "setting": "agent.backend", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.bootstrap_cpu_grace", "kind": "problem", "title": "agent.bootstrap_cpu_grace", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must be between 0s and 10m; 0s applies pressure throttling immediately.", "fix": "Must be between 0s and 10m; 0s applies pressure throttling immediately.", "verify": null, "setting": "agent.bootstrap_cpu_grace", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.bootstrap_cpu_grace_short", "kind": "problem", "title": "agent.bootstrap_cpu_grace_short", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Less than 2m of normal CPU quota before pressure throttling; registration can slow under load.", "fix": "Less than 2m of normal CPU quota before pressure throttling; registration can slow under load.", "verify": null, "setting": "agent.bootstrap_cpu_grace", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.capacity", "kind": "problem", "title": "agent.capacity", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must be at least 1, or the host can never take a runner.", "fix": "Must be at least 1, or the host can never take a runner.", "verify": null, "setting": "agent.capacity", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.docker_build_cache_mb", "kind": "problem", "title": "agent.docker_build_cache_mb", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must be between 0 and 1048576 MiB; 0 disables automatic Docker builder-cache cleanup.", "fix": "Must be between 0 and 1048576 MiB; 0 disables automatic Docker builder-cache cleanup.", "verify": null, "setting": "agent.docker_build_cache_mb", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.extra_ca", "kind": "problem", "title": "agent.extra_ca", "category": "configuration", "severity": "info", "detection": "static", "detects": "Container runners on this host trust an extra CA as well as the image's own roots. Right for your proxy's CA; clear it otherwise.", "fix": "Container runners on this host trust an extra CA as well as the image's own roots. Right for your proxy's CA; clear it otherwise.", "verify": null, "setting": "agent.extra_ca_file", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.extra_ca_invalid", "kind": "problem", "title": "agent.extra_ca_invalid", "category": "configuration", "severity": "error", "detection": "static", "detects": "Not an absolute path to a readable PEM file with at least one certificate. Point it at your organisation's root CA in PEM form, or clear it.", "fix": "Not an absolute path to a readable PEM file with at least one certificate. Point it at your organisation's root CA in PEM form, or clear it.", "verify": null, "setting": "agent.extra_ca_file", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.finished_retention", "kind": "problem", "title": "agent.finished_retention", "category": "configuration", "severity": "error", "detection": "static", "detects": "Cannot be negative.", "fix": "Cannot be negative.", "verify": null, "setting": "agent.finished_retention", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.finished_retention_long", "kind": "problem", "title": "agent.finished_retention_long", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Finished runners stay on the host this long, holding disk and, for a non-ephemeral pool, whatever the job left behind.", "fix": "Finished runners stay on the host this long, holding disk and, for a non-ephemeral pool, whatever the job left behind.", "verify": null, "setting": "agent.finished_retention", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.heartbeat_interval_long", "kind": "problem", "title": "agent.heartbeat_interval_long", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Hosts heartbeat less often than the controller's timeout, so a healthy host will be counted lost.", "fix": "Hosts heartbeat less often than the controller's timeout, so a healthy host will be counted lost.", "verify": null, "setting": "agent.heartbeat_interval", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.insecure_http", "kind": "problem", "title": "agent.insecure_http", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The agent talks to the controller over plain HTTP, so its token and every runner's credentials cross the network in the clear.", "fix": "The agent talks to the controller over plain HTTP, so its token and every runner's credentials cross the network in the clear.", "verify": null, "setting": "agent.allow_insecure_http", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.insecure_tls", "kind": "problem", "title": "agent.insecure_tls", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The agent does not verify the controller's certificate, so anything on the path can impersonate it.", "fix": "The agent does not verify the controller's certificate, so anything on the path can impersonate it.", "verify": null, "setting": "agent.insecure_skip_verify", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.none", "kind": "problem", "title": "agent.none", "category": "configuration", "severity": "info", "detection": "static", "detects": "This controller hosts no runners itself, so at least one standalone agent has to join it.", "fix": "This controller hosts no runners itself, so at least one standalone agent has to join it.", "verify": null, "setting": "agent.embedded", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.prewarm_jitter", "kind": "problem", "title": "agent.prewarm_jitter", "category": "configuration", "severity": "error", "detection": "static", "detects": "Background preparation stagger is outside 0s–5m.", "fix": "Use 30s by default, or 0s to disable, and restart agents.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "agent.prewarm_timeout", "kind": "problem", "title": "agent.prewarm_timeout", "category": "configuration", "severity": "error", "detection": "static", "detects": "Background preparation budget is outside 1s–15m.", "fix": "Use 5m by default and restart agents.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "agent.process_backend", "kind": "problem", "title": "agent.process_backend", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The process backend gives jobs no container isolation: a job can read and write anything the agent user can.", "fix": "The process backend gives jobs no container isolation: a job can read and write anything the agent user can.", "verify": null, "setting": "agent.backend", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.process_root", "kind": "problem", "title": "agent.process_root", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The process backend is running as root, so every job is root on the host.", "fix": "The process backend is running as root, so every job is root on the host.", "verify": null, "setting": "agent.backend", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.root", "kind": "problem", "title": "agent.root", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The agent process is running as root. Raised only where an agent actually runs.", "fix": "The agent process is running as root. Raised only where an agent actually runs.", "verify": null, "setting": "agent", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.unverified_runner_download", "kind": "problem", "title": "agent.unverified_runner_download", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The process backend may install a runner archive whose checksum it could not confirm.", "fix": "The process backend may install a runner archive whose checksum it could not confirm.", "verify": null, "setting": "agent.allow_unverified_runner_download", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "agent.workdir", "kind": "problem", "title": "agent.workdir", "category": "configuration", "severity": "error", "detection": "static", "detects": "Empty.", "fix": "Empty.", "verify": null, "setting": "agent.work_dir", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "ai_context.artifact_quota", "kind": "problem", "title": "ai_context.artifact_quota", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "A repository's AI Context workflow built its context, but GitHub refused to store the file that carries it between the workflow's two jobs, because the account's Actions artifact storage quota is full. Public repositories are not affected; a private one shares its owner's quota with every other workflow that uploads artifacts, so what filled it is usually another workflow's builds. The entry says what the repository's own artifacts hold. GitHub recalculates usage only every six to twelve hours, so Zoomies starts the workflow again every six hours, three times for the commit, and no sooner.", "fix": "A repository's AI Context workflow built its context, but GitHub refused to store the file that carries it between the workflow's two jobs, because the account's Actions artifact storage quota is full. Public repositories are not affected; a private one shares its owner's quota with every other workflow that uploads artifacts, so what filled it is usually another workflow's builds. The entry says what the repository's own artifacts hold. GitHub recalculates usage only every six to twelve hours, so Zoomies starts the workflow again every six hours, three times for the commit, and no sooner.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "ai_context.artifact_upload_failed", "kind": "problem", "title": "ai_context.artifact_upload_failed", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The same hand-off between the workflow's jobs failed for a reason other than the quota; GitHub's message is on that step of the run. Zoomies starts the workflow again, half an hour after it ended, twice.", "fix": "The same hand-off between the workflow's jobs failed for a reason other than the quota; GitHub's message is on that step of the run. Zoomies starts the workflow again, half an hour after it ended, twice.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "ai_context.delivery_failed", "kind": "problem", "title": "ai_context.delivery_failed", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "A Zoomies-only upload built its context but did not reach the controller. The controller must be reachable from GitHub's runners over https at its upload address and accept GitHub's identity token for the repository. Zoomies starts the workflow again half an hour after, twice.", "fix": "A Zoomies-only upload built its context but did not reach the controller. The controller must be reachable from GitHub's runners over https at its upload address and accept GitHub's identity token for the repository. Zoomies starts the workflow again half an hour after, twice.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "ai_context.generation_refused", "kind": "problem", "title": "ai_context.generation_refused", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "The generator stopped on one of its safety checks: the managed configuration no longer matches what was reviewed, the checkout was not the commit, nothing eligible was left, or the secret scan changed what it would carry. A new run would give the same answer, so Zoomies does not start one; **Reinstall / repair** re-reviews the configuration.", "fix": "The generator stopped on one of its safety checks: the managed configuration no longer matches what was reviewed, the checkout was not the commit, nothing eligible was left, or the secret scan changed what it would carry. A new run would give the same answer, so Zoomies does not start one; **Reinstall / repair** re-reviews the configuration.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "ai_context.oversized_file", "kind": "problem", "title": "ai_context.oversized_file", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "The repository's AI Context workflow was written by an earlier Zoomies release, which stops the whole run when any file is over 1 MiB. Current releases list such a file as omitted and carry on. A new run would fail the same way, so Zoomies does not start one; **Reinstall / repair** moves the workflow to the current generator.", "fix": "The repository's AI Context workflow was written by an earlier Zoomies release, which stops the whole run when any file is over 1 MiB. Current releases list such a file as omitted and carry on. A new run would fail the same way, so Zoomies does not start one; **Reinstall / repair** moves the workflow to the current generator.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "ai_context.publication_refused", "kind": "problem", "title": "ai_context.publication_refused", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "The context was built, but the workflow's last job could not update the `zoomies-ai-context` branch: usually a token that cannot write contents, a ruleset that blocks that branch, files on it that Zoomies does not own, or the trusted branch moving on during the run. Zoomies starts the workflow again once, half an hour after.", "fix": "The context was built, but the workflow's last job could not update the `zoomies-ai-context` branch: usually a token that cannot write contents, a ruleset that blocks that branch, files on it that Zoomies does not own, or the trusted branch moving on during the run. Zoomies starts the workflow again once, half an hour after.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "ai_context.run_failed", "kind": "problem", "title": "ai_context.run_failed", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The AI Context run failed in a way Zoomies does not recognise; the entry links the run. Zoomies starts the workflow again half an hour after, twice.", "fix": "The AI Context run failed in a way Zoomies does not recognise; the entry links the run. Zoomies starts the workflow again half an hour after, twice.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "ai_context.runner_unavailable", "kind": "problem", "title": "ai_context.runner_unavailable", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The AI Context run was queued but GitHub never gave its job a hosted runner, so no step ran and the job was cancelled when it timed out. It is GitHub's side and normally passes: Zoomies starts the workflow again half an hour after it ended, up to four times.", "fix": "The AI Context run was queued but GitHub never gave its job a hosted runner, so no step ran and the job was cancelled when it timed out. It is GitHub's side and normally passes: Zoomies starts the workflow again half an hour after it ended, up to four times.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "ai_context.setup_failed", "kind": "problem", "title": "ai_context.setup_failed", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "A step before generation failed: checking out the repository, installing Node, or installing the pinned generator from the npm registry. Almost always an outage at GitHub or npm; Zoomies starts the workflow again half an hour after, twice.", "fix": "A step before generation failed: checking out the repository, installing Node, or installing the pinned generator from the npm registry. Almost always an outage at GitHub or npm; Zoomies starts the workflow again half an hour after, twice.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "ai_context.startup_failed", "kind": "problem", "title": "ai_context.startup_failed", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "GitHub refused to start the AI Context run at all: it found the workflow file invalid, or the account's billing or the organisation's Actions policy does not allow it to run. Zoomies tries again six hours after, twice.", "fix": "GitHub refused to start the AI Context run at all: it found the workflow file invalid, or the account's billing or the organisation's Actions policy does not allow it to run. Zoomies tries again six hours after, twice.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "ai_context.timed_out", "kind": "problem", "title": "ai_context.timed_out", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "An AI Context job started and reached its time limit: fifteen minutes to generate, ten to publish or upload. Zoomies starts the workflow again half an hour after, twice; if it keeps happening, add exclusions for large directories.", "fix": "An AI Context job started and reached its time limit: fifteen minutes to generate, ten to publish or upload. Zoomies starts the workflow again half an hour after, twice; if it keeps happening, add exclusions for large directories.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "ai_context.too_much_source", "kind": "problem", "title": "ai_context.too_much_source", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "The generator refused to build a snapshot because the repository's eligible source is over a size or file-count limit even with the largest files left out. Zoomies does not start the workflow again; add exclusions in the repository's AI Context settings.", "fix": "The generator refused to build a snapshot because the repository's eligible source is over a size or file-count limit even with the largest files left out. Zoomies does not start the workflow again; add exclusions in the repository's AI Context settings.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "auth.cookie_insecure", "kind": "problem", "title": "auth.cookie_insecure", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Session cookies go out without `Secure`, so one plain-HTTP request to this host hands over a live session. Set an https `server.external_url`, or `security.cookie_secure` if TLS is terminated in front.", "fix": "Session cookies go out without `Secure`, so one plain-HTTP request to this host hands over a live session. Set an https `server.external_url`, or `security.cookie_secure` if TLS is terminated in front.", "verify": null, "setting": "security.cookie_secure", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "auth.disabled", "kind": "problem", "title": "auth.disabled", "category": "configuration", "severity": "error wherever the controller is reachable, warning on loopback with nothing in front", "detection": "static", "detects": "Every request is treated as an administrator. Acceptable for local development; never on a host others can reach. An external URL or a trusted proxy counts as reachable, because a loopback bind behind a proxy is not private.", "fix": "Every request is treated as an administrator. Acceptable for local development; never on a host others can reach. An external URL or a trusted proxy counts as reachable, because a loopback bind behind a proxy is not private.", "verify": null, "setting": "security.disable_auth", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "auth.no_login_limit", "kind": "problem", "title": "auth.no_login_limit", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Password guessing is not rate limited.", "fix": "Password guessing is not rate limited.", "verify": null, "setting": "security.rate_limit_logins", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "auth.session_ttl", "kind": "problem", "title": "auth.session_ttl", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must be positive.", "fix": "Must be positive.", "verify": null, "setting": "security.session_ttl", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "auth.session_ttl_long", "kind": "problem", "title": "auth.session_ttl_long", "category": "configuration", "severity": "warning", "detection": "static", "detects": "A stolen session cookie stays useful for this long.", "fix": "A stolen session cookie stays useful for this long.", "verify": null, "setting": "security.session_ttl", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "backup.failed", "kind": "problem", "title": "backup.failed", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The scheduled backup is failing, and the entry carries the error. Usually the directory `backup.directory` names is missing, not writable by the controller, or out of room for a copy of the database. The controller tries again every fifteen minutes; taking one from the Backups tab shows the same error in the page.", "fix": "The scheduled backup is failing, and the entry carries the error. Usually the directory `backup.directory` names is missing, not writable by the controller, or out of room for a copy of the database. The controller tries again every fifteen minutes; taking one from the Backups tab shows the same error in the page.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "backup.no_remote", "kind": "problem", "title": "backup.no_remote", "category": "configuration", "severity": "info", "detection": "static", "detects": "Backups are taken on a schedule and nothing in `zoomies.yaml` says where a copy goes. That is a backup against a mistake, not against the disk, the machine or the datacentre. Add a destination on the Backups page or under `backup.remotes`, or keep shipping the directory yourself; the point of the entry is that one of the three is somebody's job. It is raised from the file, so the Configuration page drops it once the fleet has a destination stored from the Backups page, which the validator cannot see.", "fix": "Backups are taken on a schedule and nothing in `zoomies.yaml` says where a copy goes. That is a backup against a mistake, not against the disk, the machine or the datacentre. Add a destination on the Backups page or under `backup.remotes`, or keep shipping the directory yourself; the point of the entry is that one of the three is somebody's job. It is raised from the file, so the Configuration page drops it once the fleet has a destination stored from the Backups page, which the validator cannot see.", "verify": null, "setting": "backup.remotes", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "backup.remote_credentials", "kind": "problem", "title": "backup.remote_credentials", "category": "configuration", "severity": "error", "detection": "static", "detects": "A remote has no access key or no secret key, so every request to it would be refused and the copies would pile up unsent.", "fix": "A remote has no access key or no secret key, so every request to it would be refused and the copies would pile up unsent.", "verify": null, "setting": "backup.remotes", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "backup.remote_duplicate", "kind": "problem", "title": "backup.remote_duplicate", "category": "configuration", "severity": "error", "detection": "static", "detects": "Two remotes share a name, so one of them would be unaddressable by the Backups tab, the log and the problems drawer alike.", "fix": "Two remotes share a name, so one of them would be unaddressable by the Backups tab, the log and the problems drawer alike.", "verify": null, "setting": "backup.remotes", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "backup.remote_endpoint", "kind": "problem", "title": "backup.remote_endpoint", "category": "configuration", "severity": "error", "detection": "static", "detects": "A remote's endpoint is not an HTTP URL. Write the service's own, such as `https://s3.eu-west-2.amazonaws.com` or `http://minio:9000`; the scheme is what decides whether the connection is encrypted.", "fix": "A remote's endpoint is not an HTTP URL. Write the service's own, such as `https://s3.eu-west-2.amazonaws.com` or `http://minio:9000`; the scheme is what decides whether the connection is encrypted.", "verify": null, "setting": "backup.remotes", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "backup.remote_failed", "kind": "problem", "title": "backup.remote_failed", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "Backups are not reaching one of the destinations under `backup.remotes`, and the entry carries the service's own refusal: a wrong secret, a bucket that is not there, a policy that does not allow writing, or a clock too far from the service's to sign with. The controller tries again every fifteen minutes and the Backups tab tests the remote on demand. The copies on this host are unaffected; they are simply all there is.", "fix": "Backups are not reaching one of the destinations under `backup.remotes`, and the entry carries the service's own refusal: a wrong secret, a bucket that is not there, a policy that does not allow writing, or a clock too far from the service's to sign with. The controller tries again every fifteen minutes and the Backups tab tests the remote on demand. The copies on this host are unaffected; they are simply all there is.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "backup.remote_incomplete", "kind": "problem", "title": "backup.remote_incomplete", "category": "configuration", "severity": "error", "detection": "static", "detects": "A remote has no endpoint or no bucket, so nothing would be copied to it and nothing would say so. Finish it, or set `disabled: true` until it is ready.", "fix": "A remote has no endpoint or no bucket, so nothing would be copied to it and nothing would say so. Finish it, or set `disabled: true` until it is ready.", "verify": null, "setting": "backup.remotes", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "backup.remote_insecure", "kind": "problem", "title": "backup.remote_insecure", "category": "configuration", "severity": "warning", "detection": "static", "detects": "A destination is reached over plain HTTP, so the access key, the signature and the backup itself cross the network in the clear, anyone on the path can read the fleet and write to the bucket afterwards. Use `https://` unless the endpoint is on this host or a network you own end to end. Raised by the validator for a destination in the file and by the controller for one added on the Backups page; loopback is not warned about.", "fix": "A destination is reached over plain HTTP, so the access key, the signature and the backup itself cross the network in the clear, anyone on the path can read the fleet and write to the bucket afterwards. Use `https://` unless the endpoint is on this host or a network you own end to end. Raised by the validator for a destination in the file and by the controller for one added on the Backups page; loopback is not warned about.", "verify": null, "setting": "backup.remotes", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "backup.remote_name", "kind": "problem", "title": "backup.remote_name", "category": "configuration", "severity": "error", "detection": "static", "detects": "A remote's name is not usable: it is a path component in the API and a word in a log line, so it is lower-case letters, digits and dashes.", "fix": "A remote's name is not usable: it is a path component in the API and a word in a log line, so it is lower-case letters, digits and dashes.", "verify": null, "setting": "backup.remotes", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "backup.remote_passphrase_short", "kind": "problem", "title": "backup.remote_passphrase_short", "category": "configuration", "severity": "warning", "detection": "static", "detects": "A remote's passphrase is shorter than the eight characters the Backups tab's encrypted download accepts. The archive is only as private as this, and it is typed once, into a file.", "fix": "A remote's passphrase is shorter than the eight characters the Backups tab's encrypted download accepts. The archive is only as private as this, and it is typed once, into a file.", "verify": null, "setting": "backup.remotes", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "backup.remote_plaintext", "kind": "problem", "title": "backup.remote_plaintext", "category": "configuration", "severity": "warning", "detection": "static", "detects": "A destination is sent the backup unencrypted. A backup is the whole fleet (every repository and job it has seen, every account, and the sealed GitHub App credentials) and in a bucket it is a file anyone who can read the bucket can open. Set a passphrase on it and keep that wherever you keep the encryption key; nothing here can recover a lost one. Raised about a destination in the file and about one added on the page alike.", "fix": "A destination is sent the backup unencrypted. A backup is the whole fleet (every repository and job it has seen, every account, and the sealed GitHub App credentials) and in a bucket it is a file anyone who can read the bucket can open. Set a passphrase on it and keep that wherever you keep the encryption key; nothing here can recover a lost one. Raised about a destination in the file and about one added on the page alike.", "verify": null, "setting": "backup.remotes", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "backup.remote_shadowed", "kind": "problem", "title": "backup.remote_shadowed", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "A destination stored from the Backups page has the same name as one `zoomies.yaml` or the environment describes, and the file has the last word, so nothing is sent to the stored one and its settings are not the ones in use. Rename one of them, or delete the stored destination and keep describing it in the file.", "fix": "A destination stored from the Backups page has the same name as one `zoomies.yaml` or the environment describes, and the file has the last word, so nothing is sent to the stored one and its settings are not the ones in use. Rename one of them, or delete the stored destination and keep describing it in the file.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "backup.remote_unreadable", "kind": "problem", "title": "backup.remote_unreadable", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "A stored destination's sealed secret key or passphrase does not open with this controller's encryption key, so nothing can be sent to it. This is what a database restored onto a host with a different key looks like: put the key this fleet was sealed with back, or open the destination on the Backups page and enter its secret key again.", "fix": "A stored destination's sealed secret key or passphrase does not open with this controller's encryption key, so nothing can be sent to it. This is what a database restored onto a host with a different key looks like: put the key this fleet was sealed with back, or open the destination on the Backups page and enter its secret key again.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "backup.restore_failed", "kind": "problem", "title": "backup.restore_failed", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "The last staged restore did not finish; the entry says why, and which database the controller started on: the one it already had when the restore was refused before anything moved, or the restored one, fenced or not, when it failed after the copy. Put right what the reason names and stage the restore again; the entry clears when it is dismissed from the Backups tab.", "fix": "The last staged restore did not finish; the entry says why, and which database the controller started on: the one it already had when the restore was refused before anything moved, or the restored one, fenced or not, when it failed after the copy. Put right what the reason names and stage the restore again; the entry clears when it is dismissed from the Backups tab.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "backup.restore_staged", "kind": "problem", "title": "backup.restore_staged", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "An administrator has staged a restore from the Backups tab, and it is waiting for the controller to restart. Nothing has changed yet: the database is swapped when the next controller starts, before it opens anything, and the restored fleet comes back fenced. Restart the controller from the tab to apply it, or cancel it there.", "fix": "An administrator has staged a restore from the Backups tab, and it is waiting for the controller to restart. Nothing has changed yet: the database is swapped when the next controller starts, before it opens anything, and the restored fleet comes back fenced. Restart the controller from the tab to apply it, or cancel it there.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "bind.empty", "kind": "problem", "title": "bind.empty", "category": "configuration", "severity": "error", "detection": "static", "detects": "Give it a `host:port`. Nothing can start without one.", "fix": "Give it a `host:port`. Nothing can start without one.", "verify": null, "setting": "server.bind", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "bind.malformed", "kind": "problem", "title": "bind.malformed", "category": "configuration", "severity": "error", "detection": "static", "detects": "It is not a `host:port` address. A bare port needs the colon: `:8080`.", "fix": "It is not a `host:port` address. A bare port needs the colon: `:8080`.", "verify": null, "setting": "server.bind", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "bind.public_no_tls", "kind": "problem", "title": "bind.public_no_tls", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Sessions and API tokens cross the network in the clear. Put TLS in front of it, or bind to loopback and tunnel.", "fix": "Sessions and API tokens cross the network in the clear. Put TLS in front of it, or bind to loopback and tunnel.", "verify": null, "setting": "server.bind", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "bootstrap.ignored", "kind": "problem", "title": "bootstrap.ignored", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The bootstrap variables are set on an instance that already has accounts, so this start ignored them; they only ever create the first account. The file they name is a credential kept for nothing: remove the variables and delete it.", "fix": "The bootstrap variables are set on an instance that already has accounts, so this start ignored them; they only ever create the first account. The file they name is a credential kept for nothing: remove the variables and delete it.", "verify": null, "setting": "-", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "bootstrap.incomplete", "kind": "problem", "title": "bootstrap.incomplete", "category": "configuration", "severity": "error", "detection": "static", "detects": "`ZOOMIES_BOOTSTRAP_ADMIN` needs exactly one of `ZOOMIES_BOOTSTRAP_PASSWORD_FILE` or `ZOOMIES_BOOTSTRAP_TOKEN_FILE`, and neither file means anything without it. Set all that is missing, or unset all three and use the setup token.", "fix": "`ZOOMIES_BOOTSTRAP_ADMIN` needs exactly one of `ZOOMIES_BOOTSTRAP_PASSWORD_FILE` or `ZOOMIES_BOOTSTRAP_TOKEN_FILE`, and neither file means anything without it. Set all that is missing, or unset all three and use the setup token.", "verify": null, "setting": "-", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "capacity_demand.cooldown", "kind": "problem", "title": "capacity_demand.cooldown", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must be positive.", "fix": "Must be positive.", "verify": null, "setting": "capacity_demand.cooldown", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "capacity_demand.delivery_failed", "kind": "problem", "title": "capacity_demand.delivery_failed", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "An external capacity provisioner did not accept the latest event, after its retries.", "fix": "An external capacity provisioner did not accept the latest event, after its retries.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "capacity_demand.secret", "kind": "problem", "title": "capacity_demand.secret", "category": "configuration", "severity": "error", "detection": "static", "detects": "Empty, so deliveries could not be signed and a receiver could not tell them from anyone else's.", "fix": "Empty, so deliveries could not be signed and a receiver could not tell them from anyone else's.", "verify": null, "setting": "capacity_demand.signing_secret", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "capacity_demand.timeout", "kind": "problem", "title": "capacity_demand.timeout", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must be positive.", "fix": "Must be positive.", "verify": null, "setting": "capacity_demand.timeout", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "capacity_demand.url", "kind": "problem", "title": "capacity_demand.url", "category": "configuration", "severity": "error", "detection": "static", "detects": "Not an absolute HTTP URL.", "fix": "Not an absolute HTTP URL.", "verify": null, "setting": "capacity_demand.destination_url", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "controller.development_update_available", "kind": "problem", "title": "controller.development_update_available", "category": "capacity", "severity": "info", "detection": "runtime", "detects": "The controller runs the moving development channel but its stamped commit is not the current head of `main`. A main CI image publication may still be running, or failed or cancelled before advancing `:dev`; an upgrade can only pull what was successfully published. Let main CI publish, then run `zoomies upgrade` again.", "fix": "The controller runs the moving development channel but its stamped commit is not the current head of `main`. A main CI image publication may still be running, or failed or cancelled before advancing `:dev`; an upgrade can only pull what was successfully published. Let main CI publish, then run `zoomies upgrade` again.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "controller.lease_lost", "kind": "problem", "title": "controller.lease_lost", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "Another controller has taken this database's lease, so two are running against it. Both are scheduling, and they will mint runners against each other and remove each other's workloads. Stop one; the survivor restarts with `--takeover`.", "fix": "Another controller has taken this database's lease, so two are running against it. Both are scheduling, and they will mint runners against each other and remove each other's workloads. Stop one; the survivor restarts with `--takeover`.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "controller.loop_panicked", "kind": "problem", "title": "controller.loop_panicked", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "A background loop panicked and was restarted. The fleet keeps running, but this is a bug: the stack is in the log, and it is worth reporting.", "fix": "A background loop panicked and was restarted. The fleet keeps running, but this is a bug: the stack is in the log, and it is worth reporting.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "controller.problems_partial", "kind": "problem", "title": "controller.problems_partial", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "One of the queries behind this list failed, so the list is incomplete and the entry names which sections are missing from it. It exists because the alternative is worse: this page used to return a 500 for any one failing query, and an operator whose drawer will not load reads that as a fleet with nothing wrong. The controller log carries the query that failed.", "fix": "One of the queries behind this list failed, so the list is incomplete and the entry names which sections are missing from it. It exists because the alternative is worse: this page used to return a 500 for any one failing query, and an operator whose drawer will not load reads that as a fleet with nothing wrong. The controller log carries the query that failed.", "verify": null, "status_sentence": "The controller could not check everything, so this status may be incomplete.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "controller.update_available", "kind": "problem", "title": "controller.update_available", "category": "capacity", "severity": "info", "detection": "runtime", "detects": "A newer release of Zoomies has been published than the one this controller was built from. Nothing is wrong and nothing updates itself: runners, pools and jobs do not depend on the controller's version. It appears only on a controller built from a release tag, and `updates.check_interval: 0` switches both the check and this entry off.", "fix": "A newer release of Zoomies has been published than the one this controller was built from. Nothing is wrong and nothing updates itself: runners, pools and jobs do not depend on the controller's version. It appears only on a controller built from a release tag, and `updates.check_interval: 0` switches both the check and this entry off.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "crypto.key_in_config", "kind": "problem", "title": "crypto.key_in_config", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The key is in the config file, so anything that reads the file (a backup, a support bundle) reads every stored secret. Point at a file instead.", "fix": "The key is in the config file, so anything that reads the file (a backup, a support bundle) reads every stored secret. Point at a file instead.", "verify": null, "setting": "security.encryption_key", "docs_html": "https://zoomies.sh/problem-codes/#configuration-storage-and-secrets", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-storage-and-secrets" }, { "id": "crypto.key_mismatch", "kind": "problem", "title": "crypto.key_mismatch", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "This instance's encryption key does not open the GitHub App credentials in its own database. Every installation fails at once and nothing can authenticate to GitHub, which is what tells this apart from `installation.unhealthy`; that one is fixed on GitHub, this one by putting the right key file back. It is what a restore that brought the database and left the key behind looks like once the instance is running; the startup refusal catches the case where the key file is missing entirely.", "fix": "This instance's encryption key does not open the GitHub App credentials in its own database. Every installation fails at once and nothing can authenticate to GitHub, which is what tells this apart from `installation.unhealthy`; that one is fixed on GitHub, this one by putting the right key file back. It is what a restore that brought the database and left the key behind looks like once the instance is running; the startup refusal catches the case where the key file is missing entirely.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "crypto.no_key", "kind": "problem", "title": "crypto.no_key", "category": "configuration", "severity": "warning", "detection": "static", "detects": "One will be generated on first start, and it is the only copy. Back it up: without it the stored GitHub App private key and webhook secrets cannot be decrypted.", "fix": "One will be generated on first start, and it is the only copy. Back it up: without it the stored GitHub App private key and webhook secrets cannot be decrypted.", "verify": null, "setting": "security.encryption_key_file", "docs_html": "https://zoomies.sh/problem-codes/#configuration-storage-and-secrets", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-storage-and-secrets" }, { "id": "db.parent_not_dir", "kind": "problem", "title": "db.parent_not_dir", "category": "configuration", "severity": "error", "detection": "static", "detects": "The parent exists and is not a directory.", "fix": "The parent exists and is not a directory.", "verify": null, "setting": "database.path", "docs_html": "https://zoomies.sh/problem-codes/#configuration-storage-and-secrets", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-storage-and-secrets" }, { "id": "db.path_missing", "kind": "problem", "title": "db.path_missing", "category": "configuration", "severity": "error", "detection": "static", "detects": "Empty.", "fix": "Empty.", "verify": null, "setting": "database.path", "docs_html": "https://zoomies.sh/problem-codes/#configuration-storage-and-secrets", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-storage-and-secrets" }, { "id": "dind.expected", "kind": "problem", "title": "dind.expected", "category": "configuration", "severity": "info", "detection": "static", "detects": "This fleet has said that a pool giving its jobs their own Docker daemon is expected, so that pool no longer raises `pool.dangerous` for the privileged container the daemon runs in. Nothing about the runners changed; a warning a fleet has already decided about, repeated for every such pool on every pass, is what teaches an operator to stop reading the list. The host Docker socket and persistent runners still warn. Turn the setting off to be told again.", "fix": "This fleet has said that a pool giving its jobs their own Docker daemon is expected, so that pool no longer raises `pool.dangerous` for the privileged container the daemon runs in. Nothing about the runners changed; a warning a fleet has already decided about, repeated for every such pool on every pass, is what teaches an operator to stop reading the list. The host Docker socket and persistent runners still warn. Turn the setting off to be told again.", "verify": null, "setting": "security.docker_in_docker_expected", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "egress.private_allowed", "kind": "problem", "title": "egress.private_allowed", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The switch that lets every URL the controller dials name this machine, a link-local address or a private network is on, so nothing checks them any more and `egress.private_target` stays silent. It is a warning, not information, because it removes the one guard between somebody with settings rights and this machine's neighbourhood; on a LAN install that needs it, the warning is the acknowledgement, and it never stops startup. Turn the setting off unless the identity provider, Enterprise Server, provider or backup remote really lives on a network you own. See [security](security.md#securityallow_private_egress-true).", "fix": "The switch that lets every URL the controller dials name this machine, a link-local address or a private network is on, so nothing checks them any more and `egress.private_target` stays silent. It is a warning, not information, because it removes the one guard between somebody with settings rights and this machine's neighbourhood; on a LAN install that needs it, the warning is the acknowledgement, and it never stops startup. Turn the setting off unless the identity provider, Enterprise Server, provider or backup remote really lives on a network you own. See [security](security.md#securityallow_private_egress-true).", "verify": null, "setting": "security.allow_private_egress", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "egress.private_target", "kind": "problem", "title": "egress.private_target", "category": "configuration", "severity": "warning", "detection": "static", "detects": "A URL the controller dials, `oidc.issuer` (when OIDC is on), `github.api_base_url`, `capacity_demand.destination_url`, `agent.runner_download_url` or a backup remote's endpoint in the file, names this machine, a link-local address such as the cloud metadata service at `169.254.169.254`, or a private range (RFC 1918, carrier-grade NAT, IPv6 unique-local), in any spelling that reaches one. At startup it is a warning and never stops the controller: that value came from whoever runs the process, and an upgrade must not stop an install that works. Written through the API instead, `PATCH /settings`, a settings import (the preview marks the row), a backup remote, a direct provider or an installation's own API base URL, the same sentence is a 422 on that field. Use the service's public address, or, when it really lives on a network you own, set `security.allow_private_egress`. See [security](security.md#securityallow_private_egress-true).", "fix": "A URL the controller dials, `oidc.issuer` (when OIDC is on), `github.api_base_url`, `capacity_demand.destination_url`, `agent.runner_download_url` or a backup remote's endpoint in the file, names this machine, a link-local address such as the cloud metadata service at `169.254.169.254`, or a private range (RFC 1918, carrier-grade NAT, IPv6 unique-local), in any spelling that reaches one. At startup it is a warning and never stops the controller: that value came from whoever runs the process, and an upgrade must not stop an install that works. Written through the API instead, `PATCH /settings`, a settings import (the preview marks the row), a backup remote, a direct provider or an installation's own API base URL, the same sentence is a 422 on that field. Use the service's public address, or, when it really lives on a network you own, set `security.allow_private_egress`. See [security](security.md#securityallow_private_egress-true).", "verify": null, "setting": "the URL's own key", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "external_url.insecure", "kind": "problem", "title": "external_url.insecure", "category": "configuration", "severity": "warning", "detection": "static", "detects": "GitHub will deliver webhooks over plaintext, so the payloads and their signatures cross the internet unencrypted.", "fix": "GitHub will deliver webhooks over plaintext, so the payloads and their signatures cross the internet unencrypted.", "verify": null, "setting": "server.external_url", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "external_url.malformed", "kind": "problem", "title": "external_url.malformed", "category": "configuration", "severity": "error", "detection": "static", "detects": "Not an absolute URL.", "fix": "Not an absolute URL.", "verify": null, "setting": "server.external_url", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "external_url.missing", "kind": "problem", "title": "external_url.missing", "category": "configuration", "severity": "warning", "detection": "static", "detects": "GitHub cannot be told where to deliver webhooks, so scaling falls back to the poller and reacts in tens of seconds.", "fix": "GitHub cannot be told where to deliver webhooks, so scaling falls back to the poller and reacts in tens of seconds.", "verify": null, "setting": "server.external_url", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "github.api_base_malformed", "kind": "problem", "title": "github.api_base_malformed", "category": "configuration", "severity": "error", "detection": "static", "detects": "Not an absolute URL.", "fix": "Not an absolute URL.", "verify": null, "setting": "github.api_base_url", "docs_html": "https://zoomies.sh/problem-codes/#configuration-github", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-github" }, { "id": "github.api_base_missing", "kind": "problem", "title": "github.api_base_missing", "category": "configuration", "severity": "error", "detection": "static", "detects": "Empty. Leave it unset for github.com, or give GitHub Enterprise Server's API base.", "fix": "Empty. Leave it unset for github.com, or give GitHub Enterprise Server's API base.", "verify": null, "setting": "github.api_base_url", "docs_html": "https://zoomies.sh/problem-codes/#configuration-github", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-github" }, { "id": "github.toolchain_scan_negative", "kind": "problem", "title": "github.toolchain_scan_negative", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must not be negative. Use a duration such as `24h`, or 0 to scan only when asked.", "fix": "Must not be negative. Use a duration such as `24h`, or 0 to scan only when asked.", "verify": null, "setting": "github.toolchain_scan_interval", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "github.toolchain_scan_too_fast", "kind": "problem", "title": "github.toolchain_scan_too_fast", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Every workflow in every repository is read more often than hourly. Each scan spends calls from the GitHub quota the scheduler uses to find queued jobs, and workflows change far less often than that.", "fix": "Every workflow in every repository is read more often than hourly. Each scan spends calls from the GitHub quota the scheduler uses to find queued jobs, and workflows change far less often than that.", "verify": null, "setting": "github.toolchain_scan_interval", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "host.auto_pool_skipped", "kind": "problem", "title": "host.auto_pool_skipped", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "A host is in none of the pools the controller keeps for each architecture and size class, for a reason you can change: its `size` tag is not small, medium or large, its architecture is neither amd64 nor arm64, or it offers neither the Docker nor the Podman backend an automatic pool uses. A cordoned host and a host that has gone quiet are not reported: both are things a fleet does, and both end by themselves. Fix the tag or the host's runtime, or leave the host to the pools you made.", "fix": "A host is in none of the pools the controller keeps for each architecture and size class, for a reason you can change: its `size` tag is not small, medium or large, its architecture is neither amd64 nor arm64, or it offers neither the Docker nor the Podman backend an automatic pool uses. A cordoned host and a host that has gone quiet are not reported: both are things a fleet does, and both end by themselves. Fix the tag or the host's runtime, or leave the host to the pools you made.", "verify": "`zoomies auto-pools` shows the host in a class's pool and the entry leaves the list.", "status_sentence": "A machine is in none of the pools the fleet keeps for its size, so it takes no work from them.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "host.cordoned_with_work", "kind": "problem", "title": "host.cordoned_with_work", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "A cordoned host could run jobs that are queued. Cordoning is deliberate, so this is a reminder rather than a fault.", "fix": "A cordoned host could run jobs that are queued. Cordoning is deliberate, so this is a reminder rather than a fault.", "verify": "The entry leaves the list once the host is uncordoned or the queued jobs have been taken by another host.", "status_sentence": "A machine is being taken out of service once its current jobs finish.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.duplicate_agent", "kind": "problem", "title": "host.duplicate_agent", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "Two agent sessions have used this host's credentials in turn. An agent takes a new session each time it starts and never returns to an old one, so alternation means a second agent holds a copy of the token, usually a cloned VM or a copied state directory. Neither is refused: both are running real jobs, and picking one would end the other's.", "fix": "Two agent sessions have used this host's credentials in turn. An agent takes a new session each time it starts and never returns to an old one, so alternation means a second agent holds a copy of the token, usually a cloned VM or a copied state directory. Neither is refused: both are running real jobs, and picking one would end the other's.", "verify": "After the copy is stopped and the token rotated, the host's sessions stop alternating and the entry leaves the list within a few heartbeats.", "status_sentence": "Two machines are claiming to be the same one, so jobs may not be placed on it.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.health_stale", "kind": "problem", "title": "host.health_stale", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "A connected host that has sent an OS report before has sent none for ten minutes, so what its page shows may no longer be true. Its collector has most likely stopped, or its clock is wrong, because the age is judged by the report's own timestamp. The host page greys a badge after three minutes; this is raised only after ten. A container's partial report is produced afresh every minute and is not judged. It clears with the next report. It never reaches the public status page.", "fix": "A connected host that has sent an OS report before has sent none for ten minutes, so what its page shows may no longer be true. Its collector has most likely stopped, or its clock is wrong, because the age is judged by the report's own timestamp. The host page greys a badge after three minutes; this is raised only after ten. A container's partial report is produced afresh every minute and is not judged. It clears with the next report. It never reaches the public status page.", "verify": "The host page's report age drops below three minutes and the entry leaves the list with the next report.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.image_pull_failed", "kind": "problem", "title": "host.image_pull_failed", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "A runner start or prewarm on a host failed because its pool's image could not be made ready. The entry names the host, the pool, the image and the registry it comes from (`docker.io` for a bare name, otherwise the reference's host with its port) so a host whose egress blocks `ghcr.io` says so instead of showing runners that never register. Check the host can reach the registry and is logged in to it for a private image (`docker pull` on the host gives the daemon's own answer), and that the tag exists. The command is offered only for an image that is shaped like one, and the error and the names in the entry are shown as prose, so text an agent reported cannot put a command of its own in front of you. Cleared by the next start or prewarm of the same pool that succeeds on that host.", "fix": "A runner start or prewarm on a host failed because its pool's image could not be made ready. The entry names the host, the pool, the image and the registry it comes from (`docker.io` for a bare name, otherwise the reference's host with its port) so a host whose egress blocks `ghcr.io` says so instead of showing runners that never register. Check the host can reach the registry and is logged in to it for a private image (`docker pull` on the host gives the daemon's own answer), and that the tag exists. The command is offered only for an image that is shaped like one, and the error and the names in the entry are shown as prose, so text an agent reported cannot put a command of its own in front of you. Cleared by the next start or prewarm of the same pool that succeeds on that host.", "verify": "The next runner start on the host succeeds: `zoomies runners list --host \u003chost-id\u003e` shows one past `provisioning`, and the entry leaves the list.", "status_sentence": "A machine cannot download the runner image, so jobs that need it may wait.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.limits_unenforceable", "kind": "problem", "title": "host.limits_unenforceable", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "A host's Docker or Podman daemon has said it cannot apply a CPU quota, a memory limit or a pids limit. A CPU quota it cannot apply is refused at create, so a pool that sets one fails every runner it starts there; a memory limit is dropped, so the pool's limit binds nothing; a pids limit is ignored. No default is given on that field either way. On a rootless daemon the fix is to delegate the controllers to its user (`Delegate=cpu cpuset io memory pids` in a drop-in for the user slice); on a root daemon it is cgroup v2, or a kernel built with the controller that is missing.", "fix": "A host's Docker or Podman daemon has said it cannot apply a CPU quota, a memory limit or a pids limit. A CPU quota it cannot apply is refused at create, so a pool that sets one fails every runner it starts there; a memory limit is dropped, so the pool's limit binds nothing; a pids limit is ignored. No default is given on that field either way. On a rootless daemon the fix is to delegate the controllers to its user (`Delegate=cpu cpuset io memory pids` in a drop-in for the user slice); on a root daemon it is cgroup v2, or a kernel built with the controller that is missing.", "verify": "Runners start on the host with their limits applied: a runner's resource sample on its page names a quota, and the entry leaves the list.", "status_sentence": "A machine cannot hold runners to their size, so jobs may run slower than usual.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.limits_unverified", "kind": "problem", "title": "host.limits_unverified", "category": "reliability", "severity": "info", "detection": "runtime", "detects": "A host's agent predates the limits probe, so the controller does not know whether its daemon can apply a limit and gives its runners no default CPU or memory, because a limit sent to a daemon that cannot apply it fails the create. Pools that set their own limits are unaffected. Upgrade the agent: the probe arrives with the next heartbeat, with no re-join, and defaults start on the next runner. Raised only while `scheduler.default_runner_limits` is on, since with it off an unverified probe changes nothing.", "fix": "A host's agent predates the limits probe, so the controller does not know whether its daemon can apply a limit and gives its runners no default CPU or memory, because a limit sent to a daemon that cannot apply it fails the create. Pools that set their own limits are unaffected. Upgrade the agent: the probe arrives with the next heartbeat, with no re-join, and defaults start on the next runner. Raised only while `scheduler.default_runner_limits` is on, since with it off an unverified probe changes nothing.", "verify": "After the agent is upgraded, the host page reports whether its daemon applies limits and the entry leaves the list.", "status_sentence": "A machine has not yet confirmed it can hold runners to their size.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.memory_pool_exhausted", "kind": "problem", "title": "host.memory_pool_exhausted", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "A runner on this host, of a pool set to lend, needed more memory than it was created with and the [memory valve](elastic-memory.md) had none to lend: either everything no runner's guarantee needs was already lent, or what was left is the free memory the valve keeps back for the host itself (its reserve and a twentieth of its RAM). The entry says how much is lent and how much is held back. The runner was left at the limit it had, as it would be with the valve off, so this is a sizing signal and not a fault: lower the host's capacity, give its runners a smaller share, or put the memory-hungry pool on a larger host. It clears fifteen minutes after the last refusal, whether or not the runner that met it is still there. A pool that only observes never raises it: nothing was lent, so a runner that would have been refused is evidence on the runner and in the decision counts.", "fix": "A runner on this host, of a pool set to lend, needed more memory than it was created with and the [memory valve](elastic-memory.md) had none to lend: either everything no runner's guarantee needs was already lent, or what was left is the free memory the valve keeps back for the host itself (its reserve and a twentieth of its RAM). The entry says how much is lent and how much is held back. The runner was left at the limit it had, as it would be with the valve off, so this is a sizing signal and not a fault: lower the host's capacity, give its runners a smaller share, or put the memory-hungry pool on a larger host. It clears fifteen minutes after the last refusal, whether or not the runner that met it is still there. A pool that only observes never raises it: nothing was lent, so a runner that would have been refused is evidence on the runner and in the decision counts.", "verify": "The next run of the job on the host completes without a kill, and the host page's memory valve shows headroom.", "status_sentence": "A machine ran out of spare memory to lend to its busiest jobs.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.os_health", "kind": "problem", "title": "host.os_health", "category": "reliability", "severity": "error when a counted check could not run, warning otherwise", "detection": "runtime", "detects": "A connected host's latest OS report has a finding in the checks `zoomies doctor` counts by default; the safe tier, without optional checks: file watches, Docker log rotation, free disk and the like, or a check that could not run properly. The entry names up to three and says how many more. The aggressive and dedicated tiers are choices rather than faults, so they are suggestions on the host's page and never raise this. A pending reboot is not counted here: it has its own entry, `host.reboot_pending`, so a host whose only finding is a reboot raises that and not this. One entry per host. Zoomies does not change the operating system from here: `sudo zoomies doctor --verbose` on the host says what each check found (from another machine, the host's page has a command for `zoomies doctor --host` with a short-lived token in it), and `sudo zoomies doctor --interactive` on the host offers each fix. Nothing is raised for a host that has sent no report, one that is offline (`host.unhealthy` speaks for it) or a container's partial report. It never reaches the public status page.", "fix": "A connected host's latest OS report has a finding in the checks `zoomies doctor` counts by default; the safe tier, without optional checks: file watches, Docker log rotation, free disk and the like, or a check that could not run properly. The entry names up to three and says how many more. The aggressive and dedicated tiers are choices rather than faults, so they are suggestions on the host's page and never raise this. A pending reboot is not counted here: it has its own entry, `host.reboot_pending`, so a host whose only finding is a reboot raises that and not this. One entry per host. Zoomies does not change the operating system from here: `sudo zoomies doctor --verbose` on the host says what each check found (from another machine, the host's page has a command for `zoomies doctor --host` with a short-lived token in it), and `sudo zoomies doctor --interactive` on the host offers each fix. Nothing is raised for a host that has sent no report, one that is offline (`host.unhealthy` speaks for it) or a container's partial report. It never reaches the public status page.", "verify": "The host page's health pill clears, or shows the check as accepted, once the next report arrives; `sudo zoomies doctor` on the host counts no finding.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.overprovisioned", "kind": "problem", "title": "host.overprovisioned", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "A measured host has more slots than its machine can carry: more than it has allocatable CPUs, or more than it has 2 GB of allocatable memory for. Where every pool that can place on the host has a minimum below a core or 2 GB, that minimum is the size a slot is judged against instead, because those runners are accepted at it. A slot is two containers wherever a `docker_mode: dind` pool places on the host, whether or not it typed its own CPU and memory; a pool that typed them is given those figures a second time for the daemon its builds run in, and a pool sized by its host still splits one slot between a runner and that same daemon, so either way the machine has to be twice the size per slot and the detail names the pool that made it so and whether it typed its limits. The detail names the machine, the capacity and, with default limits on, the share each runner is being given, which is what a job there crawls on; with them off it says nothing limits the runners at all. The share is only given where the host's daemon can enforce it and the pool runs on a container backend; elsewhere nothing limits the runners either. The fix names the largest capacity that fits, never below one, or says the machine is too small to run a runner well when even one slot does not. Lower the capacity on the host card or with `PATCH /api/v1/hosts/{id}` (a host's capacity is decided once when it joins and a heartbeat never rewrites it, so `--capacity` on a fresh join token applies only at the next join, and `agent.capacity` only sets the figure when the embedded host first enrols, not afterwards) or add a host. A `dind` pool sized by its host that cannot give both containers a comfortable share at all is refused the host outright rather than merely counted here, see [hosts and pools](hosts-and-pools.md#default-allocations).", "fix": "A measured host has more slots than its machine can carry: more than it has allocatable CPUs, or more than it has 2 GB of allocatable memory for. Where every pool that can place on the host has a minimum below a core or 2 GB, that minimum is the size a slot is judged against instead, because those runners are accepted at it. A slot is two containers wherever a `docker_mode: dind` pool places on the host, whether or not it typed its own CPU and memory; a pool that typed them is given those figures a second time for the daemon its builds run in, and a pool sized by its host still splits one slot between a runner and that same daemon, so either way the machine has to be twice the size per slot and the detail names the pool that made it so and whether it typed its limits. The detail names the machine, the capacity and, with default limits on, the share each runner is being given, which is what a job there crawls on; with them off it says nothing limits the runners at all. The share is only given where the host's daemon can enforce it and the pool runs on a container backend; elsewhere nothing limits the runners either. The fix names the largest capacity that fits, never below one, or says the machine is too small to run a runner well when even one slot does not. Lower the capacity on the host card or with `PATCH /api/v1/hosts/{id}` (a host's capacity is decided once when it joins and a heartbeat never rewrites it, so `--capacity` on a fresh join token applies only at the next join, and `agent.capacity` only sets the figure when the embedded host first enrols, not afterwards) or add a host. A `dind` pool sized by its host that cannot give both containers a comfortable share at all is refused the host outright rather than merely counted here, see [hosts and pools](hosts-and-pools.md#default-allocations).", "verify": "After the capacity is lowered or the pools' minimums raised, the host's card shows slots its machine can carry and the entry leaves the list.", "status_sentence": "A machine is promised to more runners than it can hold at once.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.reboot_pending", "kind": "problem", "title": "host.reboot_pending", "category": "reliability", "severity": "info", "detection": "runtime", "detects": "An update installed on a connected host takes effect only after it restarts, usually a newer kernel. It stays until the host has rebooted, whether or not anything is running on it. The entry says when there is no runner on the host, so it can be rebooted now. Otherwise it says to cordon the host and reboot it when `zoomies runners list --host \u003chost-id\u003e --state busy` shows nothing, because a runner kept warm for a pool waits for work and never finishes by itself, and a count of runners cannot tell it from one holding a job. Zoomies never reboots a host. It never reaches the public status page.", "fix": "An update installed on a connected host takes effect only after it restarts, usually a newer kernel. It stays until the host has rebooted, whether or not anything is running on it. The entry says when there is no runner on the host, so it can be rebooted now. Otherwise it says to cordon the host and reboot it when `zoomies runners list --host \u003chost-id\u003e --state busy` shows nothing, because a runner kept warm for a pool waits for work and never finishes by itself, and a count of runners cannot tell it from one holding a job. Zoomies never reboots a host. It never reaches the public status page.", "verify": "The host page stops showing a pending reboot after the host comes back, and the entry leaves the list.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.resources_unknown", "kind": "problem", "title": "host.resources_unknown", "category": "reliability", "severity": "warning with default limits on, note otherwise", "detection": "runtime", "detects": "One or more hosts have reported no CPUs, no memory and no disk. They are placed by slot count alone, which is how every host was placed before agents learnt to measure themselves, so nothing is broken, but a pool's resource limits cannot be fitted against a machine nobody has measured, every figure on the host's card is missing, and with `scheduler.default_runner_limits` on its runners get no default either, because their share of an unmeasured machine cannot be computed. A pool with no limits of its own therefore runs unlimited there, which is the shape defaults exist to stop, and that is what makes it a warning. Upgrading the agent fixes it on the next heartbeat, with no re-join.", "fix": "One or more hosts have reported no CPUs, no memory and no disk. They are placed by slot count alone, which is how every host was placed before agents learnt to measure themselves, so nothing is broken, but a pool's resource limits cannot be fitted against a machine nobody has measured, every figure on the host's card is missing, and with `scheduler.default_runner_limits` on its runners get no default either, because their share of an unmeasured machine cannot be computed. A pool with no limits of its own therefore runs unlimited there, which is the shape defaults exist to stop, and that is what makes it a warning. Upgrading the agent fixes it on the next heartbeat, with no re-join.", "verify": "After the agent is upgraded, the host's card shows CPUs, memory and disk and the entry leaves the list.", "status_sentence": "A machine has not reported how much room it has, so it is used cautiously.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.runtime_recovering", "kind": "problem", "title": "host.runtime_recovering", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "A host's agent reports that its container runtime failed (it could not be reached, or it did not answer in time) and it is holding new starts until one recovery attempt. Running jobs continue. The entry names which failure in a row this is, when the agent reported it and when the attempt is due, on the controller's clock, with the backend's own error, shown as prose, so a backtick in what the agent sent reads as an apostrophe and cannot offer a command to copy. For a runtime that cannot be reached, start it or give the agent's user its socket (for Docker, `sudo systemctl start docker`); for one that is slow, look at its load and disk and lower the host's capacity if it carries more than the machine can. It is kept on the host row, so a controller restart does not lose it, and clears on the next heartbeat after a start succeeds.", "fix": "A host's agent reports that its container runtime failed (it could not be reached, or it did not answer in time) and it is holding new starts until one recovery attempt. Running jobs continue. The entry names which failure in a row this is, when the agent reported it and when the attempt is due, on the controller's clock, with the backend's own error, shown as prose, so a backtick in what the agent sent reads as an apostrophe and cannot offer a command to copy. For a runtime that cannot be reached, start it or give the agent's user its socket (for Docker, `sudo systemctl start docker`); for one that is slow, look at its load and disk and lower the host's capacity if it carries more than the machine can. It is kept on the host row, so a controller restart does not lose it, and clears on the next heartbeat after a start succeeds.", "verify": "The entry leaves the list once the agent's recovery attempt succeeds; the host's card no longer shows a held start.", "status_sentence": "A machine's container runtime is recovering, so it is taking no new jobs for now.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.shared_folder_unmounted", "kind": "problem", "title": "host.shared_folder_unmounted", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "A host cannot give its runners a kept tool cache, for one of two reasons the detail names. The first: a containerised agent (the controller's embedded one, or an agent in a container) runs runners, but its container does not mount the [shared folder](configuration.md#the-shared-folder) from the host at its own path. The folder it would use is inside its own data volume, where the host's Docker daemon, which binds runners' caches by that path, cannot see it; handing it out would give every runner an empty cache folder owned by root that no setup action can write to. So the agent keeps no tool cache instead, and the pools that keep one, named in the detail, run their jobs without it. The second: a tool cache folder in the shared folder exists but the runner (uid 1001) cannot write to it (typically one the daemon created as root before the mount was fixed) which would fail every setup action that downloads with `EACCES`; runners start without it instead. Raised only while such a pool could run on the host. Add the mount, the detail gives the line, or run `zoomies upgrade --yes` on the host, which offers it; open or delete the unwritable folder as the detail says. The next heartbeat after the fix clears it.", "fix": "A host cannot give its runners a kept tool cache, for one of two reasons the detail names. The first: a containerised agent (the controller's embedded one, or an agent in a container) runs runners, but its container does not mount the [shared folder](configuration.md#the-shared-folder) from the host at its own path. The folder it would use is inside its own data volume, where the host's Docker daemon, which binds runners' caches by that path, cannot see it; handing it out would give every runner an empty cache folder owned by root that no setup action can write to. So the agent keeps no tool cache instead, and the pools that keep one, named in the detail, run their jobs without it. The second: a tool cache folder in the shared folder exists but the runner (uid 1001) cannot write to it (typically one the daemon created as root before the mount was fixed) which would fail every setup action that downloads with `EACCES`; runners start without it instead. Raised only while such a pool could run on the host. Add the mount, the detail gives the line, or run `zoomies upgrade --yes` on the host, which offers it; open or delete the unwritable folder as the detail says. The next heartbeat after the fix clears it.", "verify": "After the mount or the agent's volume is added, the host page shows the tool cache as kept and the entry leaves the list.", "status_sentence": "A machine keeps no tool cache, so jobs there download their tools again and start slower.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.slots_below_capacity", "kind": "problem", "title": "host.slots_below_capacity", "category": "reliability", "severity": "info", "detection": "runtime", "detects": "A host's capacity says it takes more runners than its machine holds of the standard runner size it was given (set to three slots, holding one because a runner is six of its seven CPUs) while a pool that reaches it has had its typical job wait at least 30 seconds for a runner, over at least five jobs in the last six hours, read from the jobs themselves, so a restart does not forget it. Without that it is silent: big runners may be exactly what was wanted, and a capacity above what they allow is only a ceiling. It is also silent for a host the controller has stepped down (`host.throttled` already says what to do), because that host is held back by load and changing its runner size would lift the step-down in the same write. The size is also withheld, with the reason in the detail, when a pool that takes its size from the host has had a job killed for memory in the last week, or its jobs used more than the smaller runner would hold with half as much again to spare: a smaller runner holds more of them, which is no use to a job that needed the memory. The detail names the machine, the capacity, the runner size and which pools waited. When the controller can price a smaller size it carries a remedy, applied with one click or `zoomies problems apply`: it asks every pool that reaches the host how many runners it would have room for there before and after, and proposes the size only if none loses any and the host gains some. The effect says how many runners the host holds, not the pools' rooms added together. Rooms are counted on what the machines can hold: a host that is reporting, uncordoned and compatible counts, and a temporary hold on new starts does not take it out. Otherwise the detail says what stands in the way; the size is below the host's own smallest runner or below a pool's, or a pool's minimum charges each runner more than the standard, in which case lower that pool's smallest runner. Give each runner less in the host's Runner sizes, or lower the capacity if the big runners are what you want.", "fix": "A host's capacity says it takes more runners than its machine holds of the standard runner size it was given (set to three slots, holding one because a runner is six of its seven CPUs) while a pool that reaches it has had its typical job wait at least 30 seconds for a runner, over at least five jobs in the last six hours, read from the jobs themselves, so a restart does not forget it. Without that it is silent: big runners may be exactly what was wanted, and a capacity above what they allow is only a ceiling. It is also silent for a host the controller has stepped down (`host.throttled` already says what to do), because that host is held back by load and changing its runner size would lift the step-down in the same write. The size is also withheld, with the reason in the detail, when a pool that takes its size from the host has had a job killed for memory in the last week, or its jobs used more than the smaller runner would hold with half as much again to spare: a smaller runner holds more of them, which is no use to a job that needed the memory. The detail names the machine, the capacity, the runner size and which pools waited. When the controller can price a smaller size it carries a remedy, applied with one click or `zoomies problems apply`: it asks every pool that reaches the host how many runners it would have room for there before and after, and proposes the size only if none loses any and the host gains some. The effect says how many runners the host holds, not the pools' rooms added together. Rooms are counted on what the machines can hold: a host that is reporting, uncordoned and compatible counts, and a temporary hold on new starts does not take it out. Otherwise the detail says what stands in the way; the size is below the host's own smallest runner or below a pool's, or a pool's minimum charges each runner more than the standard, in which case lower that pool's smallest runner. Give each runner less in the host's Runner sizes, or lower the capacity if the big runners are what you want.", "verify": "After the size or the capacity changes, the host's card shows the slots its machine holds and the entry leaves the list on the next pass.", "status_sentence": "A machine could hold more runners than it does, because each runner is sized larger than the machine divides into, while jobs have been waiting for room.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.throttled", "kind": "problem", "title": "host.throttled", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "One or more hosts have been stepped down after sustained pressure: CPU pinned with a runner no limit binds, a load average past twice the cores, or memory at the reserve. The entry carries each host's own sentence, how many of its slots it is left, which measurement did it, what the running jobs are getting (never less than half their CPU allocation), and that it lifts one step after five minutes of calm. Wait for it, lower the host's capacity or the pools' limits so its runners fit the machine, or lift it from the host card (**Lift the throttle**) once the cause is fixed; a cleared throttle comes back on the next heartbeat if the pressure is still there. A host that keeps being throttled has too many slots for its machine.", "fix": "One or more hosts have been stepped down after sustained pressure: CPU pinned with a runner no limit binds, a load average past twice the cores, or memory at the reserve. The entry carries each host's own sentence, how many of its slots it is left, which measurement did it, what the running jobs are getting (never less than half their CPU allocation), and that it lifts one step after five minutes of calm. Wait for it, lower the host's capacity or the pools' limits so its runners fit the machine, or lift it from the host card (**Lift the throttle**) once the cause is fixed; a cleared throttle comes back on the next heartbeat if the pressure is still there. A host that keeps being throttled has too many slots for its machine.", "verify": "The host's card no longer shows it stepped down and `zoomies hosts list` shows its full slot count once pressure has eased.", "status_sentence": "A machine is busy enough that it is taking new jobs more slowly.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.unhealthy", "kind": "problem", "title": "host.unhealthy", "category": "reliability", "severity": "error with runners on it, warning without", "detection": "runtime", "detects": "The host has stopped heartbeating. With runners recorded on it their state is unknown, which is worse than a spare host being down.", "fix": "The host has stopped heartbeating. With runners recorded on it their state is unknown, which is worse than a spare host being down.", "verify": "The host's card turns green and the entry leaves the problems list on the next heartbeat; `zoomies hosts list` shows it connected.", "status_sentence": "A machine has stopped checking in, so fewer jobs can run at once.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "host.version_behind", "kind": "problem", "title": "host.version_behind", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "One or more hosts run a different release from the controller. A fleet part-way through an upgrade looks like this and clears itself, which is why it is a warning; one that stays this way has a host somebody has forgotten, running an agent whose odd behaviour has an explanation nobody thinks to look for. The entry names which hosts are behind, which are *ahead* (the direction the policy calls unsupported, where the fix is to upgrade the controller) and which run a build the controller cannot order against its own. Two builds of one tag are the same release and do not appear.", "fix": "One or more hosts run a different release from the controller. A fleet part-way through an upgrade looks like this and clears itself, which is why it is a warning; one that stays this way has a host somebody has forgotten, running an agent whose odd behaviour has an explanation nobody thinks to look for. The entry names which hosts are behind, which are *ahead* (the direction the policy calls unsupported, where the fix is to upgrade the controller) and which run a build the controller cannot order against its own. Two builds of one tag are the same release and do not appear.", "verify": "`zoomies hosts list` shows every host on the controller's release and the entry leaves the list.", "status_sentence": "A machine is running an older agent than the controller.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "host.work_concentrated", "kind": "problem", "title": "host.work_concentrated", "category": "reliability", "severity": "info", "detection": "runtime", "detects": "A host has been throttled after sustained pressure while it is running runners, and one or more other hosts are available, uncordoned, running nothing and under 30% CPU, and a pool that runs on the busy host could run on them (selector, backend and platform). Raised only while `scheduler.host_order` is the default, `headroom`, which prefers the host with the most left afterwards: the largest machine (often the one that also runs the controller) is the first choice every time a job needs a runner, even throttled, because it still has more left than a small idle one. No change is proposed, because what helps is a decision that depends on what the large host is for: lower its capacity so it takes fewer runners at once. Changing `scheduler.host_order` is not the answer on its own: `best_fit` packs the fullest host first and `largest_standard` prefers the host where a pool's runner is biggest, and both keep choosing a large host that `headroom` would have left. It clears as soon as the host is no longer throttled or no other host is idle.", "fix": "A host has been throttled after sustained pressure while it is running runners, and one or more other hosts are available, uncordoned, running nothing and under 30% CPU, and a pool that runs on the busy host could run on them (selector, backend and platform). Raised only while `scheduler.host_order` is the default, `headroom`, which prefers the host with the most left afterwards: the largest machine (often the one that also runs the controller) is the first choice every time a job needs a runner, even throttled, because it still has more left than a small idle one. No change is proposed, because what helps is a decision that depends on what the large host is for: lower its capacity so it takes fewer runners at once. Changing `scheduler.host_order` is not the answer on its own: `best_fit` packs the fullest host first and `largest_standard` prefers the host where a pool's runner is biggest, and both keep choosing a large host that `headroom` would have left. It clears as soon as the host is no longer throttled or no other host is idle.", "verify": "New runners land on the idle hosts: `zoomies runners list` shows them spread, and the entry leaves the list once the busy host is no longer throttled.", "status_sentence": "One machine is being kept busy to the point of slowing down while others sit idle, because new jobs keep going to the biggest one first.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "images.refresh_negative", "kind": "problem", "title": "images.refresh_negative", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must not be negative. Use a duration, or 0 to leave images alone.", "fix": "Must not be negative. Use a duration, or 0 to leave images alone.", "verify": null, "setting": "images.refresh_interval", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "images.refresh_off", "kind": "problem", "title": "images.refresh_off", "category": "configuration", "severity": "info", "detection": "static", "detects": "Nothing refreshes runner images, so a pool naming a moving tag keeps whatever its hosts pulled first. Expected on an air-gapped fleet, or one that pins every pool to a digest.", "fix": "Nothing refreshes runner images, so a pool naming a moving tag keeps whatever its hosts pulled first. Expected on an air-gapped fleet, or one that pins every pool to a digest.", "verify": null, "setting": "images.refresh_interval", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "images.refresh_too_fast", "kind": "problem", "title": "images.refresh_too_fast", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Every pool's image is checked on every host far more often than an image is built.", "fix": "Every pool's image is checked on every host far more often than an image is built.", "verify": null, "setting": "images.refresh_interval", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "indexing.allowed", "kind": "problem", "title": "indexing.allowed", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Search engines are invited to index a fleet controller. Deliberate for a demo, rarely otherwise.", "fix": "Search engines are invited to index a fleet controller. Deliberate for a demo, rarely otherwise.", "verify": null, "setting": "server.allow_indexing", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "installation.unhealthy", "kind": "problem", "title": "installation.unhealthy", "category": "reliability", "severity": "error", "detection": "runtime", "detects": "The GitHub App installation is not usable; the App was uninstalled, its key was rotated, or its permissions were changed. Nothing can register until it is fixed. An App that is merely not subscribed to `workflow_job` is *not* this: the fleet works on the fallback poller, more slowly, and `webhook.never_received` is the entry for it.", "fix": "The GitHub App installation is not usable; the App was uninstalled, its key was rotated, or its permissions were changed. Nothing can register until it is fixed. An App that is merely not subscribed to `workflow_job` is *not* this: the fleet works on the fallback poller, more slowly, and `webhook.never_received` is the entry for it.", "verify": null, "status_sentence": "The fleet has lost a permission it needs on GitHub, so jobs from some repositories cannot start.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "jobs.duration_regressed", "kind": "problem", "title": "jobs.duration_regressed", "category": "reliability", "severity": "info", "detection": "runtime", "detects": "A job's typical run over the last day took at least half as long again, and at least a minute more, than over the seven days before, with at least ten measured runs in each window, on jobs this fleet ran. It carries no remedy: the detail says to compare a run with an older one and to group the job's runs by host and controller version, which tells a slow workflow from a slow machine.", "fix": "A job's typical run over the last day took at least half as long again, and at least a minute more, than over the seven days before, with at least ten measured runs in each window, on jobs this fleet ran. It carries no remedy: the detail says to compare a run with an older one and to group the job's runs by host and controller version, which tells a slow workflow from a slow machine.", "verify": "The job's typical duration on the Jobs page falls back towards the seven-day figure and the entry leaves the list after the next day of runs.", "status_sentence": "A job now typically takes much longer than it did the week before.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "jobs.label_advice", "kind": "problem", "title": "jobs.label_advice", "category": "cost", "severity": "info", "detection": "runtime", "detects": "Jobs whose measured runs call for something other than what their `runs-on` asks for, counted by kind, in one entry however many there are: a job that names a size class too small for what it uses (it can only ever run on hosts that are too small, and is killed when it needs more than they have), a job that names none and needs more than the default class (it is routed there best effort, which is not a promise, because GitHub decides which waiting job a runner takes), and a job that names a class larger than it uses (it occupies a host another job needs). Only a job with at least five measured runs is advised on, a job an operator has pinned is left out, and there is none while `scheduler.size_routing` is `off`. The advice for each job is on the Jobs page, in `GET /api/v1/label-advice` and in `zoomies jobs advice`, with what to write instead.", "fix": "Jobs whose measured runs call for something other than what their `runs-on` asks for, counted by kind, in one entry however many there are: a job that names a size class too small for what it uses (it can only ever run on hosts that are too small, and is killed when it needs more than they have), a job that names none and needs more than the default class (it is routed there best effort, which is not a promise, because GitHub decides which waiting job a runner takes), and a job that names a class larger than it uses (it occupies a host another job needs). Only a job with at least five measured runs is advised on, a job an operator has pinned is left out, and there is none while `scheduler.size_routing` is `off`. The advice for each job is on the Jobs page, in `GET /api/v1/label-advice` and in `zoomies jobs advice`, with what to write instead.", "verify": "The Jobs page lists no advice for the job after its next five measured runs, and `zoomies jobs advice` leaves it out.", "status_sentence": "Some workflows ask for a machine size their jobs do not need, or need more than they ask for.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "jobs.oom_killed", "kind": "problem", "title": "jobs.oom_killed", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The kernel killed a job's runner, one of its steps (exit 137), or in a Docker-in-Docker pool the sidecar its builds ran in, for its memory limit on a host in the last hour. GitHub records it as an ordinary failure; this is the fleet owning up. The entry names the job, the host and the most the job was measured using. With `scheduler.history_sizing: on` the next run of that job is placed on a host with room for half as much again; otherwise give the pool a larger minimum memory, or raise `memory_mb`.", "fix": "The kernel killed a job's runner, one of its steps (exit 137), or in a Docker-in-Docker pool the sidecar its builds ran in, for its memory limit on a host in the last hour. GitHub records it as an ordinary failure; this is the fleet owning up. The entry names the job, the host and the most the job was measured using. With `scheduler.history_sizing: on` the next run of that job is placed on a host with room for half as much again; otherwise give the pool a larger minimum memory, or raise `memory_mb`.", "verify": "The next run of the job completes without exit 137: its page shows a peak below its limit, and the entry leaves the list after an hour.", "status_sentence": "A job ran out of memory on its machine, so it failed through no fault of its own.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "jobs.runner_lost", "kind": "problem", "title": "jobs.runner_lost", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "A job's runner stopped under it, so the failure is the fleet's rather than the workflow's. When most of them share one category, the detail says so and the fix is that category's.", "fix": "A job's runner stopped under it, so the failure is the fleet's rather than the workflow's. When most of them share one category, the detail says so and the fix is that category's.", "verify": "The next run of the job finishes on a runner that outlives it: `zoomies jobs list --ours --since 1h` shows nothing new, and the entry leaves the list.", "status_sentence": "A runner stopped while running a job, so that job may fail or need a re-run.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "jobs.unmatched", "kind": "problem", "title": "jobs.unmatched", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "Jobs are queued that no enabled pool here will run. Usually their labels match no pool: they may belong to another runner provider, or a pool may be missing a label. The entry says so instead when the cause is the GitHub target rather than the labels (a pool advertising exactly those labels but belonging to another installation, or a repository no installation here covers) because those need the installation changed, not the workflow.", "fix": "Jobs are queued that no enabled pool here will run. Usually their labels match no pool: they may belong to another runner provider, or a pool may be missing a label. The entry says so instead when the cause is the GitHub target rather than the labels (a pool advertising exactly those labels but belonging to another installation, or a repository no installation here covers) because those need the installation changed, not the workflow.", "verify": "`zoomies jobs list --unmatched` is empty and the entry leaves the list once the labels match a pool or the installation covers the repository.", "status_sentence": "Some jobs ask for runner labels this fleet does not offer, so they will wait until the workflow or the fleet changes.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "jobs.workflow_failing", "kind": "problem", "title": "jobs.workflow_failing", "category": "reliability", "severity": "info", "detection": "runtime", "detects": "A job failed at the same step three or more times in the last six hours and did not pass once in that time, on this fleet's runners or on GitHub's hosted ones. It names the step and where the job ran, and has no remedy: the fix is in the workflow or the repository's settings, and the fleet only saw it. Each pattern is one problem, newest failure first.", "fix": "A job failed at the same step three or more times in the last six hours and did not pass once in that time, on this fleet's runners or on GitHub's hosted ones. It names the step and where the job ran, and has no remedy: the fix is in the workflow or the repository's settings, and the fleet only saw it. Each pattern is one problem, newest failure first.", "verify": "The job passes at the step it was failing on: `zoomies jobs list --workflow \u003cname\u003e --failed` shows no new failures and the entry leaves the list.", "status_sentence": "A workflow job has failed at the same step several times in a row and not passed once, which is the workflow's to fix rather than the fleet's.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "kennel.api_budget", "kind": "problem", "title": "kennel.api_budget", "category": "configuration", "severity": "error", "detection": "static", "detects": "Outside 5 to 50. Below 5% a refresh would not finish before the next was due; above 50% Kennel Club would compete with the scheduler for the requests it needs to place a runner. The default is 20.", "fix": "Outside 5 to 50. Below 5% a refresh would not finish before the next was due; above 50% Kennel Club would compete with the scheduler for the requests it needs to place a runner. The default is 20.", "verify": null, "setting": "kennel.api_budget_percent", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "kennel.exposure", "kind": "problem", "title": "kennel.exposure", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "A repository this fleet serves has an open, unwaived exposure finding that is an error: code from a stranger's pull request has run on your runners, or a public repository's jobs ran on a pool set up so that such code could reach the host. One entry for the whole fleet, however many repositories it is about: its title is the count and it names none of them. It links to Kennel Club's repositories narrowed to those with an error, which lists every finding with the evidence behind it and is where one is waived with a reason. Dismissing it holds until every such error has cleared. A repository with only warnings raises nothing here: it appears in Kennel Club and nowhere else. Neither does one an administrator has told Kennel Club not to look at: it is not evaluated, so there is no error to raise, and starting again raises it afresh if the error is still there. It is never on the public status page.", "fix": "A repository this fleet serves has an open, unwaived exposure finding that is an error: code from a stranger's pull request has run on your runners, or a public repository's jobs ran on a pool set up so that such code could reach the host. One entry for the whole fleet, however many repositories it is about: its title is the count and it names none of them. It links to Kennel Club's repositories narrowed to those with an error, which lists every finding with the evidence behind it and is where one is waived with a reason. Dismissing it holds until every such error has cleared. A repository with only warnings raises nothing here: it appears in Kennel Club and nowhere else. Neither does one an administrator has told Kennel Club not to look at: it is not evaluated, so there is no error to raise, and starting again raises it afresh if the error is still there. It is never on the public status page.", "verify": "Kennel Club's Overview shows no repository with an open exposure error once the pools or the repositories' settings change and the next read lands, and the entry leaves the list.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "kennel.refresh_interval", "kind": "problem", "title": "kennel.refresh_interval", "category": "configuration", "severity": "error", "detection": "static", "detects": "Shorter than an hour, or zero. Every refresh spends requests from a limit shared with everything else the installation does, and what is read changes over days, so a shorter interval buys nothing and zero would never refresh. Use `24h`, the default, or anything of an hour or more.", "fix": "Shorter than an hour, or zero. Every refresh spends requests from a limit shared with everything else the installation does, and what is read changes over days, so a shorter interval buys nothing and zero would never refresh. Use `24h`, the default, or anything of an hour or more.", "verify": null, "setting": "kennel.refresh_interval", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "kennel.scope", "kind": "problem", "title": "kennel.scope", "category": "configuration", "severity": "error", "detection": "static", "detects": "Not a scope Kennel Club knows. They are `served`, the repositories this fleet has run a job for, and `installation`, every repository the GitHub App can see.", "fix": "Not a scope Kennel Club knows. They are `served`, the repositories this fleet has run a job for, and `installation`, every repository the GitHub App can see.", "verify": null, "setting": "kennel.scope", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "kennel.unavailable", "kind": "problem", "title": "kennel.unavailable", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "Kennel Club has not been able to read an installation's repositories for six hours: GitHub refused the listing, which every App is allowed to make, or the reads keep failing or being held back by the rate limit. What it says about those repositories is as old as that. A permission you chose not to grant is never this: it is shown on the repositories it affects, naming the permission, and is not a problem. Nor is an installation on which every repository has been told not to be tracked: nothing there is read, so the failure is dropped, and it comes back if one is tracked again and the reads still fail.", "fix": "Kennel Club has not been able to read an installation's repositories for six hours: GitHub refused the listing, which every App is allowed to make, or the reads keep failing or being held back by the rate limit. What it says about those repositories is as old as that. A permission you chose not to grant is never this: it is shown on the repositories it affects, naming the permission, and is not a problem. Nor is an installation on which every repository has been told not to be tracked: nothing there is read, so the failure is dropped, and it comes back if one is tracked again and the reads still fail.", "verify": "Kennel Club's Overview shows the installation read within the last refresh interval, and the entry leaves the list after the next successful read.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "kennel.unknown_check", "kind": "problem", "title": "kennel.unknown_check", "category": "configuration", "severity": "error", "detection": "static", "detects": "A name that is neither a check's code nor an area. It would turn nothing off, so the check you meant to turn off would go on running. The finding lists the names that are valid.", "fix": "A name that is neither a check's code nor an area. It would turn nothing off, so the check you meant to turn off would go on running. The finding lists the names that are valid.", "verify": null, "setting": "kennel.disabled_checks", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "limits.loopback", "kind": "problem", "title": "limits.loopback", "category": "configuration", "severity": "warning", "detection": "static", "detects": "A ceiling is set on a controller only this machine can reach. It guards an instance many people reach against one caller spending what the others need; with nobody else able to reach this one it can only refuse you. Set it back to 0, or leave it if the controller is about to go behind a proxy or get an external URL.", "fix": "A ceiling is set on a controller only this machine can reach. It guards an instance many people reach against one caller spending what the others need; with nobody else able to reach this one it can only refuse you. Set it back to 0, or leave it if the controller is about to go behind a proxy or get an external URL.", "verify": null, "setting": "the `limits.*` key named", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "limits.negative", "kind": "problem", "title": "limits.negative", "category": "configuration", "severity": "error", "detection": "static", "detects": "A ceiling cannot be negative. Set it to 0 for no ceiling, or to the most this instance should hold.", "fix": "A ceiling cannot be negative. Set it to 0 for no ceiling, or to the most this instance should hold.", "verify": null, "setting": "the `limits.*` key named", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "log.debug", "kind": "problem", "title": "log.debug", "category": "configuration", "severity": "info", "detection": "static", "detects": "Debug logging is on, which is loud and includes request detail.", "fix": "Debug logging is on, which is loud and includes request detail.", "verify": null, "setting": "log.level", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "log.format", "kind": "problem", "title": "log.format", "category": "configuration", "severity": "error", "detection": "static", "detects": "Not a format. They are `text` and `json`.", "fix": "Not a format. They are `text` and `json`.", "verify": null, "setting": "log.format", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "log.level", "kind": "problem", "title": "log.level", "category": "configuration", "severity": "error", "detection": "static", "detects": "Not a level. They are `debug`, `info`, `warn` and `error`.", "fix": "Not a level. They are `debug`, `info`, `warn` and `error`.", "verify": null, "setting": "log.level", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "mcp_oauth.enabled", "kind": "problem", "title": "mcp_oauth.enabled", "category": "configuration", "severity": "info", "detection": "static", "detects": "Claude and other MCP clients can sign in to `/mcp` through the browser, and it names the address to give them and the `claude mcp add` line for Claude Code. It says when `server.external_url` is unset, because the addresses advertised are then taken from each request's Host header, and when `security.mcp_open_registration` is off, because only an administrator's clients can connect. Nothing to change; see [Connect Claude to Zoomies](connect-claude.md).", "fix": "Claude and other MCP clients can sign in to `/mcp` through the browser, and it names the address to give them and the `claude mcp add` line for Claude Code. It says when `server.external_url` is unset, because the addresses advertised are then taken from each request's Host header, and when `security.mcp_open_registration` is off, because only an administrator's clients can connect. Nothing to change; see [Connect Claude to Zoomies](connect-claude.md).", "verify": null, "setting": "security.mcp_oauth", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "mcp_oauth.no_external_url", "kind": "problem", "title": "mcp_oauth.no_external_url", "category": "configuration", "severity": "error", "detection": "static", "detects": "MCP sign-in has been turned on where the controller listens off this machine, and `server.external_url` is not set, so the issuer, the endpoints and the resource a token is bound to would be read from each request's `Host` header, which whoever sends the request chooses. Set `server.external_url` to the https address people use, turn `security.mcp_oauth` off, or bind to loopback behind a proxy. Left unset, `security.mcp_oauth` does not turn itself on in this shape, so an instance that started before goes on starting. On loopback nothing is refused.", "fix": "MCP sign-in has been turned on where the controller listens off this machine, and `server.external_url` is not set, so the issuer, the endpoints and the resource a token is bound to would be read from each request's `Host` header, which whoever sends the request chooses. Set `server.external_url` to the https address people use, turn `security.mcp_oauth` off, or bind to loopback behind a proxy. Left unset, `security.mcp_oauth` does not turn itself on in this shape, so an instance that started before goes on starting. On loopback nothing is refused.", "verify": null, "setting": "security.mcp_oauth", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "mcp_oauth.plain_http", "kind": "problem", "title": "mcp_oauth.plain_http", "category": "configuration", "severity": "warning", "detection": "static", "detects": "MCP sign-in has been turned on where the controller is reached over plain HTTP, so authorisation codes and the tokens they buy cross the network readable. Serve the controller over https, or set `security.mcp_oauth` to false and give MCP clients an API token.", "fix": "MCP sign-in has been turned on where the controller is reached over plain HTTP, so authorisation codes and the tokens they buy cross the network readable. Serve the controller over https, or set `security.mcp_oauth` to false and give MCP clients an API token.", "verify": null, "setting": "security.mcp_oauth", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "metrics.public", "kind": "problem", "title": "metrics.public", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The metrics endpoint answers without authentication, so anyone who can reach it learns the shape of the fleet.", "fix": "The metrics endpoint answers without authentication, so anyone who can reach it learns the shape of the fleet.", "verify": null, "setting": "metrics.public", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "oidc.incomplete", "kind": "problem", "title": "oidc.incomplete", "category": "security", "severity": "error", "detection": "static", "detects": "Single sign-on is enabled with no issuer or no client id.", "fix": "Single sign-on is enabled with no issuer or no client id.", "verify": null, "setting": "oidc", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "oidc.insecure_issuer", "kind": "problem", "title": "oidc.insecure_issuer", "category": "security", "severity": "warning", "detection": "static", "detects": "The token exchange happens in the clear.", "fix": "The token exchange happens in the clear.", "verify": null, "setting": "oidc.issuer", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "oidc.link_by_username", "kind": "problem", "title": "oidc.link_by_username", "category": "security", "severity": "warning", "detection": "static", "detects": "A first single sign-on login can take over an existing password account with the same name.", "fix": "A first single sign-on login can take over an existing password account with the same name.", "verify": null, "setting": "oidc.link_by_username", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "oidc.no_redirect", "kind": "problem", "title": "oidc.no_redirect", "category": "security", "severity": "error", "detection": "static", "detects": "Needed, and it must be an address the identity provider can reach.", "fix": "Needed, and it must be an address the identity provider can reach.", "verify": null, "setting": "oidc.redirect_url", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "oidc.open_signup", "kind": "problem", "title": "oidc.open_signup", "category": "security", "severity": "warning", "detection": "static", "detects": "Anyone your identity provider authenticates gets an account here. Narrow it at the provider, or turn signup off and create accounts yourself.", "fix": "Anyone your identity provider authenticates gets an account here. Narrow it at the provider, or turn signup off and create accounts yourself.", "verify": null, "setting": "oidc.allow_signup", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "oidc.password_login_hidden_without_sso", "kind": "problem", "title": "oidc.password_login_hidden_without_sso", "category": "security", "severity": "warning", "detection": "static", "detects": "The password form is set to be hidden, but single sign-on is off, so the setting does nothing yet, and the day single sign-on is turned on, everybody below administrator loses password sign-in. Turn the setting off, or configure single sign-on and check its button works first.", "fix": "The password form is set to be hidden, but single sign-on is off, so the setting does nothing yet, and the day single sign-on is turned on, everybody below administrator loses password sign-in. Turn the setting off, or configure single sign-on and check its button works first.", "verify": null, "setting": "oidc.hide_password_login", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "oidc.unavailable", "kind": "problem", "title": "oidc.unavailable", "category": "security", "severity": "error", "detection": "runtime", "detects": "Single sign-on is configured but not working: the identity provider could not be reached, or did not answer with an OpenID configuration, or a setting it needs is missing. Password sign-in still works and the sign-in page shows its form while this lasts, but accounts that only ever signed in through single sign-on cannot sign in. When the provider is unreachable the controller tries again by itself, backing off from five seconds to five minutes, and the entry clears once it answers; no restart. Check that this host can reach `oidc.issuer`; a missing setting is named in the entry and needs a restart once corrected.", "fix": "Single sign-on is configured but not working: the identity provider could not be reached, or did not answer with an OpenID configuration, or a setting it needs is missing. Password sign-in still works and the sign-in page shows its form while this lasts, but accounts that only ever signed in through single sign-on cannot sign in. When the provider is unreachable the controller tries again by itself, backing off from five seconds to five minutes, and the entry clears once it answers; no restart. Check that this host can reach `oidc.issuer`; a missing setting is named in the entry and needs a restart once corrected.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "origins.any", "kind": "problem", "title": "origins.any", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Any website can act with a signed-in operator's session. Name the origins you serve from.", "fix": "Any website can act with a signed-in operator's session. Name the origins you serve from.", "verify": null, "setting": "server.allowed_origins", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "origins.insecure", "kind": "problem", "title": "origins.insecure", "category": "configuration", "severity": "warning", "detection": "static", "detects": "A plaintext origin is allowed to act on this controller.", "fix": "A plaintext origin is allowed to act on this controller.", "verify": null, "setting": "server.allowed_origins", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "poll.disabled", "kind": "problem", "title": "poll.disabled", "category": "configuration", "severity": "warning", "detection": "static", "detects": "If a webhook delivery is lost or misconfigured, jobs queue for ever with nothing to notice. Leave it on unless you monitor delivery yourself.", "fix": "If a webhook delivery is lost or misconfigured, jobs queue for ever with nothing to notice. Leave it on unless you monitor delivery yourself.", "verify": null, "setting": "github.poll_fallback", "docs_html": "https://zoomies.sh/problem-codes/#configuration-github", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-github" }, { "id": "poll.too_fast", "kind": "problem", "title": "poll.too_fast", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Polling this often will consume the App's API rate limit. The poller is a safety net, not the primary path.", "fix": "Polling this often will consume the App's API rate limit. The poller is a safety net, not the primary path.", "verify": null, "setting": "github.poll_interval", "docs_html": "https://zoomies.sh/problem-codes/#configuration-github", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-github" }, { "id": "poller.paused", "kind": "problem", "title": "poller.paused", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "GitHub is rate-limiting one installation, so every background sweep is standing down from it until the moment named. It clears itself. One installation is held at a time -- the quota is per installation -- so the others are still polled, and the entry names which one.", "fix": "GitHub is rate-limiting one installation, so every background sweep is standing down from it until the moment named. It clears itself. One installation is held at a time -- the quota is per installation -- so the others are still polled, and the entry names which one.", "verify": null, "status_sentence": "The controller has stopped asking GitHub for queued jobs, so a missed notification is not caught.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "poller.stale", "kind": "problem", "title": "poller.stale", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The fallback poller is enabled and has not finished a sweep for more than two and a half intervals. It is the safety net for a fleet whose webhooks stop arriving, and a net that has stopped sweeping looks exactly like a net with nothing to catch. The controller's log carries the error that ended the sweep.", "fix": "The fallback poller is enabled and has not finished a sweep for more than two and a half intervals. It is the safety net for a fleet whose webhooks stop arriving, and a net that has stopped sweeping looks exactly like a net with nothing to catch. The controller's log carries the error that ended the sweep.", "verify": null, "status_sentence": "The controller has not heard from GitHub recently, so new jobs may be noticed late.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.auto_blocked", "kind": "problem", "title": "pool.auto_blocked", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "An automatic pool could not be made or kept, and the entry says why and what to change. Either a pool you made already has the name the controller would give the pool for that architecture and size class, or already answers to the class label `zoomies-small`, `zoomies-medium` or `zoomies-large` that pool would carry (the controller never takes over, rewrites or shares a pool of yours, so it makes none and jobs for that class wait for a pool to carry the label) or the pool kept for the installation automatic pools used to belong to still holds the name the new one needs, because pool names are unique across the instance (delete it, which you can because it is no longer kept) or `scheduler.auto_pools` is on and no installation could be chosen, because several are configured and `scheduler.auto_pools_installation` names none. Rename or relabel your pool, delete the old one, or name the installation.", "fix": "An automatic pool could not be made or kept, and the entry says why and what to change. Either a pool you made already has the name the controller would give the pool for that architecture and size class, or already answers to the class label `zoomies-small`, `zoomies-medium` or `zoomies-large` that pool would carry (the controller never takes over, rewrites or shares a pool of yours, so it makes none and jobs for that class wait for a pool to carry the label) or the pool kept for the installation automatic pools used to belong to still holds the name the new one needs, because pool names are unique across the instance (delete it, which you can because it is no longer kept) or `scheduler.auto_pools` is on and no installation could be chosen, because several are configured and `scheduler.auto_pools_installation` names none. Rename or relabel your pool, delete the old one, or name the installation.", "verify": "`zoomies auto-pools` lists the pool for that class and the entry leaves the list.", "status_sentence": "A pool the fleet should keep for a size of machine could not be made, so jobs for that size wait.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.cache_above_disk", "kind": "problem", "title": "pool.cache_above_disk", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The pool's cache size limit is above the free disk on the smallest host it can land on. The cache is evicted down to its limit between one runner and the next, so a limit above the free space is not a limit at all: the disk fills first, and a host at or below its disk reserve takes no runner of any pool.", "fix": "The pool's cache size limit is above the free disk on the smallest host it can land on. The cache is evicted down to its limit between one runner and the next, so a limit above the free space is not a limit at all: the disk fills first, and a host at or below its disk reserve takes no runner of any pool.", "verify": "The pool's cache limit is at or below the free disk its page shows for the smallest host, and the entry leaves the list.", "status_sentence": "A runner cache is set larger than the disk it lives on.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.cache_memory_unbounded", "kind": "problem", "title": "pool.cache_memory_unbounded", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The pool's cache source is under `/dev/shm`, which is the host's memory, and the cache has no size limit. The cache is evicted only down to its limit between one runner and the next, so with none it grows until the host has no memory left, which takes every runner and the agent with it. Set the cache's size limit, keeping it well inside what the smallest host can spare, or move the cache source to a directory on disk.", "fix": "The pool's cache source is under `/dev/shm`, which is the host's memory, and the cache has no size limit. The cache is evicted only down to its limit between one runner and the next, so with none it grows until the host has no memory left, which takes every runner and the agent with it. Set the cache's size limit, keeping it well inside what the smallest host can spare, or move the cache source to a directory on disk.", "verify": "The pool's cache shows a size limit on its page and the entry leaves the list.", "status_sentence": "A runner cache kept in memory has no size limit, so it could use all of a machine's memory.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.cache_shared", "kind": "problem", "title": "pool.cache_shared", "category": "security", "severity": "warning", "detection": "runtime", "detects": "Under an organisation installation, GitHub can hand this pool's runners any repository's job whose `runs-on` matches its labels, Zoomies has no say in which repository that is. A repository-scoped cache is therefore only as private as the pool's labels: give the pool a branded label that only the intended repository's workflows use. A repository-targeted installation registers runners only that repository's jobs can reach, so this never fires there.", "fix": "Under an organisation installation, GitHub can hand this pool's runners any repository's job whose `runs-on` matches its labels, Zoomies has no say in which repository that is. A repository-scoped cache is therefore only as private as the pool's labels: give the pool a branded label that only the intended repository's workflows use. A repository-targeted installation registers runners only that repository's jobs can reach, so this never fires there.", "verify": "The pool's cache scope is `installation`, or the pool is under a repository installation, and the entry leaves the list.", "status_sentence": "A runner cache is shared more widely than usual.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.daemon_share_suggested", "kind": "problem", "title": "pool.daemon_share_suggested", "category": "capacity", "severity": "info", "detection": "runtime", "detects": "A Docker-in-Docker pool divides its slot against its work: over the last six hours, one of its two containers used at least 85% of its own limit for one sample in twenty while the other used at most half of its. Raised only from runners sized by their host (a typed limit goes to both containers in full, so there is nothing to divide), only from at least 60 samples across three runners that span at least half an hour and of which no one runner is more than half, so a pool just restarted or one long job cannot speak for it, and only from samples taken under the division in force now, so an edit clears it. The notice says the time the samples really cover. CPU and memory are judged on their own, so the notice can say more CPU for a sidecar that is building and less memory for one that is not. The proposal, for each resource, is each container's own 95th-percentile use plus a quarter, as a share of the slot, rounded to five and kept between 10 and 90. A proposal is priced on the pool's hosts before it is made, because a thinner half is held to the pool's smallest runner and the slot grows to carry it: a share that would leave the hosts room for fewer runners than they hold now is stepped back toward the current one until it does not, and when no share worth a move of ten points does, the notice says what the figure would have cost and proposes none. It stays for as long as the window shows it and clears itself when the share is changed or the work is. An operator sees a button on the notice that makes the change in one click, leaving everything else about the pool as it was; the same figures are in the problem's `daemon_share` field. Or change the pool's Docker sidecar's CPU and memory shares as the notice says (`--daemon-cpu-share`, `--daemon-memory-share`, or `--daemon-share` when both want the same figure); it applies to runners created afterwards. See [Two containers, two limits](hosts-and-pools.md#keeping-the-work-folder-in-memory). It is not given for a pool the controller keeps, which divides its slot evenly and has no setting for it. Memory is sized from the most each half used, not its 95th percentile, because it is a limit a job is killed at: a runner that used 1 GiB in nearly every sample and 3.5 GiB in a few is not given less than 3.5. A share is only proposed if, turned into the limits each container would really be given on the pool's hosts, it gives the squeezed container more somewhere and leaves the other enough everywhere: where the pool has a smallest runner and the thin half sits at it, every share gives that half the same, so the notice says what sets it (raise the pool's smallest runner) and offers no share to apply.", "fix": "A Docker-in-Docker pool divides its slot against its work: over the last six hours, one of its two containers used at least 85% of its own limit for one sample in twenty while the other used at most half of its. Raised only from runners sized by their host (a typed limit goes to both containers in full, so there is nothing to divide), only from at least 60 samples across three runners that span at least half an hour and of which no one runner is more than half, so a pool just restarted or one long job cannot speak for it, and only from samples taken under the division in force now, so an edit clears it. The notice says the time the samples really cover. CPU and memory are judged on their own, so the notice can say more CPU for a sidecar that is building and less memory for one that is not. The proposal, for each resource, is each container's own 95th-percentile use plus a quarter, as a share of the slot, rounded to five and kept between 10 and 90. A proposal is priced on the pool's hosts before it is made, because a thinner half is held to the pool's smallest runner and the slot grows to carry it: a share that would leave the hosts room for fewer runners than they hold now is stepped back toward the current one until it does not, and when no share worth a move of ten points does, the notice says what the figure would have cost and proposes none. It stays for as long as the window shows it and clears itself when the share is changed or the work is. An operator sees a button on the notice that makes the change in one click, leaving everything else about the pool as it was; the same figures are in the problem's `daemon_share` field. Or change the pool's Docker sidecar's CPU and memory shares as the notice says (`--daemon-cpu-share`, `--daemon-memory-share`, or `--daemon-share` when both want the same figure); it applies to runners created afterwards. See [Two containers, two limits](hosts-and-pools.md#keeping-the-work-folder-in-memory). It is not given for a pool the controller keeps, which divides its slot evenly and has no setting for it. Memory is sized from the most each half used, not its 95th percentile, because it is a limit a job is killed at: a runner that used 1 GiB in nearly every sample and 3.5 GiB in a few is not given less than 3.5. A share is only proposed if, turned into the limits each container would really be given on the pool's hosts, it gives the squeezed container more somewhere and leaves the other enough everywhere: where the pool has a smallest runner and the thin half sits at it, every share gives that half the same, so the notice says what sets it (raise the pool's smallest runner) and offers no share to apply.", "verify": "After the share changes, both containers' samples on the pool's page sit under their limits and the entry leaves the list within six hours.", "status_sentence": "A pool's two build containers are divided unevenly against the work, so one is short of room while the other idles.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.daemon_share_unsupported", "kind": "problem", "title": "pool.daemon_share_unsupported", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "A Docker-in-Docker pool asks for something other than an even split between the runner and its daemon, and some of the hosts it can land on run an agent too old to say it follows the pool's shares. The host is charged by the share while the agent gives each container half the slot, so the daemon is given less (or more) than the pool says, and nothing on the pool shows which runners that happened to. Upgrade the agent on each host named, or point the pool at other hosts with its host selector until they are; the sidecar-share advice proposes nothing meanwhile.", "fix": "A Docker-in-Docker pool asks for something other than an even split between the runner and its daemon, and some of the hosts it can land on run an agent too old to say it follows the pool's shares. The host is charged by the share while the agent gives each container half the slot, so the daemon is given less (or more) than the pool says, and nothing on the pool shows which runners that happened to. Upgrade the agent on each host named, or point the pool at other hosts with its host selector until they are; the sidecar-share advice proposes nothing meanwhile.", "verify": "After the agents are upgraded, the pool's page lists no host that cannot follow the shares and the entry leaves the list.", "status_sentence": "Some machines give a Docker-in-Docker runner's daemon half of its slot whatever the pool asks for, until their agent is upgraded.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.dangerous", "kind": "problem", "title": "pool.dangerous", "category": "security", "severity": "warning", "detection": "runtime", "detects": "The pool was configured to weaken the isolation between a workflow job and the host it runs on; the same sentence the pool page shows for the setting itself, so an operator sees one fact in both places rather than learning it twice. Edit the pool if this was not deliberate.", "fix": "The pool was configured to weaken the isolation between a workflow job and the host it runs on; the same sentence the pool page shows for the setting itself, so an operator sees one fact in both places rather than learning it twice. Edit the pool if this was not deliberate.", "verify": "The pool's page no longer shows the weakening setting and the entry leaves the list; it stays while the setting is deliberate.", "status_sentence": "Some runners are set up with more access to their machine than the safe default.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.docker_client_missing", "kind": "problem", "title": "pool.docker_client_missing", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "The pool's `docker_mode` gives its jobs a Docker daemon, and the image its runners boot carries no client to reach it with, so the daemon comes up unused and every job fails at its first Docker step with `Unable to locate executable file: docker`, which names the missing binary and not the reason. A pool that asks for a daemon is moved onto `ghcr.io/eyupio/zoomies-runner-docker` wherever that image is known to exist; a digest names one exact image, and a tag from another build may name a run whose variant was never published, so those are left as you set them. Pin the variant at the same tag or digest, or clear the pool's image so it follows the fleet's default and is moved for you.", "fix": "The pool's `docker_mode` gives its jobs a Docker daemon, and the image its runners boot carries no client to reach it with, so the daemon comes up unused and every job fails at its first Docker step with `Unable to locate executable file: docker`, which names the missing binary and not the reason. A pool that asks for a daemon is moved onto `ghcr.io/eyupio/zoomies-runner-docker` wherever that image is known to exist; a digest names one exact image, and a tag from another build may name a run whose variant was never published, so those are left as you set them. Pin the variant at the same tag or digest, or clear the pool's image so it follows the fleet's default and is moved for you.", "verify": "A job's first Docker step on the pool succeeds and the entry leaves the list.", "status_sentence": "Jobs that use Docker may fail because their runner image lacks the Docker client.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.elastic_cpu_unsupported", "kind": "problem", "title": "pool.elastic_cpu_unsupported", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The pool lends CPU to busy runners, and some of the hosts it can land on run an agent too old to say it can move a live runner's quota. A runner placed there is held at its guaranteed share, exactly as with elastic CPU off, and nothing on the pool says which of its runners that happened to. The dry run names the hosts: upgrade the agent on each (the command is on the host's card) or keep the pool on observe, which asks nothing of the agent, until they are.", "fix": "The pool lends CPU to busy runners, and some of the hosts it can land on run an agent too old to say it can move a live runner's quota. A runner placed there is held at its guaranteed share, exactly as with elastic CPU off, and nothing on the pool says which of its runners that happened to. The dry run names the hosts: upgrade the agent on each (the command is on the host's card) or keep the pool on observe, which asks nothing of the agent, until they are.", "verify": "After the agents are upgraded, the pool's page lists no host that cannot lend CPU and the entry leaves the list.", "status_sentence": "Runners cannot borrow spare CPU on some machines, so busy jobs run at their normal size.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.elastic_memory_unsupported", "kind": "problem", "title": "pool.elastic_memory_unsupported", "category": "capacity", "severity": "warning or info", "detection": "runtime", "detects": "The pool has the [memory valve](elastic-memory.md) on, and some of the hosts it can land on run an agent that cannot carry it out: one that predates the valve, or that is not on Linux, where the host's memory cannot be read the way a loan has to be checked. A runner placed there keeps the memory it was created with, exactly as with the valve off, and a pool that is only observing collects nothing from it. A warning for a pool set to lend, which is quietly not doing what it says; a note for one that only observes, which is where a new container pool starts, so that a fleet with one Windows or macOS agent in it is not \"degraded\" over a thing nobody asked of that agent. The dry run names the hosts: upgrade the agent on a Linux host (the command is on the host's card) or keep the pool off the others with its host selector.", "fix": "The pool has the [memory valve](elastic-memory.md) on, and some of the hosts it can land on run an agent that cannot carry it out: one that predates the valve, or that is not on Linux, where the host's memory cannot be read the way a loan has to be checked. A runner placed there keeps the memory it was created with, exactly as with the valve off, and a pool that is only observing collects nothing from it. A warning for a pool set to lend, which is quietly not doing what it says; a note for one that only observes, which is where a new container pool starts, so that a fleet with one Windows or macOS agent in it is not \"degraded\" over a thing nobody asked of that agent. The dry run names the hosts: upgrade the agent on a Linux host (the command is on the host's card) or keep the pool off the others with its host selector.", "verify": "After the agents are upgraded, the pool's page lists no host that cannot carry the valve and the entry leaves the list.", "status_sentence": "Runners cannot borrow spare memory on some machines, so a job that needs more is stopped at its normal limit.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.github_rate_limited", "kind": "problem", "title": "pool.github_rate_limited", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The pool has jobs waiting and its installation is inside a GitHub rate-limit backoff, so the scheduler is creating nothing for it rather than spending another call on a quota that is already gone. It lifts on its own when `poller.paused` does.", "fix": "The pool has jobs waiting and its installation is inside a GitHub rate-limit backoff, so the scheduler is creating nothing for it rather than spending another call on a quota that is already gone. It lifts on its own when `poller.paused` does.", "verify": "The entry leaves the list when `poller.stale` clears and the pool's runners start again; the pool's page no longer names a backoff.", "status_sentence": "GitHub is limiting how fast the fleet can register runners, so jobs may wait longer.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.history_unfit", "kind": "problem", "title": "pool.history_unfit", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "A job waiting on the pool is known, from its own recent runs, to need more CPU or memory than any host that can run the pool has to give. It is placed as it always was, so it is likely to fail the way it did before, and the jobs beside it are not held back for it. Add a larger host the pool can reach, or split the job.", "fix": "A job waiting on the pool is known, from its own recent runs, to need more CPU or memory than any host that can run the pool has to give. It is placed as it always was, so it is likely to fail the way it did before, and the jobs beside it are not held back for it. Add a larger host the pool can reach, or split the job.", "verify": "The job's next run lands on a host with room for it, which its page shows as the host chosen and the class it ran in, and the entry leaves the list.", "status_sentence": "A job needs more memory or CPU than any machine that can run it has, so it may fail again.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.host_overcommitted", "kind": "problem", "title": "pool.host_overcommitted", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "A host this pool can land on promises more runner slots than its machine can back at the size this pool typed; a pool sized by its host cannot raise it, because a slot is exactly one of its runners. The slots above what fits read as free capacity everywhere they are counted, and every create for one of them is refused for want of CPU or memory. Adjust the host to the slots its machine can back, or give the pool's runners less.", "fix": "A host this pool can land on promises more runner slots than its machine can back at the size this pool typed; a pool sized by its host cannot raise it, because a slot is exactly one of its runners. The slots above what fits read as free capacity everywhere they are counted, and every create for one of them is refused for want of CPU or memory. Adjust the host to the slots its machine can back, or give the pool's runners less.", "verify": "The host's card shows slots its machine can back at the pool's size, and the entry leaves the list.", "status_sentence": "Runners are sized to more than their machines have, so jobs may run slower.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.max_above_room", "kind": "problem", "title": "pool.max_above_room", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The pool's maximum is more runners than its hosts have room for at the size one of its runners is given there; a pool with a minimum counts the one reduced runner a host too small for another standard one still takes, at what it would be given. It is not wrong (the maximum is a backstop rather than a target) but the runners above the room are runners the scheduler will never create, and the jobs that ask for them wait with nothing else on any page saying why. Lower the maximum, ask for less per runner, or give the pool more hosts.", "fix": "The pool's maximum is more runners than its hosts have room for at the size one of its runners is given there; a pool with a minimum counts the one reduced runner a host too small for another standard one still takes, at what it would be given. It is not wrong (the maximum is a backstop rather than a target) but the runners above the room are runners the scheduler will never create, and the jobs that ask for them wait with nothing else on any page saying why. Lower the maximum, ask for less per runner, or give the pool more hosts.", "verify": "The pool's maximum is at or below the room its page shows, and the entry leaves the list.", "status_sentence": "The fleet is allowed more runners than its machines have room for.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.memory_ceiling_reached", "kind": "problem", "title": "pool.memory_ceiling_reached", "category": "capacity", "severity": "info", "detection": "runtime", "detects": "A runner of this pool held everything it may be lent, its own share and the pool's [memory ceiling](elastic-memory.md#the-ceiling) together, lowered by the host's where the host names one, and still wanted more. The entry names the hosts. Raise the ceiling (**Memory ceiling** in the pool editor, or `zoomies pools edit \u003cpool\u003e --memory-burst-max`), or give each runner more to start with. It clears fifteen minutes after the last time.", "fix": "A runner of this pool held everything it may be lent, its own share and the pool's [memory ceiling](elastic-memory.md#the-ceiling) together, lowered by the host's where the host names one, and still wanted more. The entry names the hosts. Raise the ceiling (**Memory ceiling** in the pool editor, or `zoomies pools edit \u003cpool\u003e --memory-burst-max`), or give each runner more to start with. It clears fifteen minutes after the last time.", "verify": "The job's next run completes within the raised ceiling: its page shows a peak below it, and the entry leaves the list.", "status_sentence": "Some jobs needed more memory than a runner may be lent.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.minimum_held_by_job", "kind": "problem", "title": "pool.minimum_held_by_job", "category": "reliability", "severity": "info", "detection": "runtime", "detects": "One kind of job holds a Docker-in-Docker pool's floor where it is. The pool's evidence is by pool, so a single heavy job (a weekly build, a release) is all it says, and every runner carries a slot sized for it; `pool.minimum_overcharges` stays silent because the floor has to hold that job. This notice is raised when the same conditions hold (jobs have waited, at least 20 measured jobs in the last week, none killed for memory, the sidecar evidence covers the pool) and, with the heaviest job name set aside, the rest of the pool's jobs used at most a figure that would put the slot at least a quarter lower and give the pool's hosts room for more runners. It names the job, its peak and how often it ran, says what the rest would fit in, and **proposes nothing to apply**: lowering the smallest runner before that job is somewhere else is what gets it killed. Move it first (a pool of its own, with a label only that job asks for, is the certain way; with `scheduler.size_routing` on, a size label or `zoomies size pin` sends it to a larger class of the pool's own runners; `scheduler.history_sizing=on` gives it a larger runner from what it used before, though only after a run or two) and then lower the smallest runner. Silent when there are too many job names to be sure which holds it.", "fix": "One kind of job holds a Docker-in-Docker pool's floor where it is. The pool's evidence is by pool, so a single heavy job (a weekly build, a release) is all it says, and every runner carries a slot sized for it; `pool.minimum_overcharges` stays silent because the floor has to hold that job. This notice is raised when the same conditions hold (jobs have waited, at least 20 measured jobs in the last week, none killed for memory, the sidecar evidence covers the pool) and, with the heaviest job name set aside, the rest of the pool's jobs used at most a figure that would put the slot at least a quarter lower and give the pool's hosts room for more runners. It names the job, its peak and how often it ran, says what the rest would fit in, and **proposes nothing to apply**: lowering the smallest runner before that job is somewhere else is what gets it killed. Move it first (a pool of its own, with a label only that job asks for, is the certain way; with `scheduler.size_routing` on, a size label or `zoomies size pin` sends it to a larger class of the pool's own runners; `scheduler.history_sizing=on` gives it a larger runner from what it used before, though only after a run or two) and then lower the smallest runner. Silent when there are too many job names to be sure which holds it.", "verify": "Once the heavy job is pinned to a larger class or given its own pool, the entry leaves the list and `pool.minimum_overcharges` can speak for the rest.", "status_sentence": "One kind of job in a pool is why every runner there is sized so large, and the rest would fit more runners on each machine in a smaller one.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "pool.minimum_overcharges", "kind": "problem", "title": "pool.minimum_overcharges", "category": "reliability", "severity": "info", "detection": "runtime", "detects": "A Docker-in-Docker pool sized by its hosts charges every runner far more memory than its jobs have used. The pool's smallest runner is a figure per container, so a thin sidecar multiplies it: a 1.5 GB minimum with the sidecar holding 15% of the memory is a request for a 10 GB slot, because only that gives the thinner half 1.5 GB. The machines pay for it, holding fewer runners than they could. Raised only when the pool's typical job has waited at least 30 seconds for a runner in the last six hours, over at least 20 jobs with a measured peak in the last week, none killed for memory (a pool that has been killed for it is never advised to shrink) and the most any job used, with half as much again, is well under the charge (by a quarter or more). A job's peak is its two containers added together, but each has a limit of its own, so the floor is also sized from the most each half used in any job of the last week, kept per job as its heartbeats arrive, and from the last few hours of heartbeats under the same coverage rule as the sidecar share (60 samples, three runners, half an hour, none more than half), whichever is larger: a thin sidecar that has used most of what it is given, in a weekly build as much as in a recent one, keeps the floor where it is. The advice waits until at least 20 jobs have been measured that way, which after an upgrade is a week of jobs, and with no such samples it waits too. Memory only: it is a limit a job is killed at, so what jobs used is evidence of what a slot has to hold, while a build's CPU peak is a burst the elastic valve lends for. When the controller can price it, it carries a remedy that lowers the smallest runner to what keeps every slot at 1.5 times the most a job has used, proposed only if the pool's hosts then have room for more runners; otherwise the detail says lowering it would not help and the limit is elsewhere.", "fix": "A Docker-in-Docker pool sized by its hosts charges every runner far more memory than its jobs have used. The pool's smallest runner is a figure per container, so a thin sidecar multiplies it: a 1.5 GB minimum with the sidecar holding 15% of the memory is a request for a 10 GB slot, because only that gives the thinner half 1.5 GB. The machines pay for it, holding fewer runners than they could. Raised only when the pool's typical job has waited at least 30 seconds for a runner in the last six hours, over at least 20 jobs with a measured peak in the last week, none killed for memory (a pool that has been killed for it is never advised to shrink) and the most any job used, with half as much again, is well under the charge (by a quarter or more). A job's peak is its two containers added together, but each has a limit of its own, so the floor is also sized from the most each half used in any job of the last week, kept per job as its heartbeats arrive, and from the last few hours of heartbeats under the same coverage rule as the sidecar share (60 samples, three runners, half an hour, none more than half), whichever is larger: a thin sidecar that has used most of what it is given, in a weekly build as much as in a recent one, keeps the floor where it is. The advice waits until at least 20 jobs have been measured that way, which after an upgrade is a week of jobs, and with no such samples it waits too. Memory only: it is a limit a job is killed at, so what jobs used is evidence of what a slot has to hold, while a build's CPU peak is a burst the elastic valve lends for. When the controller can price it, it carries a remedy that lowers the smallest runner to what keeps every slot at 1.5 times the most a job has used, proposed only if the pool's hosts then have room for more runners; otherwise the detail says lowering it would not help and the limit is elsewhere.", "verify": "After the smallest runner is lowered, the pool's hosts show room for more runners and the entry leaves the list on the next pass.", "status_sentence": "A pool's smallest runner, multiplied by its thin Docker sidecar, charges every runner far more memory than its jobs have ever used, so machines hold fewer runners while jobs wait.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "pool.no_capacity", "kind": "problem", "title": "pool.no_capacity", "category": "capacity", "severity": "error with jobs waiting, warning without", "detection": "runtime", "detects": "The pool cannot start the runners it wants. The entry carries the scheduler's own reason: no host matches its selector, every host is full, or every host is cordoned.", "fix": "The pool cannot start the runners it wants. The entry carries the scheduler's own reason: no host matches its selector, every host is full, or every host is cordoned.", "verify": "A runner starts for the pool: `zoomies runners list --pool \u003cpool-id\u003e` shows one past `provisioning`, and the entry leaves the list.", "status_sentence": "No machine has room for some jobs right now, so they wait until one frees up or a machine is added.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.no_eligible_host", "kind": "problem", "title": "pool.no_eligible_host", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "No host can run the pool, and a host's runner profile is why for at least one of them: a host's minimum runner size is above the size the pool states, or a host's standard is below the floor of a pool that takes its size from the host. The entry names each host with the limit that left it out, in the same words the pool's page and the refusal of a host edit use. The pool looks healthy and starts no runner while this holds, so nothing else on any page says why. It is raised only where a profile is involved; a pool no host can run for any other reason keeps the explanations it always had. Change those hosts' minimum or standard runner size, change the pool's size or minimum, or add a host whose profile suits it.", "fix": "No host can run the pool, and a host's runner profile is why for at least one of them: a host's minimum runner size is above the size the pool states, or a host's standard is below the floor of a pool that takes its size from the host. The entry names each host with the limit that left it out, in the same words the pool's page and the refusal of a host edit use. The pool looks healthy and starts no runner while this holds, so nothing else on any page says why. It is raised only where a profile is involved; a pool no host can run for any other reason keeps the explanations it always had. Change those hosts' minimum or standard runner size, change the pool's size or minimum, or add a host whose profile suits it.", "verify": "The pool's page lists at least one host that can run it and the entry leaves the list.", "status_sentence": "A pool has no host that can run it, because the hosts' runner profiles keep it off them.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.profile_default", "kind": "problem", "title": "pool.profile_default", "category": "capacity", "severity": "info", "detection": "runtime", "detects": "A pool that takes its size from its host is placed on hosts whose profiles name no standard runner size, so its runners there are the fleet's default size; the same on a small machine and a large one. The entry names the hosts and the default. A pool the controller keeps for a size class is not one of these: its runners are that class's size on a host that names none, which differs with the host because the host's class does. Give each of those hosts a standard runner size on its card, or set `runners.default_cpus` and `runners.default_memory_mb` to the size most of them should run.", "fix": "A pool that takes its size from its host is placed on hosts whose profiles name no standard runner size, so its runners there are the fleet's default size; the same on a small machine and a large one. The entry names the hosts and the default. A pool the controller keeps for a size class is not one of these: its runners are that class's size on a host that names none, which differs with the host because the host's class does. Give each of those hosts a standard runner size on its card, or set `runners.default_cpus` and `runners.default_memory_mb` to the size most of them should run.", "verify": "The named hosts' profiles show a standard runner size and the entry leaves the list.", "status_sentence": "A pool sized by its host is running at the fleet's default size on a host that names none.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.provision_timeout_short", "kind": "problem", "title": "pool.provision_timeout_short", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "A pool's own provision timeout lands inside the time its runners may legitimately take to start: the agent's create budget, plus the Docker wait a pool that provides a daemon does before it registers. Runners still coming up are failed and replaced, and the replacement pulls the same image over the link that was slow to begin with. This is `scheduler.provision_timeout_short` asked of one pool, because a pool may override either half for itself. Raise the pool's provision timeout above the two together, or clear it to follow the fleet.", "fix": "A pool's own provision timeout lands inside the time its runners may legitimately take to start: the agent's create budget, plus the Docker wait a pool that provides a daemon does before it registers. Runners still coming up are failed and replaced, and the replacement pulls the same image over the link that was slow to begin with. This is `scheduler.provision_timeout_short` asked of one pool, because a pool may override either half for itself. Raise the pool's provision timeout above the two together, or clear it to follow the fleet.", "verify": "Runners of the pool reach `idle` without being failed for the timeout, and the entry leaves the list after the setting changes.", "status_sentence": "Runners are given less time to start than they usually need, so some may be retried.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.queue_wait_high", "kind": "problem", "title": "pool.queue_wait_high", "category": "reliability", "severity": "info", "detection": "runtime", "detects": "A pool that keeps no runner warm (`min_runners` 0) has jobs that waited five minutes or more at the 95th percentile for a runner in the last six hours, over at least 10 jobs, while the median is short: a burst of jobs waits for the runners ahead of it to be made. Not raised for a pool the controller keeps, whose minimum is worked out from its hosts, or for one that already has a minimum. It carries a remedy that keeps two runners warm (fewer if the pool's maximum or its hosts' room is smaller), which holds idle capacity for the pool; if no host has room for a runner of the pool, the detail says so and there is no remedy.", "fix": "A pool that keeps no runner warm (`min_runners` 0) has jobs that waited five minutes or more at the 95th percentile for a runner in the last six hours, over at least 10 jobs, while the median is short: a burst of jobs waits for the runners ahead of it to be made. Not raised for a pool the controller keeps, whose minimum is worked out from its hosts, or for one that already has a minimum. It carries a remedy that keeps two runners warm (fewer if the pool's maximum or its hosts' room is smaller), which holds idle capacity for the pool; if no host has room for a runner of the pool, the detail says so and there is no remedy.", "verify": "The p95 wait on the pool's page falls below five minutes over the next six hours once a runner is kept warm.", "status_sentence": "A pool keeps no runner warm, and the slowest jobs wait minutes for one to be made, even though most start at once.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "pool.repository_scale_up_deferred", "kind": "problem", "title": "pool.repository_scale_up_deferred", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The per-repository creation limit held runners back. Expected under a burst; standing means the limit is too tight.", "fix": "The per-repository creation limit held runners back. Expected under a burst; standing means the limit is too tight.", "verify": "The entry leaves the list once runners for the repository are created without being held back; the pool's page shows no deferred starts.", "status_sentence": "Starting runners for some repositories is held back until GitHub allows it.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.resources_unenforced", "kind": "problem", "title": "pool.resources_unenforced", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "A pool on the `process` backend sets CPU, memory or disk limits, and that backend starts a runner as a plain process with no cgroup, so the limits bind nothing. The scheduler still holds that much room on the host, so the fleet does not oversubscribe; what is missing is the enforcement, and a job that runs away can take the machine with it. Move the pool to `docker` or `podman`, where the same limits become cgroup limits, or clear them.", "fix": "A pool on the `process` backend sets CPU, memory or disk limits, and that backend starts a runner as a plain process with no cgroup, so the limits bind nothing. The scheduler still holds that much room on the host, so the fleet does not oversubscribe; what is missing is the enforcement, and a job that runs away can take the machine with it. Move the pool to `docker` or `podman`, where the same limits become cgroup limits, or clear them.", "verify": "The entry leaves the list once the pool's limits are removed or it moves to a container backend; its page no longer shows limits it cannot apply.", "status_sentence": "Runner sizes are not enforced on some machines.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.runner_group_public_repositories_blocked", "kind": "problem", "title": "pool.runner_group_public_repositories_blocked", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "GitHub reports the pool's runner group as unavailable to public repositories. The runners can register, connect and appear idle, but GitHub will leave matching public-repository jobs queued. Enable **Allow public repositories** for that runner group under the organisation's **Settings \u003e Actions \u003e Runner groups**, or use a repository-scoped installation.", "fix": "GitHub reports the pool's runner group as unavailable to public repositories. The runners can register, connect and appear idle, but GitHub will leave matching public-repository jobs queued. Enable **Allow public repositories** for that runner group under the organisation's **Settings \u003e Actions \u003e Runner groups**, or use a repository-scoped installation.", "verify": "A queued public-repository job is picked up by the pool's next idle runner, and the entry leaves the list.", "status_sentence": "Jobs from public repositories are not allowed on some runners, so they will wait.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.runner_group_unresolved", "kind": "problem", "title": "pool.runner_group_unresolved", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "A pool asked for a runner group its target does not offer, or GitHub would not say which groups exist, so its runners registered in Default instead of the isolation boundary the pool requested.", "fix": "A pool asked for a runner group its target does not offer, or GitHub would not say which groups exist, so its runners registered in Default instead of the isolation boundary the pool requested.", "verify": "The pool's next runner registers in the group the pool names, which its page and GitHub's runner list both show.", "status_sentence": "Some runners cannot be registered in the group they belong to, so jobs for them wait.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.runners_failing", "kind": "problem", "title": "pool.runners_failing", "category": "capacity", "severity": "error with jobs waiting, warning without", "detection": "runtime", "detects": "Runners are being created and dying before they register. The fix is the category of the most recent failure (an image that cannot be pulled, a registration GitHub refused, a container backend that is not answering) and the usual causes only when nothing narrowed it. This is the failure with no failed job behind it: the jobs stay queued, nothing is marked failed, and every other count reads as a fleet that is merely busy.", "fix": "Runners are being created and dying before they register. The fix is the category of the most recent failure (an image that cannot be pulled, a registration GitHub refused, a container backend that is not answering) and the usual causes only when nothing narrowed it. This is the failure with no failed job behind it: the jobs stay queued, nothing is marked failed, and every other count reads as a fleet that is merely busy.", "verify": "Runners of the pool reach `idle`: `zoomies runners list --pool \u003cpool-id\u003e --state idle` is not empty, and the entry leaves the list.", "status_sentence": "Runners are failing to start, so jobs are waiting longer than usual.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.size_strands_hosts", "kind": "problem", "title": "pool.size_strands_hosts", "category": "capacity", "severity": "info", "detection": "runtime", "detects": "A pool asks for a fixed CPU and memory on every host, and some of its hosts could hold more runners of it than their slot count allows, so the machine above that slot count goes unused. Not wrong, and often deliberate; it is the state a fixed size drifts into as a fleet acquires unequal machines. Raise those hosts' capacity, or clear the pool's CPU and memory so each runner is given one slot's share of the host it lands on, which fills every slot on every machine whatever size it is.", "fix": "A pool asks for a fixed CPU and memory on every host, and some of its hosts could hold more runners of it than their slot count allows, so the machine above that slot count goes unused. Not wrong, and often deliberate; it is the state a fixed size drifts into as a fleet acquires unequal machines. Raise those hosts' capacity, or clear the pool's CPU and memory so each runner is given one slot's share of the host it lands on, which fills every slot on every machine whatever size it is.", "verify": "After the capacity is raised or the size removed, the host's card shows the extra slots in use and the entry leaves the list.", "status_sentence": "Runners are sized larger than some machines can hold, so those machines stay idle.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.size_unlimited", "kind": "problem", "title": "pool.size_unlimited", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "A pool sets no CPU or memory limit (which normally means each of its runners is given one slot's share of the machine it lands on) while `scheduler.default_runner_limits` is off, so that share is charged against the host and never applied. A host's worth of these runners can each take every core at once, which is the shape that stops the Docker daemon answering. Turn `scheduler.default_runner_limits` back on, which is the default, or give each pool a size of its own.", "fix": "A pool sets no CPU or memory limit (which normally means each of its runners is given one slot's share of the machine it lands on) while `scheduler.default_runner_limits` is off, so that share is charged against the host and never applied. A host's worth of these runners can each take every core at once, which is the shape that stops the Docker daemon answering. Turn `scheduler.default_runner_limits` back on, which is the default, or give each pool a size of its own.", "verify": "The pool's page shows a size, or `scheduler.default_runner_limits` is on, and the entry leaves the list.", "status_sentence": "Some runners have no size limit, so one job can slow the others on its machine.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.tmpfs_auto_on_disk", "kind": "problem", "title": "pool.tmpfs_auto_on_disk", "category": "capacity", "severity": "info", "detection": "runtime", "detects": "A pool that lets each runner decide where its in-memory folders live has an automatic folder on disk on some hosts, because the runner there has no room for it to be useful: below a floor of 2 GB for the work folder, 1 GB for `/tmp` and 4 GB for the Docker image store (or the size you typed, if that is smaller) a folder fills and fails jobs with \"no space left on device\" while saving little disk traffic. The setting working as designed, not a fault. It names the hosts and what each runner has, and says what would change it, worked out from the pool's own numbers: per host, the smallest runner size at which the work folder would be in memory (the host's standard runner memory for a pool that takes its size from its hosts, its capacity for one sized by a slot's share) and the slots that leaves, and a daemon share that would put more folders in memory *without losing a runner*, when one does. A share that costs runners is never offered, because raising a thinner half to its minimum grows the slot. The same per-host answer is under the folders in the pool editor's Speed section while the pool is being made.", "fix": "A pool that lets each runner decide where its in-memory folders live has an automatic folder on disk on some hosts, because the runner there has no room for it to be useful: below a floor of 2 GB for the work folder, 1 GB for `/tmp` and 4 GB for the Docker image store (or the size you typed, if that is smaller) a folder fills and fails jobs with \"no space left on device\" while saving little disk traffic. The setting working as designed, not a fault. It names the hosts and what each runner has, and says what would change it, worked out from the pool's own numbers: per host, the smallest runner size at which the work folder would be in memory (the host's standard runner memory for a pool that takes its size from its hosts, its capacity for one sized by a slot's share) and the slots that leaves, and a daemon share that would put more folders in memory *without losing a runner*, when one does. A share that costs runners is never offered, because raising a thinner half to its minimum grows the slot. The same per-host answer is under the folders in the pool editor's Speed section while the pool is being made.", "verify": "After the runner is given more room, the pool's page shows the folders in memory on those hosts and the entry leaves the list.", "status_sentence": "Some runners keep their working files on disk because they are too small for memory to be worth using.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.tmpfs_host_off", "kind": "problem", "title": "pool.tmpfs_host_off", "category": "capacity", "severity": "info", "detection": "runtime", "detects": "The pool keeps folders in memory, and an operator turned in-memory folders off in the runner profile of some of the hosts it can land on. A runner placed there has its folders on disk, as it would with the setting off. It is information and not a warning, because a machine's owner knows how much memory it has and the pool's does not, so the host having the last word is the design; it is said where the pool is saved because nothing on the pool says which of its runners that happened to. The fallback is a tactical fix and is kept visible for that reason. Clear *Fall back to disk on this machine (temporary)* under Runner sizes on the host's card, or `zoomies hosts edit --tmpfs-off=false`, to use memory there, set the host's folder sizes instead, or point the pool at other hosts with its host selector.", "fix": "The pool keeps folders in memory, and an operator turned in-memory folders off in the runner profile of some of the hosts it can land on. A runner placed there has its folders on disk, as it would with the setting off. It is information and not a warning, because a machine's owner knows how much memory it has and the pool's does not, so the host having the last word is the design; it is said where the pool is saved because nothing on the pool says which of its runners that happened to. The fallback is a tactical fix and is kept visible for that reason. Clear *Fall back to disk on this machine (temporary)* under Runner sizes on the host's card, or `zoomies hosts edit --tmpfs-off=false`, to use memory there, set the host's folder sizes instead, or point the pool at other hosts with its host selector.", "verify": "The entry leaves the list once the hosts' profiles allow in-memory folders or the pool stops asking for them.", "status_sentence": "Some machines keep runners' working files on disk by choice, so runners placed there are not sped up.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.tmpfs_memory_tight", "kind": "problem", "title": "pool.tmpfs_memory_tight", "category": "capacity", "severity": "warning or info", "detection": "runtime", "detects": "The pool keeps its work folder or `/tmp` in memory, and the room those folders may fill is a large part of the runner's memory limit. A tmpfs is charged to that limit, so the job has what the folders leave. A warning where sizes you typed take more than half the limit; a job that fills them is killed for want of memory; information where folders left to size themselves were fitted down because the limit is small. Either way the fix names the limit that leaves the job the room it has now with the folders on top. It is a proposal and never an edit, because the limit is also what the scheduler charges the host for. A pool whose runners are sized by their host has no limit of its own to judge, so it is judged host by host instead: the limit is one runner's charge there, divided with its Docker daemon by the pool's share, and the hosts where the folders came out smaller than asked for are named. A warning where a folder came out under half of what was asked, because a job that fills it fails with \"no space left on device\"; the fix is a runner size at which the folders fit, a different daemon share, or turning off the folder that does not.", "fix": "The pool keeps its work folder or `/tmp` in memory, and the room those folders may fill is a large part of the runner's memory limit. A tmpfs is charged to that limit, so the job has what the folders leave. A warning where sizes you typed take more than half the limit; a job that fills them is killed for want of memory; information where folders left to size themselves were fitted down because the limit is small. Either way the fix names the limit that leaves the job the room it has now with the folders on top. It is a proposal and never an edit, because the limit is also what the scheduler charges the host for. A pool whose runners are sized by their host has no limit of its own to judge, so it is judged host by host instead: the limit is one runner's charge there, divided with its Docker daemon by the pool's share, and the hosts where the folders came out smaller than asked for are named. A warning where a folder came out under half of what was asked, because a job that fills it fails with \"no space left on device\"; the fix is a runner size at which the folders fit, a different daemon share, or turning off the folder that does not.", "verify": "The pool's page shows its folders' room as a small part of the runner's memory limit, and the entry leaves the list.", "status_sentence": "Some runners could run out of memory because the working space they keep in memory takes much of their limit.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.tmpfs_suggested", "kind": "problem", "title": "pool.tmpfs_suggested", "category": "capacity", "severity": "info", "detection": "runtime", "detects": "The pool's jobs have been waiting on disk, and its machines have memory to spare. Raised only when, on one host, all three hold: the pool ran jobs there in the last six hours, the host's I/O wait has stayed above 20% for at least ten minutes (a reading CPU occupancy hides, because it counts that wait as busy), and the host has free memory beyond its own reserve for a work folder's default. It stays for as long as that is true and clears itself when the pool turns the setting on, the disk calms or the memory is spent. Turn on the in-memory work folder for the pool, and raise its memory limit by the proposed amount. See [Keeping the work folder in memory](hosts-and-pools.md#keeping-the-work-folder-in-memory).", "fix": "The pool's jobs have been waiting on disk, and its machines have memory to spare. Raised only when, on one host, all three hold: the pool ran jobs there in the last six hours, the host's I/O wait has stayed above 20% for at least ten minutes (a reading CPU occupancy hides, because it counts that wait as busy), and the host has free memory beyond its own reserve for a work folder's default. It stays for as long as that is true and clears itself when the pool turns the setting on, the disk calms or the memory is spent. Turn on the in-memory work folder for the pool, and raise its memory limit by the proposed amount. See [Keeping the work folder in memory](hosts-and-pools.md#keeping-the-work-folder-in-memory).", "verify": "After the folders move to memory, the host's I/O wait on its page falls below 10% and the entry leaves the list.", "status_sentence": "Jobs on some runners are slow and their machines have spare memory, so keeping working files in memory could speed them up.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "pool.tmpfs_unsupported", "kind": "problem", "title": "pool.tmpfs_unsupported", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The pool keeps folders in memory, and some of the hosts it can land on run an agent too old to say it can mount them. A runner placed there starts with its folders on disk, exactly as with the setting off, and nothing on the pool says which of its runners that happened to. The dry run names the hosts: upgrade the agent on each, or point the pool at other hosts with its host selector until they are.", "fix": "The pool keeps folders in memory, and some of the hosts it can land on run an agent too old to say it can mount them. A runner placed there starts with its folders on disk, exactly as with the setting off, and nothing on the pool says which of its runners that happened to. The dry run names the hosts: upgrade the agent on each, or point the pool at other hosts with its host selector until they are.", "verify": "After the agents are upgraded, the pool's page lists no host that cannot mount the folders and the entry leaves the list.", "status_sentence": "Some machines cannot keep runners' working files in memory, so those runners use disk as before.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "provider.bootstrap_failed", "kind": "problem", "title": "provider.bootstrap_failed", "category": "reliability", "severity": "error", "detection": "runtime", "detects": "The machine came up and its agent never did. The guest's own standard error is on the machine's page.", "fix": "Usually the template: a missing guest agent, a missing Zoomies binary, or an `agent.json` left in the image. See [Proxmox VE](proxmox.md#preparing-the-template).", "verify": null, "status_sentence": "A newly rented machine failed to join the fleet.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-infrastructure-providers" }, { "id": "provider.contract_unsupported", "kind": "problem", "title": "provider.contract_unsupported", "category": "reliability", "severity": "error", "detection": "runtime", "detects": "A provider declares a contract version this build does not speak. Machines it already owns stay visible, drainable and deletable; a version mismatch must never strand a running machine.", "fix": "Upgrade whichever side is behind; the message names both numbers.", "verify": null, "status_sentence": "The fleet cannot rent machines from one of its providers.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-infrastructure-providers" }, { "id": "provider.credentials_refused", "kind": "problem", "title": "provider.credentials_refused", "category": "reliability", "severity": "error", "detection": "runtime", "detects": "The API token was refused, or is not allowed to do something. Where the provider named a privilege, so does this.", "fix": "Run the provider's check, which lists every missing privilege and the path it is needed on.", "verify": null, "status_sentence": "A machine provider refused the fleet's credentials, so no new machines can be added from it.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-infrastructure-providers" }, { "id": "provider.delete_grace_short", "kind": "problem", "title": "provider.delete_grace_short", "category": "configuration", "severity": "warning", "detection": "static", "detects": "A machine whose host goes briefly quiet would be destroyed mid-job. Ninety seconds of silence only makes a host unhealthy; the runners on it are not given up until five minutes, and until they are the fleet still believes they are running jobs. Set it well above that, such as `10m`.", "fix": "A machine whose host goes briefly quiet would be destroyed mid-job. Ninety seconds of silence only makes a host unhealthy; the runners on it are not given up until five minutes, and until they are the fleet still believes they are running jobs. Set it well above that, such as `10m`.", "verify": null, "setting": "provider.delete_grace", "docs_html": "https://zoomies.sh/problem-codes/#configuration-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-infrastructure-providers" }, { "id": "provider.delete_pending", "kind": "problem", "title": "provider.delete_pending", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "A delete was issued and the resource has not been confirmed gone. Something is still costing money.", "fix": "Check the provider. A delete is only complete when an inspection cannot find the resource; a 200 from the delete call is not that.", "verify": null, "status_sentence": "A machine the fleet no longer needs is still being removed.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-infrastructure-providers" }, { "id": "provider.enrol_timeout", "kind": "problem", "title": "provider.enrol_timeout", "category": "configuration", "severity": "error", "detection": "static", "detects": "Shorter than the silence that loses a host, so machines that did arrive would be given up on.", "fix": "Shorter than the silence that loses a host, so machines that did arrive would be given up on.", "verify": null, "setting": "provider.enrol_timeout", "docs_html": "https://zoomies.sh/problem-codes/#configuration-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-infrastructure-providers" }, { "id": "provider.idle_timeout_short", "kind": "problem", "title": "provider.idle_timeout_short", "category": "configuration", "severity": "warning", "detection": "static", "detects": "A quiet host carries no runner anything will place, so it reads as idle from the moment it falls silent. Set at or below the five minutes its runners are given, a machine is drained for a network blip rather than for being unwanted. Set it above that, such as `15m`.", "fix": "A quiet host carries no runner anything will place, so it reads as idle from the moment it falls silent. Set at or below the five minutes its runners are given, a machine is drained for a network blip rather than for being unwanted. Set it above that, such as `15m`.", "verify": null, "setting": "provider.idle_timeout", "docs_html": "https://zoomies.sh/problem-codes/#configuration-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-infrastructure-providers" }, { "id": "provider.interval", "kind": "problem", "title": "provider.interval", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must be positive. It is how often machines are reconciled.", "fix": "Must be positive. It is how often machines are reconciled.", "verify": null, "setting": "provider.interval", "docs_html": "https://zoomies.sh/problem-codes/#configuration-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-infrastructure-providers" }, { "id": "provider.machine_failed", "kind": "problem", "title": "provider.machine_failed", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "A machine never reached ready, and carries the provider's own words for why.", "fix": "Read the machine's page. Three failures in a row stand the provider down, so a bad template costs a few machines rather than fifty.", "verify": null, "status_sentence": "A machine the fleet asked for did not arrive, so jobs may wait for room.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-infrastructure-providers" }, { "id": "provider.no_ceiling", "kind": "problem", "title": "provider.no_ceiling", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Providers are on but the ceiling is none, so nothing will be rented however much work queues. A maximum of none is none, as it is for a pool's `max_runners`.", "fix": "Providers are on but the ceiling is none, so nothing will be rented however much work queues. A maximum of none is none, as it is for a pool's `max_runners`.", "verify": null, "setting": "provider.max_machines", "docs_html": "https://zoomies.sh/problem-codes/#configuration-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-infrastructure-providers" }, { "id": "provider.orphan_found", "kind": "problem", "title": "provider.orphan_found", "category": "reliability", "severity": "error", "detection": "runtime", "detects": "A resource wearing this controller's marks has no row behind it. It is never deleted automatically; it may belong to another live fleet.", "fix": "Review it on the provider's Orphans tab. A restored database and a row pruned early are the two ways this happens.", "verify": null, "status_sentence": "A machine provider has a machine the fleet cannot account for.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-infrastructure-providers" }, { "id": "provider.ownership_unverified", "kind": "problem", "title": "provider.ownership_unverified", "category": "reliability", "severity": "error", "detection": "runtime", "detects": "A machine is quarantined: its row and the resource disagree about who owns what. **Nothing will touch it again until a person does.**", "fix": "Look at both sides. Then either release the row, which forgets the machine without touching the resource, or remove the resource by hand.", "verify": null, "status_sentence": "The fleet cannot confirm it owns a machine, so it is leaving it alone.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-infrastructure-providers" }, { "id": "provider.paused", "kind": "problem", "title": "provider.paused", "category": "configuration", "severity": "info", "detection": "static", "detects": "New machines are held by configuration. Draining, deleting, recovery and ownership checks all continue.", "fix": "New machines are held by configuration. Draining, deleting, recovery and ownership checks all continue.", "verify": null, "setting": "provider.paused", "docs_html": "https://zoomies.sh/problem-codes/#configuration-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-infrastructure-providers" }, { "id": "provider.preflight_failed", "kind": "problem", "title": "provider.preflight_failed", "category": "configuration", "severity": "error", "detection": "static", "detects": "The provider refused its connection check.", "fix": "Read the message; it carries the provider's own words.", "verify": null, "status_sentence": "A machine provider failed its checks, so no new machines can be added from it.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "provider.provisioning_paused", "kind": "problem", "title": "provider.provisioning_paused", "category": "reliability", "severity": "info", "detection": "runtime", "detects": "The kill switch is on. Draining, deleting, recovery and ownership checks all continue; only creation is held.", "fix": "Nothing, unless you did not mean it.", "verify": null, "status_sentence": "Adding machines is paused, so jobs wait for the machines the fleet already has.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-infrastructure-providers" }, { "id": "provider.quota_exhausted", "kind": "problem", "title": "provider.quota_exhausted", "category": "reliability", "severity": "warning, or **error** with jobs queued", "detection": "runtime", "detects": "The hypervisor refused for want of capacity. New machines stand down for a while; drains and deletes continue.", "fix": "Free space or capacity, or lower the provider's limit so the fleet stops asking.", "verify": null, "status_sentence": "A machine provider has no more room for the fleet, so jobs wait for the machines it has.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-infrastructure-providers" }, { "id": "provider.scale_down_fast", "kind": "problem", "title": "provider.scale_down_fast", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Machines are removed sooner than one idle period, so the quiet between two bursts pays the creation cost again.", "fix": "Machines are removed sooner than one idle period, so the quiet between two bursts pays the creation cost again.", "verify": null, "setting": "provider.scale_down_cooldown", "docs_html": "https://zoomies.sh/problem-codes/#configuration-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-infrastructure-providers" }, { "id": "provider.template_unverified", "kind": "problem", "title": "provider.template_unverified", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "The provider's settings changed and its prerequisites have not been checked since.", "fix": "Run the check. It creates nothing.", "verify": null, "status_sentence": "A machine template has not been checked yet.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-infrastructure-providers" }, { "id": "provider.timeouts", "kind": "problem", "title": "provider.timeouts", "category": "configuration", "severity": "error", "detection": "static", "detects": "A provider operation has no bound, or `provider.ambiguity_timeout` is not longer than `provider.create_timeout`, which would quarantine machines that are merely still being built.", "fix": "A provider operation has no bound, or `provider.ambiguity_timeout` is not longer than `provider.create_timeout`, which would quarantine machines that are merely still being built.", "verify": null, "setting": "provider.create_timeout", "docs_html": "https://zoomies.sh/problem-codes/#configuration-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-infrastructure-providers" }, { "id": "provider.unreachable", "kind": "problem", "title": "provider.unreachable", "category": "reliability", "severity": "warning, or **error** with a machine mid-operation", "detection": "runtime", "detects": "The hypervisor could not be reached. This is never evidence about a resource: nothing is created, failed or deleted on the strength of it.", "fix": "Check the endpoint, the network and the certificate. Machines already running are unaffected.", "verify": null, "status_sentence": "A machine provider cannot be reached, so no new machines can be added from it.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-infrastructure-providers" }, { "id": "provider.unservable", "kind": "problem", "title": "provider.unservable", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "No pool could ever place a runner on this provider's machines, so anything it rents is money for nothing.", "fix": "Match the provider's machine shape, labels and platform to a pool, or disable it.", "verify": null, "status_sentence": "Some jobs need machines no provider can supply.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-infrastructure-providers", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-infrastructure-providers" }, { "id": "provider.zone_missing", "kind": "problem", "title": "provider.zone_missing", "category": "configuration", "severity": "error", "detection": "static", "detects": "The provider has no zone (node, region) configured, so there is nowhere to put a machine.", "fix": "Set one.", "verify": null, "status_sentence": "A machine provider is missing a location the fleet is set to use.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.bridge_missing", "kind": "problem", "title": "proxmox.bridge_missing", "category": "configuration", "severity": "error", "detection": "static", "detects": "No network bridge is configured, or the configured one is not on that node. A machine with no network cannot reach this controller to enrol.", "fix": "Choose a bridge that can reach the controller.", "verify": null, "status_sentence": "A machine provider is missing a network the fleet is set to use.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.credentials_refused", "kind": "problem", "title": "proxmox.credentials_refused", "category": "configuration", "severity": "error", "detection": "static", "detects": "The API token was refused outright.", "fix": "Check the token's user, realm, token id and secret. The form wants them exactly as Proxmox printed them: `user@realm!tokenid=secret`.", "verify": null, "status_sentence": "A machine provider refused the fleet's credentials, so no new machines can be added from it.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.insecure_tls", "kind": "problem", "title": "proxmox.insecure_tls", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Certificate verification is off, so the token crosses to whoever answered.", "fix": "Paste the cluster's CA, `/etc/pve/pve-root-ca.pem`, into the provider's CA field instead. See [Security](security.md).", "verify": null, "status_sentence": "The fleet talks to a machine provider without checking its certificate.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.node_missing", "kind": "problem", "title": "proxmox.node_missing", "category": "configuration", "severity": "error", "detection": "static", "detects": "No node is configured, or the configured one is not in the cluster.", "fix": "Choose one the credential can see; the wizard lists them.", "verify": null, "status_sentence": "A machine provider is missing a server the fleet is set to use.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.node_offline", "kind": "problem", "title": "proxmox.node_offline", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The node is configured and not currently online.", "fix": "Machines cannot be created there until it returns.", "verify": null, "status_sentence": "A server at a machine provider is offline, so fewer machines can be added.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.privilege_missing", "kind": "problem", "title": "proxmox.privilege_missing", "category": "configuration", "severity": "error", "detection": "static", "detects": "The token is valid and not allowed to do something. The finding names the privilege **and** the path it is needed on.", "fix": "Grant that privilege. [Proxmox VE](proxmox.md#the-api-token) lists every one and why it is needed.", "verify": null, "status_sentence": "The fleet lacks a permission at a machine provider, so some machines cannot be added.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.storage_inactive", "kind": "problem", "title": "proxmox.storage_inactive", "category": "configuration", "severity": "error", "detection": "static", "detects": "The storage exists and is not active.", "fix": "Bring it up, or choose another.", "verify": null, "status_sentence": "Storage at a machine provider is inactive, so no new machines can be added there.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.storage_missing", "kind": "problem", "title": "proxmox.storage_missing", "category": "configuration", "severity": "error", "detection": "static", "detects": "No storage is configured, or the configured one does not exist on that node.", "fix": "Choose one the wizard lists.", "verify": null, "status_sentence": "A machine provider is missing storage the fleet is set to use.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.storage_no_images", "kind": "problem", "title": "proxmox.storage_no_images", "category": "configuration", "severity": "error", "detection": "static", "detects": "The storage exists and does not accept disk images, so a clone has nowhere to land.", "fix": "Choose a storage whose content types include `images`.", "verify": null, "status_sentence": "Storage at a machine provider cannot hold machine images.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.template_missing", "kind": "problem", "title": "proxmox.template_missing", "category": "configuration", "severity": "error", "detection": "static", "detects": "No template VMID is configured, or nothing exists at it.", "fix": "Prepare a template as the [runbook](proxmox.md#preparing-the-template) describes and give its VMID.", "verify": null, "status_sentence": "A machine provider is missing the template new machines are made from.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.template_no_agent", "kind": "problem", "title": "proxmox.template_no_agent", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The template does not have the QEMU guest agent enabled. Enrolment reaches the guest through it, so a machine made from this template will boot, cost money and never join.", "fix": "Install `qemu-guest-agent` in the image and set `agent: enabled=1`.", "verify": null, "status_sentence": "The template new machines are made from cannot report their address.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.template_not_a_template", "kind": "problem", "title": "proxmox.template_not_a_template", "category": "configuration", "severity": "error", "detection": "static", "detects": "A VM exists at that VMID and is not a template. Cloning a running VM is not what this does.", "fix": "Convert it to a template, or point at the right VMID.", "verify": null, "status_sentence": "The template new machines are made from is set up the wrong way.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.unreachable", "kind": "problem", "title": "proxmox.unreachable", "category": "configuration", "severity": "error", "detection": "static", "detects": "The cluster could not be reached at all.", "fix": "Check the endpoint, the port (8006), the network and the certificate.", "verify": null, "status_sentence": "A machine provider cannot be reached, so no new machines can be added from it.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.version_unqualified", "kind": "problem", "title": "proxmox.version_unqualified", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The cluster is older than the release this integration was qualified against. It is not refused, but nothing about it has been tested.", "fix": "Upgrade, or proceed knowing it is unqualified.", "verify": null, "status_sentence": "A machine provider runs a version this fleet has not been tested with.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.vmid_range", "kind": "problem", "title": "proxmox.vmid_range", "category": "configuration", "severity": "error", "detection": "static", "detects": "No VMID range is configured, or its bounds are the wrong way round. The range is both a budget and a blast radius: a VM outside it is by construction not ours.", "fix": "Give a block nothing else allocates from.", "verify": null, "status_sentence": "A machine provider's numbering range is too small for the machines the fleet may need.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxmox.vmid_range_reserved", "kind": "problem", "title": "proxmox.vmid_range_reserved", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Guests already exist inside the configured range. They are not touched, but the range is meant to be Zoomies' alone.", "fix": "Move the range, or move those guests.", "verify": null, "status_sentence": "A machine provider's numbering range overlaps machines the fleet does not own.", "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "proxy.bad_cidr", "kind": "problem", "title": "proxy.bad_cidr", "category": "configuration", "severity": "error", "detection": "static", "detects": "An entry is not an IP address or a CIDR block.", "fix": "An entry is not an IP address or a CIDR block.", "verify": null, "setting": "server.trusted_proxies", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "proxy.trust_everyone", "kind": "problem", "title": "proxy.trust_everyone", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Any client can claim any address, so the audit log's IPs mean nothing. Name your proxy's range instead.", "fix": "Any client can claim any address, so the audit log's IPs mean nothing. Name your proxy's range instead.", "verify": null, "setting": "server.trusted_proxies", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "proxy.untrusted", "kind": "problem", "title": "proxy.untrusted", "category": "configuration", "severity": "info", "detection": "static", "detects": "Client IPs in the audit log are the socket's, not the header's. Correct behind nothing; wrong behind a proxy, which is what the setting is for.", "fix": "Client IPs in the audit log are the socket's, not the header's. Correct behind nothing; wrong behind a proxy, which is what the setting is for.", "verify": null, "setting": "server.trusted_proxies", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "recovery.fenced", "kind": "problem", "title": "recovery.fenced", "category": "capacity", "severity": "error", "detection": "runtime", "detects": "This fleet was restored from a backup and is held: the scheduler decides as normal and applies none of it; no runner is created, drained or removed, nothing is reaped from GitHub, and the fallback poller does not sweep. The Overview shows what it *would* do, so you can tell \"nothing to do\" from \"not allowed to\". `/readyz` answers 503 while it is on; liveness is unaffected, so a container runtime does not restart it. The fix names the three things a restore does not bring with it, and lifting is `POST /api/v1/recovery/unfence`.", "fix": "This fleet was restored from a backup and is held: the scheduler decides as normal and applies none of it; no runner is created, drained or removed, nothing is reaped from GitHub, and the fallback poller does not sweep. The Overview shows what it *would* do, so you can tell \"nothing to do\" from \"not allowed to\". `/readyz` answers 503 while it is on; liveness is unaffected, so a container runtime does not restart it. The fix names the three things a restore does not bring with it, and lifting is `POST /api/v1/recovery/unfence`.", "verify": null, "status_sentence": "The fleet is recovering from a restore and is not starting new runners yet.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "retention.audit_renamed", "kind": "problem", "title": "retention.audit_renamed", "category": "configuration", "severity": "info", "detection": "static", "detects": "The key was renamed to `retention.scaling_events`, which is all it ever bounded; audit rows are never pruned. The value is still honoured. Rename it.", "fix": "The key was renamed to `retention.scaling_events`, which is all it ever bounded; audit rows are never pruned. The value is still honoured. Rename it.", "verify": null, "setting": "retention.audit", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "retention.jobs_short", "kind": "problem", "title": "retention.jobs_short", "category": "configuration", "severity": "info", "detection": "static", "detects": "Job history is kept for less than the week the sizing advice counts. Advice to give a pool a smaller runner, or a host smaller runners, rests on what a week of jobs used, so it is not given. Keep a week or more.", "fix": "Job history is kept for less than the week the sizing advice counts. Advice to give a pool a smaller runner, or a host smaller runners, rests on what a week of jobs used, so it is not given. Keep a week or more.", "verify": null, "setting": "retention.jobs", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "runners.class_sizes", "kind": "problem", "title": "runners.class_sizes", "category": "configuration", "severity": "error", "detection": "static", "detects": "The runner sizes of the classes are not positive and growing. A job is classed as the smallest runner that holds it, so a larger class with a smaller runner would never be chosen. The defaults are 1 CPU and 2 GB, 2 CPUs and 4 GB, and 4 CPUs and 8 GB.", "fix": "The runner sizes of the classes are not positive and growing. A job is classed as the smallest runner that holds it, so a larger class with a smaller runner would never be chosen. The defaults are 1 CPU and 2 GB, 2 CPUs and 4 GB, and 4 CPUs and 8 GB.", "verify": null, "setting": "runners.small_cpus", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "runners.cleanup_failed", "kind": "problem", "title": "runners.cleanup_failed", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "Zoomies could not finish taking a runner away: a container still on its host, or a registration still on GitHub. Different from `runners.failed`, which is a job that did not run; this is something *left behind*. Most often GitHub's own bookkeeping is a few seconds behind the webhook that told Zoomies the job was done, and this clears on the next retry with nothing to do. A container Docker is still removing is waited for rather than raised, and appears here only once the removal has gone ten minutes without finishing; see [Cleanup](troubleshooting.md#what-zoomies-cleans-up-and-what-it-leaves) for the other shapes and what each needs.", "fix": "Zoomies could not finish taking a runner away: a container still on its host, or a registration still on GitHub. Different from `runners.failed`, which is a job that did not run; this is something *left behind*. Most often GitHub's own bookkeeping is a few seconds behind the webhook that told Zoomies the job was done, and this clears on the next retry with nothing to do. A container Docker is still removing is waited for rather than raised, and appears here only once the removal has gone ten minutes without finishing; see [Cleanup](troubleshooting.md#what-zoomies-cleans-up-and-what-it-leaves) for the other shapes and what each needs.", "verify": "`zoomies runners list` no longer shows the runner, the host carries no container for it, GitHub's runner list has no registration, and the entry leaves the list.", "status_sentence": "Some finished runners could not be cleaned up, which uses room new jobs need.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "runners.docker_wait", "kind": "problem", "title": "runners.docker_wait", "category": "configuration", "severity": "error", "detection": "static", "detects": "The runner image accepts a wait of one second to one hour and exits with a configuration error for anything else, so no runner on a Docker pool would take a job. Use a duration in that range, or 0 to leave the image's default.", "fix": "The runner image accepts a wait of one second to one hour and exits with a configuration error for anything else, so no runner on a Docker pool would take a job. Use a duration in that range, or 0 to leave the image's default.", "verify": null, "setting": "runners.docker_wait", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "runners.docker_wait_short", "kind": "problem", "title": "runners.docker_wait_short", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Less than the recommended 3m for a loaded DinD daemon to become ready.", "fix": "Less than the recommended 3m for a loaded DinD daemon to become ready.", "verify": null, "setting": "runners.docker_wait", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-agent", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-agent" }, { "id": "runners.env_reserved", "kind": "problem", "title": "runners.env_reserved", "category": "configuration", "severity": "error", "detection": "static", "detects": "It names a variable the controller writes for each runner individually; its JIT configuration, name, labels, group or credentials. One value for the whole fleet is wrong for every runner in it; remove it.", "fix": "It names a variable the controller writes for each runner individually; its JIT configuration, name, labels, group or credentials. One value for the whole fleet is wrong for every runner in it; remove it.", "verify": null, "setting": "runners.env", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "runners.failed", "kind": "problem", "title": "runners.failed", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "Runners are in the failed state with their reasons recorded.", "fix": "Runners are in the failed state with their reasons recorded.", "verify": "`zoomies runners list --state failed` is empty and the entry leaves the list once the failed runners are gone.", "status_sentence": "Some runners failed, so jobs may wait while they are replaced.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "runners.not_progressing", "kind": "problem", "title": "runners.not_progressing", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "Runners have sat in `provisioning` or `registering` for over half the provision timeout, so the fleet says so while there is still time to look rather than only when it fails them. The entry splits the two shapes, because they are not fixed in the same place: a runner still waiting for a container is a backend or image problem on the host, and one whose container started without registering is the runner process failing to reach GitHub.", "fix": "Runners have sat in `provisioning` or `registering` for over half the provision timeout, so the fleet says so while there is still time to look rather than only when it fails them. The entry splits the two shapes, because they are not fixed in the same place: a runner still waiting for a container is a backend or image problem on the host, and one whose container started without registering is the runner process failing to reach GitHub.", "verify": "The runner reaches `idle` or is failed with its reason: `zoomies runners list --state provisioning` and `--state registering` show nothing older than half the timeout.", "status_sentence": "Some runners are taking far longer to start than they should, so jobs are waiting.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "scheduler.auto_pools", "kind": "problem", "title": "scheduler.auto_pools", "category": "configuration", "severity": "error", "detection": "static", "detects": "Use `off`, `shadow` or `on`.", "fix": "Use `off`, `shadow` or `on`.", "verify": null, "setting": "scheduler.auto_pools", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.auto_pools_docker_mode", "kind": "problem", "title": "scheduler.auto_pools_docker_mode", "category": "configuration", "severity": "error", "detection": "static", "detects": "Use `none` or `dind`. The host's own Docker socket is not offered to a pool the controller makes.", "fix": "Use `none` or `dind`. The host's own Docker socket is not offered to a pool the controller makes.", "verify": null, "setting": "scheduler.auto_pools_docker_mode", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.auto_pools_host_grace", "kind": "problem", "title": "scheduler.auto_pools_host_grace", "category": "configuration", "severity": "error", "detection": "static", "detects": "The grace before a silent host stops counting towards its pool is not longer than the five minutes after which the fleet gives its runners up. A pool would shrink while the host's jobs were still being counted. Set it above 5m; the default is 10m.", "fix": "The grace before a silent host stops counting towards its pool is not longer than the five minutes after which the fleet gives its runners up. A pool would shrink while the host's jobs were still being counted. Set it above 5m; the default is 10m.", "verify": null, "setting": "scheduler.auto_pools_host_grace", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.auto_rerun_limit", "kind": "problem", "title": "scheduler.auto_rerun_limit", "category": "configuration", "severity": "error", "detection": "static", "detects": "The bound on automatic re-runs is outside 1–5. Zero would make `scheduler.auto_rerun` read as on while doing nothing; a large one turns a fault the fleet causes every time into a bill.", "fix": "The bound on automatic re-runs is outside 1–5. Zero would make `scheduler.auto_rerun` read as on while doing nothing; a large one turns a fault the fleet causes every time into a bill.", "verify": null, "setting": "scheduler.auto_rerun_limit", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.auto_rerun_on", "kind": "problem", "title": "scheduler.auto_rerun_on", "category": "configuration", "severity": "warning", "detection": "static", "detects": "A job whose runner died under it is sent back to GitHub automatically, without anybody asking. That spends the installation's GitHub minutes, and a job that got as far as running may have had side effects its author expected to happen once. Turn it off to leave the re-run to the button on the job, or lower `scheduler.auto_rerun_limit`.", "fix": "A job whose runner died under it is sent back to GitHub automatically, without anybody asking. That spends the installation's GitHub minutes, and a job that got as far as running may have had side effects its author expected to happen once. Turn it off to leave the re-run to the button on the job, or lower `scheduler.auto_rerun_limit`.", "verify": null, "setting": "scheduler.auto_rerun", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.burst", "kind": "problem", "title": "scheduler.burst", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must be at least 1.", "fix": "Must be at least 1.", "verify": null, "setting": "scheduler.max_creates_per_tick", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.default_runner_limits_off", "kind": "problem", "title": "scheduler.default_runner_limits_off", "category": "configuration", "severity": "warning", "detection": "static", "detects": "A pool that sets no `cpus` or `memory_mb` gets runners with no cgroup limit at all, so a host's worth of them can each take every core and all of the memory; the shape that stops Docker answering. Leave it on, or set both on every pool.", "fix": "A pool that sets no `cpus` or `memory_mb` gets runners with no cgroup limit at all, so a host's worth of them can each take every core and all of the memory; the shape that stops Docker answering. Leave it on, or set both on every pool.", "verify": null, "setting": "scheduler.default_runner_limits", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.history_sizing", "kind": "problem", "title": "scheduler.history_sizing", "category": "configuration", "severity": "error", "detection": "static", "detects": "Use `off`, `shadow` or `on`.", "fix": "Use `off`, `shadow` or `on`.", "verify": null, "setting": "scheduler.history_sizing", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.host_order", "kind": "problem", "title": "scheduler.host_order", "category": "configuration", "severity": "error", "detection": "static", "detects": "Use `headroom`, `largest_standard` or `best_fit`.", "fix": "Use `headroom`, `largest_standard` or `best_fit`.", "verify": null, "setting": "scheduler.host_order", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.host_throttling_off", "kind": "problem", "title": "scheduler.host_throttling_off", "category": "configuration", "severity": "warning", "detection": "static", "detects": "An overwhelmed host is never stepped down: the pressure holds still refuse new starts while CPU or memory is acutely short, but nothing outlasts a sample, and the runners already on the host are never slowed. Leave it on unless something outside Zoomies manages the hosts' load.", "fix": "An overwhelmed host is never stepped down: the pressure holds still refuse new starts while CPU or memory is acutely short, but nothing outlasts a sample, and the runners already on the host are never slowed. Leave it on unless something outside Zoomies manages the hosts' load.", "verify": null, "setting": "scheduler.host_throttling", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.interval", "kind": "problem", "title": "scheduler.interval", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must be positive.", "fix": "Must be positive.", "verify": null, "setting": "scheduler.interval", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.lifetime_short", "kind": "problem", "title": "scheduler.lifetime_short", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Idle runners are recycled sooner than a long job takes, so work may be interrupted.", "fix": "Idle runners are recycled sooner than a long job takes, so work may be interrupted.", "verify": null, "setting": "scheduler.max_runner_lifetime", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.placement_mode", "kind": "problem", "title": "scheduler.placement_mode", "category": "configuration", "severity": "error", "detection": "static", "detects": "Use `headroom`, `shadow` or `readiness`.", "fix": "Use `headroom`, `shadow` or `readiness`.", "verify": null, "setting": "scheduler.placement_mode", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.provision_timeout_short", "kind": "problem", "title": "scheduler.provision_timeout_short", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Runners are failed sooner than they may legitimately take to start. An agent allows itself fifteen minutes for a create, because a cold image pull on a slow link is minutes rather than seconds, and a runner on a pool that provides Docker then waits for that daemon before it registers. A timeout inside the two together condemns runners that are still coming up, and the replacement pulls the same image over the same link. Set it above both, such as `20m`.", "fix": "Runners are failed sooner than they may legitimately take to start. An agent allows itself fifteen minutes for a create, because a cold image pull on a slow link is minutes rather than seconds, and a runner on a pool that provides Docker then waits for that daemon before it registers. A timeout inside the two together condemns runners that are still coming up, and the replacement pulls the same image over the same link. Set it above both, such as `20m`.", "verify": null, "setting": "scheduler.provision_timeout", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.registration_concurrency", "kind": "problem", "title": "scheduler.registration_concurrency", "category": "configuration", "severity": "error", "detection": "static", "detects": "Credential request concurrency is outside 1–16.", "fix": "Use 1 by default; increase only with measured need.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#a-providers-preflight", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#a-providers-preflight" }, { "id": "scheduler.registration_throttled", "kind": "problem", "title": "scheduler.registration_throttled", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "The last scheduling pass had a host chosen for a runner and did not create it, because the installation was already at `scheduler.registration_concurrency` credential requests in flight. Nothing fails: the demand is kept and a later pass takes it, so it shows as runners appearing slowly. It is listed because it is the one limit that used to bind silently; the pool would report jobs waiting, and an operator reading that adds hosts, which cannot help when hosts are not what ran out. Raise the setting if the installation's GitHub quota has room; leave it if the fleet is deliberately gentle with that quota.", "fix": "The last scheduling pass had a host chosen for a runner and did not create it, because the installation was already at `scheduler.registration_concurrency` credential requests in flight. Nothing fails: the demand is kept and a later pass takes it, so it shows as runners appearing slowly. It is listed because it is the one limit that used to bind silently; the pool would report jobs waiting, and an operator reading that adds hosts, which cannot help when hosts are not what ran out. Raise the setting if the installation's GitHub quota has room; leave it if the fleet is deliberately gentle with that quota.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-pools-jobs-and-runners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-pools-jobs-and-runners" }, { "id": "scheduler.size_class_limits", "kind": "problem", "title": "scheduler.size_class_limits", "category": "configuration", "severity": "error", "detection": "static", "detects": "The limits between the size classes are not positive, or the medium ones are below the small ones, which would leave a class empty. Set `scheduler.size_small_max_*` no larger than `scheduler.size_medium_max_*`; the defaults are 4 CPUs and 16 GB, and 12 CPUs and 48 GB.", "fix": "The limits between the size classes are not positive, or the medium ones are below the small ones, which would leave a class empty. Set `scheduler.size_small_max_*` no larger than `scheduler.size_medium_max_*`; the defaults are 4 CPUs and 16 GB, and 12 CPUs and 48 GB.", "verify": null, "setting": "scheduler.size_small_max_cpus", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.size_default_class", "kind": "problem", "title": "scheduler.size_default_class", "category": "configuration", "severity": "error", "detection": "static", "detects": "Use `small`, `medium` or `large`.", "fix": "Use `small`, `medium` or `large`.", "verify": null, "setting": "scheduler.size_default_class", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.size_routing", "kind": "problem", "title": "scheduler.size_routing", "category": "configuration", "severity": "error", "detection": "static", "detects": "Use `off`, `shadow` or `on`.", "fix": "Use `off`, `shadow` or `on`.", "verify": null, "setting": "scheduler.size_routing", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.size_routing_without_pools", "kind": "problem", "title": "scheduler.size_routing_without_pools", "category": "configuration", "severity": "info", "detection": "static", "detects": "Size routing is on and automatic pools are off, so no pool is kept for a class and a job is only answered by a pool of yours that carries its size label. Turn `scheduler.auto_pools` on, or label your own pools `zoomies-small`, `zoomies-medium` and `zoomies-large`.", "fix": "Size routing is on and automatic pools are off, so no pool is kept for a class and a job is only answered by a pool of yours that carries its size label. Turn `scheduler.auto_pools` on, or label your own pools `zoomies-small`, `zoomies-medium` and `zoomies-large`.", "verify": null, "setting": "scheduler.auto_pools", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "scheduler.size_wait_negative", "kind": "problem", "title": "scheduler.size_wait_negative", "category": "configuration", "severity": "error", "detection": "static", "detects": "A size routing wait is negative. Use a duration, or 0 for none.", "fix": "A size routing wait is negative. Use a duration, or 0 for none.", "verify": null, "setting": "scheduler.size_class_hold", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "security.auto_apply_remedies", "kind": "problem", "title": "security.auto_apply_remedies", "category": "security", "severity": "error", "detection": "static", "detects": "Use `off`, `shadow` or `on` (`false` and `true` are still read as `off` and `on`).", "fix": "Use `off`, `shadow` or `on` (`false` and `true` are still read as `off` and `on`).", "verify": null, "setting": "security.auto_apply_remedies", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "settings.imported_from_file", "kind": "problem", "title": "settings.imported_from_file", "category": "configuration", "severity": "info", "detection": "static", "detects": "The settings your `zoomies.yaml` spells have been copied into the database, once, on the first start after upgrading. Nothing about what the controller runs has changed; the file is still the layer underneath them.", "fix": "The settings your `zoomies.yaml` spells have been copied into the database, once, on the first start after upgrading. Nothing about what the controller runs has changed; the file is still the layer underneath them.", "verify": null, "setting": "-", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-settings-in-the-database", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-settings-in-the-database" }, { "id": "settings.stored_invalid", "kind": "problem", "title": "settings.stored_invalid", "category": "configuration", "severity": "warning", "detection": "static", "detects": "A stored value will not parse, so the layer underneath it is in force; the configuration file, or the built-in default. Set it again on the settings page, or clear it with `zoomies config unset \u003ckey\u003e`.", "fix": "A stored value will not parse, so the layer underneath it is in force; the configuration file, or the built-in default. Set it again on the settings page, or clear it with `zoomies config unset \u003ckey\u003e`.", "verify": null, "setting": "the setting named", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-settings-in-the-database", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-settings-in-the-database" }, { "id": "settings.stored_unknown", "kind": "problem", "title": "settings.stored_unknown", "category": "configuration", "severity": "info", "detection": "static", "detects": "Settings are stored that this version does not have, usually because a newer one set them and this is a rollback. They are kept untouched, so upgrading again picks them up where it left off.", "fix": "Settings are stored that this version does not have, usually because a newer one set them and this is a rollback. They are kept untouched, so upgrading again picks them up where it left off.", "verify": null, "setting": "-", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-settings-in-the-database", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-settings-in-the-database" }, { "id": "settings.stored_unreadable", "kind": "problem", "title": "settings.stored_unreadable", "category": "configuration", "severity": "error", "detection": "static", "detects": "A stored credential was sealed with a different encryption key. Starting anyway would run the fleet with credentials silently missing. Restore the key it was sealed with, or clear the setting and set it again.", "fix": "A stored credential was sealed with a different encryption key. Starting anyway would run the fleet with credentials silently missing. Restore the key it was sealed with, or clear the setting and set it again.", "verify": null, "setting": "-", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-settings-in-the-database", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-settings-in-the-database" }, { "id": "status.mode", "kind": "problem", "title": "status.mode", "category": "configuration", "severity": "error", "detection": "static", "detects": "Not a mode. They are `off`, `authenticated` and `public`.", "fix": "Not a mode. They are `off`, `authenticated` and `public`.", "verify": null, "setting": "status.mode", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "status.public", "kind": "problem", "title": "status.public", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The fleet status at `/status`, `/status.svg` and `/api/v1/status` answers without an account. It names nothing, but anyone who can reach the controller learns that the fleet exists, which release it runs, whether it is blocked, roughly how busy it is and the codes of its current problems. Set `status.mode` to `authenticated` if only people with an account should see it.", "fix": "The fleet status at `/status`, `/status.svg` and `/api/v1/status` answers without an account. It names nothing, but anyone who can reach the controller learns that the fleet exists, which release it runs, whether it is blocked, roughly how busy it is and the codes of its current problems. Set `status.mode` to `authenticated` if only people with an account should see it.", "verify": null, "setting": "status.mode", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "status.public_no_tls", "kind": "problem", "title": "status.public_no_tls", "category": "configuration", "severity": "error", "detection": "static", "detects": "`status.mode` is `public` on a listener that is not on loopback and does not terminate TLS, so the address it invites people to would carry every sign-in in the clear. Set `server.tls.mode` to `self-signed` or `files`, bind `server.bind` to loopback behind a proxy that terminates TLS, or set `status.mode` to `authenticated`.", "fix": "`status.mode` is `public` on a listener that is not on loopback and does not terminate TLS, so the address it invites people to would carry every sign-in in the clear. Set `server.tls.mode` to `self-signed` or `files`, bind `server.bind` to loopback behind a proxy that terminates TLS, or set `status.mode` to `authenticated`.", "verify": null, "setting": "status.mode", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "tailcat.unavailable", "kind": "problem", "title": "tailcat.unavailable", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "The controller's private-connection listener has no Tailcat relay it can reach, so hosts enrolled with a [private connection](private-hosts.md) cannot heartbeat or take work; direct hosts are unaffected. The entry carries the last attempt's reason. It is the platform's, because the fix is the controller's own outbound access. Nothing needs restarting: the controller retries every twenty seconds, tries the relay its identity was sealed with first and another only while that one does not answer, and the entry clears once one does. A listener that cannot start no longer stops the controller starting.", "fix": "The controller's private-connection listener has no Tailcat relay it can reach, so hosts enrolled with a [private connection](private-hosts.md) cannot heartbeat or take work; direct hosts are unaffected. The entry carries the last attempt's reason. It is the platform's, because the fix is the controller's own outbound access. Nothing needs restarting: the controller retries every twenty seconds, tries the relay its identity was sealed with first and another only while that one does not answer, and the entry clears once one does. A listener that cannot start no longer stops the controller starting.", "verify": null, "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "tls.file_unreadable", "kind": "problem", "title": "tls.file_unreadable", "category": "configuration", "severity": "error", "detection": "static", "detects": "The file is named but cannot be read. Usually ownership after an install as another user.", "fix": "The file is named but cannot be read. Usually ownership after an install as another user.", "verify": null, "setting": "server.tls.*", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "tls.files_missing", "kind": "problem", "title": "tls.files_missing", "category": "configuration", "severity": "error", "detection": "static", "detects": "`mode: files` needs both a certificate and a key.", "fix": "`mode: files` needs both a certificate and a key.", "verify": null, "setting": "server.tls", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "tls.mode_unknown", "kind": "problem", "title": "tls.mode_unknown", "category": "configuration", "severity": "error", "detection": "static", "detects": "Not a TLS mode. The modes are `off`, `self_signed` and `files`.", "fix": "Not a TLS mode. The modes are `off`, `self_signed` and `files`.", "verify": null, "setting": "server.tls.mode", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "tls.self_signed", "kind": "problem", "title": "tls.self_signed", "category": "configuration", "severity": "info", "detection": "static", "detects": "Browsers and agents will not trust it without being told to. Fine for a private network, not for anything else.", "fix": "Browsers and agents will not trust it without being told to. Fine for a private network, not for anything else.", "verify": null, "setting": "server.tls.mode", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-listener", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-listener" }, { "id": "two_step.required", "kind": "problem", "title": "two_step.required", "category": "configuration", "severity": "info", "detection": "static", "detects": "Every account that signs in with a password is asked for an authenticator code, and one without an authenticator sets it up at its next sign-in. It says what it does not cover (sessions that already exist, API tokens, and single sign-on accounts, whose second factor belongs to the identity provider) because those are what an operator turning it on is most likely to assume it does. Nothing to change.", "fix": "Every account that signs in with a password is asked for an authenticator code, and one without an authenticator sets it up at its next sign-in. It says what it does not cover (sessions that already exist, API tokens, and single sign-on accounts, whose second factor belongs to the identity provider) because those are what an operator turning it on is most likely to assume it does. Nothing to change.", "verify": null, "setting": "security.require_two_step", "docs_html": "https://zoomies.sh/problem-codes/#configuration-authentication", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-authentication" }, { "id": "ui.capacity_map.layout", "kind": "problem", "title": "ui.capacity_map.layout", "category": "configuration", "severity": "error", "detection": "static", "detects": "Not a layout the host capacity map can open in. They are `overlay`, every host on one chart, and `split`, a chart for each.", "fix": "Not a layout the host capacity map can open in. They are `overlay`, every host on one chart, and `split`, a chart for each.", "verify": null, "setting": "ui.capacity_map.overview_layout`, `ui.capacity_map.hosts_layout", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "updates.auto", "kind": "problem", "title": "updates.auto", "category": "configuration", "severity": "info", "detection": "static", "detects": "The update mode is `auto`, which is meant to install new releases without anyone asking: the controller takes the newest once it has been public for `updates.soak`, and the hosts that have opted in then follow it, one at a time. There is no automatic rollback, because a migration is one way. Nothing to change if that is what you want; `manual` keeps the decision with a person. In this release it only shows what the mode would take on Settings → Updates; it installs nothing yet.", "fix": "The update mode is `auto`, which is meant to install new releases without anyone asking: the controller takes the newest once it has been public for `updates.soak`, and the hosts that have opted in then follow it, one at a time. There is no automatic rollback, because a migration is one way. Nothing to change if that is what you want; `manual` keeps the decision with a person. In this release it only shows what the mode would take on Settings → Updates; it installs nothing yet.", "verify": null, "setting": "updates.mode", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "updates.auto_without_soak", "kind": "problem", "title": "updates.auto_without_soak", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The mode is `auto` and the soak is 0, which removes the only wait between a release being published and every host running it. A release that turns out to be broken, or is replaced by a fix within hours, would be taken before anyone has had the chance to notice. Use `24h`, the default, or set the mode to `manual`.", "fix": "The mode is `auto` and the soak is 0, which removes the only wait between a release being published and every host running it. A release that turns out to be broken, or is replaced by a fix within hours, would be taken before anyone has had the chance to notice. Use `24h`, the default, or set the mode to `manual`.", "verify": null, "setting": "updates.soak", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "updates.interval_negative", "kind": "problem", "title": "updates.interval_negative", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must not be negative. Use a duration, or 0 to never ask.", "fix": "Must not be negative. Use a duration, or 0 to never ask.", "verify": null, "setting": "updates.check_interval", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "updates.interval_too_fast", "kind": "problem", "title": "updates.interval_too_fast", "category": "configuration", "severity": "warning", "detection": "static", "detects": "Releases are published far less often than this, and the check is unauthenticated.", "fix": "Releases are published far less often than this, and the check is unauthenticated.", "verify": null, "setting": "updates.check_interval", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "updates.mode", "kind": "problem", "title": "updates.mode", "category": "configuration", "severity": "error", "detection": "static", "detects": "Not a release update mode. Use `off`, which says that a release exists and nothing more; `manual`, which adds the Update buttons and moves nothing without a click; or `auto`, which takes a release once it has been public for `updates.soak`. In this release it only shows what the mode would take on Settings → Updates; it installs nothing yet.", "fix": "Not a release update mode. Use `off`, which says that a release exists and nothing more; `manual`, which adds the Update buttons and moves nothing without a click; or `auto`, which takes a release once it has been public for `updates.soak`. In this release it only shows what the mode would take on Settings → Updates; it installs nothing yet.", "verify": null, "setting": "updates.mode", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "updates.mode_without_check", "kind": "problem", "title": "updates.mode_without_check", "category": "configuration", "severity": "warning", "detection": "static", "detects": "The mode is `manual` or `auto` and `updates.check_interval` is 0, the air-gap switch, so this controller never learns that a release exists and the mode has nothing to act on. Give the check a duration such as `24h`, or set the mode to `off`.", "fix": "The mode is `manual` or `auto` and `updates.check_interval` is 0, the air-gap switch, so this controller never learns that a release exists and the mode has nothing to act on. Give the check a duration such as `24h`, or set the mode to `off`.", "verify": null, "setting": "updates.mode", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "updates.soak_negative", "kind": "problem", "title": "updates.soak_negative", "category": "configuration", "severity": "error", "detection": "static", "detects": "Must not be negative. Use a duration such as `24h`, or 0 for no wait.", "fix": "Must not be negative. Use a duration such as `24h`, or 0 for no wait.", "verify": null, "setting": "updates.soak", "docs_html": "https://zoomies.sh/problem-codes/#configuration-the-scheduler-and-the-rest", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#configuration-the-scheduler-and-the-rest" }, { "id": "webhook.never_received", "kind": "problem", "title": "webhook.never_received", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "No webhook has ever arrived, so scaling is running entirely on the poller.", "fix": "No webhook has ever arrived, so scaling is running entirely on the poller.", "verify": null, "status_sentence": "The controller has never heard from GitHub directly, so new jobs are noticed late.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "webhook.rejected", "kind": "problem", "title": "webhook.rejected", "category": "reliability", "severity": "warning", "detection": "runtime", "detects": "Deliveries arrived and were refused, almost always a signing-secret mismatch.", "fix": "Deliveries arrived and were refused, almost always a signing-secret mismatch.", "verify": null, "status_sentence": "GitHub's notifications are being refused because their signature does not verify, so new jobs are noticed late.", "docs_html": "https://zoomies.sh/problem-codes/#runtime-hosts-and-installations", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/problem-codes.md#runtime-hosts-and-installations" }, { "id": "capacity.job_hit_default_limit", "kind": "check", "title": "A job ran until GitHub stopped it at its six-hour default limit.", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "A job ran until GitHub stopped it at its six-hour default limit.", "fix": "Set timeout-minutes on the job from its own usual duration, so a hung run is stopped in minutes rather than hours.", "verify": "Press Recheck after the next run of the job; the finding closes once no run in the window was cancelled at the six-hour limit.", "area": "capacity", "docs_html": "https://zoomies.sh/kennel-club/#capacity-job_hit_default_limit", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#capacity-job_hit_default_limit" }, { "id": "capacity.unserved_label", "kind": "check", "title": "Jobs waited more than ten minutes for a label no pool serves.", "category": "capacity", "severity": "warning", "detection": "runtime", "detects": "Jobs waited more than ten minutes for a label no pool serves.", "fix": "Add the label to a pool that can run the job, or change the workflow's runs-on to a label a pool serves.", "verify": "Press Recheck after the pool or the workflow changes; the finding closes once no job has waited ten minutes for an unserved label in the last seven days.", "area": "capacity", "docs_html": "https://zoomies.sh/kennel-club/#capacity-unserved_label", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#capacity-unserved_label" }, { "id": "ci.action_not_pinned", "kind": "check", "title": "External actions or reusable workflows use mutable refs, or Docker actions use no image digest.", "category": "cost", "severity": "warning", "detection": "static", "detects": "External actions or reusable workflows use mutable refs, or Docker actions use no image digest.", "fix": "Pin every external action and reusable workflow to a reviewed full commit, and every Docker action to an image digest, and let an updater move the pins.", "verify": "Press Recheck after the change reaches the default branch; the finding closes when no external reference is a tag or a branch.", "area": "ci", "docs_html": "https://zoomies.sh/kennel-club/#ci-action_not_pinned", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#ci-action_not_pinned" }, { "id": "ci.no_concurrency", "kind": "check", "title": "Pull-request-only workflows do not cancel superseded runs at workflow or job level.", "category": "cost", "severity": "info", "detection": "static", "detects": "Pull-request-only workflows do not cancel superseded runs at workflow or job level.", "fix": "Where a superseded pull-request run may be cancelled, add a concurrency group scoped to the workflow and the pull request with cancel-in-progress on.", "verify": "Press Recheck after the change reaches the default branch; the finding closes when every pull-request-only workflow cancels superseded runs.", "area": "ci", "docs_html": "https://zoomies.sh/kennel-club/#ci-no_concurrency", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#ci-no_concurrency" }, { "id": "ci.no_timeout", "kind": "check", "title": "Executable jobs have no timeout-minutes.", "category": "cost", "severity": "warning", "detection": "static", "detects": "Executable jobs have no timeout-minutes.", "fix": "Set timeout-minutes on each executable job in the default-branch workflows, from its usual duration with a margin for a slow run.", "verify": "Press Recheck after the change reaches the default branch; the finding closes when every executable job declares a timeout.", "area": "ci", "docs_html": "https://zoomies.sh/kennel-club/#ci-no_timeout", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#ci-no_timeout" }, { "id": "exposure.fork_code_ran", "kind": "check", "title": "A run from a fork's pull request executed on this fleet.", "category": "security", "severity": "error", "detection": "runtime", "detects": "A run from a fork's pull request executed on this fleet.", "fix": "Require approval for workflows from fork pull requests in the repository's Actions settings, or stop routing its pull-request jobs to this fleet.", "verify": "Press Recheck after the setting changes; the finding closes once no run from a fork's pull request has executed here in the window.", "area": "exposure", "docs_html": "https://zoomies.sh/kennel-club/#exposure-fork_code_ran", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#exposure-fork_code_ran" }, { "id": "exposure.public_repo_on_fleet", "kind": "check", "title": "A public repository ran jobs on this fleet, or has jobs waiting for it.", "category": "security", "severity": "warning", "detection": "runtime", "detects": "A public repository ran jobs on this fleet, or has jobs waiting for it.", "fix": "Decide whether this fleet should run a public repository's jobs at all; if it should, keep them on an ephemeral pool with no host socket, no privileged daemon and no root, so a stranger's pull request cannot reach the host.", "verify": "Press Recheck once the pool is ephemeral and unprivileged, or once the repository no longer sends jobs here; the finding closes when the next read sees neither.", "area": "exposure", "docs_html": "https://zoomies.sh/kennel-club/#exposure-public_repo_on_fleet", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#exposure-public_repo_on_fleet" }, { "id": "exposure.public_repo_weak_pool", "kind": "check", "title": "The pool that ran a public repository's jobs is persistent, mounts the host Docker socket, gives its jobs a privileged Docker daemon, runs as root or uses no container.", "category": "security", "severity": "error", "detection": "runtime", "detects": "The pool that ran a public repository's jobs is persistent, mounts the host Docker socket, gives its jobs a privileged Docker daemon, runs as root or uses no container.", "fix": "Move the public repository's jobs to a pool that is ephemeral, does not mount the host Docker socket, runs no privileged daemon and does not run as root, or make this pool so.", "verify": "Press Recheck after the pool's settings change; the finding closes when every run of the repository in the window landed on a pool without those settings.", "area": "exposure", "docs_html": "https://zoomies.sh/kennel-club/#exposure-public_repo_weak_pool", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#exposure-public_repo_weak_pool" }, { "id": "exposure.target_event_ran", "kind": "check", "title": "A workflow that strangers can trigger (pull_request_target, workflow_run, issue_comment, issues) ran on this fleet in a public repository.", "category": "security", "severity": "warning", "detection": "runtime", "detects": "A workflow that strangers can trigger (pull_request_target, workflow_run, issue_comment, issues) ran on this fleet in a public repository.", "fix": "Read the workflows these events trigger and make sure none of them checks out or executes the pull request's own code; if one must, run it on an isolated, ephemeral pool.", "verify": "Press Recheck once the workflow has been reviewed or moved; the finding closes when no run for those events has executed here in the window.", "area": "exposure", "docs_html": "https://zoomies.sh/kennel-club/#exposure-target_event_ran", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#exposure-target_event_ran" }, { "id": "setup.code_of_conduct", "kind": "check", "title": "No repository-local code of conduct was found on the default branch.", "category": "configuration", "severity": "info", "detection": "static", "detects": "No repository-local code of conduct was found on the default branch.", "fix": "Add CODE_OF_CONDUCT.md with an enforcement contact, or confirm that the account default provides it.", "verify": "Press Recheck once the file is on the default branch; the finding closes when the next tree read finds it.", "area": "setup", "docs_html": "https://zoomies.sh/kennel-club/#setup-code_of_conduct", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#setup-code_of_conduct" }, { "id": "setup.codeowners", "kind": "check", "title": "No repository-local CODEOWNERS file was found on the default branch.", "category": "configuration", "severity": "info", "detection": "static", "detects": "No repository-local CODEOWNERS file was found on the default branch.", "fix": "Add CODEOWNERS in .github, the repository root or docs and assign owners for the CI workflows; review enforcement separately.", "verify": "Press Recheck once the file is on the default branch; the finding closes when the next tree read finds it.", "area": "setup", "docs_html": "https://zoomies.sh/kennel-club/#setup-codeowners", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#setup-codeowners" }, { "id": "setup.contributing", "kind": "check", "title": "No repository-local contribution guide was found on the default branch.", "category": "configuration", "severity": "info", "detection": "static", "detects": "No repository-local contribution guide was found on the default branch.", "fix": "Add CONTRIBUTING.md with setup, test and pull request guidance, or confirm that the account default provides it.", "verify": "Press Recheck once the file is on the default branch; the finding closes when the next tree read finds it.", "area": "setup", "docs_html": "https://zoomies.sh/kennel-club/#setup-contributing", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#setup-contributing" }, { "id": "setup.dependency_updates", "kind": "check", "title": "No repository-local dependency update configuration was found on the default branch.", "category": "configuration", "severity": "info", "detection": "static", "detects": "No repository-local dependency update configuration was found on the default branch.", "fix": "Configure Dependabot or Renovate for the package ecosystems and GitHub Actions, or confirm that an external service manages updates.", "verify": "Press Recheck once the file is on the default branch; the finding closes when the next tree read finds it.", "area": "setup", "docs_html": "https://zoomies.sh/kennel-club/#setup-dependency_updates", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#setup-dependency_updates" }, { "id": "setup.issue_template", "kind": "check", "title": "No repository-local issue template was found on the default branch.", "category": "configuration", "severity": "info", "detection": "static", "detects": "No repository-local issue template was found on the default branch.", "fix": "Add an issue form or template under .github/ISSUE_TEMPLATE, or confirm that account defaults provide one.", "verify": "Press Recheck once the file is on the default branch; the finding closes when the next tree read finds it.", "area": "setup", "docs_html": "https://zoomies.sh/kennel-club/#setup-issue_template", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#setup-issue_template" }, { "id": "setup.licence", "kind": "check", "title": "No repository-local licence was found on the default branch.", "category": "configuration", "severity": "info", "detection": "static", "detects": "No repository-local licence was found on the default branch.", "fix": "Choose an appropriate licence with the project owner and record it in a root licence file.", "verify": "Press Recheck once the file is on the default branch; the finding closes when the next tree read finds it.", "area": "setup", "docs_html": "https://zoomies.sh/kennel-club/#setup-licence", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#setup-licence" }, { "id": "setup.pull_request_template", "kind": "check", "title": "No repository-local pull request template was found on the default branch.", "category": "configuration", "severity": "info", "detection": "static", "detects": "No repository-local pull request template was found on the default branch.", "fix": "Add a pull request template with a change summary and validation prompts, or confirm that the account default provides one.", "verify": "Press Recheck once the file is on the default branch; the finding closes when the next tree read finds it.", "area": "setup", "docs_html": "https://zoomies.sh/kennel-club/#setup-pull_request_template", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#setup-pull_request_template" }, { "id": "setup.readme", "kind": "check", "title": "No repository-local README was found on the default branch.", "category": "configuration", "severity": "info", "detection": "static", "detects": "No repository-local README was found on the default branch.", "fix": "Add a README with the project purpose, prerequisites and the commands to build, test and run it.", "verify": "Press Recheck once the file is on the default branch; the finding closes when the next tree read finds it.", "area": "setup", "docs_html": "https://zoomies.sh/kennel-club/#setup-readme", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#setup-readme" }, { "id": "setup.security", "kind": "check", "title": "No repository-local security policy was found on the default branch.", "category": "configuration", "severity": "info", "detection": "static", "detects": "No repository-local security policy was found on the default branch.", "fix": "Add SECURITY.md with a private reporting route and supported versions, or confirm that the account default provides them.", "verify": "Press Recheck once the file is on the default branch; the finding closes when the next tree read finds it.", "area": "setup", "docs_html": "https://zoomies.sh/kennel-club/#setup-security", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#setup-security" }, { "id": "setup.workflows", "kind": "check", "title": "No repository-local CI workflow was found on the default branch.", "category": "configuration", "severity": "info", "detection": "static", "detects": "No repository-local CI workflow was found on the default branch.", "fix": "Add a workflow under .github/workflows that runs the relevant build and tests, or confirm that CI is provided elsewhere.", "verify": "Press Recheck once the file is on the default branch; the finding closes when the next tree read finds it.", "area": "setup", "docs_html": "https://zoomies.sh/kennel-club/#setup-workflows", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#setup-workflows" }, { "id": "token.permissions_unset", "kind": "check", "title": "Jobs inherit token permissions without a declaration at workflow or job level.", "category": "security", "severity": "warning", "detection": "static", "detects": "Jobs inherit token permissions without a declaration at workflow or job level.", "fix": "Declare the least permissions each workflow or job needs, after reading what its actions and publishing steps use.", "verify": "Press Recheck after the change reaches the default branch; the finding closes when every job has a permissions block of its own or its workflow's.", "area": "token", "docs_html": "https://zoomies.sh/kennel-club/#token-permissions_unset", "docs_md": "https://github.com/eyupio/zoomies/blob/main/docs/kennel-club.md#token-permissions_unset" } ] }