# 3pm-backend HTTP API The route table of 3pm-backend, exactly as a backend serves it for an LLM on `GET /api/v1/routes` (build {"server_version":"0.64.0","link_min":5,"link_max":5,"store":{"metrics":false,"events":false,"interval_ms":0,"sent":0,"dropped":0,"catalog_over_cap":0,"errors":0,"retries":0},"feed_app_events_dropped":0,"link":{"frames_invalid":0,"crc_full_mismatch":0}}). - Paths below are relative to **the backend's origin**: the hosted one is `https://fleet.edgecr4ft.net`, a self-hosted one is wherever `3pm-backend` listens (default `http://127.0.0.1:9616`). Not relative to this documentation site. - Authentication: `Authorization: Bearer 3pmu_` (or the dashboard's session cookie); a linked node may use its own `3pma_` bearer within the scopes its owner granted. - A live backend answers the same text on `GET /api/v1/routes`, one route in full on `?route=`, the JSON table on `?format=json`. The same full routes are here: - [`info`](https://3pm.edgecr4ft.net/docs/backend/routes/info.md): `GET /api/v1/info` - [`routes`](https://3pm.edgecr4ft.net/docs/backend/routes/routes.md): `GET /api/v1/routes` - [`client`](https://3pm.edgecr4ft.net/docs/backend/routes/client.md): `GET /api/v1/client.js` - [`register`](https://3pm.edgecr4ft.net/docs/backend/routes/register.md): `POST /api/auth/register` - [`login`](https://3pm.edgecr4ft.net/docs/backend/routes/login.md): `POST /api/auth/login` - [`logout`](https://3pm.edgecr4ft.net/docs/backend/routes/logout.md): `POST /api/auth/logout` - [`me`](https://3pm.edgecr4ft.net/docs/backend/routes/me.md): `GET /api/auth/me` - [`createUserToken`](https://3pm.edgecr4ft.net/docs/backend/routes/createUserToken.md): `POST /api/auth/tokens` - [`revokeUserToken`](https://3pm.edgecr4ft.net/docs/backend/routes/revokeUserToken.md): `DELETE /api/auth/tokens/{tokenId}` - [`listBuckets`](https://3pm.edgecr4ft.net/docs/backend/routes/listBuckets.md): `GET /api/buckets` - [`createBucket`](https://3pm.edgecr4ft.net/docs/backend/routes/createBucket.md): `POST /api/buckets` - [`getBucket`](https://3pm.edgecr4ft.net/docs/backend/routes/getBucket.md): `GET /api/buckets/{clusterId}` - [`setBucketSettings`](https://3pm.edgecr4ft.net/docs/backend/routes/setBucketSettings.md): `POST /api/buckets/{clusterId}/settings` - [`bucketToken`](https://3pm.edgecr4ft.net/docs/backend/routes/bucketToken.md): `GET /api/buckets/{clusterId}/enroll-token` - [`events`](https://3pm.edgecr4ft.net/docs/backend/routes/events.md): `GET /api/buckets/{clusterId}/events` - [`listNodes`](https://3pm.edgecr4ft.net/docs/backend/routes/listNodes.md): `GET /api/buckets/{clusterId}/nodes` - [`approveNode`](https://3pm.edgecr4ft.net/docs/backend/routes/approveNode.md): `POST /api/buckets/{clusterId}/nodes/{nodeId}/approve` - [`revokeNode`](https://3pm.edgecr4ft.net/docs/backend/routes/revokeNode.md): `POST /api/buckets/{clusterId}/nodes/{nodeId}/revoke` - [`deleteNode`](https://3pm.edgecr4ft.net/docs/backend/routes/deleteNode.md): `DELETE /api/buckets/{clusterId}/nodes/{nodeId}` - [`setNodeAccess`](https://3pm.edgecr4ft.net/docs/backend/routes/setNodeAccess.md): `POST /api/buckets/{clusterId}/nodes/{nodeId}/access` - [`listTokens`](https://3pm.edgecr4ft.net/docs/backend/routes/listTokens.md): `GET /api/buckets/{clusterId}/tokens` - [`createToken`](https://3pm.edgecr4ft.net/docs/backend/routes/createToken.md): `POST /api/buckets/{clusterId}/tokens` - [`revokeToken`](https://3pm.edgecr4ft.net/docs/backend/routes/revokeToken.md): `DELETE /api/buckets/{clusterId}/tokens/{tokenId}` - [`queryMetrics`](https://3pm.edgecr4ft.net/docs/backend/routes/queryMetrics.md): `POST /api/buckets/{clusterId}/metrics/query` - [`metricsCatalog`](https://3pm.edgecr4ft.net/docs/backend/routes/metricsCatalog.md): `GET /api/buckets/{clusterId}/metrics/catalog` - [`metricLabelValues`](https://3pm.edgecr4ft.net/docs/backend/routes/metricLabelValues.md): `GET /api/buckets/{clusterId}/metrics/labels/{label}/values` - [`metricsInventory`](https://3pm.edgecr4ft.net/docs/backend/routes/metricsInventory.md): `GET /api/buckets/{clusterId}/metrics/inventory` - [`metricsByServer`](https://3pm.edgecr4ft.net/docs/backend/routes/metricsByServer.md): `GET /api/buckets/{clusterId}/metrics/servers` - [`metricsByApp`](https://3pm.edgecr4ft.net/docs/backend/routes/metricsByApp.md): `GET /api/buckets/{clusterId}/metrics/apps` - [`eventsQuery`](https://3pm.edgecr4ft.net/docs/backend/routes/eventsQuery.md): `POST /api/buckets/{clusterId}/events/query` - [`logsQuery`](https://3pm.edgecr4ft.net/docs/backend/routes/logsQuery.md): `POST /api/buckets/{clusterId}/logs/query` - [`insights`](https://3pm.edgecr4ft.net/docs/backend/routes/insights.md): `GET /api/buckets/{clusterId}/insights` - [`insight`](https://3pm.edgecr4ft.net/docs/backend/routes/insight.md): `GET /api/buckets/{clusterId}/insights/{incident}` - [`nodeAppMetrics`](https://3pm.edgecr4ft.net/docs/backend/routes/nodeAppMetrics.md): `GET /api/buckets/{clusterId}/nodes/{nodeId}/app-metrics` - [`nodeActions`](https://3pm.edgecr4ft.net/docs/backend/routes/nodeActions.md): `GET /api/buckets/{clusterId}/nodes/{nodeId}/actions` - [`nodeHost`](https://3pm.edgecr4ft.net/docs/backend/routes/nodeHost.md): `GET /api/buckets/{clusterId}/nodes/{nodeId}/host` - [`nodeProcessTrees`](https://3pm.edgecr4ft.net/docs/backend/routes/nodeProcessTrees.md): `GET /api/buckets/{clusterId}/nodes/{nodeId}/trees` - [`nodeProcessTree`](https://3pm.edgecr4ft.net/docs/backend/routes/nodeProcessTree.md): `GET /api/buckets/{clusterId}/nodes/{nodeId}/procs/{procId}/tree` - [`nodeAction`](https://3pm.edgecr4ft.net/docs/backend/routes/nodeAction.md): `POST /api/buckets/{clusterId}/nodes/{nodeId}/actions` - [`nodeShell`](https://3pm.edgecr4ft.net/docs/backend/routes/nodeShell.md): `POST /api/buckets/{clusterId}/nodes/{nodeId}/shell` - [`nodeForward`](https://3pm.edgecr4ft.net/docs/backend/routes/nodeForward.md): `POST /api/buckets/{clusterId}/nodes/{nodeId}/forward` - [`nodeListen`](https://3pm.edgecr4ft.net/docs/backend/routes/nodeListen.md): `POST /api/buckets/{clusterId}/nodes/{nodeId}/listen` - [`nodeCall`](https://3pm.edgecr4ft.net/docs/backend/routes/nodeCall.md): `POST /api/buckets/{clusterId}/nodes/{nodeId}/calls` - [`putCallArtifact`](https://3pm.edgecr4ft.net/docs/backend/routes/putCallArtifact.md): `PUT /api/buckets/{clusterId}/calls/{callId}/artifacts` - [`getCall`](https://3pm.edgecr4ft.net/docs/backend/routes/getCall.md): `GET /api/buckets/{clusterId}/calls/{callId}` - [`listArtifacts`](https://3pm.edgecr4ft.net/docs/backend/routes/listArtifacts.md): `GET /api/buckets/{clusterId}/artifacts` - [`artifactFile`](https://3pm.edgecr4ft.net/docs/backend/routes/artifactFile.md): `GET /api/buckets/{clusterId}/artifacts/{artifactId}/file` - [`putBlob`](https://3pm.edgecr4ft.net/docs/backend/routes/putBlob.md): `PUT /api/buckets/{clusterId}/blobs/{blobId}` - [`getBlob`](https://3pm.edgecr4ft.net/docs/backend/routes/getBlob.md): `GET /api/buckets/{clusterId}/blobs/{blobId}` - [`viewBlob`](https://3pm.edgecr4ft.net/docs/backend/routes/viewBlob.md): `POST /api/buckets/{clusterId}/blobs/{blobId}/view` - [`enroll`](https://3pm.edgecr4ft.net/docs/backend/routes/enroll.md): `POST /api/v1/enroll` - [`link`](https://3pm.edgecr4ft.net/docs/backend/routes/link.md): `GET /api/v1/link` - [`pushLogs`](https://3pm.edgecr4ft.net/docs/backend/routes/pushLogs.md): `POST /api/buckets/{clusterId}/logs` - [`indexPage`](https://3pm.edgecr4ft.net/docs/backend/routes/indexPage.md): `GET /` - [`bucketPage`](https://3pm.edgecr4ft.net/docs/backend/routes/bucketPage.md): `GET /b/{clusterId}` - [`bucketSettingsPage`](https://3pm.edgecr4ft.net/docs/backend/routes/bucketSettingsPage.md): `GET /b/{clusterId}/settings` - [`asset`](https://3pm.edgecr4ft.net/docs/backend/routes/asset.md): `GET /assets/{path}` --- tokens: claude: 15159 o200k: 9914 api: 3pm-backend 0.64.0 about: "the fleet backend's HTTP API. Paths are relative to the origin that served this text; bodies and answers are JSON; an error is {error: {code, message}}" auth: "auth=user: the route needs an account — Authorization: Bearer 3pmu_, or the dashboard's session cookie — and one under /api/buckets/{clusterId} a bucket that account owns (another's answers 404); auth=public: no credential. A route may also be open to a linked node's own bearer (Authorization: Bearer 3pma_, minted in every welcome): GET /api/v1/routes?route= names the fleet scope it then needs (fleet:list, fleet:read, fleet:invoke), granted per node by its owner" more: "GET /api/v1/routes?route= answers one route in full: its details, its bounds, the types of its body and answer. ?format=json answers the whole table as JSON. other: routes that are not for an LLM (pages, the JavaScript client module, the agent's WebSockets). recipes: a body for each common question and how to read its answer, then the rules no type says. in_full: the routes given whole after them, their types once — what a bucket holds, then how to query it" routes[45]{name,method,path,body,returns,auth,doc}: info,GET,/api/v1/info,"",Info,public,"Versions, the accepted link range, how the store is doing and what the link got wrong; no credentials needed." routes,GET,/api/v1/routes,"",ApiDescriptor,public,"Every route this backend serves: an index for an LLM by default, with `metricsInventory` and `queryMetrics` given whole; one route in full with `?route=` (details, bounds, body and answer types), the JSON table with `?format=json` or `Accept: application/json` (what this client sends); `?version=` defaults to `latest`." register,POST,/api/auth/register,Register,Registered,public,"Create an account: its token comes back once (there is no other way in, keep it) with a session cookie. Only the first account unless the backend runs with `--registration open`; that first account adopts every bucket without an owner." login,POST,/api/auth/login,Login,LoggedIn,public,"Trade an account's token for a session cookie — the dashboard's way in; a script sends the token as `Authorization: Bearer` on each request instead." logout,POST,/api/auth/logout,"",void,user,Close this session and drop its cookie; the token stays valid. me,GET,/api/auth/me,"",Me,user,"The account behind this request: its name, its tokens (never their secrets) and how many buckets it owns." createUserToken,POST,/api/auth/tokens,CreateUserToken,UserTokenCreated,user,"Another token for the account (a laptop, a CI job); the secret comes back once." revokeUserToken,DELETE,/api/auth/tokens/{tokenId},"",void,user,Revoke one of the account's tokens and close every session it opened; never the last one (409 `ERR_LAST_TOKEN`). listBuckets,GET,/api/buckets,"",BucketView[],user,"The caller's buckets, oldest first." createBucket,POST,/api/buckets,CreateBucket,BucketView,user,"Create a bucket: a cluster id to paste, and nodes to enroll into it." getBucket,GET,/api/buckets/{clusterId},"",BucketDetail,user,"One bucket with its nodes, last telemetry and history — the shape the feed opens with; `history`, `match` and `view` cut it as they cut the feed's `init`." setBucketSettings,POST,/api/buckets/{clusterId}/settings,SetBucketSettings,BucketView,user,"Set the bucket's settings — `approval`: whether a host that links with a reusable token waits for `approveNode` (turning it off approves the hosts already waiting); `require_access_rights`: off, every active node may place a call on every other, as if each held `fleet:invoke`, `fleet:list` and `fleet:read`; `record_shells` (on by default): the screen of each shell session is kept as an asciicast artifact of its call. Answers the bucket as it now stands." bucketToken,GET,/api/buckets/{clusterId}/enroll-token,"",BucketToken,user,"The bucket's permanent enrollment token and its paste line: the same at every read, never expiring, any number of hosts, not revocable. Created on the first read; its hosts wait for `approveNode` unless the bucket's `approval` is off." events,GET,/api/buckets/{clusterId}/events,"",text/event-stream,user,"The bucket's live feed: `init`, then `node` / `telemetry` / `emit` / `call` / `artifact` — a call's state and an artifact's id and sizes only, never params, results or summaries (`getCall`, `listArtifacts`); `history`, `match` and `view` cut it to what the page draws." listNodes,GET,/api/buckets/{clusterId}/nodes,"",NodeView[],user,"Every enrolled node of the bucket, presence layered on." approveNode,POST,/api/buckets/{clusterId}/nodes/{nodeId}/approve,"",NodeView,user,Admit a pending node. Idempotent. revokeNode,POST,/api/buckets/{clusterId}/nodes/{nodeId}/revoke,"",NodeView,user,"Revoke a node, cut its live session and drop the bearer it was using. Idempotent." deleteNode,DELETE,/api/buckets/{clusterId}/nodes/{nodeId},"",void,user,"Delete a node: its row, its calls and its artifacts go, its live session is cut and the feed sends `reset`. Unlike `revokeNode` nothing is held against its key — the host may enroll again, as a new node." setNodeAccess,POST,/api/buckets/{clusterId}/nodes/{nodeId}/access,SetNodeAccess,NodeView,user,"What this node may ask of the bucket with its own bearer (`scopes`), who may place a call on it (`callers`), and what it lets go agent to agent with a LAN neighbour (`direct`); no scope by default, and an empty `callers` accepts every node holding `fleet:invoke`. Replaces each list it names, and takes effect on the node's next request." listTokens,GET,/api/buckets/{clusterId}/tokens,"",TokenView[],user,Every enrollment token of the bucket; never the secrets. createToken,POST,/api/buckets/{clusterId}/tokens,CreateToken,TokenCreated,user,Mint an enrollment token. The secret and its paste line are returned once. revokeToken,DELETE,/api/buckets/{clusterId}/tokens/{tokenId},"",void,user,"Revoke an enrollment token; the bucket's permanent one (`bucketToken`) is refused, 409." queryMetrics,POST,/api/buckets/{clusterId}/metrics/query,MetricsQuery,MetricsResult,user,"Time series for charts: metrics, filters, aggregation and step, scoped to the bucket." metricsCatalog,GET,/api/buckets/{clusterId}/metrics/catalog,"",MetricDef[],user,"Every metric a query may name: id, kind, unit, labels, default time function." metricLabelValues,GET,/api/buckets/{clusterId}/metrics/labels/{label}/values,"",LabelValues,user,"The values one label takes in this bucket over the last two UTC days, at most 5000." metricsInventory,GET,/api/buckets/{clusterId}/metrics/inventory,"",MetricsInventory,user,"What a dashboard or an LLM starts from, in one call: the metrics the bucket holds series for, with their descriptions, its hosts and their processes — text for an LLM by default, JSON with `?format=json` or `Accept: application/json` (what this client sends)." metricsByServer,GET,/api/buckets/{clusterId}/metrics/servers,"",MetricsByServer,user,"Which metric is stored on which server, for which app: each node with its host's own metrics, then each of its apps with theirs — text for an LLM by default, JSON with `?format=json` or `Accept: application/json` (what this client sends)." metricsByApp,GET,/api/buckets/{clusterId}/metrics/apps,"",MetricsByApp,user,"Which metric each app has on which server: each app with the servers it has series on and, on each, their metrics — text for an LLM by default, JSON with `?format=json` or `Accept: application/json` (what this client sends)." eventsQuery,POST,/api/buckets/{clusterId}/events/query,EventsQuery,EventsResult,user,"The persisted events of the bucket, newest first: a range, exact filters (kind, app, level, event_name…), a limit." logsQuery,POST,/api/buckets/{clusterId}/logs/query,LogsQuery,LogsResult,user,"The logs the bucket's nodes shipped over a range: their sources, the templates with their counts (worst first) and the shipped lines (newest first), filtered by node, source, owner, severity, template or text." insights,GET,/api/buckets/{clusterId}/insights,"",InsightsList,user,"What deserves a look on this fleet: incidents ranked by the insights detectors, best first — text for an LLM by default, JSON with `?format=json` or `Accept: application/json` (what this client sends)." insight,GET,/api/buckets/{clusterId}/insights/{incident},"",InsightDetail,user,"One incident of `insights` and the findings it groups, each with its detector, series, score and summary — text for an LLM by default, JSON with `?format=json` or `Accept: application/json`." nodeAppMetrics,GET,/api/buckets/{clusterId}/nodes/{nodeId}/app-metrics,"",AppMetricsView,user,"The application metrics of every instance of the node, as last received: sources, series, values and histogram buckets." nodeActions,GET,/api/buckets/{clusterId}/nodes/{nodeId}/actions,"",NodeActionsView,user,"The custom actions the apps of the node declare (`io.action`), per instance and source, with descriptions and timeouts — what `nodeAction` may run; none of the metrics, so a page may poll it all the time." nodeHost,GET,/api/buckets/{clusterId}/nodes/{nodeId}/host,"",NodeHostView,user,"The node's host as it is now, whole: the gauges and counters with the identity (hostname, OS, totals, interface and mount names) the feed's ticks blank; 404 before its first frame, and for a node that sends no host." nodeProcessTrees,GET,/api/buckets/{clusterId}/nodes/{nodeId}/trees,"",ProcessTreesView,user,"The process trees of every instance of the node, as last received — every process and the folded picture of `3pm ls --tree` — with the count of trees refused; what the dashboard's panel polls." nodeProcessTree,GET,/api/buckets/{clusterId}/nodes/{nodeId}/procs/{procId}/tree,"",ProcessTreeView,user,"The process tree of one instance of the node, as last received: every process, and the same folded picture as `3pm ls --tree`; 404 while the backend holds none — an instance running alone, or before its first tree." nodeAction,POST,/api/buckets/{clusterId}/nodes/{nodeId}/actions,NodeActionBody,CallView,user,"Run a custom action an app of the node declared (`io.action`), on every instance of the app or on one, optionally carrying bytes in and out; the call is answered as soon as it is placed — follow it with `getCall` or the feed's `call` events." nodeShell,POST,/api/buckets/{clusterId}/nodes/{nodeId}/shell,NodeShellBody,CallView,user,"Open an interactive login shell on the node (`3pm fleet ssh`), or run one command there (`command`, `3pm fleet ssh -- `): the answer is the call, whose `input.blob` you `PUT` keystrokes (or stdin) to and `output.blob` you `GET` the screen (or stdout and stderr) from; it replies `exited` when the shell or the command ends. Never from a page on another site." nodeForward,POST,/api/buckets/{clusterId}/nodes/{nodeId}/forward,NodeForwardBody,CallView,user,"Open one TCP connection from the node to `host:port` (`3pm fleet ssh -L`, one call per local connection): the answer is the call, whose `input.blob` you `PUT` the bytes to send and `output.blob` you `GET` the bytes that come back; it replies `exited` once both ways are closed. Never from a page on another site." nodeListen,POST,/api/buckets/{clusterId}/nodes/{nodeId}/listen,NodeListenBody,CallView,user,"Make the node listen on a port for you (`3pm fleet ssh -R`): the answer is the call, whose sealed legs carry the port it bound and one `accepted` message per connection, each of which you take over with a `nodeForward` naming it (`accepted`); it replies `exited` when you close it. Never from a page on another site." nodeCall,POST,/api/buckets/{clusterId}/nodes/{nodeId}/calls,NodeCallBody,CallView,user,"Send one generic command to the node's agent — `{\"op\": \"restart\", \"target\": {\"name\": \"api\"}}`, `stop` or `ping` (an action goes through `nodeAction`); answered as soon as it is placed — follow it with `getCall`." getCall,GET,/api/buckets/{clusterId}/calls/{callId},"",CallView,user,"One remote command: its state (`sent`, `acked`, `replied`, `unacked`, `refused`, `lost`), who placed it, the blobs it carries, its result once replied (`result.ok` says the action ran) and the artifacts it left. A node's bearer sees the calls it placed itself and nothing else." listArtifacts,GET,/api/buckets/{clusterId}/artifacts,"",ArtifactView[],user,"The files actions left (profiles, dumps), newest first, at most 200; `node` (a node id) and `app` (a name) narrow. Kept 30 days and 1 GB per bucket. A node's bearer sees what its own calls left and nothing else." viewBlob,POST,/api/buckets/{clusterId}/blobs/{blobId}/view,ViewBlobBody,ViewBlobUrl,user,"Mint the one credential a browser tag can carry: a `getBlob` URL with `?k=` on it, good for sixty seconds, for this blob, for reading — `{as: \"mjpeg\"}` adds the rendering an `` needs." other[12]{name,kind,path}: client,static,/api/v1/client.js putCallArtifact,agent-http,/api/buckets/{clusterId}/calls/{callId}/artifacts artifactFile,static,/api/buckets/{clusterId}/artifacts/{artifactId}/file putBlob,static,/api/buckets/{clusterId}/blobs/{blobId} getBlob,static,/api/buckets/{clusterId}/blobs/{blobId} enroll,agent-http,/api/v1/enroll link,agent-ws,/api/v1/link pushLogs,agent-http,/api/buckets/{clusterId}/logs indexPage,static,/ bucketPage,static,/b/{clusterId} bucketSettingsPage,static,/b/{clusterId}/settings asset,static,/assets/{path} in_full[3]: metricsInventory,queryMetrics,eventsQuery ## Recipes A body for each common question, and how to read its answer. Every body passes the validation its route applies; the metrics are static catalog lines, the node ids placeholders. ### The value right now — which processes are down POST /api/buckets/{clusterId}/metrics/query (queryMetrics) {"range":{"from":"now-10m"},"queries":[{"ref":"UP","metric":"process.up"}],"options":{"mode":"summary"}} read: one series per instance; `summary.last` is its latest value (`process.up`: 1 up, 0 down). Keep the rows whose `last` is 0 yourself: a query has no value predicate. ### A total over a period — restarts per app over the last 7 days POST /api/buckets/{clusterId}/metrics/query (queryMetrics) {"range":{"from":"now-7d"},"queries":[{"ref":"R","metric":"process.restarts","time":{"fn":"increase","window":"7d"},"aggregate":{"fn":"sum","by":["app"]}}],"options":{"mode":"summary"}} read: `summary.last` per app is the total: the window is the whole range, so the last point covers it. Never add the points up — each one is already a window's total. ### The top N — the 5 apps using the most CPU over the last hour POST /api/buckets/{clusterId}/metrics/query (queryMetrics) {"range":{"from":"now-1h"},"queries":[{"ref":"CPU","metric":"process.cpu.utilization","aggregate":{"fn":"sum","by":["app"]}}],"options":{"max_series":5,"sort":"avg","mode":"summary"}} read: the 5 highest by average, best first, `summary.avg` each. `truncated: true` says more existed; `series_total` is then a floor (the store is asked for `max_series` + 1), not a count. ### A percentage from two metrics — memory used per host, in % POST /api/buckets/{clusterId}/metrics/query (queryMetrics) {"range":{"from":"now-6h"},"queries":[{"ref":"U","metric":"system.memory.used","hidden":true},{"ref":"T","metric":"system.memory.total","hidden":true}],"formulas":[{"ref":"PCT","expr":"U / T * 100","unit":"percent"}]} read: one `PCT` series per node: a formula pairs the series of its refs on equal label sets (a lone series applies to all). Add `"options":{"mode":"summary"}` for one number per host — formulas run there too. ### A counter as a rate — network receive per host and interface POST /api/buckets/{clusterId}/metrics/query (queryMetrics) {"range":{"from":"now-3h"},"queries":[{"ref":"RX","metric":"system.interface.receive","time":{"fn":"rate"}}]} read: bytes per second, one series per `node` and `iface`. `rate` is a counter's default; `increase` gives a count per window instead. ### One host by its name — resident memory per app on that host POST /api/buckets/{clusterId}/metrics/query (queryMetrics) {"range":{"from":"now-6h"},"queries":[{"ref":"RSS","metric":"process.memory.rss","filters":[{"label":"node","op":"eq","value":"3pmn_aaaaaaaaaaaaaaaa"}],"aggregate":{"fn":"sum","by":["app"]}}]} read: a filter names a host by `node`, its node_id, never by hostname: find the hostname in the inventory's `hosts[]` and put its `node_id` in `value`. ### One value of a label — the 1-minute load average per host POST /api/buckets/{clusterId}/metrics/query (queryMetrics) {"range":{"from":"now-24h"},"queries":[{"ref":"L","metric":"system.load","filters":[{"label":"window","op":"eq","value":"1"}]}]} read: what a label holds is the inventory's `label_values` (`system.load`'s `window`: 1, 5, 15); a filter takes a value as written there. ### Error events — the error-level application events of the last day POST /api/buckets/{clusterId}/events/query (eventsQuery) {"range":{"from":"now-24h"},"filters":{"kind":"app","level":"error"},"limit":100} read: `events`, newest first; `truncated: true` says more matched than `limit`. An empty list is an answer: none happened. ### Link two machines, step 1 — let node A place calls (on the caller) POST /api/buckets/{clusterId}/nodes/{nodeId}/access (setNodeAccess) {"scopes":["fleet:list","fleet:read","fleet:invoke"]} read: sent to `/nodes/{A}/access`: the `NodeView` of A with its new `scopes`. `fleet:invoke` alone is what `io.call` / `io.open` / `3pm fleet call` need; the two others let A see the fleet. `listNodes` gives each machine's `node_id` from its `hostname`. A list left out of the body is kept as it is; a list sent replaces the old one whole. ### Link two machines, step 2 — let node A call one app's actions on node B (on the target) POST /api/buckets/{clusterId}/nodes/{nodeId}/access (setNodeAccess) {"callers":[{"from":"3pmn_aaaaaaaaaaaaaaaa","app":"kiosk","action":"show"},{"from":"3pmn_aaaaaaaaaaaaaaaa","from_app":"mac-camera","app":"tabloid-mac_camera","action":"video"}]} read: sent to `/nodes/{B}/access`: the `NodeView` of B with its `callers`. An empty `callers` lets in every node holding `fleet:invoke`; once it holds a rule, only what a rule admits goes through (403 `ERR_FORBIDDEN` "not allowed to call /" otherwise). A field left out is `*`; `from_app` is the 3pm app name of the caller as its daemon declares it; a stream action (`io.stream`) is admitted like any other. The list replaces the old one: read B's `callers` in `listNodes` first and send them back with the new rule. A node calling itself needs both steps on itself. ### Notes - Legends: every series carries `node` = a node_id; the inventory's `hosts[]` gives its hostname, and its `processes` the `app` / `instance` / `ns` values. - No value predicate: a query cannot keep the series above or below a threshold (`up == 0`, disk > 90 %) — ask `mode: summary` and filter the rows yourself. - A bool gauge (unit `bool`, as `process.up`) reads per instance: its `avg` across instances is a share of them, not a state. - An `app.*` metric: the inventory's `apps` column lists the apps that emit it; when it lists several, put `app` in `aggregate.by` or in a filter. # GET /api/buckets/{clusterId}/metrics/inventory (metricsInventory) What a dashboard or an LLM starts from, in one call: the metrics the bucket holds series for, with their descriptions, its hosts and their processes — text for an LLM by default, JSON with `?format=json` or `Accept: application/json` (what this client sends). - auth: an account — Authorization: Bearer 3pmu_, or the dashboard's session cookie - answer: MetricsInventory (JSON) ## Details Text for an LLM by default; the generated client asks for the JSON (`Accept: application/json`), and so does `?format=json`. The JSON is `{ tokens, metric_groups, label_values, hosts }`, shaped for few tokens: nothing empty, nothing derivable, no join to make. `tokens` comes first: what the whole answer costs an LLM, estimated from its bytes with the ratio measured for its shape on Claude's tokenizer (about 2.4 bytes a token; OpenAI's counts 1.5–1.7× fewer). `metric_groups` are the catalog lines that have at least one series stored over the last two UTC days — the catalog says what a query may name, this says what would draw something — grouped by the labels they carry: `{ labels, metrics: [{ id, kind, unit?, default_time?, description?, apps? }] }`. `default_time` is left out when it is the kind's (`avg_over_time` for a gauge, `rate` for a counter, `quantile` for a histogram), `unit` and `description` when empty; a line is an application's exactly when its id starts with `app.`, and `apps` lists the apps whose series carry it (one read of every application series but a histogram's buckets, at most 5000: past that, the lists may be short). `label_values` are `{ label, values, more? }`: what every other label of those lines holds over the same window (not `node`, `app`, `instance`, `ns` — `hosts` has them — nor `pid`), sorted, at most 50 values, `more` counting the rest. `hosts` are every enrolled node, `{ node_id, hostname, os, arch, state, processes: [{ app, instance, ns? }] }`: the instances the store holds a series for over the same window, label values that go into a query's `filters` as they are, next to `node` = `node_id`. Every series carries `node` = a `node_id`: a query names a host by it, never by hostname, and `hosts` turns a legend's `node` into a name. `?detail=index` leaves the descriptions out, about 40 % fewer tokens. The text (`?format=llm`, the default) is the same answer as TOON-style `text/plain` after an `about:` line saying how the lists join, about a fifth fewer tokens than the JSON (more than half fewer with `detail=index`): a header per list with its count and fields (`metrics[140]{id,kind,unit,default_time,description}:`), one row per entry, every `default_time` spelled out. An unknown parameter or value is a 400. The store is read for at most 5000 series names and 5000 processes. Without `--metrics-url` the route answers 409 `ERR_STORE_DISABLED`. # POST /api/buckets/{clusterId}/metrics/query (queryMetrics) Time series for charts: metrics, filters, aggregation and step, scoped to the bucket. - auth: an account — Authorization: Bearer 3pmu_, or the dashboard's session cookie - node: also open to a node's own bearer (3pma_…) when its row is granted fleet:read; that node's bucket only, and for the calls it placed itself - body: MetricsQuery (JSON) - answer: MetricsResult (JSON) ## Details Discovery first: `metricsCatalog` gives every metric id, the labels each one carries and its `default_time`; `metricLabelValues` gives what a label holds in this bucket. A metric the catalog lacks, a label the metric does not carry, `rate` on a gauge, a regex that does not compile or any unknown field is a 400 `ERR_BAD_REQUEST` naming what was wrong. Body `{ range, queries, formulas?, options? }`: - `range` = `{ from, to?, step? }`. `from` and `to` are epoch milliseconds or `now` / `now-` (`now-90m`, `now-7d`); `to` defaults to `now`. `step` is `30s` / `5m` / `1h` or milliseconds, and is optional: left out, it is derived from `options.max_points`. Both ends are aligned to it, and the answer says which step was used (`range.step_ms`). - `queries` = 1 to 8 `{ ref, metric, filters?, time?, aggregate?, hidden? }`. `ref` matches `^[A-Z][A-Z0-9]{0,7}$` and is unique across queries and formulas. `metric` is a catalog id (`process.cpu.utilization`, or `app.` for an application's own). `filters` are AND-ed `{ label, op, value | values }` over `eq | neq | in | nin | regex | nregex`. `time` is the time function and its window — `rate` / `increase` / `delta` on a counter, the `*_over_time` family on a gauge, `quantile` (with `q`) on a histogram, `raw` on either — the window defaulting to twice the step. `aggregate` = `{ fn: sum | avg | min | max | count | quantile, by: [labels] }` collapses the series that share `by`; without it, one series comes back per label combination the metric carries. - `formulas` = up to 8 `{ ref, expr }` over the other refs: `A / B * 100`, with `+ - * /` and parentheses. A query marked `hidden: true` is computed for a formula and not returned. - `options`: `max_points` (the step is raised until the range fits), `max_series` with `sort` (`avg` or `last`) deciding which are kept, `fill` (`null` | `zero` | `previous`), `mode` (`series`, or `summary` to drop the values and keep min/max/avg/last/count per series), `explain` to get the compiled expression back. Answer `{ range, times, results, took_ms }`: one `times[]` shared by every series, then one entry per visible ref with `series[].labels`, `series[].values` aligned on `times`, `series[].summary`, and `series_total` / `truncated` when the ranking cut some off. The bounds are `limits.queryMetrics` of `GET /api/v1/routes`, and the step may not go below the store's resolution (`store.interval_ms` of `GET /api/v1/info`). They bound the grid, not the window, and what a step is measured against is `options.max_points` — 600 by default, 2000 at most: 30 days at `5m` is 8 640 points and is refused whatever it is set to. The same 30 days draws at `2h` on the default and at `30m` (1 442 points) with `max_points: 2000`; a `5m` step fits two days on the default and just under seven at 2000. Leave `step` out and the finest step of the ladder that fits is picked for you. Example — 24 hours per app: the peak resident memory, and the average CPU turned into a percentage by a formula (the ratio itself computed and not returned). Every metric it names is a static catalog line, so it runs against any bucket: {"range":{"from":"now-24h"},"queries":[{"ref":"CPU","metric":"process.cpu.utilization","filters":[{"label":"app","op":"in","values":["api","worker"]}],"aggregate":{"fn":"avg","by":["app"]},"hidden":true},{"ref":"RSS","metric":"process.memory.rss","aggregate":{"fn":"max","by":["app"]}}],"formulas":[{"ref":"PCT","expr":"CPU * 100","unit":"percent"}],"options":{"max_points":300}} ## Bounds - default_points: 600 - default_series: 200 - max_body_bytes: 65536 - max_filters: 16 - max_formulas: 8 - max_points: 2000 - max_queries: 8 - max_range_ms: 2678400000 - max_series: 2000 - max_value_len: 256 - max_values: 64 - step_floor_from: "info.store.interval_ms" # POST /api/buckets/{clusterId}/events/query (eventsQuery) The persisted events of the bucket, newest first: a range, exact filters (kind, app, level, event_name…), a limit. - auth: an account — Authorization: Bearer 3pmu_, or the dashboard's session cookie - body: EventsQuery (JSON) - answer: EventsResult (JSON) ## Details Body `{ range, filters?, limit? }`. `range` is the metrics query's, minus `step` (there is no grid): `from` / `to` as epoch milliseconds or `now-`. `filters` are exact matches, AND-ed, on the fields the store writes — `kind` (`process`, `daemon_shutdown`, `app`, `action`, `shell`), `node`, `app`, `instance`, `src`, `level`, `event_name`, `fingerprint`, `call` (every line of one action or shell session) — no operators, no regex. `limit` is 1 to 1000 (default 200). Answer `{ events, truncated, took_ms }`, newest first, each event the record as stored (`_time`, `_msg`, `kind`, and the fields above); `truncated` says more matched than `limit`. Bounds: `limits.eventsQuery` of `GET /api/v1/routes`. Example — the last 50 error-level application events of one app over a day: {"range":{"from":"now-24h"},"filters":{"kind":"app","app":"api","level":"error"},"limit":50} ## Bounds - default_limit: 200 - max_body_bytes: 16384 - max_limit: 1000 - max_range_ms: 2678400000 - max_value_len: 256 ## Types ```ts /** `metricsInventory(clusterId)`: what a dashboard or an LLM starts from. The route answers TOON-style text by default; this client asks for this JSON (`Accept: application/json`), as does `?format=json`. `?detail=index` leaves the descriptions out. */ interface MetricsInventory { /** What this whole answer costs an LLM; first, so a reader knows before the rest. */ tokens: TokenCount; /** The catalog lines with a series stored over the last two UTC days, grouped by the labels they carry; groups and lines in catalog order. */ metric_groups: MetricGroup[]; /** What every other label of those lines holds over the same window — not `node`, `app`, `instance`, `ns` (`hosts` has them) nor `pid` —, in the order the lines name them; a label that holds nothing is left out. */ label_values: InventoryLabel[]; /** Every enrolled node of the bucket, whatever its state, with its processes; a node only the store knows comes last, `state: "unknown"`. Every series carries `node` = `node_id`: a query names a host by it, never by hostname, and this list names a legend's `node`. */ hosts: InventoryHost[]; } /** `POST /api/buckets/{id}/metrics/query`. */ interface MetricsQuery { range: TimeRange; /** 1–8. */ queries: QuerySpec[]; /** ≤ 8. */ formulas?: Formula[]; options?: QueryOptions; } interface MetricsResult { range: ResultRange; /** One timestamp per point, shared by every series. */ times: EpochMs[]; results: ResultSet[]; took_ms: number; } /** `POST /api/buckets/{id}/events/query`: the persisted events, newest first. */ interface EventsQuery { /** `step` is ignored. */ range: TimeRange; filters?: EventFilters; /** ≤ 1000, default 200. */ limit?: number; } interface EventsResult { /** The stored fields as they are (`_time`, `_msg`, `kind`, `app`, `level`, `a.*`, `exception.*`, `fingerprint`, `repeat`…). */ events: Record[]; /** More than `limit` matched. */ truncated: boolean; took_ms: number; } /** What an answer costs to put in a prompt: the first field of every answer that carries it (`metricsInventory`, `GET /api/v1/routes`). */ interface TokenCount { /** Claude's tokenizer, estimated from the bytes with a ratio measured per shape through the `count_tokens` API — within a few percent. */ claude: number; /** OpenAI's `o200k_base`, exact. Claude counts 1.5–1.7× more on this backend's answers. */ o200k: number; } interface MetricGroup { /** What `filters` and `aggregate.by` may name for every metric of the group. */ labels: QueryLabel[]; metrics: InventoryMetric[]; } interface InventoryLabel { label: QueryLabel; /** Sorted, at most 50: values a filter takes as written. */ values: string[]; /** How many more there were past the 50; absent when none. */ more?: number; } interface InventoryHost { node_id: NodeId; hostname: string; os: string; arch: string; /** As `NodeView.state`; `"unknown"` for a node only the store knows. */ state: DisplayState | "unknown"; /** The instances the store holds a series for, sorted: label values, ready for `filters` next to `node` = `node_id`. */ processes: InventoryProcess[]; } interface TimeRange { from: TimeValue; /** Default `now`. */ to?: TimeValue; /** `30s`, `5m`, `1h` or milliseconds (what `step_ms` answers); omitted ⇒ derived from `options.max_points`. Both ends are aligned to it. */ step?: DurationValue; } interface QuerySpec { /** `^[A-Z][A-Z0-9]{0,7}$`, unique across queries and formulas. */ ref: string; /** A catalog id, `process.cpu.utilization`. */ metric: string; /** AND-ed; at most 16. */ filters?: Filter[]; /** Default: the catalog's `default_time` for the metric. */ time?: TimeFn; aggregate?: Aggregate; /** Computed for a formula, not returned. */ hidden?: boolean; } interface Formula { ref: string; /** Refs, numbers, `+ - * /` and parentheses: `A / B * 100`. */ expr: string; unit?: string; } interface QueryOptions { /** 2–2000, default 600: the step is raised until the range fits. Every bound here is also `limits.queryMetrics` of `GET /api/v1/routes`, which is generated from the validator. */ max_points?: number; /** 1–2000, default 200: the series kept are the top ones by `sort` over the whole range (the engine ranks, the backend re-ranks the same way). */ max_series?: number; /** What ranks series for `max_series`, descending; default `avg`. */ sort?: "avg" | "last"; /** Default `null`. */ fill?: "null" | "zero" | "previous"; /** `series` (default) or `summary` (no values). */ mode?: "series" | "summary"; /** Add `results[].expr`: the expression as compiled for the engine. */ explain?: boolean; } interface ResultRange { /** Aligned to `step_ms`: `from` rounded down, `to` rounded up. */ from: EpochMs; to: EpochMs; step_ms: number; } /** Milliseconds since the Unix epoch. */ type EpochMs = number; interface ResultSet { ref: string; /** The catalog id; absent on a formula. */ metric?: string; kind: MetricKind; /** `ratio`, `bytes`, `seconds`, `count`, `bool`, or the formula's; absent on a formula that gave none. */ unit?: string; series: Series[]; /** The series the engine answered, at most `max_series + 1`: `truncated` is exact, this is a floor, not the bucket's count. */ series_total: number; truncated: boolean; /** With `options.explain`: the expression as compiled for the engine. */ expr?: string; } /** Exact matches on the stored fields. */ interface EventFilters { /** `process`, `daemon_shutdown`, `app`, `action`, `shell`. */ kind?: string; node?: NodeId; app?: string; instance?: number; src?: string; level?: string; event_name?: string; fingerprint?: string; /** A call id: every line of one action or shell session. */ call?: string; } /** The labels a query may filter or group on; the bucket itself is never one of them. */ /** A built-in label, or an application's own label name (`^[a-z][a-z0-9_]{0,31}$`, not reserved) on a metric that carries it. */ type QueryLabel = "node" | "app" | "instance" | "ns" | "mount" | "window" | "src" | "core" | "iface" | "blockdev" | "sensor" | "gpu" | "array" | "proc" | (string & {}); /** A catalog line, minus what is empty or derivable: it is an application's exactly when `id` starts with `app.`. */ interface InventoryMetric { id: string; kind: MetricKind; /** Absent when the metric has none. */ unit?: string; /** Absent when it is the kind's: `avg_over_time` for a gauge, `rate` for a counter, `quantile` for a histogram. */ default_time?: TimeFnName; /** Absent when the application gave none, and with `?detail=index`. */ description?: string; /** An application line only: the apps whose series carry it, sorted — absent when none did. With several, put `app` in `aggregate.by` or a filter. */ apps?: string[]; } /** `3pmn_…` — a node. */ type NodeId = string; /** `NodeView.state`: the enrollment state, or the presence of an active node. */ type DisplayState = NodeState | "online" | "stale" | "offline"; interface InventoryProcess { app: string; instance: string; ns?: string; } /** Epoch milliseconds, or `now` / `now-` (`now-1h`, `now-7d`). */ type TimeValue = EpochMs | string; /** Milliseconds, or `30s` / `5m` / `1h` / `7d` / `2w`. */ type DurationValue = number | string; interface Filter { label: QueryLabel; op: FilterOp; /** `eq`, `neq`, `regex`, `nregex`. ≤ 256 characters. */ value?: string; /** `in`, `nin`: 1–64 values. */ values?: string[]; } interface TimeFn { fn: TimeFnName; /** `1m` or milliseconds, like `range.step`; default twice the step. Not for `raw`. */ window?: DurationValue; /** 0..1, `quantile_over_time` and `quantile` (a histogram's) only. */ q?: number; } interface Aggregate { fn: AggregateFnName; /** Labels the metric carries. */ by?: QueryLabel[]; /** 0..1, `quantile` only. */ q?: number; } type MetricKind = "gauge" | "counter" | "histogram"; interface Series { /** The `by` labels, or every public label without an aggregation — never the bucket. */ labels: Record; /** Aligned on `MetricsResult.times`; absent in `summary` mode. */ values?: (number | null)[]; summary: Summary; } /** `rate`, `increase`, `delta` for counters; the `*_over_time` family for gauges; `raw` for either; `quantile` (with `q`), `rate` or `increase` for a histogram. */ type TimeFnName = | "raw" | "rate" | "increase" | "delta" | "avg_over_time" | "min_over_time" | "max_over_time" | "last_over_time" | "quantile_over_time" | "quantile"; /** Enrollment state in the database, presence aside. */ type NodeState = "active" | "pending" | "revoked" | "unknown"; type FilterOp = "eq" | "neq" | "in" | "nin" | "regex" | "nregex"; type AggregateFnName = "sum" | "avg" | "min" | "max" | "count" | "quantile"; /** Over the points the engine returned on the grid — before `fill`, so `sort` ranks what the engine ranked. */ interface Summary { min: number | null; max: number | null; avg: number | null; last: number | null; /** Points the engine returned (not the filled ones). */ count: number; } ```