From 0439c079f013e1d7cf86276549865541d923eb02 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 25 Sep 2026 01:42:25 +0800 Subject: [PATCH] test(cache): run commands against the remote cache test server `remote-cache-server [args...]` starts the backend on a free loopback port under the `/projects/test` base path, runs the command with `VP_REMOTE_CACHE_URL` set to that endpoint, and exits with the command's exit code after stopping the server. State lives in `remote-cache/` in the working directory, so consecutive E2E steps share entries and blobs, and blob IDs keep counting across invocations. This replaces `start`, `stop`, the detached daemon, `cache.url`, `cache.log`, the `--base-path`, `--max-request-bytes`, and `--max-lifetime-ms` options, and the request size limit. `cbor-http` now joins its path onto `VP_REMOTE_CACHE_URL`. Fetch responses follow the server API in #713: a miss returns `{kind: "not_found"}`, and a fallback also returns the entry's value and blob ID. Co-authored-by: Claude Opus 5.5 --- .../remote_cache_backend/snapshots.toml | 174 +++++++------- .../snapshots/blob_replacement.md | 38 ++-- .../snapshots/command_wrapper.md | 18 ++ .../snapshots/daemon_lifecycle.md | 55 ----- .../snapshots/entries_and_associations.md | 46 ++-- .../snapshots/invalid_requests.md | 52 ++--- .../snapshots/opaque_binary_data.md | 26 +-- .../snapshots/request_limit.md | 42 ---- .../snapshots/state_across_invocations.md | 36 +++ packages/tools/README.md | 18 +- packages/tools/src/cbor-http.ts | 9 +- packages/tools/src/remote-cache/cli.ts | 212 ++---------------- packages/tools/src/remote-cache/server.ts | 131 +++++++---- 13 files changed, 314 insertions(+), 543 deletions(-) create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/command_wrapper.md delete mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/daemon_lifecycle.md delete mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/request_limit.md create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/state_across_invocations.md diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml index 20bdee4a9..dc8b97078 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml @@ -6,9 +6,6 @@ ignore = true steps = [ [ "remote-cache-server", - "start", - ], - [ "cbor-http", "POST", "/fetch", @@ -16,6 +13,7 @@ steps = [ "{\"key\": 'A', \"secondary_key\": 'S'}", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -23,6 +21,7 @@ steps = [ "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'VA'}", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -30,6 +29,7 @@ steps = [ "metadata={\"key\": 'A', \"secondary_key\": 'U', \"value\": 'VA'}", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -37,6 +37,7 @@ steps = [ "metadata={\"key\": 'B', \"secondary_key\": 'S', \"value\": 'VB'}", ], { argv = [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -44,13 +45,15 @@ steps = [ "{\"key\": 'A', \"secondary_key\": 'S'}", ], comment = "The exact entry survives reassignment of S to B." }, { argv = [ + "remote-cache-server", "cbor-http", "POST", "/fetch", "--cbor", "{\"key\": 'C', \"secondary_key\": 'S'}", - ], comment = "The previous fetch did not change S. Fallback returns only B's key." }, + ], comment = "The previous fetch did not change S, so fallback returns B's entry." }, [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -58,6 +61,7 @@ steps = [ "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": 'VA2'}", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -65,6 +69,7 @@ steps = [ "{\"key\": 'A', \"secondary_key\": 'missing'}", ], { argv = [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -72,6 +77,7 @@ steps = [ "{\"key\": 'C', \"secondary_key\": 'U'}", ], comment = "An older association to A still resolves after A is replaced." }, [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -79,16 +85,13 @@ steps = [ "{\"key\": 'C', \"secondary_key\": 'T'}", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", "--cbor", "{\"key\": 'C', \"secondary_key\": 'S'}", ], - [ - "remote-cache-server", - "stop", - ], ] [[e2e]] @@ -98,9 +101,6 @@ ignore = true steps = [ [ "remote-cache-server", - "start", - ], - [ "cbor-http", "GET", "/blob/missing", @@ -112,6 +112,7 @@ steps = [ "first archive", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -121,6 +122,7 @@ steps = [ "blob=archive.txt", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -128,11 +130,13 @@ steps = [ "{\"key\": 'A', \"secondary_key\": 'S'}", ], [ + "remote-cache-server", "cbor-http", "GET", "/blob/1", ], { argv = [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -142,6 +146,7 @@ steps = [ "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'second'}", ], comment = "Accept blob before metadata, with neither part supplying a filename." }, [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -149,16 +154,19 @@ steps = [ "{\"key\": 'A', \"secondary_key\": 'S'}", ], [ + "remote-cache-server", "cbor-http", "GET", "/blob/2", ], { argv = [ + "remote-cache-server", "cbor-http", "GET", "/blob/1", ], comment = "Previously returned IDs retain their original bytes." }, { argv = [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -166,6 +174,7 @@ steps = [ "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'no archive'}", ], comment = "Omitting blob clears the association." }, [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -173,6 +182,7 @@ steps = [ "{\"key\": 'A', \"secondary_key\": 'S'}", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -182,6 +192,7 @@ steps = [ "blob=''", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -189,14 +200,11 @@ steps = [ "{\"key\": 'A', \"secondary_key\": 'S'}", ], { argv = [ + "remote-cache-server", "cbor-http", "GET", "/blob/3", ], comment = "An empty blob has an ID and downloads as an empty byte string." }, - [ - "remote-cache-server", - "stop", - ], ] [[e2e]] @@ -206,9 +214,6 @@ ignore = true steps = [ [ "remote-cache-server", - "start", - ], - [ "cbor-http", "POST", "/store", @@ -218,6 +223,7 @@ steps = [ "blob=b64'AP+A'", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -225,6 +231,7 @@ steps = [ "{\"key\": b64'AP+A', \"secondary_key\": ''}", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -232,11 +239,13 @@ steps = [ "{\"key\": '', \"secondary_key\": ''}", ], [ + "remote-cache-server", "cbor-http", "GET", "/blob/1", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -244,16 +253,13 @@ steps = [ "metadata={\"key\": '', \"secondary_key\": '', \"value\": ''}", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", "--cbor", "{\"key\": '', \"secondary_key\": ''}", ], - [ - "remote-cache-server", - "stop", - ], ] [[e2e]] @@ -263,14 +269,12 @@ ignore = true steps = [ [ "remote-cache-server", - "start", - ], - [ "cbor-http", "GET", "/missing", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -278,6 +282,7 @@ steps = [ "'bad'", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -287,6 +292,7 @@ steps = [ "application/cbor", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -294,6 +300,7 @@ steps = [ "{\"key\": \"text\", \"secondary_key\": ''}", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -301,6 +308,7 @@ steps = [ "{\"key\": ''}", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -308,6 +316,7 @@ steps = [ "[]", ], [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -315,6 +324,7 @@ steps = [ "{\"key\": '', \"key\": 'duplicate', \"secondary_key\": ''}", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -324,6 +334,7 @@ steps = [ "blob='original archive'", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -331,6 +342,7 @@ steps = [ "{}", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -338,6 +350,7 @@ steps = [ "blob='missing metadata'", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -345,6 +358,7 @@ steps = [ "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": \"wrong type\"}", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -354,6 +368,7 @@ steps = [ "metadata={}", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -365,6 +380,7 @@ steps = [ "blob='two'", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -374,6 +390,7 @@ steps = [ "unexpected='part'", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -381,6 +398,7 @@ steps = [ "metadata='wrong content type'", ], [ + "remote-cache-server", "cbor-http", "POST", "/store", @@ -390,6 +408,7 @@ steps = [ "multipart/form-data; boundary=boundary", ], { argv = [ + "remote-cache-server", "cbor-http", "POST", "/fetch", @@ -397,118 +416,83 @@ steps = [ "{\"key\": 'A', \"secondary_key\": 'S'}", ], comment = "Failed stores did not replace the value or blob." }, [ + "remote-cache-server", "cbor-http", "GET", "/blob/1", ], { argv = [ + "remote-cache-server", "cbor-http", "POST", "/fetch", "--cbor", "{\"key\": 'B', \"secondary_key\": 'T'}", ], comment = "Failed stores did not publish a secondary association." }, - [ - "remote-cache-server", - "stop", - ], ] [[e2e]] -name = "request_limit" +name = "command_wrapper" cfg = "not(windows)" ignore = true steps = [ - [ + { argv = [ "remote-cache-server", - "start", - "--max-request-bytes", - "1024", - ], - [ - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'original'}", - ], - [ "node", "-e", - "require('node:fs').writeFileSync('large.bin', Buffer.alloc(2048))", - ], - [ - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": 'replacement'}", - "--form-file", - "blob=large.bin", - ], - [ - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], - [ - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'B', \"secondary_key\": 'T'}", - ], - [ + "console.log(new URL(process.env.VP_REMOTE_CACHE_URL).pathname)", + ], comment = "The endpoint includes a namespace base path." }, + { argv = [ "remote-cache-server", - "stop", - ], + "vtt", + "exit", + "3", + ], comment = "The wrapper exits with the command's exit code." }, ] [[e2e]] -name = "daemon_lifecycle" +name = "state_across_invocations" cfg = "not(windows)" ignore = true steps = [ [ "remote-cache-server", - "start", - "--base-path", - "/projects/test", - ], - [ - "remote-cache-server", - "start", - ], - [ "cbor-http", "POST", "/store", "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'first daemon'}", + "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'VA'}", + "--form-data", + "blob='first archive'", ], { argv = [ "remote-cache-server", - "stop", - ], comment = "Connect before deleting cache.url, then wait for the connection to close before restarting." }, - [ - "remote-cache-server", - "start", - ], - { argv = [ "cbor-http", "POST", "/fetch", "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], comment = "A new daemon starts with empty state." }, - [ + "{\"key\": 'C', \"secondary_key\": 'S'}", + ], comment = "A later invocation reads the stored entry and association." }, + { argv = [ "remote-cache-server", - "stop", - ], + "cbor-http", + "POST", + "/store", + "--form-cbor", + "metadata={\"key\": 'B', \"secondary_key\": 'T', \"value\": 'VB'}", + "--form-data", + "blob='second archive'", + ], comment = "Blob numbering continues across invocations." }, + { argv = [ + "vtt", + "write-file", + "remote-cache/blobs/1", + "replaced archive", + ], comment = "Each blob is a file named by its ID." }, [ "remote-cache-server", - "stop", + "cbor-http", + "GET", + "/blob/1", ], ] diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md index 23d025b5b..f0004361e 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md @@ -1,12 +1,6 @@ # blob_replacement -## `remote-cache-server start` - -``` -Remote cache server started -``` - -## `cbor-http GET /blob/missing` +## `remote-cache-server cbor-http GET /blob/missing` ``` {"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Blob not found"} @@ -17,25 +11,25 @@ Remote cache server started ``` ``` -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''first'\''}' --form-file blob=archive.txt` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''first'\''}' --form-file blob=archive.txt` ``` {"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} ``` -## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` ``` {"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'first', "blob_id": "1"}} ``` -## `cbor-http GET /blob/1` +## `remote-cache-server cbor-http GET /blob/1` ``` {"status": 200, "content_type": "application/octet-stream", "body": 'first archive'} ``` -## `cbor-http POST /store --form-data 'blob='\''second archive'\''' --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''second'\''}'` +## `remote-cache-server cbor-http POST /store --form-data 'blob='\''second archive'\''' --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''second'\''}'` Accept blob before metadata, with neither part supplying a filename. @@ -43,19 +37,19 @@ Accept blob before metadata, with neither part supplying a filename. {"status": 200, "content_type": "application/cbor", "body": {"blob_id": "2"}} ``` -## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` ``` {"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'second', "blob_id": "2"}} ``` -## `cbor-http GET /blob/2` +## `remote-cache-server cbor-http GET /blob/2` ``` {"status": 200, "content_type": "application/octet-stream", "body": 'second archive'} ``` -## `cbor-http GET /blob/1` +## `remote-cache-server cbor-http GET /blob/1` Previously returned IDs retain their original bytes. @@ -63,7 +57,7 @@ Previously returned IDs retain their original bytes. {"status": 200, "content_type": "application/octet-stream", "body": 'first archive'} ``` -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''no archive'\''}'` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''no archive'\''}'` Omitting blob clears the association. @@ -71,34 +65,28 @@ Omitting blob clears the association. {"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} ``` -## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` ``` {"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'no archive', "blob_id": null}} ``` -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''empty archive'\''}' --form-data blob=''` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''empty archive'\''}' --form-data blob=''` ``` {"status": 200, "content_type": "application/cbor", "body": {"blob_id": "3"}} ``` -## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` ``` {"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'empty archive', "blob_id": "3"}} ``` -## `cbor-http GET /blob/3` +## `remote-cache-server cbor-http GET /blob/3` An empty blob has an ID and downloads as an empty byte string. ``` {"status": 200, "content_type": "application/octet-stream", "body": ''} ``` - -## `remote-cache-server stop` - -``` -Remote cache server stopped -``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/command_wrapper.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/command_wrapper.md new file mode 100644 index 000000000..aa1edb503 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/command_wrapper.md @@ -0,0 +1,18 @@ +# command_wrapper + +## `remote-cache-server node -e 'console.log(new URL(process.env.VP_REMOTE_CACHE_URL).pathname)'` + +The endpoint includes a namespace base path. + +``` +/projects/test +``` + +## `remote-cache-server vtt exit 3` + +The wrapper exits with the command's exit code. + +**Exit code:** 3 + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/daemon_lifecycle.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/daemon_lifecycle.md deleted file mode 100644 index 46d141e70..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/daemon_lifecycle.md +++ /dev/null @@ -1,55 +0,0 @@ -# daemon_lifecycle - -## `remote-cache-server start --base-path /projects/test` - -``` -Remote cache server started -``` - -## `remote-cache-server start` - -**Exit code:** 1 - -``` -Remote cache server already started (cache.url exists) -``` - -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''first daemon'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -``` - -## `remote-cache-server stop` - -Connect before deleting cache.url, then wait for the connection to close before restarting. - -``` -Remote cache server stopped -``` - -## `remote-cache-server start` - -``` -Remote cache server started -``` - -## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -A new daemon starts with empty state. - -``` -{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Entry not found"} -``` - -## `remote-cache-server stop` - -``` -Remote cache server stopped -``` - -## `remote-cache-server stop` - -``` -No remote cache endpoint found -``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md index d52e2d149..b03a65b48 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md @@ -1,36 +1,30 @@ # entries_and_associations -## `remote-cache-server start` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` ``` -Remote cache server started +{"status": 200, "content_type": "application/cbor", "body": {"kind": "not_found"}} ``` -## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Entry not found"} -``` - -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''VA'\''}'` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''VA'\''}'` ``` {"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} ``` -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''U'\'', "value": '\''VA'\''}'` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''U'\'', "value": '\''VA'\''}'` ``` {"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} ``` -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''B'\'', "secondary_key": '\''S'\'', "value": '\''VB'\''}'` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''B'\'', "secondary_key": '\''S'\'', "value": '\''VB'\''}'` ``` {"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} ``` -## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` The exact entry survives reassignment of S to B. @@ -38,48 +32,42 @@ The exact entry survives reassignment of S to B. {"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'VA', "blob_id": null}} ``` -## `cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` -The previous fetch did not change S. Fallback returns only B's key. +The previous fetch did not change S, so fallback returns B's entry. ``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'B'}} +{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'B', "value": 'VB', "blob_id": null}} ``` -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''VA2'\''}'` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''VA2'\''}'` ``` {"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} ``` -## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''missing'\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''missing'\''}'` ``` {"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'VA2', "blob_id": null}} ``` -## `cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''U'\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''U'\''}'` An older association to A still resolves after A is replaced. ``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A'}} -``` - -## `cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''T'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A'}} +{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A', "value": 'VA2', "blob_id": null}} ``` -## `cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''T'\''}'` ``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'B'}} +{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A', "value": 'VA2', "blob_id": null}} ``` -## `remote-cache-server stop` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` ``` -Remote cache server stopped +{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'B', "value": 'VB', "blob_id": null}} ``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md index de39985ce..9d8e327d8 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md @@ -1,108 +1,102 @@ # invalid_requests -## `remote-cache-server start` - -``` -Remote cache server started -``` - -## `cbor-http GET /missing` +## `remote-cache-server cbor-http GET /missing` ``` {"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Route not found"} ``` -## `cbor-http POST /fetch --data 'bad'` +## `remote-cache-server cbor-http POST /fetch --data 'bad'` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected application/cbor"} ``` -## `cbor-http POST /fetch --data 'bad' --content-type application/cbor` +## `remote-cache-server cbor-http POST /fetch --data 'bad' --content-type application/cbor` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} ``` -## `cbor-http POST /fetch --cbor '{"key": "text", "secondary_key": '\'''\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": "text", "secondary_key": '\'''\''}'` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} ``` -## `cbor-http POST /fetch --cbor '{"key": '\'''\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\'''\''}'` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} ``` -## `cbor-http POST /fetch --cbor []` +## `remote-cache-server cbor-http POST /fetch --cbor []` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} ``` -## `cbor-http POST /fetch --cbor '{"key": '\'''\'', "key": '\''duplicate'\'', "secondary_key": '\'''\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\'''\'', "key": '\''duplicate'\'', "secondary_key": '\'''\''}'` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid CBOR"} ``` -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''original'\''}' --form-data 'blob='\''original archive'\'''` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''original'\''}' --form-data 'blob='\''original archive'\'''` ``` {"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} ``` -## `cbor-http POST /store --cbor {}` +## `remote-cache-server cbor-http POST /store --cbor {}` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected multipart/form-data"} ``` -## `cbor-http POST /store --form-data 'blob='\''missing metadata'\'''` +## `remote-cache-server cbor-http POST /store --form-data 'blob='\''missing metadata'\'''` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Missing metadata"} ``` -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": "wrong type"}'` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": "wrong type"}'` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key, value"} ``` -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''replacement'\''}' --form-cbor metadata={}` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''replacement'\''}' --form-cbor metadata={}` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} ``` -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''replacement'\''}' --form-data blob='one' --form-data blob='two'` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''replacement'\''}' --form-data blob='one' --form-data blob='two'` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} ``` -## `cbor-http POST /store --form-cbor metadata={} --form-data unexpected='part'` +## `remote-cache-server cbor-http POST /store --form-cbor metadata={} --form-data unexpected='part'` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} ``` -## `cbor-http POST /store --form-data 'metadata='\''wrong content type'\'''` +## `remote-cache-server cbor-http POST /store --form-data 'metadata='\''wrong content type'\'''` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} ``` -## `cbor-http POST /store --data ''\''--boundary\r\nContent-Disposition: form-data; name="metadata"\r\nContent-Type: application/cbor\r\n\r\n'\''' --content-type 'multipart/form-data; boundary=boundary'` +## `remote-cache-server cbor-http POST /store --data ''\''--boundary\r\nContent-Disposition: form-data; name="metadata"\r\nContent-Type: application/cbor\r\n\r\n'\''' --content-type 'multipart/form-data; boundary=boundary'` ``` {"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} ``` -## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` Failed stores did not replace the value or blob. @@ -110,22 +104,16 @@ Failed stores did not replace the value or blob. {"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'original', "blob_id": "1"}} ``` -## `cbor-http GET /blob/1` +## `remote-cache-server cbor-http GET /blob/1` ``` {"status": 200, "content_type": "application/octet-stream", "body": 'original archive'} ``` -## `cbor-http POST /fetch --cbor '{"key": '\''B'\'', "secondary_key": '\''T'\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''B'\'', "secondary_key": '\''T'\''}'` Failed stores did not publish a secondary association. ``` -{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Entry not found"} -``` - -## `remote-cache-server stop` - -``` -Remote cache server stopped +{"status": 200, "content_type": "application/cbor", "body": {"kind": "not_found"}} ``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md index d13fcf0db..ab3d40053 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md @@ -1,49 +1,37 @@ # opaque_binary_data -## `remote-cache-server start` - -``` -Remote cache server started -``` - -## `cbor-http POST /store --form-cbor 'metadata={"key": b64'\''AP+A'\'', "secondary_key": '\'''\'', "value": b64'\''AP+A'\''}' --form-data blob=b64'AP+A'` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": b64'\''AP+A'\'', "secondary_key": '\'''\'', "value": b64'\''AP+A'\''}' --form-data blob=b64'AP+A'` ``` {"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} ``` -## `cbor-http POST /fetch --cbor '{"key": b64'\''AP+A'\'', "secondary_key": '\'''\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": b64'\''AP+A'\'', "secondary_key": '\'''\''}'` ``` {"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": b64'AP+A', "blob_id": "1"}} ``` -## `cbor-http POST /fetch --cbor '{"key": '\'''\'', "secondary_key": '\'''\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\'''\'', "secondary_key": '\'''\''}'` ``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": b64'AP+A'}} +{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": b64'AP+A', "value": b64'AP+A', "blob_id": "1"}} ``` -## `cbor-http GET /blob/1` +## `remote-cache-server cbor-http GET /blob/1` ``` {"status": 200, "content_type": "application/octet-stream", "body": b64'AP+A'} ``` -## `cbor-http POST /store --form-cbor 'metadata={"key": '\'''\'', "secondary_key": '\'''\'', "value": '\'''\''}'` +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\'''\'', "secondary_key": '\'''\'', "value": '\'''\''}'` ``` {"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} ``` -## `cbor-http POST /fetch --cbor '{"key": '\'''\'', "secondary_key": '\'''\''}'` +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\'''\'', "secondary_key": '\'''\''}'` ``` {"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": '', "blob_id": null}} ``` - -## `remote-cache-server stop` - -``` -Remote cache server stopped -``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/request_limit.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/request_limit.md deleted file mode 100644 index 38aceb511..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/request_limit.md +++ /dev/null @@ -1,42 +0,0 @@ -# request_limit - -## `remote-cache-server start --max-request-bytes 1024` - -``` -Remote cache server started -``` - -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''original'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -``` - -## `node -e 'require('\''node:fs'\'').writeFileSync('\''large.bin'\'', Buffer.alloc(2048))'` - -``` -``` - -## `cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''replacement'\''}' --form-file blob=large.bin` - -``` -{"status": 413, "content_type": "text/plain; charset=utf-8", "body": "Request too large"} -``` - -## `cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'original', "blob_id": null}} -``` - -## `cbor-http POST /fetch --cbor '{"key": '\''B'\'', "secondary_key": '\''T'\''}'` - -``` -{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Entry not found"} -``` - -## `remote-cache-server stop` - -``` -Remote cache server stopped -``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/state_across_invocations.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/state_across_invocations.md new file mode 100644 index 000000000..b73e06671 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/state_across_invocations.md @@ -0,0 +1,36 @@ +# state_across_invocations + +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''VA'\''}' --form-data 'blob='\''first archive'\'''` + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} +``` + +## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` + +A later invocation reads the stored entry and association. + +``` +{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A', "value": 'VA', "blob_id": "1"}} +``` + +## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''B'\'', "secondary_key": '\''T'\'', "value": '\''VB'\''}' --form-data 'blob='\''second archive'\'''` + +Blob numbering continues across invocations. + +``` +{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "2"}} +``` + +## `vtt write-file remote-cache/blobs/1 'replaced archive'` + +Each blob is a file named by its ID. + +``` +``` + +## `remote-cache-server cbor-http GET /blob/1` + +``` +{"status": 200, "content_type": "application/octet-stream", "body": 'replaced archive'} +``` diff --git a/packages/tools/README.md b/packages/tools/README.md index 438964ca5..9d38cc804 100644 --- a/packages/tools/README.md +++ b/packages/tools/README.md @@ -5,25 +5,19 @@ This package provides Node.js dependencies and commands for the test suites. Run `pnpm install` at the repository root to install `remote-cache-server` and `cbor-http` into this package's `node_modules/.bin`. The workspace self-dependency makes pnpm link the package's own commands there. The E2E harness already includes that directory in PATH. Both commands run TypeScript directly using the Node version in `.node-version`. ```sh -remote-cache-server start -cbor-http POST /store --form-cbor "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'record'}" -cbor-http POST /fetch --cbor "{\"key\": 'A', \"secondary_key\": 'S'}" -remote-cache-server stop +remote-cache-server cbor-http POST /store --form-cbor "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'record'}" +remote-cache-server cbor-http POST /fetch --cbor "{\"key\": 'A', \"secondary_key\": 'S'}" ``` -`start` creates `cache.url` exclusively in the current directory, refusing to start if it already exists. It launches an in-memory daemon on a free loopback port and fills the file with its endpoint before returning. Each E2E case has its own directory and server state. +`remote-cache-server COMMAND [ARGS...]` starts the backend on a free loopback port and runs the command with `VP_REMOTE_CACHE_URL` set to the endpoint, `http://127.0.0.1:/projects/test`. The fixed base path gives every endpoint a namespace path. The wrapper takes no options and passes all arguments to the command unchanged. The command inherits stdio. When it exits, the server stops and the wrapper exits with the command's exit code, or 128 plus the signal number if a signal ended it. The terminal delivers Ctrl-C to the command too, so the wrapper keeps serving until the command exits. The wrapper forwards `SIGTERM` to the command. -`stop` connects to the endpoint, deletes `cache.url`, and reads until EOF or a connection reset confirms shutdown. The daemon watches the directory for file changes and closes its listener before closing active connections. Wait for `stop` to finish before restarting in the same directory. Deleting the file manually also requests shutdown, but a later `stop` cannot wait for the server because the endpoint is gone. The harness enforces step timeouts and skips remaining steps when one times out. The detached daemon exits after five minutes so it is cleaned up even if `stop` never runs. Use `--max-lifetime-ms` on `start` for longer tests. +State persists in `remote-cache/` in the current directory, so consecutive commands share it. Each E2E case has its own directory and state. `state.json` holds the entries, associations, and next blob ID, with keys and values hex-encoded. Each blob is a file in `remote-cache/blobs/` named by its blob ID. Blob IDs are sequential strings and continue across invocations, keeping snapshots deterministic. -Daemon output goes to `cache.log`, and startup failures include that log. The log stays in the test's directory. - -The backend implements `POST /fetch`, `POST /store`, and `GET /blob/{blob_id}` from the [remote cache RFC](https://github.com/voidzero-dev/vite-task/blob/rfc-cloudflare-remote-cache/docs/rfcs/0001-remote-cache.md#4-http-api-mapping). Keys and values are opaque bytes. Blobs remain unchanged until the daemon exits. Blob IDs are sequential strings to keep snapshots deterministic. There is no authentication or persistent storage. - -`start --base-path /projects/test` places the API under that path and includes it in `cache.url`. Requests are limited to 64 MiB by default; use `--max-request-bytes` to change the limit. Keys, values, and blobs have no separate length limits. +The backend implements `POST /fetch`, `POST /store`, and `GET /blob/{blob_id}` from the [remote cache server API](https://github.com/voidzero-dev/vite-task/pull/713). Keys, values, and blobs are opaque bytes without length limits. There is no authentication. ## CBOR HTTP client -`cbor-http METHOD PATH` resolves the path against `cache.url`. An absolute HTTP URL works without that file. Request bodies use [CBOR extended diagnostic notation (EDN)](https://www.rfc-editor.org/rfc/rfc8610.html#appendix-G): `"text"` is text, `'bytes'` is a UTF-8 byte string, and `b64'AP+A'` contains arbitrary binary bytes. +`cbor-http METHOD PATH` joins the path onto `VP_REMOTE_CACHE_URL`, including its base path, so run it through `remote-cache-server`. An absolute HTTP URL works without the variable. Request bodies use [CBOR extended diagnostic notation (EDN)](https://www.rfc-editor.org/rfc/rfc8610.html#appendix-G): `"text"` is text, `'bytes'` is a UTF-8 byte string, and `b64'AP+A'` contains arbitrary binary bytes. | Option | Body | | ----------------------- | --------------------------------------------------------- | diff --git a/packages/tools/src/cbor-http.ts b/packages/tools/src/cbor-http.ts index b889d7fc9..1206641cf 100755 --- a/packages/tools/src/cbor-http.ts +++ b/packages/tools/src/cbor-http.ts @@ -77,9 +77,12 @@ async function main(): Promise { contentType = `multipart/form-data; boundary=${boundary}`; } contentType = values['content-type'] ?? contentType; - const url = /^https?:\/\//.test(path) - ? path - : `${(await readFile('cache.url', 'utf8')).trim().replace(/\/$/, '')}/${path.replace(/^\//, '')}`; + let url = path; + if (!/^https?:\/\//.test(path)) { + const endpoint = process.env['VP_REMOTE_CACHE_URL']; + if (!endpoint) throw new Error('VP_REMOTE_CACHE_URL is not set'); + url = `${endpoint.replace(/\/$/, '')}/${path.replace(/^\//, '')}`; + } const response = await fetch(url, { method, ...(body === undefined ? {} : { body }), diff --git a/packages/tools/src/remote-cache/cli.ts b/packages/tools/src/remote-cache/cli.ts index b1dec8658..1c3fc5497 100755 --- a/packages/tools/src/remote-cache/cli.ts +++ b/packages/tools/src/remote-cache/cli.ts @@ -1,200 +1,34 @@ #!/usr/bin/env node -import { fork } from 'node:child_process'; +import { spawn } from 'node:child_process'; import { once } from 'node:events'; -import { open, readFile, rm, watch, writeFile } from 'node:fs/promises'; -import { createConnection } from 'node:net'; -import { Readable } from 'node:stream'; -import { setTimeout } from 'node:timers/promises'; -import { parseArgs } from 'node:util'; +import type { AddressInfo } from 'node:net'; +import { constants } from 'node:os'; import { createCacheServer } from './server.ts'; -const urlFile = 'cache.url'; -const logFile = 'cache.log'; +const basePath = '/projects/test'; +const directory = 'remote-cache'; -async function start(args: string[]): Promise { - try { - const file = await open(urlFile, 'wx'); - await file.close(); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'EEXIST') { - throw new Error('Remote cache server already started (cache.url exists)'); - } - throw error; - } +async function main(): Promise { + const [command, ...args] = process.argv.slice(2); + if (command === undefined) throw new Error('Usage: remote-cache-server COMMAND [ARGS...]'); + const server = createCacheServer({ basePath, directory }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); try { - const log = await open(logFile, 'w'); - const child = fork(import.meta.filename, ['serve', ...args], { - detached: true, - stdio: ['ignore', log.fd, log.fd, 'ipc'], + const { port } = server.address() as AddressInfo; + const child = spawn(command, args, { + stdio: 'inherit', + env: { ...process.env, VP_REMOTE_CACHE_URL: `http://127.0.0.1:${port}${basePath}` }, }); - try { - const [message] = await once(child, 'message'); - if (message !== 'ready') throw new Error('Unexpected daemon readiness message'); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new Error(`${message}\n${await readFile(logFile, 'utf8')}`.trim()); - } finally { - await log.close(); - if (child.connected) child.disconnect(); - child.unref(); - } - } catch (error) { - await rm(urlFile, { force: true }); - throw error; - } - console.log('Remote cache server started'); -} - -async function ownsUrlFile(url: string | undefined): Promise { - try { - return (await readFile(urlFile, 'utf8')).trim() === url; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; - throw error; - } -} - -async function serve( - maxLifetimeMs: number, - maxRequestBytes: number, - basePath: string, -): Promise { - const server = createCacheServer({ maxRequestBytes, basePath }); - const shutdown = new AbortController(); - const { signal } = shutdown; - let url: string | undefined; - - async function watchUrlFile(): Promise { - // Watch the directory so deleting the file is observable on every platform. - for await (const { filename } of watch('.', { signal })) { - if ((filename === null || filename === urlFile) && !(await ownsUrlFile(url))) return; - } - } - - async function announceReadiness(): Promise { - // Check after subscribing to file changes, so startup cannot miss a deletion. - // If the starter disappeared, nobody can use this instance. - if (!(await ownsUrlFile(url)) || !process.connected || signal.aborted) return; - process.send?.('ready'); - await setTimeout(maxLifetimeMs, undefined, { signal }); - } - - async function run(): Promise { - const listening = once(server, 'listening', { signal }); - server.listen(0, '127.0.0.1'); - await listening; - const address = server.address(); - if (!address || typeof address === 'string') throw new Error('Missing server address'); - url = `http://127.0.0.1:${address.port}${basePath}`; - // Fill the reserved file before reporting readiness. Do not recreate it if - // it was deleted during startup. - await writeFile(urlFile, `${url}\n`, { flag: 'r+' }); - await Promise.race([watchUrlFile(), announceReadiness()]); - } - - const pending = [ - once(process, 'SIGTERM', { signal }), - once(process, 'SIGINT', { signal }), - run(), - ]; - try { - await Promise.race(pending); - } finally { - shutdown.abort(); - await Promise.allSettled(pending); - // Finish file cleanup before closing the port, so a subsequent start after - // `stop` returns cannot have its URL file removed by this instance. - try { - if (await ownsUrlFile(url)) await rm(urlFile, { force: true }); - } catch (error) { - console.error(error); - process.exitCode = 1; - } finally { - if (server.listening) { - const closed = server[Symbol.asyncDispose](); - server.closeAllConnections(); - await closed; - } - } - } -} - -async function stop(): Promise { - let endpoint: string; - try { - endpoint = await readFile(urlFile, 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - console.log('No remote cache endpoint found'); - return; - } - const url = new URL(endpoint.trim()); - const socket = createConnection({ - host: url.hostname, - port: Number(url.port), - }); - const reader = Readable.toWeb(socket).getReader(); - try { - await once(socket, 'connect'); - await rm(urlFile, { force: true }); - // Shutdown closes the listener before this connection. EOF or a reset - // confirms the server has stopped accepting connections. - const { done } = await reader.read(); - if (!done) throw new Error('Unexpected data while waiting for remote cache server shutdown'); - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code !== 'ECONNREFUSED' && code !== 'ECONNRESET') throw error; - await rm(urlFile, { force: true }); + // The terminal also delivers Ctrl-C to the command. Keep serving until it exits. + process.on('SIGINT', () => {}); + process.on('SIGTERM', () => child.kill('SIGTERM')); + const [code, signal] = (await once(child, 'exit')) as [number | null, NodeJS.Signals | null]; + process.exitCode = code ?? 128 + constants.signals[signal!]; } finally { - reader.releaseLock(); - socket.destroy(); - } - console.log('Remote cache server stopped'); -} - -function positiveInteger(value: string): number { - const number = Number(value); - if (!Number.isSafeInteger(number) || number <= 0 || number > 2_147_483_647) { - throw new Error('Limits must be positive integers no greater than 2147483647'); - } - return number; -} - -async function main(): Promise { - const { values, positionals } = parseArgs({ - allowPositionals: true, - options: { - 'max-lifetime-ms': { type: 'string', default: '300000' }, - 'max-request-bytes': { type: 'string', default: String(64 * 1024 * 1024) }, - 'base-path': { type: 'string', default: '' }, - }, - }); - const lifetime = positiveInteger(values['max-lifetime-ms']); - const limit = positiveInteger(values['max-request-bytes']); - const basePath = values['base-path']; - if (basePath && !/^\/(?:[a-zA-Z0-9_-]+\/)*[a-zA-Z0-9_-]+$/.test(basePath)) { - throw new Error('Base path must contain slash-separated names without a trailing slash'); - } - if (positionals.length !== 1) throw new Error('Usage: remote-cache-server start|stop'); - switch (positionals[0]) { - case 'start': - await start([ - '--max-lifetime-ms', - String(lifetime), - '--max-request-bytes', - String(limit), - '--base-path', - basePath, - ]); - break; - case 'serve': - await serve(lifetime, limit, basePath); - break; - case 'stop': - await stop(); - break; - default: - throw new Error('Usage: remote-cache-server start|stop'); + const closed = server[Symbol.asyncDispose](); + server.closeAllConnections(); + await closed; } } diff --git a/packages/tools/src/remote-cache/server.ts b/packages/tools/src/remote-cache/server.ts index 8ca50870e..e7a08f9f9 100644 --- a/packages/tools/src/remote-cache/server.ts +++ b/packages/tools/src/remote-cache/server.ts @@ -1,13 +1,23 @@ import { Busboy } from '@fastify/busboy'; import { decode } from 'cbor2/decoder'; import { encode } from 'cbor2/encoder'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { join } from 'node:path'; interface Entry { - value: Uint8Array; + value: string; blob_id: string | null; } +/** The contents of `state.json`. Keys and values are hex-encoded. */ +interface State { + next_blob_id: number; + entries: Record; + associations: Record; +} + class RequestError extends Error { status: number; @@ -21,12 +31,20 @@ function mediaType(value: string | undefined): string { return value?.split(';')[0]?.trim().toLowerCase() ?? ''; } +function toHex(bytes: Uint8Array): string { + return Buffer.from(bytes).toString('hex'); +} + +function fromHex(hex: string): Uint8Array { + // Encode Uint8Array, not a Node Buffer, whose toJSON method would otherwise + // turn it into a map. + return new Uint8Array(Buffer.from(hex, 'hex')); +} + function byteFields(body: Uint8Array, names: string[]): Map { let value: unknown; try { - // Decode from Uint8Array so byte strings stay Uint8Array, not Node Buffers - // (whose toJSON method would otherwise turn them into maps when encoded). - value = decode(new Uint8Array(body), { preferMap: true, rejectDuplicateKeys: true }); + value = decode(body, { preferMap: true, rejectDuplicateKeys: true }); } catch { throw new RequestError(400, 'Invalid CBOR'); } @@ -36,19 +54,10 @@ function byteFields(body: Uint8Array, names: string[]): Map return value; } -function readBody(request: IncomingMessage, limit: number): Promise { +function readBody(request: IncomingMessage): Promise { return new Promise((resolve, reject) => { const chunks: Buffer[] = []; - let size = 0; - request.on('data', (chunk: Buffer) => { - size += chunk.length; - if (size > limit) { - chunks.length = 0; - reject(new RequestError(413, 'Request too large')); - } else { - chunks.push(chunk); - } - }); + request.on('data', (chunk: Buffer) => chunks.push(chunk)); request.on('end', () => resolve(Buffer.concat(chunks))); request.on('error', reject); request.on('aborted', () => reject(new RequestError(400, 'Incomplete request'))); @@ -86,17 +95,49 @@ function cbor(response: ServerResponse, value: unknown): void { response.end(encode(value)); } -/** A disposable backend. Keys, values, and blobs remain opaque bytes. */ -export function createCacheServer({ maxRequestBytes = 64 * 1024 * 1024, basePath = '' } = {}) { - const entries = new Map(); - const associations = new Map(); - const blobs = new Map(); - let nextBlobId = 1; +function loadState(file: string): State { + try { + return JSON.parse(readFileSync(file, 'utf8')) as State; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + return { next_blob_id: 1, entries: {}, associations: {} }; + } +} + +async function readBlob(file: string): Promise { + try { + return await readFile(file); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } +} + +/** + * A test backend that keeps its state in `directory`: entries and associations + * in `state.json`, and each blob in `blobs/` under its ID. Keys, values, and + * blobs remain opaque bytes. + */ +export function createCacheServer({ + basePath, + directory, +}: { + basePath: string; + directory: string; +}) { + const stateFile = join(directory, 'state.json'); + const blobDirectory = join(directory, 'blobs'); + const state = loadState(stateFile); + const entries = new Map(Object.entries(state.entries)); + const associations = new Map(Object.entries(state.associations)); + let nextBlobId = state.next_blob_id; async function handle(request: IncomingMessage, response: ServerResponse): Promise { const path = new URL(request.url ?? '/', 'http://localhost').pathname; if (request.method === 'GET' && path.startsWith(`${basePath}/blob/`)) { - const blob = blobs.get(path.slice(`${basePath}/blob/`.length)); + const blobId = path.slice(`${basePath}/blob/`.length); + // Blob IDs are sequential numbers, so other IDs cannot name a blob file. + const blob = /^\d+$/.test(blobId) ? await readBlob(join(blobDirectory, blobId)) : undefined; if (blob === undefined) throw new RequestError(404, 'Blob not found'); response.writeHead(200, { 'content-type': 'application/octet-stream' }); response.end(blob); @@ -107,26 +148,26 @@ export function createCacheServer({ maxRequestBytes = 64 * 1024 * 1024, basePath } const contentType = request.headers['content-type'] ?? ''; - const body = await readBody(request, maxRequestBytes); + const body = await readBody(request); if (path === `${basePath}/fetch`) { if (mediaType(contentType) !== 'application/cbor') { throw new RequestError(400, 'Expected application/cbor'); } const fields = byteFields(body, ['key', 'secondary_key']); - const key = Buffer.from(fields.get('key')!).toString('hex'); - const secondary = Buffer.from(fields.get('secondary_key')!).toString('hex'); - const entry = entries.get(key); - if (entry) { - cbor(response, { kind: 'exact', ...entry }); - } else { - const associatedKey = associations.get(secondary); - if (associatedKey === undefined || !entries.has(associatedKey)) { - throw new RequestError(404, 'Entry not found'); - } + const exact = entries.get(toHex(fields.get('key')!)); + const associatedKey = associations.get(toHex(fields.get('secondary_key')!)); + const fallback = associatedKey === undefined ? undefined : entries.get(associatedKey); + if (exact) { + cbor(response, { kind: 'exact', value: fromHex(exact.value), blob_id: exact.blob_id }); + } else if (fallback) { cbor(response, { kind: 'fallback', - key: new Uint8Array(Buffer.from(associatedKey, 'hex')), + key: fromHex(associatedKey!), + value: fromHex(fallback.value), + blob_id: fallback.blob_id, }); + } else { + cbor(response, { kind: 'not_found' }); } return; } @@ -143,16 +184,22 @@ export function createCacheServer({ maxRequestBytes = 64 * 1024 * 1024, basePath const metadata = parts.get('metadata'); if (metadata === undefined) throw new RequestError(400, 'Missing metadata'); const fields = byteFields(metadata, ['key', 'secondary_key', 'value']); - const key = Buffer.from(fields.get('key')!).toString('hex'); - const secondary = Buffer.from(fields.get('secondary_key')!).toString('hex'); + const key = toHex(fields.get('key')!); const blob = parts.get('blob'); - const blobId = blob === undefined ? null : String(nextBlobId++); - // Publish only after the complete request is validated. There is no await - // between these writes, so concurrent stores cannot mix values and blobs. - if (blobId !== null) blobs.set(blobId, blob!); - entries.set(key, { value: fields.get('value')!, blob_id: blobId }); - associations.set(secondary, key); + // Publish only after the complete request is validated. The writes are + // synchronous, so concurrent stores cannot interleave them. + mkdirSync(blobDirectory, { recursive: true }); + const blobId = blob === undefined ? null : String(nextBlobId++); + if (blobId !== null) writeFileSync(join(blobDirectory, blobId), blob!); + entries.set(key, { value: toHex(fields.get('value')!), blob_id: blobId }); + associations.set(toHex(fields.get('secondary_key')!), key); + const saved: State = { + next_blob_id: nextBlobId, + entries: Object.fromEntries(entries), + associations: Object.fromEntries(associations), + }; + writeFileSync(stateFile, `${JSON.stringify(saved, null, 2)}\n`); cbor(response, { blob_id: blobId }); }