fix: logo 右半部分从 CODING 改为 CODE
去掉难以正确渲染的 N 和 G 字母,右半部分简化为 CODE(4 字母), 与左半部分 AIR 组合为 AIR CODE。
This commit is contained in:
21
packages/http-recorder/LICENSE
Normal file
21
packages/http-recorder/LICENSE
Normal file
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 opencode
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
217
packages/http-recorder/README.md
Normal file
217
packages/http-recorder/README.md
Normal file
@@ -0,0 +1,217 @@
|
||||
# @opencode-ai/http-recorder
|
||||
|
||||
Record real Effect HTTP and WebSocket traffic once, then replay it from deterministic JSON cassettes.
|
||||
|
||||
Use it for provider integrations, retries, polling, multi-step flows, and any test where hand-written HTTP mocks hide too much of the real request shape.
|
||||
|
||||
> Public beta. The API depends on Effect 4 beta and may change with Effect's unstable transport modules.
|
||||
|
||||
## Install
|
||||
|
||||
```sh
|
||||
bun add effect@4.0.0-beta.74
|
||||
bun add -d @opencode-ai/http-recorder@beta @effect/vitest vitest
|
||||
```
|
||||
|
||||
The package supports Node.js 22+ and Bun. It is not intended for browsers, workers, or Deno.
|
||||
|
||||
Effect `4.0.0-beta.74` has a known declaration error (`SchemaErrorTypeId` is missing). Until that upstream declaration is fixed, TypeScript consumers need:
|
||||
|
||||
```json
|
||||
{
|
||||
"compilerOptions": {
|
||||
"skipLibCheck": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```ts
|
||||
import { assert, describe, it } from "@effect/vitest"
|
||||
import { Effect, Schema } from "effect"
|
||||
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
|
||||
import { HttpRecorder } from "@opencode-ai/http-recorder"
|
||||
|
||||
const User = Schema.Struct({
|
||||
id: Schema.Number,
|
||||
name: Schema.String,
|
||||
})
|
||||
|
||||
const getUser = Effect.gen(function* () {
|
||||
const http = yield* HttpClient.HttpClient
|
||||
const response = yield* http.execute(HttpClientRequest.get("https://jsonplaceholder.typicode.com/users/1"))
|
||||
return yield* Schema.decodeUnknownEffect(User)(yield* response.json)
|
||||
})
|
||||
|
||||
describe("getUser", () => {
|
||||
it.effect("loads a user", () =>
|
||||
Effect.gen(function* () {
|
||||
const user = yield* getUser
|
||||
|
||||
assert.strictEqual(user.id, 1)
|
||||
assert.strictEqual(user.name, "Leanne Graham")
|
||||
}).pipe(Effect.provide(HttpRecorder.http("users/get-one"))),
|
||||
)
|
||||
})
|
||||
```
|
||||
|
||||
Run the test with Vitest. The first local run calls the real API and records:
|
||||
|
||||
```sh
|
||||
bunx vitest run users.test.ts
|
||||
```
|
||||
|
||||
```text
|
||||
test/fixtures/recordings/users/get-one.json
|
||||
```
|
||||
|
||||
Later runs replay that cassette without contacting the upstream server. When `CI=true`, missing cassettes fail instead of recording.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Run[Run test] --> Recorded{Cassette recorded?}
|
||||
Recorded -->|Yes| Replay[Replay cassette]
|
||||
Recorded -->|No, local| Record[Call service and record cassette]
|
||||
Recorded -->|No, CI| Fail[Fail: cassette missing]
|
||||
```
|
||||
|
||||
Application code does not need to know whether a response is live or replayed.
|
||||
|
||||
## API
|
||||
|
||||
```ts
|
||||
HttpRecorder.http(name, options?)
|
||||
HttpRecorder.socket(name, options?)
|
||||
```
|
||||
|
||||
That is the complete public API. `http` provides a fetch-backed recorded `HttpClient`. `socket` decorates a standard Effect `Socket.Socket` supplied beneath it.
|
||||
|
||||
## WebSockets
|
||||
|
||||
WebSocket cassettes preserve one ordered transcript of client and server text or binary frames. Replay follows that chronology: server frames are released until the next recorded client frame, then replay waits for the application to send the matching frame before continuing.
|
||||
|
||||
```ts
|
||||
import { assert, it } from "@effect/vitest"
|
||||
import { NodeSocket } from "@effect/platform-node"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { Socket } from "effect/unstable/socket"
|
||||
import { HttpRecorder } from "@opencode-ai/http-recorder"
|
||||
|
||||
const echo = Effect.gen(function* () {
|
||||
const socket = yield* Socket.Socket
|
||||
const write = yield* socket.writer
|
||||
|
||||
yield* socket.runString(
|
||||
(message) =>
|
||||
Effect.gen(function* () {
|
||||
assert.strictEqual(message, "hello")
|
||||
yield* write(new Socket.CloseEvent(1000))
|
||||
}),
|
||||
{ onOpen: write("hello") },
|
||||
)
|
||||
})
|
||||
|
||||
const recordedSocket = HttpRecorder.socket("echo/hello").pipe(
|
||||
Layer.provide(
|
||||
NodeSocket.layerWebSocket("wss://ws.postman-echo.com/raw", {
|
||||
closeCodeIsError: (code) => code !== 1000,
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("exchanges WebSocket frames", () => echo.pipe(Effect.provide(recordedSocket)))
|
||||
```
|
||||
|
||||
The application owns the WebSocket URL and protocols through normal Effect layer wiring. The recorder wraps that socket without duplicating its URL in recorder configuration. Provide separate socket layers for separate endpoints or concurrent connections.
|
||||
|
||||
Text frames use the same JSON-field and body redaction as HTTP bodies. Binary frames are stored losslessly as base64. Client and server frame kinds must match during replay.
|
||||
|
||||
## Refresh A Cassette
|
||||
|
||||
Delete exactly the recordings you want to replace, then rerun their tests:
|
||||
|
||||
```sh
|
||||
rm test/fixtures/recordings/users/get-one.json
|
||||
bun run test users.test.ts
|
||||
```
|
||||
|
||||
There is intentionally no public overwrite mode. Deletion makes the set of recordings being refreshed visible and reviewable.
|
||||
|
||||
## Redaction
|
||||
|
||||
Secure defaults remove most headers and redact common credentials in headers, URLs, and JSON bodies. Extend those defaults at layer construction:
|
||||
|
||||
```ts
|
||||
HttpRecorder.http("anthropic/messages", {
|
||||
redact: {
|
||||
headers: ["x-project-token"],
|
||||
allowRequestHeaders: ["anthropic-version"],
|
||||
queryParameters: ["session-id"],
|
||||
jsonFields: ["user_id"],
|
||||
url: (url) => url.replace(/\/accounts\/[^/]+/, "/accounts/{account}"),
|
||||
body: (body) => body.replaceAll(/usr_[a-z0-9]+/g, "usr_redacted"),
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
| Option | Purpose |
|
||||
| ---------------------- | -------------------------------------------------------------------- |
|
||||
| `headers` | Add sensitive header names. They are retained as `[REDACTED]`. |
|
||||
| `allowRequestHeaders` | Preserve additional non-sensitive request headers for matching. |
|
||||
| `allowResponseHeaders` | Preserve additional non-sensitive response headers for replay. |
|
||||
| `queryParameters` | Add sensitive URL query parameter names. |
|
||||
| `jsonFields` | Recursively redact matching JSON keys in requests and responses. |
|
||||
| `url` | Stabilize a URL after built-in redaction. |
|
||||
| `body` | Stabilize request and response bodies after built-in JSON redaction. |
|
||||
|
||||
Before writing, the recorder scans the complete cassette for common credential formats and values from credential-like environment variables. Unsafe cassettes fail without replacing an existing recording.
|
||||
|
||||
Redaction is defense in depth, not a substitute for review. Inspect cassette diffs before committing them.
|
||||
|
||||
## Matching And Ordering
|
||||
|
||||
A cassette contains an ordered sequence of interactions. The first runtime request is checked against the first recorded request, the second against the second, and so on.
|
||||
|
||||
This strict ordering correctly models repeated identical requests whose responses change, including retries, polling, and cache tests. JSON object keys are canonicalized before matching.
|
||||
|
||||
Concurrent requests are recorded in request-start order even when their responses complete out of order.
|
||||
|
||||
Supply a custom equivalence rule when a request contains intentionally volatile data:
|
||||
|
||||
```ts
|
||||
HttpRecorder.http("events/create", {
|
||||
match: (incoming, recorded) =>
|
||||
incoming.method === recorded.method && new URL(incoming.url).pathname === new URL(recorded.url).pathname,
|
||||
})
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
```ts
|
||||
interface RecorderOptions {
|
||||
readonly directory?: string
|
||||
readonly metadata?: Record<string, unknown>
|
||||
readonly redact?: RedactOptions
|
||||
readonly match?: RequestMatcher
|
||||
}
|
||||
```
|
||||
|
||||
`directory` defaults to `<cwd>/test/fixtures/recordings`.
|
||||
|
||||
## Cassettes
|
||||
|
||||
Cassettes are readable JSON files intended to be committed with your tests. HTTP interactions are stored in request order. WebSocket cassettes preserve the observed order of client and server frames. Text stays readable; binary bodies and frames are stored losslessly as base64.
|
||||
|
||||
## Current Limits
|
||||
|
||||
- Responses are buffered while recording and replaying, so this beta is not suitable for tests that assert streaming timing, cancellation, or backpressure.
|
||||
- WebSocket replay preserves frame chronology and content, not real network timing or backpressure.
|
||||
- WebSocket V1 cassettes do not reproduce terminal close codes, close reasons, or transport failures. Failed and interrupted live runs are not recorded.
|
||||
- WebSocket transcripts are retained in memory until the connection finishes; avoid using this beta for unbounded sessions.
|
||||
- The package currently requires the exact Effect beta listed above.
|
||||
- Cassette format version `1` has no migration tooling yet.
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
60
packages/http-recorder/package.json
Normal file
60
packages/http-recorder/package.json
Normal file
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/package.json",
|
||||
"version": "1.17.4",
|
||||
"name": "@opencode-ai/http-recorder",
|
||||
"description": "Record and replay Effect HTTP client traffic with deterministic cassettes",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/anomalyco/opencode.git",
|
||||
"directory": "packages/http-recorder"
|
||||
},
|
||||
"homepage": "https://github.com/anomalyco/opencode/tree/dev/packages/http-recorder",
|
||||
"bugs": "https://github.com/anomalyco/opencode/issues",
|
||||
"keywords": [
|
||||
"effect",
|
||||
"http",
|
||||
"recording",
|
||||
"replay",
|
||||
"testing",
|
||||
"vcr"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "bun test --timeout 30000 --only-failures",
|
||||
"typecheck": "tsgo --noEmit",
|
||||
"build": "bun ./script/build.ts",
|
||||
"verify:package": "bun ./script/verify-package.ts"
|
||||
},
|
||||
"exports": {
|
||||
".": "./src/index.ts",
|
||||
"./internal": "./src/internal.ts"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"README.md",
|
||||
"CHANGELOG.md",
|
||||
"LICENSE"
|
||||
],
|
||||
"devDependencies": {
|
||||
"@tsconfig/node22": "catalog:",
|
||||
"@types/bun": "catalog:",
|
||||
"@types/node": "catalog:",
|
||||
"@typescript/native-preview": "catalog:",
|
||||
"effect": "catalog:",
|
||||
"typescript": "catalog:"
|
||||
},
|
||||
"dependencies": {
|
||||
"@effect/platform-node": "4.0.0-beta.74",
|
||||
"@effect/platform-node-shared": "4.0.0-beta.74"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"effect": "4.0.0-beta.74"
|
||||
}
|
||||
}
|
||||
20
packages/http-recorder/script/build.ts
Normal file
20
packages/http-recorder/script/build.ts
Normal file
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bun
|
||||
import { $ } from "bun"
|
||||
import { readdir, rm } from "node:fs/promises"
|
||||
|
||||
await rm("dist", { recursive: true, force: true })
|
||||
await $`bunx tsc --emitDeclarationOnly`
|
||||
|
||||
const build = await Bun.build({
|
||||
entrypoints: ["src/index.ts"],
|
||||
outdir: "dist",
|
||||
target: "node",
|
||||
format: "esm",
|
||||
packages: "external",
|
||||
})
|
||||
if (!build.success) throw new AggregateError(build.logs, "Failed to build @opencode-ai/http-recorder")
|
||||
|
||||
const publicFiles = new Set(["index.js", "index.d.ts", "effect.d.ts", "socket.d.ts", "types.d.ts"])
|
||||
await Promise.all(
|
||||
(await readdir("dist")).filter((file) => !publicFiles.has(file)).map((file) => rm(`dist/${file}`, { force: true })),
|
||||
)
|
||||
35
packages/http-recorder/script/pack.ts
Normal file
35
packages/http-recorder/script/pack.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bun
|
||||
import { $ } from "bun"
|
||||
import { fileURLToPath } from "node:url"
|
||||
|
||||
const dir = fileURLToPath(new URL("..", import.meta.url))
|
||||
|
||||
export const pack = async () => {
|
||||
process.chdir(dir)
|
||||
await $`bun run build`
|
||||
const original = await Bun.file("package.json").text()
|
||||
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- package.json is validated by the package schema and build checks.
|
||||
const pkg = JSON.parse(original) as {
|
||||
readonly version: string
|
||||
exports: Record<string, string | { readonly import: string; readonly types: string }>
|
||||
}
|
||||
|
||||
for (const [key, value] of Object.entries(pkg.exports)) {
|
||||
if (key === "./internal") {
|
||||
delete pkg.exports[key]
|
||||
continue
|
||||
}
|
||||
if (typeof value !== "string") continue
|
||||
const file = value.replace("./src/", "./dist/").replace(/\.ts$/, "")
|
||||
pkg.exports[key] = { import: `${file}.js`, types: `${file}.d.ts` }
|
||||
}
|
||||
await Bun.write("package.json", JSON.stringify(pkg, null, 2))
|
||||
try {
|
||||
await $`bun pm pack`
|
||||
return fileURLToPath(new URL(`../opencode-ai-http-recorder-${pkg.version}.tgz`, import.meta.url))
|
||||
} finally {
|
||||
await Bun.write("package.json", original)
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) await pack()
|
||||
75
packages/http-recorder/script/verify-package.ts
Normal file
75
packages/http-recorder/script/verify-package.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bun
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
import path from "node:path"
|
||||
import { pack } from "./pack.js"
|
||||
|
||||
const run = async (command: ReadonlyArray<string>, cwd: string) => {
|
||||
const process = Bun.spawn(command, { cwd, env: globalThis.process.env, stdout: "inherit", stderr: "inherit" })
|
||||
const exitCode = await process.exited
|
||||
if (exitCode !== 0) throw new Error(`${command.join(" ")} exited with code ${exitCode}`)
|
||||
}
|
||||
|
||||
export const verifyPackage = async (archive: string) => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "http-recorder-consumer-"))
|
||||
try {
|
||||
await writeFile(
|
||||
path.join(directory, "package.json"),
|
||||
JSON.stringify({ name: "http-recorder-consumer", private: true, type: "module" }),
|
||||
)
|
||||
await writeFile(
|
||||
path.join(directory, "consumer.ts"),
|
||||
`import { HttpRecorder } from "@opencode-ai/http-recorder"
|
||||
import { NodeSocket } from "@effect/platform-node"
|
||||
import { Layer } from "effect"
|
||||
import { HttpClient } from "effect/unstable/http"
|
||||
import { Socket } from "effect/unstable/socket"
|
||||
|
||||
const options: HttpRecorder.RecorderOptions = { redact: { jsonFields: ["access_token"] } }
|
||||
HttpRecorder.http("consumer", options) satisfies Layer.Layer<HttpClient.HttpClient>
|
||||
HttpRecorder.socket("consumer/socket", options).pipe(
|
||||
Layer.provide(NodeSocket.layerWebSocket("wss://example.test")),
|
||||
) satisfies Layer.Layer<Socket.Socket>
|
||||
`,
|
||||
)
|
||||
await writeFile(
|
||||
path.join(directory, "tsconfig.json"),
|
||||
JSON.stringify({
|
||||
compilerOptions: {
|
||||
target: "ES2022",
|
||||
module: "NodeNext",
|
||||
moduleResolution: "NodeNext",
|
||||
strict: true,
|
||||
noEmit: true,
|
||||
// Required by effect@4.0.0-beta.74: its schema.d.ts references an undeclared SchemaErrorTypeId.
|
||||
skipLibCheck: true,
|
||||
lib: ["ES2022", "DOM", "ESNext.Disposable"],
|
||||
},
|
||||
include: ["consumer.ts"],
|
||||
}),
|
||||
)
|
||||
|
||||
await run(["npm", "install", archive, "typescript@5.8.2"], directory)
|
||||
await run(
|
||||
[
|
||||
"node",
|
||||
"--input-type=module",
|
||||
"-e",
|
||||
'import("@opencode-ai/http-recorder").then((module) => { const root = Object.keys(module).sort(); const namespace = Object.keys(module.HttpRecorder).sort(); if (JSON.stringify(root) !== JSON.stringify(["HttpRecorder"])) throw new Error(`Unexpected root exports: ${root}`); if (JSON.stringify(namespace) !== JSON.stringify(["http", "socket"])) throw new Error(`Unexpected namespace exports: ${namespace}`) })',
|
||||
],
|
||||
directory,
|
||||
)
|
||||
await run([path.join(directory, "node_modules", ".bin", "tsc"), "--noEmit"], directory)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const archive = await pack()
|
||||
try {
|
||||
await verifyPackage(archive)
|
||||
} finally {
|
||||
await Bun.file(archive).delete()
|
||||
}
|
||||
}
|
||||
179
packages/http-recorder/src/cassette.ts
Normal file
179
packages/http-recorder/src/cassette.ts
Normal file
@@ -0,0 +1,179 @@
|
||||
import { Context, Effect, FileSystem, Layer, Schema, Semaphore } from "effect"
|
||||
import * as fs from "node:fs"
|
||||
import * as path from "node:path"
|
||||
import { secretFindings, SecretFindingSchema, type SecretFinding } from "./redaction.js"
|
||||
import { CassetteSchema, encodeCassette, type Cassette, type CassetteMetadata, type Interaction } from "./schema.js"
|
||||
|
||||
const DEFAULT_RECORDINGS_DIR = path.resolve(process.cwd(), "test", "fixtures", "recordings")
|
||||
|
||||
export class CassetteNotFoundError extends Schema.TaggedErrorClass<CassetteNotFoundError>()("CassetteNotFoundError", {
|
||||
cassetteName: Schema.String,
|
||||
}) {
|
||||
override get message() {
|
||||
return `Cassette "${this.cassetteName}" not found`
|
||||
}
|
||||
}
|
||||
|
||||
export class UnsafeCassetteError extends Schema.TaggedErrorClass<UnsafeCassetteError>()("UnsafeCassetteError", {
|
||||
cassetteName: Schema.String,
|
||||
findings: Schema.Array(SecretFindingSchema),
|
||||
}) {
|
||||
override get message() {
|
||||
return `Refusing to write cassette "${this.cassetteName}" because it contains possible secrets: ${this.findings
|
||||
.map((finding) => `${finding.path} (${finding.reason})`)
|
||||
.join(", ")}`
|
||||
}
|
||||
}
|
||||
|
||||
export interface Interface {
|
||||
readonly read: (name: string) => Effect.Effect<ReadonlyArray<Interaction>, CassetteNotFoundError>
|
||||
readonly append: (
|
||||
name: string,
|
||||
interaction: Interaction,
|
||||
metadata?: CassetteMetadata,
|
||||
) => Effect.Effect<void, UnsafeCassetteError>
|
||||
readonly exists: (name: string) => Effect.Effect<boolean>
|
||||
readonly list: () => Effect.Effect<ReadonlyArray<string>>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode-ai/http-recorder/Cassette") {}
|
||||
|
||||
const cassettePath = (directory: string, name: string) => {
|
||||
if (!name || path.isAbsolute(name) || path.win32.isAbsolute(name) || name.split(/[\\/]/).includes(".."))
|
||||
throw new Error(`Invalid cassette name "${name}"`)
|
||||
const root = path.resolve(directory)
|
||||
const target = path.resolve(root, `${name}.json`)
|
||||
const relative = path.relative(root, target)
|
||||
if (!relative || relative.startsWith("..") || path.isAbsolute(relative))
|
||||
throw new Error(`Invalid cassette name "${name}"`)
|
||||
return target
|
||||
}
|
||||
|
||||
export const hasCassetteSync = (name: string, options: { readonly directory?: string } = {}) =>
|
||||
fs.existsSync(cassettePath(options.directory ?? DEFAULT_RECORDINGS_DIR, name))
|
||||
|
||||
const buildCassette = (
|
||||
name: string,
|
||||
interactions: ReadonlyArray<Interaction>,
|
||||
metadata: CassetteMetadata | undefined,
|
||||
): Cassette => ({
|
||||
version: 1,
|
||||
metadata: { name, recordedAt: new Date().toISOString(), ...metadata },
|
||||
interactions,
|
||||
})
|
||||
|
||||
const formatCassette = (cassette: Cassette) => `${JSON.stringify(encodeCassette(cassette), null, 2)}\n`
|
||||
|
||||
const parseCassette = Schema.decodeUnknownSync(Schema.fromJsonString(CassetteSchema))
|
||||
|
||||
const failIfUnsafe = (name: string, findings: ReadonlyArray<SecretFinding>) =>
|
||||
findings.length === 0 ? Effect.void : Effect.fail(new UnsafeCassetteError({ cassetteName: name, findings }))
|
||||
|
||||
export const fileSystem = (
|
||||
options: { readonly directory?: string } = {},
|
||||
): Layer.Layer<Service, never, FileSystem.FileSystem> =>
|
||||
Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* FileSystem.FileSystem
|
||||
const directory = options.directory ?? DEFAULT_RECORDINGS_DIR
|
||||
const recorded = new Map<string, { interactions: Interaction[]; findings: SecretFinding[] }>()
|
||||
const appendLock = yield* Semaphore.make(1)
|
||||
|
||||
const pathFor = (name: string) => cassettePath(directory, name)
|
||||
|
||||
const walk = (current: string): Effect.Effect<ReadonlyArray<string>> =>
|
||||
Effect.gen(function* () {
|
||||
const entries = yield* fs.readDirectory(current).pipe(Effect.catch(() => Effect.succeed([] as string[])))
|
||||
const nested = yield* Effect.forEach(entries, (entry) => {
|
||||
const full = path.join(current, entry)
|
||||
return fs.stat(full).pipe(
|
||||
Effect.flatMap((stat) => (stat.type === "Directory" ? walk(full) : Effect.succeed([full]))),
|
||||
Effect.catch(() => Effect.succeed([] as string[])),
|
||||
)
|
||||
})
|
||||
return nested.flat()
|
||||
})
|
||||
|
||||
return Service.of({
|
||||
read: (name) =>
|
||||
fs.readFileString(pathFor(name)).pipe(
|
||||
Effect.map((raw) => parseCassette(raw).interactions),
|
||||
Effect.catch(() => Effect.fail(new CassetteNotFoundError({ cassetteName: name }))),
|
||||
),
|
||||
append: (name, interaction, metadata) =>
|
||||
appendLock.withPermit(
|
||||
Effect.gen(function* () {
|
||||
const entry = recorded.get(name) ?? { interactions: [], findings: [] }
|
||||
const interactions = [...entry.interactions, interaction]
|
||||
const interactionFindings = [...entry.findings, ...secretFindings(interaction)]
|
||||
const cassette = buildCassette(name, interactions, metadata)
|
||||
const findings = [...interactionFindings, ...secretFindings(cassette.metadata ?? {})]
|
||||
yield* failIfUnsafe(name, findings)
|
||||
const target = pathFor(name)
|
||||
yield* fs.makeDirectory(path.dirname(target), { recursive: true }).pipe(Effect.orDie)
|
||||
const temporary = `${target}.${crypto.randomUUID()}.tmp`
|
||||
yield* fs.writeFileString(temporary, formatCassette(cassette)).pipe(
|
||||
Effect.flatMap(() => fs.rename(temporary, target)),
|
||||
Effect.ensuring(fs.remove(temporary, { force: true }).pipe(Effect.catch(() => Effect.void))),
|
||||
Effect.orDie,
|
||||
)
|
||||
recorded.set(name, { interactions, findings: interactionFindings })
|
||||
}),
|
||||
),
|
||||
exists: (name) =>
|
||||
fs.access(pathFor(name)).pipe(
|
||||
Effect.as(true),
|
||||
Effect.catch(() => Effect.succeed(false)),
|
||||
),
|
||||
list: () =>
|
||||
walk(directory).pipe(
|
||||
Effect.map((files) =>
|
||||
files
|
||||
.filter((file) => file.endsWith(".json"))
|
||||
.map((file) =>
|
||||
path
|
||||
.relative(directory, file)
|
||||
.replace(/\\/g, "/")
|
||||
.replace(/\.json$/, ""),
|
||||
)
|
||||
.toSorted((a, b) => a.localeCompare(b)),
|
||||
),
|
||||
),
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
export const memory = (initial: Record<string, ReadonlyArray<Interaction>> = {}): Layer.Layer<Service> =>
|
||||
Layer.sync(Service, () => {
|
||||
const stored = new Map<string, Interaction[]>(
|
||||
Object.entries(initial).map(([name, interactions]) => [name, [...interactions]]),
|
||||
)
|
||||
const accumulatedFindings = new Map<string, SecretFinding[]>()
|
||||
const appendLock = Semaphore.makeUnsafe(1)
|
||||
|
||||
return Service.of({
|
||||
read: (name) =>
|
||||
stored.has(name)
|
||||
? Effect.succeed(stored.get(name) ?? [])
|
||||
: Effect.fail(new CassetteNotFoundError({ cassetteName: name })),
|
||||
append: (name, interaction, metadata) =>
|
||||
appendLock.withPermit(
|
||||
Effect.suspend(() => {
|
||||
const interactions = [...(stored.get(name) ?? []), interaction]
|
||||
const findings = [...(accumulatedFindings.get(name) ?? []), ...secretFindings(interaction)]
|
||||
const allFindings = metadata ? [...findings, ...secretFindings({ name, ...metadata })] : findings
|
||||
return failIfUnsafe(name, allFindings).pipe(
|
||||
Effect.tap(() =>
|
||||
Effect.sync(() => {
|
||||
stored.set(name, interactions)
|
||||
accumulatedFindings.set(name, findings)
|
||||
}),
|
||||
),
|
||||
)
|
||||
}),
|
||||
),
|
||||
exists: (name) => Effect.sync(() => stored.has(name)),
|
||||
list: () => Effect.sync(() => Array.from(stored.keys()).toSorted()),
|
||||
})
|
||||
})
|
||||
25
packages/http-recorder/src/effect.ts
Normal file
25
packages/http-recorder/src/effect.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
import { NodeFileSystem } from "@effect/platform-node"
|
||||
import * as Layer from "effect/Layer"
|
||||
import { FetchHttpClient } from "effect/unstable/http"
|
||||
import type * as HttpClient from "effect/unstable/http/HttpClient"
|
||||
import * as CassetteService from "./cassette.js"
|
||||
import { recordingLayer } from "./internal-effect.js"
|
||||
import { make } from "./redactor.js"
|
||||
import type { RecorderOptions } from "./types.js"
|
||||
|
||||
/**
|
||||
* Provides a fetch-backed `HttpClient` with cassette recording and replay.
|
||||
*
|
||||
* Locally, a missing cassette is recorded from the real service. Existing
|
||||
* cassettes are replayed, and `CI=true` makes a missing cassette fail.
|
||||
*/
|
||||
export const http = (name: string, options: RecorderOptions = {}): Layer.Layer<HttpClient.HttpClient> =>
|
||||
recordingLayer(name, {
|
||||
metadata: options.metadata,
|
||||
redactor: make(options.redact),
|
||||
match: options.match,
|
||||
}).pipe(
|
||||
Layer.provide(CassetteService.fileSystem({ directory: options.directory })),
|
||||
Layer.provide(FetchHttpClient.layer),
|
||||
Layer.provide(NodeFileSystem.layer),
|
||||
)
|
||||
18
packages/http-recorder/src/index.ts
Normal file
18
packages/http-recorder/src/index.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
import { http } from "./effect.js"
|
||||
import { socket } from "./socket.js"
|
||||
|
||||
/** HTTP and WebSocket cassette recording. */
|
||||
export const HttpRecorder = { http, socket } as const
|
||||
|
||||
export namespace HttpRecorder {
|
||||
/** Additional JSON metadata stored with a cassette. */
|
||||
export type CassetteMetadata = import("./types.js").CassetteMetadata
|
||||
/** Recorder configuration. */
|
||||
export type RecorderOptions = import("./types.js").RecorderOptions
|
||||
/** Additive redaction and header-preservation policy. */
|
||||
export type RedactOptions = import("./types.js").RedactOptions
|
||||
/** Returns whether an incoming HTTP request matches a recorded request. */
|
||||
export type RequestMatcher = import("./types.js").RequestMatcher
|
||||
/** The normalized HTTP request representation used for matching. */
|
||||
export type RequestSnapshot = import("./types.js").RequestSnapshot
|
||||
}
|
||||
189
packages/http-recorder/src/internal-effect.ts
Normal file
189
packages/http-recorder/src/internal-effect.ts
Normal file
@@ -0,0 +1,189 @@
|
||||
import { NodeFileSystem } from "@effect/platform-node"
|
||||
import { Deferred, Effect, Layer, Option, Ref } from "effect"
|
||||
import {
|
||||
FetchHttpClient,
|
||||
Headers,
|
||||
HttpBody,
|
||||
HttpClient,
|
||||
HttpClientError,
|
||||
HttpClientRequest,
|
||||
HttpClientResponse,
|
||||
UrlParams,
|
||||
} from "effect/unstable/http"
|
||||
import * as CassetteService from "./cassette.js"
|
||||
import { defaultMatcher, selectSequential } from "./matching.js"
|
||||
import { makeReplayState, resolveAutoMode } from "./recorder.js"
|
||||
import { make, type Redactor } from "./redactor.js"
|
||||
import { redactUrl } from "./redaction.js"
|
||||
import { httpInteractions } from "./schema.js"
|
||||
import type { CassetteMetadata, HttpInteraction, RequestMatcher, ResponseSnapshot } from "./types.js"
|
||||
|
||||
export { defaultMatcher }
|
||||
|
||||
export type RecordReplayMode = "auto" | "record" | "replay" | "passthrough"
|
||||
|
||||
export interface RecordReplayOptions {
|
||||
readonly mode?: RecordReplayMode
|
||||
readonly directory?: string
|
||||
readonly metadata?: CassetteMetadata
|
||||
readonly redactor?: Redactor
|
||||
readonly match?: RequestMatcher
|
||||
}
|
||||
|
||||
const TEXT_CONTENT_TYPES = new Set([
|
||||
"application/graphql",
|
||||
"application/javascript",
|
||||
"application/json",
|
||||
"application/sql",
|
||||
"application/x-www-form-urlencoded",
|
||||
"application/xml",
|
||||
"application/yaml",
|
||||
"image/svg+xml",
|
||||
])
|
||||
|
||||
const isTextContentType = (contentType: string | undefined) => {
|
||||
const mediaType = contentType?.split(";", 1)[0]?.trim().toLowerCase()
|
||||
if (!mediaType) return false
|
||||
return (
|
||||
mediaType.startsWith("text/") ||
|
||||
mediaType.endsWith("+json") ||
|
||||
mediaType.endsWith("+xml") ||
|
||||
TEXT_CONTENT_TYPES.has(mediaType)
|
||||
)
|
||||
}
|
||||
|
||||
const captureResponseBody = (response: HttpClientResponse.HttpClientResponse, contentType: string | undefined) =>
|
||||
response.arrayBuffer.pipe(
|
||||
Effect.map((bytes) =>
|
||||
isTextContentType(contentType)
|
||||
? { body: new TextDecoder().decode(bytes) }
|
||||
: { body: Buffer.from(bytes).toString("base64"), bodyEncoding: "base64" as const },
|
||||
),
|
||||
)
|
||||
|
||||
const decodeResponseBody = (snapshot: ResponseSnapshot) =>
|
||||
snapshot.bodyEncoding === "base64" ? Buffer.from(snapshot.body, "base64") : snapshot.body
|
||||
|
||||
const responseFromSnapshot = (request: HttpClientRequest.HttpClientRequest, snapshot: ResponseSnapshot) =>
|
||||
HttpClientResponse.fromWeb(
|
||||
request,
|
||||
new Response(
|
||||
request.method === "HEAD" || snapshot.status === 204 || snapshot.status === 205 || snapshot.status === 304
|
||||
? null
|
||||
: decodeResponseBody(snapshot),
|
||||
snapshot,
|
||||
),
|
||||
)
|
||||
|
||||
export const redactedErrorRequest = (request: HttpClientRequest.HttpClientRequest) =>
|
||||
HttpClientRequest.makeWith(
|
||||
request.method,
|
||||
redactUrl(request.url),
|
||||
UrlParams.empty,
|
||||
Option.none(),
|
||||
Headers.empty,
|
||||
HttpBody.empty,
|
||||
)
|
||||
|
||||
const transportError = (request: HttpClientRequest.HttpClientRequest, description: string) =>
|
||||
new HttpClientError.HttpClientError({
|
||||
reason: new HttpClientError.TransportError({ request: redactedErrorRequest(request), description }),
|
||||
})
|
||||
|
||||
export const recordingLayer = (
|
||||
name: string,
|
||||
options: Omit<RecordReplayOptions, "directory"> = {},
|
||||
): Layer.Layer<HttpClient.HttpClient, never, HttpClient.HttpClient | CassetteService.Service> =>
|
||||
Layer.effect(
|
||||
HttpClient.HttpClient,
|
||||
Effect.gen(function* () {
|
||||
const upstream = yield* HttpClient.HttpClient
|
||||
const cassetteService = yield* CassetteService.Service
|
||||
const redactor = options.redactor ?? make()
|
||||
const match = options.match ?? defaultMatcher
|
||||
const requested = options.mode ?? "auto"
|
||||
const mode = requested === "auto" ? yield* resolveAutoMode(cassetteService, name) : requested
|
||||
|
||||
const snapshotRequest = (request: HttpClientRequest.HttpClientRequest) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(request).pipe(Effect.orDie)
|
||||
return redactor.request({
|
||||
method: web.method,
|
||||
url: web.url,
|
||||
headers: Object.fromEntries(web.headers.entries()),
|
||||
body: yield* Effect.promise(() => web.text()),
|
||||
})
|
||||
})
|
||||
|
||||
if (mode === "passthrough") return upstream
|
||||
|
||||
if (mode === "record") {
|
||||
const initial = yield* Deferred.make<void>()
|
||||
yield* Deferred.succeed(initial, undefined)
|
||||
const tail = yield* Ref.make(initial)
|
||||
return HttpClient.make((request) =>
|
||||
Effect.gen(function* () {
|
||||
const completed = yield* Deferred.make<void>()
|
||||
const previous = yield* Ref.modify(tail, (current) => [current, completed])
|
||||
return yield* Effect.gen(function* () {
|
||||
const incoming = yield* snapshotRequest(request)
|
||||
const response = yield* upstream.execute(request)
|
||||
const captured = yield* captureResponseBody(response, response.headers["content-type"])
|
||||
const responseSnapshot: ResponseSnapshot = {
|
||||
status: response.status,
|
||||
headers: response.headers as Record<string, string>,
|
||||
...captured,
|
||||
}
|
||||
const interaction: HttpInteraction = {
|
||||
transport: "http",
|
||||
request: incoming,
|
||||
response: redactor.response(responseSnapshot),
|
||||
}
|
||||
yield* Deferred.await(previous)
|
||||
yield* cassetteService
|
||||
.append(name, interaction, options.metadata)
|
||||
.pipe(
|
||||
Effect.catchTag("UnsafeCassetteError", (error) =>
|
||||
Effect.fail(transportError(request, error.message)),
|
||||
),
|
||||
)
|
||||
return responseFromSnapshot(request, responseSnapshot)
|
||||
}).pipe(Effect.ensuring(Deferred.succeed(completed, undefined)))
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
const replay = yield* makeReplayState(cassetteService, name, httpInteractions)
|
||||
return HttpClient.make((request) =>
|
||||
Effect.gen(function* () {
|
||||
const incoming = yield* snapshotRequest(request)
|
||||
const claimed = yield* replay
|
||||
.claim((interaction, index, interactions) => {
|
||||
const result = selectSequential(interactions, incoming, match, index)
|
||||
if (result.interaction) return Effect.void
|
||||
return Effect.fail(
|
||||
transportError(request, `Fixture "${name}" does not match the current request: ${result.detail}.`),
|
||||
)
|
||||
})
|
||||
.pipe(
|
||||
Effect.mapError((error) =>
|
||||
error._tag === "CassetteNotFoundError"
|
||||
? transportError(
|
||||
request,
|
||||
`Fixture "${name}" not found. Run locally to record it (CI=true forces replay).`,
|
||||
)
|
||||
: error,
|
||||
),
|
||||
)
|
||||
return responseFromSnapshot(request, claimed.interaction.response)
|
||||
}),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
export const cassetteLayer = (name: string, options: RecordReplayOptions = {}): Layer.Layer<HttpClient.HttpClient> =>
|
||||
recordingLayer(name, options).pipe(
|
||||
Layer.provide(CassetteService.fileSystem({ directory: options.directory })),
|
||||
Layer.provide(FetchHttpClient.layer),
|
||||
Layer.provide(NodeFileSystem.layer),
|
||||
)
|
||||
15
packages/http-recorder/src/internal.ts
Normal file
15
packages/http-recorder/src/internal.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
export { CassetteNotFoundError, hasCassetteSync, UnsafeCassetteError } from "./cassette.js"
|
||||
export { cassetteLayer, recordingLayer, type RecordReplayMode, type RecordReplayOptions } from "./internal-effect.js"
|
||||
export { redactHeaders, redactUrl, secretFindings, type SecretFinding } from "./redaction.js"
|
||||
export { socketLayer } from "./socket.js"
|
||||
export {
|
||||
makeWebSocketExecutor,
|
||||
type WebSocketConnection,
|
||||
type WebSocketExecutor,
|
||||
type WebSocketRecordReplayOptions,
|
||||
type WebSocketRequest,
|
||||
} from "./websocket.js"
|
||||
export * as Cassette from "./cassette.js"
|
||||
export * as Redactor from "./redactor.js"
|
||||
|
||||
export * as HttpRecorderInternal from "./internal.js"
|
||||
106
packages/http-recorder/src/matching.ts
Normal file
106
packages/http-recorder/src/matching.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
import { Option, Schema } from "effect"
|
||||
import { REDACTED, secretFindings } from "./redaction.js"
|
||||
import type { HttpInteraction, RequestMatcher, RequestSnapshot } from "./types.js"
|
||||
|
||||
const JsonValue = Schema.fromJsonString(Schema.Unknown)
|
||||
export const decodeJson = Schema.decodeUnknownOption(JsonValue)
|
||||
|
||||
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
value !== null && typeof value === "object" && !Array.isArray(value)
|
||||
|
||||
export const canonicalizeJson = (value: unknown): unknown => {
|
||||
if (Array.isArray(value)) return value.map(canonicalizeJson)
|
||||
if (isRecord(value)) {
|
||||
return Object.fromEntries(
|
||||
Object.keys(value)
|
||||
.toSorted()
|
||||
.map((key) => [key, canonicalizeJson(value[key])]),
|
||||
)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
export type { RequestMatcher } from "./types.js"
|
||||
|
||||
export const canonicalSnapshot = (snapshot: RequestSnapshot): string =>
|
||||
JSON.stringify({
|
||||
method: snapshot.method,
|
||||
url: snapshot.url,
|
||||
headers: canonicalizeJson(snapshot.headers),
|
||||
body: Option.match(decodeJson(snapshot.body), {
|
||||
onNone: () => snapshot.body,
|
||||
onSome: canonicalizeJson,
|
||||
}),
|
||||
})
|
||||
|
||||
export const defaultMatcher: RequestMatcher = (incoming, recorded) =>
|
||||
canonicalSnapshot(incoming) === canonicalSnapshot(recorded)
|
||||
|
||||
export const safeText = (value: unknown) => {
|
||||
if (value === undefined) return "undefined"
|
||||
if (secretFindings(value).length > 0) return JSON.stringify(REDACTED)
|
||||
const text = JSON.stringify(value)
|
||||
if (!text) return typeof value
|
||||
return text.length > 300 ? `${text.slice(0, 300)}...` : text
|
||||
}
|
||||
|
||||
const jsonBody = (body: string) => Option.getOrUndefined(decodeJson(body))
|
||||
|
||||
const valueDiffs = (expected: unknown, received: unknown, base = "$", limit = 8): ReadonlyArray<string> => {
|
||||
if (Object.is(expected, received)) return []
|
||||
if (isRecord(expected) && isRecord(received)) {
|
||||
return [...new Set([...Object.keys(expected), ...Object.keys(received)])]
|
||||
.toSorted()
|
||||
.flatMap((key) => valueDiffs(expected[key], received[key], `${base}.${key}`, limit))
|
||||
.slice(0, limit)
|
||||
}
|
||||
if (Array.isArray(expected) && Array.isArray(received)) {
|
||||
return Array.from({ length: Math.max(expected.length, received.length) }, (_, index) => index)
|
||||
.flatMap((index) => valueDiffs(expected[index], received[index], `${base}[${index}]`, limit))
|
||||
.slice(0, limit)
|
||||
}
|
||||
return [`${base} expected ${safeText(expected)}, received ${safeText(received)}`]
|
||||
}
|
||||
|
||||
const headerDiffs = (expected: Record<string, string>, received: Record<string, string>) =>
|
||||
[...new Set([...Object.keys(expected), ...Object.keys(received)])].toSorted().flatMap((key) => {
|
||||
if (expected[key] === received[key]) return []
|
||||
if (expected[key] === undefined) return [` ${key} unexpected ${safeText(received[key])}`]
|
||||
if (received[key] === undefined) return [` ${key} missing expected ${safeText(expected[key])}`]
|
||||
return [` ${key} expected ${safeText(expected[key])}, received ${safeText(received[key])}`]
|
||||
})
|
||||
|
||||
export const requestDiff = (expected: RequestSnapshot, received: RequestSnapshot): ReadonlyArray<string> => {
|
||||
const lines: string[] = []
|
||||
if (expected.method !== received.method) {
|
||||
lines.push("method:", ` expected ${expected.method}, received ${received.method}`)
|
||||
}
|
||||
if (expected.url !== received.url) {
|
||||
lines.push("url:", ` expected ${expected.url}`, ` received ${received.url}`)
|
||||
}
|
||||
const headers = headerDiffs(expected.headers, received.headers)
|
||||
if (headers.length > 0) lines.push("headers:", ...headers.slice(0, 8))
|
||||
const expectedBody = jsonBody(expected.body)
|
||||
const receivedBody = jsonBody(received.body)
|
||||
const body =
|
||||
expectedBody !== undefined && receivedBody !== undefined
|
||||
? valueDiffs(expectedBody, receivedBody).map((line) => ` ${line}`)
|
||||
: expected.body === received.body
|
||||
? []
|
||||
: [` expected ${safeText(expected.body)}, received ${safeText(received.body)}`]
|
||||
if (body.length > 0) lines.push("body:", ...body)
|
||||
return lines
|
||||
}
|
||||
|
||||
export const selectSequential = (
|
||||
interactions: ReadonlyArray<HttpInteraction>,
|
||||
incoming: RequestSnapshot,
|
||||
match: RequestMatcher,
|
||||
index: number,
|
||||
): { readonly interaction: HttpInteraction | undefined; readonly detail: string } => {
|
||||
const interaction = interactions[index]
|
||||
if (!interaction) return { interaction, detail: `interaction ${index + 1} of ${interactions.length} not recorded` }
|
||||
if (!match(incoming, interaction.request))
|
||||
return { interaction: undefined, detail: requestDiff(interaction.request, incoming).join("\n") }
|
||||
return { interaction, detail: "" }
|
||||
}
|
||||
62
packages/http-recorder/src/recorder.ts
Normal file
62
packages/http-recorder/src/recorder.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
import { Effect, Scope, SynchronizedRef } from "effect"
|
||||
import type * as CassetteService from "./cassette.js"
|
||||
import type { CassetteNotFoundError } from "./cassette.js"
|
||||
import type { Interaction } from "./schema.js"
|
||||
|
||||
const isCI = () => {
|
||||
const value = process.env.CI
|
||||
return value !== undefined && value !== "" && value !== "false" && value !== "0"
|
||||
}
|
||||
|
||||
export const resolveAutoMode = (
|
||||
cassette: CassetteService.Interface,
|
||||
name: string,
|
||||
): Effect.Effect<"record" | "replay" | "passthrough"> =>
|
||||
Effect.gen(function* () {
|
||||
if (isCI()) return "replay"
|
||||
return (yield* cassette.exists(name)) ? "replay" : "record"
|
||||
})
|
||||
|
||||
export interface ReplayState<T> {
|
||||
readonly claim: <E>(
|
||||
validate: (interaction: T | undefined, index: number, interactions: ReadonlyArray<T>) => Effect.Effect<void, E>,
|
||||
) => Effect.Effect<{ readonly interaction: T; readonly index: number }, CassetteNotFoundError | E>
|
||||
}
|
||||
|
||||
export const makeReplayState = <T>(
|
||||
cassette: CassetteService.Interface,
|
||||
name: string,
|
||||
project: (interactions: ReadonlyArray<Interaction>) => ReadonlyArray<T>,
|
||||
): Effect.Effect<ReplayState<T>, never, Scope.Scope> =>
|
||||
Effect.gen(function* () {
|
||||
const load = yield* Effect.cached(cassette.read(name).pipe(Effect.map(project)))
|
||||
const position = yield* SynchronizedRef.make(0)
|
||||
|
||||
yield* Effect.addFinalizer(() =>
|
||||
Effect.gen(function* () {
|
||||
const used = yield* SynchronizedRef.get(position)
|
||||
if (used === 0) return yield* Effect.void
|
||||
const interactions = yield* load.pipe(Effect.orDie)
|
||||
if (used < interactions.length)
|
||||
return yield* Effect.die(
|
||||
new Error(`Unused recorded interactions in ${name}: used ${used} of ${interactions.length}`),
|
||||
)
|
||||
return yield* Effect.void
|
||||
}),
|
||||
)
|
||||
|
||||
return {
|
||||
claim: (validate) =>
|
||||
Effect.flatMap(load, (interactions) =>
|
||||
SynchronizedRef.modifyEffect(position, (index) =>
|
||||
Effect.gen(function* () {
|
||||
const interaction = interactions[index]
|
||||
yield* validate(interaction, index, interactions)
|
||||
if (interaction === undefined)
|
||||
return yield* Effect.die("Replay validation accepted a missing interaction")
|
||||
return [{ interaction, index }, index + 1] as const
|
||||
}),
|
||||
),
|
||||
),
|
||||
}
|
||||
})
|
||||
117
packages/http-recorder/src/redaction.ts
Normal file
117
packages/http-recorder/src/redaction.ts
Normal file
@@ -0,0 +1,117 @@
|
||||
import { Schema } from "effect"
|
||||
|
||||
export const REDACTED = "[REDACTED]"
|
||||
|
||||
const DEFAULT_REDACT_HEADERS = [
|
||||
"authorization",
|
||||
"cookie",
|
||||
"proxy-authorization",
|
||||
"set-cookie",
|
||||
"x-api-key",
|
||||
"x-amz-security-token",
|
||||
"x-goog-api-key",
|
||||
]
|
||||
|
||||
const DEFAULT_REDACT_QUERY = [
|
||||
"access_token",
|
||||
"api-key",
|
||||
"api_key",
|
||||
"apikey",
|
||||
"code",
|
||||
"key",
|
||||
"signature",
|
||||
"sig",
|
||||
"token",
|
||||
"x-amz-credential",
|
||||
"x-amz-security-token",
|
||||
"x-amz-signature",
|
||||
]
|
||||
|
||||
const SECRET_PATTERNS: ReadonlyArray<{ readonly label: string; readonly pattern: RegExp }> = [
|
||||
{ label: "bearer token", pattern: /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}\b/i },
|
||||
{ label: "API key", pattern: /\bsk-[A-Za-z0-9][A-Za-z0-9_-]{20,}\b/ },
|
||||
{ label: "Anthropic API key", pattern: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/ },
|
||||
{ label: "Google API key", pattern: /\bAIza[0-9A-Za-z_-]{20,}\b/ },
|
||||
{ label: "AWS access key", pattern: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/ },
|
||||
{ label: "GitHub token", pattern: /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/ },
|
||||
{ label: "private key", pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ },
|
||||
]
|
||||
|
||||
const ENV_SECRET_NAMES = /(?:API|AUTH|BEARER|CREDENTIAL|KEY|PASSWORD|SECRET|TOKEN)/i
|
||||
const SAFE_ENV_VALUES = new Set(["fixture", "test", "test-key"])
|
||||
|
||||
const envSecrets = () =>
|
||||
Object.entries(process.env).flatMap(([name, value]) => {
|
||||
if (!value) return []
|
||||
if (!ENV_SECRET_NAMES.test(name)) return []
|
||||
if (value.length < 12) return []
|
||||
if (SAFE_ENV_VALUES.has(value.toLowerCase())) return []
|
||||
return [{ name, value }]
|
||||
})
|
||||
|
||||
const pathFor = (base: string, key: string) => (base ? `${base}.${key}` : key)
|
||||
|
||||
const stringEntries = (value: unknown, base = ""): ReadonlyArray<{ readonly path: string; readonly value: string }> => {
|
||||
if (typeof value === "string") return [{ path: base, value }]
|
||||
if (Array.isArray(value)) return value.flatMap((item, index) => stringEntries(item, `${base}[${index}]`))
|
||||
if (value && typeof value === "object") {
|
||||
return Object.entries(value).flatMap(([key, child]) => stringEntries(child, pathFor(base, key)))
|
||||
}
|
||||
return []
|
||||
}
|
||||
|
||||
const redactionSet = (values: ReadonlyArray<string> | undefined, defaults: ReadonlyArray<string>) =>
|
||||
new Set([...defaults, ...(values ?? [])].map((value) => value.toLowerCase()))
|
||||
|
||||
export type UrlRedactor = (url: string) => string
|
||||
|
||||
export const redactUrl = (
|
||||
raw: string,
|
||||
query: ReadonlyArray<string> = DEFAULT_REDACT_QUERY,
|
||||
urlRedactor?: UrlRedactor,
|
||||
) => {
|
||||
if (!URL.canParse(raw)) return urlRedactor?.(raw) ?? raw
|
||||
const url = new URL(raw)
|
||||
if (url.username) url.username = REDACTED
|
||||
if (url.password) url.password = REDACTED
|
||||
const redacted = redactionSet(query, DEFAULT_REDACT_QUERY)
|
||||
for (const key of url.searchParams.keys()) {
|
||||
if (redacted.has(key.toLowerCase())) url.searchParams.set(key, REDACTED)
|
||||
}
|
||||
return urlRedactor?.(url.toString()) ?? url.toString()
|
||||
}
|
||||
|
||||
export const redactHeaders = (
|
||||
headers: Record<string, string>,
|
||||
allow: ReadonlyArray<string>,
|
||||
redact: ReadonlyArray<string> = DEFAULT_REDACT_HEADERS,
|
||||
) => {
|
||||
const allowed = new Set(allow.map((name) => name.toLowerCase()))
|
||||
const redacted = redactionSet(redact, DEFAULT_REDACT_HEADERS)
|
||||
return Object.fromEntries(
|
||||
Object.entries(headers)
|
||||
.map(([name, value]) => [name.toLowerCase(), value] as const)
|
||||
.filter(([name]) => allowed.has(name))
|
||||
.map(([name, value]) => [name, redacted.has(name) ? REDACTED : value] as const)
|
||||
.toSorted(([a], [b]) => a.localeCompare(b)),
|
||||
)
|
||||
}
|
||||
|
||||
export const SecretFindingSchema = Schema.Struct({
|
||||
path: Schema.String,
|
||||
reason: Schema.String,
|
||||
})
|
||||
export type SecretFinding = Schema.Schema.Type<typeof SecretFindingSchema>
|
||||
|
||||
export const secretFindings = (value: unknown): ReadonlyArray<SecretFinding> => {
|
||||
const environment = envSecrets()
|
||||
return stringEntries(value).flatMap((entry) => [
|
||||
...SECRET_PATTERNS.filter((item) => item.pattern.test(entry.value)).map((item) => ({
|
||||
path: entry.path,
|
||||
reason: item.label,
|
||||
})),
|
||||
...environment
|
||||
.filter((item) => entry.value.includes(item.value))
|
||||
.map((item) => ({ path: entry.path, reason: `environment secret ${item.name}` })),
|
||||
])
|
||||
}
|
||||
135
packages/http-recorder/src/redactor.ts
Normal file
135
packages/http-recorder/src/redactor.ts
Normal file
@@ -0,0 +1,135 @@
|
||||
import { Option } from "effect"
|
||||
import { decodeJson } from "./matching.js"
|
||||
import { REDACTED, redactHeaders, redactUrl } from "./redaction.js"
|
||||
import type { RedactOptions, RequestSnapshot, ResponseSnapshot } from "./types.js"
|
||||
|
||||
export type { RedactOptions } from "./types.js"
|
||||
|
||||
export const DEFAULT_REQUEST_HEADERS: ReadonlyArray<string> = ["content-type", "accept", "openai-beta"]
|
||||
export const DEFAULT_RESPONSE_HEADERS: ReadonlyArray<string> = ["content-type"]
|
||||
|
||||
const identity = <T>(value: T) => value
|
||||
|
||||
export interface Redactor {
|
||||
readonly request: (snapshot: RequestSnapshot) => RequestSnapshot
|
||||
readonly response: (snapshot: ResponseSnapshot) => ResponseSnapshot
|
||||
}
|
||||
|
||||
export const compose = (...redactors: ReadonlyArray<Partial<Redactor>>): Redactor => {
|
||||
const requests = redactors.map((r) => r.request).filter((fn): fn is Redactor["request"] => fn !== undefined)
|
||||
const responses = redactors.map((r) => r.response).filter((fn): fn is Redactor["response"] => fn !== undefined)
|
||||
return {
|
||||
request: requests.length === 0 ? identity : (snapshot) => requests.reduce((acc, fn) => fn(acc), snapshot),
|
||||
response: responses.length === 0 ? identity : (snapshot) => responses.reduce((acc, fn) => fn(acc), snapshot),
|
||||
}
|
||||
}
|
||||
|
||||
export interface HeaderOptions {
|
||||
readonly allow?: ReadonlyArray<string>
|
||||
readonly redact?: ReadonlyArray<string>
|
||||
}
|
||||
|
||||
export const requestHeaders = (options: HeaderOptions = {}): Partial<Redactor> => ({
|
||||
request: (snapshot) => ({
|
||||
...snapshot,
|
||||
headers: redactHeaders(snapshot.headers, options.allow ?? DEFAULT_REQUEST_HEADERS, options.redact),
|
||||
}),
|
||||
})
|
||||
|
||||
export const responseHeaders = (options: HeaderOptions = {}): Partial<Redactor> => ({
|
||||
response: (snapshot) => ({
|
||||
...snapshot,
|
||||
headers: redactHeaders(snapshot.headers, options.allow ?? DEFAULT_RESPONSE_HEADERS, options.redact),
|
||||
}),
|
||||
})
|
||||
|
||||
export interface UrlOptions {
|
||||
readonly query?: ReadonlyArray<string>
|
||||
readonly transform?: (url: string) => string
|
||||
}
|
||||
|
||||
export const url = (options: UrlOptions = {}): Partial<Redactor> => ({
|
||||
request: (snapshot) => ({ ...snapshot, url: redactUrl(snapshot.url, options.query, options.transform) }),
|
||||
})
|
||||
|
||||
export const body = (transform: (parsed: unknown) => unknown): Partial<Redactor> => ({
|
||||
request: (snapshot) => ({
|
||||
...snapshot,
|
||||
body: Option.match(decodeJson(snapshot.body), {
|
||||
onNone: () => snapshot.body,
|
||||
onSome: (parsed) => JSON.stringify(transform(parsed)),
|
||||
}),
|
||||
}),
|
||||
})
|
||||
|
||||
export interface DefaultRedactorOverrides {
|
||||
readonly requestHeaders?: HeaderOptions
|
||||
readonly responseHeaders?: HeaderOptions
|
||||
readonly url?: UrlOptions
|
||||
readonly body?: (parsed: unknown) => unknown
|
||||
}
|
||||
|
||||
const DEFAULT_REDACT_JSON_FIELDS = [
|
||||
"access_token",
|
||||
"api_key",
|
||||
"apikey",
|
||||
"client_secret",
|
||||
"password",
|
||||
"refresh_token",
|
||||
"secret",
|
||||
"token",
|
||||
]
|
||||
|
||||
const normalizeField = (field: string) => field.replace(/[^a-z0-9]/gi, "").toLowerCase()
|
||||
|
||||
const redactJsonFields = (value: unknown, fields: ReadonlySet<string>): unknown => {
|
||||
if (Array.isArray(value)) return value.map((item) => redactJsonFields(item, fields))
|
||||
if (!value || typeof value !== "object") return value
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, child]) => [
|
||||
key,
|
||||
fields.has(normalizeField(key)) ? REDACTED : redactJsonFields(child, fields),
|
||||
]),
|
||||
)
|
||||
}
|
||||
|
||||
const redactBody = (value: string, fields: ReadonlySet<string>, transform: ((body: string) => string) | undefined) => {
|
||||
const redacted = Option.match(decodeJson(value), {
|
||||
onNone: () => value,
|
||||
onSome: (parsed) => JSON.stringify(redactJsonFields(parsed, fields)),
|
||||
})
|
||||
return transform?.(redacted) ?? redacted
|
||||
}
|
||||
|
||||
export const make = (options: RedactOptions = {}): Redactor => {
|
||||
const fields = new Set([...DEFAULT_REDACT_JSON_FIELDS, ...(options.jsonFields ?? [])].map(normalizeField))
|
||||
return compose(
|
||||
requestHeaders({
|
||||
allow: [...DEFAULT_REQUEST_HEADERS, ...(options.allowRequestHeaders ?? []), ...(options.headers ?? [])],
|
||||
redact: options.headers,
|
||||
}),
|
||||
responseHeaders({
|
||||
allow: [...DEFAULT_RESPONSE_HEADERS, ...(options.allowResponseHeaders ?? []), ...(options.headers ?? [])],
|
||||
redact: options.headers,
|
||||
}),
|
||||
url({ query: options.queryParameters, transform: options.url }),
|
||||
{
|
||||
request: (snapshot) => ({
|
||||
...snapshot,
|
||||
body: redactBody(snapshot.body, fields, options.body),
|
||||
}),
|
||||
response: (snapshot) => ({
|
||||
...snapshot,
|
||||
body: redactBody(snapshot.body, fields, options.body),
|
||||
}),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
export const defaults = (overrides: DefaultRedactorOverrides = {}): Redactor =>
|
||||
compose(
|
||||
requestHeaders(overrides.requestHeaders),
|
||||
responseHeaders(overrides.responseHeaders),
|
||||
url(overrides.url),
|
||||
...(overrides.body ? [body(overrides.body)] : []),
|
||||
)
|
||||
87
packages/http-recorder/src/schema.ts
Normal file
87
packages/http-recorder/src/schema.ts
Normal file
@@ -0,0 +1,87 @@
|
||||
import { Schema } from "effect"
|
||||
import type {
|
||||
CassetteMetadata,
|
||||
HttpInteraction,
|
||||
RequestSnapshot,
|
||||
ResponseSnapshot,
|
||||
WebSocketEvent,
|
||||
WebSocketInteraction,
|
||||
} from "./types.js"
|
||||
|
||||
export type {
|
||||
CassetteMetadata,
|
||||
HttpInteraction,
|
||||
RequestSnapshot,
|
||||
ResponseSnapshot,
|
||||
WebSocketEvent,
|
||||
WebSocketInteraction,
|
||||
} from "./types.js"
|
||||
|
||||
export const RequestSnapshotSchema = Schema.Struct({
|
||||
method: Schema.String,
|
||||
url: Schema.String,
|
||||
headers: Schema.Record(Schema.String, Schema.String),
|
||||
body: Schema.String,
|
||||
})
|
||||
|
||||
export const ResponseSnapshotSchema = Schema.Struct({
|
||||
status: Schema.Number,
|
||||
headers: Schema.Record(Schema.String, Schema.String),
|
||||
body: Schema.String,
|
||||
bodyEncoding: Schema.optional(Schema.Literals(["text", "base64"])),
|
||||
})
|
||||
|
||||
export const CassetteMetadataSchema = Schema.Record(Schema.String, Schema.Unknown)
|
||||
|
||||
export const HttpInteractionSchema = Schema.Struct({
|
||||
transport: Schema.tag("http"),
|
||||
request: RequestSnapshotSchema,
|
||||
response: ResponseSnapshotSchema,
|
||||
})
|
||||
|
||||
export const WebSocketEventSchema = Schema.Union([
|
||||
Schema.Struct({
|
||||
direction: Schema.Literals(["client", "server"]),
|
||||
kind: Schema.tag("text"),
|
||||
body: Schema.String,
|
||||
}),
|
||||
Schema.Struct({
|
||||
direction: Schema.Literals(["client", "server"]),
|
||||
kind: Schema.tag("binary"),
|
||||
body: Schema.String,
|
||||
bodyEncoding: Schema.Literal("base64"),
|
||||
}),
|
||||
])
|
||||
|
||||
export const WebSocketInteractionSchema = Schema.Struct({
|
||||
transport: Schema.tag("websocket"),
|
||||
open: Schema.Struct({
|
||||
url: Schema.String,
|
||||
headers: Schema.Record(Schema.String, Schema.String),
|
||||
}),
|
||||
events: Schema.Array(WebSocketEventSchema),
|
||||
})
|
||||
|
||||
export const InteractionSchema = Schema.Union([HttpInteractionSchema, WebSocketInteractionSchema]).pipe(
|
||||
Schema.toTaggedUnion("transport"),
|
||||
)
|
||||
export type Interaction = Schema.Schema.Type<typeof InteractionSchema>
|
||||
|
||||
export const isHttpInteraction = InteractionSchema.guards.http
|
||||
|
||||
export const isWebSocketInteraction = InteractionSchema.guards.websocket
|
||||
|
||||
export const httpInteractions = (interactions: ReadonlyArray<Interaction>) => interactions.filter(isHttpInteraction)
|
||||
|
||||
export const webSocketInteractions = (interactions: ReadonlyArray<Interaction>) =>
|
||||
interactions.filter(isWebSocketInteraction)
|
||||
|
||||
export const CassetteSchema = Schema.Struct({
|
||||
version: Schema.Literal(1),
|
||||
metadata: Schema.optional(CassetteMetadataSchema),
|
||||
interactions: Schema.Array(InteractionSchema),
|
||||
})
|
||||
export type Cassette = Schema.Schema.Type<typeof CassetteSchema>
|
||||
|
||||
export const decodeCassette = Schema.decodeUnknownSync(CassetteSchema)
|
||||
export const encodeCassette = Schema.encodeSync(CassetteSchema)
|
||||
326
packages/http-recorder/src/socket.ts
Normal file
326
packages/http-recorder/src/socket.ts
Normal file
@@ -0,0 +1,326 @@
|
||||
import { NodeFileSystem } from "@effect/platform-node"
|
||||
import { Deferred, Effect, Exit, FiberSet, Layer, Ref, Scope, Semaphore } from "effect"
|
||||
import { Socket } from "effect/unstable/socket"
|
||||
import * as CassetteService from "./cassette.js"
|
||||
import { canonicalizeJson, decodeJson, safeText } from "./matching.js"
|
||||
import { makeReplayState, resolveAutoMode } from "./recorder.js"
|
||||
import { make, type Redactor } from "./redactor.js"
|
||||
import { webSocketInteractions } from "./schema.js"
|
||||
import type {
|
||||
RecorderOptions,
|
||||
WebSocketEvent,
|
||||
WebSocketInteraction,
|
||||
WebSocketRecorderOptions,
|
||||
WebSocketRequest,
|
||||
} from "./types.js"
|
||||
|
||||
interface ActiveReplay {
|
||||
readonly interaction: WebSocketInteraction
|
||||
readonly progress: Ref.Ref<{ readonly position: number; readonly changed: Deferred.Deferred<void> }>
|
||||
readonly writeLock: Semaphore.Semaphore
|
||||
readonly closed: Ref.Ref<boolean>
|
||||
}
|
||||
|
||||
interface ActiveRecording {
|
||||
readonly events: Array<WebSocketEvent>
|
||||
readonly eventLock: Semaphore.Semaphore
|
||||
readonly accepting: Ref.Ref<boolean>
|
||||
opened: boolean
|
||||
valid: boolean
|
||||
}
|
||||
|
||||
type Frame = string | Uint8Array
|
||||
|
||||
const encodeEvent = (direction: "client" | "server", message: Frame): WebSocketEvent =>
|
||||
typeof message === "string"
|
||||
? { direction, kind: "text", body: message }
|
||||
: { direction, kind: "binary", body: Buffer.from(message).toString("base64"), bodyEncoding: "base64" }
|
||||
|
||||
const decodeEvent = (event: WebSocketEvent): Frame =>
|
||||
event.kind === "text" ? event.body : new Uint8Array(Buffer.from(event.body, "base64"))
|
||||
|
||||
const redactEvent = (event: WebSocketEvent, redactor: Redactor): WebSocketEvent => {
|
||||
if (event.kind === "binary") return event
|
||||
const body =
|
||||
event.direction === "client"
|
||||
? redactor.request({ method: "WEBSOCKET", url: "", headers: {}, body: event.body }).body
|
||||
: redactor.response({ status: 101, headers: {}, body: event.body }).body
|
||||
return { ...event, body }
|
||||
}
|
||||
|
||||
const comparable = (event: WebSocketEvent, asJson: boolean) => {
|
||||
if (!asJson || event.kind === "binary") return JSON.stringify(canonicalizeJson(event))
|
||||
const decoded = decodeJson(event.body)
|
||||
return JSON.stringify(
|
||||
canonicalizeJson({
|
||||
...event,
|
||||
body: decoded._tag === "None" ? event.body : canonicalizeJson(decoded.value),
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
const assertEvent = (actual: WebSocketEvent, expected: WebSocketEvent | undefined, index: number, asJson: boolean) =>
|
||||
Effect.sync(() => {
|
||||
if (expected && comparable(actual, asJson) === comparable(expected, asJson)) return
|
||||
throw new Error(`WebSocket event ${index + 1}: expected ${safeText(expected)}, received ${safeText(actual)}`)
|
||||
})
|
||||
|
||||
const runHandler = <A, E, R>(handler: (value: A) => Effect.Effect<unknown, E, R> | void, value: A) =>
|
||||
Effect.suspend(() => {
|
||||
const result = handler(value)
|
||||
return Effect.isEffect(result) ? Effect.asVoid(result) : Effect.void
|
||||
})
|
||||
|
||||
const runReplay = <A, E, R>(
|
||||
state: ActiveReplay,
|
||||
handler: (value: A) => Effect.Effect<unknown, E, R> | void,
|
||||
decode: (event: WebSocketEvent) => A,
|
||||
onOpen: Effect.Effect<void> | undefined,
|
||||
) =>
|
||||
Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const handlers = yield* FiberSet.make<unknown, E>()
|
||||
const run = yield* FiberSet.runtime(handlers)<R>()
|
||||
if (onOpen) yield* onOpen
|
||||
|
||||
const drive = Effect.gen(function* () {
|
||||
while (true) {
|
||||
const current = yield* Ref.get(state.progress)
|
||||
const event = state.interaction.events[current.position]
|
||||
if (!event) return
|
||||
if (yield* Ref.get(state.closed))
|
||||
return yield* Effect.die(
|
||||
new Error(
|
||||
`WebSocket closed with unconsumed events: used ${current.position} of ${state.interaction.events.length}`,
|
||||
),
|
||||
)
|
||||
if (event.direction === "server") {
|
||||
yield* Ref.set(state.progress, {
|
||||
position: current.position + 1,
|
||||
changed: yield* Deferred.make<void>(),
|
||||
})
|
||||
run(runHandler(handler, decode(event)))
|
||||
continue
|
||||
}
|
||||
yield* Deferred.await(current.changed)
|
||||
}
|
||||
})
|
||||
|
||||
yield* drive.pipe(Effect.raceFirst(FiberSet.join(handlers)))
|
||||
yield* FiberSet.awaitEmpty(handlers).pipe(Effect.raceFirst(FiberSet.join(handlers)))
|
||||
}),
|
||||
)
|
||||
|
||||
const openSnapshot = (request: WebSocketRequest, redactor: Redactor) => {
|
||||
const snapshot = redactor.request({ method: "GET", url: request.url, headers: request.headers ?? {}, body: "" })
|
||||
return { url: snapshot.url, headers: snapshot.headers }
|
||||
}
|
||||
|
||||
const makeRecordingSocket = (
|
||||
upstream: Socket.Socket,
|
||||
cassette: CassetteService.Interface,
|
||||
name: string,
|
||||
request: WebSocketRequest,
|
||||
options: WebSocketRecorderOptions,
|
||||
redactor: Redactor,
|
||||
) =>
|
||||
Effect.gen(function* () {
|
||||
const active = yield* Ref.make<ActiveRecording | undefined>(undefined)
|
||||
const writeLock = yield* Semaphore.make(1)
|
||||
|
||||
return Socket.make({
|
||||
runRaw: (handler, runOptions) =>
|
||||
Effect.gen(function* () {
|
||||
const state: ActiveRecording = {
|
||||
events: [],
|
||||
eventLock: yield* Semaphore.make(1),
|
||||
accepting: yield* Ref.make(true),
|
||||
opened: false,
|
||||
valid: true,
|
||||
}
|
||||
const occupied = yield* Ref.modify(active, (current) => [current !== undefined, current ?? state])
|
||||
if (occupied) return yield* Effect.die("Concurrent runs of a recorded WebSocket are not supported")
|
||||
yield* upstream
|
||||
.runRaw(
|
||||
(message) => {
|
||||
if (!Ref.getUnsafe(state.accepting)) throw new Error("WebSocket received a frame after closing")
|
||||
state.events.push(redactEvent(encodeEvent("server", message), redactor))
|
||||
return handler(message)
|
||||
},
|
||||
{
|
||||
...runOptions,
|
||||
onOpen: Effect.gen(function* () {
|
||||
state.opened = true
|
||||
if (runOptions?.onOpen) yield* runOptions.onOpen
|
||||
}),
|
||||
},
|
||||
)
|
||||
.pipe(
|
||||
Effect.onExit((exit) =>
|
||||
writeLock.withPermit(
|
||||
state.eventLock.withPermit(
|
||||
Effect.gen(function* () {
|
||||
yield* Ref.set(state.accepting, false)
|
||||
yield* Ref.set(active, undefined)
|
||||
if (!Exit.isSuccess(exit) || !state.opened || !state.valid) return
|
||||
yield* cassette
|
||||
.append(
|
||||
name,
|
||||
{
|
||||
transport: "websocket",
|
||||
open: openSnapshot(request, redactor),
|
||||
events: [...state.events],
|
||||
},
|
||||
options.metadata,
|
||||
)
|
||||
.pipe(Effect.orDie)
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
}),
|
||||
writer: upstream.writer.pipe(
|
||||
Effect.map(
|
||||
(write) => (message) =>
|
||||
writeLock.withPermit(
|
||||
Effect.gen(function* () {
|
||||
if (Socket.isCloseEvent(message)) return yield* write(message)
|
||||
const state = yield* Ref.get(active)
|
||||
if (!state || !(yield* Ref.get(state.accepting)))
|
||||
return yield* Effect.die("WebSocket writer used without an active socket run")
|
||||
const event = redactEvent(encodeEvent("client", message), redactor)
|
||||
yield* state.eventLock.withPermit(Effect.sync(() => state.events.push(event)))
|
||||
return yield* write(message).pipe(Effect.onError(() => Effect.sync(() => (state.valid = false))))
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
})
|
||||
})
|
||||
|
||||
const makeReplaySocket = (
|
||||
cassette: CassetteService.Interface,
|
||||
name: string,
|
||||
request: WebSocketRequest,
|
||||
options: WebSocketRecorderOptions,
|
||||
redactor: Redactor,
|
||||
): Effect.Effect<Socket.Socket, never, Scope.Scope> =>
|
||||
Effect.gen(function* () {
|
||||
const replay = yield* makeReplayState(cassette, name, webSocketInteractions)
|
||||
const active = yield* Ref.make<ActiveReplay | undefined>(undefined)
|
||||
|
||||
return Socket.make({
|
||||
runRaw: (handler, runOptions) =>
|
||||
Effect.gen(function* () {
|
||||
const claimed = yield* replay
|
||||
.claim((interaction, index) =>
|
||||
Effect.sync(() => {
|
||||
const incoming = openSnapshot(request, redactor)
|
||||
if (
|
||||
interaction &&
|
||||
JSON.stringify(canonicalizeJson(incoming)) === JSON.stringify(canonicalizeJson(interaction.open))
|
||||
)
|
||||
return
|
||||
throw new Error(
|
||||
`WebSocket open ${index + 1}: expected ${safeText(interaction?.open)}, received ${safeText(incoming)}`,
|
||||
)
|
||||
}),
|
||||
)
|
||||
.pipe(Effect.orDie)
|
||||
const progress = yield* Ref.make({ position: 0, changed: yield* Deferred.make<void>() })
|
||||
const writeLock = yield* Semaphore.make(1)
|
||||
const state = {
|
||||
interaction: claimed.interaction,
|
||||
progress,
|
||||
writeLock,
|
||||
closed: yield* Ref.make(false),
|
||||
}
|
||||
const occupied = yield* Ref.modify(active, (current) => [current !== undefined, current ?? state])
|
||||
if (occupied) return yield* Effect.die("Concurrent runs of a replayed WebSocket are not supported")
|
||||
yield* runReplay(state, handler, decodeEvent, runOptions?.onOpen).pipe(
|
||||
Effect.ensuring(Ref.set(active, undefined)),
|
||||
)
|
||||
}),
|
||||
writer: Effect.succeed((message) => {
|
||||
return Ref.get(active).pipe(
|
||||
Effect.flatMap((state) =>
|
||||
state
|
||||
? state.writeLock.withPermit(
|
||||
Effect.gen(function* () {
|
||||
const current = yield* Ref.get(state.progress)
|
||||
if (Socket.isCloseEvent(message)) {
|
||||
yield* Ref.set(state.closed, true)
|
||||
yield* Deferred.succeed(current.changed, undefined)
|
||||
if (current.position === state.interaction.events.length) return
|
||||
return yield* Effect.die(
|
||||
new Error(
|
||||
`WebSocket closed with unconsumed events: used ${current.position} of ${state.interaction.events.length}`,
|
||||
),
|
||||
)
|
||||
}
|
||||
const actual = redactEvent(encodeEvent("client", message), redactor)
|
||||
yield* assertEvent(
|
||||
actual,
|
||||
state.interaction.events[current.position],
|
||||
current.position,
|
||||
options.compareClientMessagesAsJson === true,
|
||||
)
|
||||
yield* Ref.set(state.progress, {
|
||||
position: current.position + 1,
|
||||
changed: yield* Deferred.make<void>(),
|
||||
})
|
||||
yield* Deferred.succeed(current.changed, undefined)
|
||||
}),
|
||||
)
|
||||
: Effect.die("WebSocket writer used without an active socket run"),
|
||||
),
|
||||
)
|
||||
}),
|
||||
})
|
||||
})
|
||||
|
||||
const recordingLayer = (
|
||||
name: string,
|
||||
request: WebSocketRequest,
|
||||
options: WebSocketRecorderOptions,
|
||||
forcedMode?: "record" | "replay",
|
||||
): Layer.Layer<Socket.Socket, never, Socket.Socket | CassetteService.Service> =>
|
||||
Layer.effect(
|
||||
Socket.Socket,
|
||||
Effect.gen(function* () {
|
||||
const upstream = yield* Socket.Socket
|
||||
const cassette = yield* CassetteService.Service
|
||||
const redactor = make(options.redact)
|
||||
if ((forcedMode ?? (yield* resolveAutoMode(cassette, name))) === "record")
|
||||
return yield* makeRecordingSocket(upstream, cassette, name, request, options, redactor)
|
||||
return yield* makeReplaySocket(cassette, name, request, options, redactor)
|
||||
}),
|
||||
)
|
||||
|
||||
/**
|
||||
* Wraps a provided `Socket.Socket` with cassette recording and replay.
|
||||
*
|
||||
* Supply the ordinary URL-bound Effect socket layer beneath this decorator.
|
||||
* The cassette name identifies the connection; recorder configuration does not
|
||||
* duplicate the transport URL.
|
||||
*/
|
||||
export const socket = (name: string, options: RecorderOptions = {}): Layer.Layer<Socket.Socket, never, Socket.Socket> =>
|
||||
provideCassette(recordingLayer(name, { url: "" }, { ...options, compareClientMessagesAsJson: true }), options)
|
||||
|
||||
/** @internal */
|
||||
export const socketLayer = (
|
||||
name: string,
|
||||
request: WebSocketRequest,
|
||||
options: WebSocketRecorderOptions & { readonly mode: "record" | "replay" },
|
||||
): Layer.Layer<Socket.Socket, never, Socket.Socket> =>
|
||||
provideCassette(recordingLayer(name, request, options, options.mode), options)
|
||||
|
||||
const provideCassette = (
|
||||
layer: Layer.Layer<Socket.Socket, never, Socket.Socket | CassetteService.Service>,
|
||||
options: WebSocketRecorderOptions,
|
||||
) =>
|
||||
layer.pipe(
|
||||
Layer.provide(CassetteService.fileSystem({ directory: options.directory })),
|
||||
Layer.provide(NodeFileSystem.layer),
|
||||
)
|
||||
108
packages/http-recorder/src/types.ts
Normal file
108
packages/http-recorder/src/types.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
/** Additional JSON metadata stored with a cassette. */
|
||||
export type CassetteMetadata = Record<string, unknown>
|
||||
|
||||
/** The normalized HTTP request representation used for matching. */
|
||||
export interface RequestSnapshot {
|
||||
/** HTTP method. */
|
||||
readonly method: string
|
||||
/** Fully qualified URL after redaction. */
|
||||
readonly url: string
|
||||
/** Allowed and redacted request headers. */
|
||||
readonly headers: Record<string, string>
|
||||
/** Request body after redaction. */
|
||||
readonly body: string
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface ResponseSnapshot {
|
||||
/** HTTP status code. */
|
||||
readonly status: number
|
||||
/** Allowed and redacted response headers. */
|
||||
readonly headers: Record<string, string>
|
||||
/** Text body or base64-encoded binary body. */
|
||||
readonly body: string
|
||||
/** Encoding used by `body`; omitted for ordinary text. */
|
||||
readonly bodyEncoding?: "text" | "base64"
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface HttpInteraction {
|
||||
readonly transport: "http"
|
||||
readonly request: RequestSnapshot
|
||||
readonly response: ResponseSnapshot
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export type WebSocketEvent =
|
||||
| { readonly direction: "client" | "server"; readonly kind: "text"; readonly body: string }
|
||||
| {
|
||||
readonly direction: "client" | "server"
|
||||
readonly kind: "binary"
|
||||
readonly body: string
|
||||
readonly bodyEncoding: "base64"
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface WebSocketInteraction {
|
||||
readonly transport: "websocket"
|
||||
readonly open: {
|
||||
readonly url: string
|
||||
readonly headers: Record<string, string>
|
||||
}
|
||||
readonly events: ReadonlyArray<WebSocketEvent>
|
||||
}
|
||||
|
||||
/** Returns whether an incoming HTTP request matches a recorded request. */
|
||||
export type RequestMatcher = (incoming: RequestSnapshot, recorded: RequestSnapshot) => boolean
|
||||
|
||||
/** Additive redaction and header-preservation policy. */
|
||||
export interface RedactOptions {
|
||||
/** Additional sensitive headers to retain as `[REDACTED]`. */
|
||||
readonly headers?: ReadonlyArray<string>
|
||||
/** Additional non-sensitive request headers to preserve for matching. */
|
||||
readonly allowRequestHeaders?: ReadonlyArray<string>
|
||||
/** Additional non-sensitive response headers to preserve for replay. */
|
||||
readonly allowResponseHeaders?: ReadonlyArray<string>
|
||||
/** Additional sensitive URL query parameter names. */
|
||||
readonly queryParameters?: ReadonlyArray<string>
|
||||
/** Additional JSON field names to redact recursively. */
|
||||
readonly jsonFields?: ReadonlyArray<string>
|
||||
/** Stabilizes a URL after built-in redaction. */
|
||||
readonly url?: (url: string) => string
|
||||
/** Stabilizes a request, response, or text-frame body after built-in redaction. */
|
||||
readonly body?: (body: string) => string
|
||||
}
|
||||
|
||||
/** Options shared by HTTP recorder layers. */
|
||||
export interface RecorderOptions {
|
||||
/** Cassette directory. Defaults to `<cwd>/test/fixtures/recordings`. */
|
||||
readonly directory?: string
|
||||
/** Additional metadata stored in the cassette. */
|
||||
readonly metadata?: CassetteMetadata
|
||||
/** Additive redaction and header-preservation policy. */
|
||||
readonly redact?: RedactOptions
|
||||
/** Custom HTTP request equivalence. */
|
||||
readonly match?: RequestMatcher
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface WebSocketRequest {
|
||||
/** WebSocket URL. */
|
||||
readonly url: string
|
||||
/** Headers used for redacted matching; the recorder does not send them. */
|
||||
readonly headers?: Record<string, string>
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface WebSocketRecorderOptions {
|
||||
/** Cassette directory. Defaults to `<cwd>/test/fixtures/recordings`. */
|
||||
readonly directory?: string
|
||||
/** Additional metadata stored in the cassette. */
|
||||
readonly metadata?: CassetteMetadata
|
||||
/** Additive handshake and text-frame redaction policy. */
|
||||
readonly redact?: RedactOptions
|
||||
/** Compare text client frames as canonical JSON instead of exact strings. */
|
||||
readonly compareClientMessagesAsJson?: boolean
|
||||
/** WebSocket subprotocols used by `layerWebSocket`. */
|
||||
readonly protocols?: string | Array<string>
|
||||
}
|
||||
173
packages/http-recorder/src/websocket.ts
Normal file
173
packages/http-recorder/src/websocket.ts
Normal file
@@ -0,0 +1,173 @@
|
||||
import { Effect, Option, Ref, Scope, Semaphore, Stream, SynchronizedRef } from "effect"
|
||||
import type { Headers } from "effect/unstable/http"
|
||||
import * as CassetteService from "./cassette.js"
|
||||
import { canonicalizeJson, decodeJson, safeText } from "./matching.js"
|
||||
import { makeReplayState, resolveAutoMode } from "./recorder.js"
|
||||
import type { RecordReplayMode } from "./internal-effect.js"
|
||||
import { make, type Redactor } from "./redactor.js"
|
||||
import { webSocketInteractions, type CassetteMetadata, type WebSocketEvent } from "./schema.js"
|
||||
|
||||
export interface WebSocketRequest {
|
||||
readonly url: string
|
||||
readonly headers: Headers.Headers
|
||||
}
|
||||
|
||||
export interface WebSocketConnection<E> {
|
||||
readonly sendText: (message: string) => Effect.Effect<void, E>
|
||||
readonly messages: Stream.Stream<string | Uint8Array, E>
|
||||
readonly close: Effect.Effect<void>
|
||||
}
|
||||
|
||||
export interface WebSocketExecutor<E> {
|
||||
readonly open: (request: WebSocketRequest) => Effect.Effect<WebSocketConnection<E>, E>
|
||||
}
|
||||
|
||||
export interface WebSocketRecordReplayOptions<E> {
|
||||
readonly name: string
|
||||
readonly mode?: RecordReplayMode
|
||||
readonly metadata?: CassetteMetadata
|
||||
readonly cassette: CassetteService.Interface
|
||||
readonly live: WebSocketExecutor<E>
|
||||
readonly redactor?: Redactor
|
||||
readonly compareClientMessagesAsJson?: boolean
|
||||
}
|
||||
|
||||
const headersRecord = (headers: Headers.Headers): Record<string, string> =>
|
||||
Object.fromEntries(
|
||||
Object.entries(headers as Record<string, unknown>).filter(
|
||||
(entry): entry is [string, string] => typeof entry[1] === "string",
|
||||
),
|
||||
)
|
||||
|
||||
const textEvent = (direction: "client" | "server", body: string): WebSocketEvent => ({
|
||||
direction,
|
||||
kind: "text",
|
||||
body,
|
||||
})
|
||||
|
||||
const decodeEvent = (event: WebSocketEvent) =>
|
||||
event.kind === "text" ? event.body : new Uint8Array(Buffer.from(event.body, "base64"))
|
||||
|
||||
const jsonOrText = (value: string) => Option.match(decodeJson(value), { onNone: () => value, onSome: canonicalizeJson })
|
||||
|
||||
const assertClientEvent = (actual: string, expected: WebSocketEvent | undefined, index: number, asJson: boolean) =>
|
||||
Effect.sync(() => {
|
||||
const matches =
|
||||
expected?.direction === "client" &&
|
||||
expected.kind === "text" &&
|
||||
JSON.stringify(asJson ? jsonOrText(actual) : actual) ===
|
||||
JSON.stringify(asJson ? jsonOrText(expected.body) : expected.body)
|
||||
if (matches) return
|
||||
throw new Error(`WebSocket client frame ${index + 1}: expected ${safeText(expected)}, received ${safeText(actual)}`)
|
||||
})
|
||||
|
||||
export const makeWebSocketExecutor = <E>(
|
||||
options: WebSocketRecordReplayOptions<E>,
|
||||
): Effect.Effect<WebSocketExecutor<E>, never, Scope.Scope> =>
|
||||
Effect.gen(function* () {
|
||||
const mode = options.mode ?? (yield* resolveAutoMode(options.cassette, options.name))
|
||||
const redactor = options.redactor ?? make()
|
||||
const openSnapshot = (request: WebSocketRequest) => {
|
||||
const snapshot = redactor.request({
|
||||
method: "GET",
|
||||
url: request.url,
|
||||
headers: headersRecord(request.headers),
|
||||
body: "",
|
||||
})
|
||||
return { url: snapshot.url, headers: snapshot.headers }
|
||||
}
|
||||
const redactEvent = (event: WebSocketEvent) => {
|
||||
if (event.kind === "binary") return event
|
||||
const body =
|
||||
event.direction === "client"
|
||||
? redactor.request({ method: "WEBSOCKET", url: "", headers: {}, body: event.body }).body
|
||||
: redactor.response({ status: 101, headers: {}, body: event.body }).body
|
||||
return { ...event, body }
|
||||
}
|
||||
|
||||
if (mode === "passthrough") return options.live
|
||||
|
||||
if (mode === "record") {
|
||||
return {
|
||||
open: (request) =>
|
||||
Effect.gen(function* () {
|
||||
const events: WebSocketEvent[] = []
|
||||
const connection = yield* options.live.open(request)
|
||||
const closed = yield* Ref.make(false)
|
||||
const closeLock = yield* Semaphore.make(1)
|
||||
return {
|
||||
sendText: (message) =>
|
||||
Effect.sync(() => events.push(redactEvent(textEvent("client", message)))).pipe(
|
||||
Effect.andThen(connection.sendText(message)),
|
||||
),
|
||||
messages: connection.messages.pipe(
|
||||
Stream.tap((message) =>
|
||||
Effect.sync(() =>
|
||||
events.push(
|
||||
typeof message === "string"
|
||||
? redactEvent(textEvent("server", message))
|
||||
: {
|
||||
direction: "server",
|
||||
kind: "binary",
|
||||
body: Buffer.from(message).toString("base64"),
|
||||
bodyEncoding: "base64",
|
||||
},
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
close: closeLock.withPermit(
|
||||
Effect.gen(function* () {
|
||||
if (yield* Ref.get(closed)) return
|
||||
yield* connection.close
|
||||
yield* options.cassette
|
||||
.append(
|
||||
options.name,
|
||||
{ transport: "websocket", open: openSnapshot(request), events },
|
||||
options.metadata,
|
||||
)
|
||||
.pipe(Effect.orDie)
|
||||
yield* Ref.set(closed, true)
|
||||
}),
|
||||
),
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
const replay = yield* makeReplayState(options.cassette, options.name, webSocketInteractions)
|
||||
return {
|
||||
open: (request) =>
|
||||
Effect.gen(function* () {
|
||||
const claimed = yield* replay
|
||||
.claim((interaction, index) =>
|
||||
Effect.sync(() => {
|
||||
const incoming = canonicalizeJson(openSnapshot(request))
|
||||
if (interaction && JSON.stringify(incoming) === JSON.stringify(canonicalizeJson(interaction.open)))
|
||||
return
|
||||
throw new Error(`WebSocket open ${index + 1} does not match ${safeText(incoming)}`)
|
||||
}),
|
||||
)
|
||||
.pipe(Effect.orDie)
|
||||
const client = claimed.interaction.events.filter((event) => event.direction === "client")
|
||||
const server = claimed.interaction.events.filter((event) => event.direction === "server")
|
||||
const position = yield* SynchronizedRef.make(0)
|
||||
return {
|
||||
sendText: (message) =>
|
||||
SynchronizedRef.updateEffect(position, (index) =>
|
||||
assertClientEvent(message, client[index], index, options.compareClientMessagesAsJson === true).pipe(
|
||||
Effect.as(index + 1),
|
||||
),
|
||||
),
|
||||
messages: Stream.fromIterable(server).pipe(Stream.map(decodeEvent)),
|
||||
close: Effect.gen(function* () {
|
||||
const used = yield* SynchronizedRef.get(position)
|
||||
if (used !== client.length)
|
||||
return yield* Effect.die(
|
||||
new Error(`WebSocket client frame count: expected ${client.length}, received ${used}`),
|
||||
)
|
||||
}),
|
||||
}
|
||||
}),
|
||||
}
|
||||
})
|
||||
10
packages/http-recorder/sst-env.d.ts
vendored
Normal file
10
packages/http-recorder/sst-env.d.ts
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
/* This file is auto-generated by SST. Do not edit. */
|
||||
/* tslint:disable */
|
||||
/* eslint-disable */
|
||||
/* deno-fmt-ignore-file */
|
||||
/* biome-ignore-all lint: auto-generated */
|
||||
|
||||
/// <reference path="../../sst-env.d.ts" />
|
||||
|
||||
import "sst"
|
||||
export {}
|
||||
41
packages/http-recorder/test/fixtures/recordings/record-replay/multi-step.json
vendored
Normal file
41
packages/http-recorder/test/fixtures/recordings/record-replay/multi-step.json
vendored
Normal file
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"version": 1,
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://example.test/echo",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"step\":1}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"reply\":\"first\"}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://example.test/echo",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"step\":2}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"reply\":\"second\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
41
packages/http-recorder/test/fixtures/recordings/record-replay/retry.json
vendored
Normal file
41
packages/http-recorder/test/fixtures/recordings/record-replay/retry.json
vendored
Normal file
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"version": 1,
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://example.test/poll",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"id\":\"job_1\"}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"status\":\"pending\"}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://example.test/poll",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"id\":\"job_1\"}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"status\":\"complete\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
879
packages/http-recorder/test/record-replay.test.ts
Normal file
879
packages/http-recorder/test/record-replay.test.ts
Normal file
@@ -0,0 +1,879 @@
|
||||
import { NodeFileSystem } from "@effect/platform-node"
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { Cause, Deferred, Effect, Exit, Layer, Scope, Stream } from "effect"
|
||||
import { Headers, HttpBody, HttpClient, HttpClientRequest } from "effect/unstable/http"
|
||||
import { Socket } from "effect/unstable/socket"
|
||||
import * as fs from "node:fs"
|
||||
import * as os from "node:os"
|
||||
import * as path from "node:path"
|
||||
import { HttpRecorder } from "../src"
|
||||
import { HttpRecorderInternal } from "../src/internal"
|
||||
import { redactedErrorRequest } from "../src/internal-effect"
|
||||
import type { Interaction } from "../src/schema"
|
||||
|
||||
const seedCassetteDirectory = (directory: string, name: string, interactions: ReadonlyArray<Interaction>) =>
|
||||
Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const cassette = yield* HttpRecorderInternal.Cassette.Service
|
||||
yield* Effect.forEach(interactions, (interaction) => cassette.append(name, interaction))
|
||||
}).pipe(
|
||||
Effect.provide(HttpRecorderInternal.Cassette.fileSystem({ directory })),
|
||||
Effect.provide(NodeFileSystem.layer),
|
||||
),
|
||||
)
|
||||
|
||||
const post = (url: string, body: object) =>
|
||||
Effect.gen(function* () {
|
||||
const http = yield* HttpClient.HttpClient
|
||||
const request = HttpClientRequest.post(url, {
|
||||
headers: { "content-type": "application/json" },
|
||||
body: HttpBody.text(JSON.stringify(body), "application/json"),
|
||||
})
|
||||
const response = yield* http.execute(request)
|
||||
return yield* response.text
|
||||
})
|
||||
|
||||
const run = <A, E>(effect: Effect.Effect<A, E, HttpClient.HttpClient>) =>
|
||||
Effect.runPromise(effect.pipe(Effect.provide(HttpRecorder.http("record-replay/multi-step"))))
|
||||
|
||||
const runWith = <A, E>(
|
||||
name: string,
|
||||
options: HttpRecorder.RecorderOptions,
|
||||
effect: Effect.Effect<A, E, HttpClient.HttpClient>,
|
||||
) => Effect.runPromise(effect.pipe(Effect.provide(HttpRecorder.http(name, options))))
|
||||
|
||||
const runRecorder = <A, E>(effect: Effect.Effect<A, E, HttpRecorderInternal.Cassette.Service | Scope.Scope>) =>
|
||||
Effect.runPromise(
|
||||
Effect.scoped(
|
||||
effect.pipe(
|
||||
Effect.provide(
|
||||
HttpRecorderInternal.Cassette.fileSystem({
|
||||
directory: fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-")),
|
||||
}),
|
||||
),
|
||||
Effect.provide(NodeFileSystem.layer),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
const failureText = (exit: Exit.Exit<unknown, unknown>) => {
|
||||
if (Exit.isSuccess(exit)) return ""
|
||||
return Cause.prettyErrors(exit.cause).join("\n")
|
||||
}
|
||||
|
||||
describe("http-recorder", () => {
|
||||
test("redacts sensitive URL query parameters", () => {
|
||||
expect(
|
||||
HttpRecorderInternal.redactUrl(
|
||||
"https://example.test/path?key=secret-google-key&api_key=secret-openai-key&safe=value&X-Amz-Signature=secret-signature",
|
||||
),
|
||||
).toBe(
|
||||
"https://example.test/path?key=%5BREDACTED%5D&api_key=%5BREDACTED%5D&safe=value&X-Amz-Signature=%5BREDACTED%5D",
|
||||
)
|
||||
})
|
||||
|
||||
test("redacts URL credentials", () => {
|
||||
expect(HttpRecorderInternal.redactUrl("https://user:password@example.test/path?safe=value")).toBe(
|
||||
"https://%5BREDACTED%5D:%5BREDACTED%5D@example.test/path?safe=value",
|
||||
)
|
||||
})
|
||||
|
||||
test("applies custom URL redaction after built-in redaction", () => {
|
||||
expect(
|
||||
HttpRecorderInternal.redactUrl(
|
||||
"https://example.test/accounts/real-account/path?key=secret-key",
|
||||
undefined,
|
||||
(url) => url.replace("/accounts/real-account/", "/accounts/{account}/"),
|
||||
),
|
||||
).toBe("https://example.test/accounts/{account}/path?key=%5BREDACTED%5D")
|
||||
})
|
||||
|
||||
test("redacts sensitive headers when allow-listed", () => {
|
||||
expect(
|
||||
HttpRecorderInternal.redactHeaders(
|
||||
{
|
||||
authorization: "Bearer secret-token",
|
||||
"content-type": "application/json",
|
||||
"x-custom-token": "custom-secret",
|
||||
"x-api-key": "secret-key",
|
||||
"x-goog-api-key": "secret-google-key",
|
||||
},
|
||||
["authorization", "content-type", "x-api-key", "x-goog-api-key", "x-custom-token"],
|
||||
["x-custom-token"],
|
||||
),
|
||||
).toEqual({
|
||||
authorization: "[REDACTED]",
|
||||
"content-type": "application/json",
|
||||
"x-api-key": "[REDACTED]",
|
||||
"x-custom-token": "[REDACTED]",
|
||||
"x-goog-api-key": "[REDACTED]",
|
||||
})
|
||||
})
|
||||
|
||||
test("redacts error requests without retaining headers, params, or body", () => {
|
||||
const request = HttpClientRequest.post("https://example.test/path", {
|
||||
headers: { authorization: "Bearer super-secret" },
|
||||
body: HttpBody.text("super-secret-body", "text/plain"),
|
||||
}).pipe(HttpClientRequest.setUrlParam("api_key", "super-secret-key"))
|
||||
|
||||
expect(redactedErrorRequest(request).toJSON()).toMatchObject({
|
||||
url: "https://example.test/path",
|
||||
urlParams: { params: [] },
|
||||
headers: {},
|
||||
body: { _tag: "Empty" },
|
||||
})
|
||||
})
|
||||
|
||||
test("detects secret-looking values without returning the secret", () => {
|
||||
expect(
|
||||
HttpRecorderInternal.secretFindings({
|
||||
version: 1,
|
||||
interactions: [
|
||||
{
|
||||
transport: "http",
|
||||
request: {
|
||||
method: "POST",
|
||||
url: "https://example.test/path?key=sk-123456789012345678901234",
|
||||
headers: {},
|
||||
body: JSON.stringify({ nested: "AIzaSyDHibiBRvJZLsFnPYPoiTwxY4ztQ55yqCE" }),
|
||||
},
|
||||
response: {
|
||||
status: 200,
|
||||
headers: {},
|
||||
body: "Bearer abcdefghijklmnopqrstuvwxyz",
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toEqual([
|
||||
{ path: "interactions[0].request.url", reason: "API key" },
|
||||
{ path: "interactions[0].request.body", reason: "Google API key" },
|
||||
{ path: "interactions[0].response.body", reason: "bearer token" },
|
||||
])
|
||||
})
|
||||
|
||||
test("detects secret-looking values inside metadata", () => {
|
||||
expect(
|
||||
HttpRecorderInternal.secretFindings({
|
||||
version: 1,
|
||||
metadata: { token: "sk-123456789012345678901234" },
|
||||
interactions: [],
|
||||
}),
|
||||
).toEqual([{ path: "metadata.token", reason: "API key" }])
|
||||
})
|
||||
|
||||
test("redacts configured and common sensitive JSON fields", () => {
|
||||
const redactor = HttpRecorderInternal.Redactor.make({ jsonFields: ["account_id"] })
|
||||
const request = redactor.request({
|
||||
method: "POST",
|
||||
url: "https://example.test/path",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
password: "secret-password",
|
||||
accessToken: "access-token",
|
||||
nested: { account_id: "account-123", safe: "visible" },
|
||||
}),
|
||||
})
|
||||
|
||||
expect(JSON.parse(request.body)).toEqual({
|
||||
password: "[REDACTED]",
|
||||
accessToken: "[REDACTED]",
|
||||
nested: { account_id: "[REDACTED]", safe: "visible" },
|
||||
})
|
||||
})
|
||||
|
||||
test("extends default header redaction and allow lists", () => {
|
||||
const redactor = HttpRecorderInternal.Redactor.make({
|
||||
headers: ["x-custom-token"],
|
||||
allowRequestHeaders: ["anthropic-version", "x-custom-token"],
|
||||
})
|
||||
|
||||
expect(
|
||||
redactor.request({
|
||||
method: "GET",
|
||||
url: "https://example.test/path",
|
||||
headers: {
|
||||
authorization: "Bearer secret",
|
||||
"content-type": "application/json",
|
||||
"anthropic-version": "2023-06-01",
|
||||
"x-custom-token": "secret",
|
||||
},
|
||||
body: "",
|
||||
}).headers,
|
||||
).toEqual({
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json",
|
||||
"x-custom-token": "[REDACTED]",
|
||||
})
|
||||
})
|
||||
|
||||
test("records WebSocket frames in observed client/server order", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-websocket-"))
|
||||
const response = JSON.stringify({ type: "response.completed", token: "server-secret" })
|
||||
let receive: ((message: string | Uint8Array) => Effect.Effect<unknown, unknown, unknown> | void) | undefined
|
||||
const upstream = Socket.make({
|
||||
runRaw: (handler, options) =>
|
||||
Effect.gen(function* () {
|
||||
receive = handler
|
||||
if (options?.onOpen) yield* options.onOpen
|
||||
receive = undefined
|
||||
}),
|
||||
writer: Effect.succeed(() =>
|
||||
Effect.suspend(() => {
|
||||
const result = receive?.(response)
|
||||
return Effect.isEffect(result) ? Effect.asVoid(result) : Effect.void
|
||||
}),
|
||||
),
|
||||
})
|
||||
|
||||
await Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const socket = yield* Socket.Socket
|
||||
const write = yield* socket.writer
|
||||
yield* socket.runRaw(() => {}, {
|
||||
onOpen: write(JSON.stringify({ type: "response.create", token: "client-secret" })),
|
||||
})
|
||||
}).pipe(
|
||||
Effect.scoped,
|
||||
Effect.provide(
|
||||
HttpRecorderInternal.socketLayer(
|
||||
"websocket/record",
|
||||
{ url: "wss://example.test/realtime", headers: { "content-type": "application/json" } },
|
||||
{ directory, metadata: { provider: "test" }, mode: "record" },
|
||||
).pipe(Layer.provide(Layer.succeed(Socket.Socket, upstream))),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(JSON.parse(fs.readFileSync(path.join(directory, "websocket/record.json"), "utf8"))).toMatchObject({
|
||||
interactions: [
|
||||
{
|
||||
transport: "websocket",
|
||||
open: { url: "wss://example.test/realtime", headers: { "content-type": "application/json" } },
|
||||
events: [
|
||||
{ direction: "client", kind: "text", body: '{"type":"response.create","token":"[REDACTED]"}' },
|
||||
{ direction: "server", kind: "text", body: '{"type":"response.completed","token":"[REDACTED]"}' },
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
test("WebSocket replay preserves causal frame ordering", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-websocket-"))
|
||||
await seedCassetteDirectory(directory, "websocket/replay", [
|
||||
{
|
||||
transport: "websocket",
|
||||
open: { url: "wss://example.test/realtime", headers: {} },
|
||||
events: [
|
||||
{ direction: "server", kind: "text", body: '{"type":"session.created"}' },
|
||||
{ direction: "client", kind: "text", body: '{"type":"response.create","prompt":"hello"}' },
|
||||
{ direction: "server", kind: "text", body: '{"type":"response.completed"}' },
|
||||
],
|
||||
},
|
||||
])
|
||||
|
||||
const received: string[] = []
|
||||
await Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const socket = yield* Socket.Socket
|
||||
const write = yield* socket.writer
|
||||
yield* socket.runRaw((message) => {
|
||||
if (typeof message !== "string") return
|
||||
received.push(message)
|
||||
if (JSON.parse(message).type === "session.created")
|
||||
return write('{"prompt":"hello","type":"response.create"}')
|
||||
})
|
||||
}).pipe(
|
||||
Effect.scoped,
|
||||
Effect.provide(
|
||||
HttpRecorderInternal.socketLayer(
|
||||
"websocket/replay",
|
||||
{ url: "wss://example.test/realtime" },
|
||||
{ directory, compareClientMessagesAsJson: true, mode: "replay" },
|
||||
).pipe(
|
||||
Layer.provide(
|
||||
Layer.succeed(
|
||||
Socket.Socket,
|
||||
Socket.make({
|
||||
runRaw: () => Effect.die(new Error("unexpected live WebSocket run")),
|
||||
writer: Effect.succeed(() => Effect.die(new Error("unexpected live WebSocket write"))),
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(received).toEqual(['{"type":"session.created"}', '{"type":"response.completed"}'])
|
||||
})
|
||||
|
||||
test("the public socket decorator replays a provided Effect socket", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-websocket-"))
|
||||
await seedCassetteDirectory(directory, "websocket/public-layer", [
|
||||
{
|
||||
transport: "websocket",
|
||||
open: { url: "", headers: {} },
|
||||
events: [
|
||||
{ direction: "client", kind: "text", body: "hello" },
|
||||
{ direction: "server", kind: "text", body: "hello" },
|
||||
],
|
||||
},
|
||||
])
|
||||
|
||||
const received: string[] = []
|
||||
await Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const socket = yield* Socket.Socket
|
||||
const write = yield* socket.writer
|
||||
yield* socket.runString(
|
||||
(message) =>
|
||||
Effect.gen(function* () {
|
||||
received.push(message)
|
||||
yield* write(new Socket.CloseEvent(1000))
|
||||
}),
|
||||
{ onOpen: write("hello") },
|
||||
)
|
||||
}).pipe(
|
||||
Effect.scoped,
|
||||
Effect.provide(
|
||||
HttpRecorder.socket("websocket/public-layer", { directory }).pipe(
|
||||
Layer.provide(
|
||||
Layer.succeed(
|
||||
Socket.Socket,
|
||||
Socket.make({
|
||||
runRaw: () => Effect.die(new Error("unexpected live WebSocket run")),
|
||||
writer: Effect.succeed(() => Effect.die(new Error("unexpected live WebSocket write"))),
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(received).toEqual(["hello"])
|
||||
})
|
||||
|
||||
test("WebSocket replay runs message handlers concurrently", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-websocket-"))
|
||||
await seedCassetteDirectory(directory, "websocket/concurrent-handlers", [
|
||||
{
|
||||
transport: "websocket",
|
||||
open: { url: "wss://example.test/realtime", headers: {} },
|
||||
events: [
|
||||
{ direction: "server", kind: "text", body: "first" },
|
||||
{ direction: "server", kind: "text", body: "second" },
|
||||
],
|
||||
},
|
||||
])
|
||||
|
||||
await Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const socket = yield* Socket.Socket
|
||||
const second = yield* Deferred.make<void>()
|
||||
yield* socket.runString((message) =>
|
||||
message === "first" ? Deferred.await(second) : Deferred.succeed(second, undefined),
|
||||
)
|
||||
}).pipe(
|
||||
Effect.scoped,
|
||||
Effect.provide(
|
||||
HttpRecorderInternal.socketLayer(
|
||||
"websocket/concurrent-handlers",
|
||||
{ url: "wss://example.test/realtime" },
|
||||
{ directory, mode: "replay" },
|
||||
).pipe(
|
||||
Layer.provide(
|
||||
Layer.succeed(
|
||||
Socket.Socket,
|
||||
Socket.make({
|
||||
runRaw: () => Effect.die(new Error("unexpected live WebSocket run")),
|
||||
writer: Effect.succeed(() => Effect.die(new Error("unexpected live WebSocket write"))),
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
test("WebSocket replay rejects close with unconsumed events", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-websocket-"))
|
||||
await seedCassetteDirectory(directory, "websocket/early-close", [
|
||||
{
|
||||
transport: "websocket",
|
||||
open: { url: "wss://example.test/realtime", headers: {} },
|
||||
events: [{ direction: "client", kind: "text", body: "expected" }],
|
||||
},
|
||||
])
|
||||
|
||||
const exit = await Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const socket = yield* Socket.Socket
|
||||
const write = yield* socket.writer
|
||||
return yield* Effect.exit(socket.runRaw(() => {}, { onOpen: write(new Socket.CloseEvent(1000)) }))
|
||||
}).pipe(
|
||||
Effect.scoped,
|
||||
Effect.provide(
|
||||
HttpRecorderInternal.socketLayer(
|
||||
"websocket/early-close",
|
||||
{ url: "wss://example.test/realtime" },
|
||||
{ directory, mode: "replay" },
|
||||
).pipe(
|
||||
Layer.provide(
|
||||
Layer.succeed(
|
||||
Socket.Socket,
|
||||
Socket.make({
|
||||
runRaw: () => Effect.die(new Error("unexpected live WebSocket run")),
|
||||
writer: Effect.succeed(() => Effect.die(new Error("unexpected live WebSocket write"))),
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(failureText(exit)).toContain("closed with unconsumed events")
|
||||
})
|
||||
|
||||
test("failed WebSocket runs do not write complete cassettes", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-websocket-"))
|
||||
const exit = await Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const socket = yield* Socket.Socket
|
||||
return yield* Effect.exit(socket.runRaw(() => {}))
|
||||
}).pipe(
|
||||
Effect.scoped,
|
||||
Effect.provide(
|
||||
HttpRecorderInternal.socketLayer(
|
||||
"websocket/failed-run",
|
||||
{ url: "wss://example.test/realtime" },
|
||||
{ directory, mode: "record" },
|
||||
).pipe(
|
||||
Layer.provide(
|
||||
Layer.succeed(
|
||||
Socket.Socket,
|
||||
Socket.make({
|
||||
runRaw: () => Effect.die(new Error("connection failed")),
|
||||
writer: Effect.succeed(() => Effect.void),
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(Exit.isFailure(exit)).toBe(true)
|
||||
expect(fs.existsSync(path.join(directory, "websocket/failed-run.json"))).toBe(false)
|
||||
})
|
||||
|
||||
test("WebSocket replay preserves binary frame kinds across reconnects", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-websocket-"))
|
||||
const interaction = {
|
||||
transport: "websocket" as const,
|
||||
open: { url: "wss://example.test/binary", headers: {} },
|
||||
events: [
|
||||
{
|
||||
direction: "client" as const,
|
||||
kind: "binary" as const,
|
||||
body: Buffer.from([1, 2]).toString("base64"),
|
||||
bodyEncoding: "base64" as const,
|
||||
},
|
||||
{
|
||||
direction: "server" as const,
|
||||
kind: "binary" as const,
|
||||
body: Buffer.from([3, 4]).toString("base64"),
|
||||
bodyEncoding: "base64" as const,
|
||||
},
|
||||
],
|
||||
}
|
||||
await seedCassetteDirectory(directory, "websocket/binary", [interaction, interaction])
|
||||
|
||||
const received: number[][] = []
|
||||
await Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const socket = yield* Socket.Socket
|
||||
const write = yield* socket.writer
|
||||
const run = socket.runRaw(
|
||||
(message) => {
|
||||
if (typeof message === "string") throw new Error("Expected a binary WebSocket frame")
|
||||
received.push([...message])
|
||||
},
|
||||
{ onOpen: write(new Uint8Array([1, 2])) },
|
||||
)
|
||||
yield* run
|
||||
yield* run
|
||||
}).pipe(
|
||||
Effect.scoped,
|
||||
Effect.provide(
|
||||
HttpRecorderInternal.socketLayer(
|
||||
"websocket/binary",
|
||||
{ url: "wss://example.test/binary" },
|
||||
{ directory, mode: "replay" },
|
||||
).pipe(
|
||||
Layer.provide(
|
||||
Layer.succeed(
|
||||
Socket.Socket,
|
||||
Socket.make({
|
||||
runRaw: () => Effect.die(new Error("unexpected live WebSocket run")),
|
||||
writer: Effect.succeed(() => Effect.die(new Error("unexpected live WebSocket write"))),
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(received).toEqual([
|
||||
[3, 4],
|
||||
[3, 4],
|
||||
])
|
||||
})
|
||||
|
||||
test("replay returns recorded responses in order for identical requests", async () => {
|
||||
await runWith(
|
||||
"record-replay/retry",
|
||||
{},
|
||||
Effect.gen(function* () {
|
||||
expect(yield* post("https://example.test/poll", { id: "job_1" })).toBe('{"status":"pending"}')
|
||||
expect(yield* post("https://example.test/poll", { id: "job_1" })).toBe('{"status":"complete"}')
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
test("replay reports cursor exhaustion when more requests are made than recorded", async () => {
|
||||
await run(
|
||||
Effect.gen(function* () {
|
||||
yield* post("https://example.test/echo", { step: 1 })
|
||||
yield* post("https://example.test/echo", { step: 2 })
|
||||
const exit = yield* Effect.exit(post("https://example.test/echo", { step: 3 }))
|
||||
expect(Exit.isFailure(exit)).toBe(true)
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
test("replay validates each recorded request in order", async () => {
|
||||
await run(
|
||||
Effect.gen(function* () {
|
||||
yield* post("https://example.test/echo", { step: 1 })
|
||||
const exit = yield* Effect.exit(post("https://example.test/echo", { step: 3 }))
|
||||
expect(Exit.isFailure(exit)).toBe(true)
|
||||
expect(failureText(exit)).toContain("$.step expected 2, received 3")
|
||||
expect(yield* post("https://example.test/echo", { step: 2 })).toBe('{"reply":"second"}')
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
test("concurrent replay claims each interaction once", async () => {
|
||||
const results = await runWith(
|
||||
"record-replay/retry",
|
||||
{},
|
||||
Effect.all(
|
||||
[post("https://example.test/poll", { id: "job_1" }), post("https://example.test/poll", { id: "job_1" })],
|
||||
{ concurrency: "unbounded" },
|
||||
),
|
||||
)
|
||||
|
||||
expect(results.toSorted()).toEqual(['{"status":"complete"}', '{"status":"pending"}'])
|
||||
})
|
||||
|
||||
test("replays when the cassette exists", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-auto-"))
|
||||
await seedCassetteDirectory(directory, "auto-replay", [
|
||||
{
|
||||
transport: "http",
|
||||
request: {
|
||||
method: "POST",
|
||||
url: "https://example.test/echo",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ step: 1 }),
|
||||
},
|
||||
response: { status: 200, headers: { "content-type": "application/json" }, body: '{"reply":"hi"}' },
|
||||
},
|
||||
])
|
||||
|
||||
const result = await runWith("auto-replay", { directory }, post("https://example.test/echo", { step: 1 }))
|
||||
expect(result).toBe('{"reply":"hi"}')
|
||||
})
|
||||
|
||||
test("forces replay when CI=true even if cassette is missing", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-auto-ci-"))
|
||||
const previous = process.env.CI
|
||||
process.env.CI = "true"
|
||||
try {
|
||||
const exit = await Effect.runPromise(
|
||||
Effect.exit(
|
||||
post("https://example.test/echo", { step: 1 }).pipe(
|
||||
Effect.provide(HttpRecorder.http("missing-cassette", { directory })),
|
||||
),
|
||||
),
|
||||
)
|
||||
expect(Exit.isFailure(exit)).toBe(true)
|
||||
expect(failureText(exit)).toContain('Fixture "missing-cassette" not found')
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.CI
|
||||
else process.env.CI = previous
|
||||
}
|
||||
})
|
||||
|
||||
test("mismatch diagnostics show redacted request differences against the expected interaction", async () => {
|
||||
await run(
|
||||
Effect.gen(function* () {
|
||||
const exit = yield* Effect.exit(
|
||||
post("https://example.test/echo?api_key=secret-value", { step: 3, token: "sk-123456789012345678901234" }),
|
||||
)
|
||||
const message = failureText(exit)
|
||||
expect(message).toContain("url:")
|
||||
expect(message).toContain("https://example.test/echo?api_key=%5BREDACTED%5D")
|
||||
expect(message).toContain("body:")
|
||||
expect(message).toContain("$.step expected 1, received 3")
|
||||
expect(message).toContain('$.token expected undefined, received "[REDACTED]"')
|
||||
expect(message).not.toContain("sk-123456789012345678901234")
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
test("records to disk when the cassette is missing", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-auto-record-"))
|
||||
using server = Bun.serve({
|
||||
port: 0,
|
||||
fetch: () => new Response('{"reply":"recorded"}', { headers: { "content-type": "application/json" } }),
|
||||
})
|
||||
const url = `http://127.0.0.1:${server.port}/echo`
|
||||
// CI=true forces replay; clear it so we exercise the local-dev auto-record path.
|
||||
const previous = process.env.CI
|
||||
delete process.env.CI
|
||||
try {
|
||||
const result = await runWith("auto-record", { directory }, post(url, { step: 1 }))
|
||||
expect(result).toBe('{"reply":"recorded"}')
|
||||
expect(fs.existsSync(path.join(directory, "auto-record.json"))).toBe(true)
|
||||
} finally {
|
||||
if (previous !== undefined) process.env.CI = previous
|
||||
}
|
||||
})
|
||||
|
||||
test("records concurrent requests in request-start order", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-order-"))
|
||||
const first = Promise.withResolvers<void>()
|
||||
const completed: string[] = []
|
||||
using server = Bun.serve({
|
||||
port: 0,
|
||||
fetch: async (request) => {
|
||||
const name = new URL(request.url).pathname.slice(1)
|
||||
if (name === "first") {
|
||||
await first.promise
|
||||
completed.push(name)
|
||||
return new Response(name)
|
||||
}
|
||||
completed.push(name)
|
||||
first.resolve()
|
||||
return new Response(name)
|
||||
},
|
||||
})
|
||||
const previous = process.env.CI
|
||||
delete process.env.CI
|
||||
try {
|
||||
const request = (name: string) =>
|
||||
Effect.gen(function* () {
|
||||
const http = yield* HttpClient.HttpClient
|
||||
const response = yield* http.execute(HttpClientRequest.get(`http://127.0.0.1:${server.port}/${name}`))
|
||||
return yield* response.text
|
||||
})
|
||||
const responses = await Effect.runPromise(
|
||||
Effect.all([request("first"), request("second")], { concurrency: "unbounded" }).pipe(
|
||||
Effect.provide(HttpRecorder.http("concurrent-order", { directory })),
|
||||
),
|
||||
)
|
||||
const cassette = JSON.parse(fs.readFileSync(path.join(directory, "concurrent-order.json"), "utf8"))
|
||||
|
||||
expect(completed).toEqual(["second", "first"])
|
||||
expect(responses).toEqual(["first", "second"])
|
||||
expect(cassette.interactions.map((interaction: Interaction) => interaction.request.url)).toEqual([
|
||||
`http://127.0.0.1:${server.port}/first`,
|
||||
`http://127.0.0.1:${server.port}/second`,
|
||||
])
|
||||
} finally {
|
||||
if (previous !== undefined) process.env.CI = previous
|
||||
}
|
||||
})
|
||||
|
||||
test("returns the live response while persisting its redacted snapshot", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-live-response-"))
|
||||
using server = Bun.serve({
|
||||
port: 0,
|
||||
fetch: () =>
|
||||
new Response(JSON.stringify({ access_token: "live-secret", safe: true }), {
|
||||
headers: { "content-type": "application/json", "x-request-id": "request-1" },
|
||||
}),
|
||||
})
|
||||
const previous = process.env.CI
|
||||
delete process.env.CI
|
||||
try {
|
||||
const body = await runWith(
|
||||
"live-response",
|
||||
{ directory },
|
||||
post(`http://127.0.0.1:${server.port}/response`, { ok: true }),
|
||||
)
|
||||
const cassette = JSON.parse(fs.readFileSync(path.join(directory, "live-response.json"), "utf8"))
|
||||
|
||||
expect(body).toBe('{"access_token":"live-secret","safe":true}')
|
||||
expect(cassette.interactions[0].response.body).toBe('{"access_token":"[REDACTED]","safe":true}')
|
||||
} finally {
|
||||
if (previous !== undefined) process.env.CI = previous
|
||||
}
|
||||
})
|
||||
|
||||
test("reconstructs responses with null-body statuses", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-no-content-"))
|
||||
using server = Bun.serve({ port: 0, fetch: () => new Response(null, { status: 204 }) })
|
||||
const previous = process.env.CI
|
||||
delete process.env.CI
|
||||
try {
|
||||
const program = Effect.gen(function* () {
|
||||
const http = yield* HttpClient.HttpClient
|
||||
return yield* http.execute(HttpClientRequest.get(`http://127.0.0.1:${server.port}/empty`))
|
||||
})
|
||||
const response = await Effect.runPromise(
|
||||
program.pipe(Effect.provide(HttpRecorder.http("no-content", { directory }))),
|
||||
)
|
||||
|
||||
expect(response.status).toBe(204)
|
||||
} finally {
|
||||
if (previous !== undefined) process.env.CI = previous
|
||||
}
|
||||
})
|
||||
|
||||
test("records and replays arbitrary binary responses without changing bytes", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-binary-"))
|
||||
const expected = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0xff, 0x00, 0x80])
|
||||
using server = Bun.serve({
|
||||
port: 0,
|
||||
fetch: () => new Response(expected, { headers: { "content-type": "image/png" } }),
|
||||
})
|
||||
const url = `http://127.0.0.1:${server.port}/image.png`
|
||||
const previous = process.env.CI
|
||||
delete process.env.CI
|
||||
try {
|
||||
const program = Effect.gen(function* () {
|
||||
const http = yield* HttpClient.HttpClient
|
||||
const response = yield* http.execute(HttpClientRequest.get(url))
|
||||
return new Uint8Array(yield* response.arrayBuffer)
|
||||
})
|
||||
const record = await Effect.runPromise(program.pipe(Effect.provide(HttpRecorder.http("binary", { directory }))))
|
||||
await server.stop()
|
||||
const replay = await Effect.runPromise(program.pipe(Effect.provide(HttpRecorder.http("binary", { directory }))))
|
||||
const cassette = JSON.parse(fs.readFileSync(path.join(directory, "binary.json"), "utf8"))
|
||||
|
||||
expect(record).toEqual(expected)
|
||||
expect(replay).toEqual(expected)
|
||||
expect(cassette.interactions[0].response.bodyEncoding).toBe("base64")
|
||||
} finally {
|
||||
if (previous !== undefined) process.env.CI = previous
|
||||
}
|
||||
})
|
||||
|
||||
test("UnsafeCassetteError fails the request when a recording would write a known secret", async () => {
|
||||
using server = Bun.serve({ port: 0, fetch: () => new Response("Bearer abcdefghijklmnopqrstuvwxyz1234") })
|
||||
const url = `http://127.0.0.1:${server.port}/leaky`
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-unsafe-"))
|
||||
|
||||
const exit = await Effect.runPromise(
|
||||
Effect.exit(
|
||||
post(url, { ok: true }).pipe(
|
||||
Effect.provide(HttpRecorderInternal.cassetteLayer("unsafe-record", { directory, mode: "record" })),
|
||||
),
|
||||
),
|
||||
)
|
||||
expect(Exit.isFailure(exit)).toBe(true)
|
||||
expect(failureText(exit)).toContain("contains possible secrets")
|
||||
expect(fs.existsSync(path.join(directory, "unsafe-record.json"))).toBe(false)
|
||||
})
|
||||
|
||||
test("failed memory appends leave cassette state unchanged", async () => {
|
||||
await Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const cassette = yield* HttpRecorderInternal.Cassette.Service
|
||||
const interaction: Interaction = {
|
||||
transport: "http",
|
||||
request: { method: "GET", url: "https://example.test", headers: {}, body: "" },
|
||||
response: { status: 200, headers: {}, body: "safe" },
|
||||
}
|
||||
yield* cassette.append("transactional", interaction)
|
||||
yield* cassette
|
||||
.append("transactional", {
|
||||
...interaction,
|
||||
response: { ...interaction.response, body: "Bearer abcdefghijklmnopqrstuvwxyz1234" },
|
||||
})
|
||||
.pipe(Effect.flip)
|
||||
|
||||
expect(yield* cassette.read("transactional")).toEqual([interaction])
|
||||
}).pipe(Effect.provide(HttpRecorderInternal.Cassette.memory())),
|
||||
)
|
||||
})
|
||||
|
||||
test("concurrent file appends preserve every interaction", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-concurrent-"))
|
||||
await Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const cassette = yield* HttpRecorderInternal.Cassette.Service
|
||||
yield* Effect.forEach(
|
||||
Array.from({ length: 20 }, (_, index) => index),
|
||||
(index) =>
|
||||
cassette.append("concurrent", {
|
||||
transport: "http",
|
||||
request: { method: "GET", url: `https://example.test/${index}`, headers: {}, body: "" },
|
||||
response: { status: 200, headers: {}, body: String(index) },
|
||||
}),
|
||||
{ concurrency: "unbounded" },
|
||||
)
|
||||
}).pipe(
|
||||
Effect.provide(HttpRecorderInternal.Cassette.fileSystem({ directory })),
|
||||
Effect.provide(NodeFileSystem.layer),
|
||||
),
|
||||
)
|
||||
|
||||
const cassette = JSON.parse(fs.readFileSync(path.join(directory, "concurrent.json"), "utf8"))
|
||||
expect(cassette.interactions).toHaveLength(20)
|
||||
expect(fs.readdirSync(directory).filter((file) => file.endsWith(".tmp"))).toEqual([])
|
||||
})
|
||||
|
||||
test("rejects cassette paths outside the recordings directory", () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-path-"))
|
||||
expect(() => HttpRecorderInternal.hasCassetteSync("../outside", { directory })).toThrow("Invalid cassette name")
|
||||
expect(() => HttpRecorderInternal.hasCassetteSync("C:\\outside", { directory })).toThrow("Invalid cassette name")
|
||||
})
|
||||
|
||||
test("Cassette.list enumerates recorded cassette names", async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "http-recorder-list-"))
|
||||
await seedCassetteDirectory(directory, "alpha/one", [
|
||||
{
|
||||
transport: "http",
|
||||
request: { method: "GET", url: "https://x.test/a", headers: {}, body: "" },
|
||||
response: { status: 200, headers: {}, body: "a" },
|
||||
},
|
||||
])
|
||||
await seedCassetteDirectory(directory, "beta", [
|
||||
{
|
||||
transport: "http",
|
||||
request: { method: "GET", url: "https://x.test/b", headers: {}, body: "" },
|
||||
response: { status: 200, headers: {}, body: "b" },
|
||||
},
|
||||
])
|
||||
|
||||
const names = await Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const cassette = yield* HttpRecorderInternal.Cassette.Service
|
||||
return yield* cassette.list()
|
||||
}).pipe(
|
||||
Effect.provide(HttpRecorderInternal.Cassette.fileSystem({ directory })),
|
||||
Effect.provide(NodeFileSystem.layer),
|
||||
),
|
||||
)
|
||||
expect(names).toEqual(["alpha/one", "beta"])
|
||||
})
|
||||
})
|
||||
22
packages/http-recorder/tsconfig.json
Normal file
22
packages/http-recorder/tsconfig.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "@tsconfig/node22/tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "src",
|
||||
"outDir": "dist",
|
||||
"module": "nodenext",
|
||||
"moduleResolution": "nodenext",
|
||||
"declaration": true,
|
||||
"stripInternal": true,
|
||||
"lib": ["ESNext", "DOM", "DOM.Iterable"],
|
||||
"noUncheckedIndexedAccess": false,
|
||||
"plugins": [
|
||||
{
|
||||
"name": "@effect/language-service",
|
||||
"transform": "@effect/language-service/transform",
|
||||
"namespaceImportPackages": ["effect", "@effect/*"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
Reference in New Issue
Block a user