Web Bot Auth: follow draft-ietf-webbotauth-httpsig-protocol-01

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- **Web Bot Auth verification follows draft-ietf-webbotauth-httpsig-protocol-01.** A `Signature-Agent` value is the URL of the agent's key set, and a key now verifies only for the directory that published it, so a key reused by another directory, or published at a different URL on the same host, no longer borrows the trusted agent's name. A bare origin, the form agents send today, still means that origin's well-known directory. Members carrying parameters are ignored, a signature must cover the `Signature-Agent` member it relies on, and directory fetches no longer follow redirects. Signatures that cannot be attributed report `claimed`, never `verified`.
+- **Breaking for direct `DirectoryCache` users:** `resolveKey(keyId)` is now `resolveKey(keyId, agent)`, where `agent` is the resolved Signature-Agent identifier. `AgentVerifier` and `detectBot()` callers are unaffected.
+- The Vercel validator's bundled clearance decision carries the same Web Bot Auth change.
+
 ## [0.22.3] - 2026-11-11
 
 ### Changed

packages/webdecoy/src/agent/directory.ts

@@ -22,8 +22,7 @@
 import { jwkThumbprint, type JWK } from './thumbprint';
 import type { VerificationKey, KeyResolver } from './signature';
 import type { SignedAgentDirectory } from './types';
-
-const WELL_KNOWN = '/.well-known/http-message-signatures-directory';
+import { resolveSignatureAgent } from './identity';
 const COLD_RETRY_BACKOFF_MS = 30_000;
 
 /**
@@ -62,10 +61,14 @@ export class DirectoryCache {
     this.opts = opts;
   }
 
-  /** Resolve a keyid to a verification key, fetching/refreshing as needed. */
-  resolveKey: KeyResolver = async (keyId: string) => {
+  /**
+   * Resolve a keyid published by one agent identity (a resolved
+   * Signature-Agent URL) to a verification key, fetching/refreshing as
+   * needed. A key is only ever attributed to the directory that published it.
+   */
+  resolveKey: KeyResolver = async (keyId: string, agent: string) => {
     await this.ensureFresh();
-    return this.keys.get(keyId);
+    return this.keys.get(cacheKey(agent, keyId));
   };
 
   /** Kick off a fetch without blocking — call at startup to warm the cache. */
@@ -110,7 +113,7 @@ export class DirectoryCache {
     for (const keys of results) {
       if (keys === null) continue;
       fetchedAny = true;
-      for (const k of keys) next.set(k.thumbprint, k.key);
+      for (const k of keys) next.set(cacheKey(k.identifier, k.thumbprint), k.key);
     }
 
     if (fetchedAny) {
@@ -127,8 +130,15 @@ export class DirectoryCache {
 
   private async fetchDirectory(
     src: SignedAgentDirectory,
-  ): Promise<{ thumbprint: string; key: VerificationKey }[] | null> {
-    const url = directoryUrl(src.directory);
+  ): Promise<{ identifier: string; thumbprint: string; key: VerificationKey }[] | null> {
+    let url: string;
+    let identifier: string;
+    try {
+      ({ fetch: url, identifier } = resolveSignatureAgent(src.directory));
+    } catch {
+      this.debug(`${src.name}: directory is not an https URL`);
+      return null;
+    }
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), this.opts.fetchTimeoutMs);
     try {
@@ -138,7 +148,8 @@ export class DirectoryCache {
           Accept:
             'application/http-message-signatures-directory+json, application/jwk-set+json, application/json',
         },
-        redirect: 'follow',
+        // Web Bot Auth draft 01 §5.5: redirects are a discovery failure.
+        redirect: 'error',
       });
       if (!res.ok) {
         this.debug(`${src.name}: HTTP ${res.status}`);
@@ -149,10 +160,10 @@ export class DirectoryCache {
         this.debug(`${src.name}: no keys in directory`);
         return null;
       }
-      const out: { thumbprint: string; key: VerificationKey }[] = [];
+      const out: { identifier: string; thumbprint: string; key: VerificationKey }[] = [];
       for (const jwk of body.keys) {
         const resolved = await importKey(jwk, src);
-        if (resolved) out.push(resolved);
+        if (resolved) out.push({ identifier, ...resolved });
       }
       this.debug(`${src.name}: ${out.length} key(s)`);
       return out;
@@ -169,12 +180,8 @@ export class DirectoryCache {
   }
 }
 
-/** Append the well-known path unless the directory string already carries one. */
-function directoryUrl(directory: string): string {
-  const u = new URL(directory);
-  if (u.pathname && u.pathname !== '/') return u.toString();
-  u.pathname = WELL_KNOWN;
-  return u.toString();
+function cacheKey(agent: string, keyId: string): string {
+  return `${agent} ${keyId}`;
 }
 
 /**

packages/webdecoy/src/agent/identity.ts

@@ -0,0 +1,45 @@
+/**
+ * Signature-Agent identity (draft-ietf-webbotauth-httpsig-protocol-01 §5.5).
+ *
+ * A Signature-Agent value is a URL that serves a JWK Set, and that URL (minus
+ * its query) is the agent's identity. A bare origin is the form signers sent
+ * under earlier drafts and still send today; it means the origin's well-known
+ * directory, because fetching the origin itself would return a web page.
+ *
+ * Mirrors `ResolveSignatureAgent` in github.com/WebDecoy/web-bot-auth. The
+ * shared vectors in `signature-agent-resolution.json` keep the two in step.
+ */
+
+export const WELL_KNOWN_DIRECTORY = '/.well-known/http-message-signatures-directory';
+
+export interface ResolvedAgent {
+  /** The URL a verifier fetches the JWK Set from. */
+  fetch: string;
+  /** The identity keys are attributed to: the fetched URL without its query. */
+  identifier: string;
+}
+
+/** Resolve a Signature-Agent value. Throws when it is not a usable https URL. */
+export function resolveSignatureAgent(value: string): ResolvedAgent {
+  // WHATWG URL parsing forgives "https:///host"; the draft does not.
+  if (!/^https:\/\/[^/\\]/i.test(value)) throw new Error('signature-agent must be an https URL');
+  const url = new URL(value);
+  if (url.protocol !== 'https:' || !url.hostname || url.username || url.password) {
+    throw new Error('signature-agent must be an https URL');
+  }
+  url.hash = '';
+  const hasQuery = url.href.includes('?');
+  const authority = canonicalAuthority(url);
+  if (url.pathname === '/' && !hasQuery) {
+    const wellKnown = `https://${authority}${WELL_KNOWN_DIRECTORY}`;
+    return { fetch: wellKnown, identifier: wellKnown };
+  }
+  const path = url.pathname.replace(/%[0-9a-f]{2}/gi, (m) => m.toUpperCase());
+  return { fetch: url.href, identifier: `https://${authority}${path}` };
+}
+
+function canonicalAuthority(url: URL): string {
+  const host = url.hostname.toLowerCase();
+  if (!url.port || url.port === '443') return host;
+  return `${host}:${url.port}`;
+}

packages/webdecoy/src/agent/sfv.ts

@@ -51,6 +51,16 @@ class Cursor {
 }
 
 /** Parse an RFC 8941 dictionary header value. Throws on malformed input. */
+/** Parse a header value that is exactly one sf-string (RFC 8941 §3.3.3). */
+export function parseBareString(input: string): string {
+  const p = new Cursor(input);
+  p.skipSP();
+  const value = parseString(p);
+  p.skipSP();
+  if (!p.eof()) throw new Error('trailing data after string');
+  return value;
+}
+
 export function parseDictionary(input: string): DictEntry[] {
   const p = new Cursor(input);
   const out: DictEntry[] = [];

packages/webdecoy/src/agent/signature-agent-resolution.json

@@ -0,0 +1,72 @@
+{
+  "_comment": "Signature-Agent member value -> (fetch URL, draft-01 identifier). Generated from and checked by ResolveSignatureAgent; the edge and SDK TypeScript ports test against a copy. Ports may differ on the fetch URL's spelling (case, default port) but must agree on the identifier.",
+  "valid": [
+    {
+      "value": "https://agent.example",
+      "fetch": "https://agent.example/.well-known/http-message-signatures-directory",
+      "identifier": "https://agent.example/.well-known/http-message-signatures-directory"
+    },
+    {
+      "value": "https://agent.example/",
+      "fetch": "https://agent.example/.well-known/http-message-signatures-directory",
+      "identifier": "https://agent.example/.well-known/http-message-signatures-directory"
+    },
+    {
+      "value": "https://Agent.Example:443",
+      "fetch": "https://Agent.Example:443/.well-known/http-message-signatures-directory",
+      "identifier": "https://agent.example/.well-known/http-message-signatures-directory"
+    },
+    {
+      "value": "https://agent.example/.well-known/http-message-signatures-directory",
+      "fetch": "https://agent.example/.well-known/http-message-signatures-directory",
+      "identifier": "https://agent.example/.well-known/http-message-signatures-directory"
+    },
+    {
+      "value": "https://agent.example/jwks.json",
+      "fetch": "https://agent.example/jwks.json",
+      "identifier": "https://agent.example/jwks.json"
+    },
+    {
+      "value": "https://agent.example/keys?rev=3",
+      "fetch": "https://agent.example/keys?rev=3",
+      "identifier": "https://agent.example/keys"
+    },
+    {
+      "value": "https://agent.example/?rev=3",
+      "fetch": "https://agent.example/?rev=3",
+      "identifier": "https://agent.example/"
+    },
+    {
+      "value": "https://agent.example/a/../keys",
+      "fetch": "https://agent.example/a/../keys",
+      "identifier": "https://agent.example/keys"
+    },
+    {
+      "value": "https://agent.example/k%c3%a9ys",
+      "fetch": "https://agent.example/k%c3%a9ys",
+      "identifier": "https://agent.example/k%C3%A9ys"
+    },
+    {
+      "value": "https://agent.example/keys#frag",
+      "fetch": "https://agent.example/keys",
+      "identifier": "https://agent.example/keys"
+    },
+    {
+      "value": "https://agent.example:8443/keys",
+      "fetch": "https://agent.example:8443/keys",
+      "identifier": "https://agent.example:8443/keys"
+    },
+    {
+      "value": "https://crawler.agent.example/jwks.json",
+      "fetch": "https://crawler.agent.example/jwks.json",
+      "identifier": "https://crawler.agent.example/jwks.json"
+    }
+  ],
+  "invalid": [
+    "http://agent.example",
+    "https:///keys",
+    "https://user:pw@agent.example/keys",
+    "agent.example",
+    ""
+  ]
+}

packages/webdecoy/src/agent/signature.ts

@@ -12,7 +12,8 @@
  * `atob`, and `URL` are used.
  */
 
-import { parseDictionary, type DictEntry, type BareItem } from './sfv';
+import { parseDictionary, parseBareString, type DictEntry, type BareItem } from './sfv';
+import { resolveSignatureAgent } from './identity';
 
 const TAG = 'web-bot-auth';
 const encoder = new TextEncoder();
@@ -33,8 +34,12 @@ export interface VerificationKey {
   exp?: number;
 }
 
-/** Resolves a JWK-thumbprint keyid to a verification key, or undefined if unknown. */
-export type KeyResolver = (keyId: string) => Promise<VerificationKey | undefined>;
+/**
+ * Resolves a JWK-thumbprint keyid, as published by one agent identity (a
+ * resolved Signature-Agent URL), to a verification key, or undefined if that
+ * identity publishes no such key.
+ */
+export type KeyResolver = (keyId: string, agent: string) => Promise<VerificationKey | undefined>;
 
 export type CoreOutcome = 'none' | 'verified' | 'impersonation' | 'claimed';
 
@@ -116,7 +121,18 @@ export async function verifyWebBotAuth(
     // Must bind the host, per the profile.
     if (!m.components.some((c) => c.name === '@authority' || c.name === '@target-uri')) continue;
 
-    const key = await resolveKey(keyid);
+    // The key is looked up under the Signature-Agent identity this signature
+    // covers, so a key reused by two directories never borrows the other's
+    // name. A signature that binds no identity cannot be attributed.
+    let agent: string;
+    try {
+      agent = coveredSignatureAgent(req, m);
+    } catch {
+      best = higher(best, { outcome: 'claimed', keyId: keyid, reason: 'unbound-agent' });
+      continue;
+    }
+
+    const key = await resolveKey(keyid, agent);
     if (!key) {
       // Unknown agent — a claim we can't confirm or refute.
       best = higher(best, { outcome: 'claimed', keyId: keyid, reason: 'unknown-key' });
@@ -248,6 +264,35 @@ function dictMemberValue(header: string, key: string): string {
   return found.raw;
 }
 
+/**
+ * The identity of the Signature-Agent member a signature covers. Draft 01
+ * §5.2.1: members are https URLs, a member with parameters MUST be ignored,
+ * and the legacy bare-string form is still accepted for deployed signers.
+ */
+function coveredSignatureAgent(req: NormalizedRequest, m: SigInputMember): string {
+  const header = req.headers.get('signature-agent');
+  if (header === null) throw new Error('missing signature-agent');
+  const component = m.components.find((c) => c.name.toLowerCase() === 'signature-agent');
+  if (!component) throw new Error('signature-agent is not covered');
+  const trimmed = header.trim();
+  let value: string;
+  if (trimmed.startsWith('"')) {
+    if (component.keyParam !== undefined) throw new Error('legacy string cannot select a member');
+    value = parseBareString(trimmed);
+  } else {
+    // Covering the whole dictionary would not say which member is ours.
+    if (component.keyParam === undefined) throw new Error('dictionary member is not covered');
+    const member = parseDictionary(trimmed).find((d) => d.key === component.keyParam);
+    const item = member?.items[0];
+    if (!member || member.isList || member.items.length !== 1 || typeof item?.value !== 'string') {
+      throw new Error('signature-agent member is not a string');
+    }
+    if (item.params.size > 0) throw new Error('signature-agent member carries parameters');
+    value = item.value;
+  }
+  return resolveSignatureAgent(value).identifier;
+}
+
 // --- header parsing ---
 
 function parseSignatureInput(value: string): SigInputMember[] {