347 lines
13 KiB
JavaScript
347 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
|
import { readFileSync } from "node:fs";
|
|
import { resolve } from "node:path";
|
|
import assert from "node:assert/strict";
|
|
|
|
const DEFAULT_FIXTURE = "apps/api/engine_gateway/golden/engine_gateway_contract.v1.json";
|
|
const DEFAULT_SCHEMA = "apps/api/engine_gateway/golden/engine_gateway_schema.v1.json";
|
|
|
|
function usage() {
|
|
return [
|
|
"Usage: node scripts/check-engine-gateway-contract.mjs [--fixture path] [--schema path] [--json]",
|
|
"",
|
|
"Validates the engine gateway v1 golden fixture without importing Python code.",
|
|
].join("\n");
|
|
}
|
|
|
|
function parseArgs(argv) {
|
|
const args = {
|
|
fixture: DEFAULT_FIXTURE,
|
|
schema: DEFAULT_SCHEMA,
|
|
json: false,
|
|
};
|
|
|
|
for (let i = 0; i < argv.length; i += 1) {
|
|
const arg = argv[i];
|
|
if (arg === "--fixture") {
|
|
args.fixture = argv[i + 1];
|
|
i += 1;
|
|
} else if (arg === "--schema") {
|
|
args.schema = argv[i + 1];
|
|
i += 1;
|
|
} else if (arg === "--json") {
|
|
args.json = true;
|
|
} else if (arg === "--help" || arg === "-h") {
|
|
console.log(usage());
|
|
process.exit(0);
|
|
} else {
|
|
throw new Error(`Unknown argument: ${arg}\n${usage()}`);
|
|
}
|
|
}
|
|
|
|
if (!args.fixture || !args.schema) {
|
|
throw new Error(`Missing path argument.\n${usage()}`);
|
|
}
|
|
return args;
|
|
}
|
|
|
|
function readJson(path) {
|
|
return JSON.parse(readFileSync(resolve(path), "utf8"));
|
|
}
|
|
|
|
function assertPlainObject(value, label) {
|
|
assert.equal(typeof value, "object", `${label} must be an object`);
|
|
assert.notEqual(value, null, `${label} must not be null`);
|
|
assert.equal(Array.isArray(value), false, `${label} must not be an array`);
|
|
}
|
|
|
|
function assertOnlyKeys(value, allowedKeys, label) {
|
|
for (const key of Object.keys(value)) {
|
|
assert.ok(allowedKeys.includes(key), `${label}.${key} is not in the v1 contract`);
|
|
}
|
|
}
|
|
|
|
function assertRequired(value, requiredKeys, label) {
|
|
for (const key of requiredKeys) {
|
|
assert.ok(Object.hasOwn(value, key), `${label}.${key} is required`);
|
|
}
|
|
}
|
|
|
|
function assertNullableObject(value, label) {
|
|
if (value === null || value === undefined) return;
|
|
assertPlainObject(value, label);
|
|
}
|
|
|
|
function assertInteger(value, label) {
|
|
assert.equal(Number.isInteger(value), true, `${label} must be an integer`);
|
|
}
|
|
|
|
function assertNumber(value, label) {
|
|
assert.equal(typeof value, "number", `${label} must be a number`);
|
|
assert.equal(Number.isFinite(value), true, `${label} must be finite`);
|
|
}
|
|
|
|
function assertString(value, label) {
|
|
assert.equal(typeof value, "string", `${label} must be a string`);
|
|
}
|
|
|
|
function assertBoolean(value, label) {
|
|
assert.equal(typeof value, "boolean", `${label} must be a boolean`);
|
|
}
|
|
|
|
function schemaDef(schema, name) {
|
|
const def = schema?.$defs?.[name];
|
|
assertPlainObject(def, `$defs.${name}`);
|
|
return def;
|
|
}
|
|
|
|
function resolveSchemaRef(schema, ref, label) {
|
|
assertString(ref, label);
|
|
const prefix = "#/$defs/";
|
|
assert.ok(ref.startsWith(prefix), `${label} must reference a schema definition`);
|
|
return schemaDef(schema, ref.slice(prefix.length));
|
|
}
|
|
|
|
function streamEventSpecs(schema) {
|
|
const packetDef = schemaDef(schema, "StreamPacket");
|
|
assert.ok(Array.isArray(packetDef.oneOf), "$defs.StreamPacket.oneOf must be an array");
|
|
const specs = new Map();
|
|
for (const [index, variant] of packetDef.oneOf.entries()) {
|
|
assertPlainObject(variant, `$defs.StreamPacket.oneOf[${index}]`);
|
|
const event = variant.properties?.event?.const;
|
|
assertString(event, `$defs.StreamPacket.oneOf[${index}].properties.event.const`);
|
|
const payloadRef = variant.properties?.payload?.$ref;
|
|
const payloadDef = resolveSchemaRef(
|
|
schema,
|
|
payloadRef,
|
|
`$defs.StreamPacket.oneOf[${index}].properties.payload.$ref`,
|
|
);
|
|
assert.ok(!specs.has(event), `duplicate stream event spec: ${event}`);
|
|
specs.set(event, { event, payloadDef });
|
|
}
|
|
return specs;
|
|
}
|
|
|
|
function defaultModelSentinel(schema) {
|
|
const value = schema["x-engine-gateway-default-model-sentinel"];
|
|
assertString(value, "x-engine-gateway-default-model-sentinel");
|
|
assert.notEqual(value.trim(), "", "x-engine-gateway-default-model-sentinel must not be blank");
|
|
return value;
|
|
}
|
|
|
|
function assertDefaultModelSentinelUsage(fixture, sentinel) {
|
|
assert.equal(
|
|
fixture.generate_request?.model,
|
|
sentinel,
|
|
"golden generate_request.model must pin the gateway default-routing sentinel",
|
|
);
|
|
assert.notEqual(
|
|
fixture.generate_response?.model,
|
|
sentinel,
|
|
"generate_response.model must report the resolved provider model, not the default-routing sentinel",
|
|
);
|
|
for (const [index, packet] of (fixture.stream_packets ?? []).entries()) {
|
|
if (packet?.event !== "done") continue;
|
|
assert.notEqual(
|
|
packet.payload?.model,
|
|
sentinel,
|
|
`stream_packets[${index}].payload.model must report the resolved provider model`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function assertRootShape(schema, fixture, streamSpecs) {
|
|
assert.deepEqual(schema["x-engine-gateway-sse-events"], [...streamSpecs.keys()]);
|
|
assertDefaultModelSentinelUsage(fixture, defaultModelSentinel(schema));
|
|
assertRequired(fixture, schema.required, "fixture");
|
|
assertOnlyKeys(fixture, Object.keys(schema.properties), "fixture");
|
|
assert.equal(fixture.version, schema.properties.version.const);
|
|
}
|
|
|
|
function assertGenerateRequest(schema, request) {
|
|
const requestDef = schemaDef(schema, "GenerateRequest");
|
|
const messageDef = schemaDef(schema, "EngineMessage");
|
|
const allowedRequestKeys = Object.keys(requestDef.properties);
|
|
const messageRoles = messageDef.properties.role.enum;
|
|
const aiRoles = requestDef.properties.ai_role.enum;
|
|
|
|
assertPlainObject(request, "generate_request");
|
|
assertRequired(request, requestDef.required, "generate_request");
|
|
assertOnlyKeys(request, allowedRequestKeys, "generate_request");
|
|
assert.ok(aiRoles.includes(request.ai_role), "generate_request.ai_role must be a known role");
|
|
assert.ok(Array.isArray(request.messages), "generate_request.messages must be an array");
|
|
assert.ok(request.messages.length > 0, "generate_request.messages must not be empty");
|
|
|
|
request.messages.forEach((message, index) => {
|
|
assertPlainObject(message, `generate_request.messages[${index}]`);
|
|
assertRequired(message, messageDef.required, `generate_request.messages[${index}]`);
|
|
assertOnlyKeys(message, Object.keys(messageDef.properties), `generate_request.messages[${index}]`);
|
|
assert.ok(messageRoles.includes(message.role), `generate_request.messages[${index}].role is invalid`);
|
|
assertString(message.content, `generate_request.messages[${index}].content`);
|
|
if (Object.hasOwn(message, "cache")) {
|
|
assert.equal(typeof message.cache, "boolean", `generate_request.messages[${index}].cache must be boolean`);
|
|
}
|
|
});
|
|
|
|
if (Object.hasOwn(request, "model")) {
|
|
assert.ok(request.model === null || typeof request.model === "string", "generate_request.model must be string or null");
|
|
}
|
|
if (Object.hasOwn(request, "session_id")) {
|
|
assert.ok(
|
|
request.session_id === null || typeof request.session_id === "string",
|
|
"generate_request.session_id must be string or null",
|
|
);
|
|
}
|
|
if (Object.hasOwn(request, "max_tokens")) assertInteger(request.max_tokens, "generate_request.max_tokens");
|
|
if (Object.hasOwn(request, "temperature")) assertNumber(request.temperature, "generate_request.temperature");
|
|
if (Object.hasOwn(request, "structured_schema")) assertNullableObject(request.structured_schema, "generate_request.structured_schema");
|
|
if (Object.hasOwn(request, "metadata")) assertPlainObject(request.metadata, "generate_request.metadata");
|
|
}
|
|
|
|
function assertGenerateResponse(schema, response) {
|
|
const responseDef = schemaDef(schema, "GenerateResponse");
|
|
assertPlainObject(response, "generate_response");
|
|
assertRequired(response, responseDef.required, "generate_response");
|
|
assertOnlyKeys(response, Object.keys(responseDef.properties), "generate_response");
|
|
assertString(response.text, "generate_response.text");
|
|
assertString(response.model, "generate_response.model");
|
|
assertString(response.provider, "generate_response.provider");
|
|
if (Object.hasOwn(response, "tokens_in")) assertInteger(response.tokens_in, "generate_response.tokens_in");
|
|
if (Object.hasOwn(response, "tokens_out")) assertInteger(response.tokens_out, "generate_response.tokens_out");
|
|
if (Object.hasOwn(response, "cost_usd")) assertNumber(response.cost_usd, "generate_response.cost_usd");
|
|
if (Object.hasOwn(response, "inference_geo")) {
|
|
assert.ok(
|
|
response.inference_geo === null || typeof response.inference_geo === "string",
|
|
"generate_response.inference_geo must be string or null",
|
|
);
|
|
}
|
|
if (Object.hasOwn(response, "structured")) assertNullableObject(response.structured, "generate_response.structured");
|
|
}
|
|
|
|
function parseSseFrame(frame, streamSpecs) {
|
|
assertString(frame, "stream_frame");
|
|
let event = null;
|
|
const dataLines = [];
|
|
|
|
for (const line of frame.split(/\r?\n/)) {
|
|
if (line === "") continue;
|
|
if (line.startsWith("event:")) {
|
|
event = line.slice("event:".length).trim();
|
|
} else if (line.startsWith("data:")) {
|
|
dataLines.push(line.slice("data:".length).trimStart());
|
|
}
|
|
}
|
|
|
|
assert.ok(event, "stream frame must contain an event line");
|
|
assert.ok(streamSpecs.has(event), `unknown stream event: ${event}`);
|
|
assert.ok(dataLines.length > 0, `stream event ${event} must contain data`);
|
|
return packetFromEventData(event, dataLines.join("\n"), streamSpecs);
|
|
}
|
|
|
|
function packetFromEventData(event, data, streamSpecs) {
|
|
if (data === "[DONE]") return null;
|
|
|
|
let payload;
|
|
try {
|
|
payload = JSON.parse(data);
|
|
} catch (error) {
|
|
if (event === "token") {
|
|
payload = { text: data };
|
|
} else {
|
|
throw new Error(`stream event ${event} must contain JSON data: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
const spec = streamSpecs.get(event);
|
|
assert.ok(spec, `unknown stream event: ${event}`);
|
|
assertPayloadMatchesSchema(payload, spec.payloadDef, `${event} payload`);
|
|
return { event, payload };
|
|
}
|
|
|
|
function assertPayloadMatchesSchema(payload, payloadDef, label) {
|
|
assertPlainObject(payload, label);
|
|
assertRequired(payload, payloadDef.required ?? [], label);
|
|
const properties = payloadDef.properties ?? {};
|
|
assertOnlyKeys(payload, Object.keys(properties), label);
|
|
for (const [key, spec] of Object.entries(properties)) {
|
|
if (!Object.hasOwn(payload, key)) continue;
|
|
assertValueMatchesSchemaType(payload[key], spec, `${label}.${key}`);
|
|
}
|
|
}
|
|
|
|
function assertValueMatchesSchemaType(value, spec, label) {
|
|
const types = Array.isArray(spec.type) ? spec.type : [spec.type];
|
|
if (value === null && types.includes("null")) return;
|
|
if (types.includes("string")) {
|
|
assertString(value, label);
|
|
} else if (types.includes("integer")) {
|
|
assertInteger(value, label);
|
|
} else if (types.includes("number")) {
|
|
assertNumber(value, label);
|
|
} else if (types.includes("boolean")) {
|
|
assertBoolean(value, label);
|
|
} else if (types.includes("object")) {
|
|
assertPlainObject(value, label);
|
|
} else if (types.includes("array")) {
|
|
assert.ok(Array.isArray(value), `${label} must be an array`);
|
|
} else {
|
|
throw new Error(`${label} has unsupported schema type: ${spec.type}`);
|
|
}
|
|
}
|
|
|
|
function assertStreamContract(fixture, streamSpecs) {
|
|
assert.ok(Array.isArray(fixture.stream_frames), "stream_frames must be an array");
|
|
assert.ok(Array.isArray(fixture.stream_packets), "stream_packets must be an array");
|
|
assert.ok(Array.isArray(fixture.compatibility_lines), "compatibility_lines must be an array");
|
|
|
|
const decodedPackets = [];
|
|
for (const frame of fixture.stream_frames) {
|
|
const packet = parseSseFrame(frame, streamSpecs);
|
|
if (packet) decodedPackets.push(packet);
|
|
}
|
|
assert.deepEqual(decodedPackets, fixture.stream_packets);
|
|
|
|
for (const line of fixture.compatibility_lines) {
|
|
assertString(line, "compatibility line");
|
|
if (line.trim() === "data: [DONE]") continue;
|
|
throw new Error(`unsupported compatibility line: ${line}`);
|
|
}
|
|
}
|
|
|
|
function validateContract(schema, fixture) {
|
|
const streamSpecs = streamEventSpecs(schema);
|
|
assertRootShape(schema, fixture, streamSpecs);
|
|
assertGenerateRequest(schema, fixture.generate_request);
|
|
assertGenerateResponse(schema, fixture.generate_response);
|
|
assertStreamContract(fixture, streamSpecs);
|
|
return {
|
|
version: fixture.version,
|
|
defaultModelSentinel: defaultModelSentinel(schema),
|
|
streamEvents: schema["x-engine-gateway-sse-events"],
|
|
streamFrameCount: fixture.stream_frames.length,
|
|
decodedPacketCount: fixture.stream_packets.length,
|
|
compatibilityLineCount: fixture.compatibility_lines.length,
|
|
};
|
|
}
|
|
|
|
function main() {
|
|
const args = parseArgs(process.argv.slice(2));
|
|
const schema = readJson(args.schema);
|
|
const fixture = readJson(args.fixture);
|
|
const result = validateContract(schema, fixture);
|
|
|
|
if (args.json) {
|
|
console.log(JSON.stringify({ ok: true, ...result }, null, 2));
|
|
} else {
|
|
console.log(
|
|
`engine_gateway_contract.v${result.version} OK: ${result.decodedPacketCount} packets, ${result.compatibilityLineCount} compatibility lines`,
|
|
);
|
|
}
|
|
}
|
|
|
|
try {
|
|
main();
|
|
} catch (error) {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exit(1);
|
|
}
|