fix: red-team round 3 hardening across desktop, mobile, core and server
Batch of red-team r3 fixes that were in the working tree before the 2026-09-28 design overhaul, committed as one unit with their tests. - desktop main: STT timeouts and sidecar, voice recording store, sync (credentials, audio, knowledge reindex, push gates), runtime provisioner, update policy, AltGr keybindings, voice-command policy, dictionary file codec/limits, meeting transcript condensing and a local recording ledger so interrupted-session recovery only closes meetings this device recorded (a phone's live meeting is left alone). - mobile: login CSRF via implicit token callbacks rejected, account deletion/retention, durable queue retention, knowledge realtime without unfiltered DELETE, meeting re-record failure paths, cloud STT client, preferences store/resync. - core: text chunking splits long unbroken transcripts to fit, template field policy, dictionary limits, meeting markdown inline handling. - server: payple webhook policy and cancellation order scope, meeting document generation quota, team RPC null-role guard, unified LLM quota in-flight accounting, knowledge chunk vector index, meeting re-record failure paths (migrations 20260929*). - ci: portable/runtime feed gates, update-policy schema, Forgejo file delete and alias planning. Four older tests are updated to the new contracts rather than the old behavior: token-pair auth callbacks are rejected, knowledge realtime no longer subscribes to DELETE, long transcript lines are split, and meeting recovery requires the local recording ledger for empty rows.
This commit is contained in:
Yun Chan
parent
2428ede03d
commit
ba9ef9741e
161 changed files with 17056 additions and 2379 deletions
|
|
@ -6,6 +6,13 @@
|
|||
-- provider call before commit rejected N-1 of them. A 'processing' request now
|
||||
-- holds one unit of the allowance until it is failed, committed or its lease
|
||||
-- expires.
|
||||
--
|
||||
-- Regression (red-team r3-13, 20260929020000): the held unit lived in a second
|
||||
-- ledger ('processing' rows) under a second advisory-lock key, so llm-proxy's
|
||||
-- reserve_llm_quota could take the same last unit while a document was being
|
||||
-- generated, and commit then threw the paid generation away with
|
||||
-- generation_quota_exceeded. The claim now reserves through
|
||||
-- reserve_llm_quota; commit/fail settle that reservation.
|
||||
|
||||
BEGIN;
|
||||
|
||||
|
|
@ -34,6 +41,26 @@ EXCEPTION WHEN OTHERS THEN
|
|||
END;
|
||||
$$;
|
||||
|
||||
-- Returns the commit payload, or {"error": SQLERRM} when the commit raises.
|
||||
CREATE OR REPLACE FUNCTION pg_temp.try_commit(p_actor uuid, p_key uuid)
|
||||
RETURNS jsonb
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
RETURN public.commit_meeting_document_generation_v1(p_actor, p_key, 'Generated body', 10, 1, 1);
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
RETURN jsonb_build_object('error', SQLERRM);
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION pg_temp.usage_today(p_actor uuid, p_feature text)
|
||||
RETURNS integer
|
||||
LANGUAGE sql
|
||||
AS $$
|
||||
SELECT coalesce(sum(count), 0)::integer FROM public.daily_usage
|
||||
WHERE user_id = p_actor AND date = CURRENT_DATE AND feature = p_feature;
|
||||
$$;
|
||||
|
||||
INSERT INTO auth.users (
|
||||
id, aud, role, email, encrypted_password, email_confirmed_at,
|
||||
raw_app_meta_data, raw_user_meta_data, created_at, updated_at
|
||||
|
|
@ -47,19 +74,28 @@ INSERT INTO auth.users (
|
|||
'37000000-0000-4000-8000-000000000002', 'authenticated', 'authenticated',
|
||||
'meeting-doc-quota-unlimited@example.invalid', crypt('fixture-password', gen_salt('bf')), now(),
|
||||
'{"provider":"email","providers":["email"]}'::jsonb, '{}'::jsonb, now(), now()
|
||||
),
|
||||
(
|
||||
'37000000-0000-4000-8000-000000000003', 'authenticated', 'authenticated',
|
||||
'meeting-doc-quota-pro@example.invalid', crypt('fixture-password', gen_salt('bf')), now(),
|
||||
'{"provider":"email","providers":["email"]}'::jsonb, '{}'::jsonb, now(), now()
|
||||
);
|
||||
|
||||
UPDATE public.subscriptions SET tier = 'free', status = 'active', overage_credits = 0
|
||||
WHERE user_id = '37000000-0000-4000-8000-000000000001';
|
||||
UPDATE public.subscriptions SET tier = 'pro_plus', status = 'active', overage_credits = 0
|
||||
WHERE user_id = '37000000-0000-4000-8000-000000000002';
|
||||
UPDATE public.subscriptions SET tier = 'pro', status = 'active', overage_credits = 0
|
||||
WHERE user_id = '37000000-0000-4000-8000-000000000003';
|
||||
|
||||
INSERT INTO public.meetings (id, user_id, title, status, raw_transcript)
|
||||
VALUES
|
||||
('37100000-0000-4000-8000-000000000001', '37000000-0000-4000-8000-000000000001',
|
||||
'Quota fixture meeting', 'completed', 'Speaker one talked about the roadmap.'),
|
||||
('37100000-0000-4000-8000-000000000002', '37000000-0000-4000-8000-000000000002',
|
||||
'Unlimited fixture meeting', 'completed', 'Speaker two talked about hiring.');
|
||||
'Unlimited fixture meeting', 'completed', 'Speaker two talked about hiring.'),
|
||||
('37100000-0000-4000-8000-000000000003', '37000000-0000-4000-8000-000000000003',
|
||||
'Pro fixture meeting', 'completed', 'Speaker three talked about the launch.');
|
||||
|
||||
INSERT INTO public.user_templates (
|
||||
id, user_id, template_kind, name, template_type, system_prompt, is_builtin
|
||||
|
|
@ -67,7 +103,9 @@ INSERT INTO public.user_templates (
|
|||
('37200000-0000-4000-8000-000000000001', '37000000-0000-4000-8000-000000000001',
|
||||
'meeting_document', 'Quota fixture template', 'custom', 'Summarize the meeting.', false),
|
||||
('37200000-0000-4000-8000-000000000002', '37000000-0000-4000-8000-000000000002',
|
||||
'meeting_document', 'Unlimited fixture template', 'custom', 'Summarize the meeting.', false);
|
||||
'meeting_document', 'Unlimited fixture template', 'custom', 'Summarize the meeting.', false),
|
||||
('37200000-0000-4000-8000-000000000003', '37000000-0000-4000-8000-000000000003',
|
||||
'meeting_document', 'Pro fixture template', 'custom', 'Summarize the meeting.', false);
|
||||
|
||||
-- One weekly Haiku unit left for the free user.
|
||||
INSERT INTO public.daily_usage (user_id, date, feature, count)
|
||||
|
|
@ -83,6 +121,7 @@ DECLARE
|
|||
parallel jsonb;
|
||||
replay jsonb;
|
||||
after_release jsonb;
|
||||
committed jsonb;
|
||||
after_commit jsonb;
|
||||
overage_claim jsonb;
|
||||
overage_parallel jsonb;
|
||||
|
|
@ -105,17 +144,21 @@ BEGIN
|
|||
replay->>'error' IS NULL AND NOT (replay->>'claimed')::boolean AND replay->>'status' = 'processing',
|
||||
'replaying the in-flight key reports processing: ' || replay::text
|
||||
);
|
||||
PERFORM pg_temp.assert_true(pg_temp.usage_today(actor, 'llm_haiku') = 250,
|
||||
'a replay takes no additional unit');
|
||||
|
||||
-- A failed request releases the unit it held.
|
||||
PERFORM public.fail_meeting_document_generation_v1(actor, '37300000-0000-4000-8000-000000000001', 'provider_timeout');
|
||||
PERFORM pg_temp.assert_true(pg_temp.usage_today(actor, 'llm_haiku') = 249,
|
||||
'failure gives the held unit back to the ledger');
|
||||
after_release := pg_temp.try_claim(actor, '37300000-0000-4000-8000-000000000003', meeting, template, haiku);
|
||||
PERFORM pg_temp.assert_true((after_release->>'claimed')::boolean,
|
||||
'failure releases the in-flight unit: ' || after_release::text);
|
||||
|
||||
-- Commit converts the held unit into recorded usage; the allowance is spent.
|
||||
PERFORM public.commit_meeting_document_generation_v1(
|
||||
actor, '37300000-0000-4000-8000-000000000003', 'Generated body', 10, 1, 1
|
||||
);
|
||||
committed := pg_temp.try_commit(actor, '37300000-0000-4000-8000-000000000003');
|
||||
PERFORM pg_temp.assert_true(committed->>'consumedFrom' = 'base',
|
||||
'commit reports the unit the claim held: ' || committed::text);
|
||||
SELECT count INTO usage_count FROM public.daily_usage
|
||||
WHERE user_id = actor AND date = CURRENT_DATE AND feature = 'llm_haiku';
|
||||
PERFORM pg_temp.assert_true(usage_count = 250, 'commit records exactly one unit');
|
||||
|
|
@ -132,10 +175,13 @@ BEGIN
|
|||
PERFORM pg_temp.assert_true(overage_parallel->>'error' = 'generation_quota_exceeded',
|
||||
'a single overage credit is not spent twice by parallel claims: ' || overage_parallel::text);
|
||||
|
||||
-- A crashed request stops holding its unit once its lease expires.
|
||||
UPDATE public.meeting_document_generation_requests
|
||||
SET created_at = now() - interval '11 minutes'
|
||||
WHERE user_id = actor AND idempotency_key = '37300000-0000-4000-8000-000000000005';
|
||||
-- A crashed request stops holding its unit once its reservation lease expires.
|
||||
UPDATE public.llm_quota_reservations AS reservation
|
||||
SET lease_expires_at = now() - interval '1 minute'
|
||||
FROM public.meeting_document_generation_requests AS request
|
||||
WHERE request.user_id = actor
|
||||
AND request.idempotency_key = '37300000-0000-4000-8000-000000000005'
|
||||
AND reservation.id = request.llm_reservation_id;
|
||||
after_lease := pg_temp.try_claim(actor, '37300000-0000-4000-8000-000000000007', meeting, template, haiku);
|
||||
PERFORM pg_temp.assert_true((after_lease->>'claimed')::boolean,
|
||||
'an expired in-flight lease no longer holds a unit: ' || after_lease::text);
|
||||
|
|
@ -159,4 +205,146 @@ BEGIN
|
|||
END;
|
||||
$$;
|
||||
|
||||
-- Cross-path (r3-13): a meeting document and llm-proxy share one ledger.
|
||||
DO $$
|
||||
DECLARE
|
||||
actor constant uuid := '37000000-0000-4000-8000-000000000003';
|
||||
meeting constant uuid := '37100000-0000-4000-8000-000000000003';
|
||||
template constant uuid := '37200000-0000-4000-8000-000000000003';
|
||||
opus constant text := 'claude-opus-4-6';
|
||||
claim jsonb;
|
||||
proxy jsonb;
|
||||
committed jsonb;
|
||||
blocked jsonb;
|
||||
legacy jsonb;
|
||||
proxy_reservation constant uuid := '37600000-0000-4000-8000-000000000001';
|
||||
credits integer;
|
||||
BEGIN
|
||||
-- One Opus unit left today (Pro: 50/day).
|
||||
INSERT INTO public.daily_usage (user_id, date, feature, count)
|
||||
VALUES (actor, CURRENT_DATE, 'llm_opus', 49);
|
||||
|
||||
-- 1) Document first, then a Talk/command request through llm-proxy.
|
||||
claim := pg_temp.try_claim(actor, '37500000-0000-4000-8000-000000000001', meeting, template, opus);
|
||||
PERFORM pg_temp.assert_true((claim->>'claimed')::boolean,
|
||||
'the document claim takes the last Opus unit: ' || claim::text);
|
||||
PERFORM pg_temp.assert_true(pg_temp.usage_today(actor, 'llm_opus') = 50,
|
||||
'the claimed unit is visible in daily_usage while the provider call runs');
|
||||
|
||||
proxy := public.reserve_llm_quota(actor, gen_random_uuid(), 'llm_opus', 50, 'daily');
|
||||
PERFORM pg_temp.assert_true((proxy->>'allowed')::boolean IS FALSE,
|
||||
'llm-proxy cannot take the unit held by an in-flight document: ' || proxy::text);
|
||||
PERFORM pg_temp.assert_true(
|
||||
(SELECT count(*) FROM public.meeting_documents WHERE user_id = actor) = 0,
|
||||
'no document exists before commit'
|
||||
);
|
||||
|
||||
committed := pg_temp.try_commit(actor, '37500000-0000-4000-8000-000000000001');
|
||||
PERFORM pg_temp.assert_true(
|
||||
committed->>'error' IS NULL AND committed->>'consumedFrom' = 'base',
|
||||
'the paid Opus generation is committed with the base unit the claim held: ' || committed::text
|
||||
);
|
||||
PERFORM pg_temp.assert_true(pg_temp.usage_today(actor, 'llm_opus') = 50,
|
||||
'commit does not count the unit a second time');
|
||||
|
||||
-- consume_quota sees the same ledger.
|
||||
PERFORM pg_temp.assert_true(
|
||||
(public.consume_quota(actor, 'llm_opus', 50, 'daily')->>'allowed')::boolean IS FALSE,
|
||||
'consume_quota counts document units'
|
||||
);
|
||||
|
||||
-- 2) llm-proxy first, then a document: the claim sees the proxy's unit.
|
||||
UPDATE public.daily_usage SET count = 49
|
||||
WHERE user_id = actor AND date = CURRENT_DATE AND feature = 'llm_opus';
|
||||
proxy := public.reserve_llm_quota(actor, proxy_reservation, 'llm_opus', 50, 'daily');
|
||||
PERFORM pg_temp.assert_true((proxy->>'allowed')::boolean, 'llm-proxy takes the last unit');
|
||||
blocked := pg_temp.try_claim(actor, '37500000-0000-4000-8000-000000000002', meeting, template, opus);
|
||||
PERFORM pg_temp.assert_true(blocked->>'error' = 'generation_quota_exceeded',
|
||||
'a document claim cannot take the unit held by llm-proxy: ' || blocked::text);
|
||||
PERFORM public.finalize_llm_quota(proxy_reservation, false);
|
||||
|
||||
-- 3) Overage is taken at claim and refunded when the generation fails.
|
||||
UPDATE public.daily_usage SET count = 50
|
||||
WHERE user_id = actor AND date = CURRENT_DATE AND feature = 'llm_opus';
|
||||
UPDATE public.subscriptions SET overage_credits = 1 WHERE user_id = actor;
|
||||
claim := pg_temp.try_claim(actor, '37500000-0000-4000-8000-000000000003', meeting, template, opus);
|
||||
PERFORM pg_temp.assert_true((claim->>'claimed')::boolean, 'an overage credit covers the claim');
|
||||
SELECT overage_credits INTO credits FROM public.subscriptions WHERE user_id = actor;
|
||||
PERFORM pg_temp.assert_true(credits = 0, 'the credit is held at claim time');
|
||||
proxy := public.reserve_llm_quota(actor, gen_random_uuid(), 'llm_opus', 50, 'daily');
|
||||
PERFORM pg_temp.assert_true((proxy->>'allowed')::boolean IS FALSE,
|
||||
'llm-proxy cannot spend the credit held by an in-flight document');
|
||||
PERFORM public.fail_meeting_document_generation_v1(actor, '37500000-0000-4000-8000-000000000003', 'provider_timeout');
|
||||
SELECT overage_credits INTO credits FROM public.subscriptions WHERE user_id = actor;
|
||||
PERFORM pg_temp.assert_true(credits = 1, 'a failed generation refunds the overage credit');
|
||||
PERFORM pg_temp.assert_true(pg_temp.usage_today(actor, 'llm_opus') = 50,
|
||||
'a failed generation gives its unit back');
|
||||
|
||||
-- 4) A row claimed before this migration (no reservation) is still charged
|
||||
-- once, at commit, through the same ledger.
|
||||
UPDATE public.daily_usage SET count = 49
|
||||
WHERE user_id = actor AND date = CURRENT_DATE AND feature = 'llm_opus';
|
||||
UPDATE public.subscriptions SET overage_credits = 0 WHERE user_id = actor;
|
||||
claim := pg_temp.try_claim(actor, '37500000-0000-4000-8000-000000000004', meeting, template, opus);
|
||||
PERFORM pg_temp.assert_true((claim->>'claimed')::boolean, 'legacy fixture claim');
|
||||
-- Simulate the pre-migration shape: no reservation, unit not in daily_usage.
|
||||
PERFORM public.finalize_llm_quota(
|
||||
(SELECT llm_reservation_id FROM public.meeting_document_generation_requests
|
||||
WHERE user_id = actor AND idempotency_key = '37500000-0000-4000-8000-000000000004'),
|
||||
false
|
||||
);
|
||||
UPDATE public.meeting_document_generation_requests SET llm_reservation_id = NULL
|
||||
WHERE user_id = actor AND idempotency_key = '37500000-0000-4000-8000-000000000004';
|
||||
legacy := pg_temp.try_commit(actor, '37500000-0000-4000-8000-000000000004');
|
||||
PERFORM pg_temp.assert_true(legacy->>'consumedFrom' = 'base',
|
||||
'a legacy in-flight row is charged at commit: ' || legacy::text);
|
||||
PERFORM pg_temp.assert_true(pg_temp.usage_today(actor, 'llm_opus') = 50,
|
||||
'a legacy commit charges exactly one unit');
|
||||
PERFORM pg_temp.assert_true(
|
||||
(SELECT llm_reservation_id IS NOT NULL FROM public.meeting_document_generation_requests
|
||||
WHERE user_id = actor AND idempotency_key = '37500000-0000-4000-8000-000000000004'),
|
||||
'the legacy commit records the reservation that charged it'
|
||||
);
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- One lock key for the whole ledger: every quota path takes it through
|
||||
-- daily_usage_lock_v1 and no quota function hardcodes an advisory-lock key.
|
||||
-- (fail_meeting_document_generation_v1 locks inside finalize_llm_quota.)
|
||||
DO $$
|
||||
DECLARE
|
||||
offenders text;
|
||||
BEGIN
|
||||
SELECT string_agg(p.proname, ', ' ORDER BY p.proname)
|
||||
INTO offenders
|
||||
FROM pg_proc AS p
|
||||
JOIN pg_namespace AS n ON n.oid = p.pronamespace
|
||||
WHERE n.nspname = 'public'
|
||||
AND p.proname IN (
|
||||
'reserve_llm_quota', 'finalize_llm_quota', 'consume_quota',
|
||||
'claim_meeting_document_generation_v1', 'commit_meeting_document_generation_v1',
|
||||
'fail_meeting_document_generation_v1'
|
||||
)
|
||||
AND (
|
||||
p.prosrc LIKE '%pg_advisory%'
|
||||
OR (
|
||||
p.proname <> 'fail_meeting_document_generation_v1'
|
||||
AND p.prosrc NOT LIKE '%daily_usage_lock_v1%'
|
||||
)
|
||||
OR (
|
||||
p.proname = 'fail_meeting_document_generation_v1'
|
||||
AND p.prosrc NOT LIKE '%finalize_llm_quota%'
|
||||
)
|
||||
);
|
||||
PERFORM pg_temp.assert_true(offenders IS NULL,
|
||||
'quota functions must lock only through daily_usage_lock_v1: ' || coalesce(offenders, ''));
|
||||
|
||||
PERFORM pg_temp.assert_true(
|
||||
NOT has_function_privilege('anon', 'public.daily_usage_lock_v1(uuid, text)', 'EXECUTE')
|
||||
AND NOT has_function_privilege('authenticated', 'public.daily_usage_lock_v1(uuid, text)', 'EXECUTE'),
|
||||
'the ledger lock is not callable by clients'
|
||||
);
|
||||
END;
|
||||
$$;
|
||||
|
||||
ROLLBACK;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue