페르소나 소스팩 동기화 정리
This commit is contained in:
Yun Chan
parent
6a81ec596c
commit
e8e08935ed
10 changed files with 1126 additions and 283 deletions
|
|
@ -43,6 +43,10 @@ class NotConfigured(RuntimeError):
|
|||
"""
|
||||
|
||||
|
||||
class IndexPolicyViolation(ValueError):
|
||||
"""Index request violates RAG source isolation policy."""
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# 1. 정책 4-튜플 (설계서 §4.3) — 사전필터 + 회수가중치 + 리랭킹목표 + 주입방식
|
||||
# 3-AI 가 *같은 물리 테이블 kb.chunk* 를 다른 정책으로 검색한다.
|
||||
|
|
@ -149,6 +153,34 @@ class RetrievalResult:
|
|||
degraded: bool = False # reranker/embed fallback 여부(투명성)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class EpisodicTurnInput:
|
||||
"""app.turn_embedding writer input.
|
||||
|
||||
Only masked, client-visible client utterances should be passed here. Raw text,
|
||||
evaluator payloads, counselor turns, and CCD/answer-key material are not part of
|
||||
this contract.
|
||||
"""
|
||||
|
||||
turn_id: str
|
||||
session_id: str
|
||||
case_id: str
|
||||
seq: int
|
||||
text_masked: str
|
||||
speaker: str = "client"
|
||||
visible_to: Sequence[str] = ("client", "counselor", "evaluator")
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class EpisodicEmbeddingWriteResult:
|
||||
"""Best-effort turn_embedding writer result."""
|
||||
|
||||
candidates: int
|
||||
inserted: int
|
||||
skipped: int
|
||||
degraded: bool = False
|
||||
|
||||
|
||||
# CRAG 게이트 임계값(설계서 §3.7 top1_score). 미달이면 호출부가 "관찰 프레이밍"으로 다운그레이드.
|
||||
# 주: 임의 가정값 — Phase 3 파일럿에서 분포 측정 후 확정(M14, "검증됨" 금지).
|
||||
CRAG_TOP1_THRESHOLD = 0.35
|
||||
|
|
@ -572,6 +604,91 @@ ORDER BY d.s_dense DESC
|
|||
LIMIT $4
|
||||
"""
|
||||
|
||||
_TURN_EMBEDDING_INSERT_SQL = """
|
||||
INSERT INTO app.turn_embedding
|
||||
(turn_id, case_id, session_id, seq, dense, sparse, context_prefix)
|
||||
VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5::vector, $6::jsonb, $7)
|
||||
ON CONFLICT (turn_id) DO NOTHING
|
||||
"""
|
||||
|
||||
|
||||
def episodic_turn_inputs_from_records(
|
||||
*,
|
||||
session_id: str,
|
||||
case_id: str,
|
||||
turns: Sequence[Any],
|
||||
) -> list[EpisodicTurnInput]:
|
||||
"""Build safe episodic embedding inputs from TurnRecord-like objects.
|
||||
|
||||
The writer is intentionally narrow: only client speaker turns that are visible
|
||||
to the client, have a DB turn_id, and have non-empty masked text are eligible.
|
||||
Raw `text`, evaluations, counselor messages, and evaluator-only turns are
|
||||
ignored by construction.
|
||||
"""
|
||||
inputs: list[EpisodicTurnInput] = []
|
||||
for turn in turns:
|
||||
turn_id = getattr(turn, "turn_id", None)
|
||||
text_masked = str(getattr(turn, "text_masked", "") or "").strip()
|
||||
visible_to = tuple(getattr(turn, "visible_to", ()) or ())
|
||||
if not turn_id or not text_masked:
|
||||
continue
|
||||
if getattr(turn, "speaker", None) != "client":
|
||||
continue
|
||||
if "client" not in visible_to:
|
||||
continue
|
||||
inputs.append(
|
||||
EpisodicTurnInput(
|
||||
turn_id=str(turn_id),
|
||||
session_id=session_id,
|
||||
case_id=case_id,
|
||||
seq=int(getattr(turn, "turn_seq", 0) or 0),
|
||||
text_masked=text_masked,
|
||||
speaker="client",
|
||||
visible_to=visible_to,
|
||||
)
|
||||
)
|
||||
return inputs
|
||||
|
||||
|
||||
async def write_persona_turn_embeddings(
|
||||
conn: "asyncpg.Connection",
|
||||
*,
|
||||
turns: Sequence[EpisodicTurnInput],
|
||||
) -> EpisodicEmbeddingWriteResult:
|
||||
"""Index masked client-visible client utterances into app.turn_embedding.
|
||||
|
||||
This is a technical writer only. It does not infer relationship or clinical
|
||||
facts, and it does not store raw text. Missing BGE-M3/pgvector is reported as
|
||||
NotConfigured so callers can skip without breaking session persistence.
|
||||
"""
|
||||
candidates = len(turns)
|
||||
inserted = 0
|
||||
for turn in turns:
|
||||
if turn.speaker != "client" or "client" not in tuple(turn.visible_to or ()):
|
||||
continue
|
||||
text = turn.text_masked.strip()
|
||||
if not text:
|
||||
continue
|
||||
eq = await asyncio.to_thread(embed_query, text)
|
||||
result = await conn.execute(
|
||||
_TURN_EMBEDDING_INSERT_SQL,
|
||||
turn.turn_id,
|
||||
turn.case_id,
|
||||
turn.session_id,
|
||||
int(turn.seq),
|
||||
_vector_literal(eq.dense),
|
||||
json.dumps(eq.sparse),
|
||||
None,
|
||||
)
|
||||
if isinstance(result, str) and result.endswith(" 1"):
|
||||
inserted += 1
|
||||
return EpisodicEmbeddingWriteResult(
|
||||
candidates=candidates,
|
||||
inserted=inserted,
|
||||
skipped=max(0, candidates - inserted),
|
||||
degraded=False,
|
||||
)
|
||||
|
||||
|
||||
async def retrieve_persona_memory(
|
||||
conn: "asyncpg.Connection",
|
||||
|
|
@ -694,6 +811,40 @@ def _content_hash(chunks: list[dict[str, Any]]) -> str:
|
|||
return h.hexdigest()
|
||||
|
||||
|
||||
def _truthy_meta_flag(meta: Any, *keys: str) -> bool:
|
||||
if not isinstance(meta, dict):
|
||||
return False
|
||||
for key in keys:
|
||||
if bool(meta.get(key)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _validate_index_chunks(req: IndexRequest) -> None:
|
||||
"""Fail closed before raw source artifacts can enter kb.chunk.
|
||||
|
||||
`sensitivity=3` means 원천격리. Those materials must stay outside the
|
||||
embedding/FTS index because kb.chunk is retrievable by design.
|
||||
"""
|
||||
for index, chunk in enumerate(req.chunks):
|
||||
meta = chunk.get("meta") or {}
|
||||
sensitivity = chunk.get("sensitivity")
|
||||
try:
|
||||
sensitivity_int = int(sensitivity) if sensitivity is not None else 0
|
||||
except (TypeError, ValueError):
|
||||
sensitivity_int = 0
|
||||
if sensitivity_int >= 3 or _truthy_meta_flag(
|
||||
meta,
|
||||
"raw_source",
|
||||
"raw_source_artifact",
|
||||
"raw_source_isolated",
|
||||
):
|
||||
raise IndexPolicyViolation(
|
||||
"raw source artifacts must not be indexed in kb.chunk "
|
||||
f"(source_id={req.source_id}, chunk_index={index})"
|
||||
)
|
||||
|
||||
|
||||
async def index_document(
|
||||
conn: "asyncpg.Connection",
|
||||
req: IndexRequest,
|
||||
|
|
@ -711,6 +862,7 @@ async def index_document(
|
|||
|
||||
Raises: NotConfigured — DB(kb 스키마/vector) 미가용.
|
||||
"""
|
||||
_validate_index_chunks(req)
|
||||
content_hash = req.content_hash or _content_hash(req.chunks)
|
||||
|
||||
# (1) 증분 — 동일 source/uri/version 활성본의 content_hash 비교
|
||||
|
|
@ -825,14 +977,19 @@ async def index_document(
|
|||
|
||||
__all__ = [
|
||||
"NotConfigured",
|
||||
"IndexPolicyViolation",
|
||||
"AIRole",
|
||||
"RetrievalPolicy",
|
||||
"POLICIES",
|
||||
"RetrievedChunk",
|
||||
"RetrievalResult",
|
||||
"EpisodicTurnInput",
|
||||
"EpisodicEmbeddingWriteResult",
|
||||
"CRAG_TOP1_THRESHOLD",
|
||||
"EmbeddedQuery",
|
||||
"embed_query",
|
||||
"episodic_turn_inputs_from_records",
|
||||
"write_persona_turn_embeddings",
|
||||
"apply_contextual_prefix",
|
||||
"search_kb",
|
||||
"retrieve_persona_memory",
|
||||
|
|
|
|||
194
apps/api/app/services/source_pack_sync.py
Normal file
194
apps/api/app/services/source_pack_sync.py
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
"""Repo-managed source pack sync helpers.
|
||||
|
||||
This keeps CLI and admin API sync behavior on the same content_hash/version
|
||||
path. The source pack content itself stays in data files owned outside this
|
||||
service.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from . import live_coach, rag
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RepoSourcePackSyncItem:
|
||||
source_id: str
|
||||
doc_uri: str
|
||||
previous_version: int | None
|
||||
new_version: int
|
||||
content_hash: str
|
||||
doc_id: int | None
|
||||
chunks_indexed: int
|
||||
skipped_unchanged: bool
|
||||
embedded: bool
|
||||
degraded: bool = False
|
||||
applied: bool = False
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RepoSourcePackSyncResult:
|
||||
sources_upserted: int
|
||||
manifest_count: int
|
||||
chunks_indexed: int
|
||||
skipped_unchanged: int
|
||||
embedded: bool
|
||||
degraded: bool
|
||||
applied: bool
|
||||
items: list[RepoSourcePackSyncItem]
|
||||
|
||||
|
||||
def build_repo_source_pack_manifest() -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
"""Return repo-managed source rows and RAG index payloads without DB access."""
|
||||
|
||||
return live_coach.build_rag_source_rows(), live_coach.build_rag_index_payloads()
|
||||
|
||||
|
||||
def _latest_version(row: Any | None) -> int | None:
|
||||
if not row:
|
||||
return None
|
||||
value = row["version"]
|
||||
return int(value) if value is not None else None
|
||||
|
||||
|
||||
async def _latest_active_document(conn: Any, *, source_id: str, doc_uri: str) -> Any | None:
|
||||
return await conn.fetchrow(
|
||||
"""
|
||||
SELECT doc_id, version, content_hash
|
||||
FROM kb.document
|
||||
WHERE source_id = $1 AND doc_uri = $2 AND is_active
|
||||
ORDER BY version DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
source_id,
|
||||
doc_uri,
|
||||
)
|
||||
|
||||
|
||||
async def _upsert_source_rows(conn: Any, source_rows: list[dict[str, Any]]) -> None:
|
||||
for row in source_rows:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO kb.source
|
||||
(source_id, title, kb_kind, license_class, origin_path, citation, external_llm_ok)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (source_id) DO UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
kb_kind = EXCLUDED.kb_kind,
|
||||
license_class = EXCLUDED.license_class,
|
||||
origin_path = EXCLUDED.origin_path,
|
||||
citation = EXCLUDED.citation,
|
||||
external_llm_ok = EXCLUDED.external_llm_ok
|
||||
""",
|
||||
row["source_id"],
|
||||
row["title"],
|
||||
row["kb_kind"],
|
||||
row["license_class"],
|
||||
row["origin_path"],
|
||||
row["citation"],
|
||||
row["external_llm_ok"],
|
||||
)
|
||||
|
||||
|
||||
async def sync_repo_source_packs(conn: Any, *, apply: bool = False) -> RepoSourcePackSyncResult:
|
||||
"""Compare or apply repo-managed source packs against kb.document.
|
||||
|
||||
Dry-run mode still reads the active DB document row so it can report whether
|
||||
the next apply would skip or materialize a new document version. DB writes
|
||||
happen only when apply=True.
|
||||
"""
|
||||
|
||||
source_rows, index_payloads = build_repo_source_pack_manifest()
|
||||
source_row_by_id = {row["source_id"]: row for row in source_rows}
|
||||
if apply:
|
||||
await _upsert_source_rows(conn, source_rows)
|
||||
|
||||
items: list[RepoSourcePackSyncItem] = []
|
||||
for payload in index_payloads:
|
||||
source_id = str(payload["source_id"])
|
||||
if source_id not in source_row_by_id:
|
||||
continue
|
||||
doc_uri = str(payload["doc_uri"])
|
||||
content_hash = str(payload["content_hash"])
|
||||
latest = await _latest_active_document(conn, source_id=source_id, doc_uri=doc_uri)
|
||||
previous_version = _latest_version(latest)
|
||||
requested_version = max(1, int(payload.get("version") or 1))
|
||||
new_version = requested_version
|
||||
if previous_version is not None:
|
||||
new_version = max(requested_version, previous_version + 1)
|
||||
|
||||
if latest and latest["content_hash"] == content_hash:
|
||||
items.append(
|
||||
RepoSourcePackSyncItem(
|
||||
source_id=source_id,
|
||||
doc_uri=doc_uri,
|
||||
previous_version=previous_version,
|
||||
new_version=previous_version or requested_version,
|
||||
content_hash=content_hash,
|
||||
doc_id=int(latest["doc_id"]),
|
||||
chunks_indexed=0,
|
||||
skipped_unchanged=True,
|
||||
embedded=False,
|
||||
degraded=False,
|
||||
applied=apply,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
if not apply:
|
||||
items.append(
|
||||
RepoSourcePackSyncItem(
|
||||
source_id=source_id,
|
||||
doc_uri=doc_uri,
|
||||
previous_version=previous_version,
|
||||
new_version=new_version,
|
||||
content_hash=content_hash,
|
||||
doc_id=None,
|
||||
chunks_indexed=0,
|
||||
skipped_unchanged=False,
|
||||
embedded=False,
|
||||
degraded=False,
|
||||
applied=False,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
index_payload = dict(payload)
|
||||
index_payload["version"] = new_version
|
||||
result = await rag.index_document(conn, rag.IndexRequest(**index_payload))
|
||||
items.append(
|
||||
RepoSourcePackSyncItem(
|
||||
source_id=source_id,
|
||||
doc_uri=doc_uri,
|
||||
previous_version=previous_version,
|
||||
new_version=new_version,
|
||||
content_hash=content_hash,
|
||||
doc_id=result.doc_id,
|
||||
chunks_indexed=result.chunks_indexed,
|
||||
skipped_unchanged=result.skipped_unchanged,
|
||||
embedded=result.embedded,
|
||||
degraded=result.degraded,
|
||||
applied=True,
|
||||
)
|
||||
)
|
||||
|
||||
return RepoSourcePackSyncResult(
|
||||
sources_upserted=len(source_rows) if apply else 0,
|
||||
manifest_count=len(index_payloads),
|
||||
chunks_indexed=sum(item.chunks_indexed for item in items),
|
||||
skipped_unchanged=sum(1 for item in items if item.skipped_unchanged),
|
||||
embedded=all(item.embedded for item in items) if items else False,
|
||||
degraded=any(item.degraded for item in items),
|
||||
applied=apply,
|
||||
items=items,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RepoSourcePackSyncItem",
|
||||
"RepoSourcePackSyncResult",
|
||||
"build_repo_source_pack_manifest",
|
||||
"sync_repo_source_packs",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue