407 lines
17 KiB
Python
407 lines
17 KiB
Python
"""지식베이스(KB) 라우트 — 하이브리드 검색 + 인덱싱 트리거(관리자).
|
|
|
|
설계서 §3.6·§4.3 / MASTERPLAN §3.6:
|
|
GET /kb/health — 라우터 + RAG 구성요소 readiness
|
|
POST /kb/search — 정적 지식 하이브리드 검색(정책 4-튜플, visible_to DB 강제)
|
|
POST /kb/eval-grounding — 평가 AI 채점 근거(evaluator 정책, label_id 동봉)
|
|
POST /kb/index — 문서 인덱싱 트리거(관리자, content_hash 증분, 오프라인 배치)
|
|
|
|
정보비대칭은 *DB WHERE* 가 강제한다(services/rag.py POLICIES). 라우트는 role 을
|
|
요청 컨텍스트로만 결정하고, 검색 함수가 정책 화이트리스트를 고정한다(코드경로 부재 1차방어).
|
|
|
|
DB/임베딩 모델 미가용(Docker off, rag 의존성 미설치)이면 rag.NotConfigured → 503 변환.
|
|
무거운 import(FlagEmbedding/torch)는 services/rag.py 가 함수 내부로 가둔다(이식성).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Annotated, Any, Literal, Optional
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, status
|
|
from pydantic import BaseModel, Field
|
|
|
|
from ..db import acquire, get_pool
|
|
from ..deps import AIView, Principal, Role, require_role
|
|
from ..services import rag, source_pack_sync
|
|
|
|
router = APIRouter(prefix="/kb", tags=["kb"])
|
|
|
|
|
|
# ── 요청/응답 모델 ──────────────────────────────────────
|
|
RoleLiteral = Literal["client", "counselor", "evaluator"]
|
|
|
|
|
|
class KBSearchRequest(BaseModel):
|
|
query: str = Field(..., min_length=1) # PII 마스킹된 질의(마스킹은 가드레일 책임)
|
|
role: RoleLiteral = "evaluator" # 검색 주체(정보비대칭 정책 선택)
|
|
k: int = Field(default=5, ge=1, le=50)
|
|
rerank: bool = True
|
|
# 정책 화이트리스트를 *좁히는* 추가 필터만 허용(넓히지 못함 — 정보비대칭 보존)
|
|
kb_kind: Optional[list[str]] = None
|
|
source_id: Optional[list[str]] = None
|
|
sensitivity_max: Optional[int] = Field(default=None, ge=0, le=3)
|
|
# 감사 귀속(선택)
|
|
session_id: Optional[str] = None
|
|
turn_id: Optional[str] = None
|
|
|
|
|
|
class ChunkOut(BaseModel):
|
|
chunk_id: int
|
|
score: float
|
|
kb_kind: str
|
|
heading_path: Optional[str] = None
|
|
context_prefix: Optional[str] = None
|
|
body: Optional[str] = None # expose_body=True(상담사/평가) 정책에서만
|
|
behavior_cue: Optional[str] = None # 내담자 정책: 본문 비노출, 행동단서만(M6)
|
|
label_id: Optional[int] = None # 평가 정책에서만
|
|
meta: dict[str, Any] = Field(default_factory=dict)
|
|
source_id: Optional[str] = None
|
|
|
|
|
|
class KBSearchResponse(BaseModel):
|
|
chunks: list[ChunkOut]
|
|
policy: str
|
|
top1_score: float
|
|
crag_pass: bool # top1 >= 임계(F-06: 미달 시 관찰 프레이밍)
|
|
latency_ms: int
|
|
degraded: bool = False # reranker/embed 폴백 투명성
|
|
|
|
|
|
class MemoryRecallRequest(BaseModel):
|
|
case_id: str = Field(..., min_length=1) # UUID — 학습자별 케이스 스코프(M5/T4)
|
|
query: str = Field(..., min_length=1)
|
|
k: int = Field(default=5, ge=1, le=20)
|
|
session_id: Optional[str] = None
|
|
turn_id: Optional[str] = None
|
|
|
|
|
|
class IndexChunkIn(BaseModel):
|
|
seq: int
|
|
chunk_text: str = Field(..., min_length=1)
|
|
heading_path: Optional[str] = None
|
|
context_prefix: Optional[str] = None # Contextual Retrieval 프리픽스(색인 대상)
|
|
kb_kind: Optional[str] = None
|
|
visible_to: Optional[list[str]] = None # 미지정 시 {client,counselor,evaluator}
|
|
sensitivity: Optional[int] = Field(default=None, ge=0, le=3)
|
|
label_id: Optional[int] = None # taxonomy 정답 라벨 FK
|
|
meta: Optional[dict[str, Any]] = None
|
|
token_count: Optional[int] = None
|
|
|
|
|
|
class IndexRequestIn(BaseModel):
|
|
source_id: str = Field(..., min_length=1)
|
|
doc_uri: str = Field(..., min_length=1)
|
|
version: int = 1
|
|
content_hash: Optional[str] = None
|
|
chunks: list[IndexChunkIn]
|
|
|
|
|
|
class IndexResponse(BaseModel):
|
|
doc_id: Optional[int]
|
|
chunks_indexed: int
|
|
skipped_unchanged: bool # content_hash 동일 → 증분 스킵
|
|
embedded: bool # 임베딩 적재 여부(모델 미가용 시 False)
|
|
degraded: bool = False
|
|
|
|
|
|
class LiveCoachSourcePackSyncItem(BaseModel):
|
|
source_id: str
|
|
doc_id: Optional[int]
|
|
chunks_indexed: int
|
|
skipped_unchanged: bool
|
|
embedded: bool
|
|
degraded: bool = False
|
|
|
|
|
|
class LiveCoachSourcePackSyncResponse(BaseModel):
|
|
sources_upserted: int
|
|
chunks_indexed: int
|
|
skipped_unchanged: int
|
|
embedded: bool
|
|
degraded: bool = False
|
|
items: list[LiveCoachSourcePackSyncItem] = Field(default_factory=list)
|
|
|
|
|
|
# ── 헬퍼: rag.NotConfigured → 503 ───────────────────────
|
|
def _to_chunk_out(c: rag.RetrievedChunk) -> ChunkOut:
|
|
return ChunkOut(
|
|
chunk_id=c.chunk_id,
|
|
score=round(c.score, 6),
|
|
kb_kind=c.kb_kind,
|
|
heading_path=c.heading_path,
|
|
context_prefix=c.context_prefix,
|
|
body=c.body,
|
|
behavior_cue=c.behavior_cue,
|
|
label_id=c.label_id,
|
|
meta=c.meta,
|
|
source_id=c.source_id,
|
|
)
|
|
|
|
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
# 헬스 — 라우터 + RAG readiness (모델/DB 미가용도 정직하게 보고)
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
@router.get("/health")
|
|
async def kb_health() -> dict[str, object]:
|
|
"""KB 라우터 + RAG 구성요소 readiness.
|
|
|
|
DB 풀/임베딩 모델 가용 여부를 *크래시 없이* 점검(미가용=degraded). 부트/디버그용.
|
|
"""
|
|
db_ready = False
|
|
try:
|
|
get_pool()
|
|
db_ready = True
|
|
except RuntimeError:
|
|
db_ready = False
|
|
# 임베딩 모델은 무거우므로 *로드하지 않고* 설치 가능성만 가볍게 확인(import 시도 X).
|
|
return {
|
|
"status": "ok" if db_ready else "degraded",
|
|
"owner": "features:rag",
|
|
"db_pool": db_ready,
|
|
"crag_threshold": rag.CRAG_TOP1_THRESHOLD,
|
|
"policies": [r.value for r in rag.POLICIES],
|
|
}
|
|
|
|
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
# 지식 검색 — 정책 4-튜플(role)로 분기, visible_to DB 강제
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
@router.post("/search", response_model=KBSearchResponse)
|
|
async def search(body: KBSearchRequest) -> KBSearchResponse:
|
|
"""정적 지식 KB 하이브리드 검색(dense pgvector cosine + sparse tsvector + 리랭킹).
|
|
|
|
role 이 정책 4-튜플(사전필터·가중치·본문노출·라벨)을 고정한다 — 호출부가 못 넓힌다.
|
|
AI 뷰 RLS 컨텍스트(app.current_ai_view)를 커넥션에 주입해 visible_to 를 2중 강제.
|
|
"""
|
|
ai_role = rag.AIRole(body.role)
|
|
filters: dict[str, Any] = {}
|
|
if body.kb_kind:
|
|
filters["kb_kind"] = body.kb_kind
|
|
if body.source_id:
|
|
filters["source_id"] = body.source_id
|
|
if body.sensitivity_max is not None:
|
|
filters["sensitivity_max"] = body.sensitivity_max
|
|
|
|
# RLS 컨텍스트(레이어1): app.current_ai_view = role → visible_to WHERE DB 강제
|
|
try:
|
|
async with acquire(ai_view=AIView(body.role).value) as conn:
|
|
result = await rag.search_kb(
|
|
conn,
|
|
query=body.query,
|
|
role=ai_role,
|
|
k=body.k,
|
|
filters=filters or None,
|
|
rerank=body.rerank,
|
|
)
|
|
# 감사 적재(best-effort — 로그 실패가 검색을 막지 않음)
|
|
try:
|
|
await rag.log_retrieval(
|
|
conn,
|
|
result=result,
|
|
ai_role=body.role,
|
|
session_id=body.session_id,
|
|
turn_id=body.turn_id,
|
|
)
|
|
except Exception:
|
|
pass
|
|
except rag.NotConfigured as e:
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
detail=f"RAG not configured: {e}",
|
|
)
|
|
except RuntimeError as e:
|
|
# DB 풀 미초기화(lifespan 밖) — 시연/테스트 degraded
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE, detail=f"DB not ready: {e}"
|
|
)
|
|
|
|
return _search_response(result)
|
|
|
|
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
# 평가 근거 — evaluator 정책 래퍼(label_id 동봉, CRAG 게이트)
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
@router.post("/eval-grounding", response_model=KBSearchResponse)
|
|
async def eval_grounding(body: KBSearchRequest) -> KBSearchResponse:
|
|
"""평가 AI 채점 근거 회수(DSM/이론/taxonomy 정답라벨 + 논평).
|
|
|
|
role 무시하고 evaluator 정책 고정(평가 전용 경로). crag_pass=False 면 호출부가
|
|
'관찰 프레이밍'으로 다운그레이드(F-06).
|
|
"""
|
|
try:
|
|
async with acquire(ai_view=AIView.EVALUATOR.value) as conn:
|
|
result = await rag.retrieve_eval_grounding(
|
|
conn,
|
|
query=body.query,
|
|
k=body.k,
|
|
kinds=body.kb_kind,
|
|
source_ids=body.source_id,
|
|
rerank=body.rerank,
|
|
)
|
|
try:
|
|
await rag.log_retrieval(
|
|
conn,
|
|
result=result,
|
|
ai_role="evaluator",
|
|
session_id=body.session_id,
|
|
turn_id=body.turn_id,
|
|
)
|
|
except Exception:
|
|
pass
|
|
except rag.NotConfigured as e:
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE, detail=f"RAG not configured: {e}"
|
|
)
|
|
except RuntimeError as e:
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE, detail=f"DB not ready: {e}"
|
|
)
|
|
|
|
return _search_response(result)
|
|
|
|
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
# 페르소나 메모리 회상 — 내담자 연속성(case 스코프, episodic)
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
@router.post("/persona-memory", response_model=KBSearchResponse)
|
|
async def persona_memory(body: MemoryRecallRequest) -> KBSearchResponse:
|
|
"""회기 시작 episodic recall(app.turn_embedding, case_id 스코프 강제).
|
|
|
|
CCD/정답/평가는 이 경로에 구조적으로 부재(코드경로 부재 1차방어). 반환은 turn_id+점수만
|
|
(본문은 호출부 memory.build_recall_context 가 turns 조인). 내담자 뷰 RLS 주입.
|
|
"""
|
|
try:
|
|
async with acquire(ai_view=AIView.CLIENT.value) as conn:
|
|
result = await rag.retrieve_persona_memory(
|
|
conn,
|
|
case_id=body.case_id,
|
|
query=body.query,
|
|
k=body.k,
|
|
)
|
|
except rag.NotConfigured as e:
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE, detail=f"RAG not configured: {e}"
|
|
)
|
|
except RuntimeError as e:
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE, detail=f"DB not ready: {e}"
|
|
)
|
|
|
|
return _search_response(result)
|
|
|
|
|
|
def _search_response(result: rag.SearchResult) -> KBSearchResponse:
|
|
"""Project every RAG policy result through the same browser-facing contract."""
|
|
|
|
return KBSearchResponse(
|
|
chunks=[_to_chunk_out(c) for c in result.chunks],
|
|
policy=result.policy_name,
|
|
top1_score=round(result.top1_score, 6),
|
|
crag_pass=result.top1_score >= rag.CRAG_TOP1_THRESHOLD,
|
|
latency_ms=result.latency_ms,
|
|
degraded=result.degraded,
|
|
)
|
|
|
|
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
# 인덱싱 트리거 — 관리자 전용(content_hash 증분, 오프라인 배치)
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
@router.post(
|
|
"/index", response_model=IndexResponse, status_code=status.HTTP_202_ACCEPTED
|
|
)
|
|
async def index_document(
|
|
body: IndexRequestIn,
|
|
principal: Annotated[Principal, Depends(require_role(Role.ADMIN))],
|
|
) -> IndexResponse:
|
|
"""문서 인덱싱(관리자, RBAC ADMIN 강제). content_hash 증분 + 청크 임베딩 적재.
|
|
|
|
⚠️ 임베딩은 무거운 작업 → 본래 BackgroundTasks/배치 워커 위임 권장(202 Accepted).
|
|
DSM verbatim 저작권(license C/D)은 source 등록 시점 external_llm_ok 가드 책임.
|
|
모델 미가용 시 embedding NULL 폴백(BM25 만, degraded=True) — 크래시 X.
|
|
"""
|
|
req = rag.IndexRequest(
|
|
source_id=body.source_id,
|
|
doc_uri=body.doc_uri,
|
|
version=body.version,
|
|
content_hash=body.content_hash,
|
|
chunks=[c.model_dump() for c in body.chunks],
|
|
)
|
|
try:
|
|
# 관리자 인덱싱은 RLS 미적용(쓰기 — kb 스키마 직접). role 주입 없이 acquire.
|
|
async with acquire() as conn:
|
|
result = await rag.index_document(conn, req)
|
|
except rag.IndexPolicyViolation as e:
|
|
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)) from e
|
|
except rag.NotConfigured as e:
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE, detail=f"RAG not configured: {e}"
|
|
)
|
|
except RuntimeError as e:
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE, detail=f"DB not ready: {e}"
|
|
)
|
|
|
|
return IndexResponse(
|
|
doc_id=result.doc_id,
|
|
chunks_indexed=result.chunks_indexed,
|
|
skipped_unchanged=result.skipped_unchanged,
|
|
embedded=result.embedded,
|
|
degraded=result.degraded,
|
|
)
|
|
|
|
|
|
@router.post(
|
|
"/live-coach/source-packs/sync",
|
|
response_model=LiveCoachSourcePackSyncResponse,
|
|
status_code=status.HTTP_202_ACCEPTED,
|
|
)
|
|
async def sync_live_coach_source_packs(
|
|
principal: Annotated[Principal, Depends(require_role(Role.ADMIN))],
|
|
) -> LiveCoachSourcePackSyncResponse:
|
|
"""허가된 라이브 코칭 source pack을 kb.source/kb.chunk RAG 색인에 적재한다.
|
|
|
|
로컬 `data/kb/live_coaching_*.json`은 UI 즉시 코칭의 기본 근거이고, 이 경로는 같은 자료를
|
|
evaluator RAG 검색에도 올린다. source row를 먼저 upsert한 뒤 content_hash 기반 증분 색인을
|
|
수행한다. 임베딩 모델 미가용 시 BM25-only degraded 색인으로 이어진다.
|
|
"""
|
|
source_rows, index_payloads = source_pack_sync.build_repo_source_pack_manifest(
|
|
refresh=True
|
|
)
|
|
if not source_rows or not index_payloads:
|
|
raise HTTPException(
|
|
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
detail="live coach source packs are empty",
|
|
)
|
|
|
|
try:
|
|
async with acquire() as conn:
|
|
result = await source_pack_sync.sync_repo_source_packs(
|
|
conn, apply=True, refresh=True
|
|
)
|
|
except rag.NotConfigured as exc:
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE, detail=f"RAG not configured: {exc}"
|
|
) from exc
|
|
except RuntimeError as exc:
|
|
raise HTTPException(
|
|
status.HTTP_503_SERVICE_UNAVAILABLE, detail=f"DB not ready: {exc}"
|
|
) from exc
|
|
|
|
return LiveCoachSourcePackSyncResponse(
|
|
sources_upserted=result.sources_upserted,
|
|
chunks_indexed=result.chunks_indexed,
|
|
skipped_unchanged=result.skipped_unchanged,
|
|
embedded=result.embedded,
|
|
degraded=result.degraded,
|
|
items=[
|
|
LiveCoachSourcePackSyncItem(
|
|
source_id=item.source_id,
|
|
doc_id=item.doc_id,
|
|
chunks_indexed=item.chunks_indexed,
|
|
skipped_unchanged=item.skipped_unchanged,
|
|
embedded=item.embedded,
|
|
degraded=item.degraded,
|
|
)
|
|
for item in result.items
|
|
],
|
|
)
|