vignette/apps/api/app/services/source_pack_sync.py

"""Repo-managed source pack sync helpers.

This keeps CLI and admin API sync behavior on the same content_hash/version
path. The source pack content itself stays in data files owned outside this
service.
"""

from __future__ import annotations

from dataclasses import dataclass
from typing import Any

from . import live_coach, rag


@dataclass(slots=True)
class RepoSourcePackSyncItem:
    source_id: str
    doc_uri: str
    previous_version: int | None
    new_version: int
    content_hash: str
    doc_id: int | None
    chunks_indexed: int
    skipped_unchanged: bool
    embedded: bool
    degraded: bool = False
    applied: bool = False


@dataclass(slots=True)
class RepoSourcePackSyncResult:
    sources_upserted: int
    manifest_count: int
    chunks_indexed: int
    skipped_unchanged: int
    embedded: bool
    degraded: bool
    applied: bool
    items: list[RepoSourcePackSyncItem]


def build_repo_source_pack_manifest(*, refresh: bool = False) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
    """Return repo-managed source rows and RAG index payloads without DB access."""

    if refresh:
        live_coach.clear_local_source_pack_cache()
    return live_coach.build_rag_source_rows(), live_coach.build_rag_index_payloads()


def _latest_version(row: Any | None) -> int | None:
    if not row:
        return None
    value = row["version"]
    return int(value) if value is not None else None


async def _latest_active_document(conn: Any, *, source_id: str, doc_uri: str) -> Any | None:
    return await conn.fetchrow(
        """
        SELECT doc_id, version, content_hash
        FROM kb.document
        WHERE source_id = $1 AND doc_uri = $2 AND is_active
        ORDER BY version DESC
        LIMIT 1
        """,
        source_id,
        doc_uri,
    )


async def _upsert_source_rows(conn: Any, source_rows: list[dict[str, Any]]) -> None:
    for row in source_rows:
        await conn.execute(
            """
            INSERT INTO kb.source
              (source_id, title, kb_kind, license_class, origin_path, citation, external_llm_ok)
            VALUES ($1, $2, $3, $4, $5, $6, $7)
            ON CONFLICT (source_id) DO UPDATE SET
              title = EXCLUDED.title,
              kb_kind = EXCLUDED.kb_kind,
              license_class = EXCLUDED.license_class,
              origin_path = EXCLUDED.origin_path,
              citation = EXCLUDED.citation,
              external_llm_ok = EXCLUDED.external_llm_ok
            """,
            row["source_id"],
            row["title"],
            row["kb_kind"],
            row["license_class"],
            row["origin_path"],
            row["citation"],
            row["external_llm_ok"],
        )


async def sync_repo_source_packs(
    conn: Any,
    *,
    apply: bool = False,
    refresh: bool = False,
) -> RepoSourcePackSyncResult:
    """Compare or apply repo-managed source packs against kb.document.

    Dry-run mode still reads the active DB document row so it can report whether
    the next apply would skip or materialize a new document version. DB writes
    happen only when apply=True.
    """

    source_rows, index_payloads = build_repo_source_pack_manifest(refresh=refresh)
    source_row_by_id = {row["source_id"]: row for row in source_rows}
    if apply:
        await _upsert_source_rows(conn, source_rows)

    items: list[RepoSourcePackSyncItem] = []
    for payload in index_payloads:
        source_id = str(payload["source_id"])
        if source_id not in source_row_by_id:
            continue
        doc_uri = str(payload["doc_uri"])
        content_hash = str(payload["content_hash"])
        latest = await _latest_active_document(conn, source_id=source_id, doc_uri=doc_uri)
        previous_version = _latest_version(latest)
        requested_version = max(1, int(payload.get("version") or 1))
        new_version = requested_version
        if previous_version is not None:
            new_version = max(requested_version, previous_version + 1)

        if latest and latest["content_hash"] == content_hash:
            items.append(
                RepoSourcePackSyncItem(
                    source_id=source_id,
                    doc_uri=doc_uri,
                    previous_version=previous_version,
                    new_version=previous_version or requested_version,
                    content_hash=content_hash,
                    doc_id=int(latest["doc_id"]),
                    chunks_indexed=0,
                    skipped_unchanged=True,
                    embedded=False,
                    degraded=False,
                    applied=apply,
                )
            )
            continue

        if not apply:
            items.append(
                RepoSourcePackSyncItem(
                    source_id=source_id,
                    doc_uri=doc_uri,
                    previous_version=previous_version,
                    new_version=new_version,
                    content_hash=content_hash,
                    doc_id=None,
                    chunks_indexed=0,
                    skipped_unchanged=False,
                    embedded=False,
                    degraded=False,
                    applied=False,
                )
            )
            continue

        index_payload = dict(payload)
        index_payload["version"] = new_version
        result = await rag.index_document(conn, rag.IndexRequest(**index_payload))
        items.append(
            RepoSourcePackSyncItem(
                source_id=source_id,
                doc_uri=doc_uri,
                previous_version=previous_version,
                new_version=new_version,
                content_hash=content_hash,
                doc_id=result.doc_id,
                chunks_indexed=result.chunks_indexed,
                skipped_unchanged=result.skipped_unchanged,
                embedded=result.embedded,
                degraded=result.degraded,
                applied=True,
            )
        )

    return RepoSourcePackSyncResult(
        sources_upserted=len(source_rows) if apply else 0,
        manifest_count=len(index_payloads),
        chunks_indexed=sum(item.chunks_indexed for item in items),
        skipped_unchanged=sum(1 for item in items if item.skipped_unchanged),
        embedded=all(item.embedded for item in items) if items else False,
        degraded=any(item.degraded for item in items),
        applied=apply,
        items=items,
    )


__all__ = [
    "RepoSourcePackSyncItem",
    "RepoSourcePackSyncResult",
    "build_repo_source_pack_manifest",
    "sync_repo_source_packs",
]