yunchan/vignette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

전 저장소 리팩터링과 SSOT 정비

Browse source
This commit is contained in:
Yun Chan 2026-07-15 21:31:30 +09:00
parent 14ecbd4e7d
commit 3dfddcac6f
173 changed files with 19679 additions and 6952 deletions
Download patch file Download diff file Expand all files Collapse all files

92
scripts/export-recursive-dataset.py
View file

@ -18,6 +18,7 @@ import asyncpg # noqa: E402
from app.services.dataset_export import ( # noqa: E402
APPROVED_EXPORT_STATUS,
DRY_RUN_EXPORT_STATUS,
DatasetManifestInput,
ExportKeyMaps,
build_dataset_record,
build_manifest,
@ -130,7 +131,9 @@ def row_to_dict(row: asyncpg.Record) -> dict[str, Any]:
return {key: row[key] for key in row.keys()}
async def fetch_rows(conn: asyncpg.Connection, args: argparse.Namespace) -> list[dict[str, Any]]:
async def fetch_rows(
conn: asyncpg.Connection, args: argparse.Namespace
) -> list[dict[str, Any]]:
rows = await conn.fetch(
TURN_QUERY,
parse_time(args.started_at),
@ -165,13 +168,19 @@ async def write_dataset_rows(
return int(dataset_id)
async def fetch_agreement(conn: asyncpg.Connection, dataset_id: int | None, args: argparse.Namespace) -> dict[str, Any]:
async def fetch_agreement(
conn: asyncpg.Connection, dataset_id: int | None, args: argparse.Namespace
) -> dict[str, Any]:
if dataset_id is None:
return {"kappa": None, "icc": None, "gold_status": "not_gold"}
rows = [row_to_dict(row) for row in await conn.fetch(ANNOTATION_QUERY, dataset_id)]
kappa = cohen_kappa(rows, args.kappa_label)
icc = intraclass_correlation(rows, args.icc_label)
gold_status = "gold_candidate" if kappa is not None and kappa >= 0.70 and icc is not None and icc >= 0.75 else "not_gold"
gold_status = (
"gold_candidate"
if kappa is not None and kappa >= 0.70 and icc is not None and icc >= 0.75
else "not_gold"
)
return {"kappa": kappa, "icc": icc, "gold_status": gold_status}
@ -196,7 +205,9 @@ async def insert_manifest_row(
async def run(args: argparse.Namespace) -> int:
if args.export_status == APPROVED_EXPORT_STATUS and not args.allow_approved:
raise SystemExit("--export-status approved_for_recursive_learning_seed requires --allow-approved")
raise SystemExit(
"--export-status approved_for_recursive_learning_seed requires --allow-approved"
)
if args.write_manifest_row and not args.write_dataset:
raise SystemExit("--write-manifest-row requires --write-dataset")
@ -248,24 +259,27 @@ async def run(args: argparse.Namespace) -> int:
approvals = {
"data_steward": args.data_steward,
"legal_or_privacy_reviewer": args.legal_or_privacy_reviewer,
"technical_operator": args.technical_operator or os.environ.get("USERNAME", ""),
"technical_operator": args.technical_operator
or os.environ.get("USERNAME", ""),
"approved_at": args.approved_at,
}
manifest = build_manifest(
export_id=args.export_id,
dataset_name=args.dataset_name,
export_status=args.export_status,
purpose=args.purpose,
records=records,
jsonl_path=str(jsonl_path.relative_to(output_root)).replace("\\", "/"),
jsonl_sha256=digest,
pii_findings=pii_findings,
participants_included=len(keys.participant),
cohort_id=args.cohort_id,
consent_version=args.consent_version,
agreement=agreement,
approvals=approvals,
known_limitations=args.known_limitation,
DatasetManifestInput(
export_id=args.export_id,
dataset_name=args.dataset_name,
export_status=args.export_status,
purpose=args.purpose,
records=records,
jsonl_path=str(jsonl_path.relative_to(output_root)).replace("\\", "/"),
jsonl_sha256=digest,
pii_findings=pii_findings,
participants_included=len(keys.participant),
cohort_id=args.cohort_id,
consent_version=args.consent_version,
agreement=agreement,
approvals=approvals,
known_limitations=args.known_limitation,
)
)
validate_manifest_gate(manifest)
manifest_path.write_text(
@ -283,18 +297,44 @@ async def run(args: argparse.Namespace) -> int:
finally:
await conn.close()
print(json.dumps({"manifest": str(manifest_path), "jsonl": str(jsonl_path), "rows": len(records)}, ensure_ascii=False))
print(
json.dumps(
{
"manifest": str(manifest_path),
"jsonl": str(jsonl_path),
"rows": len(records),
},
ensure_ascii=False,
)
)
return 0
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Export a Phase 3 recursive-learning JSONL dry-run dataset.")
parser.add_argument("--database-url", default=os.environ.get("DATABASE_URL"), required=os.environ.get("DATABASE_URL") is None)
parser.add_argument("--output-root", default=str(REPO_ROOT / "data" / "phase3-dry-run"))
parser = argparse.ArgumentParser(
description="Export a Phase 3 recursive-learning JSONL dry-run dataset."
)
parser.add_argument(
"--database-url",
default=os.environ.get("DATABASE_URL"),
required=os.environ.get("DATABASE_URL") is None,
)
parser.add_argument(
"--output-root", default=str(REPO_ROOT / "data" / "phase3-dry-run")
)
parser.add_argument("--export-id", default=make_export_id())
parser.add_argument("--dataset-name", default="vignette_phase3_recursive_learning_seed")
parser.add_argument("--purpose", default="recursive-learning seed dataset for education simulator improvement")
parser.add_argument("--export-status", choices=[DRY_RUN_EXPORT_STATUS, "blocked", APPROVED_EXPORT_STATUS], default=DRY_RUN_EXPORT_STATUS)
parser.add_argument(
"--dataset-name", default="vignette_phase3_recursive_learning_seed"
)
parser.add_argument(
"--purpose",
default="recursive-learning seed dataset for education simulator improvement",
)
parser.add_argument(
"--export-status",
choices=[DRY_RUN_EXPORT_STATUS, "blocked", APPROVED_EXPORT_STATUS],
default=DRY_RUN_EXPORT_STATUS,
)
parser.add_argument("--allow-approved", action="store_true")
parser.add_argument("--started-at")
parser.add_argument("--ended-at")