현재 작업 전체 반영
This commit is contained in:
Yun Chan
parent
5560638e54
commit
c0dddab594
85 changed files with 11322 additions and 539 deletions
|
|
@ -3,11 +3,13 @@
|
|||
Vignette 로컬 개발 스택(게이트웨이 9099 + API 8000 + 웹 5173)을 정리한다.
|
||||
.DESCRIPTION
|
||||
커맨드라인 기준으로 정확히 종료한다(고아 워커/리로더 포함). scripts/dev-up.ps1 의 짝.
|
||||
기본은 Postgres 컨테이너를 보존한다. 함께 멈추려면 -Db를 명시한다.
|
||||
.EXAMPLE
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File scripts\dev-down.ps1
|
||||
#>
|
||||
param(
|
||||
[int]$ApiPort = 8000
|
||||
[int]$ApiPort = 8000,
|
||||
[switch]$Db
|
||||
)
|
||||
$ErrorActionPreference = 'SilentlyContinue'
|
||||
|
||||
|
|
@ -28,5 +30,16 @@ Get-NetTCPConnection -LocalPort $ApiPort -State Listen -ErrorAction SilentlyCont
|
|||
Select-Object -ExpandProperty OwningProcess -Unique |
|
||||
Where-Object { $_ -and $_ -ne 0 } |
|
||||
ForEach-Object { Stop-Process -Id $_ -Force }
|
||||
if ($Db) {
|
||||
if (Get-Command docker -ErrorAction SilentlyContinue) {
|
||||
$running = docker ps --filter name=vignette-dev-db --format '{{.Names}}' 2>$null
|
||||
if ($running -match 'vignette-dev-db') {
|
||||
Write-Host ' stop db vignette-dev-db'
|
||||
docker stop vignette-dev-db 1>$null
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Write-Host ' keep db vignette-dev-db (필요하면 -Db로 중지)'
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
Write-Host "완료."
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@
|
|||
엔진 게이트웨이를 띄우지 않는다(AI 턴 생성 불가, UI만 테스트 시).
|
||||
.PARAMETER NoWeb
|
||||
vite 웹 서버를 띄우지 않는다(API만 필요할 때).
|
||||
.PARAMETER NoDb
|
||||
Docker Postgres 보장을 건너뛰고 in-memory degraded 기동을 허용한다.
|
||||
.EXAMPLE
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File scripts\dev-up.ps1
|
||||
#>
|
||||
|
|
@ -80,6 +82,78 @@ function Wait-Health([string]$url, [string]$label, [int]$timeoutSec = 40) {
|
|||
Write-Host " WARN $label 미응답 ($url) — .devlogs 로그 확인"; return $false
|
||||
}
|
||||
|
||||
function Read-EnvFileValue([string]$path, [string]$key) {
|
||||
if (!(Test-Path -LiteralPath $path)) { return $null }
|
||||
$pattern = ('^\s*{0}\s*=' -f [regex]::Escape($key))
|
||||
$line = Get-Content -LiteralPath $path -Encoding utf8 |
|
||||
Where-Object { $_ -match $pattern } |
|
||||
Select-Object -First 1
|
||||
if (-not $line) { return $null }
|
||||
$value = ($line -replace $pattern, '').Trim()
|
||||
$value = $value.Trim('"')
|
||||
$value = $value.Trim("'")
|
||||
return $value
|
||||
}
|
||||
|
||||
function Warn-ComposeEnvReadiness {
|
||||
$composeEnv = Join-Path $repo 'infra\.env'
|
||||
if (!(Test-Path -LiteralPath $composeEnv)) {
|
||||
Write-Host ' INFO infra\.env 없음 - docker compose 전체 스택은 .env 작성 후 사용.'
|
||||
return
|
||||
}
|
||||
$required = @(
|
||||
'POSTGRES_PASSWORD',
|
||||
'APP_DB_PASSWORD',
|
||||
'OPENAI_API_KEY',
|
||||
'SESSION_SECRET',
|
||||
'OAUTH_GOOGLE_CLIENT_ID',
|
||||
'OAUTH_GOOGLE_CLIENT_SECRET'
|
||||
)
|
||||
$missing = @()
|
||||
foreach ($key in $required) {
|
||||
$value = Read-EnvFileValue $composeEnv $key
|
||||
if ([string]::IsNullOrWhiteSpace($value)) { $missing += $key }
|
||||
}
|
||||
if ($missing.Count -gt 0) {
|
||||
Write-Host (' WARN infra\.env 필수값 누락: {0} - compose api/proxy 기동 전 보강 필요.' -f ($missing -join ', '))
|
||||
}
|
||||
}
|
||||
|
||||
function Test-DevDbRoleSafety([string]$ownerUser, [string]$dbName) {
|
||||
if ([string]::IsNullOrWhiteSpace($ownerUser) -or [string]::IsNullOrWhiteSpace($dbName)) { return }
|
||||
$sql = "SELECT rolname || ':' || rolsuper || ':' || rolbypassrls FROM pg_roles WHERE rolname IN ('vignette_app','vignette_owner') ORDER BY rolname;"
|
||||
$rows = & docker exec vignette-dev-db psql -U $ownerUser -d $dbName -At -c $sql 2>$null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host ' WARN db role 점검 실패 - psql 접속/권한 확인 필요.'
|
||||
return
|
||||
}
|
||||
foreach ($row in $rows) {
|
||||
$parts = "$row".Split(':')
|
||||
if ($parts.Count -lt 3) { continue }
|
||||
if ($parts[0] -eq 'vignette_app' -and ($parts[1] -eq 't' -or $parts[2] -eq 't')) {
|
||||
Write-Host ' WARN vignette_app role 이 SUPERUSER/BYPASSRLS 상태 - RLS 검증이 무의미함. 컨테이너를 owner/app 분리로 재생성 필요.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Inspect-DevDbContainer {
|
||||
$healthMode = & docker inspect -f '{{if .Config.Healthcheck}}health{{else}}none{{end}}' vignette-dev-db 2>$null
|
||||
if ($LASTEXITCODE -eq 0 -and "$healthMode" -eq 'none') {
|
||||
Write-Host ' WARN vignette-dev-db healthcheck 없음 - 기존 컨테이너라면 재생성 시 healthcheck가 붙음.'
|
||||
}
|
||||
$ownerUser = & docker exec vignette-dev-db printenv POSTGRES_USER 2>$null
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($ownerUser)) { $ownerUser = 'vignette_owner' }
|
||||
$dbName = & docker exec vignette-dev-db printenv POSTGRES_DB 2>$null
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($dbName)) { $dbName = 'vignette' }
|
||||
& docker exec vignette-dev-db pg_isready -U $ownerUser 1>$null 2>$null
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Host ' OK db (vignette-dev-db 준비됨)'
|
||||
} else {
|
||||
Write-Host ' WARN db 컨테이너는 있으나 pg_isready 실패'
|
||||
}
|
||||
Test-DevDbRoleSafety "$ownerUser" "$dbName"
|
||||
}
|
||||
|
||||
function Ensure-DevDb {
|
||||
if (!(Get-Command docker -ErrorAction SilentlyContinue)) {
|
||||
Write-Host ' WARN docker CLI 없음 - DB 없이 degraded(UI 세션 시작 제한).'
|
||||
|
|
@ -97,13 +171,13 @@ function Ensure-DevDb {
|
|||
|
||||
$running = docker ps --filter name=vignette-dev-db --format '{{.Names}}' 2>$null
|
||||
if ($running -match 'vignette-dev-db') {
|
||||
Write-Host ' OK db (vignette-dev-db 실행 중)'
|
||||
Inspect-DevDbContainer
|
||||
return
|
||||
}
|
||||
|
||||
docker rm -f vignette-dev-db 1>$null 2>$null
|
||||
$envLines = Get-Content -LiteralPath (Join-Path $api '.env')
|
||||
$dbUrl = (($envLines | Where-Object { $_ -match '^DATABASE_URL=' }) -replace '^DATABASE_URL=', '').Trim()
|
||||
$apiEnv = Join-Path $api '.env'
|
||||
$dbUrl = Read-EnvFileValue $apiEnv 'DATABASE_URL'
|
||||
if ($dbUrl -notmatch 'postgresql://([^:]+):([^@]+)@[^:]+:([0-9]+)/(\S+)') {
|
||||
Write-Host ' WARN DATABASE_URL 파싱 실패 - DB 스킵'
|
||||
return
|
||||
|
|
@ -113,16 +187,27 @@ function Ensure-DevDb {
|
|||
$p = $Matches[2]
|
||||
$port = $Matches[3]
|
||||
$db = $Matches[4]
|
||||
$ownerUser = 'vignette_owner'
|
||||
$ownerPassword = $p
|
||||
$appUser = $u
|
||||
$appPassword = $p
|
||||
if ($appUser -eq $ownerUser) {
|
||||
Write-Host ' WARN DATABASE_URL 이 owner role 을 사용 중 - RLS 검증을 위해 앱 전용 role URL 권장.'
|
||||
}
|
||||
$initPath = Join-Path $repo 'infra\db\init'
|
||||
$dockerArgs = @(
|
||||
'run', '-d',
|
||||
'--name', 'vignette-dev-db',
|
||||
'-p', ('{0}:5432' -f $port),
|
||||
'-e', ('POSTGRES_USER={0}' -f $u),
|
||||
'-e', ('POSTGRES_PASSWORD={0}' -f $p),
|
||||
'--health-cmd', ('pg_isready -U {0}' -f $ownerUser),
|
||||
'--health-interval', '10s',
|
||||
'--health-timeout', '5s',
|
||||
'--health-retries', '5',
|
||||
'-e', ('POSTGRES_USER={0}' -f $ownerUser),
|
||||
'-e', ('POSTGRES_PASSWORD={0}' -f $ownerPassword),
|
||||
'-e', ('POSTGRES_DB={0}' -f $db),
|
||||
'-e', 'APP_DB_USER=vignette_app',
|
||||
'-e', 'APP_DB_PASSWORD=vignette_app',
|
||||
'-e', ('APP_DB_USER={0}' -f $appUser),
|
||||
'-e', ('APP_DB_PASSWORD={0}' -f $appPassword),
|
||||
'-v', ('{0}:/docker-entrypoint-initdb.d:ro' -f $initPath),
|
||||
'pgvector/pgvector:pg16'
|
||||
)
|
||||
|
|
@ -130,9 +215,10 @@ function Ensure-DevDb {
|
|||
|
||||
for ($i = 0; $i -lt 24; $i++) {
|
||||
Start-Sleep -Seconds 2
|
||||
docker exec vignette-dev-db pg_isready -U $u 1>$null 2>$null
|
||||
docker exec vignette-dev-db pg_isready -U $ownerUser 1>$null 2>$null
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Host ' OK db (Postgres 준비됨, init 스키마 적용)'
|
||||
Test-DevDbRoleSafety $ownerUser $db
|
||||
Start-Sleep -Seconds 1
|
||||
return
|
||||
}
|
||||
|
|
@ -151,6 +237,7 @@ Start-Sleep -Seconds 2
|
|||
|
||||
if (-not $NoDb) {
|
||||
Write-Host '[DB] Postgres(pgvector) 보장...'
|
||||
Warn-ComposeEnvReadiness
|
||||
Ensure-DevDb
|
||||
}
|
||||
|
||||
|
|
|
|||
324
scripts/export-recursive-dataset.py
Normal file
324
scripts/export-recursive-dataset.py
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
API_ROOT = REPO_ROOT / "apps" / "api"
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
import asyncpg # noqa: E402
|
||||
|
||||
from app.services.dataset_export import ( # noqa: E402
|
||||
APPROVED_EXPORT_STATUS,
|
||||
DRY_RUN_EXPORT_STATUS,
|
||||
ExportKeyMaps,
|
||||
build_dataset_record,
|
||||
build_manifest,
|
||||
cohen_kappa,
|
||||
intraclass_correlation,
|
||||
scan_for_pii,
|
||||
sha256_file,
|
||||
validate_manifest_gate,
|
||||
write_jsonl,
|
||||
)
|
||||
|
||||
|
||||
TURN_QUERY = """
|
||||
SELECT
|
||||
s.id::text AS session_id,
|
||||
s.learner_id::text AS learner_id,
|
||||
COALESCE(pc.code, s.persona_id::text, 'unknown') AS persona_code,
|
||||
s.started_at AS session_started_at,
|
||||
s.ended_at AS session_ended_at,
|
||||
t.id::text AS turn_id,
|
||||
t.seq,
|
||||
t.speaker,
|
||||
t.stage,
|
||||
t.text_masked,
|
||||
t.created_at,
|
||||
COALESCE(
|
||||
jsonb_agg(DISTINCT jsonb_build_object(
|
||||
'dimension', fs.dimension,
|
||||
'score', fs.score,
|
||||
'rationale', fs.rationale,
|
||||
'loop', fs.loop
|
||||
)) FILTER (WHERE fs.dimension IS NOT NULL),
|
||||
'[]'::jsonb
|
||||
) AS feedback_scores,
|
||||
COALESCE(
|
||||
jsonb_agg(DISTINCT jsonb_build_object(
|
||||
'code', tech.code,
|
||||
'display_name', tech.display_name,
|
||||
'category', tech.category
|
||||
)) FILTER (WHERE tech.code IS NOT NULL),
|
||||
'[]'::jsonb
|
||||
) AS techniques,
|
||||
COALESCE(
|
||||
jsonb_agg(DISTINCT jsonb_build_object(
|
||||
'code', cs.code,
|
||||
'display_name', cs.display_name
|
||||
)) FILTER (WHERE cs.code IS NOT NULL),
|
||||
'[]'::jsonb
|
||||
) AS client_states,
|
||||
COALESCE(
|
||||
jsonb_agg(DISTINCT jsonb_build_object(
|
||||
'kind', sc.kind,
|
||||
'text', sc.text,
|
||||
'intent_deviation', sc.intent_deviation
|
||||
)) FILTER (WHERE sc.id IS NOT NULL),
|
||||
'[]'::jsonb
|
||||
) AS supervisor_comments
|
||||
FROM app.turns t
|
||||
JOIN app.sessions s ON s.id = t.session_id
|
||||
JOIN app.app_user u ON u.user_id = s.learner_id
|
||||
LEFT JOIN app.persona_card pc
|
||||
ON pc.persona_id = s.persona_id AND pc.version = s.persona_version
|
||||
LEFT JOIN app.feedback_scores fs ON fs.turn_id = t.id
|
||||
LEFT JOIN app.turn_technique tt ON tt.turn_id = t.id
|
||||
LEFT JOIN app.technique_label_def tech ON tech.label_id = tt.label_id
|
||||
LEFT JOIN app.turn_client_state tcs ON tcs.turn_id = t.id
|
||||
LEFT JOIN app.client_state_def cs ON cs.label_id = tcs.label_id
|
||||
LEFT JOIN app.supervisor_comment sc ON sc.turn_id = t.id
|
||||
WHERE u.is_active
|
||||
AND t.text_masked IS NOT NULL
|
||||
AND btrim(t.text_masked) <> ''
|
||||
AND ($1::timestamptz IS NULL OR t.created_at >= $1::timestamptz)
|
||||
AND ($2::timestamptz IS NULL OR t.created_at < $2::timestamptz)
|
||||
AND ($4::boolean = false OR u.cohort = $3::text)
|
||||
GROUP BY s.id, s.learner_id, pc.code, t.id, t.seq, t.speaker, t.stage, t.text_masked, t.created_at
|
||||
ORDER BY t.created_at, s.id, t.seq
|
||||
LIMIT $5
|
||||
"""
|
||||
|
||||
ANNOTATION_QUERY = """
|
||||
SELECT
|
||||
di.item_id::text AS item_id,
|
||||
ar.round_id::text AS round_id,
|
||||
a.annotator,
|
||||
a.labels
|
||||
FROM ds.annotation a
|
||||
JOIN ds.annotation_round ar ON ar.round_id = a.round_id
|
||||
JOIN ds.dataset_item di ON di.item_id = a.item_id
|
||||
WHERE ar.dataset_id = $1
|
||||
ORDER BY di.item_id, ar.round_id, a.annotation_id
|
||||
"""
|
||||
|
||||
|
||||
def parse_time(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
normalized = value.replace("Z", "+00:00")
|
||||
parsed = datetime.fromisoformat(normalized)
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=UTC)
|
||||
return parsed
|
||||
|
||||
|
||||
def make_export_id() -> str:
|
||||
return f"phase3-rl-seed-{datetime.now(UTC).strftime('%Y%m%d')}-001"
|
||||
|
||||
|
||||
def row_to_dict(row: asyncpg.Record) -> dict[str, Any]:
|
||||
return {key: row[key] for key in row.keys()}
|
||||
|
||||
|
||||
async def fetch_rows(conn: asyncpg.Connection, args: argparse.Namespace) -> list[dict[str, Any]]:
|
||||
rows = await conn.fetch(
|
||||
TURN_QUERY,
|
||||
parse_time(args.started_at),
|
||||
parse_time(args.ended_at),
|
||||
args.cohort_id,
|
||||
args.apply_cohort_filter,
|
||||
args.limit,
|
||||
)
|
||||
return [row_to_dict(row) for row in rows]
|
||||
|
||||
|
||||
async def write_dataset_rows(
|
||||
conn: asyncpg.Connection,
|
||||
*,
|
||||
dataset_name: str,
|
||||
purpose: str,
|
||||
raw_rows: list[dict[str, Any]],
|
||||
records: list[dict[str, Any]],
|
||||
) -> int:
|
||||
dataset_id = await conn.fetchval(
|
||||
"INSERT INTO ds.dataset (name, purpose) VALUES ($1, $2) RETURNING dataset_id",
|
||||
dataset_name,
|
||||
purpose,
|
||||
)
|
||||
for raw_row, record in zip(raw_rows, records):
|
||||
await conn.execute(
|
||||
"INSERT INTO ds.dataset_item (dataset_id, turn_id, payload) VALUES ($1, $2::uuid, $3::jsonb)",
|
||||
dataset_id,
|
||||
raw_row["turn_id"],
|
||||
json.dumps(record, ensure_ascii=False),
|
||||
)
|
||||
return int(dataset_id)
|
||||
|
||||
|
||||
async def fetch_agreement(conn: asyncpg.Connection, dataset_id: int | None, args: argparse.Namespace) -> dict[str, Any]:
|
||||
if dataset_id is None:
|
||||
return {"kappa": None, "icc": None, "gold_status": "not_gold"}
|
||||
rows = [row_to_dict(row) for row in await conn.fetch(ANNOTATION_QUERY, dataset_id)]
|
||||
kappa = cohen_kappa(rows, args.kappa_label)
|
||||
icc = intraclass_correlation(rows, args.icc_label)
|
||||
gold_status = "gold_candidate" if kappa is not None and kappa >= 0.60 and icc is not None and icc >= 0.75 else "not_gold"
|
||||
return {"kappa": kappa, "icc": icc, "gold_status": gold_status}
|
||||
|
||||
|
||||
async def insert_manifest_row(
|
||||
conn: asyncpg.Connection,
|
||||
*,
|
||||
dataset_id: int,
|
||||
agreement: dict[str, Any],
|
||||
jsonl_path: str,
|
||||
) -> None:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO ds.export_manifest (dataset_id, iaa_kappa, iaa_icc, jsonl_path)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
""",
|
||||
dataset_id,
|
||||
agreement.get("kappa"),
|
||||
agreement.get("icc"),
|
||||
jsonl_path,
|
||||
)
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> int:
|
||||
if args.export_status == APPROVED_EXPORT_STATUS and not args.allow_approved:
|
||||
raise SystemExit("--export-status approved_for_recursive_learning_seed requires --allow-approved")
|
||||
if args.write_manifest_row and not args.write_dataset:
|
||||
raise SystemExit("--write-manifest-row requires --write-dataset")
|
||||
|
||||
output_root = Path(args.output_root).resolve()
|
||||
export_dir = output_root / "03-export"
|
||||
jsonl_path = export_dir / "anonymized_dataset.jsonl"
|
||||
manifest_path = export_dir / "export_manifest.json"
|
||||
|
||||
conn = await asyncpg.connect(args.database_url)
|
||||
try:
|
||||
raw_rows = await fetch_rows(conn, args)
|
||||
keys = ExportKeyMaps()
|
||||
records = [
|
||||
build_dataset_record(
|
||||
row,
|
||||
item_index=index,
|
||||
export_manifest_id=args.export_id,
|
||||
keys=keys,
|
||||
pii_scan_status="pending",
|
||||
)
|
||||
for index, row in enumerate(raw_rows, start=1)
|
||||
]
|
||||
|
||||
pii_findings: list[dict[str, Any]] = []
|
||||
for record in records:
|
||||
for finding in scan_for_pii(record):
|
||||
pii_findings.append({"item_id": record["item_id"], **finding})
|
||||
if pii_findings and args.fail_on_pii:
|
||||
raise SystemExit(f"PII scan found {len(pii_findings)} finding(s)")
|
||||
|
||||
pii_status = "pass" if not pii_findings else "fail"
|
||||
for record in records:
|
||||
record["privacy"]["pii_scan_status"] = pii_status
|
||||
|
||||
write_jsonl(records, jsonl_path)
|
||||
digest = sha256_file(jsonl_path)
|
||||
|
||||
dataset_id: int | None = None
|
||||
if args.write_dataset:
|
||||
dataset_id = await write_dataset_rows(
|
||||
conn,
|
||||
dataset_name=args.dataset_name,
|
||||
purpose=args.purpose,
|
||||
raw_rows=raw_rows,
|
||||
records=records,
|
||||
)
|
||||
|
||||
agreement = await fetch_agreement(conn, dataset_id, args)
|
||||
approvals = {
|
||||
"data_steward": args.data_steward,
|
||||
"legal_or_privacy_reviewer": args.legal_or_privacy_reviewer,
|
||||
"technical_operator": args.technical_operator or os.environ.get("USERNAME", ""),
|
||||
"approved_at": args.approved_at,
|
||||
}
|
||||
manifest = build_manifest(
|
||||
export_id=args.export_id,
|
||||
dataset_name=args.dataset_name,
|
||||
export_status=args.export_status,
|
||||
purpose=args.purpose,
|
||||
records=records,
|
||||
jsonl_path=str(jsonl_path.relative_to(output_root)).replace("\\", "/"),
|
||||
jsonl_sha256=digest,
|
||||
pii_findings=pii_findings,
|
||||
participants_included=len(keys.participant),
|
||||
cohort_id=args.cohort_id,
|
||||
consent_version=args.consent_version,
|
||||
agreement=agreement,
|
||||
approvals=approvals,
|
||||
known_limitations=args.known_limitation,
|
||||
)
|
||||
validate_manifest_gate(manifest)
|
||||
manifest_path.write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
if args.write_manifest_row and dataset_id is not None:
|
||||
await insert_manifest_row(
|
||||
conn,
|
||||
dataset_id=dataset_id,
|
||||
agreement=agreement,
|
||||
jsonl_path=str(jsonl_path),
|
||||
)
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
print(json.dumps({"manifest": str(manifest_path), "jsonl": str(jsonl_path), "rows": len(records)}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Export a Phase 3 recursive-learning JSONL dry-run dataset.")
|
||||
parser.add_argument("--database-url", default=os.environ.get("DATABASE_URL"), required=os.environ.get("DATABASE_URL") is None)
|
||||
parser.add_argument("--output-root", default=str(REPO_ROOT / "data" / "phase3-dry-run"))
|
||||
parser.add_argument("--export-id", default=make_export_id())
|
||||
parser.add_argument("--dataset-name", default="vignette_phase3_recursive_learning_seed")
|
||||
parser.add_argument("--purpose", default="recursive-learning seed dataset for education simulator improvement")
|
||||
parser.add_argument("--export-status", choices=[DRY_RUN_EXPORT_STATUS, "blocked", APPROVED_EXPORT_STATUS], default=DRY_RUN_EXPORT_STATUS)
|
||||
parser.add_argument("--allow-approved", action="store_true")
|
||||
parser.add_argument("--started-at")
|
||||
parser.add_argument("--ended-at")
|
||||
parser.add_argument("--limit", type=int, default=1000)
|
||||
parser.add_argument("--cohort-id", default="phase3")
|
||||
parser.add_argument("--apply-cohort-filter", action="store_true")
|
||||
parser.add_argument("--consent-version", default="")
|
||||
parser.add_argument("--data-steward", default="")
|
||||
parser.add_argument("--legal-or-privacy-reviewer", default="")
|
||||
parser.add_argument("--technical-operator", default="")
|
||||
parser.add_argument("--approved-at", default="")
|
||||
parser.add_argument("--known-limitation", action="append", default=[])
|
||||
parser.add_argument("--fail-on-pii", action="store_true")
|
||||
parser.add_argument("--kappa-label", default="appropriateness")
|
||||
parser.add_argument("--icc-label", default="rapport_signal")
|
||||
parser.add_argument("--write-dataset", action="store_true")
|
||||
parser.add_argument("--write-manifest-row", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
return asyncio.run(run(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -2,6 +2,7 @@ param(
|
|||
[string]$Workspace = "D:\workspace\vignette",
|
||||
[string]$TaskName = "VignettePublicRuntimeWatchdog",
|
||||
[int]$IntervalMinutes = 5,
|
||||
[string[]]$AdditionalPublicHealthUrls = @(),
|
||||
[switch]$SkipPublicHealth,
|
||||
[switch]$SkipCloudflaredRestart,
|
||||
[switch]$RunNow
|
||||
|
|
@ -33,6 +34,10 @@ if ($SkipPublicHealth) {
|
|||
if ($SkipCloudflaredRestart) {
|
||||
$actionArguments += "-SkipCloudflaredRestart"
|
||||
}
|
||||
if ($AdditionalPublicHealthUrls.Count -gt 0) {
|
||||
$escapedUrls = $AdditionalPublicHealthUrls | ForEach-Object { "`"$_`"" }
|
||||
$actionArguments += "-AdditionalPublicHealthUrls $($escapedUrls -join ',')"
|
||||
}
|
||||
|
||||
$action = New-ScheduledTaskAction `
|
||||
-Execute $powershell `
|
||||
|
|
@ -73,6 +78,9 @@ Register-ScheduledTask -TaskName $TaskName -InputObject $task -Force | Out-Null
|
|||
Write-Output "Installed scheduled task '$TaskName' for $userId"
|
||||
Write-Output "Action: $powershell $($actionArguments -join ' ')"
|
||||
Write-Output "Interval: every $IntervalMinutes minute(s), plus at user logon"
|
||||
if ($AdditionalPublicHealthUrls.Count -gt 0) {
|
||||
Write-Output "Additional public health URLs: $($AdditionalPublicHealthUrls -join ', ')"
|
||||
}
|
||||
|
||||
if ($RunNow) {
|
||||
Start-ScheduledTask -TaskName $TaskName
|
||||
|
|
|
|||
|
|
@ -188,6 +188,7 @@ $env:ENGINE_MODE = "claude_cli"
|
|||
$env:AUTH_DEV_LOGIN_ENABLED = "false"
|
||||
$env:AUTO_SEED_PERSONAS = "false"
|
||||
$env:ALLOW_SEED_PERSONA_FALLBACK = "false"
|
||||
$env:VIGNETTE_VOICE_POC_SAMPLE_TTS = "false"
|
||||
$env:FRONTEND_BASE_URL = "https://vignette.chanpaca.net"
|
||||
$frontendOrigins = @("https://vignette.chanpaca.net", "https://vnet.18ka.net", "https://vignette-b1q.pages.dev")
|
||||
$localViteOrigins = @()
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
[string]$Cloudflared = "$env:LOCALAPPDATA\Microsoft\WinGet\Links\cloudflared.exe",
|
||||
[string]$CloudflaredConfig = "$env:USERPROFILE\.cloudflared\vignette-config.yml",
|
||||
[string]$PublicHealthUrl = "https://api-vignette.chanpaca.net/health",
|
||||
[string[]]$AdditionalPublicHealthUrls = @("https://api-vnet.18ka.net/health"),
|
||||
[string[]]$AdditionalPublicHealthUrls = @(),
|
||||
[string]$LogPath = "",
|
||||
[switch]$CheckOnly,
|
||||
[switch]$SkipPublicHealth,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue