chore: 저장소 구조 정리 및 문서화, 첫 커밋
- src/dist 산출물 분리 원칙 정리(.gitignore, .gitattributes) - 루트 및 주요 폴더(config/scripts/prompts/tests/src, 런타임 폴더 5종)에 안내용 README.md 추가 - CHANGELOG.md, LICENSE, docs/ops/05-release-and-versioning.md 추가 - docs/README.md 문서 지도 갱신
This commit is contained in:
Yun Chan
commit
56a6e2da93
159 changed files with 145825 additions and 0 deletions
370
tests/test_report_e2e_design.py
Normal file
370
tests/test_report_e2e_design.py
Normal file
|
|
@ -0,0 +1,370 @@
|
|||
from __future__ import annotations
|
||||
|
||||
"""R4 xlsx E2E/design RED.
|
||||
|
||||
이 테스트가 Green 되기 전까지 "리포트가 동작한다"고 말하지 않는다.
|
||||
실제 SQLite DB 에 기준선/둘째 날 변경분을 적재하고, build_report() 가 만든 xlsx 를
|
||||
zip/XML 로 열어 사용자가 받을 산출물이 의미 있는지 감사한다.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import xml.etree.ElementTree as ET
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from dmf_crawler import paths
|
||||
from dmf_crawler.config import AgyCfg, BackupCfg, Config, LoggingCfg, ReportCfg, StorageCfg
|
||||
from dmf_crawler.models import DiffEvent, DiffResult, DmfRecord, FetchResult, FieldChange, Gate, IntegrityReport, IntegrityVerdict
|
||||
from dmf_crawler.report import build_report
|
||||
from dmf_crawler.report.theme import SHEET_NAMES, TAB_COLORS
|
||||
from dmf_crawler.storage import db, repo
|
||||
|
||||
NS_MAIN = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
|
||||
NS_REL = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
|
||||
NS_PACKAGE_REL = "{http://schemas.openxmlformats.org/package/2006/relationships}"
|
||||
|
||||
|
||||
def _bootstrap_temp_db(tmp_path: Path) -> sqlite3.Connection:
|
||||
conn = db.connect(tmp_path / "data" / "dmf.sqlite3")
|
||||
db.apply_migrations(conn, paths.MIGRATIONS_DIR, tmp_path / "backup")
|
||||
return conn
|
||||
|
||||
|
||||
def _record_fetch(conn: sqlite3.Connection, run_id: str, total: int) -> None:
|
||||
repo.record_fetch_stats(
|
||||
conn,
|
||||
run_id,
|
||||
FetchResult(
|
||||
records=(),
|
||||
total_count_reported=total,
|
||||
pages_fetched=1,
|
||||
pages_expected=1,
|
||||
http_calls=1,
|
||||
elapsed_seconds=0.25,
|
||||
archive_dir=Path("data/raw/test"),
|
||||
body_signature_ok=True,
|
||||
result_code="00",
|
||||
result_msg="NORMAL SERVICE",
|
||||
payload_sha256=f"payload-{run_id}",
|
||||
endpoint="https://apis.data.go.kr/1471000/MdcDmfInfoService01/getMdcDmfList01?serviceKey=***",
|
||||
page_size=100,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _record_quality(conn: sqlite3.Connection, run_id: str) -> None:
|
||||
gates = (
|
||||
Gate("transport_ok", True, observed=1, threshold=1, detail="응답 본문 정상"),
|
||||
Gate("total_count_match", True, observed=3, threshold=3, detail="신고 건수와 수신 건수 일치"),
|
||||
Gate("drop_ratio", True, observed=0.0, threshold=0.05, detail="급감 없음"),
|
||||
Gate("null_ratio", True, observed=0.0, threshold=0.01, detail="필수 필드 널 없음"),
|
||||
Gate("duplicate_ratio", True, observed=0.0, threshold=0.02, detail="등록번호 중복 없음"),
|
||||
Gate("churn_ratio", True, observed=1.0, threshold=1.0, detail="테스트 fixture 변동 허용"),
|
||||
Gate("withdrawn_ratio", True, blocking=False, observed=0.5, threshold=1.0, detail="테스트 fixture 취하 1건"),
|
||||
)
|
||||
report = IntegrityReport(pre=IntegrityVerdict(ok=True, gates=gates), population=3)
|
||||
repo.record_quality_checks(conn, run_id, report)
|
||||
|
||||
|
||||
def _seed_meaningful_delta(conn: sqlite3.Connection, record_factory: Callable[..., DmfRecord]) -> str:
|
||||
"""기준선 2건 + 둘째 날 신규/변경/취하/워치리스트 fixture 를 DB 에 넣는다."""
|
||||
day1 = "run_20260903_baseline"
|
||||
original = record_factory(
|
||||
"20200101-1-A-1-1",
|
||||
ingredient_name="아토르바스타틴칼슘 삼수화물",
|
||||
manufacturer="Old Factory",
|
||||
countries=("대한민국", "이탈리아"),
|
||||
)
|
||||
withdrawn = record_factory(
|
||||
"20200102-2-A-2-1",
|
||||
ingredient_name="메트포르민염산염 초장문성분명 레이아웃검사용 가나다라마바사아자차카타파하",
|
||||
manufacturer="Withdrawn Factory",
|
||||
countries=("미국",),
|
||||
)
|
||||
repo.start_run(conn, day1, "2026-09-03", "manual")
|
||||
repo.insert_snapshot(conn, day1, [original, withdrawn])
|
||||
repo.upsert_current_records(conn, day1, [original, withdrawn])
|
||||
_record_fetch(conn, day1, 2)
|
||||
repo.finish_run(conn, day1, "SUCCESS", 0, "baseline", is_baseline=True, integrity_ok=True)
|
||||
|
||||
day2 = "run_20260904_delta"
|
||||
changed = record_factory(
|
||||
"20200101-1-A-1-1",
|
||||
ingredient_name="아토르바스타틴칼슘 삼수화물",
|
||||
manufacturer="New Factory",
|
||||
countries=("대한민국", "스위스"),
|
||||
)
|
||||
new = record_factory(
|
||||
"20200103-3-A-3-1",
|
||||
ingredient_name="세마글루타이드",
|
||||
manufacturer="Brand New Factory",
|
||||
countries=("덴마크",),
|
||||
)
|
||||
diff = DiffResult(
|
||||
new=(
|
||||
DiffEvent(
|
||||
dmf_key=new.dmf_key,
|
||||
permit_no=new.permit_no,
|
||||
event_type="NEW",
|
||||
severity="INFO",
|
||||
after=new.compare_view(),
|
||||
after_hash=new.content_hash,
|
||||
),
|
||||
),
|
||||
changed=(
|
||||
DiffEvent(
|
||||
dmf_key=changed.dmf_key,
|
||||
permit_no=changed.permit_no,
|
||||
event_type="CHANGED",
|
||||
severity="HIGH",
|
||||
changes=(
|
||||
FieldChange("manufacturer", "제조소명", "Old Factory", "New Factory", "HIGH"),
|
||||
FieldChange("countries", "제조국가", "대한민국, 이탈리아", "대한민국, 스위스", "MEDIUM"),
|
||||
),
|
||||
before=original.compare_view(),
|
||||
after=changed.compare_view(),
|
||||
before_hash=original.content_hash,
|
||||
after_hash=changed.content_hash,
|
||||
),
|
||||
),
|
||||
withdrawn=(
|
||||
DiffEvent(
|
||||
dmf_key=withdrawn.dmf_key,
|
||||
permit_no=withdrawn.permit_no,
|
||||
event_type="WITHDRAWN",
|
||||
severity="CRITICAL",
|
||||
before=withdrawn.compare_view(),
|
||||
before_hash=withdrawn.content_hash,
|
||||
),
|
||||
),
|
||||
)
|
||||
repo.start_run(conn, day2, "2026-09-04", "scheduled")
|
||||
repo.insert_snapshot(conn, day2, [changed, new])
|
||||
repo.insert_events(conn, day2, diff)
|
||||
repo.upsert_current_records(conn, day2, [changed, new])
|
||||
_record_fetch(conn, day2, 3)
|
||||
_record_quality(conn, day2)
|
||||
|
||||
now = "2026-09-04T06:00:00+09:00"
|
||||
conn.execute(
|
||||
"""INSERT INTO watchlist(axis, pattern, pattern_key, match_mode, label, enabled, created_at)
|
||||
VALUES ('ingredient', '아토르바스타틴', '아토르바스타틴', 'contains', '스타틴 계열', 1, ?)""",
|
||||
(now,),
|
||||
)
|
||||
watch_id = int(conn.execute("SELECT last_insert_rowid()").fetchone()[0])
|
||||
changed_record_id = conn.execute("SELECT record_id FROM records WHERE dmf_key = ?", (changed.dmf_key,)).fetchone()[0]
|
||||
changed_event_id = conn.execute("SELECT event_id FROM events WHERE run_seq = (SELECT run_seq FROM runs WHERE run_id = ?) AND dmf_key = ?", (day2, changed.dmf_key)).fetchone()[0]
|
||||
run_seq = conn.execute("SELECT run_seq FROM runs WHERE run_id = ?", (day2,)).fetchone()[0]
|
||||
conn.execute(
|
||||
"""INSERT INTO watchlist_hits(run_seq, watch_id, event_id, record_id, matched_on)
|
||||
VALUES (?, ?, ?, ?, ?)""",
|
||||
(run_seq, watch_id, changed_event_id, changed_record_id, "아토르바스타틴칼슘 삼수화물"),
|
||||
)
|
||||
|
||||
repo.insert_enrichment(
|
||||
conn,
|
||||
day2,
|
||||
{"headline": "취하 1건, 신규 1건, 제조소 변경 1건", "summary_md": "fixture 기반 일일 브리핑", "anomalies": []},
|
||||
[{"dmf_key": changed.dmf_key, "comment": "워치리스트 성분 제조소 변경"}],
|
||||
)
|
||||
repo.finish_run(
|
||||
conn,
|
||||
day2,
|
||||
"SUCCESS",
|
||||
0,
|
||||
"meaningful delta",
|
||||
report_path="reports/DMF_리포트_2026-09-04.xlsx",
|
||||
is_baseline=False,
|
||||
integrity_ok=True,
|
||||
diff_performed=True,
|
||||
)
|
||||
return day2
|
||||
|
||||
|
||||
def _read_workbook_sheet_order(zf: zipfile.ZipFile) -> tuple[str, ...]:
|
||||
root = ET.fromstring(zf.read("xl/workbook.xml"))
|
||||
return tuple(sheet.attrib["name"] for sheet in root.findall(f".//{NS_MAIN}sheet"))
|
||||
|
||||
|
||||
def _worksheet_paths_by_name(zf: zipfile.ZipFile) -> dict[str, str]:
|
||||
workbook = ET.fromstring(zf.read("xl/workbook.xml"))
|
||||
rels = ET.fromstring(zf.read("xl/_rels/workbook.xml.rels"))
|
||||
id_to_target = {rel.attrib["Id"]: rel.attrib["Target"] for rel in rels.findall(f".//{NS_PACKAGE_REL}Relationship")}
|
||||
result: dict[str, str] = {}
|
||||
for sheet in workbook.findall(f".//{NS_MAIN}sheet"):
|
||||
rel_id = sheet.attrib[f"{NS_REL}id"]
|
||||
target = id_to_target[rel_id].lstrip("/")
|
||||
if not target.startswith("xl/"):
|
||||
target = "xl/" + target
|
||||
result[sheet.attrib["name"]] = target
|
||||
return result
|
||||
|
||||
|
||||
def _shared_strings(zf: zipfile.ZipFile) -> list[str]:
|
||||
if "xl/sharedStrings.xml" not in zf.namelist():
|
||||
return []
|
||||
root = ET.fromstring(zf.read("xl/sharedStrings.xml"))
|
||||
strings: list[str] = []
|
||||
for si in root.findall(f"{NS_MAIN}si"):
|
||||
strings.append("".join(node.text or "" for node in si.iter(f"{NS_MAIN}t")))
|
||||
return strings
|
||||
|
||||
|
||||
def _shared_text(zf: zipfile.ZipFile) -> str:
|
||||
return "\n".join(_shared_strings(zf))
|
||||
|
||||
|
||||
def _sheet_root(zf: zipfile.ZipFile, sheet_path: str) -> ET.Element:
|
||||
return ET.fromstring(zf.read(sheet_path))
|
||||
|
||||
|
||||
def _pane(root: ET.Element) -> ET.Element | None:
|
||||
return root.find(f".//{NS_MAIN}pane")
|
||||
|
||||
|
||||
def _has_autofilter_or_table(root: ET.Element) -> bool:
|
||||
return root.find(f".//{NS_MAIN}autoFilter") is not None or root.find(f".//{NS_MAIN}tableParts") is not None
|
||||
|
||||
|
||||
def _conditional_count(root: ET.Element) -> int:
|
||||
return len(root.findall(f".//{NS_MAIN}conditionalFormatting"))
|
||||
|
||||
|
||||
def _cell_text(root: ET.Element, ref: str, shared: list[str]) -> str:
|
||||
cell = root.find(f".//{NS_MAIN}c[@r='{ref}']")
|
||||
if cell is None:
|
||||
return ""
|
||||
if cell.attrib.get("t") == "s":
|
||||
value = cell.find(f"{NS_MAIN}v")
|
||||
if value is None or value.text is None:
|
||||
return ""
|
||||
idx = int(value.text)
|
||||
return shared[idx] if 0 <= idx < len(shared) else ""
|
||||
inline = cell.find(f"{NS_MAIN}is/{NS_MAIN}t")
|
||||
if inline is not None:
|
||||
return inline.text or ""
|
||||
value = cell.find(f"{NS_MAIN}v")
|
||||
return value.text or "" if value is not None else ""
|
||||
|
||||
|
||||
def _hyperlink_locations(root: ET.Element) -> list[str]:
|
||||
return [node.attrib.get("location", "") for node in root.findall(f".//{NS_MAIN}hyperlink")]
|
||||
|
||||
|
||||
def _visible_column_widths(root: ET.Element) -> list[float]:
|
||||
widths: list[float] = []
|
||||
for node in root.findall(f".//{NS_MAIN}cols/{NS_MAIN}col"):
|
||||
if node.attrib.get("hidden") == "1":
|
||||
continue
|
||||
width = node.attrib.get("width")
|
||||
if width is not None:
|
||||
widths.append(float(width))
|
||||
return widths
|
||||
|
||||
|
||||
def test_build_report_emits_meaningful_xlsx_with_delta_and_design_contracts(
|
||||
tmp_path: Path, record_factory: Callable[..., DmfRecord]
|
||||
) -> None:
|
||||
conn = _bootstrap_temp_db(tmp_path)
|
||||
try:
|
||||
run_id = _seed_meaningful_delta(conn, record_factory)
|
||||
cfg = Config().with_overrides(
|
||||
storage=StorageCfg(sqlite_path=str(tmp_path / "data" / "dmf.sqlite3")),
|
||||
report=ReportCfg(output_dir=str(tmp_path / "reports"), filename_pattern="DMF_리포트_{date}.xlsx", latest_link_name="DMF_리포트_최신.xlsx"),
|
||||
backup=BackupCfg(dir=str(tmp_path / "backup")),
|
||||
logging=LoggingCfg(dir=str(tmp_path / "logs"), console=False),
|
||||
agy=AgyCfg(enabled=False),
|
||||
)
|
||||
|
||||
outcome = build_report(conn, run_id, cfg)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
assert outcome.path.exists(), "xlsx 파일이 실제 생성돼야 한다"
|
||||
assert outcome.path.stat().st_size > 15_000, "빈 껍데기 수준 파일은 의미 있는 리포트가 아니다"
|
||||
assert outcome.rows_written >= 20
|
||||
assert outcome.latest_link_path is not None and outcome.latest_link_path.exists()
|
||||
|
||||
with zipfile.ZipFile(outcome.path) as zf:
|
||||
names = set(zf.namelist())
|
||||
assert "[Content_Types].xml" in names
|
||||
assert "xl/workbook.xml" in names
|
||||
assert "xl/styles.xml" in names
|
||||
sheet_order = _read_workbook_sheet_order(zf)
|
||||
assert sheet_order == tuple(SHEET_NAMES[key] for key in ("dashboard", "changes", "ledger", "ingredient", "company", "watchlist", "trend", "meta"))
|
||||
paths_by_sheet = _worksheet_paths_by_name(zf)
|
||||
shared_strings = _shared_strings(zf)
|
||||
text = "\n".join(shared_strings)
|
||||
|
||||
for must in (
|
||||
"신규",
|
||||
"변경",
|
||||
"취하",
|
||||
"아토르바스타틴칼슘 삼수화물",
|
||||
"세마글루타이드",
|
||||
"메트포르민염산염 초장문성분명",
|
||||
"워치리스트 성분 제조소 변경",
|
||||
"원문 조회",
|
||||
"출처:",
|
||||
"식품의약품안전처",
|
||||
):
|
||||
assert must in text, f"리포트 내용 누락: {must}"
|
||||
|
||||
core_xml = zf.read("docProps/core.xml").decode("utf-8")
|
||||
app_xml = zf.read("docProps/app.xml").decode("utf-8")
|
||||
assert "DMF 일일 모니터링 리포트" in core_xml
|
||||
assert "규제 인텔리전스" in core_xml
|
||||
assert "DMF Crawler" in core_xml
|
||||
assert "00_대시보드" in app_xml and "99_메타" in app_xml
|
||||
|
||||
dashboard_root = _sheet_root(zf, paths_by_sheet["00_대시보드"])
|
||||
a1 = _cell_text(dashboard_root, "A1", shared_strings)
|
||||
assert "대시보드" in a1 and "신규" in a1 and "취하" in a1, "A1에는 스크린리더용 목적문이 있어야 한다"
|
||||
dashboard_links = "\n".join(_hyperlink_locations(dashboard_root))
|
||||
for target_sheet in ("01_오늘변경분", "02_전체현황", "03_성분별", "04_업체별", "05_워치리스트", "06_추이", "99_메타"):
|
||||
assert target_sheet in dashboard_links, f"대시보드 내부 이동 링크 누락: {target_sheet}"
|
||||
|
||||
for sheet_name in sheet_order[1:]:
|
||||
root = _sheet_root(zf, paths_by_sheet[sheet_name])
|
||||
assert any("00_대시보드" in loc for loc in _hyperlink_locations(root)), f"{sheet_name}: 대시보드 역링크 누락"
|
||||
max_width = max(_visible_column_widths(root) or [0.0])
|
||||
assert max_width <= 48.75, f"{sheet_name}: visible column width {max_width:.2f} is too wide for Korean xlsx scanning"
|
||||
|
||||
expected_freezes = {
|
||||
"01_오늘변경분": ("5", "3"),
|
||||
"02_전체현황": ("3", "3"),
|
||||
"03_성분별": ("2", "3"),
|
||||
"04_업체별": ("2", "3"),
|
||||
"05_워치리스트": ("3", "3"),
|
||||
"06_추이": ("1", "3"),
|
||||
"99_메타": (None, "3"),
|
||||
}
|
||||
for sheet_name, (x_split, y_split) in expected_freezes.items():
|
||||
root = _sheet_root(zf, paths_by_sheet[sheet_name])
|
||||
pane = _pane(root)
|
||||
assert pane is not None, f"{sheet_name}: freeze panes 누락"
|
||||
if x_split is not None:
|
||||
assert pane.attrib.get("xSplit") == x_split, f"{sheet_name}: xSplit mismatch"
|
||||
assert pane.attrib.get("ySplit") == y_split, f"{sheet_name}: ySplit mismatch"
|
||||
|
||||
for sheet_name in ("01_오늘변경분", "02_전체현황", "03_성분별", "04_업체별", "05_워치리스트", "06_추이"):
|
||||
root = _sheet_root(zf, paths_by_sheet[sheet_name])
|
||||
assert _has_autofilter_or_table(root), f"{sheet_name}: 필터/표 누락"
|
||||
|
||||
conditional_total = sum(
|
||||
_conditional_count(_sheet_root(zf, path)) for path in paths_by_sheet.values()
|
||||
)
|
||||
assert conditional_total >= 10, "상태/날짜/워치/게이트 조건부서식이 충분히 있어야 한다"
|
||||
|
||||
for sheet_name, color in TAB_COLORS.items():
|
||||
root = _sheet_root(zf, paths_by_sheet[sheet_name])
|
||||
tab = root.find(f".//{NS_MAIN}sheetPr/{NS_MAIN}tabColor")
|
||||
assert tab is not None, f"{sheet_name}: 탭 색 누락"
|
||||
assert str(tab.attrib.get("rgb", "")).upper().endswith(color.replace("#", "").upper())
|
||||
|
||||
table_xml = "\n".join(
|
||||
zf.read(name).decode("utf-8") for name in zf.namelist() if name.startswith("xl/tables/table")
|
||||
)
|
||||
for table_name in ("T_LEDGER", "T_INGREDIENT", "T_COMPANY", "T_WATCH", "T_TREND"):
|
||||
assert table_name in table_xml, f"Excel 표 누락: {table_name}"
|
||||
Loading…
Add table
Add a link
Reference in a new issue