"""Text-fidelity regression tests: em dashes and CJK must survive EPUB -> WeasyPrint -> pypdf merge without mojibake. Markers below were verified against test_book.epub ground truth: the source contains 3 em dashes and the CJK runs listed in CJK_MARKERS. These tests lock in the UTF-8 HTML-string decoding fix (app.py L258-265): passing decoded text (not file paths) to WeasyPrint prevents Latin-1 sniffing from turning em dashes/CJK into mojibake. """ import io from pypdf import PdfReader from tests.helpers import download_bytes, pdf_metadata_title, wait_for_job EM_DASH = "\u2014" CJK_MARKERS = ["中文测试段落", "天地玄黄", "宇宙洪荒", "日月盈昃", "辰宿列张"] EXPECTED_TITLE = "Structured Test Book" EXPECTED_AUTHOR = "Agent Zero" def _converted_pdf(client, submit_epub): job_id = submit_epub(filename="book.epub").get_json()["job_id"] job = wait_for_job(client, job_id) assert job["status"] == "done", job return download_bytes(client, job["filename"]), job def test_em_dashes_survive_conversion(client, submit_epub): data, _ = _converted_pdf(client, submit_epub) reader = PdfReader(io.BytesIO(data)) full = "\n".join((p.extract_text() or "") for p in reader.pages) assert EM_DASH in full, "em dash missing from PDF text" assert full.count(EM_DASH) >= 3, f"expected >=3 em dashes, got {full.count(EM_DASH)}" def test_cjk_text_survives_conversion(client, submit_epub): data, _ = _converted_pdf(client, submit_epub) reader = PdfReader(io.BytesIO(data)) full = "\n".join((p.extract_text() or "") for p in reader.pages) for marker in CJK_MARKERS: assert marker in full, f"CJK marker {marker!r} missing from PDF text" def test_no_latin1_mojibake(client, submit_epub): """Mojibake regression: em dash mis-decoded as Latin-1 reads 'â€"'.""" data, _ = _converted_pdf(client, submit_epub) reader = PdfReader(io.BytesIO(data)) full = "\n".join((p.extract_text() or "") for p in reader.pages) assert "â€" not in full, "Latin-1 mojibake detected for em dash" assert "ï¼" not in full, "CJK fullwidth colon mojibake detected" def test_pdf_metadata_round_trip(client, submit_epub): data, _ = _converted_pdf(client, submit_epub) assert pdf_metadata_title(data) == EXPECTED_TITLE reader = PdfReader(io.BytesIO(data)) assert reader.metadata["/Author"] == EXPECTED_AUTHOR def test_page_count_matches_fixture(client, submit_epub): """The 6-page fixture must not silently drop or merge chapters.""" data, _ = _converted_pdf(client, submit_epub) reader = PdfReader(io.BytesIO(data)) assert len(reader.pages) == 6