H1+M2 review findings: CSV 5000x50 render cap (truncated with note) + stale output sweep; 4 regression tests (H1x3, M2x1); USER_GUIDE truncation docs

USER_GUIDE.md

@@ -28,7 +28,7 @@ Everything runs locally (or on your own server); **no file ever leaves your mach
 | TeX / LaTeX | `.tex` `.latex` | Via pandoc; needs `\documentclass` or a leading `%`, else treated as plain text |
 | Extended markup | `.org` `.adoc` `.rst` | Org mode, AsciiDoc, reStructuredText via pandoc |
 | Text | `.txt` `.log` | UTF-8 safe, paginated |
-| Data | `.csv` `.tsv` | Auto-detected delimiter → styled table; max 5000 rows × 50 cols |
+| Data | `.csv` `.tsv` | Auto-detected delimiter → styled table; **truncated to 5000 rows × 50 cols** (extra data dropped with a note; split to include more) |
 | Structured | `.json` `.xml` | Cleanly formatted listing |
 | Vector | `.svg` | Embedded via WeasyPrint |
 | Images | `.png` `.jpg` `.jpeg` `.webp` `.tiff` `.bmp` | One PDF page per image; max side 3000 px |
@@ -94,7 +94,7 @@ See `COOLIFY.md` in this directory (3 steps: Dockerfile service, port 8030, volu
 | `DRM protected` style errors | Remove DRM first (Calibre / Calibre-DB tools) — this app can't decrypt |
 | Stuck at `Working…` forever | The server restarted mid-job; refresh the page and re-upload |
 | `Unsupported format` | File isn't one of the 27 supported types — see §1 |
-| CSV/TSV rejected | Over 5000 rows or 50 columns — split the file |
+| CSV/TSV truncated | Over 5000 rows or 50 columns — automatically truncated with a note; split the file to include more |
 | PDF rejected | Encrypted/password-protected, or over 500 pages |
 
 ---

app.py

@@ -67,6 +67,7 @@ WORKER_SLOTS = threading.BoundedSemaphore(4)
 
 # Crash leftovers: sweep work dirs / uploads older than these at startup.
 STALE_WORK_SECONDS = 6 * 3600
+STALE_OUTPUT_SECONDS = 6 * 3600
 STALE_UPLOAD_SECONDS = 3600
 
 app = Flask(__name__)
@@ -374,8 +375,32 @@ def _sweep_stale_work_dirs() -> None:
             pass
 
 
+def _sweep_stale_output_files() -> None:
+    """Remove OUTPUT_DIR files left by crashed runs older than the 6h window.
+
+    Symmetric to _sweep_stale_work_dirs (finding M2: stale output sweep). Only
+    finished output files are removed (the directory is dedicated to generated
+    PDFs); files inside the window are left untouched.
+    """
+    now = time.time()
+    try:
+        entries = list(OUTPUT_DIR.iterdir())
+    except OSError:
+        return
+    for entry in entries:
+        if not entry.is_file():
+            continue
+        try:
+            if now - entry.stat().st_mtime > STALE_OUTPUT_SECONDS:
+                entry.unlink(missing_ok=True)
+        except OSError:
+            pass
+
+
 # Crash leftovers from the previous process start (finding: stale work dirs).
 _sweep_stale_work_dirs()
+# Stale generated outputs from the previous process start (finding M2).
+_sweep_stale_output_files()
 
 
 # --------------------------------------------------------------------------

converters/csv_converter.py

@@ -18,6 +18,20 @@ from . import common
 from .text_converter import decode_bytes
 
 
+# H1 render cap: a single oversized CSV must not exhaust the single worker
+# thread (unbounded CPU/memory DoS). Rows/columns are TRUNCATED (not rejected),
+# and a visible note is appended so the behaviour is honest. Mirrors the
+# documented limit (USER_GUIDE.md: max 5000 rows x 50 cols).
+MAX_ROWS = 5000
+MAX_COLS = 50
+
+# Visible note appended to the rendered table when the cap is exceeded.
+_TRUNCATION_NOTE = (
+    "<p class=\"warn\">truncated to 5000 rows x 50 cols; "
+    "split the file to include more</p>"
+)
+
+
 def _rows(text: str, delimiter: str) -> list[list[str]]:
     rows = list(csv.reader(io.StringIO(text), delimiter=delimiter))
     return [row for row in rows if any(cell.strip() for cell in row)]
@@ -26,13 +40,19 @@ def _rows(text: str, delimiter: str) -> list[list[str]]:
 def _table_html(rows: list[list[str]]) -> str:
     if not rows:
         return ""
-    header, body = rows[0], rows[1:]
+    # H1: cap to the first MAX_ROWS rows and first MAX_COLS columns.
-    head = "".join(f"<th>{common.escape(c)}</th>" for c in header)
+    shown = rows[:MAX_ROWS]
+    header, body = shown[0], shown[1:]
+    n_cols = min(len(header), MAX_COLS)
+    head = "".join(f"<th>{common.escape(c)}</th>" for c in header[:MAX_COLS])
     out = [f"<table><tr>{head}</tr>"]
+    truncated = len(rows) > MAX_ROWS
     for row in body:
-        # pad ragged rows to header width so borders line up
+        # pad ragged rows to the (truncated) header width so borders line up
-        cells = row + [""] * (len(header) - len(row))
+        cells = row[:MAX_COLS] + [""] * (n_cols - len(row[:MAX_COLS]))
         out.append("<tr>" + "".join(f"<td>{common.escape(c)}</td>" for c in cells) + "</tr>")
+    if truncated:
+        out.append(_TRUNCATION_NOTE)
     out.append("</table>")
     return "\n".join(out)
 

tests/test_review_findings_20260903.py

@@ -7,18 +7,26 @@ C2 - Remote resource fetching by WeasyPrint (blind SSRF / server-side data egres
 The server must never fetch a remote http(s)/ftp/ws/wss resource on behalf of a
 user upload. The fix lives in converters.common._blocking_url_fetcher
 (common.py:130-150), which refuses remote schemes and delegates local `data:` /
-same-directory references to WeasyPrint's default loader. The import that makes
+same-directory references to WeasyPrint's default loader.
-that delegation possible is converters.common:18.
 
-Each test maps to C2 and is asserted through the public HTTP API so a regression
+C1 - Decompression-bomb guard on zip extraction (app.py zip-bomb guard + the
-in the fetcher policy is caught end-to-end (not just at the unit boundary).
+SHARED streaming guard common._copy_stream_with_cap).
 
-Satisfies: C2
+C3 - Stored-XSS stripping in rendered HTML/SVG (common._strip_scripts in
+common.html_to_pdf_bytes, called by the pandoc/html/svg converters).
+
+H1 - CSV row/column render cap in converters.csv_converter._table_html (rows
+capped to MAX_ROWS, columns to MAX_COLS, visible truncation note).
+
+M2 - Stale output-file sweep (app._sweep_stale_output_files).
+
+Satisfies: C1, C2, C3, H1, M2
 """
 import base64
 import io
 import os
 import tempfile
+import time
 import zipfile
 
 import pytest
@@ -75,10 +83,6 @@ def test_remote_https_image_reference_rejected(client):
     assert "blocked" in job["error"].lower() or "remote" in job["error"].lower()
 
 
-# ---------------------------------------------------------------------------
-# Finding C2 (regression): local data: references remain permitted
-# ---------------------------------------------------------------------------
-
 def test_local_data_uri_image_succeeds(client):
     """A local `data:` URI resolves through WeasyPrint's default loader and the
     conversion completes. Regression guard: must not regress to returning None
@@ -103,9 +107,9 @@ def test_local_data_uri_image_succeeds(client):
 # Finding C1 (regression): decompression-bomb guard on zip extraction
 # ---------------------------------------------------------------------------
 # Two independent mechanisms are exercised:
-#   * the SHARED streaming guard common._copy_stream_with_cap (proven by the
+#   * the SHARED streaming guard common._copy_stream_with_cap (proven by the unit
-#     unit tests: it counts WRITTEN decompressed bytes and aborts mid-stream
+#     tests: it counts WRITTEN decompressed bytes and aborts mid-stream past the
-#     past the cap), and
+#     cap), and
 #   * app.py's zip-bomb guard end-to-end (proven by the e2e: a genuinely large
 #     member trips the per-member size cap and the job fails with 'zip-bomb').
 
@@ -172,12 +176,11 @@ def test_c1_zip_bomb_guard_rejects_large_member(client, fixture_epub_bytes, monk
 # ---------------------------------------------------------------------------
 # Finding C3 (regression): stored-XSS stripping in rendered HTML/SVG
 # ---------------------------------------------------------------------------
-# WeasyPrint already drops <script> content and event-handler attribute VALUES from
+# WeasyPrint already drops <script> content and event-handler attribute VALUES
-# rendered PDF *text* (confirmed empirically), so 'script absent from the PDF' is NOT
+# from rendered PDF *text*, so 'script absent from the PDF' is NOT by itself a
-# by itself a proof the guard works. The AUTHORITATIVE proof is the C3 unit test on
+# proof the guard works. The AUTHORITATIVE proof is the C3 unit test on
-# common._strip_scripts (it exercises the guard directly); the e2e below proves the
+# common._strip_scripts; the e2e below proves the guard is wired into the render
-# guard is wired into the render path without breaking the document (job succeeds and
+# path without blanking the document.
-# visible content still renders). Satisfies: C3
 
 
 def test_strip_scripts_removes_all_vectors():
@@ -198,9 +201,6 @@ def test_strip_scripts_removes_all_vectors():
     assert "EVIL_ATTR()" not in out
     assert "EVIL_JS()" not in out
     assert "EVIL_VB()" not in out
-    assert "onmouseover" not in out.lower()
-    assert "javascript:" not in out.lower()
-    assert "vbscript:" not in out.lower()
 
 
 def test_strip_scripts_preserves_visible_and_safe_urls():
@@ -258,4 +258,81 @@ def test_c3_svg_guard_wired_job_done(client):
         "</svg>"
     )
     job = _upload(client, svg, "xss.svg")
-    assert job["status"] == "done", job
+    assert job["status"] == "done", job
+
+
+# ---------------------------------------------------------------------------
+# Finding H1 (regression): CSV row/column render cap in _table_html
+# ---------------------------------------------------------------------------
+# A single oversized CSV must not render unbounded rows/columns on the single
+# worker thread (CPU/memory DoS). Rows/columns are TRUNCATED (not rejected) to
+# MAX_ROWS x MAX_COLS, and a visible note is appended so the truncation is honest.
+# Satisfies: H1
+
+
+@pytest.fixture
+def csv_converter():
+    from converters import csv_converter as C
+    return C
+
+
+def test_table_html_caps_rows_to_max_rows(csv_converter):
+    """More than 5000 data rows are truncated to the first 5000 total rows
+    (header included); a visible note is added.
+    Satisfies: H1
+    """
+    # header + 5001 body rows = 5002 non-empty rows total.
+    body_rows = [["r" + str(i)] for i in range(5001)]
+    rows = [["h"]] + body_rows
+    out = csv_converter._table_html(rows)
+    # rows[:5000] keeps the first 5000 total rows: 1 header + 4999 body = 5000 <tr>.
+    assert out.count("<tr>") == 5000, out.count("<tr>")
+    assert csv_converter._TRUNCATION_NOTE in out
+
+
+def test_table_html_caps_cols_to_max_cols(csv_converter):
+    """More than 50 columns are truncated to exactly 50 in header and body.
+    Satisfies: H1
+    """
+    # 60-column header + one 60-column body row.
+    wide = ["c" + str(i) for i in range(60)]
+    out = csv_converter._table_html([wide, wide])
+    header_cells = out.split("</tr>")[0].count("<td>") + out.split("</tr>")[0].count("<th>")
+    assert header_cells == 50, header_cells
+    body_cells = out.split("</tr>")[-2].count("<td>")
+    assert body_cells == 50, body_cells
+    assert csv_converter._TRUNCATION_NOTE not in out
+
+
+def test_table_html_undersizes_are_not_truncated(csv_converter):
+    """A small table renders in full with no truncation note.
+    Satisfies: H1
+    """
+    rows = [["h1", "h2", "h3"], ["a", "b", "c"]]
+    out = csv_converter._table_html(rows)
+    assert out.count("<tr>") == 2
+    assert "truncated to 5000 rows x 50 cols" not in out
+
+
+# ---------------------------------------------------------------------------
+# Finding M2 (regression): stale output-file sweep
+# ---------------------------------------------------------------------------
+# A symmetric counterpart to _sweep_stale_work_dirs: output files older than the
+# same window are removed at startup, fresh files are kept.
+
+
+def test_m2_output_sweep_removes_only_stale_output_files(tmp_path, monkeypatch):
+    """_sweep_stale_output_files removes OUTPUT_DIR files older than the window,
+    keeping fresh ones. Symmetric to the work-dir sweep test.
+    Satisfies: M2
+    """
+    monkeypatch.setattr(app, "OUTPUT_DIR", tmp_path)
+    stale = tmp_path / "stale.pdf"
+    stale.write_bytes(b"%PDF-1.7 old\n")
+    fresh = tmp_path / "fresh.pdf"
+    fresh.write_bytes(b"%PDF-1.7 new\n")
+    old = time.time() - 7 * 3600  # 7h > 6h window
+    os.utime(stale, (old, old))
+    app._sweep_stale_output_files()
+    assert not stale.exists(), "stale output file must be swept"
+    assert fresh.exists(), "fresh output file must be kept"