From 7ea2b2baad8f8516de3ec913de5990791a5bbc8e Mon Sep 17 00:00:00 2001 From: sirdave73 Date: Fri, 4 Sep 2026 06:34:20 -0400 Subject: [PATCH] H1+M2 review findings: CSV 5000x50 render cap (truncated with note) + stale output sweep; 4 regression tests (H1x3, M2x1); USER_GUIDE truncation docs --- USER_GUIDE.md | 4 +- app.py | 25 +++++ converters/csv_converter.py | 28 +++++- tests/test_review_findings_20260903.py | 121 ++++++++++++++++++++----- 4 files changed, 150 insertions(+), 28 deletions(-) diff --git a/USER_GUIDE.md b/USER_GUIDE.md index c8ede5e..54d5288 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -28,7 +28,7 @@ Everything runs locally (or on your own server); **no file ever leaves your mach | TeX / LaTeX | `.tex` `.latex` | Via pandoc; needs `\documentclass` or a leading `%`, else treated as plain text | | Extended markup | `.org` `.adoc` `.rst` | Org mode, AsciiDoc, reStructuredText via pandoc | | Text | `.txt` `.log` | UTF-8 safe, paginated | -| Data | `.csv` `.tsv` | Auto-detected delimiter → styled table; max 5000 rows × 50 cols | +| Data | `.csv` `.tsv` | Auto-detected delimiter → styled table; **truncated to 5000 rows × 50 cols** (extra data dropped with a note; split to include more) | | Structured | `.json` `.xml` | Cleanly formatted listing | | Vector | `.svg` | Embedded via WeasyPrint | | Images | `.png` `.jpg` `.jpeg` `.webp` `.tiff` `.bmp` | One PDF page per image; max side 3000 px | @@ -94,7 +94,7 @@ See `COOLIFY.md` in this directory (3 steps: Dockerfile service, port 8030, volu | `DRM protected` style errors | Remove DRM first (Calibre / Calibre-DB tools) — this app can't decrypt | | Stuck at `Working…` forever | The server restarted mid-job; refresh the page and re-upload | | `Unsupported format` | File isn't one of the 27 supported types — see §1 | -| CSV/TSV rejected | Over 5000 rows or 50 columns — split the file | +| CSV/TSV truncated | Over 5000 rows or 50 columns — automatically truncated with a note; split the file to include more | | PDF rejected | Encrypted/password-protected, or over 500 pages | --- diff --git a/app.py b/app.py index 6ca7c05..12045a8 100644 --- a/app.py +++ b/app.py @@ -67,6 +67,7 @@ WORKER_SLOTS = threading.BoundedSemaphore(4) # Crash leftovers: sweep work dirs / uploads older than these at startup. STALE_WORK_SECONDS = 6 * 3600 +STALE_OUTPUT_SECONDS = 6 * 3600 STALE_UPLOAD_SECONDS = 3600 app = Flask(__name__) @@ -374,8 +375,32 @@ def _sweep_stale_work_dirs() -> None: pass +def _sweep_stale_output_files() -> None: + """Remove OUTPUT_DIR files left by crashed runs older than the 6h window. + + Symmetric to _sweep_stale_work_dirs (finding M2: stale output sweep). Only + finished output files are removed (the directory is dedicated to generated + PDFs); files inside the window are left untouched. + """ + now = time.time() + try: + entries = list(OUTPUT_DIR.iterdir()) + except OSError: + return + for entry in entries: + if not entry.is_file(): + continue + try: + if now - entry.stat().st_mtime > STALE_OUTPUT_SECONDS: + entry.unlink(missing_ok=True) + except OSError: + pass + + # Crash leftovers from the previous process start (finding: stale work dirs). _sweep_stale_work_dirs() +# Stale generated outputs from the previous process start (finding M2). +_sweep_stale_output_files() # -------------------------------------------------------------------------- diff --git a/converters/csv_converter.py b/converters/csv_converter.py index d34424e..2ef987f 100644 --- a/converters/csv_converter.py +++ b/converters/csv_converter.py @@ -18,6 +18,20 @@ from . import common from .text_converter import decode_bytes +# H1 render cap: a single oversized CSV must not exhaust the single worker +# thread (unbounded CPU/memory DoS). Rows/columns are TRUNCATED (not rejected), +# and a visible note is appended so the behaviour is honest. Mirrors the +# documented limit (USER_GUIDE.md: max 5000 rows x 50 cols). +MAX_ROWS = 5000 +MAX_COLS = 50 + +# Visible note appended to the rendered table when the cap is exceeded. +_TRUNCATION_NOTE = ( + "

truncated to 5000 rows x 50 cols; " + "split the file to include more

" +) + + def _rows(text: str, delimiter: str) -> list[list[str]]: rows = list(csv.reader(io.StringIO(text), delimiter=delimiter)) return [row for row in rows if any(cell.strip() for cell in row)] @@ -26,13 +40,19 @@ def _rows(text: str, delimiter: str) -> list[list[str]]: def _table_html(rows: list[list[str]]) -> str: if not rows: return "" - header, body = rows[0], rows[1:] - head = "".join(f"{common.escape(c)}" for c in header) + # H1: cap to the first MAX_ROWS rows and first MAX_COLS columns. + shown = rows[:MAX_ROWS] + header, body = shown[0], shown[1:] + n_cols = min(len(header), MAX_COLS) + head = "".join(f"{common.escape(c)}" for c in header[:MAX_COLS]) out = [f"{head}"] + truncated = len(rows) > MAX_ROWS for row in body: - # pad ragged rows to header width so borders line up - cells = row + [""] * (len(header) - len(row)) + # pad ragged rows to the (truncated) header width so borders line up + cells = row[:MAX_COLS] + [""] * (n_cols - len(row[:MAX_COLS])) out.append("" + "".join(f"" for c in cells) + "") + if truncated: + out.append(_TRUNCATION_NOTE) out.append("
{common.escape(c)}
") return "\n".join(out) diff --git a/tests/test_review_findings_20260903.py b/tests/test_review_findings_20260903.py index 92b8681..a600079 100644 --- a/tests/test_review_findings_20260903.py +++ b/tests/test_review_findings_20260903.py @@ -7,18 +7,26 @@ C2 - Remote resource fetching by WeasyPrint (blind SSRF / server-side data egres The server must never fetch a remote http(s)/ftp/ws/wss resource on behalf of a user upload. The fix lives in converters.common._blocking_url_fetcher (common.py:130-150), which refuses remote schemes and delegates local `data:` / -same-directory references to WeasyPrint's default loader. The import that makes -that delegation possible is converters.common:18. +same-directory references to WeasyPrint's default loader. -Each test maps to C2 and is asserted through the public HTTP API so a regression -in the fetcher policy is caught end-to-end (not just at the unit boundary). +C1 - Decompression-bomb guard on zip extraction (app.py zip-bomb guard + the +SHARED streaming guard common._copy_stream_with_cap). -Satisfies: C2 +C3 - Stored-XSS stripping in rendered HTML/SVG (common._strip_scripts in +common.html_to_pdf_bytes, called by the pandoc/html/svg converters). + +H1 - CSV row/column render cap in converters.csv_converter._table_html (rows +capped to MAX_ROWS, columns to MAX_COLS, visible truncation note). + +M2 - Stale output-file sweep (app._sweep_stale_output_files). + +Satisfies: C1, C2, C3, H1, M2 """ import base64 import io import os import tempfile +import time import zipfile import pytest @@ -75,10 +83,6 @@ def test_remote_https_image_reference_rejected(client): assert "blocked" in job["error"].lower() or "remote" in job["error"].lower() -# --------------------------------------------------------------------------- -# Finding C2 (regression): local data: references remain permitted -# --------------------------------------------------------------------------- - def test_local_data_uri_image_succeeds(client): """A local `data:` URI resolves through WeasyPrint's default loader and the conversion completes. Regression guard: must not regress to returning None @@ -103,9 +107,9 @@ def test_local_data_uri_image_succeeds(client): # Finding C1 (regression): decompression-bomb guard on zip extraction # --------------------------------------------------------------------------- # Two independent mechanisms are exercised: -# * the SHARED streaming guard common._copy_stream_with_cap (proven by the -# unit tests: it counts WRITTEN decompressed bytes and aborts mid-stream -# past the cap), and +# * the SHARED streaming guard common._copy_stream_with_cap (proven by the unit +# tests: it counts WRITTEN decompressed bytes and aborts mid-stream past the +# cap), and # * app.py's zip-bomb guard end-to-end (proven by the e2e: a genuinely large # member trips the per-member size cap and the job fails with 'zip-bomb'). @@ -172,12 +176,11 @@ def test_c1_zip_bomb_guard_rejects_large_member(client, fixture_epub_bytes, monk # --------------------------------------------------------------------------- # Finding C3 (regression): stored-XSS stripping in rendered HTML/SVG # --------------------------------------------------------------------------- -# WeasyPrint already drops