Add OvisOCR2 support, with a gate fix for long-period loops

README.md

@@ -17,8 +17,9 @@ It keeps olmOCR's document management, queueing, and CLI 1:1, with two additions
   SOTA 1B Markdown-OCR model — image-only prompt, rendered at 1540px by default;
   `lightonocr2-soup` is the same adapter on the more-robust
   [`-ocr-soup`](https://huggingface.co/lightonai/LightOnOCR-2-1B-ocr-soup)
-  merged checkpoint; `glm-ocr`, `qianfan-ocr`, `infinity-parser2-flash`, and
-  `surya2` add four more document-OCR VLMs, documented below).
+  merged checkpoint; `glm-ocr`, `qianfan-ocr`, `infinity-parser2-flash`,
+  `surya2`, `unlimited-ocr`, and `ovisocr2` add six more document-OCR VLMs,
+  documented below).
 - **Opt-out resume** — completed work items are skipped on restart by default
   (olmOCR's done-flag behavior). `--no-resume` wipes prior progress and
   reprocesses the workspace from scratch.
@@ -69,10 +70,19 @@ poetry run paperscale ./workspace \
   flags, `results/*.jsonl`, and (with `--markdown`) `markdown/` live.
 - `--pdfs` — local PDF/image paths, a glob (`'docs/*.pdf'`), `.tar.gz` tarballs,
   or a `.txt` file listing one path per line.
-- `--ocr-model {glm-ocr,infinity-parser2-flash,lightonocr2,lightonocr2-soup,markdown,olmocr,qianfan-ocr,surya2}`
+- `--ocr-model {glm-ocr,infinity-parser2-flash,lightonocr2,lightonocr2-soup,markdown,olmocr,ovisocr2,qianfan-ocr,surya2,unlimited-ocr}`
   — which OCR adapter drives prompting/parsing.
 - `--model` — the served model id sent in each request (or a Hugging Face path
   for the internal server).
+- `--disable-quality-check CHECK` — turn off one deterministic quality check so
+  it can no longer reject a page. Repeatable; `all` disables the gate entirely.
+  Checks: `empty_output`, `mojibake`, `control_characters`,
+  `refusal_boilerplate`, `malformed_frontmatter`, `repeated_character`,
+  `repeated_ngram`, `repeated_tail`, `truncation_indicator`, `length_anomaly`.
+  Prefer this over raising `--max_page_error_rate` when a corpus trips one
+  specific gate: it keeps the other checks armed instead of accepting every
+  failure mode at once. Pair it with `PAPERSCALE_DEBUG_REJECTS=<path>.jsonl` to
+  see what a gate was dropping before you switch it off.
 
 Inputs are added to the workspace queue, grouped into work items, and processed
 concurrently by `--workers`. Re-running the same command **resumes**: finished
@@ -240,6 +250,35 @@ poetry run paperscale ./workspace --pdfs './docs/*.pdf' \
   --ocr-model surya2 --server http://127.0.0.1:8000/v1 --markdown
 ```
 
+### OvisOCR2
+
+`--ocr-model ovisocr2` drives
+[OvisOCR2](https://huggingface.co/ATH-MaaS/OvisOCR2) (`ATH-MaaS/OvisOCR2`, 0.8B,
+`Qwen3_5ForConditionalGeneration`), a compact end-to-end page parser post-trained
+from Qwen3.5-0.8B. It scores 96.58 on OmniDocBench v1.6 — the first end-to-end
+model to top that leaderboard — and 75.06 Avg3 on PureDocBench. One call returns
+the full page as Markdown, with LaTeX formulas and HTML `<table>` markup;
+paperscale strips the `<img src="images/bbox_…">` placeholders the model emits for
+charts and figures (their crops are never written) and flags those pages as
+diagrams instead.
+
+The vendor pins `vllm==0.22.1` and selects the Triton GDN prefill backend — the
+model interleaves Gated-DeltaNet `linear_attention` layers with full attention:
+
+```bash
+vllm serve ATH-MaaS/OvisOCR2 --port 8000 \
+  --gdn-prefill-backend triton \
+  --limit-mm-per-prompt '{"image": 1}' \
+  --mm-processor-kwargs '{"images_kwargs": {"min_pixels": 200704, "max_pixels": 8294400}}'
+
+poetry run paperscale ./workspace --pdfs './docs/*.pdf' \
+  --ocr-model ovisocr2 --server http://127.0.0.1:8000/v1 --markdown
+```
+
+Pages render at 1540px by default, comfortably inside the vendor's
+`448²`–`2880²` pixel band; that band tolerates roughly 3270px on the long edge
+via `--target_longest_image_dim` if a corpus has fine print.
+
 ## Outputs
 
 For each work item, paperscale writes `workspace/results/output_<hash>.jsonl` —

src/paperscale/models/__init__.py

@@ -12,6 +12,7 @@
 from paperscale.models.lightonocr import LightOnOCRModel, LightOnOCRSoupModel
 from paperscale.models.markdown import MarkdownModel
 from paperscale.models.olmocr import OlmOCRModel
+from paperscale.models.ovisocr2 import OvisOCR2Model
 from paperscale.models.qianfan import QianfanOCRModel
 from paperscale.models.surya import Surya2Model
 from paperscale.models.unlimited_ocr import UnlimitedOCRModel
@@ -28,6 +29,7 @@
     "infinity-parser2-flash": InfinityParser2FlashModel,
     "surya2": Surya2Model,
     "unlimited-ocr": UnlimitedOCRModel,
+    "ovisocr2": OvisOCR2Model,
 }
 
 
@@ -52,6 +54,7 @@ def build_ocr_model(name: str) -> OCRModel:
     "InfinityParser2FlashModel",
     "Surya2Model",
     "UnlimitedOCRModel",
+    "OvisOCR2Model",
     "MODEL_REGISTRY",
     "DEFAULT_MODEL",
     "build_ocr_model",

src/paperscale/models/markdown.py

@@ -17,13 +17,29 @@
 
 _CODE_FENCE = re.compile(r"^\s*```(?:markdown|md)?\s*\n(?P<body>.*?)\n```\s*$", re.DOTALL | re.IGNORECASE)
 
+# A complete <think>...</think> reasoning block. Several OCR VLMs emit one when
+# thinking is enabled; paperscale never enables it, so stripping is defensive.
+# Matched-pair only, so a literal "</think>" transcribed from the page cannot
+# truncate real content.
+_THINK_BLOCK = re.compile(r"<think>.*?</think>", re.DOTALL)
+
 
 def _strip_code_fence(content: str) -> str:
     """Remove a single wrapping ```markdown ... ``` fence some models add."""
     match = _CODE_FENCE.match(content)
     return match.group("body") if match else content
 
 
+def strip_think_blocks(content: str) -> str:
+    """Drop any complete ``<think>...</think>`` reasoning block from a response."""
+    return _THINK_BLOCK.sub("", content)
+
+
+def has_html_table(markdown: str | None) -> bool:
+    """Whether Markdown carries an HTML ``<table>``, which paperscale keeps as-is."""
+    return "<table" in (markdown or "").lower()
+
+
 class MarkdownModel(OCRModel):
     """Generic adapter for OCR models whose response *is* the page Markdown.
 

src/paperscale/models/ovisocr2.py

@@ -0,0 +1,117 @@
+"""OvisOCR2 adapter: 0.8B end-to-end page parser, full-page Markdown.
+
+OvisOCR2 (https://huggingface.co/ATH-MaaS/OvisOCR2) is a 0.8B document-parsing
+VLM post-trained from Qwen3.5-0.8B (``Qwen3_5ForConditionalGeneration``,
+``model_type: qwen3_5`` — the same family as ``surya2``, but with hybrid
+Gated-DeltaNet ``linear_attention`` layers). One page image plus the vendor
+instruction yields the whole page as Markdown in reading order. It tops
+OmniDocBench v1.6 (96.58) and PureDocBench (Avg3 75.06). Selected with
+``--ocr-model ovisocr2``.
+
+Output shape: standard Markdown for prose, **LaTeX** for formulas, **HTML**
+``<table>`` for tables (paperscale accepts these as-is, like ``qianfan-ocr``),
+and visual regions as placeholder ``<img src="images/bbox_{l}_{t}_{r}_{b}.jpg" />``
+tags with coordinates scaled to [0, 1000). paperscale never writes the crop
+files those tags reference, so :meth:`parse` strips them and reports the page as
+a diagram instead.
+
+vLLM serving (vendor pins ``vllm==0.22.1``; the checkpoint declares
+``transformers_version 4.57.0.dev0``, so pair it with transformers >= 4.57.
+Verified end-to-end on vLLM 0.27.1, which registers the arch natively)::
+
+    vllm serve ATH-MaaS/OvisOCR2 --port 8000 \
+        --gdn-prefill-backend triton \
+        --limit-mm-per-prompt '{"image": 1}' \
+        --mm-processor-kwargs '{"images_kwargs": {"min_pixels": 200704, "max_pixels": 8294400}}'
+
+``--gdn-prefill-backend triton`` is the vendor's ``gdn_prefill_backend="triton"``
+LLM kwarg; the mm-processor bounds are their ``min_pixels=448*448`` /
+``max_pixels=2880*2880``. The default 1540px render sits comfortably inside that
+band (~1.8 MP for a letter page), and the band tolerates up to ~3270px on the
+long edge via ``--target_longest_image_dim`` if fine print needs it.
+
+Degenerate output: the vendor's own parser trims a repeating tail out of long
+responses before returning them. This adapter deliberately does **not** — a
+looping page is an *incomplete* page (the model spent its budget repeating
+instead of transcribing the rest), and rewriting it here would hand the pipeline
+something that reads as clean and complete. The same detection lives in the
+quality gate instead, as
+:func:`paperscale.quality.verifier._has_repeated_tail_loop`, so the page is
+rejected and retried at a higher temperature. Disable it with
+``--disable-quality-check repeated_tail`` if a corpus trips it systematically.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import replace
+
+from paperscale.models.markdown import MarkdownModel, has_html_table, strip_think_blocks
+from paperscale.prompts import PageResponse
+
+# Vendor inference prompt, verbatim from the model card's OvisOCR2Parser. The
+# leading newline and the literal ``{left}``/``{top}``/… braces are part of the
+# string (the vendor builds it as a plain, non-f string). Keep both: the chat
+# template trims only the *ends* of the rendered user turn, so with the image
+# part sent first this newline survives as the image/instruction separator,
+# reproducing the vendor's exact token sequence.
+OVIS_OCR2_PROMPT = (
+    "\nExtract all readable content from the image in natural human reading order "
+    "and output the result as a single Markdown document. For charts or images, "
+    'represent them using an HTML image tag: <img src="images/bbox_{left}_{top}_{right}_{bottom}.jpg" />, '
+    "where left, top, right, bottom are bounding box coordinates scaled to [0, 1000). "
+    "Format formulas as LaTeX. Format tables as HTML: <table>...</table>. Transcribe "
+    "all other text as standard Markdown. Preserve the original text without "
+    "translation or paraphrasing."
+)
+
+# Placeholder tags for visual regions: <img src="images/bbox_12_34_56_78.jpg" />.
+# The vendor's own parser drops these by default (filter_imgtags=True) because the
+# referenced crops only exist if you save them alongside the Markdown; paperscale
+# never does, so every occurrence — block-level or inline — is a dead reference.
+_BBOX_IMG = re.compile(r'<img\s+src="images/bbox_\d+_\d+_\d+_\d+\.jpg"\s*/?>')
+
+# Blank lines left behind where a stripped <img> tag was the whole block.
+_EXCESS_BLANKS = re.compile(r"\n{3,}")
+
+
+class OvisOCR2Model(MarkdownModel):
+    """Drives OvisOCR2, which transcribes a page image to full-page Markdown."""
+
+    default_model_name = "ATH-MaaS/OvisOCR2"
+    preferred_longest_image_dim = 1540
+
+    def __init__(self) -> None:
+        super().__init__(prompt=OVIS_OCR2_PROMPT)
+
+    def build_messages(self, image_base64: str) -> list[dict]:
+        # Image part first, then the instruction: this is the vendor's content
+        # order, and it is what keeps OVIS_OCR2_PROMPT's leading newline from
+        # being trimmed off the front of the rendered user turn.
+        return [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_base64}"}},
+                    {"type": "text", "text": self._prompt},
+                ],
+            }
+        ]
+
+    def sampling_params(self) -> dict:
+        # Vendor decodes with max_tokens=16384; full pages of dense HTML tables
+        # overrun the pipeline's 8000 default. Temperature stays pipeline-owned.
+        return {"max_tokens": 16384}
+
+    def parse(self, content: str) -> PageResponse:
+        # Defensive: the chat template pre-closes thinking unless enable_thinking is
+        # explicitly true, which paperscale never sets, so no block should appear.
+        content = strip_think_blocks(content)
+        # A bbox img tag is the model's marker for a chart/figure region, so read
+        # the diagram flag off the raw text before the dead references go away.
+        is_diagram = bool(_BBOX_IMG.search(content))
+        content = _EXCESS_BLANKS.sub("\n\n", _BBOX_IMG.sub("", content))
+        result = super().parse(content)
+        # Tables come back as HTML, which paperscale keeps as-is; recompute the
+        # flag from the emitted Markdown rather than converting it.
+        return replace(result, is_table=has_html_table(result.natural_text), is_diagram=is_diagram)

src/paperscale/models/qianfan.py

@@ -15,17 +15,11 @@
 
 from __future__ import annotations
 
-import re
 from dataclasses import replace
 
-from paperscale.models.markdown import MarkdownModel
+from paperscale.models.markdown import MarkdownModel, has_html_table, strip_think_blocks
 from paperscale.prompts import PageResponse
 
-# A complete <think>...</think> reasoning block (only emitted when
-# enable_thinking=True, which paperscale never sets). Matched-pair only, so a
-# literal "</think>" transcribed from the page can't truncate real content.
-_THINK_BLOCK = re.compile(r"<think>.*?</think>", re.DOTALL)
-
 
 class QianfanOCRModel(MarkdownModel):
     """Drives Qianfan-OCR, which transcribes a page image to reading-ordered Markdown."""
@@ -45,9 +39,8 @@ def parse(self, content: str) -> PageResponse:
         # Defensive: drop any <think>...</think> reasoning block. paperscale never
         # sets enable_thinking, so this is normally a no-op; matched-pair only, so a
         # literal "</think>" in the page can't truncate the transcription.
-        content = _THINK_BLOCK.sub("", content)
+        content = strip_think_blocks(content)
         result = super().parse(content)
         # Qianfan returns tables as HTML, which paperscale accepts as-is. Recompute
         # is_table from the emitted Markdown rather than convert it.
-        is_table = "<table" in (result.natural_text or "").lower()
-        return replace(result, is_table=is_table)
+        return replace(result, is_table=has_html_table(result.natural_text))

src/paperscale/pipeline.py

@@ -58,7 +58,7 @@
 from paperscale.metrics import MetricsKeeper, WorkerTracker
 from paperscale.models import DEFAULT_MODEL, MODEL_REGISTRY, OCRModel, build_ocr_model
 from paperscale.prompts import PageResponse
-from paperscale.quality.verifier import DeterministicQualityVerifier
+from paperscale.quality.verifier import DISABLE_ALL_CHECKS, QUALITY_CHECK_CODES, DeterministicQualityVerifier, expand_disabled_checks
 from paperscale.renderpdf import png_dark_fraction, render_pdf_to_base64png
 
 # The stderr hand-off is shared with evaluate, which needs the identical ordering
@@ -107,17 +107,28 @@
 
 # Page acceptance uses paperscale's deterministic quality gate (empty / mojibake /
 # control-chars / refusal / repetition / truncation / length checks) instead of
-# olmOCR's token-count + finish_reason + rotation heuristics.
+# olmOCR's token-count + finish_reason + rotation heuristics. main() builds the
+# run's verifier onto args from --disable-quality-check; this is the default for
+# callers that drive try_single_page directly (tests, embedders).
 _verifier = DeterministicQualityVerifier()
 
+
+def _build_verifier(disabled: list[str]) -> DeterministicQualityVerifier:
+    """Build the page verifier, honouring ``--disable-quality-check``."""
+    codes = expand_disabled_checks(disabled)
+    if codes:
+        logger.warning(f"Quality checks disabled: {', '.join(sorted(codes))}. Pages these would reject are now accepted as-is.")
+    return DeterministicQualityVerifier(disabled_checks=codes)
+
+
 # A page whose rendered image has less ink than this is treated as genuinely blank:
 # an empty/degenerate OCR result for it is accepted as a successful empty page
 # rather than retried. Blank scans sit ~0.002-0.005; content pages are higher.
 _BLANK_INK_THRESHOLD = 0.01
 
 # Quality diagnostics that, on a near-blank render, mean "genuinely blank page"
 # (empty output, or a model repetition loop on noise) rather than a read failure.
-_BLANK_ELIGIBLE_DIAGNOSTICS = frozenset({"empty_output", "repeated_ngram", "repeated_character"})
+_BLANK_ELIGIBLE_DIAGNOSTICS = frozenset({"empty_output", "repeated_ngram", "repeated_character", "repeated_tail"})
 
 
 def _render_is_blank(image_base64: str) -> bool:
@@ -235,7 +246,7 @@ async def try_single_page(
         model_response_markdown = base_response_data["choices"][0]["message"]["content"]
         page_response = args.ocr_model.parse(model_response_markdown)
 
-        finding = _verifier.classify(page_response.natural_text or "")
+        finding = args.quality_verifier.classify(page_response.natural_text or "")
         if finding.accepted:
             is_valid, is_terminal = True, False
         elif finding.kind in _BLANK_ELIGIBLE_DIAGNOSTICS and render_is_blank:
@@ -1167,6 +1178,20 @@ def _build_arg_parser() -> argparse.ArgumentParser:
     parser.add_argument("--pages_per_group", type=int, default=100, help="Aim for this many PDF pages per work item group.")
     parser.add_argument("--max_page_retries", type=int, default=8, help="Max number of times to retry a page.")
     parser.add_argument("--max_page_error_rate", type=float, default=0.004, help="Allowable fraction of fallback pages per document.")
+    parser.add_argument(
+        "--disable-quality-check",
+        dest="disabled_quality_checks",
+        action="append",
+        default=[],
+        metavar="CHECK",
+        choices=(*QUALITY_CHECK_CODES, DISABLE_ALL_CHECKS),
+        help="Skip a deterministic quality check so it cannot reject a page. Repeatable. "
+        f"'{DISABLE_ALL_CHECKS}' disables the gate entirely. Choices: {', '.join(QUALITY_CHECK_CODES)}.",
+    )
+    # main() replaces this from --disable-quality-check; the default keeps the fully
+    # armed gate available to callers that drive try_single_page without going
+    # through main() (tests, embedders).
+    parser.set_defaults(quality_verifier=_verifier)
     parser.add_argument("--workers", type=int, default=4, help="Max number of page groups processed at once.")
     parser.add_argument("--max_concurrent_requests", type=int, default=500, help="Max requests in-flight to the inference provider at once.")
     parser.add_argument("--max_server_ready_timeout", type=int, default=600, help="Seconds to wait for the server to become ready.")
@@ -1244,6 +1269,7 @@ async def main():
 
     # Resolve the OCR model adapter and the served model name.
     args.ocr_model = build_ocr_model(args.ocr_model_name)
+    args.quality_verifier = _build_verifier(args.disabled_quality_checks)
     if args.model is None:
         args.model = args.ocr_model.default_model_name
     # Render size: explicit --target_longest_image_dim wins, else the model's preferred.