Sync third-party and MCP marketplace plugins

Constraint: Public skills are published only by explicit administrator action unless they are tracked third-party market sources.
Confidence: high
Scope-risk: narrow
Directive: Keep private/internal skills out of the public marketplace and preserve normal incremental market Git history.
Tested: Marketplace validation passed.
This commit is contained in:
KeyInfo Bot committed 2026-10-06 00:02:11 +08:00
1 parent 7c1d798d93
commit be7ce9bda2
18 files changed
+1276 -180

No files matched your search

@@ -2,8 +2,8 @@
"sourceId": "grill-me",
"repo": "https://github.com/mattpocock/skills.git",
"ref": "main",
"commit": "24fe0ef7737efae15c87225755e9f6f5965e4888",
"commit": "4588b32ecab9ecc9fc8cc6b6c5e7d675b6004b0d",
"adapter": "skill-collection",
"sourcePath": "skills/productivity",
"syncedAt": "2026-10-04T16:00:00Z"
"syncedAt": "2026-10-05T16:00:00Z"
}
@@ -3,5 +3,5 @@
"name": "playwright浏览器自动化操作",
"version": "20260605",
"keySource": "none",
"syncedAt": "2026-10-04T16:01:56Z"
"syncedAt": "2026-10-05T16:02:10Z"
}
@@ -2,8 +2,8 @@
"sourceId": "next-skills",
"repo": "https://github.com/vercel/next.js.git",
"ref": "canary",
"commit": "ba80ee48fc319735151c3ad6d9bb9a8180c9f09e",
"commit": "41215d77762375b4b5154b12cfbb4372a835f8a0",
"adapter": "skill-collection",
"sourcePath": "skills",
"syncedAt": "2026-10-04T16:00:00Z"
"syncedAt": "2026-10-05T16:00:00Z"
}
@@ -2,8 +2,8 @@
"sourceId": "ppt-master",
"repo": "https://github.com/hugohe3/ppt-master.git",
"ref": "main",
"commit": "44c10ed0bc3a9e1df7a25aa179ae7c26db09469b",
"commit": "2d72da616cf9fa40d4dcaf59fd4c980ecf534b7d",
"adapter": "claude-skill",
"sourcePath": "skills/ppt-master",
"syncedAt": "2026-10-01T16:00:00Z"
"syncedAt": "2026-10-05T16:00:00Z"
}
@@ -178,13 +178,15 @@ python3 scripts/source_to_md/excel_to_md.py report.xlsx budget.xlsm
python3 scripts/source_to_md/excel_to_md.py ./workbooks
python3 scripts/source_to_md/excel_to_md.py ./workbooks -o ./markdown # explicit separate output directory
python3 scripts/source_to_md/excel_to_md.py report.xlsm --max-rows 200 --max-cols 40
python3 scripts/source_to_md/excel_to_md.py report.xlsx --include-hidden # also export hidden sheets
```
Behavior:
- preserves workbook and sheet structure in Markdown
- exports visible sheets only
- exports visible sheets by default; skipped hidden sheets are named in a warning, and `--include-hidden` exports them with a `(hidden)` heading
- trims empty outer rows and columns
- propagates merged-cell labels for readable Markdown tables
- propagates merged-cell values for readable Markdown tables; merged numeric regions are noted below the table (`> Merged cells: A2:A3 (shared value)`) so a shared amount is not counted twice
- shows percent formats as percentages and keeps leading zeros of zero-padded identifiers; other number formats keep the stored value at Excel's 15-digit precision
- exports formula cells as cached values; it does not recalculate formulas
- uses the shared best-effort conversion-profile contract after success
@@ -583,7 +583,12 @@ def _omml_to_latex(elem: ET.Element) -> str:
end = _m_pr_val(elem, "endChr")
beg = "(" if beg is None else (beg or ".")
end = ")" if end is None else (end or ".")
inner = "".join(_omml_to_latex(c) for c in elem if _local_name(c) == "e")
separator = _m_pr_val(elem, "sepChr")
separator = "|" if separator is None else separator
separator = _latex_literal(separator)
beg = {"{": r"\{", "}": r"\}"}.get(beg, beg)
end = {"{": r"\{", "}": r"\}"}.get(end, end)
inner = separator.join(_omml_to_latex(c) for c in elem if _local_name(c) == "e")
return rf"\left{beg}{inner}\right{end}"
if local == "nary":
chr_ = _m_pr_val(elem, "chr") or "∫"
@@ -608,7 +613,15 @@ def _omml_to_latex(elem: ET.Element) -> str:
cmd = ACCENT_CMDS.get(_m_pr_val(elem, "chr") or "̂", r"\hat")
return cmd + "{" + _omml_part(elem, "e") + "}"
if local == "groupChr":
return _omml_part(elem, "e")
char = _m_pr_val(elem, "chr")
char = "⏟" if char is None else char
pos = _m_pr_val(elem, "pos") or "bot"
body = _omml_part(elem, "e")
if (char, pos) in {("⏞", "top"), ("⏟", "bot")}:
cmd = r"\overbrace" if pos == "top" else r"\underbrace"
return cmd + "{" + body + "}"
cmd = r"\overset" if pos == "top" else r"\underset"
return cmd + r"{\text{" + _latex_literal(char) + "}}{" + body + "}"
if local == "m":
return _omml_matrix(elem, environment="matrix")
if local == "eqArr":
@@ -617,6 +630,13 @@ def _omml_to_latex(elem: ET.Element) -> str:
return _omml_children(elem)
def _latex_literal(text: str) -> str:
"""Escape literal characters that would otherwise change LaTeX syntax."""
escapes = {"\\": r"\backslash{}", "{": r"\{", "}": r"\}", "%": r"\%",
"&": r"\&", "#": r"\#", "_": r"\_", "$": r"\$"}
return "".join(escapes.get(char, char) for char in text)
def _make_text_run(text: str) -> ET.Element:
"""Build a <w:r><w:t xml:space="preserve">text</w:t></w:r> element."""
run = ET.Element(f"{{{W_NS}}}r")
@@ -635,6 +655,7 @@ def _make_text_paragraph(text: str) -> ET.Element:
def _docx_inject_math_latex(
input_file: Path,
warnings: list[str] | None = None,
) -> tuple[Path, dict[str, str]] | None:
"""Replace OMML equations with alphanumeric placeholders in a temp DOCX.
@@ -672,6 +693,16 @@ def _docx_inject_math_latex(
parent = parent_map.get(elem)
if parent is None:
continue
for group in elem.iter(f"{{{MATH_NS}}}groupChr"):
char = _m_pr_val(group, "chr")
char = "⏟" if char is None else char
pos = _m_pr_val(group, "pos") or "bot"
if (char, pos) not in {("⏞", "top"), ("⏟", "bot")}:
warning = (f"OMML group character {char!r} at {pos}: retained as positioned text; "
"review the original equation.")
print(f"[WARN] {warning}", file=sys.stderr)
if warnings is not None:
warnings.append(warning)
latex = _omml_to_latex(elem).strip()
position = list(parent).index(elem)
parent.remove(elem)
@@ -703,51 +734,68 @@ def _docx_inject_math_latex(
# DOCX tables → pipe Markdown
# ─────────────────────────────────────────────────────────────
def _docx_paragraph_text(paragraph: ET.Element) -> str:
def _docx_paragraph_text(paragraph: ET.Element, links: dict[str, str] | None = None) -> str:
"""Extract readable text from one Word paragraph."""
parts: list[str] = []
for elem in paragraph.iter():
def visit(elem):
local = _local_name(elem)
if local == "t":
if local == "hyperlink" and links is not None:
label = _docx_paragraph_text(elem)
target = links.get(elem.get(f"{{{DOCX_NS['r']}}}id", ""), "")
anchor = elem.get(f"{{{W_NS}}}anchor", "")
if anchor:
target += "#" + anchor
target = target.replace(" ", "%20").replace("(", "%28").replace(")", "%29")
label = label.replace("[", r"\[").replace("]", r"\]")
parts.append(f"[{label}]({target})" if target else label)
elif local == "t":
parts.append(elem.text or "")
elif local == "tab":
parts.append("\t")
elif local in {"br", "cr"}:
parts.append(" ")
elif local == "footnoteReference":
parts.append(f"[^{elem.get(f'{{{W_NS}}}id')}]")
elif local in {"footnoteReference", "endnoteReference"}:
prefix = "endnote-" if local == "endnoteReference" else ""
parts.append(f"[^{prefix}{elem.get(f'{{{W_NS}}}id')}]")
else:
for child in elem:
visit(child)
visit(paragraph)
return "".join(parts).strip()
def _docx_footnote_texts(docx: zipfile.ZipFile) -> dict[str, str]:
"""Return footnote id → plain text from ``word/footnotes.xml``."""
def _docx_note_texts(docx: zipfile.ZipFile, kind: str) -> dict[str, str | None]:
"""Return footnote/endnote ids and readable text from their package part."""
try:
root = ET.fromstring(docx.read("word/footnotes.xml"))
root = ET.fromstring(docx.read(f"word/{kind}s.xml"))
except (KeyError, ET.ParseError):
return {}
return {
note.get(f"{{{W_NS}}}id", ""): " ".join(
_docx_paragraph_text(paragraph)
for paragraph in note.findall("w:p", DOCX_NS)
).strip()
for note in root.findall("w:footnote", DOCX_NS)
if note.get(f"{{{W_NS}}}type") not in {
note.get(f"{{{W_NS}}}id", ""): None if note.get(f"{{{W_NS}}}type") in {
"separator", "continuationSeparator", "continuationNotice",
}
} else " ".join(
_docx_paragraph_text(paragraph)
for paragraph in note.findall(".//w:p", DOCX_NS)
).strip()
for note in root.findall(f"w:{kind}", DOCX_NS)
}
def _docx_table_has_media(table: ET.Element) -> bool:
"""Return whether a table contains image-bearing nodes."""
return any(_local_name(elem) in {"drawing", "imagedata"} for elem in table.iter())
def _docx_table_needs_mammoth(table: ET.Element) -> bool:
"""Keep media and nested tables out of the pipe-table projection."""
return any(
_local_name(elem) in {"drawing", "imagedata"}
or (elem is not table and _local_name(elem) == "tbl")
for elem in table.iter()
)
def _docx_table_cell_text(cell: ET.Element) -> str:
def _docx_table_cell_text(cell: ET.Element, links: dict[str, str] | None = None) -> str:
"""Extract table-cell text, preserving paragraph breaks as Markdown breaks."""
paragraphs: list[str] = []
for child in cell:
if _local_name(child) == "p":
text = _docx_paragraph_text(child)
text = _docx_paragraph_text(child, links)
if text:
paragraphs.append(text)
return "<br>".join(paragraphs)
@@ -759,12 +807,12 @@ def _markdown_table_cell(text: str) -> str:
return text.replace("|", r"\|")
def _docx_table_to_markdown(table: ET.Element) -> str:
def _docx_table_to_markdown(table: ET.Element, links: dict[str, str] | None = None) -> str:
"""Convert a Word table XML node to a pipe Markdown table."""
rows: list[list[str]] = []
for row in table.findall("w:tr", DOCX_NS):
cells = [
_markdown_table_cell(_docx_table_cell_text(cell))
_markdown_table_cell(_docx_table_cell_text(cell, links))
for cell in row.findall("w:tc", DOCX_NS)
]
if cells:
@@ -785,12 +833,22 @@ def _docx_table_to_markdown(table: ET.Element) -> str:
def _docx_inject_tables_markdown(
input_file: Path,
warnings: list[str] | None = None,
) -> tuple[Path, dict[str, str]] | None:
"""Replace text-only DOCX tables with Markdown placeholders in a temp DOCX."""
try:
with zipfile.ZipFile(input_file) as docx:
document_xml = docx.read("word/document.xml")
footnotes = _docx_footnote_texts(docx)
notes = {kind: _docx_note_texts(docx, kind) for kind in ("footnote", "endnote")}
try:
rels = ET.fromstring(docx.read("word/_rels/document.xml.rels"))
except (KeyError, ET.ParseError):
rels = ET.Element("relationships")
links = {
rel.get("Id", ""): rel.get("Target", "")
for rel in rels
if rel.get("Type", "").endswith("/hyperlink")
}
except (KeyError, zipfile.BadZipFile, OSError):
return None
try:
@@ -799,20 +857,48 @@ def _docx_inject_tables_markdown(
return None
parent_map = {child: parent for parent in root.iter() for child in parent}
controls = list(root.iter(f"{{{W_NS}}}sdt"))
# Mammoth skips content controls. Unwrap only their displayed content,
# including nested controls, while leaving runs, tables, and links intact.
for control in reversed(controls):
parent = parent_map.get(control)
content = control.find("w:sdtContent", DOCX_NS)
if parent is None or content is None:
continue
position = list(parent).index(control)
parent.remove(control)
for offset, child in enumerate(content):
parent.insert(position + offset, child)
if controls:
warning = (f"DOCX content controls: {len(controls)} flattened to displayed content; "
"interactive properties omitted.")
print(f"[WARN] {warning}", file=sys.stderr)
if warnings is not None:
warnings.append(warning)
parent_map = {child: parent for parent in root.iter() for child in parent}
token_base = uuid.uuid4().hex
replacements: dict[str, str] = {}
for index, table in enumerate(root.findall(".//w:tbl", DOCX_NS)):
if _docx_table_has_media(table):
if _docx_table_needs_mammoth(table):
continue
markdown = _docx_table_to_markdown(table)
markdown = _docx_table_to_markdown(table, links)
if not markdown:
continue
# The table never reaches mammoth, so its footnotes would lose both
# the reference and the note text; keep them under the table.
for reference in table.iter(f"{{{W_NS}}}footnoteReference"):
note_id = reference.get(f"{{{W_NS}}}id", "")
if footnotes.get(note_id):
markdown += f"\n\n[^{note_id}]: {footnotes[note_id]}"
# Projected tables never reach Mammoth; retain their notes here.
for kind, texts in notes.items():
prefix = "endnote-" if kind == "endnote" else ""
for reference in table.iter(f"{{{W_NS}}}{kind}Reference"):
note_id = reference.get(f"{{{W_NS}}}id", "")
if note_id in texts and texts[note_id] is None:
continue # Structural separators are not missing note bodies.
if texts.get(note_id):
markdown += f"\n\n[^{prefix}{note_id}]: {texts[note_id]}"
else:
warning = f"DOCX {kind} {note_id}: note body unavailable; retain the original DOCX."
markdown += f"\n\n> {warning}"
print(f"[WARN] {warning}", file=sys.stderr)
if warnings is not None:
warnings.append(warning)
parent = parent_map.get(table)
if parent is None:
continue
@@ -821,7 +907,7 @@ def _docx_inject_tables_markdown(
parent.remove(table)
parent.insert(position, _make_text_paragraph(token))
replacements[token] = markdown
if not replacements:
if not replacements and not controls:
return None
for prefix, uri in DOCX_NS.items():
@@ -1001,7 +1087,7 @@ def _convert_docx(
# Rewrite OMML equations to LaTeX placeholders before mammoth (which would
# otherwise drop them); the placeholders are swapped back below.
math_injection = _docx_inject_math_latex(input_file)
math_injection = _docx_inject_math_latex(input_file, warnings)
if math_injection is not None:
math_file, math_replacements = math_injection
else:
@@ -1009,7 +1095,7 @@ def _convert_docx(
table_file = None
table_replacements: dict[str, str] = {}
mammoth_source = math_file or input_file
table_injection = _docx_inject_tables_markdown(mammoth_source)
table_injection = _docx_inject_tables_markdown(mammoth_source, warnings)
if table_injection is not None:
table_file, table_replacements = table_injection
mammoth_source = table_file
@@ -1072,7 +1158,7 @@ def _convert_docx(
print(f" Charts: {len(chart_replacements)} rendered from their cached data")
for msg in result.messages:
if msg.type == "warning":
print(f" [warn] {msg.message}")
print(f" [warn] {msg.message}", file=sys.stderr)
if warnings is not None:
warnings.append(msg.message)
@@ -17,6 +17,7 @@ import argparse
import re
import sys
from datetime import date, datetime, time
from decimal import Decimal
from pathlib import Path
from typing import Any
@@ -67,17 +68,61 @@ def _markdown_escape(value: str) -> str:
return re.sub(r"\s*\n\s*", "<br>", value).strip()
def _format_cell_value(value: Any) -> str:
def _format_number(value: int | float, number_format: str) -> str | None:
"""Add percentage/identifier meaning without rounding the stored value."""
sections = re.split(r';(?=(?:[^"]*"[^"]*")*[^"]*$)', number_format)
section = sections[1] if value < 0 and len(sections) > 1 else sections[0]
section = _strip_display_tokens(section)
if re.fullmatch(r"[+\-()]?(?:0+|#,##0)(?:\.[0#]+)?%\)?", section):
# Match the plain-number path: Excel holds 15 significant digits, so
# scale that value rather than the 17-digit binary repr.
number = Decimal(f"{value:.15g}") * 100
if number.is_finite():
rendered = format(number, "f")
if "." in rendered:
rendered = rendered.rstrip("0").rstrip(".")
return rendered + "%"
if re.fullmatch(r"0+", section) and value == int(value):
return ("-" if value < 0 else "") + str(abs(int(value))).zfill(len(section))
return None
def _strip_display_tokens(number_format: str) -> str:
"""Remove accounting decoration while retaining units and scale tokens."""
# Accounting spacing, colors, currency locales, and currency labels do not
# change a raw number's meaning. Other literal units or scaling can do so.
cleaned = re.sub(r"_.?|\*.", "", number_format)
cleaned = re.sub(r"\[(?:Black|Blue|Cyan|Green|Magenta|Red|White|Yellow|Color\d+|\$[^\]]*)\]",
"", cleaned, flags=re.I)
cleaned = re.sub(r'"(?:USD|EUR|GBP|CNY|RMB|JPY|HKD|[$€£¥¥\s-])*"', "", cleaned)
cleaned = re.sub(r"\\([$€£¥¥ ()+-])", r"\1", cleaned)
return cleaned.strip()
def _format_needs_warning(number_format: str) -> bool:
"""Identify unhandled semantic formats, excluding ordinary display styling."""
if number_format in {"General", "@"}:
return False
cleaned = _strip_display_tokens(number_format)
return (not re.fullmatch(r"[0-9#?.,Ee+\-;/()@$€£¥¥\s]*", cleaned)
or bool(re.search(r",\s*(?:;|$)", cleaned)))
def _format_cell_value(value: Any, number_format: str = "General") -> str:
if _is_empty(value):
return ""
if isinstance(value, bool):
return "TRUE" if value else "FALSE"
if isinstance(value, datetime):
return value.isoformat(sep=" ", timespec="seconds")
return value.isoformat(sep=" ")
if isinstance(value, date):
return value.isoformat()
if isinstance(value, time):
return value.isoformat(timespec="seconds")
return value.isoformat()
if _is_numeric_value(value):
formatted = _format_number(value, number_format)
if formatted is not None:
return formatted
if isinstance(value, float):
# Excel shows 15 significant digits; that keeps 1234567.89 exact while
# 0.1 + 0.2 reads as 0.3 rather than its binary expansion.
@@ -246,11 +291,17 @@ def _column_alignments(rows: list[list[Any]]) -> list[str]:
return alignments
def _rows_to_markdown_table(rows: list[list[Any]]) -> str:
def _rows_to_markdown_table(
rows: list[list[Any]], number_formats: list[list[str]] | None = None,
) -> str:
if not rows:
return "_No tabular content found._"
formatted_rows = [[_format_cell_value(value) for value in row] for row in rows]
formatted_rows = [
[_format_cell_value(value, number_formats[i][j] if number_formats else "General")
for j, value in enumerate(row)]
for i, row in enumerate(rows)
]
width = len(formatted_rows[0])
separator = _column_alignments(rows)
lines = [
@@ -268,7 +319,10 @@ def _rows_to_markdown_table(rows: list[list[Any]]) -> str:
# Excel → Markdown
# ─────────────────────────────────────────────────────────────
def _convert_excel(input_file: Path, out_file: Path, max_rows: int, max_cols: int) -> str:
def _convert_excel(
input_file: Path, out_file: Path, max_rows: int, max_cols: int,
include_hidden: bool = False, warnings: list[str] | None = None,
) -> str:
try:
from openpyxl import load_workbook
from openpyxl.utils import get_column_letter
@@ -278,6 +332,8 @@ def _convert_excel(input_file: Path, out_file: Path, max_rows: int, max_cols: in
workbook = load_workbook(input_file, data_only=True, read_only=False)
visible_sheets = [sheet for sheet in workbook.worksheets if sheet.sheet_state == "visible"]
hidden_sheets = [sheet for sheet in workbook.worksheets if sheet.sheet_state != "visible"]
selected_sheets = workbook.worksheets if include_hidden else visible_sheets
lines: list[str] = [
f"# Spreadsheet Source: {input_file.name}",
@@ -291,15 +347,23 @@ def _convert_excel(input_file: Path, out_file: Path, max_rows: int, max_cols: in
"",
]
if not visible_sheets:
if hidden_sheets and not include_hidden:
warning = "Skipped hidden sheets: " + ", ".join(sheet.title for sheet in hidden_sheets)
warning += ". Use --include-hidden to export them."
lines.extend([f"> Warning: {warning}", ""])
print(f"[WARN] {warning}", file=sys.stderr)
if warnings is not None:
warnings.append(warning)
if not selected_sheets:
lines.extend(["_No visible sheets found._", ""])
for worksheet in visible_sheets:
for worksheet in selected_sheets:
merged_values = _merged_value_map(worksheet)
bounds = _content_bounds(worksheet, merged_values)
lines.extend([
f"## Sheet: {worksheet.title}",
f"## Sheet: {worksheet.title}" + (" (hidden)" if worksheet.sheet_state != "visible" else ""),
"",
f"- State: {_sheet_state_label(worksheet.sheet_state)}",
])
@@ -328,7 +392,34 @@ def _convert_excel(input_file: Path, out_file: Path, max_rows: int, max_cols: in
"",
])
# Propagated merged values must use the anchor's display format too.
format_anchors = {
(row, col): (region.min_row, region.min_col)
for region in worksheet.merged_cells.ranges
for row in range(region.min_row, region.max_row + 1)
for col in range(region.min_col, region.max_col + 1)
if (row, col) in merged_values
}
number_formats = []
unsupported_formats: dict[str, int] = {}
for i, row in enumerate(rows, min_row):
formats = []
for j, value in enumerate(row, min_col):
anchor = format_anchors.get((i, j), (i, j))
cell = worksheet.cell(*anchor)
fmt = cell.number_format
formats.append(fmt)
if (_is_numeric_value(value) and anchor == (i, j)
and _format_number(value, fmt) is None and _format_needs_warning(fmt)):
unsupported_formats[fmt] = unsupported_formats.get(fmt, 0) + 1
number_formats.append(formats)
original_rows = rows
captions, rows = _split_caption_rows(rows, min_row, min_col, merged_values)
for i in range(len(captions)):
j = next(j for j, value in enumerate(original_rows[i]) if not _is_empty(value))
captions[i] = _format_cell_value(original_rows[i][j], number_formats[i][j])
number_formats = number_formats[len(captions):]
for caption in captions:
lines.extend([f"> {caption}", ""])
@@ -340,8 +431,20 @@ def _convert_excel(input_file: Path, out_file: Path, max_rows: int, max_cols: in
limit_notes.append(f"columns limited to {max_cols}")
lines.extend([f"> Truncated: {', '.join(limit_notes)}.", ""])
lines.extend([_rows_to_markdown_table(rows), ""])
lines.extend([_rows_to_markdown_table(rows, number_formats), ""])
for region in sorted(worksheet.merged_cells.ranges, key=lambda r: (r.min_row, r.min_col)):
value = worksheet.cell(region.min_row, region.min_col).value
if _is_numeric_value(value):
lines.extend([f"> Merged cells: {region} (shared value)", ""])
if unsupported_formats:
formats = "; ".join(f"{fmt!r} ({count} cells)" for fmt, count in unsupported_formats.items())
warning = f"{worksheet.title}: unsupported number formats: {formats}; raw values retained."
print(f"[WARN] {warning}", file=sys.stderr)
if warnings is not None:
warnings.append(warning)
lines.extend([f"> {_markdown_escape(warning)}", ""])
workbook.close()
markdown = "\n".join(lines).rstrip() + "\n"
out_file.write_text(markdown, encoding="utf-8")
_report_result(out_file)
@@ -357,6 +460,7 @@ def convert_to_markdown(
output_path: str | None = None,
max_rows: int = 0,
max_cols: int = 0,
include_hidden: bool = False,
) -> str:
input_file = Path(input_path)
if not input_file.exists():
@@ -382,13 +486,18 @@ def convert_to_markdown(
out_file.parent.mkdir(parents=True, exist_ok=True)
print(f"[INFO] Converting Excel workbook: {input_file.name}")
markdown = _convert_excel(input_file, out_file, max_rows=max_rows, max_cols=max_cols)
warnings: list[str] = []
markdown = _convert_excel(
input_file, out_file, max_rows=max_rows, max_cols=max_cols,
include_hidden=include_hidden, warnings=warnings,
)
if markdown:
profile_path = write_conversion_profile_best_effort(
input_path=str(input_file),
markdown_path=out_file,
converter="excel_to_md.py",
conversion_type=suffix.lstrip("."),
warnings=warnings,
)
if profile_path:
print(f" Wrote conversion profile -> {profile_path}")
@@ -432,6 +541,7 @@ Unsupported by default:
default=0,
help="Maximum columns per sheet to export (0 = no limit)",
)
parser.add_argument("--include-hidden", action="store_true", help="Export hidden worksheets as well")
args = parser.parse_args()
return run_path_batch(
@@ -444,6 +554,7 @@ Unsupported by default:
str(output),
max_rows=args.max_rows,
max_cols=args.max_cols,
include_hidden=args.include_hidden,
)
),
)
@@ -46,6 +46,7 @@ FONT_H2_SIZE = 18
FONT_H3_SIZE = 14
HEADER_FOOTER_SAMPLE_LIMIT = 40
HEADER_FOOTER_EDGE_SAMPLE_SIZE = 20
HEADER_FOOTER_BAND_RATIO = 0.15
CONTROL_CHARS_RE = re.compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f]')
@@ -299,16 +300,29 @@ def remove_page_footer(text: str) -> str:
return text.rstrip()
def detect_headers_footers(doc: fitz.Document, threshold_ratio: float = 0.6) -> set[str]:
def _header_footer_band(bbox: fitz.Rect, page_rect: fitz.Rect) -> str | None:
"""Classify fully contained edge text; keep text crossing a band boundary."""
band_height = page_rect.height * HEADER_FOOTER_BAND_RATIO
if page_rect.y0 <= bbox.y0 and bbox.y1 <= page_rect.y0 + band_height:
return "header"
if page_rect.y1 - band_height <= bbox.y0 and bbox.y1 <= page_rect.y1:
return "footer"
return None
def detect_headers_footers(
doc: fitz.Document, threshold_ratio: float = 0.6,
) -> tuple[set[str], set[str]]:
"""
Detect headers and footers statistically.
Principle: Headers and footers typically appear at fixed positions (top or bottom)
on each page with the same content. We collect top and bottom text from all pages,
and if certain text appears more frequently than the threshold, it is treated as noise.
Return header and footer noise separately so filtering stays in the matching band.
"""
if len(doc) < 3:
return set()
return set(), set()
headers = []
footers = []
@@ -323,12 +337,6 @@ def detect_headers_footers(doc: fitz.Document, threshold_ratio: float = 0.6) ->
for i in pages_to_scan:
page = doc[i]
rect = page.rect
h = rect.height
# Define top and bottom regions (15% each)
top_rect = fitz.Rect(0, 0, rect.width, h * 0.15)
bottom_rect = fitz.Rect(0, h * 0.85, rect.width, h)
# Extract text blocks
blocks = page.get_text("blocks")
@@ -338,24 +346,25 @@ def detect_headers_footers(doc: fitz.Document, threshold_ratio: float = 0.6) ->
if not text:
continue
# Simple spatial determination
if b_rect.intersects(top_rect):
band = _header_footer_band(b_rect, page.rect)
if band == "header":
headers.append(text)
elif b_rect.intersects(bottom_rect):
elif band == "footer":
footers.append(text)
# Count frequencies
noise_texts = set()
header_noise = set()
footer_noise = set()
total_scanned = len(pages_to_scan)
for collection in [headers, footers]:
for collection, noise_texts in [(headers, header_noise), (footers, footer_noise)]:
counter = Counter(collection)
for text, count in counter.items():
# if text appears in > 60% of scanned pages, mark as noise
if count / total_scanned > threshold_ratio:
noise_texts.add(text)
return noise_texts
return header_noise, footer_noise
def _is_hangul(char: str) -> bool:
@@ -416,6 +425,8 @@ def merge_adjacent_headings(elements: list) -> list:
next_el = elements[j]
if next_el.get("type") != 0 or not next_el.get("is_heading"):
break
if el.get("is_footer") or next_el.get("is_footer"):
break
next_match = re.match(r'^(#{1,6})\s+(.+)$', next_el["content"])
if not next_match or next_match.group(1) != level:
@@ -478,6 +489,9 @@ NUMBER_RE = re.compile(r'[-+]?\d+(?:\.\d+)?')
MODEL_COLUMN_RE = re.compile(r'^[((]\d+[))]$')
TABLE_NOTE_PREFIX = '\u6ce8'
TABLE_CONTINUATION_Y_RATIO = 0.88
# A printed page number left between split table parts; trusted only next to a
# ``<!-- Page N -->`` marker, never on its own.
PRINTED_PAGE_NUMBER_RE = re.compile(r'\d{1,4}')
TABLE_SCAN_BOTTOM_RATIO = 0.92
@@ -914,14 +928,6 @@ def _looks_like_outcome_row(row: list[str]) -> bool:
return len(short_values) == len(values)
def _regression_group_labels(row: list[str], data_cols: int) -> list[str]:
"""Infer repeated group labels for common regression-table headings."""
compact = "".join(row)
if "总样本" in compact and "国有" in compact and "非国有" in compact and data_cols == 7:
return ["总样本"] * 3 + ["国有企业"] * 2 + ["非国有企业"] * 2
return [""] * data_cols
def _flatten_regression_header(rows: list[list[str]]) -> list[list[str]]:
"""Flatten multi-line regression headings into one Markdown header row."""
if len(rows) < 3:
@@ -942,10 +948,8 @@ def _flatten_regression_header(rows: list[list[str]]) -> list[list[str]]:
return [header] + rows[2:]
header_offset = 0
groups = [""] * (len(rows[0]) - 1)
if not _looks_like_model_row(rows[0]) and _looks_like_model_row(rows[1]):
header_offset = 1
groups = _regression_group_labels(rows[0], len(rows[1]) - 1)
if not _looks_like_model_row(rows[header_offset]):
return rows
@@ -957,8 +961,6 @@ def _flatten_regression_header(rows: list[list[str]]) -> list[list[str]]:
header = ["变量"]
for idx, model in enumerate(model_row[1:]):
pieces = []
if idx < len(groups) and groups[idx]:
pieces.append(groups[idx])
if model:
pieces.append(model)
if idx + 1 < len(outcome_row) and outcome_row[idx + 1]:
@@ -1439,24 +1441,24 @@ def _compatible_table_headers(first: list[str], second: list[str]) -> bool:
return first[0] == second[0] and _markdown_col_count(first[0]) > 2
def _is_table_continuation_noise(line: str) -> bool:
"""Allow only page/header noise between split table parts."""
text = line.strip()
if not text:
return True
if text.startswith("<!-- Page ") and text.endswith("-->"):
return True
if re.fullmatch(r'\d+', text):
return True
if "重庆大学硕士学位论文" in text:
return True
continuation_labels = [
"营改增",
"深化增值税改革",
"国有企业",
"非国有企业",
]
return any(label in text for label in continuation_labels)
def _is_page_marker(text: str) -> bool:
"""Return whether a stripped line is the converter's page-break marker."""
return text.startswith("<!-- Page ") and text.endswith("-->")
def _is_table_continuation_gap(lines: list[str]) -> bool:
"""Return whether only page-break furniture separates two table parts.
A bare number is a printed page number only when the gap crosses a
``<!-- Page N -->`` marker; anywhere else it is content (a quantity, a year).
"""
texts = [line.strip() for line in lines if line.strip()]
if not any(_is_page_marker(text) for text in texts):
return not texts
return all(
_is_page_marker(text) or PRINTED_PAGE_NUMBER_RE.fullmatch(text)
for text in texts
)
def _read_markdown_table_block(lines: list[str], start: int) -> tuple[list[str], int]:
@@ -1479,32 +1481,22 @@ def merge_markdown_continuation_tables(markdown: str) -> str:
index += 1
continue
table, table_end = _read_markdown_table_block(lines, index)
search = table_end
table, index = _read_markdown_table_block(lines, index)
while True:
between_start = search
while search < len(lines) and not _is_markdown_table_line(lines[search]):
if not _is_table_continuation_noise(lines[search]):
break
search += 1
if search >= len(lines) or not _is_markdown_table_line(lines[search]):
break
if any(
not _is_table_continuation_noise(line)
for line in lines[between_start:search]
):
next_start = index
while next_start < len(lines) and not _is_markdown_table_line(lines[next_start]):
next_start += 1
if next_start >= len(lines) or not _is_table_continuation_gap(lines[index:next_start]):
break
next_table, next_end = _read_markdown_table_block(lines, search)
next_table, next_end = _read_markdown_table_block(lines, next_start)
if not _compatible_table_headers(table, next_table):
break
table.extend(next_table[2:])
search = next_end
index = next_end
result.extend(table)
index = search if search != table_end else table_end
return "\n".join(result)
@@ -1589,10 +1581,12 @@ def extract_pdf_to_markdown(
f"H1={size_map.get('h1', 'N/A')}, H2={size_map.get('h2', 'N/A')}, H3={size_map.get('h3', 'N/A')}")
print(f"[INFO] Detecting repeated headers/footers...")
noise_texts = detect_headers_footers(doc)
if noise_texts:
print(f" Found {len(noise_texts)} repeated noise texts (will be removed):")
for t in list(noise_texts)[:3]:
header_noise, footer_noise = detect_headers_footers(doc)
noise_by_band = {"header": header_noise, "footer": footer_noise, None: set()}
repeated_texts = header_noise | footer_noise
if repeated_texts:
print(f" Found {len(repeated_texts)} repeated noise texts (will be removed):")
for t in list(repeated_texts)[:3]:
print(f" - {t[:30]}...")
markdown_content = f"# {title}\n\n"
@@ -1665,7 +1659,8 @@ def extract_pdf_to_markdown(
if block["type"] == 0:
# Check if this is noise text to be filtered (whole block match)
block_text_full = "".join([span["text"] for line in block["lines"] for span in line["spans"]]).strip()
if block_text_full in noise_texts:
block_band = _header_footer_band(block_rect, page.rect)
if block_text_full in noise_by_band[block_band]:
continue
for line in block["lines"]:
@@ -1713,9 +1708,15 @@ def extract_pdf_to_markdown(
continue
# Secondary check: line-level noise match (sometimes blocks are split)
if line_text in noise_texts:
band = _header_footer_band(fitz.Rect(line["bbox"]), page.rect)
if line_text in noise_by_band[band]:
continue
if band == "footer":
line_text = remove_page_footer(line_text)
if not line_text:
continue
line_text = merge_adjacent_formatting(line_text)
heading_level = get_heading_level(line_size, size_map, line_text, line_flags)
@@ -1739,7 +1740,8 @@ def extract_pdf_to_markdown(
"content": final_text,
"is_heading": heading_level > 0,
"is_list": is_list,
"is_code": is_code_line
"is_code": is_code_line,
"is_footer": band == "footer",
})
elif block["type"] == 1:
@@ -1771,13 +1773,15 @@ def extract_pdf_to_markdown(
next_el = page_elements[j]
if next_el["type"] != 0:
break
if el.get("is_footer") or next_el.get("is_footer"):
break
if not should_merge_lines({"content": merged_content, "is_heading": False, "is_list": False}, next_el):
break
merged_content = join_wrapped_text(merged_content, next_el["content"])
j += 1
merged_elements.append({
"type": 0,
"content": remove_page_footer(merged_content),
"content": merged_content,
"is_heading": False,
"is_list": False
})
@@ -168,7 +168,7 @@ def escape_table_cell(value: str) -> str:
"""Escape Markdown table syntax inside a cell."""
normalized = value.replace("\r\n", "\n").replace("\r", "\n")
lines = [re.sub(r"\s+", " ", line).strip() for line in normalized.split("\n")]
with_breaks = "<br>".join(lines)
with_breaks = "<br>".join(line for line in lines if line)
return with_breaks.replace("|", r"\|") or " "
@@ -309,7 +309,7 @@ def _paragraph_to_markdown(
# A soft line break inside the paragraph; python-pptx's runs skip it.
current_text += "\n"
continue
if tag != "r":
if tag not in {"r", "fld"}:
continue
run = _Run(child, paragraph)
url = _run_url(run, shape)
@@ -319,7 +319,7 @@ def _paragraph_to_markdown(
flush()
current_text = ""
current_url = url
current_text += run.text or ""
current_text += (child.findtext(qn("a:t")) or "") if tag == "fld" else (run.text or "")
flush()
text = normalize_text("".join(parts))
@@ -363,7 +363,10 @@ def _shape_click_target(shape: object) -> str | None:
def _paragraph_has_hyperlink(paragraph: object) -> bool:
"""True if any run carries an external URL or an internal slide jump."""
for run in paragraph.runs:
for child in paragraph._p:
if child.tag not in {qn("a:r"), qn("a:fld")}:
continue
run = _Run(child, paragraph)
try:
if run.hyperlink.address:
return True
@@ -378,6 +381,95 @@ def _paragraph_has_hyperlink(paragraph: object) -> bool:
return False
def _paragraph_bullet(paragraph: object, shape: object) -> tuple[object | None, bool]:
"""Resolve bullet declarations from paragraph, shape, layout, and master."""
level = f"a:lvl{paragraph.level + 1}pPr"
properties = [(paragraph._p.find(qn("a:pPr")), True)]
def add_shape_styles(source):
if source is None or not getattr(source, "has_text_frame", False):
return
body = source.text_frame._txBody
style = body.find(qn("a:lstStyle"))
if style is not None:
properties.extend((style.find(qn(name)), False) for name in (level, "a:defPPr"))
add_shape_styles(shape)
if shape is not None and hasattr(shape.part, "slide"):
slide = shape.part.slide
master = slide.slide_layout.slide_master
if getattr(shape, "is_placeholder", False):
idx = shape.placeholder_format.idx
layout_shape = slide.slide_layout.placeholders.get(idx)
if layout_shape is not None and layout_shape.has_text_frame:
inherited_paragraph = layout_shape.text_frame.paragraphs[0]
if inherited_paragraph.level == paragraph.level:
properties.append((inherited_paragraph._p.find(qn("a:pPr")), False))
add_shape_styles(layout_shape)
# Master placeholders inherit by type, not by layout idx.
master_shape = next((p for p in master.placeholders
if p.placeholder_format.type == layout_shape.placeholder_format.type), None)
if master_shape is not None and master_shape.has_text_frame:
inherited_paragraph = master_shape.text_frame.paragraphs[0]
if inherited_paragraph.level == paragraph.level:
properties.append((inherited_paragraph._p.find(qn("a:pPr")), False))
add_shape_styles(master_shape)
placeholder_type = str(shape.placeholder_format.type).split()[0]
style_name = "titleStyle" if placeholder_type in {"TITLE", "CENTER_TITLE"} else "bodyStyle"
else:
style_name = "otherStyle"
styles = master._element.find(qn("p:txStyles"))
if styles is not None:
style = styles.find(qn(f"p:{style_name}"))
if style is not None:
properties.extend((style.find(qn(name)), False) for name in (level, "a:defPPr"))
presentation = shape.part.package.presentation_part._element
style = presentation.find(qn("p:defaultTextStyle"))
if style is not None:
properties.extend((style.find(qn(name)), False) for name in (level, "a:defPPr"))
for props, direct in properties:
if props is None:
continue
for name in ("buNone", "buAutoNum", "buChar", "buBlip"):
bullet = props.find(qn(f"a:{name}"))
if bullet is not None:
return bullet, direct
return None, False
def _numbered_marker(number: int, scheme: str) -> str:
"""Keep the source numbering style; unfamiliar schemes carry their id."""
label = str(number)
if scheme.startswith("alpha") and number > 0:
label = ""
remaining = number
while remaining:
remaining, digit = divmod(remaining - 1, 26)
label = chr(ord("a") + digit) + label
if scheme.startswith("alphaUc"):
label = label.upper()
elif scheme.startswith("roman") and 0 < number < 4000:
label = ""
remaining = number
for value, digits in ((1000, "M"), (900, "CM"), (500, "D"), (400, "CD"),
(100, "C"), (90, "XC"), (50, "L"), (40, "XL"),
(10, "X"), (9, "IX"), (5, "V"), (4, "IV"), (1, "I")):
count, remaining = divmod(remaining, value)
label += digits * count
if scheme.startswith("romanLc"):
label = label.lower()
elif scheme not in {"arabicPeriod", "arabicPlain", "arabicParenR", "arabicParenBoth"}:
return f"{number}. [numbering: {scheme}]"
if scheme.endswith("ParenBoth"):
return f"({label})"
if scheme.endswith("ParenR"):
return f"{label})"
if scheme.endswith("Plain"):
return label
return f"{label}."
def text_frame_to_markdown(
text_frame: object,
shape: object = None,
@@ -399,11 +491,11 @@ def text_frame_to_markdown(
if not visible_paragraphs:
return ""
list_like = any(paragraph.level > 0 for paragraph in visible_paragraphs)
if not list_like:
list_like = len(visible_paragraphs) > 1
paragraphs = []
counters = {}
previous_list = False
number_jump = False
style_shape = shape if shape is not None else getattr(text_frame, "_parent", None)
for paragraph in visible_paragraphs:
text = _escape_readback_control_lines(
_paragraph_to_markdown(
@@ -414,15 +506,47 @@ def text_frame_to_markdown(
)
if not text:
continue
bullet, direct = _paragraph_bullet(paragraph, style_shape)
kind = bullet.tag.rsplit("}", 1)[-1] if bullet is not None else "buNone"
list_like = kind != "buNone"
if kind == "buAutoNum":
scheme = bullet.get("type", "arabicPeriod")
level = paragraph.level
for deeper in list(counters):
if deeper > level:
del counters[deeper]
previous = counters.get(level)
start = bullet.get("startAt")
if start is not None and (direct or previous is None or previous[0] != scheme):
number = int(start)
else:
number = previous[1] + 1 if previous and previous[0] == scheme else 1
if previous and previous[0] == scheme and number != previous[1] + 1:
number_jump = True
counters[level] = (scheme, number)
marker = _numbered_marker(number, scheme)
else:
marker = "-"
if kind == "buNone":
counters.clear()
if paragraphs:
literal_numbered = bool(re.match(r"^\d+[.)]\s", text))
previous_numbered = bool(re.match(r"^\d+[.)]\s", paragraphs[-1]))
paragraphs.append("\n" if (list_like and previous_list) or
(literal_numbered and previous_numbered) else "\n\n")
if list_like:
indent = " " * max(paragraph.level, 0)
paragraphs.append(f"{indent}- {text}")
paragraphs.append(f"{indent}{marker} {text}")
else:
paragraphs.append(text)
previous_list = list_like
if list_like:
return "\n".join(paragraphs)
return "\n\n".join(paragraphs)
markdown = "".join(paragraphs)
if number_jump:
# Markdown renderers renumber jumps/restarts. Literal labels retain
# each source number even when native list semantics cannot map.
markdown = re.sub(r"^(\s*)(\d+)\. ", r"\1\2\\. ", markdown, flags=re.M)
return markdown
def table_to_markdown(table: object, shape: object = None) -> str:
@@ -539,13 +539,13 @@ def fetch_url(url: str) -> tuple[str, str]:
return _decode_response_text(response), response.url
def clean_title(title: str) -> str:
"""Remove common site suffixes from a title."""
def clean_title(title: str, site_name: str = "") -> str:
"""Remove only a trailing site name verified by page metadata."""
if not title:
return ""
# Remove site name suffixes often found in Chinese titles
clean = re.sub(r"[-_|].*?(政府|门户|网站|委员会).*$", "", title)
return clean.strip()
if site_name.strip():
title = re.sub(r"\s*[-_|]\s*" + re.escape(site_name.strip()) + r"\s*$", "", title)
return title.strip()
_FILENAME_TRANSLITERATIONS = str.maketrans({
@@ -798,7 +798,7 @@ def extract_metadata(soup: BeautifulSoup, url: str) -> dict[str, str]:
# 1. Title
title_tag = soup.title
title = clean_title(title_tag.string if title_tag else "")
title = title_tag.get_text() if title_tag else ""
# 2. Meta tags
metas = {}
@@ -807,6 +807,7 @@ def extract_metadata(soup: BeautifulSoup, url: str) -> dict[str, str]:
content = meta.get("content")
if name and content:
metas[name.lower()] = content.strip()
title = clean_title(title, metas.get("og:site_name", ""))
# 3. Date Extraction Strategies
date = (
@@ -878,37 +879,46 @@ def extract_metadata(soup: BeautifulSoup, url: str) -> dict[str, str]:
def find_main_content(soup: BeautifulSoup) -> Tag | None:
"""Find the most likely main content container in a page."""
# 1. Clean up first (remove known clutter)
for tag in soup(["script", "style", "nav", "header", "footer", "aside", "noscript", "iframe"]):
# Identify explicit body containers before removing page-shell semantics.
candidates = []
for selector in CONFIG["content_selectors"]:
elements = soup.find_all(selector["name"]) if "name" in selector else soup.find_all(**selector)
for element in elements:
if not any(element is existing for existing in candidates):
candidates.append(element)
for tag in soup(["script", "style", "nav", "noscript", "iframe"]):
tag.decompose()
for tag in soup(["header", "footer", "aside"]):
if not any(tag is body or body in tag.parents for body in candidates):
tag.decompose()
# Keep the original length/score gates: selector matches can be breadcrumbs
# or related-story cards. Merge only independently qualified body roots.
qualified = []
for element in candidates:
if element.parent is None:
continue
text = element.get_text(strip=True)
score = len(text) + 2 * len(re.findall(r'[\u4e00-\u9fa5]', text))
if len(text) >= 100 and score >= 200:
qualified.append(element)
candidates = qualified
roots = [element for element in candidates if element.parent is not None
and not any(parent is other for parent in element.parents for other in candidates)]
if roots:
if len(roots) == 1:
return roots[0]
combined = soup.new_tag("div")
# find_all gives document order, regardless of selector priority.
for element in list(soup.find_all(True)):
if any(element is root for root in roots):
combined.append(element.extract())
return combined
best_element = None
max_score = 0
# 2. Strategy A: Check specific classes/ids
for selector in CONFIG["content_selectors"]:
if "name" in selector:
# Tag name match (article, main)
elements = soup.find_all(selector["name"])
else:
# Class or ID match
elements = soup.find_all(attrs=selector)
for el in elements:
# Score based on text length and chinese character count
text = el.get_text(strip=True)
length = len(text)
if length < 100:
continue
chinese_count = len(re.findall(r'[\u4e00-\u9fa5]', text))
score = length + (chinese_count * 2)
if score > max_score:
max_score = score
best_element = el
# 3. Strategy B: If no specific container found, look for dense text areas with paragraphs
# Without explicit containers, retain the existing dense-text fallback.
if not best_element or max_score < 200:
for div in soup.find_all("div"):
p_count = len(div.find_all("p", recursive=False))
@@ -1090,6 +1100,43 @@ def simple_html_to_markdown_traversal(
if node.name in ['script', 'style', 'comment', 'meta', 'link']:
return ""
if node.name in {'ol', 'ul'}:
items = node.find_all('li', recursive=False)
ordered = node.name == 'ol'
reversed_list = ordered and node.has_attr('reversed')
try:
counter = int(node.get('start', len(items) if reversed_list else 1))
except (ValueError, TypeError):
counter = 1
step = -1 if reversed_list else 1
numbers = []
for item in items:
try:
counter = int(item.get('value', counter))
except (ValueError, TypeError):
pass
numbers.append(counter)
counter += step
if ordered and (reversed_list or node.get('type', '1') != '1'
or any(b != a + 1 for a, b in zip(numbers, numbers[1:]))):
# Markdown cannot represent jumps, reversal, or letter styles.
# Resolve links/images first, then retain HTML list semantics.
for link in node.find_all('a', href=True):
href = link['href']
if href.lower().startswith('javascript:'):
del link['href']
elif not href.startswith('#') and urlparse(href).scheme.lower() not in {'mailto', 'tel'}:
link['href'] = urljoin(page_url, href)
return '\n\n' + str(node) + '\n\n'
rendered = []
for number, item in zip(numbers, items):
body = ''.join(traverse(child) for child in item.children).strip()
body_lines = body.splitlines()
marker = f'{number}. ' if ordered else '- '
indent = ' ' * len(marker)
rendered.append(marker + ('\n' + indent).join(body_lines))
return '\n\n' + '\n'.join(rendered) + '\n\n'
# Handle Block Elements
is_block = node.name in ['p', 'div', 'h1', 'h2', 'h3', 'h4',
'h5', 'h6', 'li', 'blockquote', 'pre', 'hr', 'table', 'tr',
@@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""Regression tests for position-aware PDF header and footer removal.
Repeated edge text must not erase signatures, and footer lines must not join
body dates before date/page-number cleanup.
"""
from __future__ import annotations
import sys
import tempfile
import unittest
from pathlib import Path
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
BACKEND_DIR = SCRIPTS_DIR / "source_to_md"
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
import fitz # noqa: E402
import pdf_to_md # noqa: E402
class HeaderFooterPositionTests(unittest.TestCase):
def _convert(self, last_page_lines: list[tuple[float, str]]) -> str:
with tempfile.TemporaryDirectory() as tmp:
source = Path(tmp) / "signature.pdf"
with fitz.open() as doc:
for number in range(1, 6):
page = doc.new_page(width=600, height=800)
page.insert_text((72, 40), "Example Company", fontname="helv", fontsize=12)
page.insert_text((72, 200), f"Body of section {number}.", fontname="helv", fontsize=12)
page.insert_text((72, 700), "Example Footer", fontname="helv", fontsize=12)
if number == 5:
for y, text in last_page_lines:
page.insert_text((72, y), text, fontname="helv", fontsize=12)
page.insert_text((290, 770), str(number), fontname="helv", fontsize=12)
doc.save(source)
return pdf_to_md.extract_pdf_to_markdown(str(source), images="none")
def test_signature_equal_to_header_is_kept_and_header_removed(self) -> None:
markdown = self._convert([(400, "Example Company"), (430, "October 2026")])
self.assertEqual(markdown.count("Example Company"), 1)
self.assertIn("October 2026", markdown)
self.assertIn("Example Company", markdown.split("<!-- Page 5 -->")[1])
def test_footer_page_number_does_not_swallow_body_date(self) -> None:
markdown = self._convert([(430, "October 2026")])
self.assertIn("October 2026", markdown)
self.assertNotIn("October 2026 5", markdown)
self.assertNotIn("Example Company", markdown)
def test_signature_equal_to_header_inside_footer_is_kept(self) -> None:
markdown = self._convert([(730, "Example Company"), (750, "October 2026")])
self.assertEqual(markdown.count("Example Company"), 1)
self.assertIn("October 2026", markdown)
self.assertIn("Example Company", markdown.split("<!-- Page 5 -->")[1])
self.assertNotIn("Example Footer", markdown)
def test_date_and_page_number_inside_footer_are_removed(self) -> None:
markdown = self._convert([(730, "November 2025 8")])
self.assertNotIn("November 2025", markdown)
self.assertIn("Body of section 5.", markdown)
def test_date_and_page_number_inside_body_are_kept(self) -> None:
markdown = self._convert([(430, "November 2025 8")])
self.assertIn("November 2025 8", markdown)
def test_line_crossing_header_boundary_is_kept(self) -> None:
markdown = self._convert([(125, "Example Company")])
self.assertEqual(markdown.count("Example Company"), 1)
def test_repeated_footer_text_inside_body_is_kept(self) -> None:
markdown = self._convert([(400, "Example Footer")])
self.assertEqual(markdown.count("Example Footer"), 1)
def test_body_line_in_block_crossing_header_boundary_is_kept(self) -> None:
markdown = self._convert([(110, "Example Company"), (125, "Example Company")])
self.assertEqual(markdown.count("Example Company"), 1)
@@ -0,0 +1,56 @@
#!/usr/bin/env python3
"""Regression tests for merging PDF tables split across a page break.
Only page-break furniture may separate two halves of one table; a numeric or
labelled paragraph between same-header tables is content and must survive.
"""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
BACKEND_DIR = SCRIPTS_DIR / "source_to_md"
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
import pdf_to_md # noqa: E402
FIRST = "| A | B | C |\n|---|---|---|\n| one | two | three |"
SECOND = "| A | B | C |\n|---|---|---|\n| four | five | six |"
MERGED = "| A | B | C |\n|---|---|---|\n| one | two | three |\n| four | five | six |"
def _merge(markdown: str) -> str:
return pdf_to_md.merge_markdown_continuation_tables(markdown)
class ContinuationMergeTests(unittest.TestCase):
def test_page_break_with_printed_page_number_merges(self) -> None:
source = f"{FIRST}\n\n12\n\n<!-- Page 3 -->\n\n{SECOND}"
self.assertEqual(_merge(source), MERGED)
def test_blank_gap_merges(self) -> None:
self.assertEqual(_merge(f"{FIRST}\n\n{SECOND}"), MERGED)
def test_numeric_paragraph_without_page_break_is_kept(self) -> None:
for number in ("3000000", "2026", "42"):
with self.subTest(number=number):
source = f"{FIRST}\n{number}\n{SECOND}"
self.assertEqual(_merge(source), source)
def test_long_number_at_page_break_is_kept(self) -> None:
source = f"{FIRST}\n\n3000000\n\n<!-- Page 3 -->\n\n{SECOND}"
self.assertEqual(_merge(source), source)
def test_label_paragraph_between_tables_is_kept(self) -> None:
source = f"{FIRST}\n\n国有企业\n\n<!-- Page 3 -->\n\n{SECOND}"
self.assertEqual(_merge(source), source)
def test_lines_after_a_lone_table_are_kept(self) -> None:
for tail in ("\n\nSome text", "\n\n2026\n\nSome text", "\n\n<!-- Page 2 -->\n\n7"):
with self.subTest(tail=tail):
source = FIRST + tail
self.assertEqual(_merge(source), source)
@@ -0,0 +1,128 @@
#!/usr/bin/env python3
"""PPT Master - DOCX content preservation regressions.
Usage: python3 -m unittest tests.test_source_to_md_content_loss_docx
Dependencies: python-docx, mammoth
"""
import io
import json
import sys
import tempfile
import unittest
import zipfile
from contextlib import redirect_stdout
from pathlib import Path
from xml.etree import ElementTree as ET
from docx import Document
from docx.oxml import parse_xml
from docx.oxml.ns import nsdecls
from docx.opc.constants import RELATIONSHIP_TYPE as RT
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(SCRIPTS_DIR))
sys.path.insert(0, str(SCRIPTS_DIR / "source_to_md"))
import doc_to_md # noqa: E402
class DocxContentLossTests(unittest.TestCase):
def _convert(self, document, patch_archive=None):
with tempfile.TemporaryDirectory() as tmp:
source = Path(tmp) / "source.docx"
if patch_archive:
buf = io.BytesIO()
document.save(buf)
patch_archive(buf, source)
else:
document.save(source)
with redirect_stdout(io.StringIO()):
markdown = doc_to_md.convert_to_markdown(str(source))
profile = json.loads(source.with_suffix(".conversion_profile.json").read_text(encoding="utf-8"))
return markdown, profile
def test_f01_nested_table_body_is_preserved(self) -> None:
doc = Document()
cell = doc.add_table(rows=1, cols=1).cell(0, 0)
cell.text = "Outer body"
cell.add_table(rows=1, cols=1).cell(0, 0).text = "NESTED_SENTINEL_3000000"
md, _ = self._convert(doc)
self.assertIn("Outer body", md)
self.assertIn("NESTED_SENTINEL_3000000", md)
def test_f02_content_control_display_text_is_preserved(self) -> None:
doc = Document()
cell = doc.add_table(rows=1, cols=1).cell(0, 0)
cell.text = "Before"
cell._tc.append(parse_xml('<w:sdt ' + nsdecls('w') + '><w:sdtContent><w:p><w:r>'
'<w:t>CONTROL_BODY</w:t></w:r></w:p></w:sdtContent></w:sdt>'))
md, profile = self._convert(doc)
self.assertIn("CONTROL_BODY", md)
self.assertTrue(any("content control" in w.lower() for w in profile["warnings"]))
def test_f03_table_endnote_reference_and_body_are_preserved(self) -> None:
doc = Document()
p = doc.add_table(rows=1, cols=1).cell(0, 0).paragraphs[0]
p.add_run("Reference")
p._p.append(parse_xml('<w:r ' + nsdecls('w') + '><w:endnoteReference w:id="1"/></w:r>'))
def patch_archive(buf, source):
with zipfile.ZipFile(buf) as zin, zipfile.ZipFile(source, "w") as zout:
for item in zin.infolist():
data = zin.read(item.filename)
if item.filename == "word/_rels/document.xml.rels":
root = ET.fromstring(data)
ET.SubElement(root, '{http://schemas.openxmlformats.org/package/2006/relationships}'
'Relationship', Id="rIdEndnotes", Type=RT.ENDNOTES, Target="endnotes.xml")
data = ET.tostring(root)
if item.filename == "[Content_Types].xml":
root = ET.fromstring(data)
ET.SubElement(root, '{http://schemas.openxmlformats.org/package/2006/content-types}'
'Override', PartName="/word/endnotes.xml", ContentType=
"application/vnd.openxmlformats-officedocument.wordprocessingml.endnotes+xml")
data = ET.tostring(root)
zout.writestr(item, data)
zout.writestr("word/endnotes.xml", '<w:endnotes ' + nsdecls('w') + '><w:endnote w:id="1">'
'<w:p><w:r><w:t>ENDNOTE_BODY</w:t></w:r></w:p></w:endnote></w:endnotes>')
md, _ = self._convert(doc, patch_archive)
self.assertIn("[^endnote-1]", md)
self.assertIn("[^endnote-1]: ENDNOTE_BODY", md)
def test_f09_table_hyperlink_target_is_preserved(self) -> None:
doc = Document()
p = doc.add_table(rows=1, cols=1).cell(0, 0).paragraphs[0]
rid = p.part.relate_to("https://example.invalid/evidence", RT.HYPERLINK, is_external=True)
p._p.append(parse_xml('<w:hyperlink ' + nsdecls('w', 'r') + ' r:id="' + rid + '">'
'<w:r><w:t>Evidence</w:t></w:r></w:hyperlink>'))
md, _ = self._convert(doc)
self.assertIn("| [Evidence](https://example.invalid/evidence) |", md)
self.assertIn("| --- |", md)
def _math_document(self, xml):
doc = Document()
doc.add_paragraph()._p.append(parse_xml('<m:oMath ' + nsdecls('m') + '>' + xml + '</m:oMath>'))
return doc
def test_f23_delimiter_operands_keep_separator(self) -> None:
for properties, separator in [('<m:dPr><m:sepChr m:val="|"/></m:dPr>', '|'), ('', '|'),
('<m:dPr><m:sepChr m:val=";"/></m:dPr>', ';')]:
with self.subTest(separator=separator, properties=properties):
md, _ = self._convert(self._math_document('<m:d>' + properties +
'<m:e><m:r><m:t>x</m:t></m:r></m:e><m:e><m:r><m:t>y</m:t></m:r></m:e></m:d>'))
self.assertIn('x' + separator + 'y', md)
def test_f24_group_character_and_position_are_preserved(self) -> None:
for char, pos, expected in [('⏞', 'top', r'\overbrace{x+y}'),
('⏟', 'bot', r'\underbrace{x+y}')]:
with self.subTest(char=char):
md, _ = self._convert(self._math_document('<m:groupChr><m:groupChrPr><m:chr m:val="' + char +
'"/><m:pos m:val="' + pos + '"/></m:groupChrPr><m:e><m:r><m:t>x+y</m:t></m:r>'
'</m:e></m:groupChr>'))
self.assertIn(expected, md)
def test_f24_unknown_group_character_is_retained_with_warning(self) -> None:
md, profile = self._convert(self._math_document('<m:groupChr><m:groupChrPr><m:chr m:val="?"/>'
'</m:groupChrPr><m:e><m:r><m:t>x</m:t></m:r></m:e></m:groupChr>'))
self.assertIn('?', md)
self.assertTrue(any('group character' in w for w in profile['warnings']))
@@ -0,0 +1,188 @@
#!/usr/bin/env python3
"""PPT Master - Spreadsheet content preservation regressions.
Usage: python3 -m unittest tests.test_source_to_md_content_loss_excel
Dependencies: openpyxl
"""
import io
import json
import subprocess
import sys
import tempfile
import unittest
from contextlib import redirect_stderr, redirect_stdout
from datetime import datetime, time
from pathlib import Path
from openpyxl import Workbook
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(SCRIPTS_DIR))
sys.path.insert(0, str(SCRIPTS_DIR / "source_to_md"))
import excel_to_md # noqa: E402
class ExcelContentLossTests(unittest.TestCase):
def _convert(self, workbook, **options):
with tempfile.TemporaryDirectory() as tmp:
source = Path(tmp) / "book.xlsx"
workbook.save(source)
err = io.StringIO()
with redirect_stdout(io.StringIO()), redirect_stderr(err):
md = excel_to_md.convert_to_markdown(str(source), **options)
profile = json.loads(source.with_suffix('.conversion_profile.json').read_text(encoding='utf-8'))
return md, profile, err.getvalue()
def test_f21_fractional_seconds_are_preserved(self) -> None:
wb = Workbook()
wb.active.append(['Datetime', 'Time'])
wb.active.append([datetime(2026, 10, 5, 12, 34, 56, 789000), time(12, 34, 56, 789000)])
md, _, _ = self._convert(wb)
self.assertIn('2026-10-05 12:34:56.789', md)
self.assertIn('| 12:34:56.789', md)
def test_f22_percent_and_identifier_formats_are_preserved(self) -> None:
wb = Workbook()
sheet = wb.active
sheet.append(['Percent', 'Identifier'])
sheet.append([0.15, 123])
sheet['A2'].number_format = '0%'
sheet['B2'].number_format = '000000'
md, _, _ = self._convert(wb)
self.assertIn('| 15% | 000123 |', md)
def test_f22_percent_scales_excel_precision_without_binary_noise(self) -> None:
# openpyxl reads 17-digit floats; Excel itself holds 15 significant digits.
self.assertEqual(excel_to_md._format_number(0.19497971117842003, "0.00%"), "19.497971117842%")
self.assertEqual(excel_to_md._format_number(0.1 + 0.2, "0%"), "30%")
def test_f22_unsupported_format_retains_value_and_format_with_warning(self) -> None:
wb = Workbook()
wb.active.append(['Amount'])
wb.active.append([123.456])
wb.active['A2'].number_format = '"kg "0.00'
md, profile, err = self._convert(wb)
self.assertIn('123.456', md)
self.assertIn('"kg "0.00', md)
self.assertIn('(1 cells)', md)
self.assertTrue(profile['warnings'])
self.assertIn('number format', err)
def _hidden_workbook(self):
wb = Workbook()
wb.active.append(['Public'])
for name, state in [('Appendix', 'hidden'), ('Internal', 'veryHidden')]:
sheet = wb.create_sheet(name)
sheet.append(['HIDDEN_BODY_' + name])
sheet.sheet_state = state
return wb
def test_s01_hidden_sheets_are_named_and_warned_by_default(self) -> None:
md, profile, err = self._convert(self._hidden_workbook())
for name in ['Appendix', 'Internal']:
self.assertIn(name, md)
self.assertIn(name, err)
self.assertIn(name, ' '.join(profile['warnings']))
self.assertNotIn('HIDDEN_BODY_' + name, md)
def test_s01_include_hidden_exports_and_marks_sheets(self) -> None:
md, profile, _ = self._convert(self._hidden_workbook(), include_hidden=True)
for name in ['Appendix', 'Internal']:
self.assertIn('## Sheet: ' + name + ' (hidden)', md)
self.assertIn('HIDDEN_BODY_' + name, md)
self.assertFalse(profile['warnings'])
def test_s01_include_hidden_is_forwarded_by_unified_cli(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
source = Path(tmp) / 'book.xlsx'
self._hidden_workbook().save(source)
proc = subprocess.run([sys.executable, str(SCRIPTS_DIR / 'source_to_md.py'), str(source),
'--include-hidden', '--json'], capture_output=True, text=True)
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertIn('HIDDEN_BODY_Appendix', source.with_suffix('.md').read_text(encoding='utf-8'))
def test_s02_merged_values_are_copied_with_shared_value_annotation(self) -> None:
wb = Workbook()
sheet = wb.active
sheet.append(['Amount', 'Item'])
sheet.append([3000000, 'First'])
sheet['B3'] = 'Second'
sheet['A2'].number_format = '#,##0'
sheet.merge_cells('A2:A3')
md, _, _ = self._convert(wb)
self.assertIn('| 3000000 | First |', md)
self.assertIn('| 3000000 | Second |', md)
self.assertIn('> Merged cells: A2:A3 (shared value)', md)
def test_display_formats_never_round_or_emit_per_cell_notes(self) -> None:
wb = Workbook()
sheet = wb.active
sheet.append(['Amount', 'Format'])
formats = ['0', '0.00', '#,##0', '#,##0.00', '0.00_', '0_', '"USD "0.00',
'_(* #,##0.00_);_(* (#,##0.00);_(* "-"??_);_(@_)', '@']
for fmt in formats:
sheet.append([74083.6551, fmt])
sheet.cell(sheet.max_row, 1).number_format = fmt
md, profile, err = self._convert(wb)
self.assertEqual(md.count('| 74083.6551 |'), len(formats))
self.assertFalse(profile['warnings'])
self.assertEqual(err, '')
self.assertNotIn('number format', md)
def test_percentages_keep_full_precision(self) -> None:
wb = Workbook()
sheet = wb.active
sheet.append(['Percent'])
for value in (0.036, 0.551282051282051, -0.036, 1.00001):
sheet.append([value])
sheet.cell(sheet.max_row, 1).number_format = '0.00%'
md, _, _ = self._convert(wb)
for value in ('3.6%', '55.1282051282051%', '-3.6%', '100.001%'):
self.assertIn('| ' + value + ' |', md)
def test_identifier_does_not_round_fractional_values(self) -> None:
wb = Workbook()
wb.active.append(['Identifier'])
for value in (123.456, -123):
wb.active.append([value])
wb.active.cell(wb.active.max_row, 1).number_format = '000000'
md, _, _ = self._convert(wb)
self.assertIn('| 123.456 |', md)
self.assertIn('| -000123 |', md)
def test_unknown_semantic_formats_are_summarized_once_per_sheet(self) -> None:
wb = Workbook()
wb.active.append(['Amount'])
for fmt in ('"kg "0.00', '"kg "0.00', '0.00"m"'):
wb.active.append([123.456])
wb.active.cell(wb.active.max_row, 1).number_format = fmt
md, profile, err = self._convert(wb)
self.assertEqual(md.count('unsupported number formats:'), 1)
self.assertIn('(2 cells)', md)
self.assertIn('(1 cells)', md)
self.assertEqual(len(profile['warnings']), 1)
self.assertEqual(err.count('[WARN]'), 1)
def test_text_merges_do_not_emit_annotations(self) -> None:
wb = Workbook()
wb.active.append(['Title', None])
wb.active.merge_cells('A1:B1')
wb.active.append(['Amount', 'Item'])
wb.active.append([12, 'Data'])
md, _, _ = self._convert(wb)
self.assertIn('Title', md)
self.assertNotIn('Merged cells:', md)
def test_percent_sections_and_color_keep_precision(self) -> None:
wb = Workbook()
sheet = wb.active
sheet.append(['Percent'])
for value, fmt in ((0.036, r'\+0%;\-0%'), (-0.036, r'\+0%;\-0%'),
(0.551282051282051, '0.00%;[Red](0.00%)')):
sheet.append([value])
sheet.cell(sheet.max_row, 1).number_format = fmt
md, profile, _ = self._convert(wb)
for value in ('3.6%', '-3.6%', '55.1282051282051%'):
self.assertIn('| ' + value + ' |', md)
self.assertFalse(profile['warnings'])
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""PPT Master - PowerPoint content preservation regressions.
Usage: python3 -m unittest tests.test_source_to_md_content_loss_pptx
Dependencies: python-pptx
"""
import io
import sys
import tempfile
import unittest
from contextlib import redirect_stdout
from pathlib import Path
from pptx import Presentation
from pptx.oxml.xmlchemy import OxmlElement
from pptx.util import Inches
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(SCRIPTS_DIR))
sys.path.insert(0, str(SCRIPTS_DIR / 'source_to_md'))
import ppt_to_md # noqa: E402
class PptxContentLossTests(unittest.TestCase):
def _presentation(self):
prs = Presentation()
slide = prs.slides.add_slide(prs.slide_layouts[6])
shape = slide.shapes.add_textbox(Inches(1), Inches(1), Inches(5), Inches(2))
return prs, shape
def _convert(self, prs):
with tempfile.TemporaryDirectory() as tmp, redirect_stdout(io.StringIO()):
path = Path(tmp) / 'deck.pptx'
prs.save(path)
return ppt_to_md.convert_presentation_to_markdown(str(path), str(path.with_suffix('.md')))
def test_f05_field_text_keeps_xml_order(self) -> None:
prs, shape = self._presentation()
p = shape.text_frame.paragraphs[0]
p.text = 'Date: '
field = OxmlElement('a:fld')
field.set('id', '{11111111-1111-1111-1111-111111111111}')
field.set('type', 'datetime1')
text = OxmlElement('a:t')
text.text = '2026-10-05'
field.append(text)
p._p.append(field)
p.add_line_break()
p.add_run().text = 'After'
md = self._convert(prs)
self.assertIn('Date: 2026-10-05\nAfter', md)
def test_f05_field_hyperlink_is_preserved(self) -> None:
from pptx.text.text import _Run
prs, shape = self._presentation()
p = shape.text_frame.paragraphs[0]
field = OxmlElement('a:fld')
field.set('id', '{11111111-1111-1111-1111-111111111111}')
field.set('type', 'datetime1')
text = OxmlElement('a:t')
text.text = '2026-10-05'
field.append(text)
p._p.append(field)
_Run(field, p).hyperlink.address = 'https://example.invalid/date'
self.assertIn('[2026-10-05](https://example.invalid/date)', self._convert(prs))
def test_plain_table_cell_does_not_require_shape_styles(self) -> None:
prs, shape = self._presentation()
slide = prs.slides[0]
table = slide.shapes.add_table(1, 1, Inches(1), Inches(3), Inches(4), Inches(1)).table
table.cell(0, 0).text = 'Cell body'
self.assertEqual(ppt_to_md.text_frame_to_markdown(table.cell(0, 0).text_frame), 'Cell body')
self.assertIn('Cell body', self._convert(prs))
def test_f25_explicit_start_and_continuation_are_preserved(self) -> None:
prs, shape = self._presentation()
for i, label in enumerate(['Fifth', 'Sixth', 'Seventh']):
p = shape.text_frame.paragraphs[0] if i == 0 else shape.text_frame.add_paragraph()
p.text = label
bullet = OxmlElement('a:buAutoNum')
bullet.set('type', 'arabicPeriod')
if i < 2:
bullet.set('startAt', str(5 + i))
p._p.get_or_add_pPr().append(bullet)
md = self._convert(prs)
for text in ('5. Fifth', '6. Sixth', '7. Seventh'):
self.assertIn(text, md)
def test_f25_inherited_list_style_and_explicit_none(self) -> None:
prs, shape = self._presentation()
lst = shape.text_frame._txBody.find(ppt_to_md.qn('a:lstStyle'))
level = OxmlElement('a:lvl1pPr')
bullet = OxmlElement('a:buAutoNum')
bullet.set('type', 'alphaLcParenR')
bullet.set('startAt', '5')
level.append(bullet)
lst.append(level)
for i, label in enumerate(['Fifth', 'Sixth', 'Plain']):
p = shape.text_frame.paragraphs[0] if i == 0 else shape.text_frame.add_paragraph()
p.text = label
if i == 2:
p._p.get_or_add_pPr().append(OxmlElement('a:buNone'))
md = self._convert(prs)
self.assertIn('e) Fifth', md)
self.assertIn('f) Sixth', md)
self.assertIn('\n\nPlain', md)
def test_f25_numbering_jumps_keep_literal_labels(self) -> None:
prs, shape = self._presentation()
for i, number in enumerate((5, 9)):
p = shape.text_frame.paragraphs[0] if i == 0 else shape.text_frame.add_paragraph()
p.text = f'Item {number}'
bullet = OxmlElement('a:buAutoNum')
bullet.set('type', 'arabicPeriod')
bullet.set('startAt', str(number))
p._p.get_or_add_pPr().append(bullet)
md = self._convert(prs)
self.assertIn(r'5\. Item 5', md)
self.assertIn(r'9\. Item 9', md)
def test_f25_layout_inherited_numbering(self) -> None:
prs = Presentation()
slide = prs.slides.add_slide(prs.slide_layouts[1])
body = slide.placeholders[1]
layout_body = slide.slide_layout.placeholders.get(body.placeholder_format.idx)
p = layout_body.text_frame.paragraphs[0]
props = p._p.get_or_add_pPr()
for child in list(props):
props.remove(child)
bullet = OxmlElement('a:buAutoNum')
bullet.set('type', 'arabicPeriod')
bullet.set('startAt', '5')
props.append(bullet)
body.text = 'Fifth\nSixth'
md = self._convert(prs)
self.assertIn('5. Fifth', md)
self.assertIn('6. Sixth', md)
def test_f27_plain_paragraphs_do_not_acquire_bullets(self) -> None:
prs, shape = self._presentation()
shape.text = 'First paragraph\nSecond paragraph'
md = self._convert(prs)
self.assertIn('First paragraph\n\nSecond paragraph', md)
self.assertNotIn('- First paragraph', md)
def test_f27_real_bullets_are_retained_per_paragraph(self) -> None:
prs, shape = self._presentation()
p = shape.text_frame.paragraphs[0]
p.text = 'List item'
bullet = OxmlElement('a:buChar')
bullet.set('char', '•')
p._p.get_or_add_pPr().append(bullet)
shape.text_frame.add_paragraph().text = 'Plain paragraph'
md = self._convert(prs)
self.assertIn('- List item\n\nPlain paragraph', md)
def test_literal_numbered_paragraphs_have_no_blank_separator(self) -> None:
prs, shape = self._presentation()
shape.text = '1. First\n2. Second\n3) Third'
md = self._convert(prs)
self.assertIn('1. First\n2. Second\n3) Third', md)
def test_table_paragraphs_use_one_break(self) -> None:
prs, _ = self._presentation()
table = prs.slides[0].shapes.add_table(1, 1, Inches(1), Inches(3), Inches(4), Inches(1)).table
table.cell(0, 0).text = 'First\nSecond'
md = self._convert(prs)
self.assertIn('| First<br>Second |', md)
self.assertNotIn('<br><br>', md)
@@ -0,0 +1,100 @@
#!/usr/bin/env python3
"""PPT Master - Offline web content preservation regressions.
Usage: python3 -m unittest tests.test_source_to_md_content_loss_web
Dependencies: beautifulsoup4, requests
"""
import io
import sys
import tempfile
import unittest
from contextlib import redirect_stdout
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(SCRIPTS_DIR))
sys.path.insert(0, str(SCRIPTS_DIR / 'source_to_md'))
import web_to_md # noqa: E402
class WebContentLossTests(unittest.TestCase):
def _convert(self, body, head='<title>Test page</title>'):
html = '<html><head>' + head + '</head><body>' + body + '</body></html>'
response = SimpleNamespace(url='https://example.invalid/page', content=html.encode(),
headers={'Content-Type': 'text/html; charset=utf-8'}, text=html,
encoding='utf-8', apparent_encoding='utf-8')
with tempfile.TemporaryDirectory() as tmp, redirect_stdout(io.StringIO()):
output = Path(tmp) / 'page.md'
with patch.object(web_to_md, 'fetch_response', return_value=response):
result = web_to_md.process_url(response.url, str(output), download_images=False)
self.assertTrue(result[0], result)
return output.read_text(encoding='utf-8')
def test_f06_article_semantic_content_survives_shell_cleanup(self) -> None:
for container in ('article', 'main', 'div class="article-content"'):
with self.subTest(container=container):
tag = container.split()[0]
md = self._convert('<header>SITE_HEADER</header><nav>SITE_NAV</nav><' + container + '>'
'<header><h1>ARTICLE_TITLE</h1></header><p>' + 'Body text. ' * 30 + '</p><aside>BODY_NOTE</aside>'
'<footer>AUTHOR_SIGNATURE</footer><nav>ARTICLE_NAV</nav></' + tag + '>'
'<footer>SITE_FOOTER</footer><aside>SITE_AD</aside>')
for text in ('ARTICLE_TITLE', 'BODY_NOTE', 'AUTHOR_SIGNATURE'):
self.assertIn(text, md)
for text in ('SITE_HEADER', 'SITE_NAV', 'SITE_FOOTER', 'SITE_AD', 'ARTICLE_NAV'):
self.assertNotIn(text, md)
def test_f07_parallel_articles_are_retained_in_source_order(self) -> None:
md = self._convert('<article><p>' + 'First body. ' * 40 + '</p></article>'
'<article><p>' + 'SECOND_BODY ' * 30 + '</p></article>')
self.assertIn('SECOND_BODY', md)
self.assertLess(md.index('First body.'), md.index('SECOND_BODY'))
def test_f07_parallel_identified_content_blocks_are_retained(self) -> None:
md = self._convert('<div class="article-content"><p>' + 'First body. ' * 40 + '</p></div>'
'<div class="article-content"><p>' + 'SECOND_BODY ' * 30 + '</p></div>')
self.assertIn('SECOND_BODY', md)
def test_f08_title_is_not_cut_by_site_keyword(self) -> None:
md = self._convert('<p>Body</p>', '<title>预算-政府补贴3000000元方案</title>')
self.assertIn('# 预算-政府补贴3000000元方案', md)
def test_f08_only_verified_site_suffix_is_removed(self) -> None:
md = self._convert('<p>Body</p>', '<title>Budget | Example Portal</title>'
'<meta property="og:site_name" content="Example Portal">')
self.assertIn('# Budget\n', md)
self.assertNotIn('# Budget |', md)
def test_f26_ordered_list_start_and_value_are_preserved(self) -> None:
md = self._convert('<ol start="5"><li>Fifth</li><li value="9">Ninth</li></ol>')
# Markdown renderers renumber discontinuous lists; retain HTML for jumps.
self.assertIn('<ol start="5">', md)
self.assertIn('<li value="9">', md)
self.assertIn('Fifth', md)
self.assertIn('Ninth', md)
def test_f26_continuous_and_nested_lists_keep_numbers_and_links(self) -> None:
md = self._convert('<ol start="5"><li><a href="/proof">Fifth</a>'
'<ul><li>Child</li></ul></li><li>Sixth</li></ol>')
self.assertIn('5. [Fifth](https://example.invalid/proof)', md)
self.assertIn('6. Sixth', md)
self.assertIn(' - Child', md)
def test_short_selector_matches_fall_back_to_dense_body(self) -> None:
body = ''.join(f'<p>Paragraph {i}: ' + 'Project investment details. ' * 12 + '</p>' for i in range(8))
for clutter in ('<div class="content">当前位置:首页 &gt; 新闻</div>',
'<div class="related"><article>Related story</article><article>Another story</article></div>'):
with self.subTest(clutter=clutter):
md = self._convert(clutter + '<div class="article-body">' + body + '</div>')
self.assertIn('Paragraph 7:', md)
self.assertNotIn('Related story', md)
self.assertNotIn('当前位置', md)
def test_selector_score_gate_and_nested_bodies(self) -> None:
md = self._convert('<article><p>' + 'x' * 150 + '</p></article>'
'<div><p>' + 'DENSE_BODY ' * 40 + '</p></div>')
self.assertIn('DENSE_BODY', md)
md = self._convert('<main><article><p>' + 'NESTED_BODY ' * 40 + '</p></article></main>')
self.assertEqual(md.count('NESTED_BODY'), 40)
@@ -2,8 +2,8 @@
"sourceId": "shadcn",
"repo": "https://github.com/shadcn-ui/ui.git",
"ref": "main",
"commit": "295a1f114a138f23b5dfee0e0c6812394dfeb90c",
"commit": "dba27166ad7b146b2bd840dcfb0a7d0fd95f5c8c",
"adapter": "claude-skill",
"sourcePath": "skills/shadcn",
"syncedAt": "2026-10-02T16:00:00Z"
"syncedAt": "2026-10-05T16:00:00Z"
}