enne2/limeshtml
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

cc

Browse Source
This commit is contained in:
Matteo Benedetto
2025-12-23 13:20:37 +01:00
parent c3dae45f86
commit db7895c652
2 changed files with 93 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extract_pdf.py
+42 -153
View File
@@ -4,73 +4,26 @@ import os
import sys import sys
import re import re
def extract_pdf_to_html(pdf_path, output_path): def clean_text(text):
# Fix common ligature issues and OCR errors
", "fi") text = text.replace("
text = text.replace("con itto", "conflitto")
text = text.replace("dicoltà", "difficoltà")
text = text.replace("innita", "infinita")
text = text.replace("Foreign Oce", "Foreign Office")
text = text.replace("pacico", "pacifico")
text = text.replace("di nire", "di finire")
text = text.replace("a nire", "a finire")
return text
def extract_pdf_to_md(pdf_path, output_path):
if not os.path.exists(pdf_path): if not os.path.exists(pdf_path):
print(f"Error: File {pdf_path} not found.") print(f"Error: File {pdf_path} not found.")
return return
doc = fitz.open(pdf_path) doc = fitz.open(pdf_path)
html_content = """<!DOCTYPE html> md_content = ""
<html lang="it">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Articolo Limes</title>
<style>
body {
font-family: 'Georgia', serif;
line-height: 1.6;
color: #333;
max-width: 800px;
margin: 0 auto;
padding: 20px;
background-color: #f4f4f4;
}
.article-container {
background-color: #fff;
padding: 40px;
box-shadow: 0 0 10px rgba(0,0,0,0.1);
}
h1 {
color: #b30000;
font-size: 2.2em;
margin-bottom: 10px;
line-height: 1.2;
text-transform: uppercase;
}
.subtitle {
font-size: 1.2em;
color: #555;
margin-bottom: 20px;
font-style: italic;
}
p {
margin-bottom: 15px;
text-align: justify;
}
img {
max-width: 100%;
height: auto;
display: block;
margin: 20px auto;
}
.caption {
font-size: 0.9em;
color: #666;
text-align: center;
margin-top: -10px;
margin-bottom: 20px;
font-style: italic;
}
strong {
color: #b30000;
}
</style>
</head>
<body>
<div class="article-container">
"""
exclude_keywords = [ exclude_keywords = [
"MENU", "CERCA", "CHI SIAMO", "LA SCUOLA DI LIMES", "VOLUMI", "LEGGI PDF", "MENU", "CERCA", "CHI SIAMO", "LA SCUOLA DI LIMES", "VOLUMI", "LEGGI PDF",
@@ -78,41 +31,12 @@ def extract_pdf_to_html(pdf_path, output_path):
"IN EDICOLA", "ABBONATI", "Privacy", "Cookie Policy", "Gestione cookie", "IN EDICOLA", "ABBONATI", "Privacy", "Cookie Policy", "Gestione cookie",
"Riserva TDM", "Dichiarazione di accessibilità", "GEDI Periodici", "Riserva TDM", "Dichiarazione di accessibilità", "GEDI Periodici",
"Tutti a lezione di pace", "Il mito dell'Urbe", "IL PUNTO", "CARTA INEDITA", "Tutti a lezione di pace", "Il mito dell'Urbe", "IL PUNTO", "CARTA INEDITA",
"MENU CERCA", "RUBRICHE", "IL MONDO OGGI", "EBOOK", "AUTORI", "ARGOMENTI", "PROTAGONISTI", "FAQ" "AREA GEOPOLITICA", "ARGOMENTI PIÙ TRATTATI", "Per consultare la carta",
"africa/america latina", "europa/italia", "medio oriente", "russia e csi", "usa e canada"
] ]
def clean_text(text):
# Fix ligatures and other common issues
replacements = {
"": "fi", # specific to the issue seen, might be fl or fi. "conitto" -> conflitto (fl)
"fi": "fi",
"fl": "fl",
"’": "'",
"“": '"',
"”": '"',
}
# Check for the specific char seen in output
# In the output it was "conitto".
# If I can't paste the char easily, I might need to handle it by context or unicode.
# The char in the output looked like a box or special char.
# Let's assume it's a mapping issue.
# "conitto" -> "conflitto". So  is fl.
for k, v in replacements.items():
text = text.replace(k, v)
# Generic ligature fix if they appear as special chars
# But without knowing the exact unicode, it's hard.
# Let's try to fix "con itto" if it appears with a space or weird char.
text = text.replace("conitto", "conflitto")
return text
title_found = False title_found = False
# Buffer for text to handle hyphenation across blocks/pages if needed (though blocks are usually self-contained paragraphs)
# Actually, let's just process blocks.
for page_num in range(len(doc)): for page_num in range(len(doc)):
page = doc[page_num] page = doc[page_num]
page_dict = page.get_text("dict") page_dict = page.get_text("dict")
@@ -128,107 +52,72 @@ def extract_pdf_to_html(pdf_path, output_path):
continue continue
# 2. Sidebar filtering (sidebar is on the right) # 2. Sidebar filtering (sidebar is on the right)
if bbox[0] > 430: if bbox[0] > 430: # Tightened from 440
continue continue
if block["type"] == 0: # Text block if block["type"] == 0: # Text block
# ... (text extraction logic)
text = "" text = ""
lines_text = []
for line in block["lines"]: for line in block["lines"]:
line_str = "".join([span["text"] for span in line["spans"]]) for span in line["spans"]:
lines_text.append(line_str) text += span["text"]
full_text = ""
for i, line in enumerate(lines_text):
if line.endswith("-") and i < len(lines_text) - 1:
full_text += line[:-1]
else:
full_text += line + " "
text = full_text.strip()
text = clean_text(text)
text = text.strip()
if not text: if not text:
continue continue
# Debug print # Clean text
print(f"Processing block: {text[:30]}... bbox={bbox}") text = clean_text(text)
# 3. Keyword filtering # 3. Keyword filtering
# Check if any keyword is in the text if any(kw.lower() in text.lower() for kw in exclude_keywords):
# We use a more aggressive filter for uppercase or short text continue
if any(kw in text for kw in exclude_keywords):
if text.isupper() or len(text) < 100 or "Privacy" in text or "Cookie" in text: # Specific check for the footer line with slashes
print(f"Excluded junk: {text[:30]}...") if "africa/america latina" in text.lower():
continue continue
# 4. Title detection and cleaning # 4. Title detection and cleaning
if not title_found and "Home" in text: if not title_found and "Home" in text:
# Capture the full uppercase title # Capture the full uppercase title
# Adjusted regex to be a bit more flexible match = re.search(r"Home(?:Carta Inedita della Settimana|Il Punto|[\w\s]+?)([A-Z\s,:'\"!?.—–-]{10,})", text)
match = re.search(r"Home(?:[\w\s]+?)([A-Z\s,:'\"!?.—–-]{10,})", text)
if match: if match:
title = match.group(1).strip() title = match.group(1).strip()
remaining = text[match.end():] md_content += f"# {title}\n\n"
# Fix for title grabbing the first letter of subtitle
if title and remaining and title[-1].isupper() and remaining[0].islower():
# Check if the last word of title is just 1 char
last_word = title.split()[-1]
# If it's a single letter and not a common vowel word (A, I, O, E, U, Y), assume it belongs to subtitle
if len(last_word) == 1 and last_word not in ["A", "I", "E", "O", "U"]:
char_to_move = title[-1]
title = title[:-1].strip()
remaining = char_to_move + remaining
html_content += f" <h1>{title}</h1>\n"
# Try to extract subtitle # Try to extract subtitle
remaining = text[match.end():]
subtitle_match = re.search(r"^([^.!?]+[.!?])", remaining) subtitle_match = re.search(r"^([^.!?]+[.!?])", remaining)
if subtitle_match: if subtitle_match:
subtitle = subtitle_match.group(1).strip() subtitle = subtitle_match.group(1).strip()
html_content += f" <div class='subtitle'>{subtitle}</div>\n" md_content += f"_{subtitle}_\n\n"
title_found = True title_found = True
continue continue
else:
# Fallback if regex fails but "Home" is there, maybe the title is just the rest
# Or maybe it's not the title block.
pass
# Formatting # Formatting
if len(text) < 150 and text.isupper() and not title_found and len(text) > 10: if len(text) < 100 and text.isupper() and not title_found and len(text) > 10:
html_content += f" <h1>{text}</h1>\n" md_content += f"# {text}\n\n"
title_found = True title_found = True
print(f"Added H1 (heuristic): {text[:30]}") elif len(text) < 100 and text.isupper():
elif len(text) < 150 and text.isupper(): md_content += f"**{text}**\n\n"
html_content += f" <p><strong>{text}</strong></p>\n"
print(f"Added STRONG: {text[:30]}")
else: else:
if title_found and page_num == 0 and ("Pubblicato il" in text or "di Lucio CARACCIOLO" in text or "Home" in text): if title_found and page_num == 0 and ("Pubblicato il" in text or "di Lucio CARACCIOLO" in text or "Home" in text):
print(f"Skipped (metadata): {text[:30]}")
continue continue
html_content += f" <p>{text}</p>\n" md_content += f"{text}\n\n"
print(f"Added P: {text[:30]}")
elif block["type"] == 1: # Image block elif block["type"] == 1: # Image block
try: try:
image_bytes = block["image"] image_bytes = block["image"]
base64_image = base64.b64encode(image_bytes).decode('utf-8') base64_image = base64.b64encode(image_bytes).decode('utf-8')
img_format = block.get("ext", "png") img_format = block.get("ext", "png")
html_content += f' <img src="data:image/{img_format};base64,{base64_image}" />\n' md_content += f'<img src="data:image/{img_format};base64,{base64_image}" />\n\n'
except Exception as e: except Exception as e:
pass pass
html_content += """ </div>
</body>
</html>"""
with open(output_path, "w", encoding="utf-8") as f: with open(output_path, "w", encoding="utf-8") as f:
f.write(html_content) f.write(md_content)
print(f"HTML generated successfully at {output_path}") print(f"Markdown generated successfully at {output_path}")
if __name__ == "__main__": if __name__ == "__main__":
if len(sys.argv) > 1: if len(sys.argv) > 1:
@@ -236,5 +125,5 @@ if __name__ == "__main__":
else: else:
pdf_file = "Senza un piano di tregua, le nazioni europee scivolano verso la guerra - Limes.pdf" pdf_file = "Senza un piano di tregua, le nazioni europee scivolano verso la guerra - Limes.pdf"
output_file = "public/index.html" output_file = "public/index.md"
extract_pdf_to_html(pdf_file, output_file) extract_pdf_to_md(pdf_file, output_file)
public/index.md
+50
View File
File diff suppressed because one or more lines are too long