bdbb2527e6
All checks were successful
Sync To /home/ubuntu/gfps / sync (push) Successful in 2s
Change-Id: I71e676b997159a141ca07d16f82a73b45f4b2410
146 lines
4.9 KiB
Python
146 lines
4.9 KiB
Python
import os
|
|
import glob
|
|
import email
|
|
from email import policy
|
|
import subprocess
|
|
import urllib.parse
|
|
from bs4 import BeautifulSoup
|
|
import uuid
|
|
|
|
def create_dirs():
|
|
os.makedirs('convert_md/images', exist_ok=True)
|
|
|
|
def clean_filename(name):
|
|
# remove .doc extension
|
|
if name.endswith('.doc'):
|
|
name = name[:-4]
|
|
return name
|
|
|
|
def process_file(filepath):
|
|
print(f"Processing: {filepath}")
|
|
filename = os.path.basename(filepath)
|
|
base_name = clean_filename(filename)
|
|
|
|
with open(filepath, 'rb') as f:
|
|
msg = email.message_from_bytes(f.read(), policy=policy.default)
|
|
|
|
html_content = ""
|
|
images = {}
|
|
|
|
# Extract parts
|
|
if msg.is_multipart():
|
|
for part in msg.walk():
|
|
ctype = part.get_content_type()
|
|
if ctype == 'text/html':
|
|
html_content = part.get_payload(decode=True).decode(part.get_content_charset() or 'utf-8', errors='ignore')
|
|
elif ctype.startswith('image/') or ctype == 'application/octet-stream':
|
|
cloc = part.get("Content-Location")
|
|
cid = part.get("Content-ID")
|
|
|
|
if not cloc and not cid:
|
|
continue
|
|
|
|
img_data = part.get_payload(decode=True)
|
|
if not img_data:
|
|
continue
|
|
|
|
# guess extension
|
|
ext = 'png'
|
|
if img_data.startswith(b'\xff\xd8\xff'): ext = 'jpg'
|
|
elif img_data.startswith(b'GIF8'): ext = 'gif'
|
|
|
|
safe_img_name = f"{base_name}_{uuid.uuid4().hex[:8]}.{ext}"
|
|
|
|
img_save_path = os.path.join('convert_md', 'images', safe_img_name)
|
|
with open(img_save_path, 'wb') as img_f:
|
|
img_f.write(img_data)
|
|
|
|
if cloc:
|
|
images[cloc.strip()] = f"images/{safe_img_name}"
|
|
if cid:
|
|
clean_cid = cid.strip('<> \r\n')
|
|
images[f"cid:{clean_cid}"] = f"images/{safe_img_name}"
|
|
|
|
if not html_content:
|
|
print(f" No HTML content found in {filepath}. Skipping.")
|
|
return
|
|
|
|
# Parse HTML and fix image links
|
|
soup = BeautifulSoup(html_content, 'html.parser')
|
|
|
|
# Try to find the actual content wrapper to skip headers/footers
|
|
wiki_content = soup.find(class_='wiki-content')
|
|
if wiki_content:
|
|
content_to_convert = str(wiki_content)
|
|
else:
|
|
# fallback to body
|
|
body = soup.find('body')
|
|
content_to_convert = str(body) if body else html_content
|
|
|
|
# Re-parse to fix images in the scoped content
|
|
content_soup = BeautifulSoup(content_to_convert, 'html.parser')
|
|
for img in content_soup.find_all('img'):
|
|
src = img.get('src')
|
|
if not src:
|
|
continue
|
|
|
|
src = src.strip()
|
|
|
|
# Check if the src matches any of our extracted images
|
|
matched = False
|
|
for loc, new_path in images.items():
|
|
if src == loc or src.endswith(loc) or loc.endswith(src):
|
|
img['src'] = new_path
|
|
matched = True
|
|
break
|
|
|
|
if not matched:
|
|
unquoted = urllib.parse.unquote(src)
|
|
for loc, new_path in images.items():
|
|
if unquoted == loc or unquoted.endswith(loc) or loc.endswith(unquoted):
|
|
img['src'] = new_path
|
|
matched = True
|
|
break
|
|
|
|
# Remove some common Confluence boilerplate if any
|
|
for el in content_soup.find_all(class_=['aui-message', 'table-wrap', 'confluence-embedded-file-wrapper', 'Section1']):
|
|
if el.name in ('div', 'span'):
|
|
el.unwrap()
|
|
|
|
# Clean up img tags to just src and alt so pandoc converts them to pure markdown images
|
|
for img in content_soup.find_all('img'):
|
|
src = img.get('src', '')
|
|
alt = img.get('alt', '')
|
|
img.attrs = {'src': src, 'alt': alt}
|
|
|
|
cleaned_html = str(content_soup)
|
|
|
|
# Convert HTML to Markdown using Pandoc
|
|
# We use markdown_strict or gfm (GitHub Flavored Markdown) for good table support
|
|
try:
|
|
result = subprocess.run(
|
|
['pandoc', '-f', 'html', '-t', 'gfm', '--wrap=none'],
|
|
input=cleaned_html.encode('utf-8'),
|
|
capture_output=True,
|
|
check=True
|
|
)
|
|
md_content = result.stdout.decode('utf-8')
|
|
|
|
md_filepath = os.path.join('convert_md', f"{base_name}.md")
|
|
with open(md_filepath, 'w', encoding='utf-8') as f:
|
|
f.write(md_content)
|
|
print(f" Successfully converted to {md_filepath}")
|
|
|
|
except subprocess.CalledProcessError as e:
|
|
print(f" Pandoc conversion failed for {filepath}: {e.stderr.decode('utf-8', errors='ignore')}")
|
|
|
|
|
|
def main():
|
|
create_dirs()
|
|
files = glob.glob('*.doc')
|
|
for f in files:
|
|
process_file(f)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|