option#1
from docx import Document
import os
import pandas as pd
import win32com.client

# Define the expected schema sections with full meanings
expected_sections_with_titles = {
    "PART-I": "(Intro)",
    "PART-II": "(Audit Findings)",
    "PART-II-(A)": "(Significant Audit Findings)",
    "PART-II-(B)": "(Other incidental Audit Findings)",
    "PART-III": "(Follow up on findings outstanding of previous Inspection Reports)",
    "PART-IV": "(Best Practice)",
    "PART-V": "(Acknowledgement)"
}

# Convert .doc → .docx using MS Word
def convert_doc_to_docx(doc_path):
    word = win32com.client.Dispatch("Word.Application")
    word.Visible = False
    doc = word.Documents.Open(doc_path)
    docx_path = doc_path + "x"
    doc.SaveAs(docx_path, FileFormat=16)
    doc.Close()
    word.Quit()
    return docx_path

# Extract title (first meaningful line)
def extract_title(doc):
    for para in doc.paragraphs:
        if para.text.strip():
            return para.text.strip()
    return "Blank or No Title"

# Extract summary
def extract_file_summary(doc, max_lines=5):
    lines = []
    for para in doc.paragraphs:
        line = para.text.strip()
        if line:
            lines.append(line)
        if len(lines) >= max_lines:
            break
    return " ".join(lines) if lines else "No content in document."

# Extract present/missing schema-based sections
def analyze_structure(doc):
    section_map = {key: False for key in expected_sections_with_titles}
    for para in doc.paragraphs:
        text = para.text.strip().upper()
        for section in section_map:
            if section in text:
                section_map[section] = True
    present = [f"{k} {expected_sections_with_titles[k]}" for k, v in section_map.items() if v]
    missing = [f"{k} {expected_sections_with_titles[k]}" for k, v in section_map.items() if not v]
    return present, missing

#  Extract non-schema-based unexpected section-like headings
def extract_unexpected_headings(doc):
    unexpected = []
    for para in doc.paragraphs:
        line = para.text.strip()
        if not line:
            continue
        if (line.upper().startswith("PART-") or line.isupper()):
            section_key = line.split()[0].upper()
            if section_key not in expected_sections_with_titles:
                unexpected.append(line)
    return list(set(unexpected))

# Process a single file
def process_file(file_path):
    try:
        doc = Document(file_path)
        if not doc.paragraphs or all(not p.text.strip() for p in doc.paragraphs):
            return "Blank or No Title", [], [], "Blank file.", []
        title = extract_title(doc)
        present, missing = analyze_structure(doc)
        summary = extract_file_summary(doc)
        unexpected = extract_unexpected_headings(doc)
        return title, present, missing, summary, unexpected
    except Exception as e:
        return f"Error: {str(e)}", [], [], f"Error occurred: {str(e)}", []

# Main processor for all folders
def process_all_folders(base_path, output_excel_path):
    all_results = []

    for folder_name in os.listdir(base_path):
        folder_path = os.path.join(base_path, folder_name)
        if not os.path.isdir(folder_path):
            continue

        files = [f for f in os.listdir(folder_path) if (f.endswith(".docx") or f.endswith(".doc")) and not f.startswith("~$")]
        total_files = len(files)
        folder_display_name = folder_name

        for i, file in enumerate(files):
            file_path = os.path.join(folder_path, file)
            temp_converted = False

            if file.endswith(".doc") and not file.endswith(".docx"):
                try:
                    file_path = convert_doc_to_docx(file_path)
                    temp_converted = True
                except Exception as e:
                    print(f"Error converting {file}: {e}")
                    continue

            title, present, missing, summary, unexpected = process_file(file_path)

            all_results.append({
                "Folder Name": folder_display_name,
                "File Count in Folder": total_files if i == 0 else "",
                "File Name": file,
                "Title Name": title,
                "Sections Present": ", ".join(present) if present else "None",
                "Sections Missing": ", ".join(missing) if missing else "None",
                "Summary Content": summary,
                "Other Headings Present (not in schema)": ", ".join(unexpected) if unexpected else "None"
            })

            folder_display_name = ""

            if temp_converted:
                try:
                    os.remove(file_path)
                except Exception as e:
                    print(f"Could not delete temp file {file_path}: {e}")

    df = pd.DataFrame(all_results)
    pd.set_option('display.max_colwidth', None)
    os.makedirs(os.path.dirname(output_excel_path), exist_ok=True)
    df.to_excel(output_excel_path, index=False)
    print(f"Final Excel saved at: {output_excel_path}")

base_folder = r"D:\windows(c) users _mohan folders\Downloads\331 to 413"
output_file = os.path.join(os.path.dirname(__file__), "..", "output", "output_331_413.xlsx")
process_all_folders(base_folder, output_file)


option#2
from docx import Document
import os
import pandas as pd
import win32com.client
import re

# Define expected schema sections with readable titles
expected_sections_with_titles = {
    "PART-I": "(Intro)",
    "PART-II": "(Audit Findings)",
    "PART-II-(A)": "(Significant Audit Findings)",
    "PART-II-(B)": "(Other incidental Audit Findings)",
    "PART-III": "(Follow up on findings outstanding of previous Inspection Reports)",
    "PART-IV": "(Best Practice)",
    "PART-V": "(Acknowledgement)"
}

# Create regex patterns for each section
section_patterns = {
    key: re.compile(
        re.sub(r"[-()]?", r"[-() ]*", re.escape(key)),  # make separators flexible
        re.IGNORECASE
    ) for key in expected_sections_with_titles
}

# Convert .doc → .docx using MS Word
def convert_doc_to_docx(doc_path):
    word = win32com.client.Dispatch("Word.Application")
    word.Visible = False
    doc = word.Documents.Open(doc_path)
    docx_path = doc_path + "x"
    doc.SaveAs(docx_path, FileFormat=16)
    doc.Close()
    word.Quit()
    return docx_path

# Extract first non-empty line as title
def extract_title(doc):
    for para in doc.paragraphs:
        if para.text.strip():
            return para.text.strip()
    return "Blank or No Title"

# Extract brief summary (first few non-empty lines)
def extract_file_summary(doc, max_lines=5):
    lines = []
    for para in doc.paragraphs:
        line = para.text.strip()
        if line:
            lines.append(line)
        if len(lines) >= max_lines:
            break
    return " ".join(lines) if lines else "No content in document."

# Use regex to analyze which schema sections are present
def analyze_structure(doc):
    section_map = {key: False for key in expected_sections_with_titles}
    for para in doc.paragraphs:
        text = para.text.strip()
        for key, pattern in section_patterns.items():
            if pattern.search(text):
                section_map[key] = True
    present = [f"{k} {expected_sections_with_titles[k]}" for k, v in section_map.items() if v]
    missing = [f"{k} {expected_sections_with_titles[k]}" for k, v in section_map.items() if not v]
    return present, missing

# Detect headings not in schema, using uppercase or 'PART' keywords
def extract_unexpected_headings(doc):
    unexpected = []
    for para in doc.paragraphs:
        line = para.text.strip()
        if not line:
            continue
        if re.match(r"PART[- ]?[A-Z0-9()]+", line.upper()) or line.isupper():
            matched = any(pattern.search(line) for pattern in section_patterns.values())
            if not matched:
                unexpected.append(line)
    return list(set(unexpected))

# Process one Word document
def process_file(file_path):
    try:
        doc = Document(file_path)
        if not doc.paragraphs or all(not p.text.strip() for p in doc.paragraphs):
            return "Blank or No Title", [], [], "Blank file.", []
        title = extract_title(doc)
        present, missing = analyze_structure(doc)
        summary = extract_file_summary(doc)
        unexpected = extract_unexpected_headings(doc)
        return title, present, missing, summary, unexpected
    except Exception as e:
        return f"Error: {str(e)}", [], [], f"Error occurred: {str(e)}", []

# Main logic to process all folders
def process_all_folders(base_path, output_excel_path):
    all_results = []

    for folder_name in os.listdir(base_path):
        folder_path = os.path.join(base_path, folder_name)
        if not os.path.isdir(folder_path):
            continue

        files = [f for f in os.listdir(folder_path) if (f.endswith(".docx") or f.endswith(".doc")) and not f.startswith("~$")]
        total_files = len(files)
        folder_display_name = folder_name

        for i, file in enumerate(files):
            file_path = os.path.join(folder_path, file)
            temp_converted = False

            if file.endswith(".doc") and not file.endswith(".docx"):
                try:
                    file_path = convert_doc_to_docx(file_path)
                    temp_converted = True
                except Exception as e:
                    print(f"Error converting {file}: {e}")
                    continue

            title, present, missing, summary, unexpected = process_file(file_path)

            all_results.append({
                "Folder Name": folder_display_name,
                "File Count in Folder": total_files if i == 0 else "",
                "File Name": file,
                "Title Name": title,
                "Sections Present": ", ".join(present) if present else "None",
                "Sections Missing": ", ".join(missing) if missing else "None",
                "Summary Content": summary,
                "Other Headings Present (not in schema)": ", ".join(unexpected) if unexpected else "None"
            })

            folder_display_name = ""

            if temp_converted:
                try:
                    os.remove(file_path)
                except Exception as e:
                    print(f"Could not delete temp file {file_path}: {e}")

    df = pd.DataFrame(all_results)
    pd.set_option('display.max_colwidth', None)
    os.makedirs(os.path.dirname(output_excel_path), exist_ok=True)
    df.to_excel(output_excel_path, index=False)
    print(f"✅ Final Excel saved at: {output_excel_path}")

# --- Update your input/output paths here ---
base_folder = r"D:\windows(c) users _mohan folders\Downloads\331 to 413"
output_file = os.path.join(os.path.dirname(__file__), "..", "output", "dk.xlsx")

process_all_folders(base_folder, output_file)


can you tell the main diffference between these option#1 and option#2 and pipeline structure,which is best method for useful for 2.5k files