", 0), len(records))
 completion_per_pair = _distribute_tokens(usage.get("completion_tokens", 0), len(records))

 final_records = []
 for idx, r in enumerate(records, start=1):
 rec = build_finetune_record(
 question=r["question"],
 answer=r["answer"],
 section=section,
 has_citation=r["has_citation"],
 question_type=r["question_type"],
 prompt_tokens=prompt_per_pair[idx - 1],
 completion_tokens=completion_per_pair[idx - 1],
 cfg=cfg,
 )
 final_records.append(rec)

 logging.info(
 "TOKENS %s QA-%d input=%d output=%d total=%d",
 section["chunk_id"],
 idx,
 rec["metadata"]["prompt_tokens"],
 rec["metadata"]["completion_tokens"],
 rec["metadata"]["total_tokens"],
 )

 return final_records, None

──────────────────────────────────────────────────────────────────────
File Loader — unchanged from V3
──────────────────────────────────────────────────────────────────────

def resolve_law_name(filename, cfg):
 stem = Path(filename).stem
 for key, canonical in cfg.law_name_map.items():
 if key.lower() in stem.lower() or stem.lower() in key.lower():
 return canonical
 return stem

async def load_law_files(cfg):
 input_path = Path(cfg.input_dir)
 files = list(input_path.glob(".txt")) + list(in put_path.glob(".md"))

 if not files:
 logging.warning(f"No .txt/.md files found in {cfg.input_dir}")
 return []

 all_sections = []
 for fpath in sorted(files):
 async with aiofiles.open(fpath, encoding="utf-8", errors="replace") as f:
 text = await f.read()
 law_name = resolve_law_name(fpath.name, cfg)
 sections = chunk_legal_text(text, law_name, fpath.name, cfg)
 all_sections.extend(sections)
 n_preamble = sum(1 for s in sections if s["chunk_type"] == "preamble")
 n_section = sum(1 for s in sections if s["chunk_type"] == "section")
 logging.info(
 f"Loaded '{law_name}' → {len(sections)} chunks "
 f"({n_section} sections"
 f"{f', {n_preamble} preamble' if n_preamble else ''})"
 )
 return all_sections

──────────────────────────────────────────────────────────────────────
Main Pipeline — unchanged from V3
──────────────────────────────────────────────────────────────────────

SAVE_INTERVAL = 10

async def run_pipeline(cfg: PipelineConfig):
 logging.basicConfig(
 level=logging.INFO,
 format="%(asctime)s [%(levelname)s] %(message)s",
 datefmt="%H:%M:%S",
 )

 output_path = Path(cfg.output_dir)
 output_path.mkdir(parents=True, exist_ok=True)
 out_file = output_path / cfg.output_file fail_file = output_path / cfg.failed_file
 progress_path = output_path / cfg.progress_file

 api_key = os.environ.get("OPENAI_API_KEY")
 if not api_key:
 raise EnvironmentError("OPENAI_API_KEY environment variable not set.")

 client = AsyncOpenAI(api_key=api_key)
 semaphore = asyncio.Semaphore(cfg.max_concurrent_requests)

 progress = ProgressTracker(progress_path)
 resumed = progress.load()
 if not resumed:
 progress.data["started_at"] = datetime.now(timezone.utc).isoformat()
 progress.data["status"] = "in_progress"
 await progress.save()

 logging.info("Loading law files...")
 all_sections = await load_law_files(cfg)
 logging.info(f"Total sections: {len(all_sections)}")

 pending = [s for s in all_sections if not progress.is_section_done(s["chunk_id"])]
 logging.info(f"Pending: {len(pending)} (skipping {len(all_sections) - len(pending)} done)")

 if not pending:
 logging.info("All sections already processed.")
 progress.data["status"] = "completed"
 await progress.save()
 return progress.total_pairs

 mode = "a" if resumed else "w"
 total_pairs_this_run = 0
 failed_sections = []
 sections_since_save = 0

 # ✅ V5 — track question type distribution for quality reporting
 question _type_counts = defaultdict(int)
 token_totals = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}

 async with aiofiles.open(out_file, mode, encoding="utf-8") as out_f:
 async with aiofiles.open(fail_file, mode, encoding="utf-8") as fail_f:

 async def process_section(section):
 return await generate_qa_for_section(client, section, cfg, semaphore), section

 coros = [process_section(s) for s in pending]
 pending_futures = [asyncio.ensure_future(c) for c in coros]
 pbar = atqdm(total=len(pending_futures), desc="Generating QA pairs", unit="section")

 for coro in asyncio.as_completed(pending_futures):
 try:
 (records, failed), section = await coro
 except asyncio.CancelledError:
 break

 if failed:
 await fail_f.write(json.dumps(failed, ensure_ascii=False) + "\n")
 await progress.mark_section_failed(section["chunk_id"])
 failed_sections.append(failed)
 else:
 for rec in records:
 await out_f.write(json.dumps(rec, ensure_ascii=False) + "\n")
 total_pairs_this_run += 1
 qt = rec["metadata"].get("question_type", "unknown")
 question_type_counts[qt] += 1
 token_totals["prompt_tokens"] += rec["metadata"].get("prompt_tokens", 0)
 token_totals["completion_tokens"] += rec["meta