data"].get("completion_tokens", 0)
 token_totals["total_tokens"] += rec["metadata"].get("total_tokens", 0)

 await progress.mark_section_done(
 section["chunk_id"], len(records), section["law_name"]
 )

 sections_since_save += 1
 if sections_since_save >= SAVE_INTERVAL:
 await progress.save()
 sections_since_save = 0
 pbar.update(1)

 pbar.close()

 progress.data["status"] = "completed"
 await progress.save()

 # ── Summary ──
 logging.info("=" 65)
 logging.info(f"✅ Total QA pairs (all runs) : {progress.total_pairs}")
 logging.info(f" This run : {total_pairs_this_run}")
 logging.info(
 f"🔢 Token usage (this run) : input={token_totals['prompt_tokens']} "
 f"output={token_totals['completion_tokens']} total={token_totals['total_tokens']}"
 )
 logging.info(f"❌ Failed sections : {len(failed_sections)}")
 logging.info(f"📄 Output : {out_file}")
 logging.info("")
 logging.info("Question type distribution (this run):")
 for qt, count in sorted(question_type_counts.items(), key=lambda x: -x[1]):
 logging.info(f" {qt:<15}: {count}")
 logging.info("")
 logging.info("Pairs per law:")
 for law, count in sorted(progress.data["pairs_per_law"].items(), key=lambda x: -x[1]):
 logging.info(f" {law}: {count}")
 logging.info("=" 65)

 summary = {
 "total_qa_pairs": progress.total_pairs,
 "this_run_pairs": total_pairs_this_run,
 "failed_sections": len(failed_sections),
 "question_type_distribution": dict(question_type_counts),
 "token_usage_this_run": token_totals,
 "total_sections_processed": len(progress.data["completed_section_ids"]),
 "total_sections_in_corpus": len(all_sections),
 "pairs_per_law": progress.data["pairs_per_law"],
 "pipeline_version": "v5",
 "config": {k: v for k, v in asdict(cfg).items() if k != "law_name_map"},
 "generated_at": datetime.now(timezone.utc).isoformat(),
 }
 async with aiofiles.open(output_path / "generation_summary.json", "w", encoding="utf-8") as sf:
 await sf.write(json.dumps(summary, indent=2, ensure_ascii=False))

 return progress.total_pairs

──────────────────────────────────────────────────────────────────────
CLI
──────────────────────────────────────────────────────────────────────

def parse_args():
 parser = argparse.ArgumentParser(description="Nepal Legal QA Generator V5 — Strict QA format")
 parser.add_argument("--input_dir", default="./laws")
 parser.add_argument("--output_dir", default="./output_v5")
 parser.add_argument("--model ", default="gpt-4.1-mini-2025-04-14")
 parser.add_argument("--qa_per_section", type=int, default=3)
 parser.add_argument("--concurrency", type=int, default=20)
 parser.add_argument("--temperature", type=float, default=0.75)
 parser.add_argument("--reset", action="store_true")
 return parser.parse_args()

if __name__ == "__main__":
 args = parse_args()
 cfg = PipelineConfig(
 input_dir=args.input_dir,
 output_dir=args.output_dir,
 model=args.model,
 qa_pairs_per_section=args.qa_per_section,
 max_concurrent_requests=args.concurrency,
 temperature=args.temperature,
 )
 if args.reset:
 p = Path(cfg.output_dir) / cfg.progress_file
 if p.exists():
 p.unlink()
 print(f"Reset: deleted {p}")

 import sys
 if sys.platform == "win32":
 asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncio.run(run_pipeline(cfg))


We will use the deepseek-v4-flash model with ate-limited to 10 requests per minute (RPM) and 100,000 tokens per minute (TPM). and content limit 1M input & 8K output.