Spaces:
Sleeping
Sleeping
| import requests | |
| from bs4 import BeautifulSoup | |
| import os | |
| from urllib.parse import urljoin, urlparse | |
| import time | |
| import zipfile | |
| import tempfile | |
| import shutil | |
| import gradio as gr | |
| def extract_detail_page_links(url, headers): | |
| """ | |
| Extract all detail page links from the main listing page. | |
| Args: | |
| url: Main page URL | |
| headers: Request headers | |
| Returns: | |
| list of detail page URLs | |
| """ | |
| response = requests.get(url, headers=headers, timeout=30) | |
| response.raise_for_status() | |
| soup = BeautifulSoup(response.content, 'html.parser') | |
| detail_links = [] | |
| for link in soup.find_all('a', href=True): | |
| href = link['href'] | |
| # Look for detail page patterns (adjust pattern as needed) | |
| if 'Details.aspx' in href or 'PUB_ID=' in href: | |
| full_url = urljoin(url, href) | |
| if full_url not in detail_links: | |
| detail_links.append(full_url) | |
| return detail_links | |
| def extract_pdf_links_from_page(url, headers): | |
| """ | |
| Extract PDF links from a single page. | |
| Args: | |
| url: Page URL to scrape | |
| headers: Request headers | |
| Returns: | |
| list of PDF URLs | |
| """ | |
| try: | |
| response = requests.get(url, headers=headers, timeout=30) | |
| response.raise_for_status() | |
| soup = BeautifulSoup(response.content, 'html.parser') | |
| pdf_links = [] | |
| for link in soup.find_all('a', href=True): | |
| href = link['href'] | |
| if '.pdf' in href.lower(): | |
| full_url = urljoin(url, href) | |
| if full_url not in pdf_links: | |
| pdf_links.append(full_url) | |
| return pdf_links | |
| except Exception as e: | |
| print(f"Error extracting PDFs from {url}: {str(e)}") | |
| return [] | |
| def download_pdfs_from_page(url, progress=gr.Progress()): | |
| """ | |
| Download all PDFs from a webpage by navigating through detail pages. | |
| Args: | |
| url: The main webpage URL to scrape | |
| progress: Gradio progress tracker | |
| Returns: | |
| tuple of (zip_file_path, summary_message) | |
| """ | |
| headers = { | |
| 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' | |
| } | |
| try: | |
| # Step 1: Extract detail page links from main page | |
| progress(0, desc="Fetching main page and extracting detail links...") | |
| detail_page_links = extract_detail_page_links(url, headers) | |
| if len(detail_page_links) == 0: | |
| return None, "❌ No detail page links found on the main page." | |
| progress(0.1, desc=f"Found {len(detail_page_links)} detail pages to process") | |
| # Step 2: Visit each detail page and collect PDF links | |
| all_pdf_links = [] | |
| for idx, detail_url in enumerate(detail_page_links, 1): | |
| progress(0.1 + (0.3 * idx / len(detail_page_links)), | |
| desc=f"[{idx}/{len(detail_page_links)}] Scanning detail page...") | |
| pdf_links = extract_pdf_links_from_page(detail_url, headers) | |
| all_pdf_links.extend(pdf_links) | |
| # Be polite - small delay between page requests | |
| time.sleep(0.5) | |
| # Remove duplicates | |
| all_pdf_links = list(set(all_pdf_links)) | |
| if len(all_pdf_links) == 0: | |
| return None, f"❌ No PDF links found across {len(detail_page_links)} detail pages." | |
| progress(0.4, desc=f"Found {len(all_pdf_links)} unique PDFs to download") | |
| # Step 3: Create temporary directory for downloads | |
| temp_dir = tempfile.mkdtemp() | |
| # Step 4: Download each PDF | |
| successful = 0 | |
| failed = 0 | |
| failed_urls = [] | |
| for idx, pdf_url in enumerate(all_pdf_links, 1): | |
| try: | |
| parsed_url = urlparse(pdf_url) | |
| path_without_query = parsed_url.path | |
| filename = os.path.basename(path_without_query) | |
| # Handle empty filenames | |
| if not filename or filename == '': | |
| filename = f"document_{idx}.pdf" | |
| filepath = os.path.join(temp_dir, filename) | |
| # Skip if file already exists | |
| if os.path.exists(filepath): | |
| progress(0.4 + (0.5 * idx / len(all_pdf_links)), | |
| desc=f"[{idx}/{len(all_pdf_links)}] Skipping: {filename}") | |
| successful += 1 | |
| continue | |
| progress(0.4 + (0.5 * idx / len(all_pdf_links)), | |
| desc=f"[{idx}/{len(all_pdf_links)}] Downloading: {filename}") | |
| # Download PDF | |
| pdf_response = requests.get(pdf_url, headers=headers, timeout=60) | |
| pdf_response.raise_for_status() | |
| # Verify it's actually a PDF | |
| if pdf_response.headers.get('content-type', '').lower() not in ['application/pdf', 'application/octet-stream']: | |
| failed += 1 | |
| failed_urls.append(f"{filename}: Not a valid PDF file") | |
| continue | |
| # Save PDF | |
| with open(filepath, 'wb') as f: | |
| f.write(pdf_response.content) | |
| successful += 1 | |
| time.sleep(1) # Be polite | |
| except Exception as e: | |
| failed += 1 | |
| failed_urls.append(f"{filename}: {str(e)}") | |
| continue | |
| # Step 5: Generate summary | |
| summary = f""" | |
| ✅ **Download Complete!** | |
| 📊 **Summary:** | |
| - Detail pages scanned: {len(detail_page_links)} | |
| - Total PDFs found: {len(all_pdf_links)} | |
| - Successfully downloaded: {successful} | |
| - Failed: {failed} | |
| """ | |
| if failed > 0: | |
| summary += f"\n\n⚠️ **Failed Downloads:**\n" | |
| for fail in failed_urls[:10]: | |
| summary += f"- {fail}\n" | |
| if len(failed_urls) > 10: | |
| summary += f"- ... and {len(failed_urls) - 10} more\n" | |
| # Step 6: Create zip file | |
| progress(0.9, desc="Creating zip file...") | |
| zip_path = os.path.join(tempfile.gettempdir(), f"pdfs_{int(time.time())}.zip") | |
| with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: | |
| for root, dirs, files in os.walk(temp_dir): | |
| for file in files: | |
| file_path = os.path.join(root, file) | |
| zipf.write(file_path, arcname=file) | |
| # Clean up | |
| shutil.rmtree(temp_dir) | |
| progress(1.0, desc="Complete!") | |
| return zip_path, summary | |
| except requests.exceptions.RequestException as e: | |
| return None, f"❌ Error fetching webpage: {str(e)}" | |
| except Exception as e: | |
| return None, f"❌ Unexpected error: {str(e)}" | |
| # Create Gradio interface | |
| def create_interface(): | |
| with gr.Blocks(title="PDF Downloader", theme=gr.themes.Soft()) as demo: | |
| gr.Markdown( | |
| """ | |
| # 📥 Two-Level PDF Downloader | |
| Download all PDFs from webpages with intermediate detail pages! | |
| **Instructions:** | |
| 1. Enter the URL of the main listing page | |
| 2. Click "Download PDFs" | |
| 3. The tool will navigate through all detail pages | |
| 4. Download your ZIP file with all PDFs | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| url_input = gr.Textbox( | |
| label="Main Page URL", | |
| placeholder="https://armypubs.army.mil/ProductMaps/PubForm/AR.aspx", | |
| lines=1 | |
| ) | |
| download_btn = gr.Button("📥 Download PDFs", variant="primary", size="lg") | |
| with gr.Row(): | |
| with gr.Column(): | |
| output_file = gr.File(label="Download ZIP") | |
| summary_output = gr.Markdown(label="Summary") | |
| download_btn.click( | |
| fn=download_pdfs_from_page, | |
| inputs=[url_input], | |
| outputs=[output_file, summary_output] | |
| ) | |
| gr.Markdown( | |
| """ | |
| --- | |
| ### 💡 Features: | |
| - **Two-level navigation**: Scans main page → visits detail pages → downloads PDFs | |
| - **Duplicate removal**: Ensures each PDF is downloaded only once | |
| - **Polite scraping**: Includes delays between requests | |
| - **Error handling**: Continues even if some downloads fail | |
| - **Progress tracking**: Real-time updates on scanning and downloading | |
| """ | |
| ) | |
| return demo | |
| if __name__ == "__main__": | |
| demo = create_interface() | |
| demo.launch(share=True) |