import os
import re
import requests
from bs4 import BeautifulSoup
from PIL import Image
from io import BytesIO
from urllib.parse import urljoin
from tqdm import tqdm
import zipfile
import time
import random

# Constants
BASE_URL = "https://hdri-haven.com"
HDRI_URL_PATTERN = re.compile(r"https://hdri-haven\.com/hdri/[^/]+")

# Directories
DOWNLOAD_DIR = "downloads"
WEBP_DIR = "webp_images"

def create_directories():
    os.makedirs(DOWNLOAD_DIR, exist_ok=True)
    os.makedirs(WEBP_DIR, exist_ok=True)

def get_all_hdri_pages():
    """
    Scans the BASE_URL to find all HDRI page URLs matching the pattern.
    """
    response = requests.get(BASE_URL)
    response.raise_for_status()
    soup = BeautifulSoup(response.text, 'html.parser')
    
    links = set()
    for a_tag in soup.find_all('a', href=True):
        href = urljoin(BASE_URL, a_tag['href'])
        if HDRI_URL_PATTERN.match(href):
            links.add(href)
    return list(links)

def get_2k_image_link(hdri_page_url):
    """
    Scrapes the 2K image download link from an HDRI page.
    """
    response = requests.get(hdri_page_url)
    response.raise_for_status()
    soup = BeautifulSoup(response.text, 'html.parser')
    
    link_2k = soup.find('a', class_='downloadable link-2k', href=True)
    if link_2k:
        return link_2k['href']
    return None

def download_image(image_url):
    """
    Downloads an image from the given URL and saves it to the DOWNLOAD_DIR.
    Returns the file path of the downloaded image.
    """
    local_filename = os.path.join(DOWNLOAD_DIR, os.path.basename(image_url))
    with requests.get(image_url, stream=True) as r:
        r.raise_for_status()
        with open(local_filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=8192):
                f.write(chunk)
    return local_filename

def convert_to_webp(image_path):
    """
    Converts the given image to WebP format and saves it in the WEBP_DIR.
    """
    base_name = os.path.splitext(os.path.basename(image_path))[0]
    webp_path = os.path.join(WEBP_DIR, f"{base_name}.webp")
    
    with Image.open(image_path) as img:
        img.save(webp_path, 'WEBP')
    return webp_path

def zip_webp_images(zip_name="webp_images.zip"):
    """
    Creates a zip file containing all WebP images in the WEBP_DIR.
    """
    with zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_DEFLATED) as zipf:
        for root, _, files in os.walk(WEBP_DIR):
            for file in files:
                file_path = os.path.join(root, file)
                zipf.write(file_path, arcname=file)
    print(f"Zip file '{zip_name}' created successfully.")

def main():
    create_directories()
    
    print("Scanning website for HDRI pages...")
    hdri_pages = get_all_hdri_pages()
    print(f"Found {len(hdri_pages)} HDRI pages.")
    
    downloaded_images = []
    for page in tqdm(hdri_pages, desc="Processing HDRI pages"):
        image_link = get_2k_image_link(page)
        if image_link:
            try:
                image_path = download_image(image_link)
                downloaded_images.append(image_path)
                # Random delay between 2 to 3 seconds
                time.sleep(random.uniform(2, 3))
            except Exception as e:
                print(f"Failed to download {image_link}: {e}")
    
    print("Converting images to WebP format...")
    for image_path in tqdm(downloaded_images, desc="Converting images"):
        try:
            convert_to_webp(image_path)
        except Exception as e:
            print(f"Failed to convert {image_path}: {e}")
    
    print("Creating zip file of WebP images...")
    zip_webp_images()
    
    print("Process completed successfully.")

if __name__ == "__main__":
    main()

here this code is working perfectly. this is my website im downloading some images from here. its only downloading 48 images with this code. i need it to find more.

"https://hdri-haven.com/hdri/rusty-control-station"

all of the pages that has images structured same. no page number no anything. just "https://hdri-haven.com/hdri/" + "name for the image"

you need to find more. above code is working dont mess it up.