Modify this to save the data after every loop initially only, so that if the software breaks somewhere in between then still the extracted data is saved. And also for a particular search its just got stuck in a loop, and its incrementing page number and getting the same data that was on initial page url, so find the total page numbers in the start only if possible, or some other solution to not get stuck in such a loop - 
import requests
from bs4 import BeautifulSoup
import csv
import time

def scrape_practo_psychologists(cities, designations):
    headers = {
        "User-Agent": (
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
            "AppleWebKit/537.36 (KHTML, like Gecko) "
            "Chrome/58.0.3029.110 Safari/537.3"
        )
    }

    all_doctors = []

    for city in cities:
        for designation in designations:
            base_url = f"https://www.practo.com/{city}/{designation}"
            page = 1  # Start from the first page

            while True:
                # Construct the URL for the current page
                if page == 1:
                    url = base_url
                else:
                    url = f"{base_url}?page={page}"

                print(f"Scraping Page {page}: {url}")
                try:
                    response = requests.get(url, headers=headers, timeout=10)
                    response.raise_for_status()
                except requests.exceptions.RequestException as e:
                    print(f"Request failed for page {page}: {e}")
                    break

                soup = BeautifulSoup(response.content, "html.parser")

                # Find all doctor cards on the current page
                doctor_divs = soup.find_all("div", class_="u-border-general--bottom")

                # If no doctor cards are found, assume we've reached the last page
                if not doctor_divs:
                    print(f"No doctor entries found on page {page}. Ending scraping for {designation} in {city}.")
                    break

                # Iterate through each doctor card and extract details
                for div in doctor_divs:
                    # Extract doctor's name
                    name_tag = div.find("h2", attrs={"data-qa-id": "doctor_name"})
                    name = name_tag.get_text(strip=True) if name_tag else None

                    # Extract address
                    locality_tag = div.find("span", attrs={"data-qa-id": "practice_locality"})
                    city_tag = div.find("span", attrs={"data-qa-id": "practice_city"})
                    locality = locality_tag.get_text(strip=True).rstrip(',') if locality_tag else ""
                    city_name = city_tag.get_text(strip=True) if city_tag else ""
                    address = f"{locality}, {city_name}" if locality and city_name else None

                    # Extract consultation fee
                    fee_tag = div.find("span", attrs={"data-qa-id": "consultation_fee"})
                    fee = fee_tag.get_text(strip=True).replace('₹', '').replace(',', '') if fee_tag else None

                    # Check for mandatory fields
                    if name and address and fee:
                        try:
                            fee = int(fee)  # Convert fee to integer for consistency
                        except ValueError:
                            fee = None

                    # Only append if all required fields are present and valid
                    if name and address and fee:
                        doctor_info = {
                            "Name": name,
                            "Address": address,
                            "Consultation Fee": fee,
                            "City": city_name,
                            "Designation": designation
                        }
                        all_doctors.append(doctor_info)
                        print(f"Scraped Doctor {len(all_doctors)}: {name}, {address}, ₹{fee}")
                    else:
                        print(f"Skipped a doctor entry due to missing information.")

                # Move to the next page
                page += 1
                time.sleep(1)  # Delay to be polite and avoid overloading the server

    # Define CSV file name
    csv_file = "practo_doctors.csv"

    # Define CSV headers
    fieldnames = ["Name", "Address", "Consultation Fee", "City", "Designation"]

    try:
        with open(csv_file, "w", newline='', encoding="utf-8") as file:
            writer = csv.DictWriter(file, fieldnames=fieldnames)
            writer.writeheader()
            writer.writerows(all_doctors)
        print(f"\nSuccessfully saved {len(all_doctors)} records to {csv_file}")
    except Exception as e:
        print(f"Error writing to CSV file: {e}")


if __name__ == "__main__":
    cities = ["mumbai", "bangalore", "delhi", "chennai", "hyderabad", "kolkata", "gurgaon", "gurugram", "chandigadh", "mohali"]
    designations = ["psychiatrist", "psychologist", "neurologist"]
    scrape_practo_psychologists(cities, designations)