# url = input("Enter Restro Url : ") from urllib.parse import urlparse import pymongo from config import settings def main(url,location,inside_location): client = pymongo.MongoClient(settings.MONGO_URL) def restro_details(url,location_area,location_name): db = client.Restaurants_in_dubai collection = db.Restaurant_details parsed_url = urlparse(url) platform_name = parsed_url.netloc.split('.')[1].capitalize() restaurant_name = parsed_url.path.strip('/').split('/')[-1] restaurant_details = { 'url' : url, 'platform_name' : platform_name, "restaurant_name" : restaurant_name, "location_area" : location_area , "location_name" : location_name } result = collection.insert_one(restaurant_details) print("Inserted document IDs:", result.inserted_id) def main_excel_file(url_input): import requests import json import pandas as pd from bs4 import BeautifulSoup from urllib.parse import urlparse import math payload1 = {'restId': '17902'} files = [] headers2 = { 'Content-Type': 'application/x-www-form-urlencoded', 'Cookie': 'eateasy-ae-website=lai3mvcb9hd99nnivbt0pn68ibfjsd6g' } headers = { 'Content-Type': 'application/x-www-form-urlencoded', } parsed_url = urlparse(url_input) restaurant_code = parsed_url.path.strip('/').split('/')[-1] url = "https://www.eateasy.ae/dubai/food/getFilteredMenus" data = { "restCode": restaurant_code } response = requests.post(url, data=data) menu_items_list = [] category_name_list = [] j = 0 if response.status_code == 200: soup = BeautifulSoup(response.text, 'html.parser') menu_item_boxes = soup.find_all('div', class_='menu-item-box') # Find all divs with class 'menu-item-box' for i, item_box in enumerate(menu_item_boxes): # Iterate over each menu item box img = item_box.find('img') # Find the img tag within the div if img: # Check if img tag exists image_url = img.get('data-image') # Get the value of 'data-image' attribute else: image_url = None # If image URL is not found menu_item_details = item_box.find('div', class_='menu-item-details') # Find menu-item-details div if menu_item_details: type_ = menu_item_details.find('p', class_='type').text.strip() name = menu_item_details.find('h5', class_='menu-food-title').text.strip() description = menu_item_details.find('p', itemprop='description').text.strip() price = menu_item_details.find('div', class_='menu-item-price').text.strip() else: # If menu-item-details not found, set defaults type_ = "" name = "" description = "" price = "" menu_list_options = item_box.find('a', class_='menu-list-options') # Find the menu-list-options anchor tag if menu_list_options: value = menu_list_options.get('value') # Get the value attribute else: value = None if type_ not in category_name_list: category_name_list.append(type_) j = j+1 Category_position = j else: Category_position = j menu_item = { "Category": type_, "Category_position": Category_position, "Item_name": name, "Item_position": i, "Image": image_url, "description": description, "price": price, "id": value, } menu_items_list.append(menu_item) # Append menu item before the request if value is not None: option_url = f"https://www.eateasy.ae/dubai/order/add_to_cart_v1/{value}/1/" option_response = requests.post(option_url, headers=headers2, data=payload1) if option_response.status_code == 200: try: json_data = json.loads(option_response.text) extracted_data = [] if 'arrResult' in json_data and 'arrFoodChoice' in json_data['arrResult']: for choice in json_data['arrResult']['arrFoodChoice']: extracted_data.append({ 'Option_group_name': choice['choice_name'], 'Option_name': choice['name'], 'Extra_price': choice['price'], 'Min': choice.get('mandatory', 0), 'Max': choice.get('max_choice', 1) }) grouped_data = {} for choice in extracted_data: group_name = choice['Option_group_name'] if group_name not in grouped_data: grouped_data[group_name] = { 'Option_group_name': group_name, 'Min': choice['Min'], 'Max': max(choice['Max'], '1'), } num_options = sum(key.startswith('Option ') for key in grouped_data[group_name]) option_index = num_options + 1 # Index for the new option grouped_data[group_name][f"Option {option_index} Name"] = choice['Option_name'] grouped_data[group_name][f"Option {option_index} Price"] = choice['Extra_price'] for group_data in grouped_data.values(): menu_items_list.append(group_data) except json.JSONDecodeError: print("JSON decoding error. Response content may not be in valid JSON format.") else: print(f"Failed to get data for item with value {value}. Status code: {option_response.status_code}") df = pd.DataFrame(menu_items_list) # Ensure 'Max' column exists before renaming or shifting if 'Max' in df.columns: max_column_index = df.columns.get_loc('Max') for i in range(max_column_index + 1, len(df.columns)): df.rename(columns={df.columns[i]: ''}, inplace=True) option_group_name_index = df.columns.get_loc('Option_group_name') for i in range(option_group_name_index, len(df.columns)): df.iloc[:, i] = df.iloc[:, i].shift(-1) excel_file = f"{restaurant_code}_menu.xlsx" df.to_excel(excel_file, index=False) print(f"Menu items saved to {excel_file}") else: print(f"Failed to get menu items. Status code: {response.status_code}") def main_json(url_input): import requests import json from bs4 import BeautifulSoup from urllib.parse import urlparse payload1 = {'restId': '17902'} files = [] headers2 = { 'Content-Type': 'application/x-www-form-urlencoded', 'Cookie': 'eateasy-ae-website=lai3mvcb9hd99nnivbt0pn68ibfjsd6g' } headers = { 'Content-Type': 'application/x-www-form-urlencoded', } parsed_url = urlparse(url_input) restaurant_code = parsed_url.path.strip('/').split('/')[-1] url = "https://www.eateasy.ae/dubai/food/getFilteredMenus" data = { "restCode": restaurant_code } response = requests.post(url_input, data=data) menu_items_list = [] category_name_list = [] j = 0 if response.status_code == 200: soup = BeautifulSoup(response.text, 'html.parser') menu_item_boxes = soup.find_all('div', class_='menu-item-box') # Find all divs with class 'menu-item-box' location_area = soup.find('div', class_='location-area').text location_name = soup.find('div', class_='location').text for i,item_box in enumerate(menu_item_boxes): # Iterate over each menu item box img = item_box.find('img') # Find the img tag within the div if img: # Check if img tag exists image_url = img.get('data-image') # Get the value of 'data-image' attribute else: image_url = None # If image URL is not found menu_item_details = item_box.find('div', class_='menu-item-details') # Find menu-item-details div if menu_item_details: category = menu_item_details.find('p', class_='type').text.strip() name = menu_item_details.find('h5', class_='menu-food-title').text.strip() description = menu_item_details.find('p', itemprop='description').text.strip() price = menu_item_details.find('div', class_='menu-item-price').text.strip() else: # If menu-item-details not found, set defaults category = "" name = "" description = "" price = "" menu_list_options = item_box.find('a', class_='menu-list-options') # Find the menu-list-options anchor tag if menu_list_options: item_id = menu_list_options.get('value') # Get the value attribute else: item_id = None if category not in category_name_list: category_name_list.append(category) j = j+1 Category_position = j else: Category_position = j menu_item = { "Category": category, "Category_position": Category_position, "Item_name": name, "Item_position": i, "Image": image_url, "Description": description, "Price": price, "ID": item_id, "Option_groups": [] } if item_id is not None: url1 = f"https://www.eateasy.ae/dubai/order/add_to_cart_v1/{item_id}/1/" response = requests.request("POST", url1, headers=headers2, data=payload1, files=files) if response.status_code == 200: try: json_data = response.json() option_group_data = {} if 'arrResult' in json_data and 'arrFoodChoice' in json_data['arrResult']: for option_group in json_data['arrResult']['arrFoodChoice']: group_name = option_group['choice_name'] min_quantity = option_group['mandatory'] max_quantity = max((option_group['max_choice']), "1") option_group_item = next((x for x in menu_item['Option_groups'] if x['Option_group_name'] == group_name), None) if option_group_item: option_group_item['Option_group_names'].append({ "Option_name": option_group['name'], "Option_price": option_group['price'] }) else: menu_item['Option_groups'].append({ "Option_group_name": group_name, "Min": min_quantity, "Max": max_quantity, "Option_group_names": [{ "Option_name": option_group['name'], "Option_price": option_group['price'] }] }) except json.JSONDecodeError: print("JSON decoding error. Response content may not be in valid JSON format.") else: print(f"Failed to get data for item with value {item_id}. Status code: {response.status_code}") if not menu_item['Option_groups']: menu_item["Has_choice"] = False else: menu_item["Has_choice"] = True menu_items_list.append(menu_item) else: print(f"Failed to get menu items. Status code: {response.status_code}") # Convert the list of menu items into JSON # with open(f'{restaurant_code}.json', 'w') as json_file: # json.dump(menu_items_list, json_file, indent=4) print(f"Menu items saved to {restaurant_code}.json file.") # main_excel_file(url_input) return menu_items_list,location_area,location_name def extract_category(items,url): db = client.Restaurants_in_dubai collection = db.Category json_data = items restro_collection = db['Restaurant_details'] def item_extract_category(json,name): items_list = [] for item in json: if item['Category'] == name: item_info = { "id": item["ID"], "name": item["Item_name"] } items_list.append(item_info) return items_list categories_info = [] existing_categories = set() for item in json_data: name = item['Category'] if name not in existing_categories: # category_position = 1 # category_isActive = True items = item_extract_category(json_data, name) restro = restro_collection.find_one({'url': url}) if restro: restro_id = restro['_id'] restro_ref_id = restro_id category_info = { 'category_name': name, 'items': items, 'restro_ref_id' : restro_ref_id } categories_info.append(category_info) existing_categories.add(name) result = collection.insert_many(categories_info) print("Inserted document IDs:", result.inserted_ids) def extract_item(items,url): db = client.Restaurants_in_dubai collection = db.Items json_data = items category_collection = db['Category'] restro_collection = db['Restaurant_details'] items_info = [] for item in json_data: id = item['ID'] name = item['Item_name'] description = item['Description'] price = item['Price'] img_url= item['Image'] category_name = item['Category'] item_position = item['Item_position'] has_choices = item['Has_choice'] if has_choices == True: modifires = [] for option_group in item.get('Option_groups', []): modifires.append(option_group['Option_group_name']) else: modifires = "None" restro = restro_collection.find_one({'url': url}) if restro: restro_id = restro['_id'] restro_ref_id = restro_id category = category_collection.find_one( { 'category_name': category_name, 'restro_ref_id': restro_ref_id }) if category: category_id = category['_id'] ref_id = category_id item_info = { 'item_id': id, 'name': name, 'description': description, 'amount': price, 'image': img_url, 'category_name':category_name, 'modifires':modifires, 'ref_id_category' : ref_id, 'restro_ref_id' : restro_ref_id } items_info.append(item_info) result = collection.insert_many(items_info) print("Inserted document IDs:", result.inserted_ids) def extract_option_group(items,url): db = client.Restaurants_in_dubai collection = db.OptionGroup option_group_info = [] existing_categories = [] option_group_names_list = [] restro_collection = db['Restaurant_details'] for item in items: for option_group in item.get('Option_groups', []): flag = 1 restro = restro_collection.find_one({'url': url}) if restro: restro_id = restro['_id'] restro_ref_id = restro_id option_group_name = option_group["Option_group_name"] min_quantity = option_group["Min"] max_quantity = option_group["Max"] option_names = [] option_names_trial_all = [] for option in option_group.get("Option_group_names", []): option_name = option["Option_name"] option_price = option["Option_price"] option_names.append(option_name) option_names_trial = { 'Option_name': option_name, 'Option_price': option_price } option_names_trial_all.append(option_names_trial) item_id = [] for item in items: for option_group in item.get('Option_groups', []): option_group_name2 = option_group["Option_group_name"] option_group_names1 = option_group.get('Option_group_names',[]) if(option_group_name2 == option_group_name and (sorted(option_group_names1, key=lambda x: x['Option_name'])) == sorted(option_names_trial_all, key=lambda x: x['Option_name']) ): item_id.append(item['ID']) option_group_information = { "option_group_name" : option_group_name, "min_quantity" : min_quantity, "max_quantity" : max_quantity, "option_names" : option_names, "item_id" : item_id, "restro_ref_id" : restro_ref_id } option_group_check ={ "option_group_name" : option_group_name, "option_names" : option_names } for category in existing_categories: if (category['option_group_name'] == option_group_check['option_group_name'] and sorted(category['option_names']) == sorted(option_group_check['option_names'])): flag = 0 if flag==1: option_group_info.append(option_group_information) existing_categories.append(option_group_check) result = collection.insert_many(option_group_info) print("Inserted document IDs:", result.inserted_ids) def extract_option_group_names(items,url): db = client.Restaurants_in_dubai collection = db.OptionName option_group_collection = db['OptionGroup'] json_data = items option_names = [] option_names_list = [] restro_collection = db['Restaurant_details'] for item in json_data: has_choices = item['Has_choice'] if has_choices == True: flag = 1 for option_group in item.get('Option_groups', []): option_group_name2 = option_group["Option_group_name"] for option in option_group.get("Option_group_names", []): restro = restro_collection.find_one({'url': url}) if restro: restro_id = restro['_id'] restro_ref_id = restro_id option_name = option["Option_name"] option_price = option["Option_price"] query = { 'restro_ref_id': restro_id, 'option_names': option_name } matching_documents = option_group_collection.find(query) matching_ids = [doc['_id'] for doc in matching_documents] # if option_group_name_true: # option_group_id = option_group_name_true['_id'] # restro = restro_collection.find_one({'url': url}) option_group_name = { "option_name" : option_name, "option_price" : option_price, "ref_option_group_id" : matching_ids, "restro_ref_id" : restro_ref_id } if (option_name in option_names_list ): flag = 0 if flag==1: option_names.append(option_group_name) option_names_list.append(option_name) result = collection.insert_many(option_names) print("Inserted document IDs:", result.inserted_ids) items,location_area,location_name = main_json(url) restro_details(url,location,inside_location) extract_category(items,url) extract_item(items,url) extract_option_group(items,url) extract_option_group_names(items,url) return True # main(url)