from config import settings import pymongo from urllib.parse import urlparse def EatEasy_Mongo_Data_add(url): client = pymongo.MongoClient(settings.MONGO_URL) def restro_details(url,location_area,location_name): db = client.Restaurants_in_dubai collection = db.Restaurant_details parsed_url = urlparse(url) platform_name = parsed_url.netloc.split('.')[1].capitalize() restaurant_name = parsed_url.path.strip('/').split('/')[-1] restaurant_details = { 'url' : url, 'platform_name' : platform_name, "restaurant_name" : restaurant_name.strip(), "location_area" : location_area , "location_name" : location_name } result = collection.insert_one(restaurant_details) print("Inserted document IDs:", result.inserted_id) def main_json(url_input): import requests import json from bs4 import BeautifulSoup from urllib.parse import urlparse payload1 = {'restId': '17902'} files = [] headers2 = { 'Content-Type': 'application/x-www-form-urlencoded', 'Cookie': 'eateasy-ae-website=lai3mvcb9hd99nnivbt0pn68ibfjsd6g' } headers = { 'Content-Type': 'application/x-www-form-urlencoded', } parsed_url = urlparse(url_input) restaurant_code = parsed_url.path.strip('/').split('/')[-1] url = "https://www.eateasy.ae/dubai/food/getFilteredMenus" data = { "restCode": restaurant_code } response = requests.post(url_input, data=data) menu_items_list = [] category_name_list = [] j = 0 if response.status_code == 200: soup = BeautifulSoup(response.text, 'html.parser') menu_item_boxes = soup.find_all('div', class_='menu-item-box') # Find all divs with class 'menu-item-box' location_area = soup.find('div', class_='location-area').text input_element = soup.find('input', {'id': 'menuDeliveryLocationNewModalInput'}) # Get the value of the value attribute location_name = input_element.get('value') # location_name = soup.find('div', class_='location').text for i,item_box in enumerate(menu_item_boxes): # Iterate over each menu item box img = item_box.find('img') # Find the img tag within the div if img: # Check if img tag exists image_url = img.get('data-image') # Get the value of 'data-image' attribute else: image_url = None # If image URL is not found menu_item_details = item_box.find('div', class_='menu-item-details') # Find menu-item-details div if menu_item_details: category = menu_item_details.find('p', class_='type').text.strip() name = menu_item_details.find('h5', class_='menu-food-title').text.strip() description = menu_item_details.find('p', itemprop='description').text.strip() price = menu_item_details.find('div', class_='menu-item-price').text.strip() else: # If menu-item-details not found, set defaults category = "" name = "" description = "" price = "" menu_list_options = item_box.find('a', class_='menu-list-options') # Find the menu-list-options anchor tag if menu_list_options: item_id = menu_list_options.get('value') # Get the value attribute else: item_id = None if category not in category_name_list: category_name_list.append(category) j = j+1 Category_position = j else: Category_position = j menu_item = { "Category": category, "Category_position": Category_position, "Item_name": name, "Item_position": i, "Image": image_url, "Description": description, "Price": price, "ID": item_id, "Option_groups": [] } if item_id is not None: url1 = f"https://www.eateasy.ae/dubai/order/add_to_cart_v1/{item_id}/1/" response = requests.request("POST", url1, headers=headers2, data=payload1, files=files) if response.status_code == 200: try: json_data = response.json() option_group_data = {} if 'arrResult' in json_data and 'arrFoodChoice' in json_data['arrResult']: for option_group in json_data['arrResult']['arrFoodChoice']: group_name = option_group['choice_name'] min_quantity = option_group['mandatory'] max_quantity = max((option_group['max_choice']), "1") option_group_item = next((x for x in menu_item['Option_groups'] if x['Option_group_name'] == group_name), None) if option_group_item: option_group_item['Option_group_names'].append({ "Option_name": option_group['name'], "Option_price": option_group['price'] }) else: menu_item['Option_groups'].append({ "Option_group_name": group_name, "Min": min_quantity, "Max": max_quantity, "Option_group_names": [{ "Option_name": option_group['name'], "Option_price": option_group['price'] }] }) except json.JSONDecodeError: print("JSON decoding error. Response content may not be in valid JSON format.") else: print(f"Failed to get data for item with value {item_id}. Status code: {response.status_code}") if not menu_item['Option_groups']: menu_item["Has_choice"] = False else: menu_item["Has_choice"] = True menu_items_list.append(menu_item) else: print(f"Failed to get menu items. Status code: {response.status_code}") # Convert the list of menu items into JSON # with open(f'{restaurant_code}.json', 'w') as json_file: # json.dump(menu_items_list, json_file, indent=4) # print(f"Menu items saved to {restaurant_code}.json file.") # main_excel_file(url_input) return menu_items_list,location_area,location_name def extract_category(items,url): db = client.Restaurants_in_dubai collection = db.Category json_data = items restro_collection = db['Restaurant_details'] def item_extract_category(json,name): items_list = [] for item in json: if item['Category'] == name: item_info = { "id": item["ID"], "name": item["Item_name"] } items_list.append(item_info) return items_list categories_info = [] existing_categories = set() for item in json_data: name = item['Category'] if name not in existing_categories: # category_position = 1 # category_isActive = True items = item_extract_category(json_data, name) restro = restro_collection.find_one({'url': url}) if restro: restro_id = restro['_id'] restro_ref_id = restro_id category_info = { 'category_name': name, 'items': items, 'restro_ref_id' : restro_ref_id } categories_info.append(category_info) existing_categories.add(name) result = collection.insert_many(categories_info) print("Inserted Category document IDs:", result.inserted_ids) def extract_item(items,url): db = client.Restaurants_in_dubai collection = db.Items json_data = items category_collection = db['Category'] restro_collection = db['Restaurant_details'] items_info = [] for item in json_data: id = item['ID'] name = item['Item_name'] description = item['Description'] price = item['Price'] img_url= item['Image'] category_name = item['Category'] item_position = item['Item_position'] has_choices = item['Has_choice'] if has_choices == True: modifires = [] for option_group in item.get('Option_groups', []): modifires.append(option_group['Option_group_name']) else: modifires = "None" restro = restro_collection.find_one({'url': url}) if restro: restro_id = restro['_id'] restro_ref_id = restro_id category = category_collection.find_one( { 'category_name': category_name, 'restro_ref_id': restro_ref_id }) if category: category_id = category['_id'] ref_id = category_id item_info = { 'item_id': id, 'name': name, 'description': description, 'amount': price, 'image': img_url, 'category_name':category_name, 'modifires':modifires, 'ref_id_category' : ref_id, 'restro_ref_id' : restro_ref_id } items_info.append(item_info) result = collection.insert_many(items_info) print("Inserted Items document IDs:", result.inserted_ids) def extract_option_group(items,url): db = client.Restaurants_in_dubai collection = db.OptionGroup option_group_info = [] existing_categories = [] option_group_names_list = [] restro_collection = db['Restaurant_details'] for item in items: for option_group in item.get('Option_groups', []): flag = 1 restro = restro_collection.find_one({'url': url}) if restro: restro_id = restro['_id'] restro_ref_id = restro_id option_group_name = option_group["Option_group_name"] min_quantity = option_group["Min"] max_quantity = option_group["Max"] option_names = [] option_names_trial_all = [] for option in option_group.get("Option_group_names", []): option_name = option["Option_name"] option_price = option["Option_price"] option_names.append(option_name) option_names_trial = { 'Option_name': option_name, 'Option_price': option_price } option_names_trial_all.append(option_names_trial) item_id = [] for item in items: for option_group in item.get('Option_groups', []): option_group_name2 = option_group["Option_group_name"] option_group_names1 = option_group.get('Option_group_names',[]) if(option_group_name2 == option_group_name and (sorted(option_group_names1, key=lambda x: x['Option_name'])) == sorted(option_names_trial_all, key=lambda x: x['Option_name']) ): item_id.append(item['ID']) option_group_information = { "option_group_name" : option_group_name, "min_quantity" : min_quantity, "max_quantity" : max_quantity, "option_names" : option_names, "item_id" : item_id, "restro_ref_id" : restro_ref_id } option_group_check ={ "option_group_name" : option_group_name, "option_names" : option_names } for category in existing_categories: if (category['option_group_name'] == option_group_check['option_group_name'] and sorted(category['option_names']) == sorted(option_group_check['option_names'])): flag = 0 if flag==1: option_group_info.append(option_group_information) existing_categories.append(option_group_check) result = collection.insert_many(option_group_info) print("Inserted Option_Groups document IDs:", result.inserted_ids) def extract_option_group_names(items,url): db = client.Restaurants_in_dubai collection = db.OptionName option_group_collection = db['OptionGroup'] json_data = items option_names = [] option_names_list = [] restro_collection = db['Restaurant_details'] for item in json_data: has_choices = item['Has_choice'] if has_choices == True: flag = 1 for option_group in item.get('Option_groups', []): option_group_name2 = option_group["Option_group_name"] for option in option_group.get("Option_group_names", []): restro = restro_collection.find_one({'url': url}) if restro: restro_id = restro['_id'] restro_ref_id = restro_id option_name = option["Option_name"] option_price = option["Option_price"] query = { 'restro_ref_id': restro_id, 'option_names': option_name } matching_documents = option_group_collection.find(query) matching_ids = [doc['_id'] for doc in matching_documents] # if option_group_name_true: # option_group_id = option_group_name_true['_id'] # restro = restro_collection.find_one({'url': url}) option_group_name = { "option_name" : option_name, "option_price" : option_price, "ref_option_group_id" : matching_ids, "restro_ref_id" : restro_ref_id } if (option_name in option_names_list ): flag = 0 if flag==1: option_names.append(option_group_name) option_names_list.append(option_name) result = collection.insert_many(option_names) print("Inserted Options_name document IDs:", result.inserted_ids) items,location_area,location_name = main_json(url) restro_details(url,location_area,location_name) extract_category(items,url) extract_item(items,url) extract_option_group(items,url) extract_option_group_names(items,url) return True