Upload download_images.py with huggingface_hub

download_images.py

@@ -0,0 +1,91 @@
+# import json
+# import requests
+# import os
+
+# # Create a directory to save the images
+# os.makedirs('images', exist_ok=True)
+
+# # Function to download an image from a URL
+# def download_image(url, save_path):
+#     response = requests.get(url)
+#     if response.status_code == 200:
+#         with open(save_path, 'wb') as file:
+#             file.write(response.content)
+#     else:
+#         print(f"Failed to download image from {url}")
+
+# # Read the JSONL file and download images
+# # download the ith to jth images
+# def download_itoj_images(i, j):
+#     with open('transformed_data.jsonl', 'r') as file:
+#         for idx, line in enumerate(file):
+#             if idx < i:
+#                 continue
+#             if idx > j:
+#                 break
+#             item = json.loads(line)
+#             image_path = item['image_path'].split('/')[-1]
+#             # image_id = image_path.split('_')[-1].split('.')[0]
+#             url = f"http://images.cocodataset.org/train2014/{image_path}"
+#             save_path = os.path.join('images', f"{image_path}")
+#             download_image(url, save_path)
+#             print(f"Downloaded {image_path}, id: {idx}")
+#             # save the message to a file
+#             with open('downloaded_images_log.txt', 'a') as f:
+#                 f.write(f"Downloaded {image_path}, id: {idx}\n")
+
+# # read i and j from the arguments
+# import sys
+# i = int(sys.argv[1])
+# j = int(sys.argv[2])
+# download_itoj_images(i, j)
+
+import json
+import requests
+import os
+from concurrent.futures import ThreadPoolExecutor
+
+# Create a directory to save the images
+os.makedirs('images', exist_ok=True)
+
+# Function to download an image from a URL
+def download_image(item):
+    image_path = item['image_path'].split('/')[-1]
+    # image_id = image_path.split('_')[-1].split('.')[0]
+    url = f"http://images.cocodataset.org/train2014/{image_path}"
+    save_path = os.path.join('images', f"{image_path}")
+    
+    try:
+        response = requests.get(url)
+        if response.status_code == 200:
+            with open(save_path, 'wb') as file:
+                file.write(response.content)
+            log_message(f"Downloaded {image_path}, id: {item['id']}")
+        else:
+            log_message(f"Failed to download image from {url}")
+    except Exception as e:
+        log_message(f"Exception occurred while downloading {url}: {e}")
+
+# Function to log a message to log.txt
+def log_message(message):
+    with open('downloaded_images_log.txt', 'a') as log_file:
+        log_file.write(message + '\n')
+
+# Function to download images from index i to j using ThreadPoolExecutor
+def download_itoj_images(i, j):
+    with open('transformed_data.jsonl', 'r') as file:
+        items = [json.loads(line) for idx, line in enumerate(file) if i <= idx <= j]
+        # note that idx starts from 0. id = idx + 1
+        # i and j are idxes, so in the log it will show i+1 and j+1
+    
+    with ThreadPoolExecutor(max_workers=10) as executor:
+        executor.map(download_image, items)
+
+# read i and j from the arguments
+import sys
+i = int(sys.argv[1])
+j = int(sys.argv[2])
+download_itoj_images(i, j)
+
+
+