fix 2

hf_scrapper.py

@@ -1,8 +1,8 @@
 import os
 import json
 import aiohttp
-import aiofiles
 import asyncio
+import aiofiles
 import urllib.request
 from aiohttp import ClientSession, ClientTimeout
 from aiohttp.client_exceptions import ClientError
@@ -13,7 +13,7 @@ CACHE_JSON_PATH = os.path.join(CACHE_DIR, "cached_films.json")
 
 download_progress = {}
 
-def get_system_proxies():
+async def get_system_proxies():
     """
     Retrieves the system's HTTP and HTTPS proxies.
 
@@ -70,4 +70,4 @@ async def write_file_structure_to_json(file_structure, file_path):
             await json_file.write(json.dumps(file_structure, indent=2))
         print(f'File structure written to {file_path}')
     except IOError as e:
-        print(f"Error writing file structure to JSON: {e}")
+        print(f"Error writing file structure to JSON: {e}")

indexer.py

@@ -1,73 +1,45 @@
-import os
 import json
-import aiohttp
+import logging
 import asyncio
-import aiofiles
-import urllib.request
-from aiohttp import ClientSession, ClientTimeout
-from aiohttp.client_exceptions import ClientError
-from tqdm.asyncio import tqdm
-
-CACHE_DIR = os.getenv("CACHE_DIR")
-CACHE_JSON_PATH = os.path.join(CACHE_DIR, "cached_films.json")
-
-download_progress = {}
+from hf_scrapper import get_system_proxies, get_file_structure, write_file_structure_to_json
+from dotenv import load_dotenv
+import os
 
-async def get_system_proxies():
-    """
-    Retrieves the system's HTTP and HTTPS proxies.
+load_dotenv()
 
-    Returns:
-        dict: A dictionary containing the proxies.
-    """
+async def index_repository(token, repo, current_path="", proxies=None):
     try:
-        proxies = urllib.request.getproxies()
-        print("System proxies:", proxies)
-        return {
-            "http": proxies.get("http"),
-            "https": proxies.get("http")
-        }
+        file_structure = await get_file_structure(repo, token, current_path, proxies)
+        full_structure = []
+        for item in file_structure:
+            if item['type'] == 'directory':
+                sub_directory_structure = await index_repository(token, repo, item['path'], proxies)
+                full_structure.append({
+                    "type": "directory",
+                    "path": item['path'],
+                    "contents": sub_directory_structure
+                })
+            else:
+                full_structure.append(item)
+        return full_structure
     except Exception as e:
-        print(f"Error getting system proxies: {e}")
-        return {}
-
-async def get_file_structure(repo, token, path="", proxies=None):
-    """
-    Fetches the file structure of a specified Hugging Face repository.
-
-    Args:
-        repo (str): The name of the repository.
-        token (str): The authorization token for the request.
-        path (str, optional): The specific path in the repository. Defaults to "".
-        proxies (dict, optional): The proxies to use for the request. Defaults to None.
-
-    Returns:
-        list: A list of file structure information.
-    """
-    api_url = f"https://huggingface.co/api/models/{repo}/tree/main/{path}"
-    headers = {'Authorization': f'Bearer {token}'}
-    timeout = ClientTimeout(total=10)
-    async with ClientSession(timeout=timeout) as session:
-        print(f"Fetching file structure from URL: {api_url} with proxies: {proxies}")
-        try:
-            async with session.get(api_url, headers=headers, proxy=proxies.get("http")) as response:
-                response.raise_for_status()
-                return await response.json()
-        except ClientError as e:
-            print(f"Error fetching file structure: {e}")
-            return []
-
-async def write_file_structure_to_json(file_structure, file_path):
-    """
-    Writes the file structure to a JSON file.
-
-    Args:
-        file_structure (list): The file structure data.
-        file_path (str): The path where the JSON file will be saved.
-    """
+        logging.error(f"Error indexing repository: {e}")
+        raise
+
+async def indexer():
+    token = os.getenv("TOKEN")
+    repo = os.getenv("REPO")
+    output_path = os.getenv("INDEX_FILE")
+    
+    if not token or not repo or not output_path:
+        logging.error("Environment variables TOKEN, REPO, or INDEX_FILE are not set.")
+        return
+
+    proxies = await get_system_proxies()
+    
     try:
-        async with aiofiles.open(file_path, 'w') as json_file:
-            await json_file.write(json.dumps(file_structure, indent=2))
-        print(f'File structure written to {file_path}')
-    except IOError as e:
-        print(f"Error writing file structure to JSON: {e}")
+        full_structure = await index_repository(token, repo, "", proxies)
+        await write_file_structure_to_json(full_structure, output_path)
+        logging.info(f"Full file structure for repository '{repo}' has been indexed and saved to {output_path}")
+    except Exception as e:
+        logging.error(f"Error during indexing: {e}")