Update file_utils.py

file_utils.py

@@ -1,30 +1,38 @@
-import gradio as gr
-from repo_utils import extract_repo_content
-from display_utils import format_output
+import os
+from magika import Magika
 
-# Extract and display function
-def extract_and_display(url):
-    extracted_content = extract_repo_content(url, hf_token, hf_user)
-    formatted_output = format_output(extracted_content, url)
-    return formatted_output
+SUPPORTED_FILE_TYPES = ["txt", "shell", "python", "markdown", "yaml", "json", "csv", "tsv", "xml", "html", "ini", "jsonl", "ipynb"]
 
-app = gr.Blocks(theme="sudeepshouche/minimalist")
+def get_file_summary(file_path, file_type):
+    size = os.path.getsize(file_path)
+    return {
+        "name": os.path.relpath(file_path),
+        "type": file_type,
+        "size": size,
+        "creation_date": os.path.getctime(file_path),
+        "modification_date": os.path.getmtime(file_path)
+    }
 
-with app:
-    gr.Markdown("# Hugging Face Space / Model Repository Content Extractor")
-    url_input = gr.Textbox(label="https:// URL of Repository", placeholder="Enter the repository URL here OR select an example below...")
-    url_examples = gr.Examples(
-        examples=[
-            ["https://huggingface.co/spaces/big-vision/paligemma-hf"],
-            ["https://huggingface.co/google/paligemma-3b-mix-224"],
-            ["https://huggingface.co/microsoft/Phi-3-vision-128k-instruct"],
-            ["https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf"]
-        ],
-        inputs=url_input
-    )
-    output_display = gr.Textbox(label="Extracted Repository Content", show_copy_button=True, lines=20, placeholder="Repository content will be extracted here...\n\nMetadata is captured for all files, but text content provided only for files less than 32 kb\n\n\n\nReview and search through the content here OR simply copy it for offline analysis!!. 🤖")
-    extract_button = gr.Button("Extract Content")
-    
-    extract_button.click(fn=extract_and_display, inputs=url_input, outputs=output_display)
+def read_file_content(file_path, max_size=32*1024):
+    with open(file_path, "r", encoding="utf-8", errors="ignore") as file:
+        if os.path.getsize(file_path) > max_size:
+            return file.read(max_size) + "\n... [Content Truncated] ..."
+        else:
+            return file.read()
 
-app.launch()
+def validate_file_types(directory):
+    m = Magika()
+    file_types = {}
+    for root, _, files in os.walk(directory):
+        if '.git' in root:
+            continue
+        for file_name in files:
+            file_path = os.path.join(root, file_name)
+            try:
+                with open(file_path, 'rb') as file:
+                    file_bytes = file.read()
+                result = m.identify_bytes(file_bytes)
+                file_types[file_path] = result.output.ct_label
+            except Exception as e:
+                file_types[file_path] = f"Error: {str(e)}"
+    return file_types