Merge branch 'main' of hf.co:/k4d3/toolkit

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "dataset-tools"]
 	path = dataset-tools
-	url = https://github.com/ka-de/dataset-tools
+	url = git@github.com:ka-de/dataset-tools

dataset-tools

@@ -1 +1 @@
-Subproject commit 0af984140656aa40d830d3317fc6c00ac0fc9ba3
+Subproject commit a8c787494826e9ae2d1e386be8ce304eafb3a0e6

utils/crc32.py

@@ -0,0 +1,18 @@
+import zlib
+import sys
+
+def calculate_crc32(file_path):
+    with open(file_path, 'rb') as file:
+        data = file.read()
+        crc32_checksum = zlib.crc32(data)
+        return crc32_checksum
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python script.py <file_path>")
+        sys.exit(1)
+
+    file_path = sys.argv[1]
+    checksum = calculate_crc32(file_path)
+    print(f"CRC32 checksum of {file_path}: {checksum:#010x}")
+

utils/remove_grandfathered.py

@@ -1,42 +1,26 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-#
-import os
-import sys
-# Add the parent directory to the Python path
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from utils.file_processor import FileProcessor, ProcessorOptions
-import re
-from pathlib import Path
 
-class GrandfatheredRemovalProcessor(FileProcessor):
-    def __init__(self, options: ProcessorOptions, pattern: str):
-        super().__init__(options)
-        self.pattern = pattern
-    
-    def process_content(self, content: str) -> str:
-        # Remove occurrences of the grandfathered content pattern
-        content = re.sub(self.pattern, '', content)
-        # Normalize whitespace and commas
-        content = re.sub(r'\s+,', ',', content)
-        content = re.sub(r',\s+', ',', content)
-        return re.sub(r'\s+', ' ', content).strip()
+import os
 
-def main():
-    target_dir = sys.argv[1] if len(sys.argv) > 1 else '.'
+def remove_text_from_file(file_path, text_to_remove):
+    with open(file_path, 'r') as file:
+        content = file.read()
     
-    options = ProcessorOptions(
-        recursive=True,
-        dry_run=False,
-        file_extensions={'.txt', '.tags'}
-    )
+    content = content.replace(text_to_remove, "")
     
-    # Define the pattern for grandfathered content (modify as needed)
-    grandfathered_pattern = r'grandfathered_content_pattern'
-    
-    processor = GrandfatheredRemovalProcessor(options, grandfathered_pattern)
-    processor.process_directory(Path(target_dir))
+    with open(file_path, 'w') as file:
+        file.write(content)
+
+def remove_text_from_files(root_dir, text_to_remove):
+    for subdir, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith('.txt') or file.endswith('.tags'):
+                file_path = os.path.join(subdir, file)
+                remove_text_from_file(file_path, text_to_remove)
 
 if __name__ == "__main__":
-    main()
+    root_directory = '.'  # Change this to your root directory
+    text_to_remove = "grandfathered content, "
+    remove_text_from_files(root_directory, text_to_remove)
+