ome stuff

Signed-off-by: Balazs Horvath <acsipont@gmail.com>

.zshrc

@@ -10,6 +10,35 @@ alias 🐺='GGML_CUDA_NO_PINNED=1 ollama serve & conda activate openwebui && ope
 alias ngrok-webui='ngrok http --url=hedgehog-lasting-mutually.ngrok-free.app 6969'
 alias gsa='git submodule add'
 
+# The cweb function compresses all .css, .html, .js, and .json files in the ./web directory
+# using gzip and brotli.
+#
+# Then, it finds all matching files and compresses them with brotli.
+#
+# The function prints out the name of each file being processed.
+cweb() {
+    # First find and compress with gzip
+    find ./web -type f \( -name "*.css" -o -name "*.html" -o -name "*.js" -o -name "*.json" \) \
+        ! -name "*.gz" ! -name "*.br" ! -name "*.zst" \
+        -print0 | xargs -0 -P $(nproc) -I {} bash -c '
+            echo "Processing (gzip): {}"
+            gzip -k "{}"
+        '
+
+    # Then find and compress with brotli
+    find ./web -type f \( -name "*.css" -o -name "*.html" -o -name "*.js" -o -name "*.json" \) \
+        ! -name "*.gz" ! -name "*.br" ! -name "*.zst" \
+        -print0 | xargs -0 -P $(nproc) -I {} bash -c '
+            echo "Processing (brotli): {}"
+            brotli --best "{}"
+        '
+}
+
+# Automatically commits and pushes changes in all git repositories found in ~/datasets/_
+# Checks each immediate subdirectory, and if it's a git repo with changes:
+# - Adds all changes
+# - Creates a signed commit
+# - Pushes to remote
 function _up() {
     local base_dir="$HOME/datasets/_"
     
@@ -35,6 +64,9 @@ function _up() {
     cd "$current_dir"
 }
 
+# Finds and deletes .tags files that don't have a corresponding image file
+# Checks for matching .png, .jpg, .jpeg, .jxl, or .webp files
+# If no matching image is found, the .tags file is deleted with verbose output
 deleteorphantags() {
   # Loop through all .tags files
   for tag_file in **/*.tags; do
@@ -47,6 +79,9 @@ deleteorphantags() {
   done
 }
 
+# Finds and deletes .txt files that don't have a corresponding image file
+# Checks for matching .png, .jpg, .jpeg, .jxl, or .webp files
+# If no matching image is found, the .txt file is deleted with verbose output
 deleteorphantxt() {
   # Loop through all .txt files
   for txt_file in **/*.txt; do
@@ -59,6 +94,9 @@ deleteorphantxt() {
   done
 }
 
+# Lists all .tags files that don't have a corresponding image file
+# Checks for matching .png, .jpg, .jpeg, .jxl, or .webp files
+# Prints the paths of orphaned .tags files without deleting them
 look4orphantags() {
   # Loop through all .tags files
   for tag_file in **/*.tags; do
@@ -71,6 +109,9 @@ look4orphantags() {
   done
 }
 
+# Lists all .txt files that don't have a corresponding image file
+# Checks for matching .png, .jpg, .jpeg, .jxl, or .webp files
+# Prints the paths of orphaned .txt files without deleting them
 look4orphantxt() {
   # Loop through all .txt files
   for txt_file in **/*.txt; do
@@ -709,7 +750,6 @@ replace_text_in_files() {
 # The script checks if the specified directory exists and iterates over each text file in the directory.
 # For each text file, it creates a temporary file with the modified content and then replaces the original file with the temporary file.
 # If the directory does not exist, it prints an error message.
-
 inject_to_txt() {
     local dir="$1"
     local prefix="$2"

utils/extract_description_flickr.py

@@ -0,0 +1,25 @@
+import os
+import json
+
+def extract_description(json_path):
+    with open(json_path, 'r', encoding='utf-8') as file:
+        data = json.load(file)
+        if 'description' in data:
+            base_filename = os.path.splitext(json_path)[0]
+            description_filename = f"{base_filename}.description"
+            with open(description_filename, 'w', encoding='utf-8') as desc_file:
+                desc_file.write(data['description'])
+            print(f"Created: {description_filename}")
+        else:
+            print(f"No description key in: {json_path}")
+
+def process_directory(directory):
+    for root, _, files in os.walk(directory):
+        for file in files:
+            if file.lower().endswith('.json'):
+                extract_description(os.path.join(root, file))
+
+if __name__ == "__main__":
+    current_directory = os.getcwd()
+    process_directory(current_directory)
+

utils/remove_extra_file_extension_txt.py

@@ -0,0 +1,17 @@
+import os
+
+def rename_description_files(directory):
+    for root, _, files in os.walk(directory):
+        for file in files:
+            if file.endswith('.description'):
+                base_name = file.rsplit('.', 2)[0]  # Split twice to remove the extra extension
+                new_name = f"{base_name}.txt"
+                old_path = os.path.join(root, file)
+                new_path = os.path.join(root, new_name)
+                os.rename(old_path, new_path)
+                print(f"Renamed: {old_path} to {new_path}")
+
+if __name__ == "__main__":
+    current_directory = os.getcwd()
+    rename_description_files(current_directory)
+

utils/rename_description_file.py

@@ -0,0 +1,17 @@
+import os
+
+def rename_description_files(directory):
+    for root, _, files in os.walk(directory):
+        for file in files:
+            if file.endswith('.description'):
+                base_name = file.rsplit('.', 2)[0]  # Split twice to remove the extra extension
+                new_name = f"{base_name}.description"
+                old_path = os.path.join(root, file)
+                new_path = os.path.join(root, new_name)
+                os.rename(old_path, new_path)
+                print(f"Renamed: {old_path} to {new_path}")
+
+if __name__ == "__main__":
+    current_directory = os.getcwd()
+    rename_description_files(current_directory)
+