Update dlmodels.sh

dlmodels.sh

@@ -8,64 +8,73 @@ fi
 
 # Set the input file from the first argument
 INPUT_FILE="$1"
+CHECKSUM_FILE=".checksums"
 
-# Function to reliably get the remote file size
-get_remote_size() {
+# Function to generate SHA-256 checksum for a given file
+generate_checksum() {
+    local filepath="$1"
+    sha256sum "$filepath" | awk '{print $1}'
+}
+
+# Function to retrieve the remote checksum if available
+get_remote_checksum() {
     local url="$1"
-    # Attempt to retrieve Content-Length header
-    local remote_size=$(curl -sI "$url" | grep -i "Content-Length" | awk '{print $2}' | tr -d '\r')
+    local checksum_url="${url}.sha256"  # Assuming the checksum URL is <file_url>.sha256
 
-    # If Content-Length is not available, try downloading a small byte range
-    if [[ -z "$remote_size" || "$remote_size" -lt 2 ]]; then
-        remote_size=$(curl -s --range 0-1 "$url" | wc -c)
+    # Attempt to fetch the remote checksum
+    remote_checksum=$(curl -s "$checksum_url")
+    if [[ -z "$remote_checksum" ]]; then
+        echo "no_remote_checksum"
+    else
+        echo "$remote_checksum"
     fi
-
-    # Return the size, or empty if it can't be determined
-    echo "$remote_size"
 }
 
-# Function to check file existence and download integrity
+# Function to download the file and verify its integrity
 download_file() {
     local filepath="$1"
     local url="$2"
 
     echo -e "\nChecking file: $filepath"
 
-    # If file already exists, verify integrity
+    # Check if file exists locally and get its checksum
     if [[ -f "$filepath" ]]; then
-        # Get remote file size using get_remote_size function
-        remote_size=$(get_remote_size "$url")
-        local_size=$(stat -c%s "$filepath")
-
-        # Debug output to confirm size checks
-        echo "Remote size: ${remote_size:-Unknown}, Local size: $local_size for file $filepath"
+        local_checksum=$(generate_checksum "$filepath")
+        stored_checksum=$(grep "$filepath" "$CHECKSUM_FILE" | awk '{print $1}')
 
-        # Skip file if Content-Length is not available or remote size can't be determined
-        if [[ -z "$remote_size" || "$remote_size" -lt 2 ]]; then
-            echo -e "[SKIP] Unable to verify size for $filepath. Assuming complete."
+        # Check if local file's checksum matches stored checksum
+        if [[ "$local_checksum" == "$stored_checksum" ]]; then
+            echo -e "[SKIP] $filepath already exists and matches the stored checksum."
             return 0
         fi
+    fi
 
-        # If the file sizes match, skip re-downloading
-        if [[ "$remote_size" == "$local_size" ]]; then
-            echo -e "[SKIP] $filepath already exists and is complete."
+    # Check remote checksum if available
+    remote_checksum=$(get_remote_checksum "$url")
+    if [[ "$remote_checksum" != "no_remote_checksum" ]]; then
+        echo "Remote checksum found: $remote_checksum"
+        # Compare with local file's checksum if it exists
+        if [[ -f "$filepath" && "$remote_checksum" == "$local_checksum" ]]; then
+            echo -e "[SKIP] $filepath already matches the remote checksum."
             return 0
-        else
-            echo -e "[RE-DOWNLOAD] $filepath is incomplete or corrupted. Re-downloading..."
-            rm -f "$filepath"  # Remove the corrupted file before re-downloading
         fi
     else
-        echo -e "[DOWNLOAD] Starting download of $filepath"
+        echo "[INFO] No remote checksum available for $url. Proceeding with download."
     fi
 
-    # Download the file using wget, with progress shown in a clean format
+    # Download the file with wget, showing progress
+    echo -e "[DOWNLOAD] Downloading $filepath..."
     wget -c --progress=dot -O "$filepath" "$url" 2>&1 | \
     grep --line-buffered "%" | \
     sed -u -e "s,\.,,g" | \
     awk '{printf("\r[DOWNLOADING] %s - %s", "'$filepath'", $2)}'
 
-    # Confirm download completion
+    # Confirm completion
     echo -e "\n[COMPLETE] Downloaded $filepath"
+
+    # Generate and store checksum after download
+    new_checksum=$(generate_checksum "$filepath")
+    echo "$new_checksum $filepath" >> "$CHECKSUM_FILE"
 }
 
 # Read and process each line in the specified input file
@@ -74,7 +83,7 @@ while IFS= read -r line; do
     filepath=$(echo "$line" | awk '{print $1}')
     url=$(echo "$line" | awk '{print $2}')
 
-    # Call the download function for each file in the list
+    # Call the download function for each file
     download_file "$filepath" "$url" &
 done < "$INPUT_FILE"