app.py

import os
import streamlit as st
import pandas as pd
from huggingface_hub import (
    HfApi,
    upload_folder,
    create_repo,
    login,
    hf_hub_download,
    list_repo_files,
)
import logging
import time
import json
import keyring  # Secure token storage
import socket  # Offline detection
import hashlib  # Data integrity
from pathlib import Path
from threading import Thread
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import schedule
import datetime
from streamlit_option_menu import option_menu

# Set page configuration
st.set_page_config(page_title="InfiniteStorageFace", layout="wide")

# Initialize logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    handlers=[logging.StreamHandler()]
)

# Initialize session state variables
session_defaults = {
    'logs': [],
    'uploading': False,
    'cancel': False,
    'observer': None,
    'selected_files': [],
    'sync_paused': False,
    'token': "",
    'settings_loaded': False,
    'remote_files': {},
    'queued_files': [],
    'scheduled_sync': False,
    'repo_id': "",
    'repo_exists': False,
    'folder_path': "",
    'private': True,  # Default to private repositories
    'ignore_patterns_selected': [],
    'process_individually': False,
    'subfolder': "",
    'total_files_synced': 0,
    'total_files': 0,
    'sync_option': 'Sync',  # Default option is Sync
}
for key, default_value in session_defaults.items():
    if key not in st.session_state:
        st.session_state[key] = default_value

# Centralized ignore patterns mapping
IGNORE_PATTERNS_MAP = {
    "Ignore __pycache__": "**/__pycache__/**",
    "Ignore .git": ".git/**",
    "Ignore .venv": "venv/**",
    "Ignore *.pyc": "*.pyc",
    "Ignore *.log": "*.log",
    "Ignore *.tmp": "*.tmp",
    "Ignore *.DS_Store": "*.DS_Store"
}

# Default values
DEFAULT_REPO = "your_username/your_private_vault"
DEFAULT_LOCAL_PATH = str(Path.home())

# Helper Functions
def is_connected():
    """Check for internet connectivity."""
    try:
        socket.create_connection(("1.1.1.1", 53), timeout=3)
        return True
    except OSError:
        return False

def log(message, level="INFO"):
    """Log messages with timestamp."""
    timestamp = time.strftime("[%Y-%m-%d %H:%M:%S]")
    full_message = f"{timestamp} {message}"
    st.session_state.logs.append(full_message)
    if level == "INFO":
        logging.info(message)
    elif level == "WARNING":
        logging.warning(message)
    elif level == "ERROR":
        logging.error(message)
    elif level == "DEBUG":
        logging.debug(message)

def authenticate(token):
    """Authenticate user with Hugging Face token."""
    if not token:
        log("❌ Hugging Face Token is required.", level="ERROR")
        return False
    try:
        login(token)
        keyring.set_password("huggingface", "token", token)
        log("✅ Authenticated successfully!")
        return True
    except Exception as e:
        log(f"❌ Authentication failed: {e}", level="ERROR")
        return False

def create_repo_if_not_exists(repo_id, token, private):
    """Create a repository if it doesn't exist."""
    api = HfApi()
    try:
        api.list_repo_files(repo_id=repo_id, repo_type="dataset", token=token)
        log(f"✅ Repository '{repo_id}' exists.")
        st.session_state.repo_exists = True
        return True
    except Exception:
        log(f"⚠️ Repository '{repo_id}' does not exist. Creating it...", level="WARNING")
        try:
            create_repo(
                repo_id=repo_id,
                token=token,
                private=private,
                repo_type="dataset",
                exist_ok=True,
            )
            log(f"✅ Created new repository: '{repo_id}'.")
            st.session_state.repo_exists = True
            return True
        except Exception as create_err:
            log(f"❌ Failed to create repository '{repo_id}': {create_err}", level="ERROR")
            return False

def compute_checksum(file_path):
    """Compute the checksum of a file for data integrity."""
    sha256 = hashlib.sha256()
    try:
        with open(file_path, "rb") as f:
            for chunk in iter(lambda: f.read(4096), b""):
                sha256.update(chunk)
        return sha256.hexdigest()
    except Exception as e:
        log(f"❌ Failed to compute checksum for '{file_path}': {e}")
        return None

def upload_folder_structure(folder_path, repo_id, token, target_path, ignore_patterns, retry=3):
    """Upload a folder structure with error handling and retries."""
    upload_params = {
        "folder_path": folder_path,
        "repo_id": repo_id,
        "repo_type": "dataset",
        "token": token,
        "path_in_repo": target_path,
        "ignore_patterns": ignore_patterns,
    }
    log(f"🚀 Uploading folder '{folder_path}' to '{target_path}' in repository '{repo_id}'...")
    for attempt in range(1, retry + 1):
        try:
            upload_folder(**upload_params)
            log(f"✅ Upload completed for '{folder_path}'!")
            return True
        except Exception as upload_err:
            log(f"❌ Upload failed for '{folder_path}' on attempt {attempt}: {upload_err}", level="ERROR")
            if attempt < retry:
                log(f"🔄 Retrying upload ({attempt}/{retry})...", level="WARNING")
                time.sleep(2 ** attempt)  # Exponential backoff
            else:
                log(f"❌ All retry attempts failed for '{folder_path}'.", level="ERROR")
                return False

# Function to handle uploads
def upload_files():
    st.session_state.uploading = True
    token = st.session_state.token
    repo_id = st.session_state.repo_id
    private = st.session_state.private
    folder_path = st.session_state.folder_path
    subfolder = st.session_state.subfolder
    process_individually = st.session_state.process_individually
    ignore_patterns_selected = st.session_state.ignore_patterns_selected
    ignore_patterns = [IGNORE_PATTERNS_MAP[pattern] for pattern in ignore_patterns_selected]
    target_path = subfolder.replace("\\", "/") if subfolder else ""

    if not is_connected():
        log("❌ No internet connection. Sync will resume when connection is restored.", level="ERROR")
        st.session_state.uploading = False
        return

    if not authenticate(token):
        st.session_state.uploading = False
        return

    if not create_repo_if_not_exists(repo_id, token, private):
        st.session_state.uploading = False
        return

    if not os.path.isdir(folder_path):
        log(f"❌ The folder path '{folder_path}' does not exist.", level="ERROR")
        st.session_state.uploading = False
        return

    # Count the total files in folders
    st.session_state.total_files = sum(len(files) for _, _, files in os.walk(folder_path))
    st.session_state.total_files_synced = 0

    # Upload only folders (no individual files except in root)
    for item in os.listdir(folder_path):
        item_path = os.path.join(folder_path, item)
        if os.path.isdir(item_path):
            # Upload each folder
            success = upload_folder_structure(item_path, repo_id, token, f"{target_path}/{item}", ignore_patterns)
            if success:
                for root, _, files in os.walk(item_path):
                    for file in files:
                        local_file = os.path.join(root, file)
                        relative_path = os.path.relpath(local_file, folder_path).replace("\\", "/")
                        remote_file = f"{target_path}/{relative_path}"
                        st.session_state.total_files_synced += 1
                        local_checksum = compute_checksum(local_file)
                        remote_checksum = get_remote_file_checksum(repo_id, token, remote_file)
                        if local_checksum and remote_checksum and local_checksum == remote_checksum:
                            log(f"✅ Data integrity verified for '{relative_path}'.")
                        else:
                            log(f"❌ Data integrity verification failed for '{relative_path}'.", level="ERROR")
        elif os.path.isfile(item_path):
            # Upload files only if they are in the root directory
            relative_path = os.path.relpath(item_path, folder_path).replace("\\", "/")
            if "/" not in relative_path:  # Check if the file is in the root
                success = upload_folder_structure(item_path, repo_id, token, f"{target_path}/{item}", ignore_patterns)
                if success:
                    st.session_state.total_files_synced += 1
                    local_checksum = compute_checksum(item_path)
                    remote_checksum = get_remote_file_checksum(repo_id, token, f"{target_path}/{item}")
                    if local_checksum and remote_checksum and local_checksum == remote_checksum:
                        log(f"✅ Data integrity verified for '{relative_path}'.")
                    else:
                        log(f"❌ Data integrity verification failed for '{relative_path}'.", level="ERROR")

    st.session_state.uploading = False
    log("🚀 Upload process completed.")

def get_remote_file_checksum(repo_id, token, file_path):
    # Placeholder: Hugging Face Hub does not provide file checksums directly
    return None

# Function to monitor folder changes with real-time sync and offline queueing
class ChangeHandler(FileSystemEventHandler):
    def on_modified(self, event):
        if not st.session_state.uploading and not st.session_state.sync_paused:
            if is_connected():
                log("🔄 Changes detected. Starting upload...")
                upload_thread = Thread(target=upload_files)
                upload_thread.start()
            else:
                log("❌ No internet connection. Queuing changes for later upload.", level="WARNING")
                queue_changes(event.src_path)

# Queue changes when offline
def queue_changes(file_path):
    queued_files = st.session_state.get("queued_files", [])
    queued_files.append(file_path)
    st.session_state["queued_files"] = queued_files
    log(f"🕒 Queued file for upload: {file_path}")

# Check and upload queued changes when back online
def check_queued_uploads():
    if is_connected() and st.session_state.get("queued_files"):
        log("🔄 Uploading queued files...")
        for file in st.session_state["queued_files"]:
            upload_files_specific(file)
        st.session_state["queued_files"] = []

# Upload a specific file (for queued uploads)
def upload_files_specific(file_path):
    token = st.session_state.token
    repo_id = st.session_state.repo_id
    private = st.session_state.private
    folder_path = st.session_state.folder_path
    subfolder = st.session_state.subfolder
    ignore_patterns_selected = st.session_state.ignore_patterns_selected
    ignore_patterns = [IGNORE_PATTERNS_MAP[pattern] for pattern in ignore_patterns_selected]
    target_path = subfolder.replace("\\", "/") if subfolder else ""

    if not authenticate(token):
        return

    if not create_repo_if_not_exists(repo_id, token, private):
        return

    if not os.path.isfile(file_path):
        log(f"❌ The file path '{file_path}' does not exist.", level="ERROR")
        return

    relative_path = os.path.relpath(file_path, folder_path).replace("\\", "/")
    remote_path = f"{target_path}/{relative_path}"

    success = upload_folder_structure(file_path, repo_id, token, remote_path, ignore_patterns)
    if success:
        log(f"✅ Uploaded queued file '{relative_path}'.")
    else:
        log(f"❌ Failed to upload queued file '{relative_path}'.", level="ERROR")

# Function to get version history
def get_version_history():
    api = HfApi()
    token = st.session_state.token
    repo_id = st.session_state.repo_id
    try:
        commits = api.list_repo_commits(repo_id=repo_id, repo_type="dataset", token=token)
        history = []
        for commit in commits:
            date_str = commit.created_at.strftime('%Y-%m-%d %H:%M:%S')
            history.append(f"Commit {commit.commit_id[:7]} by {commit.author_name} on {date_str}: {commit.title}")
        return "\n".join(history)
    except Exception as e:
        log(f"❌ Error fetching version history: {e}", level="ERROR")
        return "Error fetching version history."

# Function to download from remote
def download_from_remote():
    token = st.session_state.token
    repo_id = st.session_state.repo_id
    folder_path = st.session_state.folder_path
    subfolder = st.session_state.subfolder
    target_path = subfolder.replace("\\", "/") if subfolder else ""

    api = HfApi()
    try:
        remote_files = api.list_repo_files(repo_id=repo_id, repo_type="dataset", token=token)
        for file in remote_files:
            local_file_path = os.path.join(folder_path, file)
            os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
            hf_hub_download(repo_id, file, repo_type="dataset", token=token, local_dir=os.path.dirname(local_file_path), local_dir_use_symlinks=False)
            log(f"✅ Downloaded '{file}' to '{local_file_path}'")
    except Exception as e:
        log(f"❌ Error downloading from remote: {e}", level="ERROR")

def pause_sync():
    st.session_state.sync_paused = True
    log("⏸️ Sync paused.", level="WARNING")

def resume_sync():
    st.session_state.sync_paused = False
    log("▶️ Sync resumed.", level="INFO")
    upload_thread = Thread(target=upload_files)
    upload_thread.start()

def save_settings():
    settings = {
        "repo_id": st.session_state.repo_id,
        "private": st.session_state.private,
        "folder_path": st.session_state.folder_path,
        "subfolder": st.session_state.subfolder,
        "process_individually": st.session_state.process_individually,
        "ignore_patterns_selected": st.session_state.ignore_patterns_selected,
        "selected_files": st.session_state.selected_files
    }
    with open("settings.json", "w") as f:
        json.dump(settings, f)
    log("💾 Settings saved.", level="INFO")

def load_settings():
    if os.path.exists("settings.json"):
        with open("settings.json", "r") as f:
            settings = json.load(f)
        st.session_state.repo_id = settings.get("repo_id", DEFAULT_REPO)
        st.session_state.private = settings.get("private", True)
        st.session_state.folder_path = settings.get("folder_path", DEFAULT_LOCAL_PATH)
        st.session_state.subfolder = settings.get("subfolder", "")
        st.session_state.process_individually = settings.get("process_individually", False)
        st.session_state.ignore_patterns_selected = settings.get("ignore_patterns_selected", [])
        st.session_state.selected_files = settings.get("selected_files", [])
        log("🔄 Settings loaded.", level="INFO")
    else:
        log("❌ No saved settings found.", level="ERROR")

def get_local_files(folder_path):
    files = []
    for root, dirs, filenames in os.walk(folder_path):
        for filename in filenames:
            relative_path = os.path.relpath(os.path.join(root, filename), folder_path)
            files.append(relative_path.replace("\\", "/"))
    return files

def schedule_sync():
    def scheduled_upload():
        if is_connected() and not st.session_state.uploading and not st.session_state.sync_paused:
            log("⏰ Scheduled sync triggered.", level="INFO")
            upload_files()

    schedule.every().day.at("02:00").do(scheduled_upload)
    st.session_state.scheduled_sync = True
    log("⏰ Scheduled daily sync at 02:00.", level="INFO")

def run_scheduler():
    while True:
        schedule.run_pending()
        time.sleep(1)

# --------------------------- Main Interface ---------------------------

st.title("🚀 InfiniteStorageFace")
st.write("Effortlessly sync your local folders to your private Hugging Face repository!")

# Create tabs for navigation
tabs = st.tabs(["Home", "Vault", "Settings", "Logs", "Help"])

with tabs[0]:
    st.header("Welcome to InfiniteStorageFace")
    st.write("Use the tabs to navigate through the application.")

    st.subheader("Vault Overview")
    st.write(f"**Repository ID:** {st.session_state.repo_id or 'Not Set'}")
    st.write(f"**Private Repository:** {'Yes' if st.session_state.private else 'No'}")
    st.write(f"**Total Files Synced:** {st.session_state.total_files_synced}")
    st.write(f"**Total Files in Folder:** {st.session_state.total_files}")

    # Display repository contents
    st.subheader("Repository Contents")
    if st.session_state.repo_exists:
        repo_files = list_repo_files(repo_id=st.session_state.repo_id, token=st.session_state.token)
        if repo_files:
            for file in repo_files:
                st.write(f"📄 {file}")
        else:
            st.write("Repository is empty.")
    else:
        st.write("Repository not found or not authenticated.")

with tabs[1]:
    st.header("Vault Sync and Upload")

    # Select Sync or Upload
    st.session_state.sync_option = st.radio("Choose an option:", ["Sync", "Upload"], index=0)

    # Folder selection using file browser
    st.subheader("Select Folder to Sync/Upload")
    st.session_state.folder_path = st.text_input("Folder Path", value=st.session_state.folder_path or DEFAULT_LOCAL_PATH)
    # Alternatively, use a file browser component
    # st.session_state.folder_path = file_browser()

    if os.path.isdir(st.session_state.folder_path):
        # Display folders only
        folders = [f for f in os.listdir(st.session_state.folder_path) if os.path.isdir(os.path.join(st.session_state.folder_path, f))]
        st.session_state.selected_files = st.multiselect(
            "Select Folders to Sync/Upload (leave empty to include all):",
            folders,
            default=st.session_state.get('selected_files', []),
            help="Select specific folders to include."
        )
    else:
        st.error("❌ Invalid folder path.")

    # Sync Controls
    col_start, col_stop = st.columns(2)
    with col_start:
        if st.session_state.sync_option == 'Sync':
            start_sync = st.button("Start Sync", key="start_sync")
        else:
            start_upload = st.button("Start Upload", key="start_upload")
    with col_stop:
        stop_sync = st.button("Stop", key="stop_sync")

    # Handle buttons
    if st.session_state.sync_option == 'Sync':
        if start_sync:
            st.session_state.cancel = False
            if not st.session_state.observer:
                event_handler = ChangeHandler()
                st.session_state.observer = Observer()
                st.session_state.observer.schedule(event_handler, st.session_state.folder_path, recursive=True)
                st.session_state.observer.start()
                log("👀 Started monitoring for changes.", level="INFO")
            log("🔄 Sync started.", level="INFO")
            upload_thread = Thread(target=upload_files)
            upload_thread.start()

        if stop_sync:
            st.session_state.cancel = True
            if st.session_state.observer:
                st.session_state.observer.stop()
                st.session_state.observer.join()
                st.session_state.observer = None
            log("🛑 Sync stopped.", level="INFO")
    else:
        if start_upload:
            st.session_state.cancel = False
            log("🔄 Upload started.", level="INFO")
            upload_thread = Thread(target=upload_files)
            upload_thread.start()

        if stop_sync:
            st.session_state.cancel = True
            log("🛑 Upload stopped.", level="INFO")

    # Display sync status and statistics
    st.subheader("Status")
    if st.session_state.uploading:
        st.info("🚀 Uploading...")
    elif st.session_state.sync_paused:
        st.warning("⏸️ Sync Paused.")
    else:
        st.success("✅ Idle.")

    st.write(f"**Total Files Synced:** {st.session_state.total_files_synced}")
    st.write(f"**Total Files in Folder:** {st.session_state.total_files}")

with tabs[2]:
    st.header("Settings")

    # Securely retrieve token
    if not st.session_state.token:
        stored_token = keyring.get_password("huggingface", "token")
        if stored_token:
            st.session_state.token = stored_token

    st.session_state.token = st.text_input(
        "Hugging Face Token",
        type="password",
        value=st.session_state.token,
        help="Enter your Hugging Face API token. It will be securely stored."
    )

    st.session_state.repo_id = st.text_input(
        "Vault ID (Repository ID)",
        value=st.session_state.get('repo_id', DEFAULT_REPO),
        help="Format: username/repo-name"
    )

    st.session_state.private = st.checkbox(
        "Make Vault Private",
        value=st.session_state.get('private', True),
        help="Private vaults are not publicly accessible."
    )

    st.session_state.subfolder = st.text_input(
        "Subfolder in Vault (Optional)",
        value=st.session_state.get('subfolder', ""),
        help="Specify a subdirectory within the vault."
    )

    st.session_state.process_individually = st.checkbox(
        "Process First-Level Folders Individually",
        value=st.session_state.get('process_individually', False),
        help="Upload each first-level folder individually."
    )

    st.session_state.ignore_patterns_selected = st.multiselect(
        "Select Patterns to Ignore",
        options=list(IGNORE_PATTERNS_MAP.keys()),
        default=st.session_state.get('ignore_patterns_selected', ["Ignore __pycache__", "Ignore .git", "Ignore *.pyc"]),
        help="Select file patterns to exclude."
    )

    save_settings_button = st.button("Save Settings", key="save_settings")
    load_settings_button = st.button("Load Settings", key="load_settings")

    if save_settings_button:
        save_settings()

    if load_settings_button:
        load_settings()

with tabs[3]:
    st.header("Logs")
    # Integrated terminal-like logs (using hypothetical package)
    # st_terminal(st.session_state.logs)
    logs_text = "\n".join(st.session_state.logs[-100:])
    st.text_area("Logs", value=logs_text, height=300)

with tabs[4]:
    st.header("Help and Documentation")
    st.markdown("""
    ### InfiniteStorageFace Documentation

    **Getting Started:**

    - **Vault ID**: This is your repository ID in the format `username/repo-name`. Treat this as your personal storage vault.
    - **Hugging Face Token**: Obtain your API token from your [Hugging Face account settings](https://huggingface.co/settings/tokens).
    - **Folder Selection**: Use the file browser or enter the path to the folder you want to sync or upload.

    **Sync vs Upload:**

    - **Sync**: Continuously monitors the selected folder for changes and syncs them to your vault.
    - **Upload**: Performs a one-time upload of the selected folder or files to your vault.

    **Settings:**

    - **Private Vault**: By default, your vault is private. Only you can access it.
    - **Ignore Patterns**: Select file patterns that you want to exclude from syncing or uploading.

    **Logs and Status:**

    - View real-time logs in the **Logs** tab.
    - Check the sync status and statistics in the **Vault** tab.

    **Support:**

    - For any issues or questions, please refer to the official documentation or contact support.

    """)

# Check queued uploads
check_queued_uploads()

# Cleanup on exit
def cleanup():
    """Cleanup observers and threads on exit."""
    if st.session_state.observer is not None:
        st.session_state.observer.stop()
        st.session_state.observer.join()

# Run scheduled sync if enabled
if st.session_state.scheduled_sync:
    schedule_sync()
    if 'scheduler_thread' not in st.session_state:
        scheduler_thread = Thread(target=run_scheduler, daemon=True)
        scheduler_thread.start()
        st.session_state['scheduler_thread'] = scheduler_thread
        log("🕒 Scheduler started.", level="INFO")

# Handle session end
# st.on_session_end(cleanup)