sync_data.sh

#!/bin/bash

# 检查环境变量
if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
    echo "Starting without backup functionality - missing HF_TOKEN or DATASET_ID"
    exec python main.py
fi

# 登录HuggingFace (使用环境变量方式避免交互问题)
export HUGGING_FACE_HUB_TOKEN=$HF_TOKEN

# 同步函数
sync_data() {
    while true; do
        echo "Starting sync process at $(date)"
        
        # 创建临时压缩文件
        cd /app
        timestamp=$(date +%Y%m%d_%H%M%S)
        backup_file="backup_${timestamp}.tar.gz"
        
        tar -czf "/tmp/${backup_file}" data/
        
        python3 -c "
from huggingface_hub import HfApi
import os
def manage_backups(api, repo_id, max_files=50):
    files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')
    backup_files = [f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')]
    backup_files.sort()
    
    if len(backup_files) >= max_files:
        files_to_delete = backup_files[:(len(backup_files) - max_files + 1)]
        for file_to_delete in files_to_delete:
            try:
                api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type='dataset')
                print(f'Deleted old backup: {file_to_delete}')
            except Exception as e:
                print(f'Error deleting {file_to_delete}: {str(e)}')
try:
    api = HfApi()
    api.upload_file(
        path_or_fileobj='/tmp/${backup_file}',
        path_in_repo='${backup_file}',
        repo_id='${DATASET_ID}',
        repo_type='dataset'
    )
    print('Backup uploaded successfully')
    
    manage_backups(api, '${DATASET_ID}')
except Exception as e:
    print(f'Backup failed: {str(e)}')
"
        # 清理临时文件
        rm -f "/tmp/${backup_file}"
        
        # 设置同步间隔
        SYNC_INTERVAL=${SYNC_INTERVAL:-7200}
        echo "Next sync in ${SYNC_INTERVAL} seconds..."
        sleep $SYNC_INTERVAL
    done
}

# 恢复函数
restore_latest() {
    echo "Attempting to restore latest backup..."
    python3 -c "
try:
    from huggingface_hub import HfApi
    import os
    
    api = HfApi()
    files = api.list_repo_files('${DATASET_ID}', repo_type='dataset')
    backup_files = [f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')]
    
    if backup_files:
        latest = sorted(backup_files)[-1]
        api.hf_hub_download(
            repo_id='${DATASET_ID}',
            filename=latest,
            repo_type='dataset',
            local_dir='/tmp'
        )
        os.system(f'tar -xzf /tmp/{latest} -C /app')
        os.remove(f'/tmp/{latest}')
        print(f'Restored from {latest}')
    else:
        print('No backup found')
except Exception as e:
    print(f'Restore failed: {str(e)}')
"
}

# 主程序
(
    # 尝试恢复
    restore_latest
    
    # 启动同步进程
    sync_data &
    
    # 启动主应用
    exec python main.py
) 2>&1 | tee -a /app/data/backup.log