p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] convert_wandb_logs_to_tb.py (python)
@tyro.conf.configure(tyro.conf.FlagConversionOff)
def main(
    run_ids: Optional[List[str]] = tyro.conf.arg(
        default=None,
        help="List of wandb run IDs to download"
    ),
    project: Optional[str] = tyro.conf.arg(
        default=None,
        help="Wandb project name (optional)"
    ),
    output_dir: str = tyro.conf.arg(
        default="./logs",
        help="Output directory for TensorBoard logs"
    ),
    list_runs: bool = tyro.conf.arg(
        default=False,
        help="List available runs instead of downloading"
    ),
    limit: int = tyro.conf.arg(
        default=50,
        help="Limit number of runs to list"
    ),
) -> None:
    """Download wandb runs and convert to TensorBoard format."""
[content] convert_wandb_logs_to_tb.py (python)
@dataclass
class Args:
    # Run selection
    run_ids: Optional[List[str]] = None
    project: Optional[str] = None
    
    # Output settings
    output_dir: str = "./logs"
    
    # Listing options
    list_runs: bool = False
    limit: int = 50
[content] convert_wandb_logs_to_tb.py (python)
@tyro.conf.configure(tyro.conf.FlagConversionOff)
def main(
    run_ids: Optional[List[str]] = tyro.conf.arg(
        default=None,
        help="List of wandb run IDs to download"
    ),
    project: Optional[str] = tyro.conf.arg(
        default=None,
        help="Wandb project name (optional)"
    ),
    output_dir: str = tyro.conf.arg(
        default="./logs",
        help="Output directory for TensorBoard logs"
    ),
    list_runs: bool = tyro.conf.arg(
        default=False,
        help="List available runs instead of downloading"
    ),
    limit: int = tyro.conf.arg(
        default=50,
        help="Limit number of runs to list"
    ),
) -> None:
[tab] slurm/dev/mihir/horeka/train_tokenizer.sh (shellscript)
#!/usr/bin/env bash

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name="debug"
slurm_job_id="0000"

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=12 \
    --min_lr=4.24e-4 \
    --max_lr=4.24e-4 \
    --log_image_interval=3 \
    --log \
    --name=test-wandb-tags-$slurm_job_id \
    --tags test tokenizer debug \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
[content] convert_wandb_logs_to_tb.py (python)
    
    def get_all_run_ids(self, project_name: str) -> List[str]:
        """
        Get all run IDs from a project.
        
        Args:
            project_name: Project name
            
        Returns:
            List of run IDs
        """
        try:
            api = wandb.Api()
            runs = api.runs(project_name)
            run_ids = [run.id for run in runs]
            logger.info(f"Found {len(run_ids)} runs in project {project_name}")
            return run_ids
            
        except Exception as e:
            logger.error(f"Error getting run IDs: {e}")
            return []
[content] convert_wandb_logs_to_tb.py (python)
        print("Please specify one of the following options:")
        print("  --all-runs: Download all runs from the project")
        print("  --run-ids <id1> <id2> ...: Download specific run IDs")
        print("  --list-runs: List available runs")
[content] convert_wandb_logs_to_tb.py (python)
    elif args.all_runs:
        logger.info(f"Getting all run IDs from project {args.project}...")
        all_run_ids = converter.get_all_run_ids(args.project)
        
        if all_run_ids:
            logger.info(f"Downloading and converting all {len(all_run_ids)} runs...")
            converted_paths = converter.download_and_convert(all_run_ids, args.project)
            
            if converted_paths:
                print(f"
Successfully converted {len(converted_paths)} runs:")
                for path in converted_paths:
                    print(f"  {path}")
                
                print(f"
To view with TensorBoard, run:")
                print(f"tensorboard --logdir {args.output_dir}")
            else:
                print("No runs were successfully converted.")
        else:
            print("No runs found in the project.")
    
    elif args.run_ids and args.run_ids != [""]:
[content] convert_wandb_logs_to_tb.py (python)
            # Convert wandb summary to a serializable format
            summary_dict = {}
            for key, value in run_data['summary'].items():
                try:
                    # Try to convert to a basic type
                    if hasattr(value, '__dict__'):
                        summary_dict[key] = str(value)
                    else:
                        summary_dict[key] = value
                except:
                    summary_dict[key] = str(value)
            
            summary_text = json.dumps(summary_dict, indent=2, default=str)
[content] convert_wandb_logs_to_tb.py (python)
                            # Clean column name for TensorBoard (remove special characters)
                            clean_column = column.replace('/', '_').replace('\\', '_').replace(' ', '_')
                            
                            try:
                                if isinstance(value, (int, float)):
                                    tf.summary.scalar(clean_column, float(value), step=int(step))
                                elif isinstance(value, str):
                                    tf.summary.text(clean_column, value, step=int(step))
                                # Add more data type handlers as needed
                            except Exception as e:
                                logger.warning(f"Could not log {clean_column}: {e}")
                                continue
[content] convert_wandb_logs_to_tb.py (python)
                            if isinstance(value, (int, float)):
                                tf.summary.scalar(column, value, step=step)
                            elif isinstance(value, str):
                                tf.summary.text(column, value, step=step)
                            # Add more data type handlers as needed
[content] convert_wandb_logs_to_tb.py (python)
            try:
                config_text = json.dumps(run_data['config'], indent=2, default=str)
                with writer.as_default():
                    tf.summary.text("config", config_text, step=0)
            except Exception as e:
                logger.warning(f"Could not serialize config: {e}")
                # Try to convert config to string representation
                config_str = str(run_data['config'])
                with writer.as_default():
                    tf.summary.text("config", config_str, step=0)