[tab] logs/logs_training/train_tokenizer_batch_size_scaling_32_node_3292333.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=32
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=logs/logs_training/%x_%j.log
#SBATCH --error=logs/logs_training/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_32_node
#SBATCH --mem=400G

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=$ws_dir/knoms_tfrecords_500_shards
ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1536 \
    --min_lr=3e-4 \
    --max_lr=3e-4 \
    --log_image_interval=250 \
    --log \
    --name=tokenizer-batch-size-scaling-32-node-$slurm_job_id \
    --tags tokenizer batch-size-scaling 32-node \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4(x32)
SLURM_JOB_UID=999226
SLURM_TASK_PID=3324541
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar
SLURMD_NODENAME=hkn0404
SLURM_JOB_START_TIME=1750872539
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1750890539
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24(x32)
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=32
SLURM_JOBID=3292333
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=128
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e11.hkn0404
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_MEM_PER_NODE=409600
SLURM_NODELIST=hkn[0404,0504-0505,0507,0509-0513,0515,0517,0519,0523-0528,0605,0617,0621-0624,0631,0636,0802,0807,0809,0811,0814-0815]
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=128
SLURM_NNODES=32
SLURM_SUBMIT_HOST=hkn1993.localdomain
SLURM_JOB_ID=3292333
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_tokenizer_batch_size_scaling_32_node
SLURM_NTASKS_PER_NODE=4
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn[0404,0504-0505,0507,0509-0513,0515,0517,0519,0523-0528,0605,0617,0621-0624,0631,0636,0802,0807,0809,0811,0814-0815]
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
GpuFreq=control_disabled
2025-06-25 19:29:38.692066: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:38.692150: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:38.692072: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:38.692070: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872578.741316 3006796 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872578.741667 3006794 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872578.741553 3006795 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872578.741697 3006793 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872578.746476 3006794 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872578.746696 3006796 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872578.746932 3006795 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872578.747059 3006793 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2025-06-25 19:29:38.833257: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:38.833156: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:38.833167: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:38.833154: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
W0000 00:00:1750872578.854070 3006793 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854105 3006793 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854107 3006793 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854109 3006793 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854077 3006794 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854107 3006794 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854109 3006794 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854111 3006794 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854074 3006795 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854107 3006795 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854109 3006795 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854111 3006795 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854070 3006796 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854100 3006796 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854102 3006796 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.854104 3006796 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872578.866889 3457572 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872578.867002 3457573 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872578.866842 3457574 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872578.867109 3457571 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872578.872780 3457571 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872578.872782 3457573 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872578.872798 3457574 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872578.873009 3457572 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2025-06-25 19:29:38.938017: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:38.938110: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:38.938016: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:38.938143: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872578.975969 2295737 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872578.976076 2295738 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872578.975971 2295739 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872578.976210 2295736 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
W0000 00:00:1750872578.979616 3457571 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.979644 3457571 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.979645 3457571 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.979647 3457571 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.979763 3457573 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.979782 3457573 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.979784 3457573 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.979786 3457573 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.979642 3457574 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.979666 3457574 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.979668 3457574 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.979670 3457574 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.980074 3457572 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.980138 3457572 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.980146 3457572 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872578.980151 3457572 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
E0000 00:00:1750872578.982055 2295736 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872578.982053 2295737 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872578.982051 2295738 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872578.982053 2295739 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2025-06-25 19:29:39.022135: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.022131: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.022135: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.022132: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.074815 2622598 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.074958 2622599 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.074810 2622600 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.075027 2622601 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872579.087429 2622599 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.087406 2622601 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.087581 2622600 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.087815 2622598 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1750872579.092605 2295736 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092637 2295736 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092639 2295736 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092641 2295736 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092611 2295737 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092640 2295737 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092642 2295737 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092644 2295737 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092605 2295738 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092637 2295738 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092639 2295738 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092641 2295738 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092612 2295739 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092639 2295739 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092641 2295739 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.092642 2295739 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200197 2622598 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200221 2622598 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200223 2622598 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200225 2622598 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200194 2622599 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200220 2622599 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200222 2622599 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200224 2622599 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200191 2622600 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200217 2622600 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200219 2622600 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200221 2622600 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200199 2622601 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200217 2622601 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200219 2622601 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.200221 2622601 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
2025-06-25 19:29:39.272050: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.272054: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.272049: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.272050: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.306474 1475818 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.306417 1475820 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.306581 1475821 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.306879 1475819 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872579.312163 1475818 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.312165 1475820 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.312168 1475821 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.312342 1475819 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2025-06-25 19:29:39.336833: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.336844: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.336830: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.336932: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.356499: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.356590: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.356497: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.356527: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.363865: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.363786: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.363780: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.363787: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.370450: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.370442: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.370443: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.370446: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.385705  234522 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.385710  234523 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.385703  234524 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.385780  234521 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.394183 3324617 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.394258 3324619 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.394485 3324618 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.394600 3324616 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
2025-06-25 19:29:39.395350: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.395353: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.395344: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.395347: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
E0000 00:00:1750872579.399618 3324616 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.399618 3324617 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.399699 3324618 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.399613 3324619 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.402073 3540254 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.402133 3540255 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.402068 3540256 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.402083 3540257 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872579.405154  234521 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.405157  234523 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.405774  234524 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.405837  234522 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.407298 3387921 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.407188 3387922 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.407203 3387924 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.407425 3387923 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872579.413319 3387921 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.413321 3387922 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.413322 3387923 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.413318 3387924 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.428347 3540254 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.428345 3540256 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.428627 3540255 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.428787 3540257 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.429699 3434622 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.429602 3434623 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.429783 3434624 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.429616 3434625 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
W0000 00:00:1750872579.433395 1475819 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.433420 1475819 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.433422 1475819 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.433424 1475819 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.433909 1475818 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.433930 1475818 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.433933 1475818 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.433934 1475818 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.433908 1475820 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.433930 1475820 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.433932 1475820 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.433933 1475820 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.433998 1475821 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.434015 1475821 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.434018 1475821 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.434019 1475821 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
2025-06-25 19:29:39.442603: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.442603: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.442600: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.442605: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
E0000 00:00:1750872579.450077 3434622 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.450080 3434623 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.450090 3434624 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.450182 3434625 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.471813 1702547 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.471882 1702549 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.472201 1702546 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.472126 1702548 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872579.477126 1702546 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.477077 1702547 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.477075 1702549 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.477293 1702548 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1750872579.525595  234521 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525628  234521 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525630  234521 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525632  234521 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525595  234522 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525632  234522 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525634  234522 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525636  234522 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525588  234523 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525627  234523 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525629  234523 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525631  234523 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525593  234524 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525630  234524 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525632  234524 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.525633  234524 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530405 3324616 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530432 3324616 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530434 3324616 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530436 3324616 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530401 3324617 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530427 3324617 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530429 3324617 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530431 3324617 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530396 3324618 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530425 3324618 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530428 3324618 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530430 3324618 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530397 3324619 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530427 3324619 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530429 3324619 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.530431 3324619 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535840 3540254 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535879 3540254 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535882 3540254 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535884 3540254 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535838 3540255 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535872 3540255 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535875 3540255 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535877 3540255 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535834 3540256 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535877 3540256 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535879 3540256 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535881 3540256 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535838 3540257 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535885 3540257 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535887 3540257 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.535889 3540257 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548205 3387921 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548236 3387921 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548238 3387921 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548240 3387921 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548204 3387922 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548232 3387922 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548234 3387922 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548236 3387922 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548206 3387923 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548236 3387923 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548239 3387923 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548241 3387923 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548205 3387924 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548228 3387924 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548230 3387924 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.548232 3387924 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552246 3434622 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552276 3434622 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552278 3434622 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552279 3434622 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552256 3434623 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552285 3434623 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552287 3434623 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552289 3434623 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552248 3434624 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552280 3434624 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552281 3434624 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552283 3434624 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552252 3434625 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552283 3434625 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552285 3434625 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.552287 3434625 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
2025-06-25 19:29:39.558321: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.558321: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.558324: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.558319: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.570457: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.570456: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.570458: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.570456: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
W0000 00:00:1750872579.584516 1702546 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584550 1702546 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584553 1702546 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584555 1702546 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584524 1702547 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584561 1702547 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584564 1702547 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584565 1702547 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584522 1702548 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584559 1702548 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584562 1702548 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584563 1702548 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584516 1702549 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584553 1702549 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584555 1702549 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.584558 1702549 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.598504 1199569 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.598447 1199570 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.598452 1199571 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.598522 1199572 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.605453 3756298 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.605411 3756299 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.605393 3756301 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.605548 3756300 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872579.611922 3756298 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.611924 3756299 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.611923 3756300 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.611919 3756301 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.615083 1199570 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.615078 1199572 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.615334 1199571 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.615490 1199569 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2025-06-25 19:29:39.626919: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.626914: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.626927: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.626923: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.676605  634354 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.676704  634355 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.676594  634356 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.676814  634353 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
2025-06-25 19:29:39.678617: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.678621: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.678697: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.678622: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
E0000 00:00:1750872579.682794  634353 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.682795  634355 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.682790  634356 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.682953  634354 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.729894 3914048 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.729890 3914050 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.729906 3914051 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.730085 3914049 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872579.744164 3914048 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.744164 3914050 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.744165 3914051 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.744560 3914049 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1750872579.748561 3756298 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748584 3756298 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748586 3756298 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748588 3756298 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748558 3756299 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748584 3756299 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748586 3756299 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748588 3756299 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748563 3756300 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748583 3756300 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748585 3756300 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748587 3756300 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748554 3756301 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748583 3756301 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748585 3756301 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.748587 3756301 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756597 1199569 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756626 1199569 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756628 1199569 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756630 1199569 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756594 1199570 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756620 1199570 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756622 1199570 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756624 1199570 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756601 1199571 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756633 1199571 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756635 1199571 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756637 1199571 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756599 1199572 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756630 1199572 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756632 1199572 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.756634 1199572 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
2025-06-25 19:29:39.765695: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.765692: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.765698: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.765760: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.775876: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.775887: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.775878: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.775870: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.800439 3429174 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.800654 3429171 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.800760 3429172 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.800768 3429173 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872579.807346 3429172 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.807313 3429174 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.808125 3429171 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.808189 3429173 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.810519 3068277 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.810609 3068279 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.810527 3068280 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.810717 3068278 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
2025-06-25 19:29:39.812959: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.812963: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.812960: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.812958: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
E0000 00:00:1750872579.816435 3068277 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.816437 3068278 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.816432 3068279 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.816438 3068280 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1750872579.822452  634353 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822473  634353 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822474  634353 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822476  634353 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822466  634354 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822530  634354 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822537  634354 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822541  634354 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822410  634355 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822441  634355 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822442  634355 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822444  634355 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822428  634356 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822476  634356 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822482  634356 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.822488  634356 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
2025-06-25 19:29:39.830542: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.830543: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.830539: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.830784: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.844080: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.844081: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.844083: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.844082: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.850751 2591965 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.850921 2591963 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.851019 2591964 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.851025 2591966 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872579.856410 2591963 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.856424 2591964 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.856408 2591965 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.856410 2591966 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1750872579.860217 3914050 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860244 3914050 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860246 3914050 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860248 3914050 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860219 3914051 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860246 3914051 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860249 3914051 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860250 3914051 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860334 3914048 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860356 3914048 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860358 3914048 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860360 3914048 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860352 3914049 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860371 3914049 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860373 3914049 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.860374 3914049 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.866507  368273 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.866499  368275 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.866621  368276 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.866731  368274 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872579.872538  368273 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.872535  368274 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.872534  368276 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.872706  368275 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.875492  793336 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.875892  793334 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.875776  793337 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.875892  793335 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872579.880809  793334 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.880805  793336 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.880805  793337 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.881046  793335 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2025-06-25 19:29:39.886822: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.886844: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.886916: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.886844: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.901820: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.901705: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.901707: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:39.901700: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
W0000 00:00:1750872579.908931 3429171 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908958 3429171 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908960 3429171 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908961 3429171 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908933 3429172 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908963 3429172 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908966 3429172 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908967 3429172 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908933 3429173 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908960 3429173 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908962 3429173 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908963 3429173 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908934 3429174 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908964 3429174 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908966 3429174 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.908967 3429174 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.922607 2368643 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.922492 2368644 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.922516 2368646 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.922725 2368645 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
W0000 00:00:1750872579.924695 3068277 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924733 3068277 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924737 3068277 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924740 3068277 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924691 3068278 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924716 3068278 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924718 3068278 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924720 3068278 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924689 3068279 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924721 3068279 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924724 3068279 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924725 3068279 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924692 3068280 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924726 3068280 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924732 3068280 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.924734 3068280 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
E0000 00:00:1750872579.928279 2368644 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.928279 2368645 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.928276 2368646 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.928502 2368643 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.930859 1154496 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.930756 1154498 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.930701 1154499 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872579.930894 1154497 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872579.944420 1154499 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.944680 1154496 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.944676 1154497 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872579.944894 1154498 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1750872579.982905 2591963 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982932 2591963 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982934 2591963 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982935 2591963 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982909 2591964 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982939 2591964 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982941 2591964 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982942 2591964 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982904 2591965 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982935 2591965 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982938 2591965 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982939 2591965 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982900 2591966 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982930 2591966 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982932 2591966 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872579.982934 2591966 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007155  368273 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007235  368273 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007243  368273 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007249  368273 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007168  368274 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007313  368274 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007323  368274 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007328  368274 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007168  368275 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007289  368275 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007298  368275 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007303  368275 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007174  368276 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007252  368276 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007259  368276 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.007264  368276 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025353  793334 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025436  793334 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025443  793334 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025448  793334 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025366  793335 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025469  793335 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025471  793335 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025473  793335 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025344  793336 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025429  793336 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025437  793336 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025442  793336 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025357  793337 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025472  793337 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025475  793337 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.025476  793337 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029806 2368643 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029834 2368643 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029836 2368643 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029838 2368643 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029812 2368644 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029839 2368644 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029841 2368644 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029842 2368644 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029806 2368645 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029836 2368645 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029838 2368645 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029840 2368645 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029814 2368646 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029840 2368646 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029842 2368646 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.029843 2368646 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
2025-06-25 19:29:40.057698: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.057804: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.057848: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.057909: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
W0000 00:00:1750872580.062493 1154496 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062523 1154496 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062525 1154496 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062526 1154496 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062490 1154497 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062526 1154497 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062528 1154497 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062530 1154497 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062491 1154498 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062521 1154498 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062524 1154498 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062525 1154498 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062490 1154499 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062518 1154499 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062520 1154499 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.062522 1154499 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.097068  172360 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.097075  172361 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.097179  172362 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.097174  172363 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872580.103155  172360 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.103157  172361 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.103154  172362 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.103158  172363 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2025-06-25 19:29:40.164289: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.164292: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.164286: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.164294: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.179137: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.179141: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.179139: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.179137: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.202697 3592132 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.202926 3592134 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.203008 3592131 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.202908 3592133 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872580.207964 3592134 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.208148 3592131 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.207965 3592132 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.208304 3592133 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2025-06-25 19:29:40.209441: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.209443: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.209866: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.209731: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.210279: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.210187: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.210184: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.210185: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.216617 3659729 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.216558 3659730 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.216721 3659727 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.216923 3659728 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
W0000 00:00:1750872580.221418  172360 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221451  172360 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221453  172360 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221455  172360 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221421  172361 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221456  172361 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221458  172361 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221460  172361 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221415  172362 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221452  172362 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221455  172362 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221456  172362 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221420  172363 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221461  172363 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221463  172363 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.221465  172363 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
E0000 00:00:1750872580.222046 3659727 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.222108 3659729 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.222050 3659730 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.222278 3659728 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.239952 2003850 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.239841 2003851 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.239942 2003852 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.239834 2003853 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.242506 1422709 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.242501 1422710 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.242701 1422711 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.242579 1422712 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872580.248607 1422709 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.248605 1422710 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.248609 1422711 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.248614 1422712 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.257456 2003850 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.257460 2003851 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.257457 2003852 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.257455 2003853 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2025-06-25 19:29:40.287593: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.287586: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.287700: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.287587: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.320054 3165841 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.320122 3165843 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.320235 3165842 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.320327 3165844 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
2025-06-25 19:29:40.321070: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.321068: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.321064: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.321064: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
E0000 00:00:1750872580.325783 3165841 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.325785 3165842 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.325789 3165843 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.325781 3165844 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1750872580.327141 3592131 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327167 3592131 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327169 3592131 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327171 3592131 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327147 3592132 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327173 3592132 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327176 3592132 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327177 3592132 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327144 3592133 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327171 3592133 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327173 3592133 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327175 3592133 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327140 3592134 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327173 3592134 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327175 3592134 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.327177 3592134 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356067 2003850 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356096 2003850 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356098 2003850 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356100 2003850 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356065 2003851 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356093 2003851 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356095 2003851 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356096 2003851 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356063 2003852 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356091 2003852 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356094 2003852 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356095 2003852 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356063 2003853 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356093 2003853 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356095 2003853 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.356096 2003853 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.357135 3940704 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.357202 3940706 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.357480 3940705 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.357594 3940707 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
W0000 00:00:1750872580.360547 3659727 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360576 3659727 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360578 3659727 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360580 3659727 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360555 3659728 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360580 3659728 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360582 3659728 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360583 3659728 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360551 3659729 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360579 3659729 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360581 3659729 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360582 3659729 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360558 3659730 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360579 3659730 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360580 3659730 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.360582 3659730 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
E0000 00:00:1750872580.362751 3940704 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.362750 3940705 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.362783 3940706 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.362782 3940707 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1750872580.369092 1422709 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369121 1422709 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369123 1422709 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369124 1422709 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369096 1422710 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369126 1422710 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369129 1422710 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369130 1422710 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369092 1422711 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369130 1422711 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369132 1422711 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369133 1422711 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369095 1422712 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369126 1422712 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369128 1422712 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.369130 1422712 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
2025-06-25 19:29:40.432783: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.432790: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.432787: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.432781: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
W0000 00:00:1750872580.438216 3165841 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438240 3165841 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438244 3165841 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438247 3165841 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438213 3165842 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438240 3165842 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438242 3165842 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438244 3165842 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438210 3165843 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438239 3165843 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438241 3165843 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438243 3165843 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438219 3165844 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438239 3165844 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438241 3165844 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.438243 3165844 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.474600 3710320 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.474558 3710321 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.474536 3710322 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.474838 3710323 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
W0000 00:00:1750872580.479493 3940704 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479516 3940704 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479518 3940704 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479520 3940704 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479492 3940705 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479514 3940705 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479516 3940705 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479518 3940705 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479495 3940706 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479515 3940706 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479517 3940706 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479518 3940706 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479488 3940707 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479513 3940707 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479516 3940707 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.479518 3940707 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
E0000 00:00:1750872580.480510 3710320 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.480513 3710321 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.480511 3710322 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.480511 3710323 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2025-06-25 19:29:40.492247: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.492247: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.492249: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.492248: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.527349: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.527397: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.527351: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-06-25 19:29:40.527477: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.535575 1764602 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.535622 1764603 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.535718 1764601 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.535822 1764604 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872580.541672 1764601 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.541672 1764602 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.541671 1764603 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.541668 1764604 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.567531 3602515 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.567610 3602513 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.567948 3602514 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1750872580.568040 3602516 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1750872580.585290 3602514 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.585359 3602515 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.585368 3602516 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1750872580.585457 3602513 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1750872580.601619 3710320 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601647 3710320 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601650 3710320 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601652 3710320 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601622 3710321 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601650 3710321 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601653 3710321 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601655 3710321 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601619 3710322 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601647 3710322 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601649 3710322 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601651 3710322 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601615 3710323 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601645 3710323 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601648 3710323 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.601650 3710323 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644510 1764601 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644550 1764601 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644552 1764601 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644554 1764601 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644507 1764602 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644548 1764602 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644550 1764602 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644552 1764602 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644589 1764603 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644612 1764603 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644614 1764603 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644615 1764603 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644549 1764604 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644573 1764604 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644575 1764604 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.644576 1764604 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691674 3602513 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691697 3602513 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691699 3602513 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691701 3602513 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691628 3602514 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691679 3602514 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691686 3602514 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691691 3602514 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691612 3602515 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691643 3602515 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691644 3602515 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691646 3602515 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691617 3602516 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691683 3602516 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691690 3602516 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872580.691695 3602516 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1750872599.008844 1475818 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.008785 1475819 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.008692 1475820 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.008687 1475821 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.008997  793334 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.008995  793336 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.009129  793335 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.009167  793337 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.009295 3068277 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.009273 3068278 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.009294 3068279 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.009282 3068280 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.020850 3006793 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.020898 3006794 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.020837 3006795 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.020991 3006796 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.027088  172360 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.027085  172362 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.027097  172363 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.027210  172361 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.050488 3659727 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.050478 3659729 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.050628 3659728 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.050639 3659730 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.058637 3602513 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.058649 3602514 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.058640 3602515 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.058634 3602516 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.064580 1199569 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.064572 1199570 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.064575 1199571 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.064585 1199572 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.069417 2591963 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.069394 2591965 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.069396 2591966 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.069543 2591964 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.075096  368274 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.075465  368273 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.075456  368275 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.075534  368276 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.080007 2295736 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.080013 2295737 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.080010 2295738 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.080011 2295739 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.080699 3914048 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.080688 3914049 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.080688 3914050 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.080806 3914051 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.084142 1702546 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.084147 1702547 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.084137 1702548 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.084141 1702549 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.091944 2368645 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.092092 2368643 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.092194 2368644 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.092223 2368646 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.094077 3940705 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.094349 3940704 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.094314 3940706 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.094264 3940707 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.095971 2003850 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.095983 2003851 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.095965 2003852 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.095993 2003853 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.099905 1422710 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.099902 1422712 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.099980 1422711 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.100052 1422709 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.109359  634353 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.109368  634354 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.109356  634356 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.109497  634355 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.109712 3540254 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.109690 3540255 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.109714 3540256 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.109700 3540257 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.116668 3457571 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.116696 3457572 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.116668 3457573 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.116676 3457574 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.118565 2622598 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.119181 2622600 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.119087 2622601 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.119196 2622599 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.124529 1154497 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.124934 1154496 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.124857 1154498 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.124904 1154499 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.127184 3429171 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.127194 3429173 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.127277 3429172 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.127291 3429174 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.127896 3710320 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.127832 3592132 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.127797 3592133 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.127975 3592131 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.127922 3592134 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.127894 3710322 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.128051 3710321 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.128079 3710323 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.130642 3756298 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.130839 3165841 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.130639 3756300 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.130801 3165842 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.130634 3756301 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.130814 3165843 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.130810 3165844 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.130817 3756299 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.131401 3434622 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.131325 3434623 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.131404 3434624 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.131437 3434625 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.131753 3324616 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.131754 3324617 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.131753 3324619 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.132081 3324618 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.136156 3387921 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.136220 3387922 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.136148 3387923 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.136192 3387924 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.142961  234521 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.143002  234522 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.142971  234523 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.143019  234524 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.267918 1764601 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.267915 1764602 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.267933 1764603 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1750872599.267923 1764604 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: Tracking run with wandb version 0.19.11
wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/wandb/run-20250625_193002-024yh92w
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run tokenizer-batch-size-scaling-32-node-3292333
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/024yh92w
2025-06-25 19:42:18.275175: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.275663: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.275916: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.276626: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.276066: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.276068: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.278109: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.002285063s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.281923: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.281931: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.281954: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.283461: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.285322: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.285591: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.286438: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.286441: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.286439: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.287820: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.289325: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.013424518s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.291342: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.015348701s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.291492: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.292055: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.291970: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.291837: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.292304: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.292350: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.292986: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.294091: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.012297646s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.294899: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.009686669s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.296152: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.298689: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.298850: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.298781: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.299079: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.299103: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.299369: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.006996676s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.301108: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.019293108s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.300937: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.301566: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.018232557s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.301676: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.019822176s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.303321: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.307387: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.307281: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.307408: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.308110: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.004087424s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.308756: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.004736411s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.309987: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.013999581s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.310671: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.311277: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.311239: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.311284: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.311257: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.311282: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.311366: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.311378: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.311920: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.312045: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.312059: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.312140: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.312979: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.01967992s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.312930: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.00524802s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.313128: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.005378496s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.313271: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.313164: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.313274: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.313358: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.314836: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.039783139s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.315253: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.029765143s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.315950: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.039496674s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.316309: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.318738: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.008635537s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.319578: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.321344: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.0109651s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.322096: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.322781: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.012752451s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.325014: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.325431: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.327439: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.327968: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.328613: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.034015242s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.327920: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.328273: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.328448: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.328044: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.328708: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.328460: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.328219: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.328471: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.328766: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.00084552s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.329304: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.328757: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.329565: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.03460028s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.330972: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.04875086s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.331467: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.036783322s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.331978: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.332285: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.00401438s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.331955: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.331966: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.332387: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.040426005s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.333298: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.333458: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.333560: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.333560: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.334191: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.005585967s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.335125: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.335508: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.335615: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.007493667s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.335613: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.007072337s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.336980: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.045579069s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.337263: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.026987081s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.337372: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.002343567s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.337220: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.00889263s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.337823: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.021640893s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.337797: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.045616818s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.337689: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.002275725s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.338232: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.046025894s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.337626: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.008706738s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.339105: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.055272388s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.339334: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.031110862s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.339627: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.051898161s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.340958: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.031559194s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.340811: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.342378: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.013150744s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.342832: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.014652287s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.342691: E external/xla/xla/service/slow_operation_alarm.cc:73] Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.344312: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.019400408s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.346270: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.036643443s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.346381: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.037062972s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.348185: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.019840843s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.348281: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.01907383s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.349650: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.042119745s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.453134: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.142666564s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.459825: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.140360414s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.476285: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.149073754s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.477267: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.189275724s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.477974: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.18514555s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.479773: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.157892588s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.490278: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.199291587s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.493597: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.218054907s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.495897: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.170905585s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.496176: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.171099907s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.499306: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.174398549s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.499620: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.157039398s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.500268: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.199465244s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.512213: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.172006363s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.513344: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.209569198s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.520808: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.212775802s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.525473: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.214789798s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.529912: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.202134962s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.530592: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.202654105s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.537656: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.23442036s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.541883: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.259389154s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.546680: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.21883327s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.548312: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.222961007s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.548224: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.240113395s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.548244: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.239715208s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
2025-06-25 19:42:18.550167: E external/xla/xla/service/slow_operation_alarm.cc:140] The operation took 1.221963685s
Trying algorithm eng3{k11=0} for conv %cudnn-conv = (f32[51840,1,150]{2,1,0}, u8[0]{0}) custom-call(%bitcast.1664, %bitcast.1665), window={size=11}, dim_labels=bf0_oi0->bf0, operand_precision={highest,highest}, custom_call_target="__cudnn$convForward", metadata={op_name="jit(train_step)/jit(main)/jvp(vmap(jit(convolve)))/jit(_conv)/conv_general_dilated" source_file="/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/dm_pix/_src/metrics.py" source_line=234}, backend_config={"operation_queue_id":"0","wait_on_operation_queues":[],"cudnn_conv_backend_config":{"conv_result_scale":1,"activation_mode":"kNone","side_input_scale":0,"leakyrelu_alpha":0},"force_earliest_schedule":false,"reification_cost":[]} is taking a while...
srun: Job step aborted: Waiting up to 32 seconds for job step to finish.
slurmstepd: error: *** JOB 3292333 ON hkn0404 CANCELLED AT 2025-06-26T00:29:01 DUE TO TIME LIMIT ***
slurmstepd: error: *** STEP 3292333.0 ON hkn0404 CANCELLED AT 2025-06-26T00:29:01 DUE TO TIME LIMIT ***
srun: got SIGCONT
srun: forcing job termination

============================= JOB FEEDBACK =============================

Job ID: 3292333
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: TIMEOUT (exit code 0)
Partition: accelerated
Nodes: 32
Cores per node: 24
Nodelist: hkn[0404,0504-0505,0507,0509-0513,0515,0517,0519,0523-0528,0605,0617,0621-0624,0631,0636,0802,0807,0809,0811,0814-0815]
CPU Utilized: 34-07:31:10
CPU Efficiency: 21.44% of 160-00:25:36 core-walltime
Job Wall-clock time: 05:00:02
Starttime: Wed Jun 25 19:28:59 2025
Endtime: Thu Jun 26 00:29:01 2025
Memory Utilized: 1.15 TB (estimated maximum)
Memory Efficiency: 9.20% of 12.50 TB (400.00 GB/node)
Energy Consumed: 423577740 Joule / 117660.483333333 Watthours
Average node power draw: 23529.4822797467 Watt