init

.gitignore

@@ -0,0 +1,2 @@
+wandb/
+script.sh

arguments.yaml

@@ -0,0 +1,49 @@
+data_cfgs:
+  eval_data_files: null
+  eval_datasets: null
+  eval_optional_args: []
+  eval_size: null
+  eval_split: null
+  eval_subset: null
+  eval_template: null
+  train_data_files: 8713_tokenized_v2.pt
+  train_datasets: /aifs4su/yaodong/datasets/llava_annotated
+  train_optional_args: []
+  train_size: null
+  train_split: train
+  train_subset: null
+  train_template: ANYTHING_TI2TI
+logger_cfgs:
+  cache_dir: null
+  log_project: align-anything
+  log_run_name: sft
+  log_type: wandb
+  output_dir: ../outputs/sft_chameleon_0727_0802_v2.1_1e-4
+  save_interval: 500.0
+model_cfgs:
+  model_max_length: 4096
+  model_name_or_path: /aifs4su/yaodong/projects/hantao/anole/facilitating_image_generation/model/chameleon_hf_0727
+  trust_remote_code: true
+special_tokens: null
+train_cfgs:
+  adam_betas:
+  - 0.9
+  - 0.95
+  adam_epsilon: 1.0e-08
+  bf16: true
+  ds_cfgs: ds_z3_config.json
+  epochs: 3
+  eval_interval: 1000
+  eval_strategy: steps
+  fp16: false
+  freeze_language_model: false
+  gradient_accumulation_steps: 2.0
+  gradient_checkpointing: true
+  learning_rate: 0.0001
+  lr_scheduler_type: cosine
+  lr_warmup_ratio: 0.03
+  max_grad_norm: 1.0
+  per_device_eval_batch_size: 2.0
+  per_device_train_batch_size: 2.0
+  seed: 42
+  weight_decay: 0.0

environ.txt

@@ -0,0 +1,166 @@
+BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
+ if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
+ case "$-" in 
+ *v*x*)
+ set +vx;
+ _mlshdbg='vx'
+ ;;
+ *v*)
+ set +v;
+ _mlshdbg='v'
+ ;;
+ *x*)
+ set +x;
+ _mlshdbg='x'
+ ;;
+ *)
+ _mlshdbg=''
+ ;;
+ esac;
+ fi;
+ unset _mlre _mlIFS;
+ if [ -n "${IFS+x}" ]; then
+ _mlIFS=$IFS;
+ fi;
+ IFS=' ';
+ for _mlv in ${MODULES_RUN_QUARANTINE:-};
+ do
+ if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
+ if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
+ _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
+ fi;
+ _mlrv="MODULES_RUNENV_${_mlv}";
+ _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
+ fi;
+ done;
+ if [ -n "${_mlre:-}" ]; then
+ eval `eval ${_mlre} /usr/bin/tclsh /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl bash '"$@"'`;
+ else
+ eval `/usr/bin/tclsh /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl bash "$@"`;
+ fi;
+ _mlstatus=$?;
+ if [ -n "${_mlIFS+x}" ]; then
+ IFS=$_mlIFS;
+ else
+ unset IFS;
+ fi;
+ unset _mlre _mlv _mlrv _mlIFS;
+ if [ -n "${_mlshdbg:-}" ]; then
+ set -$_mlshdbg;
+ fi;
+ unset _mlshdbg;
+ return $_mlstatus
+}
+BASH_FUNC_ml%%=() {  module ml "$@"
+}
+BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
+}
+BASH_FUNC_switchml%%=() {  typeset swfound=1;
+ if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
+ typeset swname='main';
+ if [ -e /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl ]; then
+ typeset swfound=0;
+ unset MODULES_USE_COMPAT_VERSION;
+ fi;
+ else
+ typeset swname='compatibility';
+ if [ -e /cm/local/apps/environment-modules/4.5.3/libexec/modulecmd-compat ]; then
+ typeset swfound=0;
+ MODULES_USE_COMPAT_VERSION=1;
+ export MODULES_USE_COMPAT_VERSION;
+ fi;
+ fi;
+ if [ $swfound -eq 0 ]; then
+ echo "Switching to Modules $swname version";
+ source /cm/local/apps/environment-modules/4.5.3/init/bash;
+ else
+ echo "Cannot switch to Modules $swname version, command not found";
+ return 1;
+ fi
+}
+CMD_WLM_CLUSTER_NAME=slurm
+CONDA_DEFAULT_ENV=hantao_new
+CONDA_EXE=/aifs4su/yaodong/miniconda3/bin/conda
+CONDA_MKL_INTERFACE_LAYER_BACKUP=
+CONDA_PREFIX=/aifs4su/yaodong/miniconda3/envs/hantao_new
+CONDA_PREFIX_1=/aifs4su/yaodong/miniconda3
+CONDA_PROMPT_MODIFIER=(hantao_new) 
+CONDA_PYTHON_EXE=/aifs4su/yaodong/miniconda3/bin/python
+CONDA_SHLVL=2
+CPATH=/cm/shared/apps/slurm/current/include
+CPATH_modshare=/cm/shared/apps/slurm/current/include:1
+CROSS_RANK=0
+CROSS_SIZE=1
+CUDA_HOME=/aifs4su/yaodong/miniconda3/envs/hantao_new
+CUDA_MODULE_LOADING=LAZY
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1028/bus
+ENABLE_LMOD=0
+GSETTINGS_SCHEMA_DIR=/aifs4su/yaodong/miniconda3/envs/hantao_new/share/glib-2.0/schemas
+GSETTINGS_SCHEMA_DIR_CONDA_BACKUP=
+HOME=/home/yangyaodong
+LANG=C.UTF-8
+LD_LIBRARY_PATH=/usr/mpi/gcc/openmpi-4.1.7a1/lib:/usr/local/cuda/lib64:/usr/local/cuda/lib64:/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64
+LD_LIBRARY_PATH_modshare=/cm/shared/apps/slurm/current/lib64:1:/usr/mpi/gcc/openmpi-4.1.7a1/lib:1:/cm/shared/apps/slurm/current/lib64/slurm:1:/usr/local/cuda/lib64:1
+LD_RUN_PATH=/usr/mpi/gcc/openmpi-4.1.7a1/lib
+LD_RUN_PATH_modshare=/usr/mpi/gcc/openmpi-4.1.7a1/lib:1
+LESSCLOSE=/usr/bin/lesspipe %s %s
+LESSOPEN=| /usr/bin/lesspipe %s
+LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64
+LIBRARY_PATH_modshare=/cm/shared/apps/slurm/current/lib64:1:/cm/shared/apps/slurm/current/lib64/slurm:1
+LOADEDMODULES=slurm/slurm/23.02.6:gcc/64/4.1.7a1
+LOADEDMODULES_modshare=slurm/slurm/23.02.6:1:gcc/64/4.1.7a1:1
+LOCAL_RANK=0
+LOCAL_SIZE=8
+LOGLEVEL=WARNING
+LOGNAME=yangyaodong
+LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
+MANPATH=/usr/mpi/gcc/openmpi-4.1.7a1/share/man:/cm/shared/apps/slurm/current/man:/cm/local/apps/environment-modules/4.5.3/share/man:/usr/local/man:/usr/local/share/man:/usr/share/man:/cm/local/apps/environment-modules/current/share/man:/cm/local/apps/environment-modules/current/share/man
+MANPATH_modshare=/usr/local/share/man:1:/usr/mpi/gcc/openmpi-4.1.7a1/share/man:1:/cm/local/apps/environment-modules/current/share/man:1:/cm/local/apps/environment-modules/4.5.3/share/man:1:/usr/local/man:1:/usr/share/man:1:/cm/shared/apps/slurm/current/man:1
+MASTER_ADDR=127.0.0.1
+MASTER_PORT=50216
+MIG_PARTED_CHECKPOINT_FILE=/var/lib/nvidia-mig-manager/checkpoint.json
+MIG_PARTED_CONFIG_FILE=/etc/nvidia-mig-manager/config.yaml
+MIG_PARTED_HOOKS_FILE=/etc/nvidia-mig-manager/hooks.yaml
+MKL_INTERFACE_LAYER=LP64,GNU
+MODULEPATH=/cm/local/modulefiles:/cm/shared/modulefiles
+MODULESHOME=/cm/local/apps/environment-modules/4.5.3
+MODULES_CMD=/cm/local/apps/environment-modules/4.5.3/libexec/modulecmd.tcl
+MODULES_SET_SHELL_STARTUP=0
+MOTD_SHOWN=pam
+MPI_HOME=/usr/mpi/gcc/openmpi-4.1.7a1
+MPI_RUN=/usr/mpi/gcc/openmpi-4.1.7a1/bin/mpirun
+OLDPWD=/aifs4su/yaodong/projects/hantao/dev_cham/align-anything/projects/chameleon
+PATH=/aifs4su/yaodong/google-cloud-sdk/bin:/aifs4su/yaodong/miniconda3/envs/hantao_new/bin:/aifs4su/yaodong/miniconda3/condabin:/usr/mpi/gcc/openmpi-4.1.7a1/bin:/usr/local/cuda/bin:/usr/lpp/mmfs/bin:/usr/local/cuda/bin:/opt/bin:/usr/local/cuda/bin:/usr/lpp/mmfs/bin:/cm/shared/apps/slurm/current/sbin:/cm/shared/apps/slurm/current/bin:/usr/local/cuda/bin:/opt/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/sbin:/usr/sbin:/cm/local/apps/environment-modules/4.5.3/bin
+PATH_modshare=/usr/mpi/gcc/openmpi-4.1.7a1/bin:1:/opt/bin/:1:/usr/bin:1:/usr/local/bin:1:/cm/shared/apps/slurm/current/bin:1:/cm/shared/apps/slurm/current/sbin:1:/bin:1:/snap/bin:1:/sbin:1:/usr/sbin:1:/cm/local/apps/environment-modules/4.5.3/bin:1:/usr/games:1:/usr/local/sbin:1:/usr/lpp/mmfs/bin:1:/usr/local/cuda/bin:1:/usr/local/games:1
+PWD=/aifs4su/yaodong/projects/hantao/dev_cham/align-anything/scripts
+PYTHONHASHSEED=42
+PYTHONPATH=/aifs4su/yaodong/projects/hantao/dev_cham/align-anything
+RANK=0
+SHELL=/bin/bash
+SHLVL=3
+SLURM_CONF=/cm/shared/apps/slurm/var/etc/slurm/slurm.conf
+SSH_CLIENT=10.33.5.36 36286 22
+SSH_CONNECTION=10.33.5.36 36286 10.33.4.214 22
+SSH_TTY=/dev/pts/1
+TERM=screen
+TERM_PROGRAM=tmux
+TERM_PROGRAM_VERSION=3.2a
+TMUX=/tmp/tmux-1028/default,3233837,0
+TMUX_PANE=%0
+TRITON_CACHE_DIR=/aifs4su/yaodong/cache/triton
+USER=yangyaodong
+WANDB_API_KEY=547f38af44135ca76a4f4eed9c8d135532da4960
+WANDB_SERVICE=2-361042-tcp-localhost-58047
+WORLD_SIZE=8
+XDG_DATA_DIRS=/usr/local/share:/usr/share:/var/lib/snapd/desktop
+XDG_RUNTIME_DIR=/run/user/1028
+XDG_SESSION_CLASS=user
+XDG_SESSION_ID=626
+XDG_SESSION_TYPE=tty
+XML_CATALOG_FILES=file:///aifs4su/yaodong/miniconda3/envs/hantao_new/etc/xml/catalog file:///etc/xml/catalog
+_=/aifs4su/yaodong/miniconda3/envs/hantao_new/bin/deepspeed
+_CE_CONDA=
+_CE_M=
+_LMFILES_=/cm/local/modulefiles/slurm/slurm/23.02.6:/cm/local/modulefiles/gcc/64/4.1.7a1
+_LMFILES__modshare=/cm/local/modulefiles/slurm/slurm/23.02.6:1:/cm/local/modulefiles/gcc/64/4.1.7a1:1

preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "crop_size": {
+    "height": 512,
+    "width": 512
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    1.0,
+    1.0,
+    1.0
+  ],
+  "image_processor_type": "ChameleonImageProcessor",
+  "image_std": [
+    1.0,
+    1.0,
+    1.0
+  ],
+  "processor_class": "ChameleonProcessor",
+  "resample": 1,
+  "rescale_factor": 0.0078,
+  "size": {
+    "shortest_edge": 512
+  }
+}

processor_config.json

@@ -0,0 +1,5 @@
+{
+  "image_seq_length": 1024,
+  "image_token": "<image>",
+  "processor_class": "ChameleonProcessor"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:281fb5305fc5d9528c8647f87e2ee4e918af417f338610052d09b3da0cc333fe
+size 14165009930

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "<reserved08706>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}