# pip install -v -U git+https://github.com/facebookresearch/xformers.git@v0.0.22 && \
# git clone https://github.com/NVIDIA/apex.git && cd apex && pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ && cd .. && rm -rf apex && 
# pip install -U ./torchscale ./open_clip ./fairseq ./infinibatch && \
python3 -m torch.distributed.launch --nproc_per_node=1 --nnodes=1 \
  app.py None \
  --task kosmosg \
  --criterion kosmosg \
  --arch kosmosg_xl \
  --required-batch-size-multiple 1 \
  --dict-path data/dict.txt \
  --spm-model data/sentencepiece.bpe.model \
  --memory-efficient-fp16 \
  --ddp-backend=no_c10d \
  --distributed-no-spawn \
  --subln \
  --sope-rel-pos \
  --checkpoint-activations \
  --flash-attention \
  --pretrained-ckpt-path ./kosmosg_checkpoints/checkpoint_final.pt