p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/jobs/mihir/berlin/crowd-pilot/run_sqwen3-0.6b-sft_dev.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=10
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --mem=600GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft/%x_%j.log
#SBATCH --job-name=run_qwen3-0.6b-sft

cat $0

source ~/.bashrc

micromamba activate miles

sh scripts/run-qwen3-0.6B-base-sft_dev.sh
[terminal_command] TERMINAL ()
import torchTraceback (most recent call last):  File "<stdin>", line 1, in <module>  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/torch/__init__.py", line 415, in <module>    _load_global_deps()  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/torch/__init__.py", line 371, in _load_global_deps    raise err  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/torch/__init__.py", line 320, in _load_global_deps    ctypes.CDLL(global_deps_lib_path, mode=ctypes.RTLD_GLOBAL)  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/ctypes/__init__.py", line 379, in __init__    self._handle = _dlopen(self._name, mode)                   ^^^^^^^^^^^^^^^^^^^^^^^^^OSError: /home/mihir.mahajan/.local/lib/python3.12/site-packages/torch/lib/libtorch_global_deps.so: cannot open shared object file: No such file or directory
[terminal_command] TERMINAL ()
(miles) [mihir.mahajan@hai005.haicore.berlin:~/Projects/miles] $ conda deactivate[mihir.mahajan@hai005.haicore.berlin:~/Projects/miles] $ micromamba activate miles(miles) [mihir.mahajan@hai005.haicore.berlin:~/Projects/miles] $ pythonPython 3.12.12 | packaged by conda-forge | (main, Oct 22 2025, 23:25:55) [GCC 14.3.0] on linuxType "help", "copyright", "credits" or "license" for more information.Ctrl click to launch VS Code Native REPL
[terminal_command] TERMINAL ()
import torchTraceback (most recent call last):  File "<stdin>", line 1, in <module>  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/torch/__init__.py", line 415, in <module>    _load_global_deps()  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/torch/__init__.py", line 371, in _load_global_deps    raise err  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/torch/__init__.py", line 320, in _load_global_deps    ctypes.CDLL(global_deps_lib_path, mode=ctypes.RTLD_GLOBAL)  File "/opt/miniforge3/lib/python3.12/ctypes/__init__.py", line 379, in __init__    self._handle = _dlopen(self._name, mode)                   ^^^^^^^^^^^^^^^^^^^^^^^^^OSError: /home/mihir.mahajan/.local/lib/python3.12/site-packages/torch/lib/libtorch_global_deps.so: cannot open shared object file: No such file or directory
[terminal_command] TERMINAL ()
(miles) [mihir.mahajan@hai005.haicore.berlin:~/Projects/miles] $ pip uninstall torchFound existing installation: torch 2.8.0+cu129^CERROR: Operation cancelled by user(miles) [mihir.mahajan@hai005.haicore.berlin:~/Projects/miles] $ pip show torchName: torchVersion: 2.8.0+cu129Summary: Tensors and Dynamic neural networks in Python with strong GPU accelerationHome-page: https://pytorch.org/Author: PyTorch TeamAuthor-email: packages@pytorch.orgLicense: BSD-3-ClauseLocation: /fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packagesRequires: filelock, fsspec, jinja2, networkx, nvidia-cublas-cu12, nvidia-cuda-cupti-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-runtime-cu12, nvidia-cudnn-cu12, nvidia-cufft-cu12, nvidia-cufile-cu12, nvidia-curand-cu12, nvidia-cusolver-cu12, nvidia-cusparse-cu12, nvidia-cusparselt-cu12, nvidia-nccl-cu12, nvidia-nvjitlink-cu12, nvidia-nvtx-cu12, setuptools, sympy, triton, typing-extensionsRequired-by: torchaudio, torchvision(miles) [mihir.mahajan@hai005.haicore.berlin:~/Projects/miles] $ pythonPython 3.12.12 | packaged by conda-forge | (main, Oct 22 2025, 23:25:55) [GCC 14.3.0] on linuxType "help", "copyright", "credits" or "license" for more information.Ctrl click to launch VS Code Native REPL
[content] build_conda.sh (shellscript)
# micromamba install -n miles cuda cuda-nvtx cuda-nvtx-dev nccl -c nvidia/label/cuda-12.9.1 -y
# micromamba install -n miles -c conda-forge cudnn -y

# prevent installing cuda 13.0 for sglang
# pip install cuda-python==12.9.1
# pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu129

# install sglang
# git clone https://github.com/sgl-project/sglang.git
cd sglang
git checkout 303cc957e62384044dfa8e52d7d8af8abe12f0ac
# Install the python packages
# pip install -e "python[all]"


# pip install cmake ninja



cd $BASE_DIR
python -c "import torch; print(torch.__version__)"
# git clone https://github.com/Dao-AILab/flash-attention.git
# cd flash-attention/hopper
# python setup.py install
echo "DONE FOR NOW"

# flash attn
# the newest version megatron supports is v2.7.4.post1
# MAX_JOBS=64 pip -v install flash-attn==2.7.4.post1 --no-build-isolation
# pip install https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.3.18/flash_attn-2.7.4%2Bcu128torch2.8-cp312-cp312-linux_x86_64.whl

# pip install git+https://github.com/ISEEKYAN/mbridge.git@89eb10887887bc74853f89a4de258c0702932a1c --no-deps
# pip install --no-build-isolation "transformer_engine[pytorch]==2.8.0" --no-cache-dir
# pip install flash-linear-attention==0.4.0
# NVCC_APPEND_FLAGS="--threads 4" \
  # pip -v install --disable-pip-version-check --no-cache-dir \
  # --no-build-isolation \
  # --config-settings "--build-option=--cpp_ext --cuda_ext --parallel 8" git+https://github.com/NVIDIA/apex.git@10417aceddd7d5d05d7cbf7b0fc2daad1105f8b4

# git clone https://github.com/NVIDIA/Megatron-LM.git --recursive && \
    # cd Megatron-LM && git checkout ${MEGATRON_COMMIT} && \
    # pip install -e .

# pip install git+https://github.com/fzyzcjy/torch_memory_saver.git@9b8b788fdeb9c2ee528183214cef65a99b71e7d5 --no-cache-dir --force-reinstall
# pip install git+https://github.com/fzyzcjy/Megatron-Bridge.git@dev_rl --no-build-isolation
# pip install nvidia-modelopt[torch]>=0.37.0 --no-build-isolation

# megatron
# cd $BASE_DIR
# git clone https://github.com/NVIDIA/Megatron-LM.git --recursive && \
  # cd Megatron-LM/ && git checkout core_v0.14.0 && \
  # pip install -e .

# install miles and apply patches

# if miles does not exist locally, clone it
# if [ ! -d "$BASE_DIR/miles" ]; then
  # cd $BASE_DIR
  # git clone  https://github.com/radixark/miles.git
  # cd miles/
  # export MILES_DIR=$BASE_DIR/miles
  # pip install -e .
# else
  # export MILES_DIR=$BASE_DIR/
  # pip install -e .
# fi

# apply patch
# cd $BASE_DIR/sglang
# git apply $MILES_DIR/docker/patch/v0.5.5.post1/sglang.patch
# cd $BASE_DIR/Megatron-LM
# git apply $MILES_DIR/docker/patch/v0.5.5.post1/megatron.patch
[terminal_command] TERMINAL ()
(miles) [mihir.mahajan@hai005.haicore.berlin:~/Projects/miles] $ python --versionPython 3.12.12(miles) [mihir.mahajan@hai005.haicore.berlin:~/Projects/miles] $ which python/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/bin/python(miles) [mihir.mahajan@hai005.haicore.berlin:~/Projects/miles] $ rm -rf /home/mihir.mahajan/.local/lib/python3.12/site-packages/torch(miles) [mihir.mahajan@hai005.haicore.berlin:~/Projects/miles] $ python --versionPython 3.12.12(miles) [mihir.mahajan@hai005.haicore.berlin:~/Projects/miles] $ pythonPython 3.12.12 | packaged by conda-forge | (main, Oct 22 2025, 23:25:55) [GCC 14.3.0] on linuxType "help", "copyright", "credits" or "license" for more information.Ctrl click to launch VS Code Native REPL
[content] build_conda.sh (shellscript)
set -ex

# create conda
yes '' | "${SHELL}" <(curl -L micro.mamba.pm/install.sh)
export PS1=tmp
mkdir -p /root/.cargo/
touch /root/.cargo/env
source ~/.bashrc

micromamba create -n miles python=3.12 pip -c conda-forge -y
micromamba activate miles
export CUDA_HOME="$CONDA_PREFIX"

export BASE_DIR=${BASE_DIR:-"/root"}
cd $BASE_DIR

# install cuda 12.9 as it's the default cuda version for torch
micromamba install -n miles cuda cuda-nvtx cuda-nvtx-dev nccl -c nvidia/label/cuda-12.9.1 -y
micromamba install -n miles -c conda-forge cudnn -y

# prevent installing cuda 13.0 for sglang
pip install cuda-python==12.9.1
pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu129

# install sglang
git clone https://github.com/sgl-project/sglang.git
cd sglang
git checkout 303cc957e62384044dfa8e52d7d8af8abe12f0ac
# Install the python packages
pip install -e "python[all]"


pip install cmake ninja

# flash attn
# the newest version megatron supports is v2.7.4.post1
MAX_JOBS=64 pip -v install flash-attn==2.7.4.post1 --no-build-isolation

pip install git+https://github.com/ISEEKYAN/mbridge.git@89eb10887887bc74853f89a4de258c0702932a1c --no-deps
pip install --no-build-isolation "transformer_engine[pytorch]==2.8.0"
pip install flash-linear-attention==0.4.0
NVCC_APPEND_FLAGS="--threads 4" \
  pip -v install --disable-pip-version-check --no-cache-dir \
  --no-build-isolation \
  --config-settings "--build-option=--cpp_ext --cuda_ext --parallel 8" git+https://github.com/NVIDIA/apex.git@10417aceddd7d5d05d7cbf7b0fc2daad1105f8b4

git clone https://github.com/NVIDIA/Megatron-LM.git --recursive && \
    cd Megatron-LM && git checkout ${MEGATRON_COMMIT} && \
    pip install -e .

pip install git+https://github.com/fzyzcjy/torch_memory_saver.git@9b8b788fdeb9c2ee528183214cef65a99b71e7d5 --no-cache-dir --force-reinstall
pip install git+https://github.com/fzyzcjy/Megatron-Bridge.git@dev_rl --no-build-isolation
pip install nvidia-modelopt[torch]>=0.37.0 --no-build-isolation

# megatron
cd $BASE_DIR
git clone https://github.com/NVIDIA/Megatron-LM.git --recursive && \
  cd Megatron-LM/ && git checkout core_v0.14.0 && \
  pip install -e .

# install miles and apply patches

# if miles does not exist locally, clone it
if [ ! -d "$BASE_DIR/miles" ]; then
  cd $BASE_DIR
  git clone  https://github.com/radixark/miles.git
  cd miles/
  export MILES_DIR=$BASE_DIR/miles
  pip install -e .
else
  export MILES_DIR=$BASE_DIR/
  pip install -e .
fi

# apply patch
cd $BASE_DIR/sglang
git apply $MILES_DIR/docker/patch/v0.5.5.post1/sglang.patch
cd $BASE_DIR/Megatron-LM
git apply $MILES_DIR/docker/patch/v0.5.5.post1/megatron.patch
[terminal_command] TERMINAL ()
import torchTraceback (most recent call last):  File "<stdin>", line 1, in <module>  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/torch/__init__.py", line 416, in <module>    from torch._C import *  # noqa: F403    ^^^^^^^^^^^^^^^^^^^^^^ImportError: libcusparseLt.so.0: cannot open shared object file: No such file or directory
[terminal_command] TERMINAL ()
(miles) [mihir.mahajan@hai005.haicore.berlin:/fast/home/mihir.mahajan/Projects/miles] $ export LD_LIBRARY_PATH=$CONDA_PREFIX/lib/python3.12/site-packages/torch/lib:$LD_LIBRARY_PATH(miles) [mihir.mahajan@hai005.haicore.berlin:/fast/home/mihir.mahajan/Projects/miles] $ pythonPython 3.12.12 | packaged by conda-forge | (main, Oct 22 2025, 23:25:55) [GCC 14.3.0] on linuxType "help", "copyright", "credits" or "license" for more information.Ctrl click to launch VS Code Native REPL
[terminal_command] TERMINAL ()
(miles) [mihir.mahajan@hai005.haicore.berlin:/fast/home/mihir.mahajan/Projects/miles] $ cat ~/.bashrc # .bashrc# Source global definitionsif [ -f /etc/bashrc ]; then        . /etc/bashrcfi# User specific environmentif ! [[ "$PATH" =~ "$HOME/.local/bin:$HOME/bin:" ]]then    PATH="$HOME/.local/bin:$HOME/bin:$PATH"fiexport PATH# Uncomment the following line if you don't like systemctl's auto-paging feature:# export SYSTEMD_PAGER=# User specific aliases and functionsif [ -d ~/.bashrc.d ]; then        for rc in ~/.bashrc.d/*; do                if [ -f "$rc" ]; then                        . "$rc"                fi        donefiunset rcexport PATH=$HOME/npm-global/bin:$PATHexport PROJECT="/fast/project/HFMI_SynergyUnit/tab_model"export HF_HOME=$PROJECT/huggingfacealias dev="cd /home/mihir.mahajan/Projects/miles"alias queue="watch -n1 squeue --me"alias fqueue="watch -n1 squeue"alias wsdir="cd /fast/project/HFMI_SynergyUnit/mihir"umask 007# >>> mamba initialize >>># !! Contents within this block are managed by 'micromamba shell init' !!export MAMBA_EXE='/fast/project/HFMI_SynergyUnit/mihir/bin/micromamba';export MAMBA_ROOT_PREFIX='/fast/project/HFMI_SynergyUnit/mihir/micromamba';__mamba_setup="$("$MAMBA_EXE" shell hook --shell bash --root-prefix "$MAMBA_ROOT_PREFIX" 2> /dev/null)"if [ $? -eq 0 ]; then    eval "$__mamba_setup"else    alias micromamba="$MAMBA_EXE"  # Fallback on help from micromamba activatefiunset __mamba_setup# <<< mamba initialize <<<(miles) [mihir.mahajan@hai005.haicore.berlin:/fast/home/mihir.mahajan/Projects/miles] $ vim ~/.bashrc (miles) [mihir.mahajan@hai005.haicore.berlin:/fast/home/mihir.mahajan/Projects/miles] $ source ~/.bashrc (miles) [mihir.mahajan@hai005.haicore.berlin:/fast/home/mihir.mahajan/Projects/miles] $ pythonPython 3.12.12 | packaged by conda-forge | (main, Oct 22 2025, 23:25:55) [GCC 14.3.0] on linuxType "help", "copyright", "credits" or "license" for more information.Ctrl click to launch VS Code Native REPL
[content] build_conda.sh (shellscript)
# micromamba install -n miles cuda cuda-nvtx cuda-nvtx-dev nccl -c nvidia/label/cuda-12.9.1 -y
# micromamba install -n miles -c conda-forge cudnn -y

# prevent installing cuda 13.0 for sglang
# pip install cuda-python==12.9.1
# pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu129

# install sglang
# git clone https://github.com/sgl-project/sglang.git
cd sglang
git checkout 303cc957e62384044dfa8e52d7d8af8abe12f0ac
# Install the python packages
# pip install -e "python[all]"


# pip install cmake ninja

# flash attn
cd $BASE_DIR
# git clone https://github.com/Dao-AILab/flash-attention.git
cd flash-attention/hopper
python setup.py install
echo "DONE FOR NOW"
# the newest version megatron supports is v2.7.4.post1
# MAX_JOBS=64 pip -v install flash-attn==2.7.4.post1 --no-build-isolation

# pip install git+https://github.com/ISEEKYAN/mbridge.git@89eb10887887bc74853f89a4de258c0702932a1c --no-deps
# pip install --no-build-isolation "transformer_engine[pytorch]==2.8.0"
# pip install flash-linear-attention==0.4.0
# NVCC_APPEND_FLAGS="--threads 4" \
  # pip -v install --disable-pip-version-check --no-cache-dir \
  # --no-build-isolation \
  # --config-settings "--build-option=--cpp_ext --cuda_ext --parallel 8" git+https://github.com/NVIDIA/apex.git@10417aceddd7d5d05d7cbf7b0fc2daad1105f8b4

# git clone https://github.com/NVIDIA/Megatron-LM.git --recursive && \
    # cd Megatron-LM && git checkout ${MEGATRON_COMMIT} && \
    # pip install -e .

# pip install git+https://github.com/fzyzcjy/torch_memory_saver.git@9b8b788fdeb9c2ee528183214cef65a99b71e7d5 --no-cache-dir --force-reinstall
# pip install git+https://github.com/fzyzcjy/Megatron-Bridge.git@dev_rl --no-build-isolation
# pip install nvidia-modelopt[torch]>=0.37.0 --no-build-isolation

# # megatron
# cd $BASE_DIR
# git clone https://github.com/NVIDIA/Megatron-LM.git --recursive && \
  # cd Megatron-LM/ && git checkout core_v0.14.0 && \
  # pip install -e .

# # install miles and apply patches

# # if miles does not exist locally, clone it
# if [ ! -d "$BASE_DIR/miles" ]; then
  # cd $BASE_DIR
  # git clone  https://github.com/radixark/miles.git
  # cd miles/
  # export MILES_DIR=$BASE_DIR/miles
  # pip install -e .
# else
  # export MILES_DIR=$BASE_DIR/
  # pip install -e .
# fi

# # apply patch
# cd $BASE_DIR/sglang
# git apply $MILES_DIR/docker/patch/v0.5.5.post1/sglang.patch
# cd $BASE_DIR/Megatron-LM
# git apply $MILES_DIR/docker/patch/v0.5.5.post1/megatron.patch
[content] /home/mihir.mahajan/Projects/miles_bak/build_conda.sh (shellscript)
#!/bin/bash

# set -ex

# create conda
# yes '' | "${SHELL}" <(curl -L micro.mamba.pm/install.sh)
export PS1=tmp
# mkdir -p /fast/project/HFMI_SynergyUnit/mihir/.cargo/
# touch /fast/project/HFMI_SynergyUnit/mihir/.cargo/env
source ~/.bashrc

# micromamba create -n miles python=3.12 pip -c conda-forge -y
micromamba activate miles
export CUDA_HOME="$CONDA_PREFIX"
export CPATH=$CUDA_HOME/targets/x86_64-linux/include:$CPATH

export BASE_DIR=${BASE_DIR:-"/fast/project/HFMI_SynergyUnit/mihir/"}
cd $BASE_DIR

# install cuda 12.9 as it's the default cuda version for torch
# micromamba install -n miles cuda cuda-nvtx cuda-nvtx-dev nccl -c nvidia/label/cuda-12.9.1 -y
# micromamba install -n miles -c conda-forge cudnn -y

# prevent installing cuda 13.0 for sglang
# pip install cuda-python==12.9.1
# pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu129

# install sglang
# git clone https://github.com/sgl-project/sglang.git
cd sglang
git checkout 303cc957e62384044dfa8e52d7d8af8abe12f0ac
# Install the python packages
# pip install -e "python[all]"


# pip install cmake ninja

# flash attn
cd $BASE_DIR
# git clone https://github.com/Dao-AILab/flash-attention.git
cd flash-attention/hopper
python setup.py install
echo "DONE FOR NOW"
[content] build_conda.sh (shellscript)

# flash attn
# the newest version megatron supports is v2.7.4.post1
# MAX_JOBS=64 pip -v install flash-attn==2.7.4.post1 --no-build-isolation
# pip install https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.3.18/flash_attn-2.7.4%2Bcu128torch2.8-cp312-cp312-linux_x86_64.whl

# pip install git+https://github.com/ISEEKYAN/mbridge.git@89eb10887887bc74853f89a4de258c0702932a1c --no-deps
# pip install --no-build-isolation "transformer_engine[pytorch]==2.8.0" --no-cache-dir
# pip install flash-linear-attention==0.4.0
# NVCC_APPEND_FLAGS="--threads 4" \
  # pip -v install --disable-pip-version-check --no-cache-dir \
  # --no-build-isolation \
  # --config-settings "--build-option=--cpp_ext --cuda_ext --parallel 8" git+https://github.com/NVIDIA/apex.git@10417aceddd7d5d05d7cbf7b0fc2daad1105f8b4

# git clone https://github.com/NVIDIA/Megatron-LM.git --recursive && \
    # cd Megatron-LM && git checkout ${MEGATRON_COMMIT} && \
    # pip install -e .

# pip install git+https://github.com/fzyzcjy/torch_memory_saver.git@9b8b788fdeb9c2ee528183214cef65a99b71e7d5 --no-cache-dir --force-reinstall
# pip install git+https://github.com/fzyzcjy/Megatron-Bridge.git@dev_rl --no-build-isolation
# pip install nvidia-modelopt[torch]>=0.37.0 --no-build-isolation

# megatron
# cd $BASE_DIR
# git clone https://github.com/NVIDIA/Megatron-LM.git --recursive && \
  # cd Megatron-LM/ && git checkout core_v0.14.0 && \
  # pip install -e .

# install miles and apply patches

# if miles does not exist locally, clone it
# if [ ! -d "$BASE_DIR/miles" ]; then
  # cd $BASE_DIR
  # git clone  https://github.com/radixark/miles.git
  # cd miles/
  # export MILES_DIR=$BASE_DIR/miles
  # pip install -e .
# else
  # export MILES_DIR=$BASE_DIR/
  # pip install -e .
# fi

# apply patch
[content] build_conda.sh (shellscript)
pip install git+https://github.com/ISEEKYAN/mbridge.git@89eb10887887bc74853f89a4de258c0702932a1c --no-deps
pip install --no-build-isolation "transformer_engine[pytorch]==2.8.0" --no-cache-dir
pip install flash-linear-attention==0.4.0
NVCC_APPEND_FLAGS="--threads 4" \
pip -v install --disable-pip-version-check --no-cache-dir \