Delete taiji

taiji/drun

@@ -1,35 +0,0 @@
-#!/bin/bash
-DOCKER_IMAGE="mirrors.tencent.com/ronnysong_rd/fastdet:torch2.0.1-cuda11.7"
-
-if [ ! -n "$DEBUG" ]; then
-    COMMAND_PREFIX="pip3 install -e ."
-else
-    COMMAND_PREFIX="pip3 install -q -e third_party/mmengine;
-                    pip3 install -q -e third_party/mmdetection;
-                    pip3 install -q -e third_party/mmcv;
-                    pip3 install -q -e third_party/mmyolo;
-                    pip3 install -q -e ."
-fi
-
-sudo nvidia-docker run \
-    --rm \
-    -it \
-    -e NVIDIA_VISIBLE_DEVICES=all \
-    --env="DISPLAY" \
-    --env="QT_X11_NO_MITSHM=1" \
-    --volume="$HOME/.Xauthority:/root/.Xauthority:rw" \
-    --shm-size=20gb \
-    --network=host \
-    -v /apdcephfs/:/apdcephfs/ \
-    -v /apdcephfs_cq2/:/apdcephfs_cq2/ \
-    -v /apdcephfs_cq3/:/apdcephfs_cq3/ \
-    -v /data/:/data/ \
-    -w $PWD \
-    $DOCKER_IMAGE \
-    bash -c "export TRANSFORMERS_CACHE=$PWD/work_dirs/.cache/transformers;
-             export TORCH_HOME=$PWD/work_dirs/.cache/torch;
-             export CLIP_CACHE=$PWD/work_dirs/.cache/clip;
-             export HF_HOME=$PWD/work_dirs/.cache/hf;
-             export TOKENIZERS_PARALLELISM=false;
-             $COMMAND_PREFIX
-             $*"

taiji/erun

@@ -1,23 +0,0 @@
-#!/bin/bash
-export NCCL_IB_GID_INDEX=3
-
-export TRANSFORMERS_CACHE=$PWD/work_dirs/.cache/transformers
-export TORCH_HOME=$PWD/work_dirs/.cache/torch
-export CLIP_CACHE=$PWD/work_dirs/.cache/clip
-export HF_HOME=$PWD/work_dirs/.cache/hf
-export TOKENIZERS_PARALLELISM=false
-export MKL_NUM_THREADS=1
-export OMP_NUM_THREADS=1
-export TORCH_DISTRIBUTED_DEBUG=INFO
-export HF_DATASETS_OFFLINE=1
-export TRANSFORMERS_OFFLINE=1
-export http_proxy="http://star-proxy.oa.com:3128"
-export https_proxy="http://star-proxy.oa.com:3128"
-export ftp_proxy="http://star-proxy.oa.com:3128"
-export no_proxy=".woa.com,mirrors.cloud.tencent.com,tlinux-mirror.tencent-cloud.com,tlinux-mirrorlist.tencent-cloud.com,localhost,127.0.0.1,mirrors-tlinux.tencentyun.com,.oa.com,.local,.3gqq.com,.7700.org,.ad.com,.ada_sixjoy.com,.addev.com,.app.local,.apps.local,.aurora.com,.autotest123.com,.bocaiwawa.com,.boss.com,.cdc.com,.cdn.com,.cds.com,.cf.com,.cjgc.local,.cm.com,.code.com,.datamine.com,.dvas.com,.dyndns.tv,.ecc.com,.expochart.cn,.expovideo.cn,.fms.com,.great.com,.hadoop.sec,.heme.com,.home.com,.hotbar.com,.ibg.com,.ied.com,.ieg.local,.ierd.com,.imd.com,.imoss.com,.isd.com,.isoso.com,.itil.com,.kao5.com,.kf.com,.kitty.com,.lpptp.com,.m.com,.matrix.cloud,.matrix.net,.mickey.com,.mig.local,.mqq.com,.oiweb.com,.okbuy.isddev.com,.oss.com,.otaworld.com,.paipaioa.com,.qqbrowser.local,.qqinternal.com,.qqwork.com,.rtpre.com,.sc.oa.com,.sec.com,.server.com,.service.com,.sjkxinternal.com,.sllwrnm5.cn,.sng.local,.soc.com,.t.km,.tcna.com,.teg.local,.tencentvoip.com,.tenpayoa.com,.test.air.tenpay.com,.tr.com,.tr_autotest123.com,.vpn.com,.wb.local,.webdev.com,.webdev2.com,.wizard.com,.wqq.com,.wsd.com,.sng.com,.music.lan,.mnet2.com,.tencentb2.com,.tmeoa.com,.pcg.com,www.wip3.adobe.com,www-mm.wip3.adobe.com,mirrors.tencent.com,csighub.tencentyun.com"
-sed -i 's/np.float/float/g' /usr/local/python/lib/python3.8/site-packages/lvis/eval.py
-touch /tmp/.unhold
-
-pip3 install -e . 
-$*
-rm /tmp/.unhold

taiji/etorchrun

@@ -1,51 +0,0 @@
-#!/bin/bash
-if [ ! -n "$SH" ]; then
-    #export NCCL_IB_GID_INDEX=3
-    export NCCL_IB_DISABLE=1
-    export NCCL_P2P_DISABLE=1
-    export NCCL_SOCKET_IFNAME=eth1
-else
-    export NCCL_IB_GID_INDEX=3
-    export NCCL_IB_SL=3
-    export NCCL_CHECKS_DISABLE=1
-    export NCCL_P2P_DISABLE=0
-    export NCCL_IB_DISABLE=0
-    export NCCL_LL_THRESHOLD=16384
-    export NCCL_IB_CUDA_SUPPORT=1
-    export NCCL_SOCKET_IFNAME=bond1
-    export UCX_NET_DEVICES=bond1
-    export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
-    export NCCL_COLLNET_ENABLE=0
-    export SHARP_COLL_ENABLE_SAT=0
-    export NCCL_NET_GDR_LEVEL=2
-    export NCCL_IB_QPS_PER_CONNECTION=4
-    export NCCL_IB_TC=160
-    export NCCL_PXN_DISABLE=1
-    export GLOO_SOCKET_IFNAME=bond1
-    export NCCL_DEBUG=info
-fi
-
-export TRANSFORMERS_CACHE=$PWD/work_dirs/.cache/transformers
-export TORCH_HOME=$PWD/work_dirs/.cache/torch
-export CLIP_CACHE=$PWD/work_dirs/.cache/clip
-export HF_HOME=$PWD/work_dirs/.cache/hf
-export TOKENIZERS_PARALLELISM=false
-export MKL_NUM_THREADS=1
-export OMP_NUM_THREADS=1
-export TORCH_DISTRIBUTED_DEBUG=INFO
-export HF_DATASETS_OFFLINE=1
-export TRANSFORMERS_OFFLINE=1
-
-export http_proxy="http://star-proxy.oa.com:3128"
-export https_proxy="http://star-proxy.oa.com:3128"
-export ftp_proxy="http://star-proxy.oa.com:3128"
-export no_proxy=".woa.com,mirrors.cloud.tencent.com,tlinux-mirror.tencent-cloud.com,tlinux-mirrorlist.tencent-cloud.com,localhost,127.0.0.1,mirrors-tlinux.tencentyun.com,.oa.com,.local,.3gqq.com,.7700.org,.ad.com,.ada_sixjoy.com,.addev.com,.app.local,.apps.local,.aurora.com,.autotest123.com,.bocaiwawa.com,.boss.com,.cdc.com,.cdn.com,.cds.com,.cf.com,.cjgc.local,.cm.com,.code.com,.datamine.com,.dvas.com,.dyndns.tv,.ecc.com,.expochart.cn,.expovideo.cn,.fms.com,.great.com,.hadoop.sec,.heme.com,.home.com,.hotbar.com,.ibg.com,.ied.com,.ieg.local,.ierd.com,.imd.com,.imoss.com,.isd.com,.isoso.com,.itil.com,.kao5.com,.kf.com,.kitty.com,.lpptp.com,.m.com,.matrix.cloud,.matrix.net,.mickey.com,.mig.local,.mqq.com,.oiweb.com,.okbuy.isddev.com,.oss.com,.otaworld.com,.paipaioa.com,.qqbrowser.local,.qqinternal.com,.qqwork.com,.rtpre.com,.sc.oa.com,.sec.com,.server.com,.service.com,.sjkxinternal.com,.sllwrnm5.cn,.sng.local,.soc.com,.t.km,.tcna.com,.teg.local,.tencentvoip.com,.tenpayoa.com,.test.air.tenpay.com,.tr.com,.tr_autotest123.com,.vpn.com,.wb.local,.webdev.com,.webdev2.com,.wizard.com,.wqq.com,.wsd.com,.sng.com,.music.lan,.mnet2.com,.tencentb2.com,.tmeoa.com,.pcg.com,www.wip3.adobe.com,www-mm.wip3.adobe.com,mirrors.tencent.com,csighub.tencentyun.com"
-
-sed -i 's/np.float/float/g' /usr/local/python/lib/python3.8/site-packages/lvis/eval.py
-
-touch /tmp/.unhold
-
-pip3 install -e .
-torchrun --nnodes=$1 --nproc_per_node=$2 --node_rank=$INDEX --master_addr=$CHIEF_IP ${@:3}
-
-rm /tmp/.unhold

taiji/jizhi_run_vanilla

@@ -1,105 +0,0 @@
-#!/bin/bash
-if [[ $1 = "--help" ]] || [[ $1 = "-h" ]]
-then
-    echo "Usage: jizhi_run NUM_MECHINES NUM_GPUS TASK_NAME <CMDS>"
-fi
-
-# user configuration
-TOKEN=$TOKEN
-if [ ! -n "$IMAGE_FULL_NAME" ]; then
-    IMAGE_FULL_NAME="mirrors.tencent.com/ronnysong_rd/fastdet:torch2.0.1-cuda11.7"
-fi
-if [ ! -n "$BUSINESS_FLAG" ]; then
-    BUSINESS_FLAG="TEG_AILab_CVC_chongqing"
-fi
-if [ ! -n "$CEPH_BUSINESS_FLAG" ]; then
-    CEPH_BUSINESS_FLAG="TEG_AILab_CVC_chongqing"
-fi
-if [ ! -n "$GPU_NAME" ]; then
-    GPU_NAME="V100"
-fi
-if [ ! -n "$PRIORITY_LEVEL" ]; then
-    PRIORITY_LEVEL="HIGH"
-fi
-if [ ! -n "$ELASTIC_LEVEL" ]; then
-    ELASTIC_LEVEL=1
-fi
-if [ ! -n "$RDMA" ]; then
-    RDMA="false"
-fi
-if [ ! -n "$CUDA" ]; then
-    CUDA="11.0"
-fi
-
-CMD_PATH="start.sh"
-CONF_PATH="jizhi_conf.json"
-ROOT_PATH=$PWD
-UUID=$(date +%s)
-
-rm -f $CMD_PATH
-
-echo 'cd '$ROOT_PATH >> $CMD_PATH
-echo 'export HF_HOME="'$ROOT_PATH'/work_dirs/.cache/hf"' >> $CMD_PATH
-echo 'export TORCH_HOME="'$ROOT_PATH'/work_dirs/.cache/torch"' >> $CMD_PATH
-echo 'export CLIP_CACHE="'$ROOT_PATH'/work_dirs/.cache/clip"' >> $CMD_PATH
-echo 'export TRANSFORMERS_CACHE="'$ROOT_PATH'/work_dirs/.cache/transformers"' >> $CMD_PATH
-echo 'export MKL_NUM_THREADS=1' >> $CMD_PATH
-echo 'export OMP_NUM_THREADS=1' >> $CMD_PATH
-echo 'export TOKENIZERS_PARALLELISM=false' >> $CMD_PATH
-echo 'export TORCH_DISTRIBUTED_DEBUG=INFO' >> $CMD_PATH
-echo 'export NCCL_IB_GID_INDEX=3' >> $CMD_PATH
-if [ $BUSINESS_FLAG = "TaiJi_HYAide_BUFFER_SH_A800H" ]; then
-    echo 'export NCCL_IB_GID_INDEX=3' >> $CMD_PATH
-    echo 'export NCCL_IB_SL=3' >> $CMD_PATH
-    echo 'export NCCL_CHECKS_DISABLE=1' >> $CMD_PATH
-    echo 'export NCCL_P2P_DISABLE=0' >> $CMD_PATH
-    echo 'export NCCL_IB_DISABLE=0' >> $CMD_PATH
-    echo 'export NCCL_LL_THRESHOLD=16384' >> $CMD_PATH
-    echo 'export NCCL_IB_CUDA_SUPPORT=1' >> $CMD_PATH
-    echo 'export NCCL_SOCKET_IFNAME=bond1' >> $CMD_PATH
-    echo 'export UCX_NET_DEVICES=bond1' >> $CMD_PATH
-    echo 'export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6' >> $CMD_PATH
-    echo 'export NCCL_COLLNET_ENABLE=0' >> $CMD_PATH
-    echo 'export SHARP_COLL_ENABLE_SAT=0' >> $CMD_PATH
-    echo 'export NCCL_NET_GDR_LEVEL=2' >> $CMD_PATH
-    echo 'export NCCL_IB_QPS_PER_CONNECTION=4' >> $CMD_PATH
-    echo 'export NCCL_IB_TC=160' >> $CMD_PATH
-    echo 'export NCCL_PXN_DISABLE=1' >> $CMD_PATH
-fi
-echo ${@:4} >> $CMD_PATH
-
-chmod +x $CMD_PATH
-
-rm -f $CONF_PATH
-
-#INIT_CMD="jizhi_client mount -bf TEG_AILab_CVC_chongqing -tk $TOKEN"
-INIT_CMD=""
-
-echo '{' > $CONF_PATH
-echo '"Token": "'$TOKEN'",' >> $CONF_PATH
-echo '"business_flag": "'$BUSINESS_FLAG'",' >> $CONF_PATH
-echo '"model_local_file_path": "'$ROOT_PATH'/'$CMD_PATH'",' >> $CONF_PATH
-echo '"host_num": '$1',' >> $CONF_PATH
-echo '"host_gpu_num": '$2',' >> $CONF_PATH
-echo '"task_flag": "'$3'_'$UUID'",' >> $CONF_PATH
-echo '"priority_level": "'$PRIORITY_LEVEL'",' >> $CONF_PATH
-echo '"elastic_level": '$ELASTIC_LEVEL',' >> $CONF_PATH
-echo '"cuda_version": "'$CUDA'",' >> $CONF_PATH
-echo '"image_full_name": "'$IMAGE_FULL_NAME'",' >> $CONF_PATH
-echo '"GPUName": "'$GPU_NAME'",' >> $CONF_PATH
-echo '"mount_ceph_business_flag": "'$CEPH_BUSINESS_FLAG'",' >> $CONF_PATH
-echo '"exec_start_in_all_mpi_pods": true,' >> $CONF_PATH
-echo '"enable_rdma": '$RDMA',' >> $CONF_PATH
-echo '"init_cmd": "'$INIT_CMD'",' >> $CONF_PATH
-echo '"envs": {' >> $CONF_PATH
-echo '    "HUNYUAN_TASK_CATEGORY": "LLM",' >> $CONF_PATH
-echo '    "HUNYUAN_TASK_MODEL_TYPE": "SFT",' >> $CONF_PATH
-echo '    "HUNYUAN_TASK_DOMAIN": "NLP",' >> $CONF_PATH
-echo '    "HUNYUAN_TASK_START_MODEL_TYPE": "7B冷启"}' >> $CONF_PATH
-echo '}' >> $CONF_PATH
-
-jizhi_client start -scfg $CONF_PATH
-
-rm -f $CMD_PATH
-rm -f $CONF_PATH
-