p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] pyproject.toml (plaintext)
[build-system]
requires = [
    "packaging",
    "setuptools >= 49.4.0",
    "wheel",
]
build-backend = "setuptools.build_meta"

[tool.isort]
profile = "black"  # black-compatible
line_length = 119  # should match black parameters
ignore_whitespace = true  # ignore whitespace for compatibility with the initial style
py_version = 310  # python 3.10 as a target version
sections = ["FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"]
default_section = "THIRDPARTY"
extend_skip = ["setup.py", "docs/source/conf.py"]
known_first_party = ["miles", "miles_plugins"]
known_third_party = ["megatron", "wandb", "ray", "transformers"]
src_paths = ["miles", "miles_plugins"]


[tool.black]
line_length = 119

[tool.ruff]
line-length = 320  # TODO
select = [
    "E",      # Pycodestyle Errors (Structural/Fundamental Errors like bad indentation)
    "F",      # Pyflakes (Core Errors: Unused imports, undefined names)
    "B",      # Flake8-Bugbear (Logic Bugs: Variable shadowing, dangerous default arguments)
    "UP",     # pyupgrade (Modernization and compatibility issues)
]
ignore = [
    "E402", # module-import-not-at-top-of-file
    "E501", # Line too long # TODO handle it later
]

[tool.pytest.ini_options]
# durations=0 will display all tests execution time, sorted in ascending order starting from from the slowest one.
# -vv will also display tests with duration = 0.00s
addopts = "--verbose --pyargs --durations=0 --strict-markers"  # always add these arguments to pytest
testpaths = ["./tests"]  # must be an explicit path to avoid importing another "tests" module
# directories to ignore when discovering tests
norecursedirs = [
    "external",
    "examples",
    "docs",
    "scripts",
    "tools",
    "tutorials",
    "*.egg",
    ".*",
    "_darcs",
    "build",
    "CVS",
    "dist",
    "venv",
    "{arch}",
]
# markers to select tests, use `pytest --markers` to see all available markers, `pytest -m "<marker>"` to select tests
markers = [
    "unit: marks unit test, i.e. testing a single, well isolated functionality (deselect with '-m \"not unit\"')",
    "integration: marks test checking the elements when integrated into subsystems (deselect with '-m \"not integration\"')",
    "system: marks test working at the highest integration level (deselect with '-m \"not system\"')",
    "acceptance: marks test checking whether the developed product/model passes the user defined acceptance criteria (deselect with '-m \"not acceptance\"')",
    "docs: mark tests related to documentation (deselect with '-m \"not docs\"')",
    "skipduringci: marks tests that are skipped ci as they are addressed by Jenkins jobs but should be run to test user setups",
    "pleasefixme: marks tests that are broken and need fixing",
]
[tab] setup.py (python)
import sys
import platform

from setuptools import find_packages, setup
from wheel.bdist_wheel import bdist_wheel as _bdist_wheel


def _fetch_requirements(path):
    with open(path) as fd:
        return [r.strip() for r in fd.readlines() if r.strip() and not r.startswith("#")]


# Custom wheel class to modify the wheel name
class bdist_wheel(_bdist_wheel):
    def finalize_options(self):
        _bdist_wheel.finalize_options(self)
        self.root_is_pure = False

    def get_tag(self):
        python_version = f"cp{sys.version_info.major}{sys.version_info.minor}"
        abi_tag = f"{python_version}"

        if platform.system() == "Linux":
            platform_tag = "manylinux1_x86_64"
        else:
            platform_tag = platform.system().lower()

        return python_version, abi_tag, platform_tag


# Setup configuration
setup(
    author="miles Team",
    name="miles",
    version="0.2.0.post1",
    packages=find_packages(include=["miles*", "miles_plugins*"]),
    include_package_data=True,
    install_requires=_fetch_requirements("requirements.txt"),
    extras_require={
        "fsdp": [
            "torch>=2.0",
        ]
    },
    python_requires=">=3.10",
    classifiers=[
        "Programming Language :: Python :: 3.10",
        "Programming Language :: Python :: 3.11",
        "Programming Language :: Python :: 3.12",
        "Environment :: GPU :: NVIDIA CUDA",
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
        "Topic :: System :: Distributed Computing",
    ],
    cmdclass={"bdist_wheel": bdist_wheel},
)
[tab] requirements.txt (pip-requirements)
accelerate
blobfile
datasets
httpx[http2]
mcp[cli]
memray  # needed for debugging (but is lightweight), we can put it to dev mode when using pyproject.toml
omegaconf
pillow
pylatexenc
pyyaml
ray[default]
ring_flash_attn
sglang-router>=0.2.3
tensorboard
transformers
wandb
[tab] .pre-commit-config.yaml (yaml)
default_language_version:
  python: python3

ci:
  autofix_prs: true
  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
  autoupdate_schedule: quarterly

repos:
  - repo: https://github.com/pre-commit/pre-commit-hooks
    rev: v4.5.0
    hooks:
      - id: check-yaml
      - id: check-case-conflict
      - id: detect-private-key
      - id: check-added-large-files
        args: ['--maxkb=1000']
      - id: requirements-txt-fixer

  - repo: https://github.com/astral-sh/ruff-pre-commit
    rev: v0.14.7
    hooks:
      - id: ruff-check
        args: [ --fix ]

  - repo: https://github.com/PyCQA/autoflake
    rev: v2.0.2
    hooks:
      - id: autoflake
        args: [--remove-all-unused-imports, --in-place]

  - repo: https://github.com/pycqa/isort
    rev: 5.13.2          # 选一个稳定版本
    hooks:
      - id: isort
        args:
          - "--profile=black"        # 常见：与 Black 对齐风格
          - "--filter-files"         # 忽略已在 .gitignore 的文件
        additional_dependencies: []  # 需要插件时在这里加

  - repo: https://github.com/psf/black
    rev: 24.3.0
    hooks:
      - id: black
        name: Format code
        additional_dependencies: ['click==8.0.2']
[tab] scripts/models/kimi-k2-thinking.sh (shellscript)
NLAYERS=61
FIRST_K_DENSE_REPLACE=1

arr=()
for ((i=0; i<NLAYERS; i++)); do
  if (( i < FIRST_K_DENSE_REPLACE )); then
    arr+=(0)
  else
    arr+=(1)
  fi
done

printf -v MOE_LAYER_FREQ "[%s]" "$(IFS=', '; echo "${arr[*]}")"

# kimi-k2-thinking
MODEL_ARGS=(
    --disable-bias-linear
    --num-layers 61
    --hidden-size 7168
    --ffn-hidden-size 18432
    --num-attention-heads 64
    --kv-channels 64
    --normalization RMSNorm
    --position-embedding-type rope
    --norm-epsilon 1e-5
    --swiglu
    --untie-embeddings-and-output-weights
    --vocab-size 163840
    
    --multi-latent-attention
    --q-lora-rank 1536
    --kv-lora-rank 512
    --qk-head-dim 128
    --qk-pos-emb-head-dim 64
    --v-head-dim 128
    --qk-layernorm
    --rotary-scaling-factor 64.0
    --rotary-base 50000
    --mscale 1.0
    --mscale-all-dim 1.0
    --attention-softmax-in-fp32
    --no-rope-fusion

    # moe
    --num-experts 384
    --moe-layer-freq $MOE_LAYER_FREQ
    --moe-ffn-hidden-size 2048
    --moe-router-topk 8
    --moe-shared-expert-intermediate-size 2048
    --moe-router-pre-softmax
    --moe-router-score-function sigmoid
    --moe-router-enable-expert-bias
    --moe-router-load-balancing-type seq_aux_loss
    --moe-token-dispatcher-type alltoall
    --moe-aux-loss-coeff 0
    --moe-router-bias-update-rate 0
    --moe-router-group-topk 1
    --moe-router-num-groups 1
    --moe-grouped-gemm
    --moe-router-topk-scaling-factor 2.827
    --moe-router-dtype fp32
    --moe-permute-fusion
)