p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] requirements.txt (pip-requirements)
absl-py
aqtp
array-record
cloud-accelerator-diagnostics
cloud-tpu-diagnostics
datasets>=4.0.0
flax>=0.11.0
gcsfs
google-api-python-client
google-cloud-aiplatform==1.61.0
google-cloud-monitoring
google-jetstream@git+https://github.com/AI-Hypercomputer/JetStream.git
grain[parquet]>=0.2.6
huggingface_hub
jax[cuda12]>=0.4.30
jaxlib>=0.4.30
jaxtyping
jsonlines
ml-collections
ml-goodput-measurement
mlperf-logging@git+https://github.com/mlperf/logging.git
numpy
omegaconf
optax
orbax-checkpoint>=0.11.22
pathwaysutils>=0.1.1
pillow>=11.1.0
pre-commit
protobuf==3.20.3
pyink
pylint
pytest
pytype
qwix@git+https://github.com/google/qwix.git
sentencepiece==0.2.0
tensorboard-plugin-profile
tensorboardx>=2.6.2.2
tensorflow-datasets
tensorflow-text>=2.17.0
tensorflow>=2.16.0
tiktoken
transformers
tunix@git+https://github.com/google/tunix.git
[tab] /Users/franzsrambical/Library/Application Support/Cursor/User/settings.json (jsonc)
{
    "git.autofetch": true,
    "vim.sneakUseIgnorecaseAndSmartcase": true,
    "diffEditor.codeLens": true,
    "python.analysis.typeCheckingMode": "standard",
    "workbench.editorAssociations": {
        "git-rebase-todo": "default"
    },
    "cursorpyright.analysis.typeCheckingMode": "standard",
    "remote.SSH.remotePlatform": {
        "juwels-cluster.fz-juelich.de": "linux",
        "hpc-submit02.scidom.de": "linux",
        "horeka.scc.kit.edu": "linux",
        "hpc-submit01.scidom.de": "linux",
        "login.haicore.berlin": "linux"
    },
    "workbench.preferredLightColorTheme": "Visual Studio Light",
    "workbench.colorTheme": "Default Light Modern",
    "remote.autoForwardPortsSource": "hybrid",
    "crowdCode.export.exportPath": "/tmp/franz/crowd-code/",
    "vim.easymotion": true
}
[tab] MaxText/__init__.py (python)
# Copyright 2023–2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__author__ = "Google LLC"
__version__ = "2025.07.24"
__description__ = (
    "MaxText is a high performance, highly scalable, open-source LLM written in pure Python/Jax and "
    "targeting Google Cloud TPUs and GPUs for training and **inference."
)


# maxtext/__init__.py

from collections.abc import Sequence

from MaxText import maxtext_utils
from MaxText import train_utils
from MaxText import pyconfig
from MaxText.layers import models
import jax
from jax.sharding import Mesh

Transformer = models.Transformer


def from_pretrained(
    config: pyconfig.HyperParameters,
    devices: Sequence[jax.Device] | None = None,
) -> Transformer:
  """Load a pretrained MaxText model from checkpoint.

  This function loads a model from a checkpoint.

  Args:
      config: Config object.

  Returns:
      Transformer: The loaded model instance (only the model)

  Example:
      model = from_pretrained(config)
  """
  devices_array = maxtext_utils.create_device_mesh(config, devices)
  mesh = Mesh(devices_array, config.mesh_axes)
  model = train_utils.create_model(config, mesh)

  # Return only the model
  return model
[tab] MaxText/max_logging.py (python)
# Copyright 2023–2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Stub for logging utilities. Right now just meant to avoid raw prints."""


def log(user_str):
  print(user_str, flush=True)