upload model

README.md

@@ -1,3 +1,61 @@
 ---
 license: mit
+language:
+- ja
+library_name: transformers
+pipeline_tag: text-generation
+tags:
+- gpt_neox
+- gpt-neox
+- japanese
+inference:
+  parameters:
+    max_new_tokens: 128
+    do_sample: false
+    repetition_penalty: 1.1
 ---
+
+# stockmark/gpt-neox-japanese-1.4b
+
+This repository provides a GPT-NeoX based model with 1.4B parameters pre-trained on Japanese corpus of about 20B tokens. This model is developed by [Stockmark Inc.](https://stockmark.co.jp/)
+
+## How to use
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+# Use torch.bfloat16 for A100 GPU and torch.flaot16 for the older generation GPUs
+torch_dtype = torch.bfloat16 if torch.cuda.is_available() and hasattr(torch.cuda, "is_bf16_supported") and torch.cuda.is_bf16_supported() else torch.float16
+
+model = AutoModelForCausalLM.from_pretrained("stockmark/gpt-neox-japanese-1.4b", device_map="auto", torch_dtype=torch_dtype)
+tokenizer = AutoTokenizer.from_pretrained("stockmark/gpt-neox-japanese-1.4b")
+
+inputs = tokenizer("自然言語処理とは", return_tensors="pt").to(model.device)
+with torch.no_grad():
+    tokens = model.generate(
+        **inputs,
+        max_new_tokens=128,
+        repetition_penalty=1.1
+    )
+    
+output = tokenizer.decode(tokens[0], skip_special_tokens=True)
+print(output)
+```
+
+## Training dataset
+- Japanese Web Corpus (ja): 8.6B tokens (This dataset will not be released.)
+- Wikipedia (ja): 0.88B tokens
+- CC100 (ja): 10.5B tokens
+
+## Training setting
+- Trained using HuggingFace Trainer and DeepSpeed (ZeRO-2)
+- 8 A100 GPUs (40GB) at ABCI
+- Mixed Precision (BF16)
+
+## License
+[The MIT license](https://opensource.org/licenses/MIT)
+
+## Developed by
+Stockmark Inc.
+https://stockmark.co.jp/

config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "classifier_dropout": 0.1,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 1024,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 1,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.30.2",
+  "use_cache": true,
+  "use_parallel_residual": false,
+  "vocab_size": 50000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_id": 1,
+  "transformers_version": "4.30.2"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ce45432f8538cee7b698f847dcb7f31cea2e27d0322bcf2fa7bf12796246baf
+size 2852015168

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2039be3d49e59904559ab49df85fa8ed58a35967e535d19f174b296a8695b300
+size 2852090557

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|padding|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|padding|>",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}