Update README.md

README.md

@@ -1,3 +1,59 @@
 ---
-{}
+datasets:
+- roneneldan/TinyStories
 ---
+---
+Model trained on the TinyStories Dataset, replicating https://arxiv.org/abs/2305.07759, based on LLaMA architecture.
+
+---
+Hyperparams used to train this model:
+```
+        "batch_size": 32,
+
+        "block_size": 256,
+
+        "lr": 5e-4,
+
+        "num_hidden_layers": 8,
+
+        "num_attention_heads": 8,
+
+        "hidden_size": 160,
+
+        "dropout": 0.1,
+
+        "weight_decay": 0.01,
+
+        "epochs": 1,
+
+        "eval_interval": 200,
+
+        "eval_steps": 50,
+
+        "vocab_size": 50257, 
+
+        "warmup_tokens": 10000,
+
+        "gradient_accumulation_steps": 16, 
+```
+---
+EXAMPLE USAGE 
+```py
+  !pip install --quiet transformers 
+  from transformers import AutoModelForCausalLM, AutoTokenizer
+  from huggingface_hub import notebook_login, login
+  import os
+
+  #login to hf to check for llama access
+  hf_token = os.getenv('HF_TOKEN')
+  login(token=hf_token)
+
+  model = AutoModelForCausalLM.from_pretrained('AnirudhRajagopalan1201/tinyllama-20M')
+  tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
+  prompt = "Lily likes cats and dogs. She asked her mom for a dog and her mom said no, so instead she asked"
+  input_ids = tokenizer.encode(prompt, return_tensors="pt")
+  output = model.generate(input_ids, temperature=0.1, max_length = 100, do_sample=True)
+  output_text = tokenizer.decode(output[0], skip_special_tokens=True)
+  print(output_text)
+  
+```