Initial commit

.gitignore

@@ -0,0 +1,2 @@
+notebooks/
+venv/

logs/generation_error.log

@@ -0,0 +1,3 @@
+
+2024-07-22 14:34:20,942 ERROR testing path
+2024-07-22 14:34:57,915 ERROR testing path

main.py

@@ -0,0 +1,37 @@
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from src.TinyLLama import text_generation
+from src.classmodels.inputforgeneration import InputForGeneration
+from src.classmodels.generatedoutput import GeneratedOutput
+import uvicorn
+
+app = FastAPI()
+
+origins = ["*"]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"] 
+)
+
+@app.get("/cmsai/warmuptextgenerationmodel")
+def warmupGenerationModel():
+    warmupModelMessage = text_generation.warmupTextGenerationModel()
+    return warmupModelMessage   
+
+@app.post("/cmsai/generatetext")
+async def generateTextUsingLLama(inputSettings:InputForGeneration) -> GeneratedOutput:
+    try:       
+        output = text_generation.generateText(inputSettings)
+        if output is not None:
+            return GeneratedOutput(status_code=200, generated_text=output)
+        else:
+            return GeneratedOutput(status_code=400, message="error when generating text")
+    except Exception as e:
+        return GeneratedOutput(status_code=500, message=str(e))
+    
+if __name__ == "__main__":
+    uvicorn.run(app=app)

requirements.txt

@@ -0,0 +1,6 @@
+fastapi
+uvicorn
+transformers
+torch
+bitsandbytes
+peft

src/TinyLLama/model/config.json

@@ -0,0 +1,44 @@
+{
+  "_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 22,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_storage": "uint8",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": false,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
+    "quant_method": "bitsandbytes"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.42.3",
+  "use_cache": false,
+  "vocab_size": 32000
+}

src/TinyLLama/model/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "max_length": 2048,
+  "pad_token_id": 0,
+  "transformers_version": "4.42.3"
+}

src/TinyLLama/model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29e7e4f127f5b404eb938d84dd96724e5f519b49e23fd7ef490d98cdfd030307
+size 807426286

src/TinyLLama/text_generation.py

@@ -0,0 +1,72 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from src.classmodels.inputforgeneration import InputForGeneration
+from errorlog.errorlog import log_error
+from pathlib import Path
+
+#MODEL NAME AS PER IN HUGGING FACE :- TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
+current_folderpath = Path(__file__).resolve().parent
+
+tokenizer = None
+quantised_model = None
+tokenizer_path = None
+model_path = None
+
+additional_kwargs = {
+  "do_sample" : True,
+  "early_stopping" :True,
+   "num_beams" : 5,
+   "no_repeat_ngram_size" : 5,
+    "truncation" : True
+}
+
+TASK_NAME = "text-generation"
+
+def isModelAvailable():    
+    model_path = current_folderpath / "model"
+    if model_path is not None and len(str(model_path).strip()) > 0:
+        return True
+    else:
+        return False
+
+def isTokenizerAvailable():
+    tokenizer_path = current_folderpath / "tokenizer"
+    if tokenizer_path is not None and len(str(tokenizer_path).strip()) > 0:
+        return True
+    else:
+        return False
+    
+def warmupTextGenerationModel():
+    try:
+        if isModelAvailable() and isTokenizerAvailable():
+            tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
+            quantised_model = AutoModelForCausalLM.from_pretrained(model_path, use_safetensors = True)
+            return "text generation model is warm up"
+        else:
+            return "No model/tokenizer folder found..."
+    except Exception as ex:
+        log_error(str(ex))
+        return "Issue occured when warming up the text generation model. Please try again.."
+
+def generateText(inputSettings: InputForGeneration):
+    try:
+        if tokenizer is not None and quantised_model is not None:
+            pipe = pipeline(task= TASK_NAME, model= quantised_model, tokenizer = tokenizer, device_map = "auto")
+        
+            #formatted prompt for LLama Model 
+            prompt = f"<s>[INST] {inputSettings.input_for_generation} [/INST]"
+            generated_text = pipe(prompt,temperature = inputSettings.temperature, max_length = inputSettings.max_length,
+                               **additional_kwargs)
+        
+            if generated_text is not None and generated_text[0]['generated_text'] is not None:
+                 return generated_text[0]['generated_text'].replace("<s>","").replace("[INST]","").replace("[/INST]","")
+            
+        else:
+            #If tokenizer or model is not captured, notify as an issue in generation
+            return None
+                    
+    except Exception as ex:
+        log_error(str(ex))
+        return ""
+        
+        
+

src/TinyLLama/tokenizer/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

src/TinyLLama/tokenizer/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

src/TinyLLama/tokenizer/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding": true,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "truncation": true,
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

src/classmodels/inputforgeneration.py

@@ -0,0 +1,6 @@
+from pydantic import BaseModel, Field
+
+class InputForGeneration(BaseModel):
+    input_for_generation : str = Field(..., description="Input text for which generation should happen")
+    temperature : float = Field(0.8, description="Define the degree of randomness")
+    max_length: int = Field(250, description="set the maximum length for the output of generated text")

src/errorlog/errorlog.py

@@ -0,0 +1,30 @@
+import logging
+import json
+from logging.handlers import TimedRotatingFileHandler
+from datetime import datetime
+from pathlib import Path
+
+
+parent_path = Path(__file__).resolve().parent.parent.parent
+final_path = parent_path/'logs'/'generation_error.log'
+
+# Configure logging with TimedRotatingFileHandler
+logging.basicConfig(level=logging.ERROR,
+                    format='%(asctime)s %(levelname)s %(message)s')
+
+# Create a TimedRotatingFileHandler
+handler = TimedRotatingFileHandler(filename=final_path, when='W0', interval=1, backupCount=0, encoding='utf-8')
+
+# Set the log file name format (optional)
+handler.suffix = "%Y-%m-%d_%H-%M-%S.log"
+
+# Set the logging format
+handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
+                    
+logger = logging.getLogger().addHandler(handler)
+
+def log_error(error_message):  
+   logging.error(error_message)
+
+
+