upload model

MAR-INF/MANIFEST.json

@@ -0,0 +1,11 @@
+{
+  "createdOn": "08/06/2021 07:26:17",
+  "runtime": "python",
+  "model": {
+    "modelName": "gpt-2-en-small-finetune",
+    "serializedFile": "pytorch_model.bin",
+    "handler": "handler.py",
+    "modelVersion": "1.0"
+  },
+  "archiverVersion": "0.3.0"
+}

config.json

@@ -0,0 +1,37 @@
+{
+  "_name_or_path": "/model",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "gradient_checkpointing": false,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "resid_pdrop": 0.1,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "transformers_version": "4.6.1",
+  "use_cache": true,
+  "vocab_size": 50257
+}

handler.py

@@ -0,0 +1,76 @@
+import torch
+import gc
+from ts.torch_handler.base_handler import BaseHandler
+from transformers import GPT2LMHeadModel
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class SampleTransformerModel(BaseHandler):
+    def __init__(self):
+        super(SampleTransformerModel, self).__init__()
+        self.model = None
+        self.device = None
+        self.initialized = False
+
+    def load_model(self, model_dir):
+        self.model = GPT2LMHeadModel.from_pretrained(model_dir, return_dict=True)
+        self.model.to(self.device)
+
+    def initialize(self, ctx):
+        # self.manifest = ctx.manifest
+        properties = ctx.system_properties
+        model_dir = properties.get("model_dir")
+        self.device = torch.device("cuda:" + str(properties.get("gpu_id")) if torch.cuda.is_available() else "cpu")
+
+        self.load_model(model_dir)
+
+        self.model.eval()
+        self.initialized = True
+
+    def preprocess(self, requests):
+        input_batch = {}
+        for idx, data in enumerate(requests):
+            input_ids = torch.tensor([data.get("body").get("text")]).to(self.device)
+            input_batch["input_ids"] = input_ids
+            input_batch["num_samples"] = data.get("body").get("num_samples")
+            input_batch["length"] = data.get("body").get("length") + len(data.get("body").get("text"))
+        del requests
+        gc.collect()
+        return input_batch
+
+    def inference(self, input_batch):
+        input_ids = input_batch["input_ids"]
+        length = input_batch["length"]
+
+        inference_output = self.model.generate(input_ids,
+                                               bos_token_id=self.model.config.bos_token_id,
+                                               eos_token_id=self.model.config.eos_token_id,
+                                               pad_token_id=self.model.config.eos_token_id,
+                                               do_sample=True,
+                                               max_length=length,
+                                               top_k=50,
+                                               top_p=0.95,
+                                               no_repeat_ngram_size=2,
+                                               num_return_sequences=input_batch["num_samples"])
+
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        del input_batch
+        gc.collect()
+        return inference_output
+
+    def postprocess(self, inference_output):
+        output = inference_output.cpu().numpy().tolist()
+        del inference_output
+        gc.collect()
+        return [output]
+
+    def handle(self, data, context):
+        # self.context = context
+        data = self.preprocess(data)
+        data = self.inference(data)
+        data = self.postprocess(data)
+        return data

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:502013578c9ea5ad0e8fc054739b8ade4d931710ff0e69b1ecc0f43dc344c6d6
+size 510408315

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}