inference.py

from peft import LoraConfig,PeftModel,PeftConfig
from transformers import AutoTokenizer,AutoModelForCausalLM
import torch



def make_inference(context, question):
    batch = tokenizer(f"### CONTEXT\n{context}\n\n### QUESTION\n{question}\n\n### ANSWER\n", return_tensors='pt')

    if torch.cuda.is_available():
        batch = {k: v.to('cuda') for k, v in batch.items()}

    with torch.cuda.amp.autocast():
        output_tokens = qa_model.generate(**batch, max_new_tokens=200)

    return tokenizer.decode(output_tokens[0], skip_special_tokens=True)



model_name = "bloom7b__finetune_sample"
HUGGING_FACE_USER_NAME = "james92"

# Do the inference 
peft_model_id = f'{HUGGING_FACE_USER_NAME}/{model_name}'
config = PeftConfig.from_pretrained(peft_model_id)
inference_model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path,return_dict=True,load_in_8bit=False,device_map='auto')
tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)

# Do the inference 
qa_model = PeftModel.from_pretrained(inference_model,peft_model_id)

# Print the model.
print(qa_model)


context = "Cheese is the best food."
question = "What is the best food?"

print(make_inference(context=context,question=question))


context = "The Moon orbits Earth at an average distance of 384,400 km (238,900 mi), or about 30 times Earth's diameter. Its gravitational influence is the main driver of Earth's tides and very slowly lengthens Earth's day. The Moon's orbit around Earth has a sidereal period of 27.3 days. During each synodic period of 29.5 days, the amount of visible surface illuminated by the Sun varies from none up to 100%, resulting in lunar phases that form the basis for the months of a lunar calendar. The Moon is tidally locked to Earth, which means that the length of a full rotation of the Moon on its own axis causes its same side (the near side) to always face Earth, and the somewhat longer lunar day is the same as the synodic period. However, 59% of the total lunar surface can be seen from Earth through cyclical shifts in perspective known as libration."
question = "At what distance does the Moon orbit the Earth?"

print(make_inference(context=context,question=question))


context = "Cheese is the best food."
question = "How far away is the Moon from the Earth?"


print(make_inference(context=context,question=question))


context = '''
Content - 0:product.partNumber=CO-AM-S185,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=242,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=3844150,status=Active,value=69938,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 1:product.partNumber=CO-AM-S185,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=303,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=5227245,status=Active,value=87567,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 2:product.partNumber=CO-AM-S393,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=222,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=9329151,status=Active,value=40626,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 3:product.partNumber=CO-AM-B209,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=88,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=1716142,status=Active,value=10560,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 4:product.partNumber=CO-AM-S393,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=89,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=6375908,status=Active,value=16287,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 5:product.partNumber=CO-RD-I380,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=168,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=286860,status=Active,value=13104,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 6:product.partNumber=CO-RD-I380,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=189,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=2162419,status=Active,value=14742,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 7:product.partNumber=CO-DL-T316,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=270,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=9656094,status=Active,value=14850,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 8:product.partNumber=CO-DL-A230,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=125,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=6219859,status=Active,value=6375,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 9:product.partNumber=CO-DL-A239,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=103,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=3352121,status=Active,value=4944,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 10:product.partNumber=CO-AM-B209,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=384,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=3024598,status=Active,value=46080,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 11:product.partNumber=CO-DL-A230,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=148,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=8276232,status=Active,value=7548,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 12:product.partNumber=CO-DL-A239,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=222,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=23233,status=Active,value=10656,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 13:product.partNumber=CO-DL-T316,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=50,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=6529379,status=Active,value=2750,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 14:product.partNumber=CO-RD-S282,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=352,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=3932441,status=Active,value=30976,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 15:product.partNumber=CO-RD-S282,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=388,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=1501513,status=Active,value=34144,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 16:product.partNumber=CO-RD-R296,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=125,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=3090203,status=Active,value=11625,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 17:product.partNumber=PS-PL-R186,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=38,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=7999926,status=Active,value=47500,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 18:product.partNumber=PS-PL-R186,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=39,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=5754848,status=Active,value=48750,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 19:product.partNumber=CO-RD-R296,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=162,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=6526123,status=Active,value=15066,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 20:product.partNumber=CO-CO-N311,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=256,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=8095102,status=Active,value=64000,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 21:product.partNumber=CO-CO-N311,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=124,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=4709951,status=Active,value=31000,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 22:product.partNumber=CO-CO-V310,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=59,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=8601192,status=Active,value=5900,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 23:product.partNumber=CO-CO-V310,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=121,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=3952909,status=Active,value=12100,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 24:product.partNumber=PS-SL-C193,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=49,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=2450098,status=Active,value=61250,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 25:product.partNumber=CO-CO-M328,location.locationIdentifier=LT-1,inventoryType=PRODUCT,quantity=214,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=6340793,status=Active,value=26750,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 26:product.partNumber=CO-CO-M328,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=230,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=4564154,status=Active,value=28750,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\nContent - 27:product.partNumber=PS-SL-C193,location.locationIdentifier=LT-2,inventoryType=PRODUCT,quantity=41,quantityUnits=EA,expirationDate=2022-12-31T00:00:00,inventoryParentType=ONHAND,class=NEW,segment=INDUSTRIAL,lotCode=9412870,status=Active,value=51250,valueCurrency=USD,sourceLink=https://foo.com,storageDate=2022-01-01T00:00:00\n\n
'''
question = "give me details about the product whose partnumber is CO-AM-S185"


print(make_inference(context=context,question=question))