Sample

sample/ct2-bloom.py

@@ -0,0 +1,24 @@
+import ctranslate2
+import transformers
+import datetime
+
+for prompt in ["Hello, I am Joan and I am from Barcelona.", "Translate to Catalan: how are you today?", "Translate to Catalan:  It is a British public service broadcaster, Headquartered in London, and it is the world's oldest national broadcaster.", "Parafraseja en català: Avui serà un dia molt llarg perquè farem moltes coses junts."]:
+    for model_name in [#"bloom-560m", 
+        "bloomz-7b1-mt"]:
+
+        generator = ctranslate2.Generator(model_name, compute_type="int8")
+        tokenizer = transformers.AutoTokenizer.from_pretrained(model_name)
+
+        start_time = datetime.datetime.now()
+
+        start_tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(prompt))
+        results = generator.generate_batch([start_tokens], max_length=90)
+        result = tokenizer.decode(results[0].sequences_ids[0])
+        t = str(datetime.datetime.now() - start_time)
+        print(f"Model: {model_name}")
+        print(f"Prompt: {prompt}")    
+        print(f"Result: {result}")
+        print(f"Time: {t}")
+        print("")
+
+        del generator, tokenizer