up

create_confidence_scores.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
+from datasets import load_dataset
+import datasets
+import torch
+
+model = Wav2Vec2ForCTC.from_pretrained("facebook/data2vec-audio-base-10m")
+processor = Wav2Vec2Processor.from_pretrained("facebook/data2vec-audio-base-10m")
+
+minds14 = load_dataset("PolyAI/minds14", "en-US", split="train")
+minds14 = minds14.cast_column("audio", datasets.Audio(sampling_rate=16_000))
+
+input_values = processor(minds14[0]["audio"]["array"], return_tensors="pt", sampling_rate=minds14[0]["audio"]["sampling_rate"]).input_values
+
+with torch.no_grad():
+    logits = model(input_values).logits
+    scores = torch.nn.functional.softmax(logits, dim=-1)
+    pred_ids = torch.argmax(logits, dim=-1)
+    pred_scores = scores.gather(1, pred_ids.unsqueeze(-1))[:, :, 0]
+
+output = processor.batch_decode(pred_ids, output_word_offsets=True)
+
+# add confidence
+def confidence_score(word_dict):
+    probs = pred_scores[0, word_dict["start_offset"]: word_dict["end_offset"]]
+    return torch.mean(probs)
+
+output["confidence_scores"] = {d["word"]: confidence_score(d) for d in output.word_offsets[0]}
+
+print(output["confidence_scores"])

example.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+from transformers import RobertaTokenizer, RobertaForTokenClassification
+import torch
+tokenizer = RobertaTokenizer.from_pretrained("Jean-Baptiste/roberta-large-ner-english")
+model = RobertaForTokenClassification.from_pretrained("Jean-Baptiste/roberta-large-ner-english")
+inputs = tokenizer("HuggingFace is a company based in Paris and New York", add_special_tokens=False, return_tensors="pt")
+
+with torch.no_grad():
+    logits = model(**inputs).logits
+
+predicted_token_class_ids = logits.argmax(-1)
+# Note that tokens are classified rather then input words which means that
+# there might be more predicted token classes than words.
+# Multiple token classes might account for the same word
+predicted_tokens_classes = [model.config.id2label[t.item()] for t in predicted_token_class_ids[0]]
+assert predicted_tokens_classes == ['O', 'ORG', 'ORG', 'O', 'O', 'O', 'O', 'O', 'LOC', 'O', 'LOC', 'LOC']
+
+labels = predicted_token_class_ids
+loss = model(**inputs, labels=labels).loss
+ab = round(loss.item(), 2)
+import ipdb; ipdb.set_trace()

get_sample_code.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+import sys
+name = sys.argv[1]
+processor_class = sys.argv[2]
+model_class = sys.argv[3]
+checkpoint = sys.argv[4]
+mask = sys.argv[5]
+
+with open("/home/patrick/transformers/src/transformers/file_utils.py", "r") as f:
+    lines = f.readlines()
+
+format_dict = {"processor_class": processor_class, "model_class": model_class, "checkpoint": checkpoint, "mask": mask}
+with open("./example.py", "w") as f:
+    f.write("#!/usr/bin/env python3\n")
+
+    is_in = False
+    is_in_code = False
+    for line in lines:
+        if line.strip() == (name + ' = r"""'):
+            is_in = True
+        if is_in and "```python" in line:
+            is_in_code = True
+        if is_in_code:
+            if ">>>" in line:
+                f.write(line.split(">>> ")[-1].format(**format_dict))
+            elif "..." in line:
+                f.write(line.split("... ")[-1].format(**format_dict))
+        if is_in_code and (line.strip() == '"""'):
+            is_in = is_in_code = False