Commit
•
6fdd844
1
Parent(s):
3d7b044
Update README.md
Browse files
README.md
CHANGED
@@ -44,8 +44,8 @@ tokenizer = AutoTokenizer.from_pretrained("InstaDeepAI/nucleotide-transformer-v2
|
|
44 |
model = AutoModelForMaskedLM.from_pretrained("InstaDeepAI/nucleotide-transformer-v2-500m-multi-species", trust_remote_code=True)
|
45 |
|
46 |
# Create a dummy dna sequence and tokenize it
|
47 |
-
sequences = [
|
48 |
-
tokens_ids = tokenizer.batch_encode_plus(sequences, return_tensors="pt")["input_ids"]
|
49 |
|
50 |
# Compute the embeddings
|
51 |
attention_mask = tokens_ids != tokenizer.pad_token_id
|
@@ -61,8 +61,11 @@ embeddings = torch_outs['hidden_states'][-1].detach().numpy()
|
|
61 |
print(f"Embeddings shape: {embeddings.shape}")
|
62 |
print(f"Embeddings per token: {embeddings}")
|
63 |
|
|
|
|
|
|
|
64 |
# Compute mean embeddings per sequence
|
65 |
-
mean_sequence_embeddings = torch.sum(attention_mask
|
66 |
print(f"Mean sequence embeddings: {mean_sequence_embeddings}")
|
67 |
```
|
68 |
|
|
|
44 |
model = AutoModelForMaskedLM.from_pretrained("InstaDeepAI/nucleotide-transformer-v2-500m-multi-species", trust_remote_code=True)
|
45 |
|
46 |
# Create a dummy dna sequence and tokenize it
|
47 |
+
sequences = ["ATTCCGATTCCGATTCCG", "ATTTCTCTCTCTCTCTGAGATCGATCGATCGAT"]
|
48 |
+
tokens_ids = tokenizer.batch_encode_plus(sequences, return_tensors="pt", padding="max_length", max_length = max_length)["input_ids"]
|
49 |
|
50 |
# Compute the embeddings
|
51 |
attention_mask = tokens_ids != tokenizer.pad_token_id
|
|
|
61 |
print(f"Embeddings shape: {embeddings.shape}")
|
62 |
print(f"Embeddings per token: {embeddings}")
|
63 |
|
64 |
+
# Add embed dimension axis
|
65 |
+
attention_mask = torch.unsqueeze(attention_mask, dim=-1)
|
66 |
+
|
67 |
# Compute mean embeddings per sequence
|
68 |
+
mean_sequence_embeddings = torch.sum(attention_mask*embeddings, axis=-2)/torch.sum(attention_mask, axis=1)
|
69 |
print(f"Mean sequence embeddings: {mean_sequence_embeddings}")
|
70 |
```
|
71 |
|