Update README.md
Browse files
README.md
CHANGED
@@ -158,9 +158,69 @@ This dual capability makes it an excellent tool for multimodal retrieval-augment
|
|
158 |
|
159 |
## Usage
|
160 |
|
161 |
-
|
162 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
163 |
|
|
|
164 |
```python
|
165 |
# !pip install transformers einops timm pillow
|
166 |
from transformers import AutoModel
|
@@ -170,20 +230,20 @@ model = AutoModel.from_pretrained("jinaai/jina-clip-v2", trust_remote_code=True)
|
|
170 |
|
171 |
# Corpus
|
172 |
sentences = [
|
173 |
-
"
|
174 |
-
"
|
175 |
-
"Un
|
176 |
-
"Ein
|
177 |
-
"
|
178 |
-
"
|
179 |
-
"
|
180 |
-
"
|
181 |
-
"
|
182 |
]
|
183 |
|
184 |
|
185 |
# Public image URLs or Pil
|
186 |
-
image_urls = ["https://i.ibb.co/
|
187 |
# Choose a matryoshka dimension, set to None to get the full 1024-dim vectors
|
188 |
truncate_dim = 512
|
189 |
|
@@ -194,13 +254,15 @@ image_embeddings = model.encode_image(
|
|
194 |
) # also accepts PIL.image, local filenames, dataURI
|
195 |
|
196 |
# Encode query text
|
197 |
-
query = "
|
198 |
query_embeddings = model.encode_text(
|
199 |
-
query, task=
|
200 |
)
|
201 |
|
202 |
# text to image
|
203 |
print("En -> Img: " + str(query_embeddings @ image_embeddings[0].T))
|
|
|
|
|
204 |
# text to text
|
205 |
print("En -> Ar: " + str(query_embeddings @ text_embeddings[0].T))
|
206 |
print("En -> Zh: " + str(query_embeddings @ text_embeddings[1].T))
|
@@ -212,9 +274,13 @@ print("En -> It: " + str(query_embeddings @ text_embeddings[6].T))
|
|
212 |
print("En -> Jp: " + str(query_embeddings @ text_embeddings[7].T))
|
213 |
print("En -> Ko: " + str(query_embeddings @ text_embeddings[8].T))
|
214 |
```
|
|
|
|
|
215 |
|
216 |
-
or via sentence-transformers:
|
217 |
|
|
|
|
|
|
|
218 |
```python
|
219 |
# !pip install sentence-transformers einops timm pillow
|
220 |
from sentence_transformers import SentenceTransformer
|
@@ -227,26 +293,26 @@ model = SentenceTransformer(
|
|
227 |
|
228 |
# Corpus
|
229 |
sentences = [
|
230 |
-
"
|
231 |
-
"
|
232 |
-
"Un
|
233 |
-
"Ein
|
234 |
-
"
|
235 |
-
"
|
236 |
-
"
|
237 |
-
"
|
238 |
-
"
|
239 |
]
|
240 |
|
241 |
# Public image URLs or Pil
|
242 |
-
image_urls = ["https://i.ibb.co/
|
243 |
|
244 |
text_embeddings = model.encode(sentences)
|
245 |
image_embeddings = model.encode(image_urls)
|
246 |
-
query = "
|
247 |
-
query_embeddings = model.encode(query)
|
248 |
```
|
249 |
-
|
250 |
|
251 |
## Contact
|
252 |
|
|
|
158 |
|
159 |
## Usage
|
160 |
|
161 |
+
<details>
|
162 |
+
<summary>via Jina AI [Embedding API](https://jina.ai/embeddings/)<</summary>
|
163 |
+
|
164 |
+
```bash
|
165 |
+
curl https://api.jina.ai/v1/embeddings \
|
166 |
+
-H "Content-Type: application/json" \
|
167 |
+
-H "Authorization: Bearer [JINA_AI_API_TOKEN]" \
|
168 |
+
-d @- <<EOFEOF
|
169 |
+
{
|
170 |
+
"model": "jina-clip-v2",
|
171 |
+
"dimensions": 1024,
|
172 |
+
"task": "retrieval.query",
|
173 |
+
"normalized": true,
|
174 |
+
"embedding_type": "float",
|
175 |
+
"input": [
|
176 |
+
{
|
177 |
+
"text": "غروب جميل على الشاطئ"
|
178 |
+
},
|
179 |
+
{
|
180 |
+
"text": "海滩上美丽的日落"
|
181 |
+
},
|
182 |
+
{
|
183 |
+
"text": "A beautiful sunset over the beach"
|
184 |
+
},
|
185 |
+
{
|
186 |
+
"text": "Un beau coucher de soleil sur la plage"
|
187 |
+
},
|
188 |
+
{
|
189 |
+
"text": "Ein wunderschöner Sonnenuntergang am Strand"
|
190 |
+
},
|
191 |
+
{
|
192 |
+
"text": "Ένα όμορφο ηλιοβασίλεμα πάνω από την παραλία"
|
193 |
+
},
|
194 |
+
{
|
195 |
+
"text": "समुद्र तट पर एक खूबसूरत सूर्यास्त"
|
196 |
+
},
|
197 |
+
{
|
198 |
+
"text": "Un bellissimo tramonto sulla spiaggia"
|
199 |
+
},
|
200 |
+
{
|
201 |
+
"text": "浜辺に沈む美しい夕日"
|
202 |
+
},
|
203 |
+
{
|
204 |
+
"text": "해변 위로 아름다운 일몰"
|
205 |
+
},
|
206 |
+
{
|
207 |
+
"image": "https://i.ibb.co/nQNGqL0/beach1.jpg"
|
208 |
+
},
|
209 |
+
{
|
210 |
+
"image": "https://i.ibb.co/r5w8hG8/beach2.jpg"
|
211 |
+
}
|
212 |
+
]
|
213 |
+
}
|
214 |
+
EOFEOF
|
215 |
+
```
|
216 |
+
|
217 |
+
</details>
|
218 |
+
|
219 |
+
|
220 |
+
<details>
|
221 |
+
<summary>via transformers:<</summary>
|
222 |
|
223 |
+
```python
|
224 |
```python
|
225 |
# !pip install transformers einops timm pillow
|
226 |
from transformers import AutoModel
|
|
|
230 |
|
231 |
# Corpus
|
232 |
sentences = [
|
233 |
+
"غروب جميل على الشاطئ", # Arabic
|
234 |
+
"海滩上美丽的日落", # Chinese
|
235 |
+
"Un beau coucher de soleil sur la plage", # French
|
236 |
+
"Ein wunderschöner Sonnenuntergang am Strand", # German
|
237 |
+
"Ένα όμορφο ηλιοβασίλεμα πάνω από την παραλία", # Greek
|
238 |
+
"समुद्र तट पर एक खूबसूरत सूर्यास्त", # Hindi
|
239 |
+
"Un bellissimo tramonto sulla spiaggia", # Italian
|
240 |
+
"浜辺に沈む美しい夕日", # Japanese
|
241 |
+
"해변 위로 아름다운 일몰", # Korean
|
242 |
]
|
243 |
|
244 |
|
245 |
# Public image URLs or Pil
|
246 |
+
image_urls = ["https://i.ibb.co/nQNGqL0/beach1.jpg", "https://i.ibb.co/r5w8hG8/beach2.jpg"]
|
247 |
# Choose a matryoshka dimension, set to None to get the full 1024-dim vectors
|
248 |
truncate_dim = 512
|
249 |
|
|
|
254 |
) # also accepts PIL.image, local filenames, dataURI
|
255 |
|
256 |
# Encode query text
|
257 |
+
query = "beautiful sunset over the beach" # English
|
258 |
query_embeddings = model.encode_text(
|
259 |
+
query, task='retrieval.query', truncate_dim=truncate_dim
|
260 |
)
|
261 |
|
262 |
# text to image
|
263 |
print("En -> Img: " + str(query_embeddings @ image_embeddings[0].T))
|
264 |
+
# image similarity
|
265 |
+
print("Img -> Img: " + str(image_embeddings[0] @ image_embeddings[1].T))
|
266 |
# text to text
|
267 |
print("En -> Ar: " + str(query_embeddings @ text_embeddings[0].T))
|
268 |
print("En -> Zh: " + str(query_embeddings @ text_embeddings[1].T))
|
|
|
274 |
print("En -> Jp: " + str(query_embeddings @ text_embeddings[7].T))
|
275 |
print("En -> Ko: " + str(query_embeddings @ text_embeddings[8].T))
|
276 |
```
|
277 |
+
```
|
278 |
+
</details>
|
279 |
|
|
|
280 |
|
281 |
+
<details>
|
282 |
+
<summary>via sentence-transformers:<</summary>
|
283 |
+
|
284 |
```python
|
285 |
# !pip install sentence-transformers einops timm pillow
|
286 |
from sentence_transformers import SentenceTransformer
|
|
|
293 |
|
294 |
# Corpus
|
295 |
sentences = [
|
296 |
+
"غروب جميل على الشاطئ", # Arabic
|
297 |
+
"海滩上美丽的日落", # Chinese
|
298 |
+
"Un beau coucher de soleil sur la plage", # French
|
299 |
+
"Ein wunderschöner Sonnenuntergang am Strand", # German
|
300 |
+
"Ένα όμορφο ηλιοβασίλεμα πάνω από την παραλία", # Greek
|
301 |
+
"समुद्र तट पर एक खूबसूरत सूर्यास्त", # Hindi
|
302 |
+
"Un bellissimo tramonto sulla spiaggia", # Italian
|
303 |
+
"浜辺に沈む美しい夕日", # Japanese
|
304 |
+
"해변 위로 아름다운 일몰", # Korean
|
305 |
]
|
306 |
|
307 |
# Public image URLs or Pil
|
308 |
+
image_urls = ["https://i.ibb.co/nQNGqL0/beach1.jpg", "https://i.ibb.co/r5w8hG8/beach2.jpg"]
|
309 |
|
310 |
text_embeddings = model.encode(sentences)
|
311 |
image_embeddings = model.encode(image_urls)
|
312 |
+
query = "beautiful sunset over the beach" # English
|
313 |
+
query_embeddings = model.encode(query)
|
314 |
```
|
315 |
+
</details>
|
316 |
|
317 |
## Contact
|
318 |
|