{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "1aafbf18-de38-4fcf-8245-e2e9a584971f", "metadata": { "tags": [] }, "outputs": [], "source": [ "# ! pip install pymilvus==2.3.4\n", "# ! pip install pyarrow==12.0.0\n", "# !pip install -U sentence-transformers" ] }, { "cell_type": "code", "execution_count": 2, "id": "f1d8f101-f51b-4a50-b150-86e87c50c453", "metadata": { "tags": [] }, "outputs": [], "source": [ "import numpy as np\n", "import tensorflow as tf\n", "from tqdm import tqdm\n", "from dotenv import load_dotenv\n", "import os\n", "import pandas as pd\n", "from pymilvus import connections, utility\n", "from pymilvus import Collection, DataType, FieldSchema, CollectionSchema\n", "import multiprocessing\n", "from sentence_transformers import SentenceTransformer" ] }, { "cell_type": "code", "execution_count": null, "id": "4ad4e3ac-9685-4f12-8043-5fbcc373d3e1", "metadata": {}, "outputs": [], "source": [ "tf.config.list_physical_devices('GPU')" ] }, { "cell_type": "code", "execution_count": 3, "id": "da71d832-b8a7-452b-b736-538a3c069b54", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", " | index | \n", "category | \n", "short_description | \n", "
---|---|---|---|
0 | \n", "0 | \n", "SCIENCE | \n", "A closer look at water-splitting's solar fuel ... | \n", "
1 | \n", "1 | \n", "SCIENCE | \n", "An irresistible scent makes locusts swarm, stu... | \n", "
2 | \n", "2 | \n", "SCIENCE | \n", "Artificial intelligence warning: AI will know ... | \n", "
3 | \n", "3 | \n", "SCIENCE | \n", "Glaciers Could Have Sculpted Mars Valleys: Study | \n", "
4 | \n", "4 | \n", "SCIENCE | \n", "Perseid meteor shower 2020: What time and how ... | \n", "
... | \n", "... | \n", "... | \n", "... | \n", "
311171 | \n", "311171 | \n", "TECH | \n", "RIM CEO Thorsten Heins' 'Significant' Plans Fo... | \n", "
311172 | \n", "311172 | \n", "SPORTS | \n", "Maria Sharapova Stunned By Victoria Azarenka I... | \n", "
311173 | \n", "311173 | \n", "SPORTS | \n", "Giants Over Patriots, Jets Over Colts Among M... | \n", "
311174 | \n", "311174 | \n", "SPORTS | \n", "Aldon Smith Arrested: 49ers Linebacker Busted ... | \n", "
311175 | \n", "311175 | \n", "SPORTS | \n", "Dwight Howard Rips Teammates After Magic Loss ... | \n", "
311176 rows × 3 columns
\n", "