channel id added to config

data/hugging_face_videos_dataset.py

@@ -0,0 +1,151 @@
+import os
+import re
+import time
+
+import torch
+import scrapetube
+from pytube import YouTube
+from faster_whisper import WhisperModel
+from tqdm import tqdm
+
+
+# Available models:
+# tiny.en, tiny, base.en, base, small.en, small, medium.en, medium
+# large-v1, large-v2, large-v3, large
+MODEL_NAME = "large-v3"
+AUDIO_SAVE_PATH = 'datasets/huggingface_audio/'
+TRANSCRIPTS_SAVE_PATH = 'datasets/huggingface_audio_transcribed/'
+
+if torch.cuda.is_available():
+    # requires: conda install -c anaconda cudnn
+    print(f"Using {MODEL_NAME} on GPU and float16")
+    model = WhisperModel(MODEL_NAME, device="cuda", compute_type="float16", device_index=[5])
+else:
+    print(f"Using {MODEL_NAME} on CPU and int8")
+    model = WhisperModel(MODEL_NAME, device="cpu", compute_type="int8")
+
+
+def replace_unallowed_chars(filename: str) -> str:
+    unallowed_chars = [' ', '/', '\\', ':', '*', '?', '"', '<', '>', '|']
+    for char in unallowed_chars:
+        filename = filename.replace(char, '_')
+    return filename
+
+
+def get_videos_urls(channel_url: str) -> list[str]:
+    videos = scrapetube.get_channel(channel_url=channel_url)
+    return [
+        f"https://www.youtube.com/watch?v={video['videoId']}"
+        for video in videos
+    ]
+
+
+def get_audio_from_video(video_url: str, save_path: str) -> tuple[str, int, str, int]:
+    yt = YouTube(video_url)
+    if check_if_file_exists(yt.title, save_path):
+        print(f'Audio already exists for: {yt.title}')
+        return (video_url, yt.title.replace(" ", "_")+".mp3", yt.title, yt.length)
+    else:
+        print(f'Downloading audio for: {yt.title}')
+        video = yt.streams.filter(only_audio=True).first()
+        out_file = video.download(output_path=save_path) 
+        base, ext = os.path.splitext(out_file)
+        new_filename = save_path + replace_unallowed_chars(yt.title) + '.mp3'
+        print(f'Saving audio to: {new_filename}')
+        os.rename(out_file, new_filename)
+        print(f'Video length: {yt.length} seconds')
+        return (video_url, new_filename, yt.title, yt.length)
+
+
+def check_if_file_exists(filename: str, save_path: str) -> bool:
+    title = filename.replace(' ', '_')
+    return any([
+        title in filename_
+        for filename_ in os.listdir(save_path)
+    ])
+
+
+def transcript_from_audio(audio_path: str) -> dict[str, list[str]]:
+    segments, info = model.transcribe(audio_path, beam_size=10)
+    return list(segments)
+
+
+def process_text(text: str) -> str:
+    text = text.strip()
+    text = re.sub('\s+', ' ', text)
+    return text
+
+
+def merge_transcripts_segements(
+    segments: list[str],
+    file_title: str,
+    num_segments_to_merge: int = 5,
+    ) -> dict[str, list[str]]:
+
+    merged_segments = {}
+    temp_text = ''
+    start_time = None
+    end_time = None
+
+    for i, segment in enumerate(segments):
+        if i % num_segments_to_merge == 0:
+            start_time = segment.start
+        end_time = segment.end
+        temp_text += segment.text + ' '
+
+        if (i + 1) % num_segments_to_merge == 0 or i == len(segments) - 1:
+            key = f'{start_time:.2f}_{end_time:.2f}'
+            merged_segments[key] = process_text(temp_text)
+            temp_text = ''
+
+    return merged_segments
+
+
+def main():
+    if not os.path.exists(AUDIO_SAVE_PATH):
+        os.makedirs(AUDIO_SAVE_PATH)
+    if not os.path.exists(TRANSCRIPTS_SAVE_PATH):
+        os.makedirs(TRANSCRIPTS_SAVE_PATH)
+
+    print('Getting videos urls')
+    videos_urls = get_videos_urls('https://www.youtube.com/@HuggingFace')
+
+    print('Downloading audio files')
+    audio_data = []
+    for video_url in tqdm(videos_urls):
+        try:
+            audio_data.append(
+                get_audio_from_video(video_url, save_path=AUDIO_SAVE_PATH)
+            )
+        except Exception as e:
+            print(f'Error downloading video: {video_url}')
+            print(e)
+
+    print('Transcribing audio files')
+    for video_url, filename, title, audio_length in tqdm(audio_data):
+        if check_if_file_exists(title, TRANSCRIPTS_SAVE_PATH):
+            print(f'Transcript already exists for: {title}')
+            continue
+        try: 
+            print(f'Transcribing: {title}')
+            start_time = time.time()
+            segments = transcript_from_audio(filename)
+            print(f'Transcription took: {time.time() - start_time:.1f} seconds')
+            merged_segments = merge_transcripts_segements(
+                segments,
+                title,
+                num_segments_to_merge=10
+            )
+            # save transcripts to separate files
+            title = replace_unallowed_chars(title)
+            for segment, text in merged_segments.items():
+                with open(f'{TRANSCRIPTS_SAVE_PATH}{title}_{segment}.txt', 'w') as f:
+                    video_url_with_time = f'{video_url}&t={float(segment.split("_")[0]):.0f}'
+                    f.write(f'source: {video_url_with_time}\n\n' + text)
+        except Exception as e:
+            print(f'Error transcribing: {title}')
+            print(e)
+
+
+if __name__ == '__main__':
+    main()

data/indexer.ipynb

@@ -7,16 +7,18 @@
    "outputs": [],
    "source": [
     "import math\n",
-    "import numpy as np\n",
     "from pathlib import Path\n",
+    "from typing import Any\n",
+    "\n",
+    "import numpy as np\n",
     "from tqdm import tqdm\n",
-    "from typing import List, Any\n",
     "from langchain.chains import RetrievalQA\n",
     "from langchain.embeddings import HuggingFaceEmbeddings, HuggingFaceInstructEmbeddings\n",
     "from langchain.document_loaders import TextLoader\n",
     "from langchain.indexes import VectorstoreIndexCreator\n",
     "from langchain.text_splitter import CharacterTextSplitter\n",
-    "from langchain.vectorstores import FAISS"
+    "from langchain.vectorstores import FAISS\n",
+    "from huggingface_hub import HfApi"
    ]
   },
   {
@@ -25,16 +27,32 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "def collect_docs(directory: str, docs: list[str], metadata: list[Any]):\n",
+    "    for p in Path(directory).iterdir():\n",
+    "        if not p.is_dir():\n",
+    "            with open(p) as f:\n",
+    "                # the first line is the source of the text\n",
+    "                source = f.readline().strip().replace('source: ', '')\n",
+    "                docs.append(f.read())\n",
+    "                metadata.append({\"source\": source})\n",
+    "        # break"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "DIRECTORIES = [\n",
+    "    \"./datasets/huggingface_docs/\",\n",
+    "    \"./datasets/huggingface_audio_transcribed/\"\n",
+    "]\n",
+    "\n",
     "docs = []\n",
     "metadata = []\n",
-    "for p in Path(\"./datasets/huggingface_docs/\").iterdir():\n",
-    "    if not p.is_dir():\n",
-    "        with open(p) as f:\n",
-    "            # the first line is the source of the text\n",
-    "            source = f.readline().strip().replace('source: ', '')\n",
-    "            docs.append(f.read())\n",
-    "            metadata.append({\"source\": source})\n",
-    "    # break\n",
+    "for directory in DIRECTORIES:\n",
+    "    collect_docs(directory, docs, metadata)\n",
     "\n",
     "print(f'number of documents: {len(docs)}')"
    ]
@@ -88,7 +106,7 @@
     "        if self.max_length < 0:\n",
     "            print('max_length is not specified, using model default max_seq_length')\n",
     "\n",
-    "    def embed_documents(self, texts: List[str]) -> List[List[float]]:\n",
+    "    def embed_documents(self, texts: list[str]) -> list[list[float]]:\n",
     "        all_embeddings = []\n",
     "        for text in tqdm(texts, desc=\"Embedding documents\"):\n",
     "            if len(text) > self.max_length and self.max_length > -1:\n",
@@ -109,7 +127,8 @@
     "        return all_embeddings\n",
     "\n",
     "\n",
-    "# max length fed to the model, if longer than max then chunks + averaging\n",
+    "# max length fed to the mode\n",
+    "# if longer than CHUNK_SIZE in previous steps: then N chunks + averaging of embeddings\n",
     "max_length = 512\n",
     "embedding_model = AverageInstructEmbeddings(  \n",
     "    model_name=model_name,\n",
@@ -143,8 +162,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "index_name = f'index-{model_name}-{chunk_size}-m{max_length}-notebooks'\n",
-    "index_name"
+    "index_name = f'index-{model_name}-{chunk_size}-m{max_length}-11_Jan_2024'\n",
+    "index_name = index_name.replace('/', '_')"
    ]
   },
   {
@@ -189,8 +208,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from huggingface_hub import HfApi\n",
-    "\n",
     "api = HfApi()\n",
     "api.create_repo(\n",
     "    repo_id=f'KonradSzafer/{index_name}',\n",
@@ -204,13 +221,6 @@
     "    repo_type='dataset',\n",
     ")"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
@@ -229,7 +239,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.11.5"
   },
   "orig_nbformat": 4
  },

data/requirements-audio.txt

@@ -0,0 +1,5 @@
+scrapetube>=2.5.1
+pytube>=15.0.0
+faster-whisper>=0.10.0
+torch>=2.0.1
+tqdm>=4.66.1

discord_bot/__main__.py

@@ -16,6 +16,7 @@ qa_engine = QAEngine(
 )
 client = DiscordClient(
     qa_engine=qa_engine,
+    channel_ids=config.discotd_channel_ids,
     num_last_messages=config.num_last_messages,
     use_names_in_context=config.use_names_in_context,
     enable_commands=config.enable_commands,

discord_bot/client/client.py

@@ -31,6 +31,7 @@ class DiscordClient(discord.Client):
     def __init__(
         self,
         qa_engine: QAEngine,
+        channel_ids: list[int] = [],
         num_last_messages: int = 5,
         use_names_in_context: bool = True,
         enable_commands: bool = True,
@@ -45,6 +46,7 @@ class DiscordClient(discord.Client):
             'The number of last messages in context should be at least 1'
 
         self.qa_engine: QAEngine = qa_engine
+        self.channel_ids: list[int] = channel_ids
         self.num_last_messages: int = num_last_messages
         self.use_names_in_context: bool = use_names_in_context
         self.enable_commands: bool = enable_commands
@@ -98,38 +100,34 @@ class DiscordClient(discord.Client):
 
 
     async def on_message(self, message):
-        if message.channel.id == 1162396480825462935:
-            """
-            Callback function to be called when a message is received.
-    
-            Args:
-                message (discord.Message): The received message.
-            """
-            if message.author == self.user:
-                return
-    
-            """
-            if self.enable_commands and message.content.startswith('!'):
-                if message.content == '!clear':
-                    await message.channel.purge()
-                    return        
-            """
-    
-    
-            last_messages = await self.get_last_messages(message)
-            context = '\n'.join(last_messages)
-    
-            logger.info('Received message: {0.content}'.format(message))
-            response = self.qa_engine.get_response(
-                question=message.content,
-                messages_context=context
+
+        if self.channel_ids and message.channel.id not in self.channel_ids:
+            return
+        
+        if message.author == self.user:
+            return
+        
+        """
+        if self.enable_commands and message.content.startswith('!'):
+            if message.content == '!clear':
+                await message.channel.purge()
+                return        
+        """
+
+        last_messages = await self.get_last_messages(message)
+        context = '\n'.join(last_messages)
+
+        logger.info('Received message: {0.content}'.format(message))
+        response = self.qa_engine.get_response(
+            question=message.content,
+            messages_context=context
+        )
+        logger.info('Sending response: {0}'.format(response))
+        try:
+            await self.send_message(
+                message,
+                response.get_answer(),
+                response.get_sources_as_text()
             )
-            logger.info('Sending response: {0}'.format(response))
-            try:
-                await self.send_message(
-                    message,
-                    response.get_answer(),
-                    response.get_sources_as_text()
-                )
-            except Exception as e:
-                logger.error('Failed to send response: {0}'.format(e))
+        except Exception as e:
+            logger.error('Failed to send response: {0}'.format(e))

qa_engine/config.py

@@ -36,6 +36,7 @@ class Config:
 
     # Discord bot config - optional
     discord_token: str = get_env('DISCORD_TOKEN', '-', warn=False)
+    discotd_channel_ids: list[int] = eval(get_env('DISCORD_CHANNEL_IDS', [], warn=False))
     num_last_messages: int = int(get_env('NUM_LAST_MESSAGES', 2, warn=False))
     use_names_in_context: bool = eval(get_env('USE_NAMES_IN_CONTEXT', 'False', warn=False))
     enable_commands: bool = eval(get_env('ENABLE_COMMANDS', 'True', warn=False))