Update utils.py

utils.py

@@ -1,3 +1,83 @@
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple, Type
+import logging
+import json
+import os
+from datetime import datetime
+import hashlib
+import csv
+import requests
+import re
+import html
+import markdown2
+import torch 
+import sys
+import gc
+from pygments.lexers import guess_lexer, ClassNotFound
+import time
+import json
+import operator
+from typing import Annotated, Sequence, TypedDict
+import pprint
+
+import gradio as gr
+from pypinyin import lazy_pinyin
+import tiktoken
+import mdtex2html
+from markdown import markdown
+from pygments import highlight
+from pygments.lexers import guess_lexer,get_lexer_by_name
+from pygments.formatters import HtmlFormatter
+
+from langchain.chains import LLMChain, RetrievalQA
+from langgraph.graph import END, StateGraph
+from langchain_openai import ChatOpenAI
+from langchain_community.document_loaders import PyPDFLoader, WebBaseLoader, UnstructuredWordDocumentLoader, DirectoryLoader
+from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader
+from langchain.document_loaders.generic import GenericLoader
+from langchain.document_loaders.parsers import OpenAIWhisperParser
+from langchain.schema import AIMessage, HumanMessage
+from langchain_community.llms import HuggingFaceHub
+from langchain_community.llms import HuggingFaceTextGenInference
+from langchain_community.embeddings import HuggingFaceInstructEmbeddings, HuggingFaceEmbeddings, HuggingFaceBgeEmbeddings, HuggingFaceInferenceAPIEmbeddings
+from langchain_community.tools import DuckDuckGoSearchRun
+from langchain.retrievers.tavily_search_api import TavilySearchAPIRetriever
+from typing import Dict, TypedDict
+from langchain_core.messages import BaseMessage
+from langchain_openai import OpenAIEmbeddings
+from langchain.prompts import PromptTemplate
+
+
+from langchain import hub
+from langchain.output_parsers.openai_tools import PydanticToolsParser
+from langchain.prompts import PromptTemplate
+from langchain.schema import Document
+from langchain_community.tools.tavily_search import TavilySearchResults
+from langchain_community.vectorstores import Chroma
+from langchain_core.messages import BaseMessage, FunctionMessage
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.pydantic_v1 import BaseModel, Field
+from langchain_core.runnables import RunnablePassthrough
+from langchain_core.utils.function_calling import convert_to_openai_tool
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import Chroma
+from chromadb.errors import InvalidDimensionException
+import io
+from PIL import Image, ImageDraw, ImageOps, ImageFont
+import base64
+from tempfile import NamedTemporaryFile
+
+import nltk
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+from nltk.stem import WordNetLemmatizer
+nltk.download('punkt')
+
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+
+
+
 ################################################
 #Pfad, wo Docs/Bilder/Filme abgelegt werden können - lokal, also hier im HF Space (sonst auf eigenem Rechner)
 PATH_WORK = "."