i get this errror

Token indices sequence length is longer than the specified maximum sequence length for this model (2341 > 1024). Running this sequence through the model will result in indexing errors


for following code

import os
import json
import boto3
from botocore.exceptions import NoCredentialsError, ClientError
from typing import List
import faiss
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import HuggingFaceBgeEmbeddings
from langchain.docstore.document import Document
from langchain.vectorstores import FAISS
from langchain.chains.question_answering import load_qa_chain
from langchain.llms import Ollama
from dotenv import load_dotenv

# Load environment variables
load_dotenv()

# Initialize the text splitter with reduced chunk size
text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)

S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME")
S3_REGION = os.getenv("S3_REGION")
S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL")
AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")

def get_s3_client():
  return boto3.client(
      's3',
      region_name=S3_REGION,
      endpoint_url=S3_ENDPOINT_URL,
      aws_access_key_id=AWS_ACCESS_KEY_ID,
      aws_secret_access_key=AWS_SECRET_ACCESS_KEY
  )

def list_s3_text_files():
  s3_client = get_s3_client()
  try:
      response = s3_client.list_objects_v2(Bucket=S3_BUCKET_NAME)
      if 'Contents' in response:
          all_files = [obj['Key'] for obj in response['Contents']]
          print(f"All files in bucket: {all_files}")
          text_files = [f for f in all_files if f.lower().endswith('.json')]
          print(f"Text files found: {text_files}")
          return text_files
      else:
          print(f"Bucket '{S3_BUCKET_NAME}' is empty or doesn't exist.")
          return []
  except NoCredentialsError:
      print("Error: No AWS credentials found.")
      return []
  except ClientError as e:
      print(f"Error listing bucket contents: {e}")
      return []

def fetch_text_from_s3(s3_client, file_key):
  try:
      response = s3_client.get_object(Bucket=S3_BUCKET_NAME, Key=file_key)
      content = response['Body'].read().decode('utf-8')
      return content
  except Exception as e:
      print(f"Error fetching {file_key} from S3: {e}")
      return ""

def process_s3_text_files():
  documents = []
  s3_client = get_s3_client()

  text_files = list_s3_text_files()
  print(f"Found {len(text_files)} text files in S3 bucket")

  for file_key in text_files:
      print(f"Processing {file_key}")

      # Fetch text content from S3
      text_content = fetch_text_from_s3(s3_client, file_key)

      if text_content:
          # Log the first 100 characters of the content
          print(f"Content fetched from {file_key}: {text_content[:100]}...")

          # Create a Document object with the text content
          doc = Document(page_content=text_content, metadata={"source": file_key})
          documents.append(doc)
      else:
          print(f"Warning: No content fetched from {file_key}")

  print(f"Total documents processed: {len(documents)}")
  return documents

def flatten_json(y):
  out = {}

  def flatten(x, name=''):
      if isinstance(x, dict):
          for a in x:
              flatten(x[a], name + a + '.')
      elif isinstance(x, list):
          i = 0
          for a in x:
              flatten(a, name + str(i) + '.')
              i += 1
      else:
          out[name[:-1]] = x

  flatten(y)
  return out

def create_and_save_faiss_index(documents: List[Document], index_path: str):
  if not documents:
      print("No documents to process. Skipping index creation.")
      return

  all_texts = []

  for doc in documents:
      json_data = json.loads(doc.page_content)

      # Flatten the JSON data
      flat_json = flatten_json(json_data)

      # Combine the flattened key-value pairs into text
      combined_text = "\n".join(f"{key}: {value}" for key, value in flat_json.items())

      # Add the combined text to the list using the updated text_splitter
      texts = text_splitter.split_text(combined_text)
      all_texts.extend(texts)

  # Log the number of texts
  print(f"Number of texts after splitting: {len(all_texts)}")

  # Convert texts into Document objects
  documents_to_index = [Document(page_content=text, metadata={}) for text in all_texts]

  # Initialize embeddings
  embeddings = HuggingFaceBgeEmbeddings(model_name='BAAI/bge-large-en')

  try:
      print("Creating FAISS index...")
      db = FAISS.from_documents(documents_to_index, embeddings)
      print("FAISS index created successfully.")

      # Ensure the directory exists
      os.makedirs(index_path, exist_ok=True)

      # Save the FAISS index
      db.save_local(index_path)
      print(f"FAISS index saved to {index_path}.")
  except Exception as e:
      print(f"Error creating or saving FAISS index: {e}")

def load_faiss_index(index_path: str):
  # Initialize embeddings
  embeddings = HuggingFaceBgeEmbeddings(model_name='BAAI/bge-large-en')
  try:
      print(f"Attempting to load FAISS index from {index_path}")
      if not os.path.exists(index_path):
          print(f"Error: Index directory {index_path} does not exist.")
          return None
      index_files = os.listdir(index_path)
      print(f"Files in index directory: {index_files}")
      # Set allow_dangerous_deserialization to True if you trust the source
      return FAISS.load_local(index_path, embeddings, allow_dangerous_deserialization=True)
  except Exception as e:
      print(f"Error loading FAISS index: {e}")
      return None

def query_documents(query: str, db: FAISS, k: int = 10):
  # Retrieve relevant documents
  docs = db.similarity_search(query, k=k)

  # Use Ollama for question answering
  llm = Ollama(base_url="https://artglobal-lisa.hf.space", model="phi3.5:latest")

  # Load the QA chain with map_reduce
  chain = load_qa_chain(llm, chain_type="map_reduce")

  # Run the chain using invoke instead of __call__
  result = chain.invoke({"input_documents": docs, "question": query}, return_only_outputs=True)

  return result.get('output_text', 'No answer generated.')

def process_documents_and_create_index():
  index_path = "/home/ec2-user/lisa/api/apps/pdfs/faiss_index"

  # Process text files from S3
  documents = process_s3_text_files()

  if not documents:
      print("No documents processed. Skipping index creation.")
      return

  # Create and save FAISS index
  create_and_save_faiss_index(documents, index_path)

  print("Documents processed and index created.")

def main_query_script():
  index_path = "/home/ec2-user/lisa/api/apps/pdfs/faiss_index"

  # Load the FAISS index
  db = load_faiss_index(index_path)
  if db is None:
      print("Failed to load FAISS index. Exiting.")
      return

  while True:
      query = input("Enter your query (or 'quit' to exit): ")
      if query.lower() == 'quit':
          break

      result = query_documents(query, db)
      print("Answer:", result)
      print()

if __name__ == "__main__":
  # Uncomment the following line to process documents and create index
  process_documents_and_create_index()

  # Run the main query script
  main_query_script()