train.py

from pathlib import Path

import openai
import pinecone
from llama_index import StorageContext, VectorStoreIndex, download_loader
from llama_index.vector_stores import PineconeVectorStore

from environments import PINECONE_API_KEY, PINECONE_INDEX, OPENAI_API_KEY

openai.api_key = OPENAI_API_KEY

print('Start Loading Data ...')

PagedCSVReader = download_loader("PagedCSVReader")
loader = PagedCSVReader(encoding="utf-8")
documents = loader.load_data(file=Path('train-assets/training-target-simple.csv'))

# PDFReader = download_loader("PDFReader")
# loader = PDFReader()
# documents = loader.load_data(file=Path('./train-assets/training-target.pdf'))

UnstructuredURLLoader = download_loader("UnstructuredURLLoader")
# urls = [
#     'https://mosdecc.elchk.org.hk/index.php',
#     'https://mosdecc.elchk.org.hk/news_details.php?pkey=3762',
#     'https://mosdecc.elchk.org.hk/center_aboutus.php',
#     'https://mosdecc.elchk.org.hk/activity_details.php?pkey=1559&pg=1&news_cat=&news_year=&news_month=',
#     'https://mosdecc.elchk.org.hk/activity_details.php?pkey=1291&pg=1&news_cat=&news_year=&news_month=',
#     'https://mosdecc.elchk.org.hk/activity_details.php?pkey=1132&pg=1&news_cat=&news_year=&news_month=',
#     'https://mosdecc.elchk.org.hk/activity_details.php?pkey=1068&pg=2&news_cat=&news_year=all&news_month=all',
#     'https://mosdecc.elchk.org.hk/activity_details.php?pkey=1008&pg=2&news_cat=&news_year=all&news_month=all',
#     'https://mosdecc.elchk.org.hk/course.php',
#     'https://mosdecc.elchk.org.hk/service_member.php',
#     'https://mosdecc.elchk.org.hk/contactus.php',
# ]

urls = [
    'https://service.elchk.org.hk/unit_service3.php?center=77#intro',
    'https://service.elchk.org.hk/hot_project.php?pkey=102&tab=1&sub_tab=2',
    'https://www.etnet.com.hk/www/tc/soin/seg_detail.php?id=561',
]

# loader = UnstructuredURLLoader(urls=urls, continue_on_failure=True, headers={
#     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
#                   "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537",
# })
# documents = loader.load()

pinecone.init(
    api_key=PINECONE_API_KEY,
    environment='gcp-starter'
)
pinecone_index = pinecone.Index(PINECONE_INDEX)
vector_store = PineconeVectorStore(pinecone_index=pinecone_index)
storage_context = StorageContext.from_defaults(vector_store=vector_store)
index = VectorStoreIndex.from_documents(documents, storage_context=storage_context)

print('Done!')