File size: 3,230 Bytes
efe0924
1ec3d3a
 
 
 
efe0924
 
1ec3d3a
24b4b28
efe0924
 
 
 
 
1ec3d3a
 
 
efe0924
 
30e5d19
1ec3d3a
 
 
efe0924
65121b5
efe0924
5cf48e0
 
1ec3d3a
6a0a9f7
 
5cf48e0
efe0924
1ec3d3a
 
efe0924
 
1ec3d3a
efe0924
1ec3d3a
efe0924
 
 
 
 
 
1ec3d3a
8d30b62
 
efe0924
 
 
8d30b62
30e5d19
 
1ec3d3a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8d30b62
1ec3d3a
 
8d30b62
 
 
 
 
 
 
30e5d19
8d30b62
 
 
 
 
 
 
 
1ec3d3a
8d30b62
 
 
 
 
1ec3d3a
 
8d30b62
 
 
 
 
 
30e5d19
 
8d30b62
 
 
 
30e5d19
 
1ec3d3a
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
# for generate (gradio server) and finetune
datasets==2.13.0
sentencepiece==0.1.99
gradio==3.35.2
huggingface_hub==0.15.1
appdirs==1.4.4
fire==0.5.0
docutils==0.20.1
torch==2.0.1
evaluate==0.4.0
rouge_score==0.1.2
sacrebleu==2.3.1
scikit-learn==1.2.2
alt-profanity-check==1.2.2
better-profanity==0.7.0
numpy==1.24.3
pandas==2.0.2
matplotlib==3.7.1
loralib==0.1.1
bitsandbytes==0.39.0
accelerate==0.20.3
git+https://github.com/huggingface/peft.git@0b62b4378b4ce9367932c73540349da9a41bdea8
transformers==4.30.2
tokenizers==0.13.3
APScheduler==3.10.1

# optional for generate
pynvml==11.5.0
psutil==5.9.5
boto3==1.26.101
botocore==1.29.101

# optional for finetune
tensorboard==2.13.0
neptune==1.2.0

# for gradio client
gradio_client==0.2.7
beautifulsoup4==4.12.2
markdown==3.4.3

# data and testing
pytest==7.2.2
pytest-xdist==3.2.1
nltk==3.8.1
textstat==0.7.3
# pandoc==2.3
#pypandoc==1.11
pypandoc_binary==1.11
openpyxl==3.1.2
lm_dataformat==0.0.20
bioc==2.0

# falcon
einops==0.6.1
instructorembedding==1.0.1

# for gpt4all .env file, but avoid worrying about imports
python-dotenv==1.0.0

text-generation==0.6.0
# for tokenization when don't have HF tokenizer
tiktoken==0.4.0
# optional: for OpenAI endpoint or embeddings (requires key)
openai==0.27.8
# optional for chat with PDF
langchain==0.0.202
pypdf==3.9.1
# avoid textract, requires old six
#textract==1.6.5

# for HF embeddings
sentence_transformers==2.2.2

# local vector db
chromadb==0.3.25
# server vector db
#pymilvus==2.2.8

# weak url support, if can't install opencv etc. If comment-in this one, then comment-out unstructured[local-inference]==0.6.6
# unstructured==0.6.6

# strong support for images
# Requires on Ubuntu: sudo apt-get install libmagic-dev poppler-utils tesseract-ocr libreoffice
unstructured[local-inference]==0.7.4
#pdf2image==1.16.3
#pytesseract==0.3.10
pillow

pdfminer.six==20221105
urllib3
requests_file

#pdf2image==1.16.3
#pytesseract==0.3.10
tabulate==0.9.0
# FYI pandoc already part of requirements.txt

# JSONLoader, but makes some trouble for some users
# jq==1.4.1

# to check licenses
# Run: pip-licenses|grep -v 'BSD\|Apache\|MIT'
pip-licenses==4.3.0

# weaviate vector db
weaviate-client==3.20.0
# optional for chat with PDF
langchain==0.0.202
pypdf==3.9.1
# avoid textract, requires old six
#textract==1.6.5

# for HF embeddings
sentence_transformers==2.2.2

# local vector db
chromadb==0.3.25
# server vector db
#pymilvus==2.2.8

# weak url support, if can't install opencv etc. If comment-in this one, then comment-out unstructured[local-inference]==0.6.6
# unstructured==0.6.6

# strong support for images
# Requires on Ubuntu: sudo apt-get install libmagic-dev poppler-utils tesseract-ocr libreoffice
unstructured[local-inference]==0.7.4
#pdf2image==1.16.3
#pytesseract==0.3.10
pillow

pdfminer.six==20221105
urllib3
requests_file

#pdf2image==1.16.3
#pytesseract==0.3.10
tabulate==0.9.0
# FYI pandoc already part of requirements.txt

# JSONLoader, but makes some trouble for some users
# jq==1.4.1

# to check licenses
# Run: pip-licenses|grep -v 'BSD\|Apache\|MIT'
pip-licenses==4.3.0

# weaviate vector db
weaviate-client==3.20.0
faiss-gpu==1.7.2
arxiv==1.4.7
pymupdf==1.22.3 # AGPL license
# extract-msg==0.41.1  # GPL3