Hugging Face's logo

Spaces:
lemesdaniel
/
Brunarize
Paused

App Files Community
lemesdaniel commited on
Commit
e00b837
·
verified ·
1 Parent(s): d91a6ca

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .env +3 -0
  2. .gitattributes +22 -0
  3. .idea/.gitignore +8 -0
  4. .idea/inspectionProfiles/Project_Default.xml +207 -0
  5. .idea/inspectionProfiles/profiles_settings.xml +6 -0
  6. .idea/linuxtips-llm.iml +10 -0
  7. .idea/misc.xml +7 -0
  8. .idea/modules.xml +8 -0
  9. .idea/vcs.xml +6 -0
  10. .idea/workspace.xml +97 -0
  11. README.md +1 -7
  12. app.py +61 -0
  13. docs/.ipynb_checkpoints/CLT-checkpoint.txt +0 -0
  14. docs/CLT.pdf +3 -0
  15. docs/CLT.txt +0 -0
  16. docs/M92TB4_2023-24_online.pdf +3 -0
  17. ingestion.py +45 -0
  18. llm-workshop-parte-1.ipynb +1013 -0
  19. llm-workshop-parte-2.ipynb +467 -0
  20. requirements.txt +14 -0
  21. venv/.gitignore +2 -0
  22. venv/bin/activate +87 -0
  23. venv/bin/activate.csh +55 -0
  24. venv/bin/activate.fish +103 -0
  25. venv/bin/activate.nu +96 -0
  26. venv/bin/activate.ps1 +61 -0
  27. venv/bin/activate_this.py +36 -0
  28. venv/bin/distro +8 -0
  29. venv/bin/dotenv +8 -0
  30. venv/bin/f2py +8 -0
  31. venv/bin/fastavro +8 -0
  32. venv/bin/fonttools +8 -0
  33. venv/bin/google +118 -0
  34. venv/bin/google-oauthlib-tool +8 -0
  35. venv/bin/gradio +8 -0
  36. venv/bin/httpx +8 -0
  37. venv/bin/huggingface-cli +8 -0
  38. venv/bin/jsondiff +41 -0
  39. venv/bin/jsonpatch +107 -0
  40. venv/bin/jsonpointer +69 -0
  41. venv/bin/jsonschema +8 -0
  42. venv/bin/langchain-server +8 -0
  43. venv/bin/langsmith +8 -0
  44. venv/bin/llmx +8 -0
  45. venv/bin/markdown-it +8 -0
  46. venv/bin/normalizer +8 -0
  47. venv/bin/openai +8 -0
  48. venv/bin/pip +8 -0
  49. venv/bin/pip3 +8 -0
  50. venv/bin/pip3.11 +8 -0
.env ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ OPENAI_API_KEY="sk-proj-lUsrclXDFwIkWRSHnjGmT3BlbkFJUzdAcPFIFiPVNYATNEh5"
2
+ PINECONE_API_KEY="189db110-8c84-4ef8-865d-eace31d1f1a7"
3
+ PINECONE_ENV="xxxxx"
.gitattributes CHANGED
@@ -33,3 +33,25 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ docs/CLT.pdf filter=lfs diff=lfs merge=lfs -text
37
+ docs/M92TB4_2023-24_online.pdf filter=lfs diff=lfs merge=lfs -text
38
+ venv/bin/ruff filter=lfs diff=lfs merge=lfs -text
39
+ venv/lib/python3.11/site-packages/PIL/.dylibs/libfreetype.6.dylib filter=lfs diff=lfs merge=lfs -text
40
+ venv/lib/python3.11/site-packages/PIL/.dylibs/libharfbuzz.0.dylib filter=lfs diff=lfs merge=lfs -text
41
+ venv/lib/python3.11/site-packages/altair/vegalite/v5/schema/__pycache__/channels.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
42
+ venv/lib/python3.11/site-packages/altair/vegalite/v5/schema/__pycache__/core.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
43
+ venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust.abi3.so filter=lfs diff=lfs merge=lfs -text
44
+ venv/lib/python3.11/site-packages/fontTools/misc/bezierTools.cpython-311-darwin.so filter=lfs diff=lfs merge=lfs -text
45
+ venv/lib/python3.11/site-packages/gradio/frpc_darwin_arm64_v0.2 filter=lfs diff=lfs merge=lfs -text
46
+ venv/lib/python3.11/site-packages/gradio/templates/frontend/assets/Index-d168f24a.js.map filter=lfs diff=lfs merge=lfs -text
47
+ venv/lib/python3.11/site-packages/numpy/.dylibs/libgfortran.5.dylib filter=lfs diff=lfs merge=lfs -text
48
+ venv/lib/python3.11/site-packages/numpy/.dylibs/libopenblas64_.0.dylib filter=lfs diff=lfs merge=lfs -text
49
+ venv/lib/python3.11/site-packages/numpy/core/_multiarray_umath.cpython-311-darwin.so filter=lfs diff=lfs merge=lfs -text
50
+ venv/lib/python3.11/site-packages/pandas/_libs/algos.cpython-311-darwin.so filter=lfs diff=lfs merge=lfs -text
51
+ venv/lib/python3.11/site-packages/pandas/_libs/groupby.cpython-311-darwin.so filter=lfs diff=lfs merge=lfs -text
52
+ venv/lib/python3.11/site-packages/pandas/_libs/hashtable.cpython-311-darwin.so filter=lfs diff=lfs merge=lfs -text
53
+ venv/lib/python3.11/site-packages/pandas/_libs/interval.cpython-311-darwin.so filter=lfs diff=lfs merge=lfs -text
54
+ venv/lib/python3.11/site-packages/pandas/_libs/join.cpython-311-darwin.so filter=lfs diff=lfs merge=lfs -text
55
+ venv/lib/python3.11/site-packages/pydantic_core/_pydantic_core.cpython-311-darwin.so filter=lfs diff=lfs merge=lfs -text
56
+ venv/lib/python3.11/site-packages/tiktoken/_tiktoken.cpython-311-darwin.so filter=lfs diff=lfs merge=lfs -text
57
+ venv/lib/python3.11/site-packages/tokenizers/tokenizers.cpython-311-darwin.so filter=lfs diff=lfs merge=lfs -text
.idea/.gitignore ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ # Default ignored files
2
+ /shelf/
3
+ /workspace.xml
4
+ # Editor-based HTTP Client requests
5
+ /httpRequests/
6
+ # Datasource local storage ignored files
7
+ /dataSources/
8
+ /dataSources.local.xml
.idea/inspectionProfiles/Project_Default.xml ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <component name="InspectionProjectProfileManager">
2
+ <profile version="1.0">
3
+ <option name="myName" value="Project Default" />
4
+ <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
5
+ <option name="ignoredPackages">
6
+ <value>
7
+ <list size="187">
8
+ <item index="0" class="java.lang.String" itemvalue="google-pasta" />
9
+ <item index="1" class="java.lang.String" itemvalue="tensorflow-estimator" />
10
+ <item index="2" class="java.lang.String" itemvalue="tzlocal" />
11
+ <item index="3" class="java.lang.String" itemvalue="greenlet" />
12
+ <item index="4" class="java.lang.String" itemvalue="pyathena" />
13
+ <item index="5" class="java.lang.String" itemvalue="scikit-learn" />
14
+ <item index="6" class="java.lang.String" itemvalue="tabulate" />
15
+ <item index="7" class="java.lang.String" itemvalue="cython" />
16
+ <item index="8" class="java.lang.String" itemvalue="unidecode" />
17
+ <item index="9" class="java.lang.String" itemvalue="validators" />
18
+ <item index="10" class="java.lang.String" itemvalue="pycparser" />
19
+ <item index="11" class="java.lang.String" itemvalue="python-slugify" />
20
+ <item index="12" class="java.lang.String" itemvalue="rfc3986" />
21
+ <item index="13" class="java.lang.String" itemvalue="country-converter" />
22
+ <item index="14" class="java.lang.String" itemvalue="markupsafe" />
23
+ <item index="15" class="java.lang.String" itemvalue="redis" />
24
+ <item index="16" class="java.lang.String" itemvalue="pyasn1-modules" />
25
+ <item index="17" class="java.lang.String" itemvalue="patsy" />
26
+ <item index="18" class="java.lang.String" itemvalue="mccabe" />
27
+ <item index="19" class="java.lang.String" itemvalue="pycryptodomex" />
28
+ <item index="20" class="java.lang.String" itemvalue="astunparse" />
29
+ <item index="21" class="java.lang.String" itemvalue="lxml" />
30
+ <item index="22" class="java.lang.String" itemvalue="petl" />
31
+ <item index="23" class="java.lang.String" itemvalue="soupsieve" />
32
+ <item index="24" class="java.lang.String" itemvalue="pycountry" />
33
+ <item index="25" class="java.lang.String" itemvalue="jsonschema" />
34
+ <item index="26" class="java.lang.String" itemvalue="xlrd" />
35
+ <item index="27" class="java.lang.String" itemvalue="libclang" />
36
+ <item index="28" class="java.lang.String" itemvalue="pydantic" />
37
+ <item index="29" class="java.lang.String" itemvalue="werkzeug" />
38
+ <item index="30" class="java.lang.String" itemvalue="mutagen" />
39
+ <item index="31" class="java.lang.String" itemvalue="rapidfuzz" />
40
+ <item index="32" class="java.lang.String" itemvalue="tensorboard-data-server" />
41
+ <item index="33" class="java.lang.String" itemvalue="typing-extensions" />
42
+ <item index="34" class="java.lang.String" itemvalue="loguru" />
43
+ <item index="35" class="java.lang.String" itemvalue="click" />
44
+ <item index="36" class="java.lang.String" itemvalue="spotipy" />
45
+ <item index="37" class="java.lang.String" itemvalue="sqlalchemy" />
46
+ <item index="38" class="java.lang.String" itemvalue="attrs" />
47
+ <item index="39" class="java.lang.String" itemvalue="psutil" />
48
+ <item index="40" class="java.lang.String" itemvalue="simplejson" />
49
+ <item index="41" class="java.lang.String" itemvalue="pyyaml" />
50
+ <item index="42" class="java.lang.String" itemvalue="boto3" />
51
+ <item index="43" class="java.lang.String" itemvalue="flatbuffers" />
52
+ <item index="44" class="java.lang.String" itemvalue="numpy-financial" />
53
+ <item index="45" class="java.lang.String" itemvalue="tensorboard" />
54
+ <item index="46" class="java.lang.String" itemvalue="execnet" />
55
+ <item index="47" class="java.lang.String" itemvalue="platformdirs" />
56
+ <item index="48" class="java.lang.String" itemvalue="awslambdaric" />
57
+ <item index="49" class="java.lang.String" itemvalue="redshift-connector" />
58
+ <item index="50" class="java.lang.String" itemvalue="humanize" />
59
+ <item index="51" class="java.lang.String" itemvalue="sqlparams" />
60
+ <item index="52" class="java.lang.String" itemvalue="idna" />
61
+ <item index="53" class="java.lang.String" itemvalue="cramjam" />
62
+ <item index="54" class="java.lang.String" itemvalue="rsa" />
63
+ <item index="55" class="java.lang.String" itemvalue="decorator" />
64
+ <item index="56" class="java.lang.String" itemvalue="networkx" />
65
+ <item index="57" class="java.lang.String" itemvalue="jellyfish" />
66
+ <item index="58" class="java.lang.String" itemvalue="isodate" />
67
+ <item index="59" class="java.lang.String" itemvalue="jaraco-functools" />
68
+ <item index="60" class="java.lang.String" itemvalue="pluggy" />
69
+ <item index="61" class="java.lang.String" itemvalue="cffi" />
70
+ <item index="62" class="java.lang.String" itemvalue="pycodestyle" />
71
+ <item index="63" class="java.lang.String" itemvalue="numpy" />
72
+ <item index="64" class="java.lang.String" itemvalue="pyasn1" />
73
+ <item index="65" class="java.lang.String" itemvalue="requests" />
74
+ <item index="66" class="java.lang.String" itemvalue="jpype1" />
75
+ <item index="67" class="java.lang.String" itemvalue="pyrsistent" />
76
+ <item index="68" class="java.lang.String" itemvalue="exceptiongroup" />
77
+ <item index="69" class="java.lang.String" itemvalue="tensorflow" />
78
+ <item index="70" class="java.lang.String" itemvalue="pyathenajdbc" />
79
+ <item index="71" class="java.lang.String" itemvalue="pytest-mock" />
80
+ <item index="72" class="java.lang.String" itemvalue="tenacity" />
81
+ <item index="73" class="java.lang.String" itemvalue="markdown" />
82
+ <item index="74" class="java.lang.String" itemvalue="ml-dtypes" />
83
+ <item index="75" class="java.lang.String" itemvalue="pyee" />
84
+ <item index="76" class="java.lang.String" itemvalue="aioitertools" />
85
+ <item index="77" class="java.lang.String" itemvalue="pygments" />
86
+ <item index="78" class="java.lang.String" itemvalue="pyarrow" />
87
+ <item index="79" class="java.lang.String" itemvalue="uritemplate" />
88
+ <item index="80" class="java.lang.String" itemvalue="scipy" />
89
+ <item index="81" class="java.lang.String" itemvalue="botocore" />
90
+ <item index="82" class="java.lang.String" itemvalue="google-auth-oauthlib" />
91
+ <item index="83" class="java.lang.String" itemvalue="backoff" />
92
+ <item index="84" class="java.lang.String" itemvalue="black" />
93
+ <item index="85" class="java.lang.String" itemvalue="et-xmlfile" />
94
+ <item index="86" class="java.lang.String" itemvalue="tensorflow-io-gcs-filesystem" />
95
+ <item index="87" class="java.lang.String" itemvalue="fastparquet" />
96
+ <item index="88" class="java.lang.String" itemvalue="pytz-deprecation-shim" />
97
+ <item index="89" class="java.lang.String" itemvalue="marko" />
98
+ <item index="90" class="java.lang.String" itemvalue="pandas" />
99
+ <item index="91" class="java.lang.String" itemvalue="termcolor" />
100
+ <item index="92" class="java.lang.String" itemvalue="blue" />
101
+ <item index="93" class="java.lang.String" itemvalue="cachetools" />
102
+ <item index="94" class="java.lang.String" itemvalue="statsmodels" />
103
+ <item index="95" class="java.lang.String" itemvalue="multidict" />
104
+ <item index="96" class="java.lang.String" itemvalue="google-api-python-client" />
105
+ <item index="97" class="java.lang.String" itemvalue="isort" />
106
+ <item index="98" class="java.lang.String" itemvalue="yarl" />
107
+ <item index="99" class="java.lang.String" itemvalue="pytz" />
108
+ <item index="100" class="java.lang.String" itemvalue="pyjwt" />
109
+ <item index="101" class="java.lang.String" itemvalue="absl-py" />
110
+ <item index="102" class="java.lang.String" itemvalue="protobuf" />
111
+ <item index="103" class="java.lang.String" itemvalue="stringcase" />
112
+ <item index="104" class="java.lang.String" itemvalue="joblib" />
113
+ <item index="105" class="java.lang.String" itemvalue="threadpoolctl" />
114
+ <item index="106" class="java.lang.String" itemvalue="googleapis-common-protos" />
115
+ <item index="107" class="java.lang.String" itemvalue="opt-einsum" />
116
+ <item index="108" class="java.lang.String" itemvalue="python-dateutil" />
117
+ <item index="109" class="java.lang.String" itemvalue="scramp" />
118
+ <item index="110" class="java.lang.String" itemvalue="psycopg2-binary" />
119
+ <item index="111" class="java.lang.String" itemvalue="convertdate" />
120
+ <item index="112" class="java.lang.String" itemvalue="gast" />
121
+ <item index="113" class="java.lang.String" itemvalue="frozenlist" />
122
+ <item index="114" class="java.lang.String" itemvalue="fsspec" />
123
+ <item index="115" class="java.lang.String" itemvalue="holidays" />
124
+ <item index="116" class="java.lang.String" itemvalue="playwright" />
125
+ <item index="117" class="java.lang.String" itemvalue="certifi" />
126
+ <item index="118" class="java.lang.String" itemvalue="oauthlib" />
127
+ <item index="119" class="java.lang.String" itemvalue="keras" />
128
+ <item index="120" class="java.lang.String" itemvalue="pyparsing" />
129
+ <item index="121" class="java.lang.String" itemvalue="commonmark" />
130
+ <item index="122" class="java.lang.String" itemvalue="karnak" />
131
+ <item index="123" class="java.lang.String" itemvalue="google-api-core" />
132
+ <item index="124" class="java.lang.String" itemvalue="beautifulsoup4" />
133
+ <item index="125" class="java.lang.String" itemvalue="pymeeus" />
134
+ <item index="126" class="java.lang.String" itemvalue="h5py" />
135
+ <item index="127" class="java.lang.String" itemvalue="iniconfig" />
136
+ <item index="128" class="java.lang.String" itemvalue="langdetect" />
137
+ <item index="129" class="java.lang.String" itemvalue="wrapt" />
138
+ <item index="130" class="java.lang.String" itemvalue="cryptography" />
139
+ <item index="131" class="java.lang.String" itemvalue="deezer-py" />
140
+ <item index="132" class="java.lang.String" itemvalue="orjson" />
141
+ <item index="133" class="java.lang.String" itemvalue="jinja2" />
142
+ <item index="134" class="java.lang.String" itemvalue="korean-lunar-calendar" />
143
+ <item index="135" class="java.lang.String" itemvalue="charset-normalizer" />
144
+ <item index="136" class="java.lang.String" itemvalue="deemix" />
145
+ <item index="137" class="java.lang.String" itemvalue="pandarallel" />
146
+ <item index="138" class="java.lang.String" itemvalue="shellingham" />
147
+ <item index="139" class="java.lang.String" itemvalue="oauth2client" />
148
+ <item index="140" class="java.lang.String" itemvalue="pathspec" />
149
+ <item index="141" class="java.lang.String" itemvalue="async-timeout" />
150
+ <item index="142" class="java.lang.String" itemvalue="frictionless" />
151
+ <item index="143" class="java.lang.String" itemvalue="scrapfly-sdk" />
152
+ <item index="144" class="java.lang.String" itemvalue="brotli" />
153
+ <item index="145" class="java.lang.String" itemvalue="cheroot" />
154
+ <item index="146" class="java.lang.String" itemvalue="hijri-converter" />
155
+ <item index="147" class="java.lang.String" itemvalue="more-itertools" />
156
+ <item index="148" class="java.lang.String" itemvalue="mypy-extensions" />
157
+ <item index="149" class="java.lang.String" itemvalue="httplib2" />
158
+ <item index="150" class="java.lang.String" itemvalue="pytest-xdist" />
159
+ <item index="151" class="java.lang.String" itemvalue="flake8" />
160
+ <item index="152" class="java.lang.String" itemvalue="newrelic" />
161
+ <item index="153" class="java.lang.String" itemvalue="requests-oauthlib" />
162
+ <item index="154" class="java.lang.String" itemvalue="google-auth-httplib2" />
163
+ <item index="155" class="java.lang.String" itemvalue="s3fs" />
164
+ <item index="156" class="java.lang.String" itemvalue="aiobotocore" />
165
+ <item index="157" class="java.lang.String" itemvalue="tomli" />
166
+ <item index="158" class="java.lang.String" itemvalue="urllib3" />
167
+ <item index="159" class="java.lang.String" itemvalue="funcy" />
168
+ <item index="160" class="java.lang.String" itemvalue="six" />
169
+ <item index="161" class="java.lang.String" itemvalue="typer" />
170
+ <item index="162" class="java.lang.String" itemvalue="pyflakes" />
171
+ <item index="163" class="java.lang.String" itemvalue="asn1crypto" />
172
+ <item index="164" class="java.lang.String" itemvalue="pytest" />
173
+ <item index="165" class="java.lang.String" itemvalue="wheel" />
174
+ <item index="166" class="java.lang.String" itemvalue="text-unidecode" />
175
+ <item index="167" class="java.lang.String" itemvalue="tzdata" />
176
+ <item index="168" class="java.lang.String" itemvalue="rich" />
177
+ <item index="169" class="java.lang.String" itemvalue="dill" />
178
+ <item index="170" class="java.lang.String" itemvalue="packaging" />
179
+ <item index="171" class="java.lang.String" itemvalue="pydash" />
180
+ <item index="172" class="java.lang.String" itemvalue="web-py" />
181
+ <item index="173" class="java.lang.String" itemvalue="chardet" />
182
+ <item index="174" class="java.lang.String" itemvalue="jmespath" />
183
+ <item index="175" class="java.lang.String" itemvalue="pandera" />
184
+ <item index="176" class="java.lang.String" itemvalue="tqdm" />
185
+ <item index="177" class="java.lang.String" itemvalue="s3transfer" />
186
+ <item index="178" class="java.lang.String" itemvalue="xlsx2csv" />
187
+ <item index="179" class="java.lang.String" itemvalue="typing-inspect" />
188
+ <item index="180" class="java.lang.String" itemvalue="colorama" />
189
+ <item index="181" class="java.lang.String" itemvalue="aiohttp" />
190
+ <item index="182" class="java.lang.String" itemvalue="grpcio" />
191
+ <item index="183" class="java.lang.String" itemvalue="aiosignal" />
192
+ <item index="184" class="java.lang.String" itemvalue="simpleeval" />
193
+ <item index="185" class="java.lang.String" itemvalue="google-auth" />
194
+ <item index="186" class="java.lang.String" itemvalue="openpyxl" />
195
+ </list>
196
+ </value>
197
+ </option>
198
+ </inspection_tool>
199
+ <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
200
+ <option name="ignoredIdentifiers">
201
+ <list>
202
+ <option value="str.*" />
203
+ </list>
204
+ </option>
205
+ </inspection_tool>
206
+ </profile>
207
+ </component>
.idea/inspectionProfiles/profiles_settings.xml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ <component name="InspectionProjectProfileManager">
2
+ <settings>
3
+ <option name="USE_PROJECT_PROFILE" value="false" />
4
+ <version value="1.0" />
5
+ </settings>
6
+ </component>
.idea/linuxtips-llm.iml ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <module type="PYTHON_MODULE" version="4">
3
+ <component name="NewModuleRootManager">
4
+ <content url="file://$MODULE_DIR$">
5
+ <excludeFolder url="file://$MODULE_DIR$/venv" />
6
+ </content>
7
+ <orderEntry type="inheritedJdk" />
8
+ <orderEntry type="sourceFolder" forTests="false" />
9
+ </component>
10
+ </module>
.idea/misc.xml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="Black">
4
+ <option name="sdkName" value="Python 3.11 (linuxtips-llm)" />
5
+ </component>
6
+ <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.11 (linuxtips-llm)" project-jdk-type="Python SDK" />
7
+ </project>
.idea/modules.xml ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="ProjectModuleManager">
4
+ <modules>
5
+ <module fileurl="file://$PROJECT_DIR$/.idea/linuxtips-llm.iml" filepath="$PROJECT_DIR$/.idea/linuxtips-llm.iml" />
6
+ </modules>
7
+ </component>
8
+ </project>
.idea/vcs.xml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="VcsDirectoryMappings">
4
+ <mapping directory="" vcs="Git" />
5
+ </component>
6
+ </project>
.idea/workspace.xml ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="AutoImportSettings">
4
+ <option name="autoReloadType" value="SELECTIVE" />
5
+ </component>
6
+ <component name="ChangeListManager">
7
+ <list default="true" id="7c6da864-434d-4e98-82d7-f0f1516bb908" name="Changes" comment="">
8
+ <change beforePath="$PROJECT_DIR$/.env" beforeDir="false" afterPath="$PROJECT_DIR$/.env" afterDir="false" />
9
+ <change beforePath="$PROJECT_DIR$/app.py" beforeDir="false" afterPath="$PROJECT_DIR$/app.py" afterDir="false" />
10
+ <change beforePath="$PROJECT_DIR$/ingestion.py" beforeDir="false" afterPath="$PROJECT_DIR$/ingestion.py" afterDir="false" />
11
+ <change beforePath="$PROJECT_DIR$/requirements.txt" beforeDir="false" afterPath="$PROJECT_DIR$/requirements.txt" afterDir="false" />
12
+ </list>
13
+ <option name="SHOW_DIALOG" value="false" />
14
+ <option name="HIGHLIGHT_CONFLICTS" value="true" />
15
+ <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
16
+ <option name="LAST_RESOLUTION" value="IGNORE" />
17
+ </component>
18
+ <component name="Git.Settings">
19
+ <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
20
+ </component>
21
+ <component name="ProjectColorInfo"><![CDATA[{
22
+ "associatedIndex": 8
23
+ }]]></component>
24
+ <component name="ProjectId" id="2fO9m3t33ty8re9vXMQLNYWZTCN" />
25
+ <component name="ProjectViewState">
26
+ <option name="hideEmptyMiddlePackages" value="true" />
27
+ <option name="showLibraryContents" value="true" />
28
+ </component>
29
+ <component name="PropertiesComponent"><![CDATA[{
30
+ "keyToString": {
31
+ "Python.app.executor": "Run",
32
+ "Python.ingestion.executor": "Run",
33
+ "RunOnceActivity.OpenProjectViewOnStart": "true",
34
+ "RunOnceActivity.ShowReadmeOnStart": "true",
35
+ "git-widget-placeholder": "main",
36
+ "last_opened_file_path": "/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm",
37
+ "node.js.detected.package.eslint": "true",
38
+ "node.js.detected.package.tslint": "true",
39
+ "node.js.selected.package.eslint": "(autodetect)",
40
+ "node.js.selected.package.tslint": "(autodetect)",
41
+ "nodejs_package_manager_path": "npm",
42
+ "vue.rearranger.settings.migration": "true"
43
+ }
44
+ }]]></component>
45
+ <component name="RunManager">
46
+ <configuration name="app" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
47
+ <module name="linuxtips-llm" />
48
+ <option name="ENV_FILES" value="" />
49
+ <option name="INTERPRETER_OPTIONS" value="" />
50
+ <option name="PARENT_ENVS" value="true" />
51
+ <envs>
52
+ <env name="PYTHONUNBUFFERED" value="1" />
53
+ </envs>
54
+ <option name="SDK_HOME" value="" />
55
+ <option name="SDK_NAME" value="Python 3.11" />
56
+ <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
57
+ <option name="IS_MODULE_SDK" value="false" />
58
+ <option name="ADD_CONTENT_ROOTS" value="true" />
59
+ <option name="ADD_SOURCE_ROOTS" value="true" />
60
+ <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
61
+ <option name="SCRIPT_NAME" value="$PROJECT_DIR$/app.py" />
62
+ <option name="PARAMETERS" value="" />
63
+ <option name="SHOW_COMMAND_LINE" value="false" />
64
+ <option name="EMULATE_TERMINAL" value="false" />
65
+ <option name="MODULE_MODE" value="false" />
66
+ <option name="REDIRECT_INPUT" value="false" />
67
+ <option name="INPUT_FILE" value="" />
68
+ <method v="2" />
69
+ </configuration>
70
+ </component>
71
+ <component name="SharedIndexes">
72
+ <attachedChunks>
73
+ <set>
74
+ <option value="bundled-python-sdk-09665e90c3a7-b11f5e8da5ad-com.jetbrains.pycharm.pro.sharedIndexes.bundled-PY-233.15026.15" />
75
+ </set>
76
+ </attachedChunks>
77
+ </component>
78
+ <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
79
+ <component name="TaskManager">
80
+ <task active="true" id="Default" summary="Default task">
81
+ <changelist id="7c6da864-434d-4e98-82d7-f0f1516bb908" name="Changes" comment="" />
82
+ <created>1713661412583</created>
83
+ <option name="number" value="Default" />
84
+ <option name="presentableId" value="Default" />
85
+ <updated>1713661412583</updated>
86
+ <workItem from="1713661413735" duration="6137000" />
87
+ </task>
88
+ <servers />
89
+ </component>
90
+ <component name="TypeScriptGeneratedFilesManager">
91
+ <option name="version" value="3" />
92
+ </component>
93
+ <component name="com.intellij.coverage.CoverageDataManagerImpl">
94
+ <SUITE FILE_PATH="coverage/linuxtips_llm$ingestion.coverage" NAME="ingestion Coverage Results" MODIFIED="1713740874365" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
95
+ <SUITE FILE_PATH="coverage/linuxtips_llm$app.coverage" NAME="app Coverage Results" MODIFIED="1713739572419" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
96
+ </component>
97
+ </project>
README.md CHANGED
@@ -1,12 +1,6 @@
1
  ---
2
  title: Brunarize
3
- emoji: 📉
4
- colorFrom: indigo
5
- colorTo: blue
6
  sdk: gradio
7
  sdk_version: 4.27.0
8
- app_file: app.py
9
- pinned: false
10
  ---
11
-
12
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
  title: Brunarize
3
+ app_file: app.py
 
 
4
  sdk: gradio
5
  sdk_version: 4.27.0
 
 
6
  ---
 
 
app.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+
4
+ from pinecone import Pinecone as PineconeClient
5
+
6
+ import gradio as gr
7
+ from langchain.chains import LLMChain
8
+ from langchain.prompts import PromptTemplate
9
+ from langchain_community.chat_models import ChatOpenAI
10
+ from langchain_community.vectorstores import Pinecone
11
+ from langchain_openai import OpenAIEmbeddings
12
+
13
+ from dotenv import load_dotenv, find_dotenv
14
+ load_dotenv(find_dotenv(), override=True)
15
+
16
+ PINECONE_API_KEY = os.environ.get('PINECONE_API_KEY')
17
+ PINECONE_ENV = os.environ.get('PINECONE_ENV')
18
+
19
+ embeddings = OpenAIEmbeddings()
20
+
21
+ pc = PineconeClient(api_key=PINECONE_API_KEY)
22
+
23
+ index_name = "linuxtips"
24
+ index = pc.Index(index_name)
25
+
26
+ index.describe_index_stats()
27
+
28
+ llm = ChatOpenAI(model='gpt-4-1106-preview', temperature=0)
29
+
30
+ template="""Assistente é uma IA jurídica que tira dúvidas.
31
+ Assistente elabora repostas simplificadas, com base no contexto fornecido.
32
+ Assistente fornece referências extraídas do contexto abaixo. Não gere links ou referência adicionais.
33
+ Assistente fornece a resposta em duas versões, português e inglês.
34
+ Ao final da resposta exiba no formato de lista as referências extraídas.
35
+ Caso não consiga encontrar no contexto abaixo ou caso a pergunta não esteja relacionada do contexto jurídico,
36
+ diga apenas 'Eu não sei!'
37
+
38
+ Question: {query}
39
+
40
+ Context: {context}
41
+ """
42
+
43
+ prompt = PromptTemplate(
44
+ template=template,
45
+ input_variables=["query", "context"],
46
+ )
47
+
48
+ def search(query):
49
+ docsearch = Pinecone.from_existing_index(index_name, embeddings)
50
+ docs = docsearch.similarity_search(query, k=3)
51
+ context = docs[0].page_content + docs[1].page_content + docs[2].page_content
52
+ resp = LLMChain(prompt=prompt, llm=llm)
53
+ return resp.run(query=query, context=context)
54
+
55
+ with gr.Blocks(title="Brunarize GPT", theme=gr.themes.Soft()) as ui:
56
+ gr.Markdown("# Sou Brunarize, uma IA que tem a seguros como base de conhecimento")
57
+ query = gr.Textbox(label='Faça a sua pergunta:', placeholder="EX: please tell me what are the main functions of an autarchy department?")
58
+ text_output = gr.Textbox(label="Resposta")
59
+ btn = gr.Button("Perguntar")
60
+ btn.click(fn=search, inputs=query, outputs=[text_output])
61
+ ui.launch(share=True)
docs/.ipynb_checkpoints/CLT-checkpoint.txt ADDED
The diff for this file is too large to render. See raw diff
 
docs/CLT.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e991030e7f5fce82b4a156636eeea6cd50d5baff7c93848d49930e157bf440b2
3
+ size 2116110
docs/CLT.txt ADDED
The diff for this file is too large to render. See raw diff
 
docs/M92TB4_2023-24_online.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49ee24ffbfe2af403bb86972e8464736e966643b3b938bf121fdf395b2118f30
3
+ size 5835181
ingestion.py ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from pinecone import Pinecone as PineconeClient, ServerlessSpec
3
+ from langchain_community.vectorstores import Pinecone
4
+ from langchain_openai import OpenAIEmbeddings
5
+ from langchain_community.document_loaders import PyPDFLoader
6
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
7
+ from dotenv import load_dotenv, find_dotenv
8
+
9
+ load_dotenv(find_dotenv(), override=True)
10
+
11
+ PINECONE_API_KEY = os.environ.get('PINECONE_API_KEY')
12
+ PINECONE_ENV = os.environ.get('PINECONE_ENV')
13
+
14
+ embeddings = OpenAIEmbeddings()
15
+
16
+ loader = PyPDFLoader("docs/M92TB4_2023-24_online.pdf")
17
+ data = loader.load()
18
+
19
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
20
+ texts = text_splitter.split_documents(data)
21
+
22
+ pinecone = PineconeClient(
23
+ api_key=PINECONE_API_KEY
24
+ )
25
+
26
+ index_name = "linuxtips"
27
+ if index_name not in pinecone.list_indexes().names():
28
+ pinecone.create_index(
29
+ name=index_name,
30
+ dimension=1536,
31
+ metric='euclidean',
32
+ spec=ServerlessSpec(
33
+ cloud="aws",
34
+ region="us-east-1"
35
+ )
36
+ )
37
+
38
+ index = pinecone.Index(index_name)
39
+
40
+ docsearch = Pinecone.from_texts([t.page_content for t in texts], embeddings, index_name=index_name)
41
+
42
+ # Test
43
+ query = "Assistant, please tell me what are the main functions of an autarchy department"
44
+ docs = docsearch.similarity_search(query)
45
+ print(docs[0].page_content)
llm-workshop-parte-1.ipynb ADDED
@@ -0,0 +1,1013 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 1,
6
+ "id": "85488dad-010f-4700-ab38-b1ec404c0beb",
7
+ "metadata": {},
8
+ "outputs": [],
9
+ "source": [
10
+ "import os\n",
11
+ "from dotenv import load_dotenv"
12
+ ]
13
+ },
14
+ {
15
+ "cell_type": "code",
16
+ "execution_count": 2,
17
+ "id": "c2cf39ce-0968-41d2-b7d6-b1748559dea8",
18
+ "metadata": {},
19
+ "outputs": [
20
+ {
21
+ "data": {
22
+ "text/plain": [
23
+ "'gcp-starter'"
24
+ ]
25
+ },
26
+ "execution_count": 2,
27
+ "metadata": {},
28
+ "output_type": "execute_result"
29
+ }
30
+ ],
31
+ "source": [
32
+ "load_dotenv('./.env')\n",
33
+ "os.environ.get('PINECONE_ENV')"
34
+ ]
35
+ },
36
+ {
37
+ "cell_type": "markdown",
38
+ "id": "751cbd61-ffd6-4b55-aafb-391ad04f3445",
39
+ "metadata": {},
40
+ "source": [
41
+ "## LangChain primeiros passos"
42
+ ]
43
+ },
44
+ {
45
+ "cell_type": "code",
46
+ "execution_count": 3,
47
+ "id": "96ac731e-db16-41d5-ab31-ba3b06ea0a7f",
48
+ "metadata": {},
49
+ "outputs": [
50
+ {
51
+ "name": "stdout",
52
+ "output_type": "stream",
53
+ "text": [
54
+ "\u001b[1mOpenAI\u001b[0m\n",
55
+ "Params: {'model_name': 'text-davinci-003', 'temperature': 0.7, 'max_tokens': 512, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'n': 1, 'request_timeout': None, 'logit_bias': {}}\n"
56
+ ]
57
+ }
58
+ ],
59
+ "source": [
60
+ "from langchain.llms import OpenAI\n",
61
+ "llm = OpenAI(model_name='text-davinci-003', temperature=0.7, max_tokens=512)\n",
62
+ "print(llm)"
63
+ ]
64
+ },
65
+ {
66
+ "cell_type": "code",
67
+ "execution_count": 4,
68
+ "id": "328bf9d9-9622-4334-aa1f-a83fbce99fad",
69
+ "metadata": {},
70
+ "outputs": [],
71
+ "source": [
72
+ "output = llm('explique como funções python funcionam')"
73
+ ]
74
+ },
75
+ {
76
+ "cell_type": "code",
77
+ "execution_count": 5,
78
+ "id": "c5f8d4f2-b06c-4364-880e-3be101ea95c6",
79
+ "metadata": {},
80
+ "outputs": [
81
+ {
82
+ "name": "stdout",
83
+ "output_type": "stream",
84
+ "text": [
85
+ "\n",
86
+ "\n",
87
+ "Em Python, uma função é um bloco de código que realiza uma tarefa específica. Uma função começa com a palavra-chave def, seguida pelo nome da função e parênteses. Os parâmetros (argumentos) são opcionais e são passados entre os parênteses. O corpo da função é identado e contém o código que é executado quando a função é chamada. O corpo da função pode conter declarações, instruções e outras funções. Quando a função é chamada, os argumentos fornecidos são passados para a função e ela executa o código no corpo da função. Uma função também pode retornar um valor ou um objeto usando a palavra-chave return. Se uma função não retornar nada, ela implícita retorna o valor None.\n"
88
+ ]
89
+ }
90
+ ],
91
+ "source": [
92
+ "print(output)"
93
+ ]
94
+ },
95
+ {
96
+ "cell_type": "code",
97
+ "execution_count": 6,
98
+ "id": "4ec80f14-ea2c-442f-a175-1fd756d4e1de",
99
+ "metadata": {},
100
+ "outputs": [
101
+ {
102
+ "name": "stdout",
103
+ "output_type": "stream",
104
+ "text": [
105
+ "13\n"
106
+ ]
107
+ }
108
+ ],
109
+ "source": [
110
+ "print(llm.get_num_tokens('explique como funções python funcionam'))"
111
+ ]
112
+ },
113
+ {
114
+ "cell_type": "code",
115
+ "execution_count": 7,
116
+ "id": "4e896bf2-733c-4b2b-a169-aa5f84a7d06d",
117
+ "metadata": {},
118
+ "outputs": [],
119
+ "source": [
120
+ "output = llm.generate(['... é a capital da França',\n",
121
+ " 'qual é a formula da área de um círculo?'])"
122
+ ]
123
+ },
124
+ {
125
+ "cell_type": "code",
126
+ "execution_count": 8,
127
+ "id": "3ddeffc8-1958-4ed8-8b94-9f351afad7df",
128
+ "metadata": {},
129
+ "outputs": [
130
+ {
131
+ "name": "stdout",
132
+ "output_type": "stream",
133
+ "text": [
134
+ "[[Generation(text='\\n\\nParis.', generation_info={'finish_reason': 'stop', 'logprobs': None})], [Generation(text='\\n\\nA área de um círculo é dada pela fórmula A = π * r², onde A representa a área e r é o raio do círculo.', generation_info={'finish_reason': 'stop', 'logprobs': None})]]\n"
135
+ ]
136
+ }
137
+ ],
138
+ "source": [
139
+ "print(output.generations)"
140
+ ]
141
+ },
142
+ {
143
+ "cell_type": "code",
144
+ "execution_count": 9,
145
+ "id": "3530c0d4-b489-4ec2-8cd4-34ce73f8a9e2",
146
+ "metadata": {},
147
+ "outputs": [
148
+ {
149
+ "data": {
150
+ "text/plain": [
151
+ "'\\n\\nParis.'"
152
+ ]
153
+ },
154
+ "execution_count": 9,
155
+ "metadata": {},
156
+ "output_type": "execute_result"
157
+ }
158
+ ],
159
+ "source": [
160
+ "output.generations[0][0].text"
161
+ ]
162
+ },
163
+ {
164
+ "cell_type": "code",
165
+ "execution_count": 10,
166
+ "id": "f1430cb9-367c-4b6e-a9ca-4aff15d465ba",
167
+ "metadata": {},
168
+ "outputs": [],
169
+ "source": [
170
+ "output = llm.generate(['escreva um slogan original para um restaurante de sushi'] * 3)"
171
+ ]
172
+ },
173
+ {
174
+ "cell_type": "code",
175
+ "execution_count": 11,
176
+ "id": "493450f7-1c88-4e88-9671-a4e232e670d7",
177
+ "metadata": {},
178
+ "outputs": [
179
+ {
180
+ "name": "stdout",
181
+ "output_type": "stream",
182
+ "text": [
183
+ "\n",
184
+ "\n",
185
+ "\"Sabor de Oriente: Sushi como nunca experimentou!\"\n",
186
+ "\n",
187
+ "\"Sushi com sabor de tradição - Delícia no seu prato!\"\n",
188
+ "\n",
189
+ "\"Uma experiência única de sabores: Sushi Time!\""
190
+ ]
191
+ }
192
+ ],
193
+ "source": [
194
+ "for o in output.generations:\n",
195
+ " print(o[0].text, end='')"
196
+ ]
197
+ },
198
+ {
199
+ "cell_type": "markdown",
200
+ "id": "f084ee4e-37a8-4eff-918b-e76fc287216c",
201
+ "metadata": {},
202
+ "source": [
203
+ "## ChatModels: GPT-3.5-Turbo e GPT-4"
204
+ ]
205
+ },
206
+ {
207
+ "cell_type": "code",
208
+ "execution_count": 12,
209
+ "id": "e419a09e-c68b-4199-a35c-6312bfac7701",
210
+ "metadata": {},
211
+ "outputs": [],
212
+ "source": [
213
+ "from langchain.schema import(\n",
214
+ " AIMessage,\n",
215
+ " HumanMessage,\n",
216
+ " SystemMessage\n",
217
+ ")\n",
218
+ "\n",
219
+ "from langchain.chat_models import ChatOpenAI"
220
+ ]
221
+ },
222
+ {
223
+ "cell_type": "code",
224
+ "execution_count": 13,
225
+ "id": "a819d353-0736-4d73-9ae5-3ffa43f63b97",
226
+ "metadata": {},
227
+ "outputs": [],
228
+ "source": [
229
+ "chat = ChatOpenAI(model_name='gpt-3.5-turbo', temperature=0.5, max_tokens=1024)\n",
230
+ "messages = [\n",
231
+ " SystemMessage(content='Você é um especialista em machine learning que responde tudo em português.'),\n",
232
+ " HumanMessage(content='explique em um parágrafo o que é machine learning.')\n",
233
+ "]\n",
234
+ "output = chat(messages)"
235
+ ]
236
+ },
237
+ {
238
+ "cell_type": "code",
239
+ "execution_count": 14,
240
+ "id": "4997eb2c-0312-432b-b9ca-563b1f867ec3",
241
+ "metadata": {},
242
+ "outputs": [
243
+ {
244
+ "name": "stdout",
245
+ "output_type": "stream",
246
+ "text": [
247
+ "Machine learning, ou aprendizado de máquina, é uma subárea da inteligência artificial que se concentra no desenvolvimento de algoritmos e modelos computacionais capazes de aprender e tomar decisões a partir de dados. Ao invés de serem explicitamente programados, esses algoritmos são treinados com exemplos e dados relevantes, permitindo que eles identifiquem padrões, façam previsões e tomem decisões autônomas. O objetivo do machine learning é capacitar as máquinas a aprenderem e melhorarem seu desempenho ao longo do tempo, sem a necessidade de intervenção humana constante.\n"
248
+ ]
249
+ }
250
+ ],
251
+ "source": [
252
+ "print(output.content)"
253
+ ]
254
+ },
255
+ {
256
+ "cell_type": "markdown",
257
+ "id": "7b053d3d-0f7a-47ff-ab69-dee325ad100f",
258
+ "metadata": {},
259
+ "source": [
260
+ "## Prompt Templates"
261
+ ]
262
+ },
263
+ {
264
+ "cell_type": "code",
265
+ "execution_count": 15,
266
+ "id": "c668f7d9-888e-4a68-8cfe-6cf1d420d285",
267
+ "metadata": {},
268
+ "outputs": [],
269
+ "source": [
270
+ "from langchain.prompts import PromptTemplate"
271
+ ]
272
+ },
273
+ {
274
+ "cell_type": "code",
275
+ "execution_count": 16,
276
+ "id": "a9c59544-91e4-4471-a05b-a5043be6e1a3",
277
+ "metadata": {},
278
+ "outputs": [
279
+ {
280
+ "name": "stdout",
281
+ "output_type": "stream",
282
+ "text": [
283
+ "input_variables=['idioma', 'virus'] template='Você é um virologista experiente.\\nEscreva algumas frases sobre o sequinte {virus} em {idioma}.'\n"
284
+ ]
285
+ }
286
+ ],
287
+ "source": [
288
+ "template = \"\"\"Você é um virologista experiente.\n",
289
+ "Escreva algumas frases sobre o sequinte {virus} em {idioma}.\"\"\"\n",
290
+ "\n",
291
+ "prompt = PromptTemplate(\n",
292
+ " input_variables=['virus', 'idioma'],\n",
293
+ " template=template\n",
294
+ ")\n",
295
+ "print(prompt)"
296
+ ]
297
+ },
298
+ {
299
+ "cell_type": "code",
300
+ "execution_count": 17,
301
+ "id": "aac42a33-a6b3-4216-a8f9-daf3d2a0f86a",
302
+ "metadata": {},
303
+ "outputs": [
304
+ {
305
+ "name": "stdout",
306
+ "output_type": "stream",
307
+ "text": [
308
+ "\n",
309
+ "\n",
310
+ "1. HIV is a complex virus that continues to challenge the scientific community.\n",
311
+ "2. HIV infection can be prevented through safe sexual practices and needle exchange programs.\n",
312
+ "3. Antiretroviral therapy is the standard of care for HIV infection.\n",
313
+ "4. Early detection of HIV is essential to successful treatment.\n",
314
+ "5. HIV is a global health issue that requires collaborative efforts.\n"
315
+ ]
316
+ }
317
+ ],
318
+ "source": [
319
+ "from langchain.llms import OpenAI\n",
320
+ "llm = OpenAI(model_name='text-davinci-003', temperature=0.7)\n",
321
+ "output = llm(prompt.format(virus='HIV', idioma='Inglês'))\n",
322
+ "print(output)"
323
+ ]
324
+ },
325
+ {
326
+ "cell_type": "markdown",
327
+ "id": "8bd30f0e-8d81-4e4b-a0ae-692a4d8fbdc7",
328
+ "metadata": {},
329
+ "source": [
330
+ "## Simple Chains"
331
+ ]
332
+ },
333
+ {
334
+ "cell_type": "code",
335
+ "execution_count": 18,
336
+ "id": "b4af114a-398b-4845-a1cb-0f4745d4e798",
337
+ "metadata": {},
338
+ "outputs": [],
339
+ "source": [
340
+ "from langchain.chat_models import ChatOpenAI\n",
341
+ "from langchain.prompts import PromptTemplate\n",
342
+ "from langchain.chains import LLMChain\n",
343
+ "\n",
344
+ "llm = ChatOpenAI(model_name='gpt-3.5-turbo', temperature=0.5)\n",
345
+ "\n",
346
+ "template = \"\"\"Você é um virologista experiente.\n",
347
+ "Escreva um resumo sobre o sequinte {virus} em {idioma}.\"\"\"\n",
348
+ "\n",
349
+ "prompt = PromptTemplate(\n",
350
+ " input_variables=['virus', 'idioma'],\n",
351
+ " template=template\n",
352
+ ")\n",
353
+ "\n",
354
+ "chain = LLMChain(llm=llm, prompt=prompt)\n",
355
+ "output = chain.run({'virus': 'HIV', 'idioma': 'english'})"
356
+ ]
357
+ },
358
+ {
359
+ "cell_type": "code",
360
+ "execution_count": 19,
361
+ "id": "005417b0-82ce-44d2-b1be-34338711e94c",
362
+ "metadata": {},
363
+ "outputs": [
364
+ {
365
+ "data": {
366
+ "text/plain": [
367
+ "'HIV, or Human Immunodeficiency Virus, is a retrovirus that attacks the immune system, specifically the CD4 cells (also known as T cells) which play a crucial role in fighting off infections. This virus is primarily transmitted through sexual contact, blood transfusion, sharing needles, or from an infected mother to her child during childbirth or breastfeeding.\\n\\nOnce HIV enters the body, it replicates rapidly and progressively weakens the immune system. As a result, individuals infected with HIV become more susceptible to opportunistic infections and certain types of cancers. Without treatment, HIV can progress to AIDS (Acquired Immunodeficiency Syndrome), the most advanced stage of the infection.\\n\\nThe symptoms of HIV can vary from person to person, and some individuals may experience flu-like symptoms shortly after infection. However, in many cases, individuals may not show any symptoms for several years. This is why it is crucial to get tested for HIV if there has been any potential exposure.\\n\\nWhile there is currently no cure for HIV, significant advancements have been made in antiretroviral therapy (ART). ART involves a combination of medications that can effectively suppress viral replication, allowing individuals with HIV to live long and healthy lives. Additionally, early diagnosis and treatment can significantly reduce the risk of transmitting the virus to others.\\n\\nPrevention plays a vital role in controlling the spread of HIV. This includes practicing safe sex by using condoms, getting tested regularly, and avoiding sharing needles or other drug paraphernalia. Education and awareness programs are also crucial in promoting HIV prevention and reducing stigma surrounding the virus.\\n\\nIn conclusion, HIV is a complex virus that affects the immune system and can lead to AIDS if left untreated. However, with advancements in treatment and prevention methods, individuals with HIV can lead fulfilling lives while managing their condition.'"
368
+ ]
369
+ },
370
+ "execution_count": 19,
371
+ "metadata": {},
372
+ "output_type": "execute_result"
373
+ }
374
+ ],
375
+ "source": [
376
+ "output"
377
+ ]
378
+ },
379
+ {
380
+ "cell_type": "markdown",
381
+ "id": "7b8754c4-0f98-4bec-b746-0fe63ae01bba",
382
+ "metadata": {},
383
+ "source": [
384
+ "## Sequential Chains"
385
+ ]
386
+ },
387
+ {
388
+ "cell_type": "code",
389
+ "execution_count": 20,
390
+ "id": "8a208c4a-55dc-4c81-9e52-7e03b16225a2",
391
+ "metadata": {},
392
+ "outputs": [
393
+ {
394
+ "name": "stdout",
395
+ "output_type": "stream",
396
+ "text": [
397
+ "\n",
398
+ "\n",
399
+ "\u001b[1m> Entering new SimpleSequentialChain chain...\u001b[0m\n",
400
+ "\u001b[36;1m\u001b[1;3m\n",
401
+ "\n",
402
+ "def softmax(x):\n",
403
+ " \"\"\"Calcula a função softmax para uma lista de números x.\"\"\"\n",
404
+ " # Use exp para obter os exponenciais de cada elemento da lista.\n",
405
+ " exp_list = [np.exp(i) for i in x]\n",
406
+ "\n",
407
+ " # Calcule a soma dos exponenciais.\n",
408
+ " sum_exp_list = sum(exp_list)\n",
409
+ "\n",
410
+ " # Calcule a probabilidade softmax.\n",
411
+ " softmax_list = [j/sum_exp_list for j in exp_list]\n",
412
+ "\n",
413
+ " return softmax_list\u001b[0m\n",
414
+ "\u001b[33;1m\u001b[1;3mA função softmax calcula a probabilidade de cada elemento em uma lista de números através da aplicação de uma função exponencial normalizada. \n",
415
+ "\n",
416
+ "Primeiro, a função utiliza a biblioteca numpy (importada como \"np\") para calcular o exponencial de cada elemento da lista de números \"x\" utilizando a função np.exp(). Esses exponenciais são armazenados em uma nova lista chamada \"exp_list\".\n",
417
+ "\n",
418
+ "Em seguida, a função realiza a soma de todos os elementos da lista \"exp_list\" utilizando a função sum(). Essa soma é armazenada em uma variável chamada \"sum_exp_list\". \n",
419
+ "\n",
420
+ "Por fim, a função calcula a probabilidade softmax de cada elemento da lista original \"x\" dividindo cada elemento pelo valor da soma dos exponenciais (\"sum_exp_list\"). Os resultados são armazenados em uma lista chamada \"softmax_list\".\n",
421
+ "\n",
422
+ "Essencialmente, a probabilidade softmax é calculada aplicando a função exponencial em cada elemento, normalizando os resultados pela soma dos exponenciais. O resultado final é uma lista de probabilidades, onde cada elemento representa a probabilidade do elemento correspondente na lista original \"x\".\u001b[0m\n",
423
+ "\n",
424
+ "\u001b[1m> Finished chain.\u001b[0m\n"
425
+ ]
426
+ }
427
+ ],
428
+ "source": [
429
+ "from langchain.chat_models import ChatOpenAI\n",
430
+ "from langchain.prompts import PromptTemplate\n",
431
+ "from langchain.llms import OpenAI\n",
432
+ "from langchain.chains import LLMChain, SimpleSequentialChain\n",
433
+ "\n",
434
+ "llm1 = OpenAI(model_name='text-davinci-003', temperature=0.7, max_tokens=1024)\n",
435
+ "prompt1 = PromptTemplate(\n",
436
+ " input_variables=['conceito'],\n",
437
+ " template=\"\"\"Você é um cientista experiente e programador Python.\n",
438
+ " Escreva uma função que implementa o conceito de {conceito}.\"\"\"\n",
439
+ ")\n",
440
+ "\n",
441
+ "chain1 = LLMChain(llm=llm1, prompt=prompt1)\n",
442
+ "\n",
443
+ "#--- segunda chain ----\n",
444
+ "\n",
445
+ "llm2 = ChatOpenAI(model_name='gpt-3.5-turbo', temperature=1.2)\n",
446
+ "prompt2 = PromptTemplate(\n",
447
+ " input_variables=['function'],\n",
448
+ " template=\"\"\"Dada a função {function} Python, descreva como funciona da forma mais detalhada possível.\"\"\"\n",
449
+ ")\n",
450
+ "\n",
451
+ "chain2 = LLMChain(llm=llm2, prompt=prompt2)\n",
452
+ "\n",
453
+ "overall_chain = SimpleSequentialChain(chains=[chain1, chain2], verbose=True)\n",
454
+ "output = overall_chain.run(\"softmax\")"
455
+ ]
456
+ },
457
+ {
458
+ "cell_type": "markdown",
459
+ "id": "cf609a40-895b-4883-9e82-9e84b6890c59",
460
+ "metadata": {},
461
+ "source": [
462
+ "## LangChain Agents"
463
+ ]
464
+ },
465
+ {
466
+ "cell_type": "code",
467
+ "execution_count": 21,
468
+ "id": "feaecdf7-3567-491c-902d-057dfd941354",
469
+ "metadata": {},
470
+ "outputs": [
471
+ {
472
+ "data": {
473
+ "text/plain": [
474
+ "146306.05007233328"
475
+ ]
476
+ },
477
+ "execution_count": 21,
478
+ "metadata": {},
479
+ "output_type": "execute_result"
480
+ }
481
+ ],
482
+ "source": [
483
+ "# exemplo exponenciação\n",
484
+ "\n",
485
+ "5.1 ** 7.3"
486
+ ]
487
+ },
488
+ {
489
+ "cell_type": "code",
490
+ "execution_count": 22,
491
+ "id": "87e7d9c9-ed99-4bc3-9153-16607dea7f62",
492
+ "metadata": {},
493
+ "outputs": [],
494
+ "source": [
495
+ "from langchain_experimental.agents.agent_toolkits import create_python_agent\n",
496
+ "from langchain_experimental.tools.python.tool import PythonAstREPLTool\n",
497
+ "from langchain.llms import OpenAI"
498
+ ]
499
+ },
500
+ {
501
+ "cell_type": "code",
502
+ "execution_count": 23,
503
+ "id": "57b7a63c-7dea-42c3-af36-5359438d9853",
504
+ "metadata": {},
505
+ "outputs": [
506
+ {
507
+ "name": "stdout",
508
+ "output_type": "stream",
509
+ "text": [
510
+ "\n",
511
+ "\n",
512
+ "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
513
+ "\u001b[32;1m\u001b[1;3m I need to calculate the power of 5.1 to 7.3\n",
514
+ "Action: python_repl_ast\n",
515
+ "Action Input: 5.1 ** 7.3\u001b[0m\n",
516
+ "Observation: \u001b[36;1m\u001b[1;3m146306.05007233328\u001b[0m\n",
517
+ "Thought:\u001b[32;1m\u001b[1;3m I now know the final answer\n",
518
+ "Final Answer: 146306.05007233328\u001b[0m\n",
519
+ "\n",
520
+ "\u001b[1m> Finished chain.\u001b[0m\n"
521
+ ]
522
+ },
523
+ {
524
+ "data": {
525
+ "text/plain": [
526
+ "'146306.05007233328'"
527
+ ]
528
+ },
529
+ "execution_count": 23,
530
+ "metadata": {},
531
+ "output_type": "execute_result"
532
+ }
533
+ ],
534
+ "source": [
535
+ "llm = OpenAI(temperature=0)\n",
536
+ "agent_executor = create_python_agent(\n",
537
+ " llm=llm,\n",
538
+ " tool=PythonAstREPLTool(),\n",
539
+ " verbose=True\n",
540
+ ")\n",
541
+ "agent_executor.run('qual a resposta para 5.1 elevado à potência de 7.3')"
542
+ ]
543
+ },
544
+ {
545
+ "cell_type": "code",
546
+ "execution_count": 24,
547
+ "id": "f72f929d-202f-4506-8760-5bf1ed06f9f2",
548
+ "metadata": {},
549
+ "outputs": [
550
+ {
551
+ "name": "stdout",
552
+ "output_type": "stream",
553
+ "text": [
554
+ "1559776268.6285\n"
555
+ ]
556
+ }
557
+ ],
558
+ "source": [
559
+ "from math import factorial; print(f\"{round(factorial(20)**0.5, 4)}\")"
560
+ ]
561
+ },
562
+ {
563
+ "cell_type": "markdown",
564
+ "id": "1b7c0c91-d5dc-4321-bede-997a5b0fe9ad",
565
+ "metadata": {},
566
+ "source": [
567
+ "### Splitting e Embedding de texto"
568
+ ]
569
+ },
570
+ {
571
+ "cell_type": "code",
572
+ "execution_count": 56,
573
+ "id": "6a40be96-10d1-41f8-8190-8b353ac25c71",
574
+ "metadata": {},
575
+ "outputs": [],
576
+ "source": [
577
+ "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
578
+ "\n",
579
+ "with open('docs/CLT.txt') as f:\n",
580
+ " clt = f.read()\n",
581
+ "\n",
582
+ "text_splitter = RecursiveCharacterTextSplitter(\n",
583
+ " chunk_size=1000,\n",
584
+ " chunk_overlap=10,\n",
585
+ " length_function=len\n",
586
+ ")"
587
+ ]
588
+ },
589
+ {
590
+ "cell_type": "code",
591
+ "execution_count": 57,
592
+ "id": "3c76b64d-d57f-498b-ac4a-9501f40252db",
593
+ "metadata": {},
594
+ "outputs": [],
595
+ "source": [
596
+ "chunks = text_splitter.create_documents([clt])"
597
+ ]
598
+ },
599
+ {
600
+ "cell_type": "code",
601
+ "execution_count": 58,
602
+ "id": "7561894c-1f2f-40be-9a98-2506d24f0ce5",
603
+ "metadata": {},
604
+ "outputs": [
605
+ {
606
+ "data": {
607
+ "text/plain": [
608
+ "Document(page_content='CLT E A REFORMA TRABALHISTA: EVOLUÇÃO\\n NAS RELAÇÕES DE TRABALHO\\n\\n Eliezer de Queiroz Noleto1')"
609
+ ]
610
+ },
611
+ "execution_count": 58,
612
+ "metadata": {},
613
+ "output_type": "execute_result"
614
+ }
615
+ ],
616
+ "source": [
617
+ "chunks[0]"
618
+ ]
619
+ },
620
+ {
621
+ "cell_type": "code",
622
+ "execution_count": 59,
623
+ "id": "86e7511c-b978-4b3c-bc0d-0545b4498dcf",
624
+ "metadata": {},
625
+ "outputs": [
626
+ {
627
+ "data": {
628
+ "text/plain": [
629
+ "Document(page_content='Ao falarmos sobre a Consolidação das Leis do Trabalho (CLT), somos reme-\\n tidos ao período conhecido como Era Vargas. Editada no ano de 1943, a CLT resultou\\n da coletânea de inúmeras leis esparsas então existentes que disciplinavam aspectos\\n sobre direito individual do trabalho, direito coletivo, fiscalização do trabalho, direito\\n processual do trabalho, além de legislações específicas de determinadas profissões.\\n Esse era o espírito do trabalho desenvolvido pela comissão constituída\\n para elaborar a CLT, evidenciado na exposição de motivos que acompanhou o\\n texto final do projeto. Segundo esse escrito, a consolidação nada mais é do que “a\\n fase própria da concatenação dos textos e da coordenação dos princípios, quando\\n já se denuncia primeiro o pensamento do sistema depois de haverem sido regu-')"
630
+ ]
631
+ },
632
+ "execution_count": 59,
633
+ "metadata": {},
634
+ "output_type": "execute_result"
635
+ }
636
+ ],
637
+ "source": [
638
+ "chunks[1]"
639
+ ]
640
+ },
641
+ {
642
+ "cell_type": "code",
643
+ "execution_count": 60,
644
+ "id": "9bc2c799-e0d8-4c95-b026-11ed0fba1f59",
645
+ "metadata": {},
646
+ "outputs": [
647
+ {
648
+ "data": {
649
+ "text/plain": [
650
+ "Document(page_content='majoritário dos legisladores, o Parlamento aprovou a chamada Reforma Tra-\\nbalhista (Lei nº 13.467/2017), a modificação mais profunda na legislação traba-\\nlhista desde a edição da própria CLT. Entendeu-se que a legislação não havia\\nacompanhado a evolução vivenciada nas relações de trabalho. Isso teria se\\ndado, por exemplo, em relação às novas formas de trabalho que surgiram com\\nas inovações tecnológicas ou em virtude da atuação mais presente das enti-\\ndades de classe, condições que teriam abrandado a hipossuficiência do traba-\\nlhador. Assim, a intenção manifesta do Poder Legislativo com a aprovação da\\nLei nº 13.467/2017 foi a de atualizar a CLT aos novos tempos, promovendo-se\\numa série de mudanças, algumas pontuais, outras de ordem mais geral. Cabe,\\nnesta oportunidade, destacar as principais modificações.\\n A nova lei incluiu um capítulo no Título II da CLT (Capítulo II-A) para dis-\\nciplinar o teletrabalho, conceituado como “a prestação de serviços preponde-')"
651
+ ]
652
+ },
653
+ "execution_count": 60,
654
+ "metadata": {},
655
+ "output_type": "execute_result"
656
+ }
657
+ ],
658
+ "source": [
659
+ "chunks[10]"
660
+ ]
661
+ },
662
+ {
663
+ "cell_type": "code",
664
+ "execution_count": 61,
665
+ "id": "0a4e0c53-7255-4b98-9cc7-ae2efca2003b",
666
+ "metadata": {},
667
+ "outputs": [
668
+ {
669
+ "data": {
670
+ "text/plain": [
671
+ "'majoritário dos legisladores, o Parlamento aprovou a chamada Reforma Tra-\\nbalhista (Lei nº 13.467/2017), a modificação mais profunda na legislação traba-\\nlhista desde a edição da própria CLT. Entendeu-se que a legislação não havia\\nacompanhado a evolução vivenciada nas relações de trabalho. Isso teria se\\ndado, por exemplo, em relação às novas formas de trabalho que surgiram com\\nas inovações tecnológicas ou em virtude da atuação mais presente das enti-\\ndades de classe, condições que teriam abrandado a hipossuficiência do traba-\\nlhador. Assim, a intenção manifesta do Poder Legislativo com a aprovação da\\nLei nº 13.467/2017 foi a de atualizar a CLT aos novos tempos, promovendo-se\\numa série de mudanças, algumas pontuais, outras de ordem mais geral. Cabe,\\nnesta oportunidade, destacar as principais modificações.\\n A nova lei incluiu um capítulo no Título II da CLT (Capítulo II-A) para dis-\\nciplinar o teletrabalho, conceituado como “a prestação de serviços preponde-'"
672
+ ]
673
+ },
674
+ "execution_count": 61,
675
+ "metadata": {},
676
+ "output_type": "execute_result"
677
+ }
678
+ ],
679
+ "source": [
680
+ "chunks[10].page_content"
681
+ ]
682
+ },
683
+ {
684
+ "cell_type": "code",
685
+ "execution_count": 62,
686
+ "id": "7ee73214-a94f-4ea7-9a20-738763dc396e",
687
+ "metadata": {},
688
+ "outputs": [
689
+ {
690
+ "data": {
691
+ "text/plain": [
692
+ "1167"
693
+ ]
694
+ },
695
+ "execution_count": 62,
696
+ "metadata": {},
697
+ "output_type": "execute_result"
698
+ }
699
+ ],
700
+ "source": [
701
+ "len(chunks)"
702
+ ]
703
+ },
704
+ {
705
+ "cell_type": "code",
706
+ "execution_count": 63,
707
+ "id": "689e072e-c429-45ea-9bc7-daaf9cdea44f",
708
+ "metadata": {},
709
+ "outputs": [],
710
+ "source": [
711
+ "def embedding_cost(texts):\n",
712
+ " import tiktoken\n",
713
+ " enc = tiktoken.encoding_for_model('text-embedding-ada-002')\n",
714
+ " total_tokens = sum([len(enc.encode(page.page_content)) for page in texts])\n",
715
+ " print(f'Total de tokens: {total_tokens}')\n",
716
+ " print(f'Custo de Embedding em USD: {total_tokens / 1000 * 0.0001:.6f}')"
717
+ ]
718
+ },
719
+ {
720
+ "cell_type": "code",
721
+ "execution_count": 64,
722
+ "id": "abb2a0e2-d2d7-4544-a3bd-45ce94891e02",
723
+ "metadata": {},
724
+ "outputs": [
725
+ {
726
+ "name": "stdout",
727
+ "output_type": "stream",
728
+ "text": [
729
+ "Total de tokens: 257483\n",
730
+ "Custo de Embedding em USD: 0.025748\n"
731
+ ]
732
+ }
733
+ ],
734
+ "source": [
735
+ "embedding_cost(chunks)"
736
+ ]
737
+ },
738
+ {
739
+ "cell_type": "markdown",
740
+ "id": "85b06ed0-2189-4a76-af82-c6db56e42df1",
741
+ "metadata": {},
742
+ "source": [
743
+ "### Creating embeddings"
744
+ ]
745
+ },
746
+ {
747
+ "cell_type": "code",
748
+ "execution_count": 65,
749
+ "id": "89fe2740-0544-48e5-996e-48927b41fda5",
750
+ "metadata": {},
751
+ "outputs": [],
752
+ "source": [
753
+ "from langchain.embeddings import OpenAIEmbeddings\n",
754
+ "embeddings = OpenAIEmbeddings()"
755
+ ]
756
+ },
757
+ {
758
+ "cell_type": "code",
759
+ "execution_count": 66,
760
+ "id": "c0e0f39b-6cdb-45ba-a739-22262fb7d4a5",
761
+ "metadata": {},
762
+ "outputs": [],
763
+ "source": [
764
+ "vector = embeddings.embed_query(chunks[0].page_content)"
765
+ ]
766
+ },
767
+ {
768
+ "cell_type": "code",
769
+ "execution_count": 45,
770
+ "id": "a467ad2a-1e1f-43cb-98cc-9f17a56821b8",
771
+ "metadata": {},
772
+ "outputs": [],
773
+ "source": [
774
+ "#vector"
775
+ ]
776
+ },
777
+ {
778
+ "cell_type": "code",
779
+ "execution_count": 67,
780
+ "id": "1b6983b7-5663-4f52-bd43-6ccb4148d8e2",
781
+ "metadata": {},
782
+ "outputs": [],
783
+ "source": [
784
+ "import os\n",
785
+ "import pinecone\n",
786
+ "from langchain.vectorstores import Pinecone\n",
787
+ "\n",
788
+ "pinecone.init(api_key=os.environ.get('PINECONE_API_KEY'), environment=os.environ.get('PINECONE_ENV'))"
789
+ ]
790
+ },
791
+ {
792
+ "cell_type": "code",
793
+ "execution_count": 68,
794
+ "id": "7c5c9195-1b4c-46dd-a17b-1c50bc19ddb1",
795
+ "metadata": {},
796
+ "outputs": [
797
+ {
798
+ "name": "stdout",
799
+ "output_type": "stream",
800
+ "text": [
801
+ "Feito!\n"
802
+ ]
803
+ }
804
+ ],
805
+ "source": [
806
+ "indexes = pinecone.list_indexes()\n",
807
+ "for i in indexes:\n",
808
+ " pinecone.delete_index(i)\n",
809
+ " print('Feito!')"
810
+ ]
811
+ },
812
+ {
813
+ "cell_type": "code",
814
+ "execution_count": 69,
815
+ "id": "2cef266d-912a-467d-bd6a-751c7f58132e",
816
+ "metadata": {},
817
+ "outputs": [
818
+ {
819
+ "name": "stdout",
820
+ "output_type": "stream",
821
+ "text": [
822
+ "Feito!\n"
823
+ ]
824
+ }
825
+ ],
826
+ "source": [
827
+ "index_name = 'linuxtips'\n",
828
+ "if index_name not in pinecone.list_indexes():\n",
829
+ " pinecone.create_index(index_name, dimension=1536, metric='cosine')\n",
830
+ " print('Feito!')"
831
+ ]
832
+ },
833
+ {
834
+ "cell_type": "code",
835
+ "execution_count": 70,
836
+ "id": "b165319d-4e84-4f02-8684-ebcce0b914ef",
837
+ "metadata": {},
838
+ "outputs": [],
839
+ "source": [
840
+ "vector_store = Pinecone.from_documents(chunks, embeddings, index_name=index_name)"
841
+ ]
842
+ },
843
+ {
844
+ "cell_type": "markdown",
845
+ "id": "b9d00249-b72b-451c-970b-9bd6be2f944b",
846
+ "metadata": {},
847
+ "source": [
848
+ "### Conversando com os dados (similarity search)"
849
+ ]
850
+ },
851
+ {
852
+ "cell_type": "code",
853
+ "execution_count": 71,
854
+ "id": "b70b037f-9848-4bee-aefd-484ca6ec8ebe",
855
+ "metadata": {},
856
+ "outputs": [
857
+ {
858
+ "name": "stdout",
859
+ "output_type": "stream",
860
+ "text": [
861
+ "[Document(page_content='tativos da respectiva categoria profissional, e pro- da remuneração da tarefa na data da concessão\\nvidenciará a afixação de aviso nos locais de tra- das férias. (Parágrafo acrescido pelo Decreto-Lei nº 1.535,\\nbalho. (Parágrafo acrescido pelo Decreto-Lei nº 1.535, de de 13/4/1977)\\n13/4/1977) § 3º Quando o salário for pago por percenta-\\nArt. 140. Os empregados contratados há menos gem, comissão ou viagem, apurar-se-á a média\\nde 12 (doze) meses gozarão, na oportunidade, percebida pelo empregado nos 12 (doze) meses\\nférias proporcionais, iniciando-se, então, novo que precederem à concessão das férias. (Parágrafo\\nperíodo aquisitivo. (Artigo com redação dada pelo acrescido pelo Decreto-Lei nº 1.535, de 13/4/1977)\\nDecreto-Lei nº 1.535, de 13/4/1977) § 4º A parte do salário paga em utilidades será'), Document(page_content='com redação dada pelo Decreto-Lei nº 1.535, de 13/4/1977)\\n § 1º É vedado descontar, do período de férias, as\\n I – deixar o emprego e não for readmitido dentro\\nfaltas do empregado ao serviço. (Parágrafo acrescido\\n dos 60 (sessenta) dias subsequentes à sua saída;\\npelo Decreto-Lei nº 1.535, de 13/4/1977)\\n (Inciso acrescido pelo Decreto-Lei nº 1.535, de 13/4/1977)\\n § 2º O período das férias será computado, para\\n II – permanecer em gozo de licença, com per-\\ntodos os efeitos, como tempo de serviço. (Parágrafo\\n cepção de salários, por mais de 30 (trinta) dias;\\nacrescido pelo Decreto-Lei nº 1.535, de 13/4/1977)'), Document(page_content='Art. 129. Todo empregado terá direito anual- 5/11/1993)\\nmente ao gozo de um período de férias, sem pre- IV – justificada pela empresa, entendendo-se\\njuízo da remuneração. (Artigo com redação dada pelo como tal a que não tiver determinado o desconto\\nDecreto-Lei nº 1.535, de 13/4/1977) (Vide art. 7º, XVII, da do correspondente salário; (Inciso acrescido pelo\\nConstituição Federal de 1988) Decreto-Lei nº 1.535, de 13/4/1977)\\n V – durante a suspensão preventiva para respon-\\nArt. 130. Após cada período de 12 (doze) meses\\n der a inquérito administrativo ou de prisão preven-\\nde vigência do contrato de trabalho, o empregado\\n tiva, quando for impronunciado ou absolvido; e'), Document(page_content='empregador, em um só período, nos 12 (doze) pagará em dobro a respectiva remuneração. (Caput\\n meses subsequentes à data em que o empregado do artigo com redação dada pelo Decreto-Lei nº 1.535, de\\n tiver adquirido o direito. (Caput do artigo com redação 13/4/1977)\\n dada pelo Decreto-Lei nº 1.535, de 13/4/1977) § 1º Vencido o mencionado prazo sem que o\\n § 1º Desde que haja concordância do empre- empregador tenha concedido as férias, o empre-\\n gado, as férias poderão ser usufruídas em até gado poderá ajuizar reclamação pedindo a fixação,\\n três períodos, sendo que um deles não poderá por sentença, da época de gozo das mesmas. (Pa-\\n ser inferior a quatorze dias corridos e os demais rágrafo acrescido pelo Decreto-Lei nº 1.535, de 13/4/1977)\\n não poderão ser inferiores a cinco dias corridos, § 2º A sentença cominará pena diária de 5%')]\n"
862
+ ]
863
+ }
864
+ ],
865
+ "source": [
866
+ "query = 'explique a remuneração das férias'\n",
867
+ "result = vector_store.similarity_search(query)\n",
868
+ "print(result)"
869
+ ]
870
+ },
871
+ {
872
+ "cell_type": "code",
873
+ "execution_count": 72,
874
+ "id": "bcc0bcad-d629-48bc-a8fc-89a6387bb7a7",
875
+ "metadata": {},
876
+ "outputs": [
877
+ {
878
+ "name": "stdout",
879
+ "output_type": "stream",
880
+ "text": [
881
+ "tativos da respectiva categoria profissional, e pro- da remuneração da tarefa na data da concessão\n",
882
+ "videnciará a afixação de aviso nos locais de tra- das férias. (Parágrafo acrescido pelo Decreto-Lei nº 1.535,\n",
883
+ "balho. (Parágrafo acrescido pelo Decreto-Lei nº 1.535, de de 13/4/1977)\n",
884
+ "13/4/1977) § 3º Quando o salário for pago por percenta-\n",
885
+ "Art. 140. Os empregados contratados há menos gem, comissão ou viagem, apurar-se-á a média\n",
886
+ "de 12 (doze) meses gozarão, na oportunidade, percebida pelo empregado nos 12 (doze) meses\n",
887
+ "férias proporcionais, iniciando-se, então, novo que precederem à concessão das férias. (Parágrafo\n",
888
+ "período aquisitivo. (Artigo com redação dada pelo acrescido pelo Decreto-Lei nº 1.535, de 13/4/1977)\n",
889
+ "Decreto-Lei nº 1.535, de 13/4/1977) § 4º A parte do salário paga em utilidades será\n",
890
+ "--------------------------------------------------\n",
891
+ "com redação dada pelo Decreto-Lei nº 1.535, de 13/4/1977)\n",
892
+ " § 1º É vedado descontar, do período de férias, as\n",
893
+ " I – deixar o emprego e não for readmitido dentro\n",
894
+ "faltas do empregado ao serviço. (Parágrafo acrescido\n",
895
+ " dos 60 (sessenta) dias subsequentes à sua saída;\n",
896
+ "pelo Decreto-Lei nº 1.535, de 13/4/1977)\n",
897
+ " (Inciso acrescido pelo Decreto-Lei nº 1.535, de 13/4/1977)\n",
898
+ " § 2º O período das férias será computado, para\n",
899
+ " II – permanecer em gozo de licença, com per-\n",
900
+ "todos os efeitos, como tempo de serviço. (Parágrafo\n",
901
+ " cepção de salários, por mais de 30 (trinta) dias;\n",
902
+ "acrescido pelo Decreto-Lei nº 1.535, de 13/4/1977)\n",
903
+ "--------------------------------------------------\n",
904
+ "Art. 129. Todo empregado terá direito anual- 5/11/1993)\n",
905
+ "mente ao gozo de um período de férias, sem pre- IV – justificada pela empresa, entendendo-se\n",
906
+ "juízo da remuneração. (Artigo com redação dada pelo como tal a que não tiver determinado o desconto\n",
907
+ "Decreto-Lei nº 1.535, de 13/4/1977) (Vide art. 7º, XVII, da do correspondente salário; (Inciso acrescido pelo\n",
908
+ "Constituição Federal de 1988) Decreto-Lei nº 1.535, de 13/4/1977)\n",
909
+ " V – durante a suspensão preventiva para respon-\n",
910
+ "Art. 130. Após cada período de 12 (doze) meses\n",
911
+ " der a inquérito administrativo ou de prisão preven-\n",
912
+ "de vigência do contrato de trabalho, o empregado\n",
913
+ " tiva, quando for impronunciado ou absolvido; e\n",
914
+ "--------------------------------------------------\n",
915
+ "empregador, em um só período, nos 12 (doze) pagará em dobro a respectiva remuneração. (Caput\n",
916
+ " meses subsequentes à data em que o empregado do artigo com redação dada pelo Decreto-Lei nº 1.535, de\n",
917
+ " tiver adquirido o direito. (Caput do artigo com redação 13/4/1977)\n",
918
+ " dada pelo Decreto-Lei nº 1.535, de 13/4/1977) § 1º Vencido o mencionado prazo sem que o\n",
919
+ " § 1º Desde que haja concordância do empre- empregador tenha concedido as férias, o empre-\n",
920
+ " gado, as férias poderão ser usufruídas em até gado poderá ajuizar reclamação pedindo a fixação,\n",
921
+ " três períodos, sendo que um deles não poderá por sentença, da época de gozo das mesmas. (Pa-\n",
922
+ " ser inferior a quatorze dias corridos e os demais rágrafo acrescido pelo Decreto-Lei nº 1.535, de 13/4/1977)\n",
923
+ " não poderão ser inferiores a cinco dias corridos, § 2º A sentença cominará pena diária de 5%\n",
924
+ "--------------------------------------------------\n"
925
+ ]
926
+ }
927
+ ],
928
+ "source": [
929
+ "for r in result:\n",
930
+ " print(r.page_content)\n",
931
+ " print('-' * 50)"
932
+ ]
933
+ },
934
+ {
935
+ "cell_type": "markdown",
936
+ "id": "91e5add4-94e1-4337-9690-5791958b0e93",
937
+ "metadata": {},
938
+ "source": [
939
+ "### Gerando respostas com LLM"
940
+ ]
941
+ },
942
+ {
943
+ "cell_type": "code",
944
+ "execution_count": 74,
945
+ "id": "9dd47d56-4f67-424f-8071-84ae370e37a3",
946
+ "metadata": {},
947
+ "outputs": [],
948
+ "source": [
949
+ "from langchain.chains import RetrievalQA\n",
950
+ "from langchain.chat_models import ChatOpenAI\n",
951
+ "\n",
952
+ "llm = ChatOpenAI(model='gpt-3.5-turbo-16k', temperature=0.5)\n",
953
+ "\n",
954
+ "retriever = vector_store.as_retriever(search_type='similarity', search_kwargs={'k': 3})\n",
955
+ "\n",
956
+ "chain = RetrievalQA.from_chain_type(llm=llm, chain_type='stuff', retriever=retriever)"
957
+ ]
958
+ },
959
+ {
960
+ "cell_type": "code",
961
+ "execution_count": 76,
962
+ "id": "75a5ed98-1a83-4fc4-b680-cf5a114a1a39",
963
+ "metadata": {},
964
+ "outputs": [
965
+ {
966
+ "name": "stdout",
967
+ "output_type": "stream",
968
+ "text": [
969
+ "O décimo terceiro salário, também conhecido como gratificação de Natal, é um benefício garantido por lei aos trabalhadores brasileiros. Ele consiste no pagamento de uma parcela adicional do salário no final do ano, equivalente a 1/12 avos da remuneração devida ao empregado em cada mês trabalhado no ano.\n",
970
+ "\n",
971
+ "O décimo terceiro salário deve ser pago em duas parcelas: a primeira deve ser paga até o dia 30 de novembro e corresponde a metade do salário do empregado. Já a segunda parcela deve ser paga até o dia 20 de dezembro e corresponde à outra metade do salário, descontando-se o valor do Imposto de Renda, se houver.\n",
972
+ "\n",
973
+ "É importante ressaltar que o décimo terceiro salário é um direito de todos os trabalhadores com carteira assinada, inclusive os empregados domésticos. Além disso, o valor do décimo terceiro salário deve ser proporcional ao tempo de serviço prestado durante o ano, ou seja, se o empregado trabalhou apenas alguns meses no ano, ele receberá o décimo terceiro proporcional a esse período.\n"
974
+ ]
975
+ }
976
+ ],
977
+ "source": [
978
+ "query = 'como funciona o décimo terceiro salário?'\n",
979
+ "resp = chain.run(query)\n",
980
+ "print(resp)"
981
+ ]
982
+ },
983
+ {
984
+ "cell_type": "code",
985
+ "execution_count": null,
986
+ "id": "de9f97cf-1757-41f6-8abd-1f3df433fab2",
987
+ "metadata": {},
988
+ "outputs": [],
989
+ "source": []
990
+ }
991
+ ],
992
+ "metadata": {
993
+ "kernelspec": {
994
+ "display_name": "Python 3 (ipykernel)",
995
+ "language": "python",
996
+ "name": "python3"
997
+ },
998
+ "language_info": {
999
+ "codemirror_mode": {
1000
+ "name": "ipython",
1001
+ "version": 3
1002
+ },
1003
+ "file_extension": ".py",
1004
+ "mimetype": "text/x-python",
1005
+ "name": "python",
1006
+ "nbconvert_exporter": "python",
1007
+ "pygments_lexer": "ipython3",
1008
+ "version": "3.11.6"
1009
+ }
1010
+ },
1011
+ "nbformat": 4,
1012
+ "nbformat_minor": 5
1013
+ }
llm-workshop-parte-2.ipynb ADDED
@@ -0,0 +1,467 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 1,
6
+ "id": "ad4030a6-e372-4353-936b-18c29f5eeda5",
7
+ "metadata": {},
8
+ "outputs": [
9
+ {
10
+ "data": {
11
+ "text/plain": [
12
+ "True"
13
+ ]
14
+ },
15
+ "execution_count": 1,
16
+ "metadata": {},
17
+ "output_type": "execute_result"
18
+ }
19
+ ],
20
+ "source": [
21
+ "import os\n",
22
+ "from dotenv import load_dotenv, find_dotenv\n",
23
+ "load_dotenv(find_dotenv(), override=True)"
24
+ ]
25
+ },
26
+ {
27
+ "cell_type": "code",
28
+ "execution_count": 18,
29
+ "id": "7b8a555e-af55-4abb-8811-af91e5816b2a",
30
+ "metadata": {},
31
+ "outputs": [],
32
+ "source": [
33
+ "def load_document(file):\n",
34
+ " import os\n",
35
+ " name, extension = os.path.splitext(file)\n",
36
+ "\n",
37
+ " if extension == '.pdf':\n",
38
+ " from langchain.document_loaders import PyPDFLoader\n",
39
+ " print(f'Carregando {file}')\n",
40
+ " loader = PyPDFLoader(file)\n",
41
+ " elif extension == '.docx':\n",
42
+ " from langchain.document_loaders import Docx2txtLoader\n",
43
+ " print(f'Carregando {file}')\n",
44
+ " loader = Docx2txtLoader(file)\n",
45
+ " else:\n",
46
+ " print('Formato não suportado!')\n",
47
+ " return None\n",
48
+ "\n",
49
+ " data = loader.load()\n",
50
+ " return data\n",
51
+ "\n",
52
+ "# Wikipedia Loader\n",
53
+ "def load_from_wikipedia(query, lang='pt', load_max_docs=2):\n",
54
+ " from langchain.document_loaders import WikipediaLoader\n",
55
+ " loader = WikipediaLoader(query=query, lang=lang, load_max_docs=load_max_docs)\n",
56
+ " data = loader.load()\n",
57
+ " return data"
58
+ ]
59
+ },
60
+ {
61
+ "cell_type": "code",
62
+ "execution_count": 24,
63
+ "id": "d0a680f5-34ff-4e25-9c82-ffee69eb7e96",
64
+ "metadata": {},
65
+ "outputs": [],
66
+ "source": [
67
+ "# Teste carregando dados de um pdf\n",
68
+ "# documento = load_document('docs/CLT.pdf')\n",
69
+ "# documento[100].page_content"
70
+ ]
71
+ },
72
+ {
73
+ "cell_type": "code",
74
+ "execution_count": 25,
75
+ "id": "86329dea-7d2b-4527-9fdb-af354d8dbb56",
76
+ "metadata": {},
77
+ "outputs": [],
78
+ "source": [
79
+ "# Teste carregando dados da wikipedia\n",
80
+ "# data = load_from_wikipedia('GPT-4')\n",
81
+ "# print(data[0].page_content)"
82
+ ]
83
+ },
84
+ {
85
+ "cell_type": "code",
86
+ "execution_count": 26,
87
+ "id": "f6cd5a76-7467-47d3-853d-7538979a84e1",
88
+ "metadata": {},
89
+ "outputs": [],
90
+ "source": [
91
+ "def chunk_data(data, chunk_size=1000):\n",
92
+ " from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
93
+ " text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=0)\n",
94
+ " chunks = text_splitter.split_documents(data)\n",
95
+ " return chunks"
96
+ ]
97
+ },
98
+ {
99
+ "cell_type": "code",
100
+ "execution_count": 32,
101
+ "id": "a066a7cb-de35-44c7-a47d-e7f0ad674172",
102
+ "metadata": {},
103
+ "outputs": [],
104
+ "source": [
105
+ "def embedding_cost(texts):\n",
106
+ " import tiktoken\n",
107
+ " enc = tiktoken.encoding_for_model('text-embedding-ada-002')\n",
108
+ " total_tokens = sum([len(enc.encode(page.page_content)) for page in texts])\n",
109
+ " print(f'Total de tokens: {total_tokens}')\n",
110
+ " print(f'Custo de Embedding em USD: {total_tokens / 1000 * 0.0001:.6f}')"
111
+ ]
112
+ },
113
+ {
114
+ "cell_type": "code",
115
+ "execution_count": 36,
116
+ "id": "bcfe04be-9886-4b13-af30-9e91e520d04e",
117
+ "metadata": {},
118
+ "outputs": [
119
+ {
120
+ "name": "stdout",
121
+ "output_type": "stream",
122
+ "text": [
123
+ "Carregando docs/CLT.pdf\n"
124
+ ]
125
+ }
126
+ ],
127
+ "source": [
128
+ "data = load_document('docs/CLT.pdf')\n",
129
+ "chunks = chunk_data(data)"
130
+ ]
131
+ },
132
+ {
133
+ "cell_type": "code",
134
+ "execution_count": 38,
135
+ "id": "8914219c-c640-4e7f-b278-8852802e6676",
136
+ "metadata": {},
137
+ "outputs": [],
138
+ "source": [
139
+ "# Teste print chunks\n",
140
+ "# print(chunks[100].page_content)\n",
141
+ "# print(len(chunks))"
142
+ ]
143
+ },
144
+ {
145
+ "cell_type": "code",
146
+ "execution_count": 39,
147
+ "id": "a14b9cd9-9da8-4534-b59e-580f6a1d90a5",
148
+ "metadata": {},
149
+ "outputs": [
150
+ {
151
+ "name": "stdout",
152
+ "output_type": "stream",
153
+ "text": [
154
+ "Total de tokens: 252697\n",
155
+ "Custo de Embedding em USD: 0.025270\n"
156
+ ]
157
+ }
158
+ ],
159
+ "source": [
160
+ "# Custo total para transformar os chunks em embeddings\n",
161
+ "embedding_cost(chunks)"
162
+ ]
163
+ },
164
+ {
165
+ "cell_type": "code",
166
+ "execution_count": 40,
167
+ "id": "dd8cfb4d-1f88-4e49-9507-b77198e33434",
168
+ "metadata": {},
169
+ "outputs": [],
170
+ "source": [
171
+ "def insert_embeddings(index_name):\n",
172
+ " import pinecone\n",
173
+ " from langchain.vectorstores import Pinecone\n",
174
+ " from langchain.embeddings.openai import OpenAIEmbeddings\n",
175
+ "\n",
176
+ " embeddings = OpenAIEmbeddings()\n",
177
+ " pinecone.init(api_key=os.environ.get('PINECONE_API_KEY'), environment=os.environ.get('PINECONE_ENV'))\n",
178
+ "\n",
179
+ " if index_name in pinecone.list_indexes():\n",
180
+ " print(f'Index {index_name}')\n",
181
+ " vector_store = Pinecone.from_existing_index(index_name, embeddings)\n",
182
+ " print('Ok')\n",
183
+ " else:\n",
184
+ " print(f'Criando index {index_name}')\n",
185
+ " pinecone.create_index(index_name, dimension=1536, metric='cosine')\n",
186
+ " vector_store = Pinecone.from_documents(chunks, embeddings, index_name=index_name)\n",
187
+ " print('Ok')\n",
188
+ " return vector_store"
189
+ ]
190
+ },
191
+ {
192
+ "cell_type": "code",
193
+ "execution_count": 41,
194
+ "id": "17d5a6d7-0f50-4cb5-8f99-68f7da27ef69",
195
+ "metadata": {},
196
+ "outputs": [],
197
+ "source": [
198
+ "def delete_index(index_name='all'):\n",
199
+ " import pinecone\n",
200
+ " pinecone.init(api_key=os.environ.get('PINECONE_API_KEY'), environment=os.environ.get('PINECONE_ENV'))\n",
201
+ "\n",
202
+ " if index_name == 'all':\n",
203
+ " indexes = pinecone.list_indexes()\n",
204
+ " print('Deletando todos os índices...')\n",
205
+ " for index in indexes:\n",
206
+ " pinecone.delete_index(index)\n",
207
+ " else:\n",
208
+ " print(f'Deletando índice {index_name}...')\n",
209
+ " pinecone.delete_index(index_name)"
210
+ ]
211
+ },
212
+ {
213
+ "cell_type": "code",
214
+ "execution_count": 44,
215
+ "id": "4c39ef6f-1fc8-424b-aaa6-a1c6c7f45f20",
216
+ "metadata": {},
217
+ "outputs": [],
218
+ "source": [
219
+ "# Teste deletando um índice existente\n",
220
+ "# delete_index()"
221
+ ]
222
+ },
223
+ {
224
+ "cell_type": "code",
225
+ "execution_count": 43,
226
+ "id": "cabeb040-0fd3-4433-9c04-208b55b060f4",
227
+ "metadata": {},
228
+ "outputs": [
229
+ {
230
+ "name": "stdout",
231
+ "output_type": "stream",
232
+ "text": [
233
+ "Criando index linuxtips\n",
234
+ "Ok\n"
235
+ ]
236
+ }
237
+ ],
238
+ "source": [
239
+ "index_name = 'linuxtips'\n",
240
+ "vector_store = insert_embeddings(index_name)"
241
+ ]
242
+ },
243
+ {
244
+ "cell_type": "code",
245
+ "execution_count": 63,
246
+ "id": "384f79f7-17af-4569-b3fa-8bd9578882ca",
247
+ "metadata": {},
248
+ "outputs": [],
249
+ "source": [
250
+ "def get_answer(vector_store, q):\n",
251
+ " from langchain.chains import RetrievalQA\n",
252
+ " from langchain.chat_models import ChatOpenAI\n",
253
+ "\n",
254
+ " llm = ChatOpenAI(model='gpt-3.5-turbo', temperature=1)\n",
255
+ "\n",
256
+ " retriever = vector_store.as_retriever(search_type='similarity', search_kwargs={'k': 3})\n",
257
+ "\n",
258
+ " chain = RetrievalQA.from_chain_type(llm=llm, chain_type='stuff', retriever=retriever)\n",
259
+ "\n",
260
+ " answer = chain.run(q)\n",
261
+ " return answer\n",
262
+ "\n",
263
+ "def ask_with_memory(vector_store, question, chat_history=[]):\n",
264
+ " from langchain.chains import ConversationalRetrievalChain\n",
265
+ " from langchain.chat_models import ChatOpenAI\n",
266
+ " \n",
267
+ " llm = ChatOpenAI(temperature=1)\n",
268
+ " retriever = vector_store.as_retriever(search_type='similarity', search_kwargs={'k': 3})\n",
269
+ " \n",
270
+ " crc = ConversationalRetrievalChain.from_llm(llm, retriever)\n",
271
+ " result = crc({'question': question, 'chat_history': chat_history})\n",
272
+ " chat_history.append((question, result['answer']))\n",
273
+ " \n",
274
+ " return result, chat_history"
275
+ ]
276
+ },
277
+ {
278
+ "cell_type": "code",
279
+ "execution_count": 68,
280
+ "id": "a4d4929d-7625-4222-a97a-3b4928e4e785",
281
+ "metadata": {},
282
+ "outputs": [
283
+ {
284
+ "name": "stdout",
285
+ "output_type": "stream",
286
+ "text": [
287
+ "A CLT, Consolidação das Leis do Trabalho, foi criada em 1943.\n"
288
+ ]
289
+ }
290
+ ],
291
+ "source": [
292
+ "q = 'em que ano a clt foi criada?'\n",
293
+ "answer = get_answer(vector_store, q)\n",
294
+ "print(answer)"
295
+ ]
296
+ },
297
+ {
298
+ "cell_type": "code",
299
+ "execution_count": 48,
300
+ "id": "ba4fff3b-4632-4d1d-9266-01e8426eebf4",
301
+ "metadata": {},
302
+ "outputs": [
303
+ {
304
+ "name": "stdout",
305
+ "output_type": "stream",
306
+ "text": [
307
+ "Digite sair para encerrar.\n"
308
+ ]
309
+ },
310
+ {
311
+ "name": "stdin",
312
+ "output_type": "stream",
313
+ "text": [
314
+ "Pergunta: #1: o que é o décimo terceiro salário?\n"
315
+ ]
316
+ },
317
+ {
318
+ "name": "stdout",
319
+ "output_type": "stream",
320
+ "text": [
321
+ "\n",
322
+ "Resposta: O décimo terceiro salário é um benefício pago aos trabalhadores no Brasil, também conhecido como \"gratificação de Natal\". Ele corresponde a um salário extra que é pago uma vez por ano, geralmente em dezembro, e equivale a 1/12 avos da remuneração total recebida pelo trabalhador no ano, incluindo salário-base, horas extras, comissões e adicionais. Esse benefício tem como objetivo proporcionar aos trabalhadores uma renda adicional para despesas extras no período de fim de ano.\n",
323
+ "\n",
324
+ " -------------------------------------------------- \n",
325
+ "\n"
326
+ ]
327
+ },
328
+ {
329
+ "name": "stdin",
330
+ "output_type": "stream",
331
+ "text": [
332
+ "Pergunta: #2: qual foi a minha pergunta anterior?\n"
333
+ ]
334
+ },
335
+ {
336
+ "name": "stdout",
337
+ "output_type": "stream",
338
+ "text": [
339
+ "\n",
340
+ "Resposta: Desculpe, mas eu não tenho acesso à pergunta anterior. Posso ajudar com alguma outra informação?\n",
341
+ "\n",
342
+ " -------------------------------------------------- \n",
343
+ "\n"
344
+ ]
345
+ },
346
+ {
347
+ "name": "stdin",
348
+ "output_type": "stream",
349
+ "text": [
350
+ "Pergunta: #3: sair\n"
351
+ ]
352
+ },
353
+ {
354
+ "name": "stdout",
355
+ "output_type": "stream",
356
+ "text": [
357
+ "Encerrando...\n"
358
+ ]
359
+ }
360
+ ],
361
+ "source": [
362
+ "import time\n",
363
+ "i = 1\n",
364
+ "print('Digite sair para encerrar.')\n",
365
+ "while True:\n",
366
+ " q = input(f'Pergunta: #{i}: ')\n",
367
+ " i = i+1\n",
368
+ " if q.lower() in ['sair']:\n",
369
+ " print('Encerrando...')\n",
370
+ " time.sleep(2)\n",
371
+ " break\n",
372
+ "\n",
373
+ " answer = get_answer(vector_store, q)\n",
374
+ " print(f'\\nResposta: {answer}')\n",
375
+ " print(f'\\n {\"-\" * 50} \\n')"
376
+ ]
377
+ },
378
+ {
379
+ "cell_type": "code",
380
+ "execution_count": 75,
381
+ "id": "eba0c508-453f-416a-9d8c-b0edb7dfcc51",
382
+ "metadata": {},
383
+ "outputs": [
384
+ {
385
+ "name": "stdout",
386
+ "output_type": "stream",
387
+ "text": [
388
+ "A Consolidação das Leis do Trabalho (CLT) foi criada em 1943.\n"
389
+ ]
390
+ }
391
+ ],
392
+ "source": [
393
+ "chat_history = []\n",
394
+ "question = 'em que ano a clt foi criada?'\n",
395
+ "result, chat_history = ask_with_memory(vector_store, question, chat_history)\n",
396
+ "print(result['answer'])"
397
+ ]
398
+ },
399
+ {
400
+ "cell_type": "code",
401
+ "execution_count": 76,
402
+ "id": "b34ff3ab-6de4-44b0-a441-6462b85d2d14",
403
+ "metadata": {},
404
+ "outputs": [
405
+ {
406
+ "name": "stdout",
407
+ "output_type": "stream",
408
+ "text": [
409
+ "Desde 1943 até 2023, se passarão 80 anos.\n"
410
+ ]
411
+ }
412
+ ],
413
+ "source": [
414
+ "question = 'estamos em 2023, quantos anos passaram?'\n",
415
+ "result, chat_history = ask_with_memory(vector_store, question, chat_history)\n",
416
+ "print(result['answer'])"
417
+ ]
418
+ },
419
+ {
420
+ "cell_type": "code",
421
+ "execution_count": 84,
422
+ "id": "504fbca7-f317-4cfb-99cd-c36a3dd4039d",
423
+ "metadata": {},
424
+ "outputs": [
425
+ {
426
+ "name": "stdout",
427
+ "output_type": "stream",
428
+ "text": [
429
+ "[('em que ano a clt foi criada?', 'A Consolidação das Leis do Trabalho (CLT) foi criada em 1943.'), ('estamos em 2023, quantos anos passaram?', 'Desde 1943 até 2023, se passarão 80 anos.')]\n"
430
+ ]
431
+ }
432
+ ],
433
+ "source": [
434
+ "print(chat_history)"
435
+ ]
436
+ },
437
+ {
438
+ "cell_type": "code",
439
+ "execution_count": null,
440
+ "id": "c26b532e-e78e-4101-9261-d0a79d5ebdf5",
441
+ "metadata": {},
442
+ "outputs": [],
443
+ "source": []
444
+ }
445
+ ],
446
+ "metadata": {
447
+ "kernelspec": {
448
+ "display_name": "Python 3 (ipykernel)",
449
+ "language": "python",
450
+ "name": "python3"
451
+ },
452
+ "language_info": {
453
+ "codemirror_mode": {
454
+ "name": "ipython",
455
+ "version": 3
456
+ },
457
+ "file_extension": ".py",
458
+ "mimetype": "text/x-python",
459
+ "name": "python",
460
+ "nbconvert_exporter": "python",
461
+ "pygments_lexer": "ipython3",
462
+ "version": "3.11.6"
463
+ }
464
+ },
465
+ "nbformat": 4,
466
+ "nbformat_minor": 5
467
+ }
requirements.txt ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ openai
2
+ langchain-community
3
+ googleapi
4
+ langchain-openai
5
+ protoc-gen-openapiv2
6
+ gradio~=4.27.0
7
+ python-dotenv~=1.0.1
8
+ tiktoken
9
+ pinecone-client
10
+ wikipedia
11
+ pypdf
12
+ llmx
13
+ langchain~=0.1.16
14
+ cryptography
venv/.gitignore ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ # created by virtualenv automatically
2
+ *
venv/bin/activate ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # This file must be used with "source bin/activate" *from bash*
2
+ # you cannot run it directly
3
+
4
+
5
+ if [ "${BASH_SOURCE-}" = "$0" ]; then
6
+ echo "You must source this script: \$ source $0" >&2
7
+ exit 33
8
+ fi
9
+
10
+ deactivate () {
11
+ unset -f pydoc >/dev/null 2>&1 || true
12
+
13
+ # reset old environment variables
14
+ # ! [ -z ${VAR+_} ] returns true if VAR is declared at all
15
+ if ! [ -z "${_OLD_VIRTUAL_PATH:+_}" ] ; then
16
+ PATH="$_OLD_VIRTUAL_PATH"
17
+ export PATH
18
+ unset _OLD_VIRTUAL_PATH
19
+ fi
20
+ if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
21
+ PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
22
+ export PYTHONHOME
23
+ unset _OLD_VIRTUAL_PYTHONHOME
24
+ fi
25
+
26
+ # The hash command must be called to get it to forget past
27
+ # commands. Without forgetting past commands the $PATH changes
28
+ # we made may not be respected
29
+ hash -r 2>/dev/null
30
+
31
+ if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
32
+ PS1="$_OLD_VIRTUAL_PS1"
33
+ export PS1
34
+ unset _OLD_VIRTUAL_PS1
35
+ fi
36
+
37
+ unset VIRTUAL_ENV
38
+ unset VIRTUAL_ENV_PROMPT
39
+ if [ ! "${1-}" = "nondestructive" ] ; then
40
+ # Self destruct!
41
+ unset -f deactivate
42
+ fi
43
+ }
44
+
45
+ # unset irrelevant variables
46
+ deactivate nondestructive
47
+
48
+ VIRTUAL_ENV='/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv'
49
+ if ([ "$OSTYPE" = "cygwin" ] || [ "$OSTYPE" = "msys" ]) && $(command -v cygpath &> /dev/null) ; then
50
+ VIRTUAL_ENV=$(cygpath -u "$VIRTUAL_ENV")
51
+ fi
52
+ export VIRTUAL_ENV
53
+
54
+ _OLD_VIRTUAL_PATH="$PATH"
55
+ PATH="$VIRTUAL_ENV/bin:$PATH"
56
+ export PATH
57
+
58
+ if [ "x" != x ] ; then
59
+ VIRTUAL_ENV_PROMPT=""
60
+ else
61
+ VIRTUAL_ENV_PROMPT=$(basename "$VIRTUAL_ENV")
62
+ fi
63
+ export VIRTUAL_ENV_PROMPT
64
+
65
+ # unset PYTHONHOME if set
66
+ if ! [ -z "${PYTHONHOME+_}" ] ; then
67
+ _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
68
+ unset PYTHONHOME
69
+ fi
70
+
71
+ if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
72
+ _OLD_VIRTUAL_PS1="${PS1-}"
73
+ PS1="(${VIRTUAL_ENV_PROMPT}) ${PS1-}"
74
+ export PS1
75
+ fi
76
+
77
+ # Make sure to unalias pydoc if it's already there
78
+ alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
79
+
80
+ pydoc () {
81
+ python -m pydoc "$@"
82
+ }
83
+
84
+ # The hash command must be called to get it to forget past
85
+ # commands. Without forgetting past commands the $PATH changes
86
+ # we made may not be respected
87
+ hash -r 2>/dev/null
venv/bin/activate.csh ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # This file must be used with "source bin/activate.csh" *from csh*.
2
+ # You cannot run it directly.
3
+ # Created by Davide Di Blasi <davidedb@gmail.com>.
4
+
5
+ set newline='\
6
+ '
7
+
8
+ alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH:q" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT:q" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc'
9
+
10
+ # Unset irrelevant variables.
11
+ deactivate nondestructive
12
+
13
+ setenv VIRTUAL_ENV '/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv'
14
+
15
+ set _OLD_VIRTUAL_PATH="$PATH:q"
16
+ setenv PATH "$VIRTUAL_ENV:q/bin:$PATH:q"
17
+
18
+
19
+
20
+ if ('' != "") then
21
+ setenv VIRTUAL_ENV_PROMPT ''
22
+ else
23
+ setenv VIRTUAL_ENV_PROMPT "$VIRTUAL_ENV:t:q"
24
+ endif
25
+
26
+ if ( $?VIRTUAL_ENV_DISABLE_PROMPT ) then
27
+ if ( $VIRTUAL_ENV_DISABLE_PROMPT == "" ) then
28
+ set do_prompt = "1"
29
+ else
30
+ set do_prompt = "0"
31
+ endif
32
+ else
33
+ set do_prompt = "1"
34
+ endif
35
+
36
+ if ( $do_prompt == "1" ) then
37
+ # Could be in a non-interactive environment,
38
+ # in which case, $prompt is undefined and we wouldn't
39
+ # care about the prompt anyway.
40
+ if ( $?prompt ) then
41
+ set _OLD_VIRTUAL_PROMPT="$prompt:q"
42
+ if ( "$prompt:q" =~ *"$newline:q"* ) then
43
+ :
44
+ else
45
+ set prompt = '('"$VIRTUAL_ENV_PROMPT:q"') '"$prompt:q"
46
+ endif
47
+ endif
48
+ endif
49
+
50
+ unset env_name
51
+ unset do_prompt
52
+
53
+ alias pydoc python -m pydoc
54
+
55
+ rehash
venv/bin/activate.fish ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # This file must be used using `source bin/activate.fish` *within a running fish ( http://fishshell.com ) session*.
2
+ # Do not run it directly.
3
+
4
+ function _bashify_path -d "Converts a fish path to something bash can recognize"
5
+ set fishy_path $argv
6
+ set bashy_path $fishy_path[1]
7
+ for path_part in $fishy_path[2..-1]
8
+ set bashy_path "$bashy_path:$path_part"
9
+ end
10
+ echo $bashy_path
11
+ end
12
+
13
+ function _fishify_path -d "Converts a bash path to something fish can recognize"
14
+ echo $argv | tr ':' '\n'
15
+ end
16
+
17
+ function deactivate -d 'Exit virtualenv mode and return to the normal environment.'
18
+ # reset old environment variables
19
+ if test -n "$_OLD_VIRTUAL_PATH"
20
+ # https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling
21
+ if test (echo $FISH_VERSION | head -c 1) -lt 3
22
+ set -gx PATH (_fishify_path "$_OLD_VIRTUAL_PATH")
23
+ else
24
+ set -gx PATH $_OLD_VIRTUAL_PATH
25
+ end
26
+ set -e _OLD_VIRTUAL_PATH
27
+ end
28
+
29
+ if test -n "$_OLD_VIRTUAL_PYTHONHOME"
30
+ set -gx PYTHONHOME "$_OLD_VIRTUAL_PYTHONHOME"
31
+ set -e _OLD_VIRTUAL_PYTHONHOME
32
+ end
33
+
34
+ if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
35
+ and functions -q _old_fish_prompt
36
+ # Set an empty local `$fish_function_path` to allow the removal of `fish_prompt` using `functions -e`.
37
+ set -l fish_function_path
38
+
39
+ # Erase virtualenv's `fish_prompt` and restore the original.
40
+ functions -e fish_prompt
41
+ functions -c _old_fish_prompt fish_prompt
42
+ functions -e _old_fish_prompt
43
+ set -e _OLD_FISH_PROMPT_OVERRIDE
44
+ end
45
+
46
+ set -e VIRTUAL_ENV
47
+ set -e VIRTUAL_ENV_PROMPT
48
+
49
+ if test "$argv[1]" != 'nondestructive'
50
+ # Self-destruct!
51
+ functions -e pydoc
52
+ functions -e deactivate
53
+ functions -e _bashify_path
54
+ functions -e _fishify_path
55
+ end
56
+ end
57
+
58
+ # Unset irrelevant variables.
59
+ deactivate nondestructive
60
+
61
+ set -gx VIRTUAL_ENV '/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv'
62
+
63
+ # https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling
64
+ if test (echo $FISH_VERSION | head -c 1) -lt 3
65
+ set -gx _OLD_VIRTUAL_PATH (_bashify_path $PATH)
66
+ else
67
+ set -gx _OLD_VIRTUAL_PATH $PATH
68
+ end
69
+ set -gx PATH "$VIRTUAL_ENV"'/bin' $PATH
70
+
71
+ # Prompt override provided?
72
+ # If not, just use the environment name.
73
+ if test -n ''
74
+ set -gx VIRTUAL_ENV_PROMPT ''
75
+ else
76
+ set -gx VIRTUAL_ENV_PROMPT (basename "$VIRTUAL_ENV")
77
+ end
78
+
79
+ # Unset `$PYTHONHOME` if set.
80
+ if set -q PYTHONHOME
81
+ set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
82
+ set -e PYTHONHOME
83
+ end
84
+
85
+ function pydoc
86
+ python -m pydoc $argv
87
+ end
88
+
89
+ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
90
+ # Copy the current `fish_prompt` function as `_old_fish_prompt`.
91
+ functions -c fish_prompt _old_fish_prompt
92
+
93
+ function fish_prompt
94
+ # Run the user's prompt first; it might depend on (pipe)status.
95
+ set -l prompt (_old_fish_prompt)
96
+
97
+ printf '(%s) ' $VIRTUAL_ENV_PROMPT
98
+
99
+ string join -- \n $prompt # handle multi-line prompts
100
+ end
101
+
102
+ set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
103
+ end
venv/bin/activate.nu ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # virtualenv activation module
2
+ # Activate with `overlay use activate.nu`
3
+ # Deactivate with `deactivate`, as usual
4
+ #
5
+ # To customize the overlay name, you can call `overlay use activate.nu as foo`,
6
+ # but then simply `deactivate` won't work because it is just an alias to hide
7
+ # the "activate" overlay. You'd need to call `overlay hide foo` manually.
8
+
9
+ export-env {
10
+ def is-string [x] {
11
+ ($x | describe) == 'string'
12
+ }
13
+
14
+ def has-env [...names] {
15
+ $names | each {|n|
16
+ $n in $env
17
+ } | all {|i| $i == true}
18
+ }
19
+
20
+ # Emulates a `test -z`, but btter as it handles e.g 'false'
21
+ def is-env-true [name: string] {
22
+ if (has-env $name) {
23
+ # Try to parse 'true', '0', '1', and fail if not convertible
24
+ let parsed = (do -i { $env | get $name | into bool })
25
+ if ($parsed | describe) == 'bool' {
26
+ $parsed
27
+ } else {
28
+ not ($env | get -i $name | is-empty)
29
+ }
30
+ } else {
31
+ false
32
+ }
33
+ }
34
+
35
+ let virtual_env = '/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv'
36
+ let bin = 'bin'
37
+
38
+ let is_windows = ($nu.os-info.family) == 'windows'
39
+ let path_name = (if (has-env 'Path') {
40
+ 'Path'
41
+ } else {
42
+ 'PATH'
43
+ }
44
+ )
45
+
46
+ let venv_path = ([$virtual_env $bin] | path join)
47
+ let new_path = ($env | get $path_name | prepend $venv_path)
48
+
49
+ # If there is no default prompt, then use the env name instead
50
+ let virtual_env_prompt = (if ('' | is-empty) {
51
+ ($virtual_env | path basename)
52
+ } else {
53
+ ''
54
+ })
55
+
56
+ let new_env = {
57
+ $path_name : $new_path
58
+ VIRTUAL_ENV : $virtual_env
59
+ VIRTUAL_ENV_PROMPT : $virtual_env_prompt
60
+ }
61
+
62
+ let new_env = (if (is-env-true 'VIRTUAL_ENV_DISABLE_PROMPT') {
63
+ $new_env
64
+ } else {
65
+ # Creating the new prompt for the session
66
+ let virtual_prefix = $'(char lparen)($virtual_env_prompt)(char rparen) '
67
+
68
+ # Back up the old prompt builder
69
+ let old_prompt_command = (if (has-env 'PROMPT_COMMAND') {
70
+ $env.PROMPT_COMMAND
71
+ } else {
72
+ ''
73
+ })
74
+
75
+ let new_prompt = (if (has-env 'PROMPT_COMMAND') {
76
+ if 'closure' in ($old_prompt_command | describe) {
77
+ {|| $'($virtual_prefix)(do $old_prompt_command)' }
78
+ } else {
79
+ {|| $'($virtual_prefix)($old_prompt_command)' }
80
+ }
81
+ } else {
82
+ {|| $'($virtual_prefix)' }
83
+ })
84
+
85
+ $new_env | merge {
86
+ PROMPT_COMMAND : $new_prompt
87
+ VIRTUAL_PREFIX : $virtual_prefix
88
+ }
89
+ })
90
+
91
+ # Environment variables that will be loaded as the virtual env
92
+ load-env $new_env
93
+ }
94
+
95
+ export alias pydoc = python -m pydoc
96
+ export alias deactivate = overlay hide activate
venv/bin/activate.ps1 ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ $script:THIS_PATH = $myinvocation.mycommand.path
2
+ $script:BASE_DIR = Split-Path (Resolve-Path "$THIS_PATH/..") -Parent
3
+
4
+ function global:deactivate([switch] $NonDestructive) {
5
+ if (Test-Path variable:_OLD_VIRTUAL_PATH) {
6
+ $env:PATH = $variable:_OLD_VIRTUAL_PATH
7
+ Remove-Variable "_OLD_VIRTUAL_PATH" -Scope global
8
+ }
9
+
10
+ if (Test-Path function:_old_virtual_prompt) {
11
+ $function:prompt = $function:_old_virtual_prompt
12
+ Remove-Item function:\_old_virtual_prompt
13
+ }
14
+
15
+ if ($env:VIRTUAL_ENV) {
16
+ Remove-Item env:VIRTUAL_ENV -ErrorAction SilentlyContinue
17
+ }
18
+
19
+ if ($env:VIRTUAL_ENV_PROMPT) {
20
+ Remove-Item env:VIRTUAL_ENV_PROMPT -ErrorAction SilentlyContinue
21
+ }
22
+
23
+ if (!$NonDestructive) {
24
+ # Self destruct!
25
+ Remove-Item function:deactivate
26
+ Remove-Item function:pydoc
27
+ }
28
+ }
29
+
30
+ function global:pydoc {
31
+ python -m pydoc $args
32
+ }
33
+
34
+ # unset irrelevant variables
35
+ deactivate -nondestructive
36
+
37
+ $VIRTUAL_ENV = $BASE_DIR
38
+ $env:VIRTUAL_ENV = $VIRTUAL_ENV
39
+
40
+ if ("" -ne "") {
41
+ $env:VIRTUAL_ENV_PROMPT = ""
42
+ }
43
+ else {
44
+ $env:VIRTUAL_ENV_PROMPT = $( Split-Path $env:VIRTUAL_ENV -Leaf )
45
+ }
46
+
47
+ New-Variable -Scope global -Name _OLD_VIRTUAL_PATH -Value $env:PATH
48
+
49
+ $env:PATH = "$env:VIRTUAL_ENV/bin:" + $env:PATH
50
+ if (!$env:VIRTUAL_ENV_DISABLE_PROMPT) {
51
+ function global:_old_virtual_prompt {
52
+ ""
53
+ }
54
+ $function:_old_virtual_prompt = $function:prompt
55
+
56
+ function global:prompt {
57
+ # Add the custom prefix to the existing prompt
58
+ $previous_prompt_value = & $function:_old_virtual_prompt
59
+ ("(" + $env:VIRTUAL_ENV_PROMPT + ") " + $previous_prompt_value)
60
+ }
61
+ }
venv/bin/activate_this.py ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Activate virtualenv for current interpreter:
3
+
4
+ Use exec(open(this_file).read(), {'__file__': this_file}).
5
+
6
+ This can be used when you must use an existing Python interpreter, not the virtualenv bin/python.
7
+ """ # noqa: D415
8
+ from __future__ import annotations
9
+
10
+ import os
11
+ import site
12
+ import sys
13
+
14
+ try:
15
+ abs_file = os.path.abspath(__file__)
16
+ except NameError as exc:
17
+ msg = "You must use exec(open(this_file).read(), {'__file__': this_file}))"
18
+ raise AssertionError(msg) from exc
19
+
20
+ bin_dir = os.path.dirname(abs_file)
21
+ base = bin_dir[: -len("bin") - 1] # strip away the bin part from the __file__, plus the path separator
22
+
23
+ # prepend bin to PATH (this file is inside the bin directory)
24
+ os.environ["PATH"] = os.pathsep.join([bin_dir, *os.environ.get("PATH", "").split(os.pathsep)])
25
+ os.environ["VIRTUAL_ENV"] = base # virtual env is right above bin directory
26
+ os.environ["VIRTUAL_ENV_PROMPT"] = "" or os.path.basename(base) # noqa: SIM222
27
+
28
+ # add the virtual environments libraries to the host python import mechanism
29
+ prev_length = len(sys.path)
30
+ for lib in "../lib/python3.11/site-packages".split(os.pathsep):
31
+ path = os.path.realpath(os.path.join(bin_dir, lib))
32
+ site.addsitedir(path.decode("utf-8") if "" else path)
33
+ sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
34
+
35
+ sys.real_prefix = sys.prefix
36
+ sys.prefix = base
venv/bin/distro ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from distro.distro import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/dotenv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from dotenv.__main__ import cli
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(cli())
venv/bin/f2py ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from numpy.f2py.f2py2e import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/fastavro ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from fastavro.__main__ import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/fonttools ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from fontTools.__main__ import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/google ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+
3
+ # Copyright (c) 2009-2020, Mario Vilas
4
+ # All rights reserved.
5
+ #
6
+ # Redistribution and use in source and binary forms, with or without
7
+ # modification, are permitted provided that the following conditions are met:
8
+ #
9
+ # * Redistributions of source code must retain the above copyright notice,
10
+ # this list of conditions and the following disclaimer.
11
+ # * Redistributions in binary form must reproduce the above copyright
12
+ # notice,this list of conditions and the following disclaimer in the
13
+ # documentation and/or other materials provided with the distribution.
14
+ # * Neither the name of the copyright holder nor the names of its
15
+ # contributors may be used to endorse or promote products derived from
16
+ # this software without specific prior written permission.
17
+ #
18
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19
+ # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20
+ # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21
+ # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
22
+ # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23
+ # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24
+ # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25
+ # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26
+ # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27
+ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28
+ # POSSIBILITY OF SUCH DAMAGE.
29
+
30
+ import sys
31
+
32
+ from googlesearch import search, get_random_user_agent
33
+
34
+ # TODO port to argparse
35
+ from optparse import OptionParser, IndentedHelpFormatter
36
+
37
+
38
+ class BannerHelpFormatter(IndentedHelpFormatter):
39
+
40
+ "Just a small tweak to optparse to be able to print a banner."
41
+
42
+ def __init__(self, banner, *argv, **argd):
43
+ self.banner = banner
44
+ IndentedHelpFormatter.__init__(self, *argv, **argd)
45
+
46
+ def format_usage(self, usage):
47
+ msg = IndentedHelpFormatter.format_usage(self, usage)
48
+ return '%s\n%s' % (self.banner, msg)
49
+
50
+
51
+ def main():
52
+
53
+ # Parse the command line arguments.
54
+ formatter = BannerHelpFormatter(
55
+ "Python script to use the Google search engine\n"
56
+ "By Mario Vilas (mvilas at gmail dot com)\n"
57
+ "https://github.com/MarioVilas/googlesearch\n"
58
+ )
59
+ parser = OptionParser(formatter=formatter)
60
+ parser.set_usage("%prog [options] query")
61
+ parser.add_option(
62
+ '--tld', metavar='TLD', type='string', default='com',
63
+ help="top level domain to use [default: com]")
64
+ parser.add_option(
65
+ '--lang', metavar='LANGUAGE', type='string', default='en',
66
+ help="produce results in the given language [default: en]")
67
+ parser.add_option(
68
+ '--tbs', metavar='TBS', type='string', default='0',
69
+ help="produce results from period [default: 0]")
70
+ parser.add_option(
71
+ '--safe', metavar='SAFE', type='string', default='off',
72
+ help="kids safe search [default: off]")
73
+ parser.add_option(
74
+ '--country', metavar='COUNTRY', type='string', default='',
75
+ help="region to restrict search on [default: not restricted]")
76
+ parser.add_option(
77
+ '--num', metavar='NUMBER', type='int', default=10,
78
+ help="number of results per page [default: 10]")
79
+ parser.add_option(
80
+ '--start', metavar='NUMBER', type='int', default=0,
81
+ help="first result to retrieve [default: 0]")
82
+ parser.add_option(
83
+ '--stop', metavar='NUMBER', type='int', default=0,
84
+ help="last result to retrieve [default: unlimited]")
85
+ parser.add_option(
86
+ '--pause', metavar='SECONDS', type='float', default=2.0,
87
+ help="pause between HTTP requests [default: 2.0]")
88
+ parser.add_option(
89
+ '--rua', action='store_true', default=False,
90
+ help="Randomize the User-Agent [default: no]")
91
+ parser.add_option(
92
+ '--insecure', dest="verify_ssl", action='store_false', default=True,
93
+ help="Randomize the User-Agent [default: no]")
94
+ (options, args) = parser.parse_args()
95
+ query = ' '.join(args)
96
+ if not query:
97
+ parser.print_help()
98
+ sys.exit(2)
99
+ params = [
100
+ (k, v) for (k, v) in options.__dict__.items()
101
+ if not k.startswith('_')]
102
+ params = dict(params)
103
+
104
+ # Randomize the user agent if requested.
105
+ if 'rua' in params and params.pop('rua'):
106
+ params['user_agent'] = get_random_user_agent()
107
+
108
+ # Run the query.
109
+ for url in search(query, **params):
110
+ print(url)
111
+ try:
112
+ sys.stdout.flush()
113
+ except Exception:
114
+ pass
115
+
116
+
117
+ if __name__ == '__main__':
118
+ main()
venv/bin/google-oauthlib-tool ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from google_auth_oauthlib.tool.__main__ import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/gradio ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from gradio.cli import cli
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(cli())
venv/bin/httpx ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from httpx import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/huggingface-cli ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from huggingface_hub.commands.huggingface_cli import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/jsondiff ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+
4
+ from __future__ import print_function
5
+
6
+ import sys
7
+ import json
8
+ import jsonpatch
9
+ import argparse
10
+
11
+
12
+ parser = argparse.ArgumentParser(description='Diff two JSON files')
13
+ parser.add_argument('FILE1', type=argparse.FileType('r'))
14
+ parser.add_argument('FILE2', type=argparse.FileType('r'))
15
+ parser.add_argument('--indent', type=int, default=None,
16
+ help='Indent output by n spaces')
17
+ parser.add_argument('-u', '--preserve-unicode', action='store_true',
18
+ help='Output Unicode character as-is without using Code Point')
19
+ parser.add_argument('-v', '--version', action='version',
20
+ version='%(prog)s ' + jsonpatch.__version__)
21
+
22
+
23
+ def main():
24
+ try:
25
+ diff_files()
26
+ except KeyboardInterrupt:
27
+ sys.exit(1)
28
+
29
+
30
+ def diff_files():
31
+ """ Diffs two JSON files and prints a patch """
32
+ args = parser.parse_args()
33
+ doc1 = json.load(args.FILE1)
34
+ doc2 = json.load(args.FILE2)
35
+ patch = jsonpatch.make_patch(doc1, doc2)
36
+ if patch.patch:
37
+ print(json.dumps(patch.patch, indent=args.indent, ensure_ascii=not(args.preserve_unicode)))
38
+ sys.exit(1)
39
+
40
+ if __name__ == "__main__":
41
+ main()
venv/bin/jsonpatch ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+
4
+ import sys
5
+ import os.path
6
+ import json
7
+ import jsonpatch
8
+ import tempfile
9
+ import argparse
10
+
11
+
12
+ parser = argparse.ArgumentParser(
13
+ description='Apply a JSON patch on a JSON file')
14
+ parser.add_argument('ORIGINAL', type=argparse.FileType('r'),
15
+ help='Original file')
16
+ parser.add_argument('PATCH', type=argparse.FileType('r'),
17
+ nargs='?', default=sys.stdin,
18
+ help='Patch file (read from stdin if omitted)')
19
+ parser.add_argument('--indent', type=int, default=None,
20
+ help='Indent output by n spaces')
21
+ parser.add_argument('-b', '--backup', action='store_true',
22
+ help='Back up ORIGINAL if modifying in-place')
23
+ parser.add_argument('-i', '--in-place', action='store_true',
24
+ help='Modify ORIGINAL in-place instead of to stdout')
25
+ parser.add_argument('-v', '--version', action='version',
26
+ version='%(prog)s ' + jsonpatch.__version__)
27
+ parser.add_argument('-u', '--preserve-unicode', action='store_true',
28
+ help='Output Unicode character as-is without using Code Point')
29
+
30
+ def main():
31
+ try:
32
+ patch_files()
33
+ except KeyboardInterrupt:
34
+ sys.exit(1)
35
+
36
+
37
+ def patch_files():
38
+ """ Diffs two JSON files and prints a patch """
39
+ args = parser.parse_args()
40
+ doc = json.load(args.ORIGINAL)
41
+ patch = json.load(args.PATCH)
42
+ result = jsonpatch.apply_patch(doc, patch)
43
+
44
+ if args.in_place:
45
+ dirname = os.path.abspath(os.path.dirname(args.ORIGINAL.name))
46
+
47
+ try:
48
+ # Attempt to replace the file atomically. We do this by
49
+ # creating a temporary file in the same directory as the
50
+ # original file so we can atomically move the new file over
51
+ # the original later. (This is done in the same directory
52
+ # because atomic renames do not work across mount points.)
53
+
54
+ fd, pathname = tempfile.mkstemp(dir=dirname)
55
+ fp = os.fdopen(fd, 'w')
56
+ atomic = True
57
+
58
+ except OSError:
59
+ # We failed to create the temporary file for an atomic
60
+ # replace, so fall back to non-atomic mode by backing up
61
+ # the original (if desired) and writing a new file.
62
+
63
+ if args.backup:
64
+ os.rename(args.ORIGINAL.name, args.ORIGINAL.name + '.orig')
65
+ fp = open(args.ORIGINAL.name, 'w')
66
+ atomic = False
67
+
68
+ else:
69
+ # Since we're not replacing the original file in-place, write
70
+ # the modified JSON to stdout instead.
71
+
72
+ fp = sys.stdout
73
+
74
+ # By this point we have some sort of file object we can write the
75
+ # modified JSON to.
76
+
77
+ json.dump(result, fp, indent=args.indent, ensure_ascii=not(args.preserve_unicode))
78
+ fp.write('\n')
79
+
80
+ if args.in_place:
81
+ # Close the new file. If we aren't replacing atomically, this
82
+ # is our last step, since everything else is already in place.
83
+
84
+ fp.close()
85
+
86
+ if atomic:
87
+ try:
88
+ # Complete the atomic replace by linking the original
89
+ # to a backup (if desired), fixing up the permissions
90
+ # on the temporary file, and moving it into place.
91
+
92
+ if args.backup:
93
+ os.link(args.ORIGINAL.name, args.ORIGINAL.name + '.orig')
94
+ os.chmod(pathname, os.stat(args.ORIGINAL.name).st_mode)
95
+ os.rename(pathname, args.ORIGINAL.name)
96
+
97
+ except OSError:
98
+ # In the event we could not actually do the atomic
99
+ # replace, unlink the original to move it out of the
100
+ # way and finally move the temporary file into place.
101
+
102
+ os.unlink(args.ORIGINAL.name)
103
+ os.rename(pathname, args.ORIGINAL.name)
104
+
105
+
106
+ if __name__ == "__main__":
107
+ main()
venv/bin/jsonpointer ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+
4
+ from __future__ import print_function
5
+
6
+ import sys
7
+ import os.path
8
+ import json
9
+ import jsonpointer
10
+ import argparse
11
+
12
+
13
+ parser = argparse.ArgumentParser(
14
+ description='Resolve a JSON pointer on JSON files')
15
+
16
+ # Accept pointer as argument or as file
17
+ ptr_group = parser.add_mutually_exclusive_group(required=True)
18
+
19
+ ptr_group.add_argument('-f', '--pointer-file', type=argparse.FileType('r'),
20
+ nargs='?',
21
+ help='File containing a JSON pointer expression')
22
+
23
+ ptr_group.add_argument('POINTER', type=str, nargs='?',
24
+ help='A JSON pointer expression')
25
+
26
+ parser.add_argument('FILE', type=argparse.FileType('r'), nargs='+',
27
+ help='Files for which the pointer should be resolved')
28
+ parser.add_argument('--indent', type=int, default=None,
29
+ help='Indent output by n spaces')
30
+ parser.add_argument('-v', '--version', action='version',
31
+ version='%(prog)s ' + jsonpointer.__version__)
32
+
33
+
34
+ def main():
35
+ try:
36
+ resolve_files()
37
+ except KeyboardInterrupt:
38
+ sys.exit(1)
39
+
40
+
41
+ def parse_pointer(args):
42
+ if args.POINTER:
43
+ ptr = args.POINTER
44
+ elif args.pointer_file:
45
+ ptr = args.pointer_file.read().strip()
46
+ else:
47
+ parser.print_usage()
48
+ sys.exit(1)
49
+
50
+ return ptr
51
+
52
+
53
+ def resolve_files():
54
+ """ Resolve a JSON pointer on JSON files """
55
+ args = parser.parse_args()
56
+
57
+ ptr = parse_pointer(args)
58
+
59
+ for f in args.FILE:
60
+ doc = json.load(f)
61
+ try:
62
+ result = jsonpointer.resolve_pointer(doc, ptr)
63
+ print(json.dumps(result, indent=args.indent))
64
+ except jsonpointer.JsonPointerException as e:
65
+ print('Could not resolve pointer: %s' % str(e), file=sys.stderr)
66
+
67
+
68
+ if __name__ == "__main__":
69
+ main()
venv/bin/jsonschema ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from jsonschema.cli import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/langchain-server ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from langchain.server import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/langsmith ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from langsmith.cli.main import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/llmx ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from llmx.cli import run
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(run())
venv/bin/markdown-it ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from markdown_it.cli.parse import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/normalizer ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from charset_normalizer.cli import cli_detect
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(cli_detect())
venv/bin/openai ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from openai.cli import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/pip ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from pip._internal.cli.main import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/pip3 ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from pip._internal.cli.main import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())
venv/bin/pip3.11 ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/Users/daniellemes/projetos/pessoal/llma/linuxtips-llm/venv/bin/python
2
+ # -*- coding: utf-8 -*-
3
+ import re
4
+ import sys
5
+ from pip._internal.cli.main import main
6
+ if __name__ == '__main__':
7
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
8
+ sys.exit(main())