Improve deployment issues - reqs, dockerfile, gitigore,...

.gitignore

@@ -10,18 +10,12 @@ conf/local/**
 conf/**/*credentials*
 
 # ignore everything in the following folders
-data/**
 logs/**
+data/03_indexed/**
 
 # except their sub-folders
-!data/**/
 !logs/**/
 
-# also keep the example dataset
-!data/01_raw/*.csv
-!data/02_processed/**
-!data/02_processed/**/**
-
 # also keep all .gitkeep files
 !.gitkeep
 

Dockerfile

@@ -15,6 +15,9 @@ ENV REQUIREMENTS_PATH=$REQUIREMENTS_PATH
 ARG HF_HOME=".cache/huggingface/hub"
 ENV HF_HOME=$HF_HOME
 
+ARG MPLCONFIGDIR=".config/matplotlib"
+ENV MPLCONFIGDIR=$MPLCONFIGDIR
+
 ARG ENTRYPOINT_PATH="./entrypoint.sh"
 ENV ENTRYPOINT_PATH=$ENTRYPOINT_PATH
 
@@ -26,10 +29,10 @@ RUN mkdir -p /code/&& \
 WORKDIR /code
 
 # Create a virtual environment in the directory /venv
-RUN python -m venv venv
+RUN python -m venv .venv
 
 #  Activate the virtual environment by adding it to the PATH environment variable
-ENV PATH="/venv/bin:$PATH"
+ENV PATH="/code/.venv/bin:$PATH"
 
 RUN apt update && \
     python -m ensurepip --upgrade && \
@@ -41,13 +44,18 @@ RUN pip install --no-cache-dir -r ./requirements.txt
 
 RUN mkdir -p $HF_HOME && \
     chmod -R 777 $HF_HOME && \
+    export HF_HOME=$HF_HOME && \
     export TRANSFORMERS_CACHE=$HF_HOME && \
-    export HF_HOME=$HF_HOME
+    mkdir -p $MPLCONFIGDIR && \
+    chmod -R 777 $MPLCONFIGDIR && \
+    export MPLCONFIGDIR=$MPLCONFIGDIR
 
 COPY . .
 
 RUN pip install -e . && \
-    python src/resume_worth/pipelines/data_indexing/pipeline.py && \
+    python src/resume_worth/pipelines/data_indexing/pipeline.py
+
+RUN python src/resume_worth/pipelines/text_generation/pipeline.py && \
     chmod +x $ENTRYPOINT_PATH
 
 ENTRYPOINT $ENTRYPOINT_PATH

data/04_prompts/prompt_template_for_explaning_why_is_a_good_fit.json

@@ -0,0 +1,16 @@
+{
+    "name": null,
+    "input_variables": [
+        "job",
+        "resume"
+    ],
+    "input_types": {},
+    "output_parser": null,
+    "partial_variables": {},
+    "metadata": null,
+    "tags": null,
+    "template": "<|im_start|>user\nExplain why the following RESUME is a good match for the presented JOB VACANCY.\nKeep your answer grounded in the facts of the RESUME and JOB VACANCY.\nWrite a maximum of three points in clear and concise language.\n\nRESUME: \n{resume}\n\nJOB VACANCY: \n{job}<|im_end|>\n<|im_start|>assistant\n",
+    "template_format": "f-string",
+    "validate_template": false,
+    "_type": "prompt"
+}

requirements.in

@@ -10,6 +10,10 @@ langchain-community
 sentence-transformers
 chromadb
 
+# to generate text
+transformers 
+torch
+
 # to build the user infertafe
 gradio
 

requirements.txt

@@ -6,7 +6,7 @@
 #
 aiofiles==23.2.1
     # via gradio
-aiohttp==3.9.3
+aiohttp==3.9.4
     # via
     #   langchain
     #   langchain-community
@@ -73,7 +73,7 @@ fastapi==0.110.1
     #   gradio
 ffmpy==0.3.2
     # via gradio
-filelock==3.13.3
+filelock==3.13.4
     # via
     #   huggingface-hub
     #   torch
@@ -95,9 +95,9 @@ google-auth==2.29.0
     # via kubernetes
 googleapis-common-protos==1.63.0
     # via opentelemetry-exporter-otlp-proto-grpc
-gradio==4.25.0
+gradio==4.26.0
     # via -r requirements.in
-gradio-client==0.15.0
+gradio-client==0.15.1
     # via gradio
 grpcio==1.62.1
     # via
@@ -124,7 +124,7 @@ huggingface-hub==0.22.2
     #   transformers
 humanfriendly==10.0
     # via coloredlogs
-idna==3.6
+idna==3.7
     # via
     #   anyio
     #   httpx
@@ -141,7 +141,7 @@ jinja2==3.1.3
     #   altair
     #   gradio
     #   torch
-joblib==1.3.2
+joblib==1.4.0
     # via scikit-learn
 jsonpatch==1.33
     # via
@@ -157,20 +157,20 @@ kiwisolver==1.4.5
     # via matplotlib
 kubernetes==29.0.0
     # via chromadb
-langchain==0.1.14
+langchain==0.1.16
     # via -r requirements.in
-langchain-community==0.0.31
+langchain-community==0.0.32
     # via
     #   -r requirements.in
     #   langchain
-langchain-core==0.1.40
+langchain-core==0.1.42
     # via
     #   langchain
     #   langchain-community
     #   langchain-text-splitters
 langchain-text-splitters==0.0.1
     # via langchain
-langsmith==0.1.40
+langsmith==0.1.47
     # via
     #   langchain
     #   langchain-community
@@ -221,7 +221,7 @@ oauthlib==3.2.2
     # via
     #   kubernetes
     #   requests-oauthlib
-onnxruntime==1.17.1
+onnxruntime==1.17.3
     # via chromadb
 opentelemetry-api==1.24.0
     # via
@@ -279,7 +279,7 @@ packaging==23.2
     #   matplotlib
     #   onnxruntime
     #   transformers
-pandas==2.2.1
+pandas==2.2.2
     # via
     #   -r requirements.in
     #   altair
@@ -296,7 +296,7 @@ protobuf==4.25.3
     #   googleapis-common-protos
     #   onnxruntime
     #   opentelemetry-proto
-pulsar-client==3.4.0
+pulsar-client==3.5.0
     # via chromadb
 pyasn1==0.6.0
     # via
@@ -304,7 +304,7 @@ pyasn1==0.6.0
     #   rsa
 pyasn1-modules==0.4.0
     # via google-auth
-pydantic==2.6.4
+pydantic==2.7.0
     # via
     #   chromadb
     #   fastapi
@@ -312,7 +312,7 @@ pydantic==2.6.4
     #   langchain
     #   langchain-core
     #   langsmith
-pydantic-core==2.16.3
+pydantic-core==2.18.1
     # via pydantic
 pydub==0.25.1
     # via gradio
@@ -376,11 +376,11 @@ rpds-py==0.18.0
     #   referencing
 rsa==4.9
     # via google-auth
-ruff==0.3.5
+ruff==0.3.7
     # via gradio
 safetensors==0.4.2
     # via transformers
-scikit-learn==1.4.1.post1
+scikit-learn==1.4.2
     # via sentence-transformers
 scipy==1.13.0
     # via
@@ -428,7 +428,9 @@ tomlkit==0.12.0
 toolz==0.12.1
     # via altair
 torch==2.2.2
-    # via sentence-transformers
+    # via
+    #   -r requirements.in
+    #   sentence-transformers
 tqdm==4.66.2
     # via
     #   chromadb
@@ -436,8 +438,10 @@ tqdm==4.66.2
     #   sentence-transformers
     #   transformers
 transformers==4.39.3
-    # via sentence-transformers
-typer[all]==0.12.1
+    # via
+    #   -r requirements.in
+    #   sentence-transformers
+typer[all]==0.12.3
     # via
     #   chromadb
     #   gradio

src/app/app.py

@@ -54,7 +54,7 @@ def run():
     )
 
     # Use share=True to create a public link to share. This share link expires in 72 hours.
-    app.launch(server_name=app_config['host'], server_port=app_config['port'], max_threads=8)
+    app.launch(server_name=app_config['host'], server_port=app_config['port'])
 
 
 if __name__ == "__main__":