| # for generate (gradio server) and finetune | |
| datasets==2.13.0 | |
| sentencepiece==0.1.99 | |
| gradio==3.35.2 | |
| huggingface_hub==0.15.1 | |
| appdirs==1.4.4 | |
| fire==0.5.0 | |
| docutils==0.20.1 | |
| torch==2.0.1; sys_platform != "darwin" and platform_machine != "arm64" | |
| evaluate==0.4.0 | |
| rouge_score==0.1.2 | |
| sacrebleu==2.3.1 | |
| scikit-learn==1.2.2 | |
| alt-profanity-check==1.2.2 | |
| better-profanity==0.7.0 | |
| numpy==1.24.3 | |
| pandas==2.0.2 | |
| matplotlib==3.7.1 | |
| loralib==0.1.1 | |
| bitsandbytes==0.39.0 | |
| accelerate==0.20.3 | |
| git+https://github.com/huggingface/peft.git@06fd06a4d2e8ed8c3a253c67d9c3cb23e0f497ad | |
| transformers==4.30.2 | |
| tokenizers==0.13.3 | |
| APScheduler==3.10.1 | |
| # optional for generate | |
| pynvml==11.5.0 | |
| psutil==5.9.5 | |
| boto3==1.26.101 | |
| botocore==1.29.101 | |
| # optional for finetune | |
| tensorboard==2.13.0 | |
| neptune==1.2.0 | |
| # for gradio client | |
| gradio_client==0.2.7 | |
| beautifulsoup4==4.12.2 | |
| markdown==3.4.3 | |
| # data and testing | |
| pytest==7.2.2 | |
| pytest-xdist==3.2.1 | |
| nltk==3.8.1 | |
| textstat==0.7.3 | |
| # pandoc==2.3 | |
| pypandoc==1.11; sys_platform == "darwin" and platform_machine == "arm64" | |
| pypandoc_binary==1.11; platform_machine == "x86_64" | |
| openpyxl==3.1.2 | |
| lm_dataformat==0.0.20 | |
| bioc==2.0 | |
| # falcon | |
| einops==0.6.1 | |
| instructorembedding==1.0.1 | |
| # for gpt4all .env file, but avoid worrying about imports | |
| python-dotenv==1.0.0 | |
| text-generation==0.6.0 | |
| # for tokenization when don't have HF tokenizer | |
| tiktoken==0.4.0 | |
| # optional: for OpenAI endpoint or embeddings (requires key) | |
| openai==0.27.8 | |