Package: mlipy
mlipyPurePython-basedMachine Learning Interfacefor multiple engines with multi-modal support.Python HTTP Server/Client (including WebSocket streaming support) for:llama.cppLangChainPrerequisitesDebian/Ubuntusudoaptupdate-y
sudoaptinstallbuild-essentialgitcurllibssl-devlibffi-devpkg-configPythonInstall Python using internal repository:sudoaptinstallpython3.11python3.11-devpython3.11-venvInstall Python using external repository:sudoadd-apt-repositoryppa:deadsnakes/ppa
sudoaptupdate-y
sudoaptinstallpython3.11python3.11-devpython3.11-venvllama.cppcd~
gitclonehttps://github.com/ggerganov/llama.cpp.gitcdllama.cpp
make-jRun Development ServerSetup virtualenv and install requirements:gitclonehttps://github.com/mtasic85/mlipy.gitcdmlipy

python3.11-mvenvvenvsourcevenv/bin/activate
pipinstallpoetry
pipinstalltorchtorchvisiontorchaudio--index-urlhttps://download.pytorch.org/whl/cpu
poetryinstallDownload one of popular models to try them:# NOTE: login in case you need to accept terms and conditions for some models# huggingface-cli login# gemmahuggingface-clidownloadsecond-state/Gemma-2b-it-GGUFgemma-2b-it-Q4_K_M.gguf
huggingface-clidownloadsecond-state/Gemma-7b-it-GGUFgemma-7b-it-Q4_K_M.gguf# qwenhuggingface-clidownloadQwen/Qwen1.5-0.5B-Chat-GGUFqwen1_5-0_5b-chat-q4_k_m.gguf
huggingface-clidownloadQwen/Qwen1.5-1.8B-Chat-GGUFqwen1_5-1_8b-chat-q4_k_m.gguf
huggingface-clidownloadQwen/Qwen1.5-4B-Chat-GGUFqwen1_5-4b-chat-q4_k_m.gguf
huggingface-clidownloadQwen/Qwen1.5-7B-Chat-GGUFqwen1_5-7b-chat-q4_k_m.gguf
huggingface-clidownloadQwen/Qwen1.5-14B-Chat-GGUFqwen1_5-14b-chat-q4_k_m.gguf# mistral aihuggingface-clidownloadTheBloke/dolphin-2.7-mixtral-8x7b-GGUFdolphin-2.7-mixtral-8x7b.Q3_K_M.gguf
huggingface-clidownloadTheBloke/dolphin-2.7-mixtral-8x7b-GGUFdolphin-2.7-mixtral-8x7b.Q4_K_M.gguf
huggingface-clidownloadTheBloke/dolphin-2.6-mistral-7B-GGUFdolphin-2.6-mistral-7b.Q4_K_M.gguf
huggingface-clidownloadsecond-state/MistralLite-7B-GGUFMistralLite-Q4_K_M.gguf
huggingface-clidownloadTheBloke/Mistral-7B-Instruct-v0.2-GGUFmistral-7b-instruct-v0.2.Q2_K.gguf
huggingface-clidownloadTheBloke/Mistral-7B-Instruct-v0.2-GGUFmistral-7b-instruct-v0.2.Q4_K_M.gguf
huggingface-clidownloadTheBloke/zephyr-7B-beta-GGUFzephyr-7b-beta.Q4_K_M.gguf
huggingface-clidownloadTheBloke/openchat-3.5-0106-GGUFopenchat-3.5-0106.Q4_K_M.gguf# stability ai# huggingface-cli download lmz/candle-stablelmhuggingface-clidownloadstabilityai/stablelm-2-zephyr-1_6bstablelm-2-zephyr-1_6b-Q4_1.gguf
huggingface-clidownloadTheBloke/stablelm-zephyr-3b-GGUFstablelm-zephyr-3b.Q4_K_M.gguf
huggingface-clidownloadTheBloke/stable-code-3b-GGUFstable-code-3b.Q4_K_M.gguf# technology innovation institute (tii)huggingface-clidownloadmaddes8cht/tiiuae-falcon-40b-instruct-gguftiiuae-falcon-40b-instruct-Q3_K_M.gguf
huggingface-clidownloadmaddes8cht/tiiuae-falcon-7b-instruct-gguftiiuae-falcon-7b-instruct-Q4_K_M.gguf# meta llamahuggingface-clidownloadTheBloke/Orca-2-7B-GGUForca-2-7b.Q4_K_M.gguf
huggingface-clidownloadafrideva/MiniChat-2-3B-GGUFminichat-2-3b.q4_k_m.gguf
huggingface-clidownloadazarovalex/MobileLLaMA-1.4B-Chat-GGUFMobileLLaMA-1.4B-Chat-Q4_K.gguf
huggingface-clidownloadTheBloke/TinyLlama-1.1B-Chat-v1.0-GGUFtinyllama-1.1b-chat-v1.0.Q4_K_M.gguf# microsoft phihuggingface-clidownloadmicrosoft/phi-2
huggingface-clidownloadmicrosoft/phi-1_5
huggingface-clidownloadOpen-Orca/oo-phi-1_5# huggingface-cli download lmz/candle-quantized-phihuggingface-clidownloadTheBloke/dolphin-2_6-phi-2-GGUFdolphin-2_6-phi-2.Q4_K_M.gguf
huggingface-clidownloadTheBloke/phi-2-GGUFphi-2.Q4_K_M.gguf
huggingface-clidownloadTKDKid1000/phi-1_5-GGUFphi-1_5-Q4_K_M.ggufRun server:python-B-mmli.server--llama-cpp-path='~/llama.cpp'Run ExamplesUsing GPU:NGL=99python-Bexamples/sync_demo.pyUsing CPU:python-Bexamples/sync_demo.py
python-Bexamples/async_demo.py
python-Bexamples/langchain_sync_demo.py
python-Bexamples/langchain_async_demo.pyRun Production Serverpython3.11-mvenvvenvsourcevenv/bin/activate
pipinstall-Umlipy
python-B-mmli.server